From 2c324fbb924bbd4a1e0fb410cf13bf4fdde7fd0b Mon Sep 17 00:00:00 2001 From: billwuhao Date: Tue, 3 Jun 2025 23:31:15 +0800 Subject: [PATCH] v1.2.0 --- README-CN.md | 23 +- README.md | 75 +- __init__.py | 8 + audio_separate.py | 179 ++++ audiotoolsnode.py | 1 + clearvoicenode.py | 57 +- images/2025-06-03_20-46-28.png | Bin 0 -> 41473 bytes images/2025-06-03_20-50-29.png | Bin 0 -> 43045 bytes images/2025-06-03_22-45-13.png | Bin 0 -> 70825 bytes images/2025-06-03_23-21-05.png | Bin 0 -> 81492 bytes look2hear/datas/Libri2Mix16.py | 281 ++++++ look2hear/datas/__init__.py | 9 + look2hear/datas/echosetdatamodule.py | 283 ++++++ look2hear/datas/lrs2datamodule.py | 358 +++++++ look2hear/layers/__init__.py | 50 + look2hear/layers/activations.py | 68 ++ look2hear/layers/cnnlayers.py | 887 +++++++++++++++++ look2hear/layers/enc_dec.py | 463 +++++++++ look2hear/layers/normalizations.py | 183 ++++ look2hear/layers/rnnlayers.py | 921 ++++++++++++++++++ look2hear/layers/stft.py | 797 +++++++++++++++ look2hear/layers/stft_tfgn.py | 246 +++++ look2hear/losses/__init__.py | 37 + look2hear/losses/matrix.py | 200 ++++ look2hear/losses/mixit.py | 311 ++++++ look2hear/losses/pit_wrapper.py | 180 ++++ look2hear/losses/sisnri.py | 42 + look2hear/metrics/__init__.py | 4 + look2hear/metrics/splitwrapper.py | 115 +++ look2hear/metrics/wrapper.py | 84 ++ look2hear/models/__init__.py | 42 + look2hear/models/base_model.py | 94 ++ look2hear/models/tiger.py | 649 ++++++++++++ look2hear/models/tiger_dnr.py | 878 +++++++++++++++++ look2hear/system/__init__.py | 11 + look2hear/system/audio_litmodule.py | 247 +++++ .../system/audio_litmodule_multidecoder.py | 258 +++++ look2hear/system/optimizers.py | 106 ++ look2hear/system/schedulers.py | 115 +++ look2hear/utils/__init__.py | 41 + look2hear/utils/complex_utils.py | 191 ++++ look2hear/utils/get_layer_from_string.py | 43 + look2hear/utils/inversible_interface.py | 13 + look2hear/utils/lightning_utils.py | 104 ++ look2hear/utils/nets_utils.py | 503 ++++++++++ look2hear/utils/parser_utils.py | 151 +++ look2hear/utils/separator.py | 69 ++ look2hear/utils/stft.py | 797 +++++++++++++++ look2hear/utils/torch_utils.py | 42 + pyproject.toml | 2 +- requirements.txt | 1 + 51 files changed, 10166 insertions(+), 53 deletions(-) create mode 100644 audio_separate.py create mode 100644 images/2025-06-03_20-46-28.png create mode 100644 images/2025-06-03_20-50-29.png create mode 100644 images/2025-06-03_22-45-13.png create mode 100644 images/2025-06-03_23-21-05.png create mode 100644 look2hear/datas/Libri2Mix16.py create mode 100644 look2hear/datas/__init__.py create mode 100644 look2hear/datas/echosetdatamodule.py create mode 100644 look2hear/datas/lrs2datamodule.py create mode 100644 look2hear/layers/__init__.py create mode 100644 look2hear/layers/activations.py create mode 100644 look2hear/layers/cnnlayers.py create mode 100644 look2hear/layers/enc_dec.py create mode 100644 look2hear/layers/normalizations.py create mode 100644 look2hear/layers/rnnlayers.py create mode 100644 look2hear/layers/stft.py create mode 100644 look2hear/layers/stft_tfgn.py create mode 100644 look2hear/losses/__init__.py create mode 100644 look2hear/losses/matrix.py create mode 100644 look2hear/losses/mixit.py create mode 100644 look2hear/losses/pit_wrapper.py create mode 100644 look2hear/losses/sisnri.py create mode 100644 look2hear/metrics/__init__.py create mode 100644 look2hear/metrics/splitwrapper.py create mode 100644 look2hear/metrics/wrapper.py create mode 100644 look2hear/models/__init__.py create mode 100644 look2hear/models/base_model.py create mode 100644 look2hear/models/tiger.py create mode 100644 look2hear/models/tiger_dnr.py create mode 100644 look2hear/system/__init__.py create mode 100644 look2hear/system/audio_litmodule.py create mode 100644 look2hear/system/audio_litmodule_multidecoder.py create mode 100644 look2hear/system/optimizers.py create mode 100644 look2hear/system/schedulers.py create mode 100644 look2hear/utils/__init__.py create mode 100644 look2hear/utils/complex_utils.py create mode 100644 look2hear/utils/get_layer_from_string.py create mode 100644 look2hear/utils/inversible_interface.py create mode 100644 look2hear/utils/lightning_utils.py create mode 100644 look2hear/utils/nets_utils.py create mode 100644 look2hear/utils/parser_utils.py create mode 100644 look2hear/utils/separator.py create mode 100644 look2hear/utils/stft.py create mode 100644 look2hear/utils/torch_utils.py diff --git a/README-CN.md b/README-CN.md index 232f3ad..28a0b8b 100644 --- a/README-CN.md +++ b/README-CN.md @@ -4,6 +4,8 @@ ## 📣 更新 +[2025-06-03]⚒️: v1.2.0. 增加 音乐/人声分离, 人声提取, 音频合并节点. 下载模型 [TIGER-speech](https://hf-mirror.com/JusperLee/TIGER-speech/tree/main), [TIGER-DnR](https://hf-mirror.com/JusperLee/TIGER-DnR/tree/main), 整个文件夹放到 `models\TTS` 目录下. + [2025-05-27]⚒️: 增加音频去噪增强节点. 下载模型 [last_best_checkpoint.pt](https://huggingface.co/alibabasglab/MossFormer2_SE_48K/blob/main/last_best_checkpoint.pt), 放到 `models\TTS\MossFormer2_SE_48K` 目录下. [2025-05-23]⚒️: 修复暂停节点逻辑问题, 现在暂停节点串联/并联首次执行都会暂停, 再次执行如果前置节点无变化自动通过. @@ -21,6 +23,8 @@ ## 📖 介绍 音频是连接文本, 视频, 图像的桥梁, 没有音频或文字的视频是无味的. 这个项目目前包括以下几个主要节点: + +- 音乐/人声分离, 人声提取, 音频合并, 音频连接 - 音频去噪增强 - 任何地方暂停工作流 - 音频加载, 可自定义加载路径, 包含子目录 @@ -35,9 +39,21 @@ 示例: -- 任何地方暂停工作流: +- 音乐/人声分离: -![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/20250426115357.png) +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-06-03_23-21-05.png) + +- 人声分离提取: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-06-03_22-45-13.png) + +- 合并音频: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-06-03_20-50-29.png) + +- 去噪增强: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-06-03_20-46-28.png) - 音频加载: @@ -85,4 +101,5 @@ pip install -r requirements.txt ## 鸣谢 -- [ClearerVoice-Studio](https://github.com/modelscope/ClearerVoice-Studio) \ No newline at end of file +- [ClearerVoice-Studio](https://github.com/modelscope/ClearerVoice-Studio) +- [TIGER](https://github.com/JusperLee/TIGER) \ No newline at end of file diff --git a/README.md b/README.md index 18aa5f9..21e7292 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,93 @@ [中文](README-CN.md)|[English](README.md) -# Audio Processing Related ComfyUI Nodes +# ComfyUI Nodes for Audio Processing and Related Tasks ## 📣 Updates -[2025-05-27]⚒️: Add audio denoising enhancement node. Download the model [last_best_checkpoint.pt](https://huggingface.co/alibabasglab/MossFormer2_SE_48K/blob/main/last_best_checkpoint.pt), put it in the directory of `models\TTS\MossFormer2_SE_48K`. +[2025-06-03]⚒️: v1.2.0. Added Music/Vocal Separation, Vocal Extraction, and Audio Merging nodes. Download models [TIGER-speech](https://hf-mirror.com/JusperLee/TIGER-speech/tree/main), [TIGER-DnR](https://hf-mirror.com/JusperLee/TIGER-DnR/tree/main), and place the entire folders into the `models\TTS` directory. -[2025-05-23]⚒️: Fixed the logic issue with the pause node. Now, both serial and parallel execution of pause nodes will pause for the first time. If there is no change in the preceding node, it will automatically pass again. +[2025-05-27]⚒️: Added Audio Denoising and Enhancement node. Download model [last_best_checkpoint.pt](https://huggingface.co/alibabasglab/MossFormer2_SE_48K/blob/main/last_best_checkpoint.pt) and place it into the `models\TTS\MossFormer2_SE_48K` directory. -[2025-04-28]⚒️: Audio loading, customizable loading path, including subdirectories. +[2025-05-23]⚒️: Fixed logic issues with the Pause node. Now, the Pause node will pause on the first execution when connected in series or parallel. On subsequent executions, it will automatically pass if the preceding nodes have not changed. + +[2025-04-28]⚒️: Audio Loading, with custom loading paths including subdirectories. [2025-04-26]⚒️: Pause workflow anywhere. -[2025-04-25]⚒️: String editing. +[2025-04-25]⚒️: String Editing. -[2025-03-28]⚒️: Added watermark embedding node. +[2025-03-28]⚒️: Added Watermark Embedding node. [2025-03-26]⚒️: Released version v1.0.0. ## 📖 Introduction -Audio is the bridge connecting text, video, and images. Videos without audio or text are bland. This project currently includes the following main nodes: -- Audio denoising enhancement +Audio acts as a bridge connecting text, video, and images. A video without audio or text is tasteless. This project currently includes the following main nodes: + +- Music/Vocal Separation, Vocal Extraction, Audio Merging, Audio Concatenation +- Audio Denoising and Enhancement - Pause workflow anywhere -- Audio loading, customizable loading path, including subdirectories - - Please rename the file `extra_help_file.yaml.example` to `extra_help_file.yaml` and remove the annotation `# `. Add a custom loading directory such as `audios_dir: D:\AIGC\ComfyUI-Data\audios_input`, Linux is `/`. -- String editing -- Automatically add subtitles to videos -- Arbitrary time scale audio cropping -- Audio volume, speed, pitch, echo processing, etc. -- Remove silent parts from audio -- Recording +- Audio Loading, with custom loading paths including subdirectories + - Please rename the `extra_help_file.yaml.example` file to `extra_help_file.yaml`, uncomment `# `, and add custom loading directories like `audios_dir: D:\AIGC\ComfyUI-Data\audios_input`. For Linux, use `/`. +- String Editing. +- Automatic Video Subtitling +- Audio Trimming at Arbitrary Time Markers +- Audio Volume, Speed, Pitch, Echo Processing, etc. +- Remove Silent Parts from Audio +- Audio Recording - Audio Watermark Embedding Examples: -- Pause workflow anywhere: +- Music/Vocal Separation: -![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/20250426115357.png) +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-06-03_23-21-05.png) -- Audio loading: +- Vocal Separation and Extraction: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-06-03_22-45-13.png) + +- Merge Audio: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-06-03_20-50-29.png) + +- Denoising and Enhancement: + +![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-06-03_20-46-28.png) + +- Audio Loading: ![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-04-28_00-34-19.png) -- String editing: +- String Editing. ![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-05-27_16-35-09.png) -- Add subtitles to video: +- Add Subtitles to Video: ![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_14-00-28.png) -- Arbitrary time scale cropping of audio: +- Trim Audio at Arbitrary Time Markers: ![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-14-52.png) -- Audio volume, speed, pitch, echo processing, etc.: +- Audio Volume, Speed, Pitch, Echo Processing, etc.: ![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-02-40.png) -- Recording and removing silent parts from audio: +- Audio Recording and Remove Silent Parts: ![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-25_13-20-30.png) -- Audio Watermark Embedding (Disable watermark embedding; if a watermark exists, it will be automatically detected): +- Audio Watermark Embedding (Embedding disabled, if watermark exists, it will be automatically detected): ![](https://github.com/billwuhao/ComfyUI_AudioTools/blob/main/images/2025-03-28_22-18-04.png) - - To use this node, download all [SilentCipher](https://huggingface.co/Sony/SilentCipher/tree/main/44_1_khz/73999_iteration) models and place them in the `ComfyUI\models\TTS\SilentCipher\44_1_khz\73999_iteration` directory. + - To use this node, download all models from [SilentCipher](https://huggingface.co/Sony/SilentCipher/tree/main/44_1_khz/73999_iteration) and place them into the `ComfyUI\models\TTS\SilentCipher\44_1_khz\73999_iteration` directory. ## Installation -Install sox and add it to the system path. +Install sox and add it to your system's PATH. ``` cd ComfyUI/custom_nodes @@ -83,6 +99,7 @@ pip install -r requirements.txt ./python_embeded/python.exe -m pip install -r requirements.txt ``` -## Acknowledgement +## Acknowledgments -- [ClearerVoice-Studio](https://github.com/modelscope/ClearerVoice-Studio) \ No newline at end of file +- [ClearerVoice-Studio](https://github.com/modelscope/ClearerVoice-Studio) +- [TIGER](https://github.com/JusperLee/TIGER) \ No newline at end of file diff --git a/__init__.py b/__init__.py index 806329d..a62e098 100644 --- a/__init__.py +++ b/__init__.py @@ -3,6 +3,8 @@ from .minimal_pause_node import MinimalPauseNode from .AddSubtitlesToVideo import AddSubtitlesToTensor from .MWAudioRecorderAT import AudioRecorderAT from .clearvoicenode import ClearVoiceRun +from .audio_separate import MusicSeparation, SpeechSeparation, MergeAudioMW + NODE_CLASS_MAPPINGS = { "ClearVoiceRun": ClearVoiceRun, @@ -18,6 +20,9 @@ NODE_CLASS_MAPPINGS = { "AddSubtitlesToVideo": AddSubtitlesToTensor, "MultiLinePromptAT": MultiLinePromptAT, "StringEditNode": StringEditNode, + "MusicSeparation": MusicSeparation, + "SpeechSeparation": SpeechSeparation, + "MergeAudioMW": MergeAudioMW, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -34,6 +39,9 @@ NODE_DISPLAY_NAME_MAPPINGS = { "AddSubtitlesToVideo": "Add Subtitles To Video", "MultiLinePromptAT": "Multi-Line Prompt", "StringEditNode": "String Edit", + "MusicSeparation": "Music Separation", + "SpeechSeparation": "Speech Separation", + "MergeAudioMW": "Merge Audio", } __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] diff --git a/audio_separate.py b/audio_separate.py new file mode 100644 index 0000000..8f8a285 --- /dev/null +++ b/audio_separate.py @@ -0,0 +1,179 @@ +import os +import torch +import folder_paths +import torchaudio.transforms as T +import torch.nn.functional as F +import sys + +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.append(current_dir) + +import look2hear.models + +device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") + +models_dir = folder_paths.models_dir +music_model_path = os.path.join(models_dir, "TTS", "TIGER-DnR") +speech_model_path = os.path.join(models_dir, "TTS", "TIGER-speech") + +class MusicSeparation: + @classmethod + def INPUT_TYPES(cls): + + return { + "required": { + "audio": ("AUDIO",), + }, + } + + CATEGORY = "🎤MW/MW-Audio-Tools" + RETURN_TYPES = ("AUDIO", "AUDIO", "AUDIO") + RETURN_NAMES = ("dialog", "effect", "music") + FUNCTION = "separate" + + def separate(self, audio): + model = look2hear.models.TIGERDNR.from_pretrained(music_model_path) + model.to(device) + model.eval() + + waveform = audio["waveform"].squeeze(0).to(device) # (1, C, T) -> (C, T) + + with torch.no_grad(): + all_target_dialog, all_target_effect, all_target_music = model(waveform[None]) + + return ({"waveform": all_target_dialog.unsqueeze(0).cpu(), "sample_rate": 44100}, + {"waveform": all_target_effect.unsqueeze(0).cpu(), "sample_rate": 44100}, + {"waveform": all_target_music.unsqueeze(0).cpu(), "sample_rate": 44100}) + + +class SpeechSeparation: + @classmethod + def INPUT_TYPES(cls): + + return { + "required": { + "audio": ("AUDIO",), + }, + } + + CATEGORY = "🎤MW/MW-Audio-Tools" + RETURN_TYPES = ("AUDIO", "AUDIO", "AUDIO") + RETURN_NAMES = ("audio_1", "audio_2", "audio_3") + FUNCTION = "separate" + + def separate(self, audio): + model = look2hear.models.TIGER.from_pretrained(speech_model_path) + model.to(device) + model.eval() + + waveform = audio["waveform"].squeeze(0) # (1, C, T) -> (C, T) + original_sr = audio["sample_rate"] + num_channels = waveform.shape[0] + if num_channels > 1: + print(f"Input audio has {num_channels} channels. Converting to mono...") + # 将多通道平均为一个通道 + waveform = torch.mean(waveform, dim=0, keepdim=True) + + target_sr = 16000 + duration = 1 # 1秒 + empty_audio = torch.zeros(1, target_sr * duration) + + if original_sr != target_sr: + print(f"Resampling audio from {original_sr} Hz to {target_sr} Hz...") + resampler = T.Resample(orig_freq=original_sr, new_freq=target_sr) + waveform = resampler(waveform) + print("Resampling complete.") + + audio = waveform.to(device) + + with torch.no_grad(): + ests_speech = model(audio) # Expected output shape: [B, num_spk, T] + + print(f"Estimated speech shape: {ests_speech.shape}") + ests_speech = ests_speech.squeeze(0) + num_speakers = ests_speech.shape[0] + if num_speakers == 0: + raise ValueError("No speakers detected.") + elif num_speakers == 1: + audio_1 = ests_speech[0].unsqueeze(0).unsqueeze(0).cpu() + audio_2 = empty_audio.unsqueeze(0) + audio_3 = empty_audio.unsqueeze(0) + elif num_speakers == 2: + audio_1 = ests_speech[0].unsqueeze(0).unsqueeze(0).cpu() + audio_2 = ests_speech[1].unsqueeze(0).unsqueeze(0).cpu() + audio_3 = empty_audio.unsqueeze(0) + else: + audio_1 = ests_speech[0].unsqueeze(0).unsqueeze(0).cpu() + audio_2 = ests_speech[1].unsqueeze(0).unsqueeze(0).cpu() + audio_3 = ests_speech[2].unsqueeze(0).unsqueeze(0).cpu() + + return ({"waveform": audio_1, "sample_rate": target_sr}, + {"waveform": audio_2, "sample_rate": target_sr}, + {"waveform": audio_3, "sample_rate": target_sr}) + + +class MergeAudioMW: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "audio_1": ("AUDIO",), + "audio_2": ("AUDIO",), + }, + } + + CATEGORY = "🎤MW/MW-Audio-Tools" + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("merged_audio",) + FUNCTION = "merge" + + def merge(self, audio_1, audio_2): + waveform_1 = audio_1['waveform'].squeeze(0) + sample_rate_1 = audio_1['sample_rate'] + waveform_2 = audio_2['waveform'].squeeze(0) + sample_rate_2 = audio_2['sample_rate'] + + target_sample_rate = max(sample_rate_1, sample_rate_2) + + if sample_rate_1 != target_sample_rate: + resampler_1 = T.Resample(orig_freq=sample_rate_1, new_freq=target_sample_rate) + waveform_1 = resampler_1(waveform_1) + + if sample_rate_2 != target_sample_rate: + resampler_2 = T.Resample(orig_freq=sample_rate_2, new_freq=target_sample_rate) + waveform_2 = resampler_2(waveform_2) + + channels_1 = waveform_1.shape[0] + channels_2 = waveform_2.shape[0] + target_channels = max(channels_1, channels_2) + + if channels_1 < target_channels: + repeat_factor = target_channels // channels_1 + remaining_channels = target_channels % channels_1 + waveform_1 = waveform_1.repeat(repeat_factor, 1) + if remaining_channels > 0: + waveform_1 = torch.cat([waveform_1, waveform_1[:remaining_channels, :]], dim=0) + + if channels_2 < target_channels: + repeat_factor = target_channels // channels_2 + remaining_channels = target_channels % channels_2 + waveform_2 = waveform_2.repeat(repeat_factor, 1) + if remaining_channels > 0: + waveform_2 = torch.cat([waveform_2, waveform_2[:remaining_channels, :]], dim=0) + + len_1 = waveform_1.shape[-1] + len_2 = waveform_2.shape[-1] + + if len_1 > len_2: + padding = len_1 - len_2 + waveform_2 = F.pad(waveform_2, (0, padding)) + elif len_2 > len_1: + padding = len_2 - len_1 + waveform_1 = F.pad(waveform_1, (0, padding)) + + merged_waveform = waveform_1 + waveform_2 + + merged_waveform = torch.clamp(merged_waveform, -1.0, 1.0) + + return ({"waveform": merged_waveform.unsqueeze(0), "sample_rate": target_sample_rate},) diff --git a/audiotoolsnode.py b/audiotoolsnode.py index 20da68f..82362ee 100644 --- a/audiotoolsnode.py +++ b/audiotoolsnode.py @@ -314,6 +314,7 @@ class AudioAddWatermark: def watermarkgen(self, audio, add_watermark, key, unload_model): """Main watermark processing pipeline""" ckpt_path = os.path.join(models_dir, "TTS", "SilentCipher", "44_1_khz", "73999_iteration") + os.makedirs(ckpt_path, exist_ok=True) config_path = os.path.join(models_dir, ckpt_path, "hparams.yaml") global MODEL_CACHE if MODEL_CACHE is None: diff --git a/clearvoicenode.py b/clearvoicenode.py index a870ca2..3757b67 100644 --- a/clearvoicenode.py +++ b/clearvoicenode.py @@ -1,7 +1,6 @@ from easydict import EasyDict import folder_paths import os -import sys import torch import tempfile import torchaudio @@ -13,24 +12,21 @@ models_dir = folder_paths.models_dir checkpoint_dir = os.path.join(models_dir, "TTS", 'MossFormer2_SE_48K') cache_dir = folder_paths.get_temp_directory() - -args = { - 'checkpoint_dir': checkpoint_dir, - 'output_dir': 'output', - 'input_path': 'data/input', - 'task': 'speech_enhancement', - 'use_cuda': 1, - 'num_gpu': 1, - 'sampling_rate': 48000, - 'network': "MossFormer2_SE_48K", - 'one_time_decode_length': 20, - 'decode_window': 4, - 'win_type': 'hamming', - 'win_len': 1920, - 'win_inc': 384, - 'fft_len': 1920, - 'num_mels': 60, -} +def statistical_compare(tensor1, tensor2): + """通过统计特征快速比较""" + stats1 = { + 'mean': tensor1.mean(), + 'std': tensor1.std(), + 'max': tensor1.max(), + 'min': tensor1.min() + } + stats2 = { + 'mean': tensor2.mean(), + 'std': tensor2.std(), + 'max': tensor2.max(), + 'min': tensor2.min() + } + return all(torch.allclose(stats1[k], stats2[k], rtol=1e-3) for k in stats1) def cache_audio_tensor( cache_dir, @@ -56,6 +52,8 @@ def cache_audio_tensor( MF2SE_MODEL = None class ClearVoiceRun: + def __init__(self): + self.audio_tensor = None @classmethod def INPUT_TYPES(s): return { @@ -73,6 +71,24 @@ class ClearVoiceRun: CATEGORY = "🎤MW/MW-Audio-Tools" def run_inference(self, audio, unload_model=False): + args = { + 'checkpoint_dir': checkpoint_dir, + 'output_dir': 'output', + 'input_path': 'data/input', + 'task': 'speech_enhancement', + 'use_cuda': 1, + 'num_gpu': 1, + 'sampling_rate': 48000, + 'network': "MossFormer2_SE_48K", + 'one_time_decode_length': 20, + 'decode_window': 4, + 'win_type': 'hamming', + 'win_len': 1920, + 'win_inc': 384, + 'fft_len': 1920, + 'num_mels': 60, + } + audio_data = audio["waveform"].squeeze(0) sr = audio["sample_rate"] audio_path = cache_audio_tensor( @@ -84,7 +100,7 @@ class ClearVoiceRun: _args = EasyDict(args) global MF2SE_MODEL - if MF2SE_MODEL is None: + if MF2SE_MODEL is None or self.audio_tensor is None or not statistical_compare(audio_data, self.audio_tensor): MF2SE_MODEL = MF2SE(_args) audio_output = MF2SE_MODEL.process() @@ -94,6 +110,7 @@ class ClearVoiceRun: if unload_model: MF2SE_MODEL.clean() MF2SE_MODEL = None + self.audio_tensor = None torch.cuda.empty_cache() return ({"waveform": audio_output, "sample_rate": 48000},) diff --git a/images/2025-06-03_20-46-28.png b/images/2025-06-03_20-46-28.png new file mode 100644 index 0000000000000000000000000000000000000000..d27a2daa694acea28acf4e4ce2abbf2b0a95ee97 GIT binary patch literal 41473 zcmX`T1zc3!_dN_!gP;r}-3Cp@l5^^Bq5xCRORQ?Hk!L*mv zaY90R+V$`cIiBSyDY%L2ET!$NW@ql~YT{^yq-OiU-o(k-%z$us9tnvGNlsE+-7RgW z*;8FztLDD(yGwHx)8kw>CT3YYalAt`WJO8Y9+~0sS&x{^hzp_3F+stF#payYw-QGq zmpMKBtq}r$>24co8Ph*H&WQFY#ggqt$ieCSi17XFqF{cK;tR@5%6O-9)#oifgLRc^ z?4jTNzGrV(n3}4WG@s2BC3v!LWE?kcz|1TxVr8awT_l3_*%%GKe$Zu(M|W{XXGS!8 zk-hrwPn&NeAv~b0HWDQmh3H`O9c2ME#E%N%H;5Et?}{Y;2ti*QUM&!ETmZvmG56A$ z8Z)52I133bBV!?S_u`X)q9(@&b8>NAva=4rB)Z-?C!<218b@7-`MN~UvsEbhM0WTm z1#1+`3xE1czwlYy4~2-1-&Y(qfWm|;?LcLPZ?DbtJx7!J10{yNIr^^--JQAOYAFac ztI3OZVF|w*;#SFts9l%1Ke1mtHnF3`fMP(aX=RIFIaS411~w9oJ$*BU=|VHt{0a<& zT3N_Xl9<(`b3TBW#>M-&-yjiqOtI8Af$QnfvVj4n1n4`9W)B~c8Y+f&g|Z~ODO}t# zQB3|D|Lk2&0*(0@<_um%=>!B+Yj<=9<-*H59M@vBs7vmWf@}r{e>d-ZJO(W#CSl*Lb`s#|BtkHBgJE@c| zl4m!5?ISO4*xsk4#K5LNhiE!4-TwgN_B`7^)OI=_oT4*E;TZ;+x!wE7cx5BuF#x33=mr?Wq!t1^rwXIFMc zuC;HVs3F?#dM1gQTk)D{dv{E>)%5O9{)#n&f!r4F+p4AsdiLk#QkD9iS05zwZ?}5g z{yWT(zzJdy&yoA(eD2biwpXG*oE7vA`=%C6X{Qyl;}TladXTR)SBx(?wcohv6&fo9%_d7yW34XmEboGh79&^ zw>{sV_uL&Q`5ZXR(*{LwPx55moR~TpIEfp6-gnIKYx4WC@k8upjFK(AWvvx6fJ&Cm zH>EO)HtT~{>S!~ax{SqFiK$tf3N*4dx$QDmtQ99huoTsMj_D3_ZYLL`B}z7*&Yd9c zx3#4oVcY`)38X#YV3DL2;LFKkc6ZUQWzKdMowjzKEBbBx-h6YV+1$fFOnJA7(~%!R zCRovAT_5RaLK#6JcJ*_AkqS{q%CZpnXuqv&@iV_~ixN&nz=tg{mLp+1q)G)Rf@Rm> z+Q&)pJaN0LNW*PNq22Eyw@=)jt^Ju+ixnhw*`M=TibsLc9P(rAF4Q;fIg)c*lJVVb zj`%#94C_gm&6q`36#w?ExULuszt-_i(yPcDY!x-pnhk|B@#7{G+GP-KP8>YCo_OTDo2j!dQ3(N7%lWqFbaEW&Y}dn*0PTzh35ypw9yQz^P4( z(h0#J&P893p7hjv;-r$yFcM7i_x$WFm|W8DRAN7>tVcW+xR0!4qNdv>3Tn3WPVH{) z!ujqmHd1}y&sKP-trzt;y6;mc1{QBtQ^9g(3VO;j*^5I>f4&;hS@Ye1aFR}4{n2#W z^}W9XCe}WfZ8v@lD?7hC6=}IEtSq>g?#iLq1?FYeZN@3)z`%8CL{|S*z)wd!<+H2rnDhF zcUL>UCa!q=#CD-y;P#`Zy0j1&w#3kM?nW>0hKIFL8&6Z5`IC8f!|0tJQR_(IRg~eQ zmSTo$_$Pu&C2(pekT-%-VC z%<>aW{y-+EU1g3 zCLwV#{Zbh+Nc(NU4JfY{nwMA*&9}kiudqHQZ$-2$IC@^h9(M*YCG<)t&*FD_2VH!l>K`0jYb|4a14%Y*kZkh*Un=>(j8cx$3v-bW#@#aFH7~ez7LY{jn6yqzG>| zpOh=Y83}_U0VB#Jxlm9x`)O#-3Vq8;lS%7qb2hmqNg5UA_!dTeRPmTq`%pTju6O0~c`@>N(mBz! zQooqQqo{D7mCF}rTZYO8D?2gJ?Hq9inn0-Wq!bI$UgE0J$g{o?t^@1L}5B%L@5*)Sq{ z^A@VxVymIeh@WKQdZ^<0{%dP%7&sJBNrw@#d8(BA0SIWss$U_;<|cIfX5QkfU9R8K;7*_^C6S=@78S{7W%{``1AH6fpYw z`fh|2@P>$}Ib^a2$PrX380WztJ9d5Dz!uL(`IYt(`tPgj@Zr%#{cs6=xhkpp{danX*0h(V+_>)}Ec-Jh;3$jR})-+F7z zi->A!YFb`i2E)3!xiKCjF{#A`%V^3FEuZ(53R#ctC_S2s&TOc!4|j$J0S(OtSWQh$ z%_1M=E;c<~gr9$EdRj+Yn`p0vs<)W_h*1VL#47u(6YsNy=r3Qe?L4OkQ~B(yt+9m% zKbQO5-wh8B7Zzzu=XsX~$)O>-}J`iRWUqh%l7~3oiH}N|bI>Lgkzo1q| z9c=BTWA9d#S&UG)5u*eK1&Q6?IGLciMIQ0H?=7|(l}_?;b01LZr{HI3=;#nrQttoe zhfzyLZ;I`11Z%S<40zeNlfen9BO;zguBWG`AFQ39oxvo27_~lK2a5=%K%vyv-~YX| z^l$1ZL)IsO^}%GWgZiv2Y9XQ7JPC7ib9woGu=GDC<=(u}Yjm-(WE4kD_Cn1OK+(Tj z=`gyb8sBTF_rAT_7)qUww+JI-r=XxXJM$PD8++EY7ml$Irau_m-!Jc)*E(&zQ2stB zj2;;3)5>Y=Od%nmfJzXVKI2G?l)o)F=EbB9zE@~;F-2FXJUl!s%vezw&&b_51g4 zsH^%bC8fw|Yr`hD@-pR|(cIkJs@htm#ZT^33UJ^`gwMS`eL4km)6~^{TuFd*jBjUJ z2WLQ>oSz@7m`n6FxgTSrcOD!ZfNw6YfUO<$2T`?i>@MI&tPYx2{<8Sz=V?i+--cb` zbMKwlvFRBt0qr8)+}sqm8YW;joSUBBRE4}*;3f%wqb*4^wz;#jmYxVTT`oFYK!cq4 zS`K!vBSmN|(;poj_2y`iM-l?FgjA9qgnMphLBplhl$DVPh>U#P+=$@3K~KrS78x~r zRR5CW;^$9FOzfFub9Z;&-Q8`9qTIn=AgpjZTsb>CLm&`4JJ$SYJ27(pGZ4<_Tv_Dq z2TPy%oDlh&(b3Vw|2lKqw!_|-5;CAAX}ayx<2Jo|wT?ibdM~IHs}8(Nae(AAh^Q-_+Cq%95G=vfJ{B<(>cd_fR45Xnh|O-xLTh`PCfXZ@zUp%F|UHq(aE zahNm7!L_s!hs?(|ojH@S<$w0{V4$PZ^vEIgvnNXAjPeT$>nkh&`t|GJ@^9w;Vk@PD z!0GAfQZ7A0T}#Vf`U{j(SeR9Rd*Jxt<*9Oi9F*k>5>t5&Y_98=u4@K5yXO*-`E~#w zh>L#?a~7edM!risPEJ8ys9t)1W%cpnDi_+p_UV#@M8VTK5yVeq*_Zk!_8^?;;RJAn zfH;{tB%PyTiFA0uL4uBfVS;aBYx|>7tgy9}GL0Tg2>!QSdUJPIYFW0cV}z8fr=HII z)N5r&2`0wn#4YyT5K>CQ5M0WzVkA8 z+Yw(MEeI7mH|PON^~mM@)`_B9$aJT58%pke*Vo%?Kk4m0C*QH^NV)+1oWmvvqx4n6 zyZebXxsB$3kL3sALmS)YQi*36qK%E8+r3hOneNrCuL=pK|0pSW^0-~vp&klTIXDjF zP=eDjhGb?H7Yl0^ja&1siozj&7e8}ya1Yd+CKeWkJzu&Tm!u6~i4d{vy(l(lba8XL z_?7kA7y~j->Ax8Z1@iErp$5ggxm;J(fdE=v_55B8NF3iub4Scss zcnKs(9VRkyiEeD0tRN}~r-t8Tn%CXsU*NA2VO{SH|72uj0K>OlXiBa{q2j~}c@prw z6vR)Lg(i3JK*kdfFJM*5oy(5Yk=viz^_STDEx3-SUK0um3Q|dQY3=RoAV$hE18cuI zeEeASMl0at?2G^puTGl+-5=y?tN>csZ&1w2V2Pz^8JI@o3~q@gDgNaFE}c9v85ux^ zU&*5f)?=+?@=qYwKHLcWd_gX7a68YZRz!)2lHOuOCSSIcklzBpBQ)ei<0N}* zl22qJ$fheTa9ZCd9zd=6R&5r^P%+spBOdrBW@EJMH+~;Dcb_PmjpZYx ziuP#`3gHz^?@2p{LS3UTcw{r8jSLLErGy%wW)o9W)Ok$s;kL^Cp$Rm^moJ!1$nagZ~ZO zk4iQPQ#|J4U+7zE*+1V6@AB!-?c!`{DE&Z#M26+3u^i#Txf+ubVd; z$^0Y7%3VQMOH3%jFPXGovS=IV;`- zFfDFHq4c*AtHxn(?KrS$VIey%LUz@@;T@!G*qsm?5~{c4wJnafM?3@W{^lpkmZyZ{ z>AP7zx?rD#VZ{O+6|9{cY%PTD--VC&x#n5eFZQwa?$<^CogC>OT0|6MhMONw&w;D8o*wBRLW%%SSii_jKer<3752#8Y$e+|>HVkG z`>W%7d8N0B1M6^;jSdBQ`M!0>a!G}d=Lu%SjIYPIUvoVT>NIiv#l2f5w8JaprVP&& za~ug^j_o_UwKb|>)6xE>IB|DddUto>Lt*#98kYpD#KGB<{diz=m>FLOaRW`jz;#Pp zo-6e#9%vh}5*8(X>(TpZ&|j@<0|4*fos#gOBmhlnYO%Fpq5$h^l^QnNFFzW*SX~wS zxXagi`Go8RE32e9CMW%4EUdr7=~xq;86uwj{rxRhTP%@0%Moav=OK(4_F@emC)xO~ z2DwIVHuzc{2tmP=5SRCyIlVtmpMd?1UbH!JC1fbNdw^hMbVzQ3{$p%4b9`uw^1=>M z>X3PInf8!o*!0n=&ocsyUHji}8FzEZt2&pB)4qTlJ4Mc0G-Oq73-`zgd(ly8_BUZd zuY{m7fc-FOI^DGsl!*yu?LMn8^%b#KfBII}-ISdIyB>{|2!shf;*;Tz;7T1L>*+J) zOnhA&d+L$EINAH}AJaX0a#Ylk?cxEeew}iOCLbT)GYNpz7Z(@zcP&9Bk+ltd!eCq*fI2>s% zlKjAq4hc%{g7Wg062Y29r~CtrWT&@&VN`5!@$skU=l%+)bhHar5s{H2=|Yp4Il3hi z@5#%x(-Kl#To$Jd#v4r!#jXf-Qf{v%jS9?i>+7D_8Z=9daZ9O?^Rmi+%j%DbFJ*y76%j|F< z>v06^*oS|7kb(8WBX=LMk!De=;V1|rCyxcUq7xY*xMnDYyijc;eBbb@r1HPyUG}cp zUS<0^%z6=ob|l~^5nMevb4i)*Ux3bNQ&U$*BR89CQE>bKxh^X4Gp1T^Z`~yY?RfY0Z){hkFV6Av8Jr7tgx`pKXYs`#lM^EZUma!tU3W%s^4BB_{#mdYQ8(-WJp06JBSjck2%2T)`g2y(wQ-?^x8NM0u ztD%xwk&u(ehYA;PZ{C(JEZlEtyimtY&~A7^v5oZg*ta(JeNP0y`B?c!H8s2(_#m?M zYkA+_o*RLr1&Z$2zAn-vHW}&!pPi|fX=>1W`)1RL)M_?7 zZa)Q@6l;Ej!*B3>G>IU>a>B(|jb6uR{TeJ@e`@E>&4LdgL$AC;FSm+H-MG~Fh3ZE3 zJ78ZhXg^cm97{u%3y7A7`WAyI;xSqxJ?LJ(9L>R`kxZha;0$|J&p}Lrl=Y4h`Mqd5 z$;eP(>Se%X*fC+_ACL|+;TM=we7+y%B!7m4DGw@vjnrpLWLXmnNSHHx|1$*y!T-ki zMpl7I+rWOck<_KpfFbs9tb28tso8ez%?aF-lIEYygLtxIg0tMn0JHV z5$3;HXEgfFEBO4moModgqPshv`Vmez-zl(v5RA>M4@a}>o&GY%%bSn`JTxlXWi*W-F>C9IAJwv-11AnP z&X=5@OQ}cEW%vufjl)zT@gLqm2sVTkaS&PQdDa1u{EvY~Rw@nS)(UP+?cQ`|prfnf zHvmm5Y7r5UvaZBc{(61U4`t@^k!ANXueCp~@`p>WHZu4D_Q0nPeyezEy#H05Sy5It z_=eoee<)f)MO{6~;6GF+BuwP-YjRu$5<mW@RNjw{m zLp)A?{kuj{mG16*qifE`QF1>jm2ez6Pg9tire=okdlmr!0a!?GOUr^)HTuLOKS_QN zTz`f?&>`Rzz%4?@Krw?Moj`ZxsMggQR6YGBWwhO*A!q zSL@o~;^19?6Y7s=K~4*!QbCMc?f&~mL{4r{V?7OMP9)Ko&Efv7zoSr1h}(iaGLJ>C zHBD6tm-j-HEM1cTtPjhp^I5S<(YR_Mo((RY3lf@~x%o!3=Xu9`;xvHPQPVPCJ`Q)h zj{w~CT%AK_W(rNTCBPt0gI2@XTaxw+-A(rwnpam>8Q-Qs2F7t+zkmM@$RJRrUB4`YyL{WvTUC?AY%m8}=0^_ugF>?n1hD`IK z@83g1LvMEc_xl0gsiC0($X)N6z(FGLEj~Ve$HQQnPJX|kV6mI7pyz>ncT@jAEdWn- zMa5CN>w^1wyw+Ea7cXqi_vS|cdw4t|*6Mk_S5;L-)u^tcb9r~Yuce{!LU*movfaQ~yo5(8|GKk7Zj!)ZiUzGWk21tkXwY9Ecqiha08KzJ` zXNWTU%B&@}-k3Q$9(_6OC#Cc{Q32%-R6P^F(fW1d-1V-J%UnfZ&r;aalnjI zXQ$~9LGTb6!Z_ci$JV|Rf1ANZt40$Ilh95er`XwTxxQ`Qw+oNF9sEGjw86Rs@fV1R ziu$#%uy+`BAK3dD1;d!~K*)73jo)FV?K27jI@S0Ou$N#w2gy)X1A`~@0|_s}A|thm z#?32rw5Rjdb93pYx-1u4yeQ!Z?KL*DRN78UZJ%puYIr_QKqb)8(YLm?zRt9@A=y=^ zYiMAy3!avX!w-jr_R~bYNO+IXt*oq`Qi|$pXvC+aY#EC2@Zb;PN(8qX%PJ`WqWjOL z^Xu2ImH7cbGPWr()hQPaD;fuW$(%Uw-oz1806?{DLIY`Ylfzgn$^k-x(C(E#UoH!Y zkGTj~t*rSM7uN+pbj5A3Vz-mj>+iKxXc46({~?%e@$&TXxo`Ietry`s)4rIUot^Uq z&q!g;M=>!m%q%Qo7fb%&MZ-py@I`lF5s};N(pFJGuQl$sylK+I2L1|oc3K3OF$UYq zySceJ@XOxd2jJ04Vz=D)#5L5y_DgN)LatxSDk@a#_^4sRz|RYmGF_gZ>`vLg5{=Ra0Wp(wn_oy}%BV$2co|&L78#{ZDR*G-3Y8gM$+1`Ak*Y%mb z0XZhjrs7AL8$jeVrh9&A@$~BrUoQ88uCA_#0z(3#h7?k1t*U3tE9>!o=u~dZyDpyg zK#>9NOk~x0bT16%>x@Yq`Q;0lup5*$1aO=7*+}B(sU}8_mX`o`-~8?B>H-{TwXWCU ziWCzW5s`Ka+?``mB0C+pn(Lz1IpzH!r9pp*Cd-~FDUvs+!|BwUZqH_$I)!8%wTg8v zv@`Yvod{X`^>lU&DTGoglb=oiy~{b?)#MLA5ADu?u;;orR}TnTInu(C5)kL7YV9cD zB5ns?-=K-cadl$lU9jOwfDv09{SFplgV6_%9z>IvnDjabruRfrih1AEjaoVj0J@Bk zfgyi2yE0Xd#{$r?xx)qr+AT&cMLkDaE>%3PiyxiD@IZ?mut;Q2gGMd?SXJ-Y3H%+& z6!W=j%9O@zJ#qn23^+UfQta^kqNmBu4m>n zHKip6A3h6fBvaIO`#VfdIWH?OKT~VB2uhf*(}3~CmADxB)eP4AS2k|@HGZ1xL&+=g zjDW|A>lFxM;^N4I_#VwI+1WHaJQ@XK%^Wd0CI4$Q&}0Apgz-?G82>&5X?g}Eu-v_V|zhFueF|j$3bL#Woc!#GI07ir_aN&eZ8a^cI1v!NNnriV7qmNDn;0LD zITW3poeg3_+hPm~##8IXmfNn7XH8@_rlz@|WmZ?0H2Li_kD8&O=%&;&80;y9FoXa* zcyP(d#^y!u->gFL6iR-3q%Qjrk#jNyTI1q2eGC$wf8#A)*I)|28q%MFw5#aww|`N+?MLn)F>;HU$O{h+_5D_7voXj0GYSDNB;Jr*&$GFBE?$c6j+ z|5}WR%6Q#ACPaZy8dls#0!i})9fb^@6vV`G^72QwkV{^9(W`$sVEAn;exP8Gscmh| zG-6fL*Twhe8Y#wrAQSPx)4V|s$qa;0qryL_d3EE4b$8r% zVnm{zL5^iE=XNQ!5AYdm+GjV{Mr&ZAB!|H=Lpp8P*x0^=PN1JlyJ`_aneamIXQLpS zQ44k@YVYXRsKgw60s=6uq#i(Jx6dy7Z)}(!{nNmZG(QjkXmjdMcNpRM?kt8{f^Lbe zl@$%b+1c6GZX`$g$xw9i=aHO$(GL+|!5u?%^3R{PFep5}U;h+SG^$Ue^@nzeCIE3h zLTi7y(CI>Oc4>vj#~4efqGi29C)@NHSRFp+e@M(h-Vu5E@?~^FLPAD{XknQ@pFCiq z^`CxzfZhBWZgH71h=iV89nDJuH@lJ^UN5+Geu#+YSzR0?$jjb7%Jus*3E_Mz5%`6@ ztYbJ*ruS8$SRka#ax}Zqb^q-tj3Ztdk7qN32U)m-O@`KKgd{Sha+69_w57I|P!y{< zch9j3G=P*p_#zY!M;8E2TGpcerV^%L=&?yJN}r?rS5he7SZ;PV!@w8Wa`?g><-GVX zv~$_g&dz72yayQB&0$N1n2$HQM*tOa8^@r}O^?roj{Ep`jc3oEQBRYxOdMnhV7&l8 z+zj9lp>~Cl4b4}=I6Cq#PScw9yXI-c!*xwN!esS{yf}7zXlc<$?q;-qr3F}wkQ9sw zaB{{ZCjR|9%rk~ss^f&5=|Z<=S*;r{KRz=cM_4It?PAO!wkcg7?&)*EfpoGzk!>|4ArOyISnmX)&%^7>X&F^K@b8C#KXh$;>9k&Rh6O^ue5zt$cc&Dn*t!PUBnn!be}0` zr>hoL0fU@6H|@}$KYwy$A|>kWN0^NP>MtxT z1dZ(n-()FmdK3Vj1mdnYXn|$=++Bjg7ob+Ov6rFQqPHhwe=|IXDRWTcX0QlYAseBU z=-i-{hYLe@0-@`Jv9Yo0E;A9${msZdE4(zF7$l@c&;4fN7N(U4)O|FYwsm?1h`Eb2 zP!c#iv;PW^LwQti7$k3Xj19=P*tv@t-Y!kkN9+Q&NKgmsm4d7;Y zc(@?YC>jR_209BG$aafe<(n~1$q%lv=4wPiNU!2OJt;{^)UctzyirBDa9OCB;DQX%WCK6=Ru)A)#4?j zf07d^IsXiSv^P?UIP-X#P znRloi6!kE+y@yY z1u_E`0P2^ZcoE^@xDueKgFSmTZuLn`?HM)dgx=f2q7L4ko}SAvz-0rKK2!prC14EX z_Gf_I1>HUt;GtZciqD`Gh2OsmIL8gobY0CAAsA?%Uhj?Ol&$ zavrMtb))+$E!P}VXJ>9vQE&N$=xKm^p%2`(LrzW(2sxr#v=B~u&>HG~2jpOQQYv`# z3Vre%EJe^{1@p>j{l{vmsz9`nUj>&g(ZtfycB2Jw3CdY=)*eIu=4KNU#yp%MqNkz) zAe0Mm1^Za21Sp-GzgTfYF`f z&BY-oX77#?Th*EHVry2}{w$HxNx1R#o#@a3Y%Acf(#6daf!g`&7nCw=bkli>l#2*k zg1*?WG&AB2P+^gYdC$#5sS!-^@{}a`+OO$g<*e5S%dZ0@K%!l96qvCBD?@c=!kG+! z7EvHr9^+u&7cj+{=;YQlpdIOP8|vypWJ=4asf_`n1LkbI(1aNp1-Nea;_1;*@J3$i zDf^YK5KzjWoH*Of;QpIZ_Za~03QVDW4q1_<6$x;VpnN(5WBjuF5X0WQH*{ML0Kras z5sO$70m`vp!_v}k-}vDd2f!e((NUP*0Pt{e+6n4LYHq*!TgYV-m8TKJk2$7va5F@Z1NE2d-p=d_dq_{L^-Ibp;My zTzq(3JJ;k6CCU7*0rpn-T5mMyV?RvW=W;y}lN!K^iA`r^S;*&9 zYLFd4HG+=Y2g*R8jriK}v9uX|oP?A#l0xL_u!r)QL<#x$SdG&M9d$bAIVfxpVpauB8WVK?EO(Iatl^w8{))hskP6m43?nsq!n zPm-hoID)N2>J-UO;E|Q9-Aj_ViaTr|HRVkOauYpWUHIgh2z^I0hFIlaWRGZJQScBXd-DIwg2iG{dLIUAhImB~t89Q4f(znRW(|azPDrS+s0fReXTkSe zKva9xK{ViT3+bUsd*)bQ4$%#FwO9#0O_mquEsE?Ix0~cw$t-0&F&+II({jOO3q5t0 z`P~L}r_Jk){pBJc>iwIIbH2<<)Mm*o>3fg_WFa{~CPhYvY#Q)55XdprQ)#%qs21v1 zS>j5mB*p%ugbxTsV1fkV7gw+qHrbXMI&qsF*Ztcu8BARUGAyuKG_|!n7Xe z0FYb5U8B$MnN?bZBj5GzCF?vaM6%@>rt~$EG>^$Z5I-+3L*hW<3%!ZSNsQ?UTvN^e zR}_GNq_6Bx^|{?4ZQiza+WKw5GYS0Zu*w1E{scB7=9^jT%IP1I2Bkt;e7v0K^so?A z&B&1nvE@q4X!{L8^rye1ehv|Fn>-QRg7n9;*9ix-bymZP6RXoVk+V;v!z>P*igoHU_b#JakO-s$Sc9$*AHy?Zg#!iV$#~aBcT45joVst%}lFe@qty%M z=MG<8oiQybOEGCvHq!ISSq_{x$N2PJfkBN4Po+Rz3J%WDJSd%V>aZlMoT|)H`!z*c z`pq&c8f{#+t+r-PY!2{IX%-y6)2Jvy261I$DB(g7KQ~1FhsA4sD%q+whcwWkpSHeX zK%zjS0Zp2TalJ$fBt2njCTkqHqH zC+L+-Q1WTuhG`ZXsx%b-F)I~1MqKuHvn;0H+rEQ8lBa`(NM!jY8mfOEe&@#{BT0i> z>L(4QTYE)90Sb(N0r?+f%|LUL|816f@od?!h<~t2n|T0|+<1PY*7u?4m-4Q&FgjyDcb4~t{n}l+fjs^ll0*393AX{sa>rfEWMw(QG3jo&`jT*I_8PRURG!R>WDR2ztl64#dBSl-mF0PMJ(A%qcO z4?t5|pK107K>(#v1VgD_qA8A{`y)#h8;1d!r_fqBflDDM*Nd!;Y8F5kU{eo*eFc?2 zp*$@8%>A64T!?W7g=fGVZ21?Fh$WdqAAN=#3S6|G`}_t67sV3x+n}f)+rF6TOqg<( zvxcdVqGG~;B8Gz~?Jc9iKX5Q0auU(_5wb$8UH7qa8Uh-nlEek9?%BB)eJiL#BX=~| zCr{z7ca5oucAUeMwEPmEAfcG z*4cF}py9w=E&U??b`VjeW3!sG3Ws?`%dYJ;L{1}P%boj`Lf1Cm7Q~wAfCCT^+z?3s zgp5pob3E}wph1JDta-h!U!44m-`0F=!7=qNzon){aS=TMX7YOQ3Cgyvg`ij}o+7@N5W$T&) z|NK=h9$!lx+Wfpgdh2=*4kjENx8J>*guu$nSU=Xz20V%eEg<2R!$a@|QY5ew0an29 zd&`Oa1$7*xHYs`YEiJABQ#0+_C<}R0om1q+^eMx@2A*laK@E-=ZxR6z=L}Rl4FZnd zveyFs?$A%OOqQ&vF)$9v^|K6a@BM3Wzhu~ zSGeTUw*_M$IA-XEs8@V9)%f6E$Y#J;Y);0*3!k;W>@Z6mN2dpoHcOc)zCU^ICF}`; zb38{D$u77nPTI+X!ms*^zaesY65CKkX(X;$tW*AODD-UN?AT%$=LGxJnOTC_C3Yb7 z{bvg9Ba$wn z{77#Q@I83JRI!{dNKM?h#h`NoW?pemmW^XryKYmHLYhFLU1NFUW^RrWlZMV6^Urf&QA|<41zM9h@LGY zhKQ^!xflw+hQQeHgB)!YegGP7HcVJuu^P2;I&l1j3O0D3e z^dLD1e|H-PP-9?{^>4Zp&I8QsBiwzl8zL9-kS!8_XfpZKXlykLoo@|~d--g8wnzeb z2;O5^8sVH~8FMrcE+Y>Hr;z;e5rhHwKVB5yG|G#@QEm=yamI}_UGIKj8a)qHW&Bo2 zF#!C0bANAtdn>H18RzgRHVj0&kRfD{_OFemOVe{ciT#v{tQUt%K8 zBAggUKDoM-mg?WHv1i2fnJL>=YqO^}|8c8-neM$~* z)1q9vSec2VQiv142hKN0IRM7DAneRm{5*fpw4t>o@Gw_?|B#KLmLq~_P68beK?t$dusd2#TBPO0cBQv)91 z?i%x{4`2oJ%IA2G_5DMsD)E1{>Xm5PTg&Fg@Z^1z&tM(R+vZtQV9onXVr6B8!wJ`KkO7fAFA#AYHGLbcJ)WL*NBuez znTMtq+m~;;I6x_b_YdD%#hi3+M_NGZZNcJ1#t)m81#D@|d^O3a>Cf}FLE*GZ+c!Ix zk#_JVV~x-D1&^j2v=V;Qa6Mj4>!Kyb(B<}JcCkxX%@dg@Vx*zyX{G(!1s1{|an`I3V$Sz_w`gGmZK!d) ze7l#6!eUcm+MfO6YJMv`s3DJu0v-6t;O4FYs^zww4FJj-{CRJ1-q0d+Z*UxL4yHf{ z>oct&{lu42%G*=4y1KIZv*#zH!ZVW#WLa!~9i7*5bT~^D*=DJ(wi}h(Uqs17?nV!R z(+e~Rq}JFlGqb^5JiqyT+b~re8)vwXSHW8&O0rb~Z!%VYKsPFazGOWD>0qM67@8aZ zHsUnmO1$CK`iE)~FJw{emv;gduA7x<6uyzfWJqozNk1(Y_Wkdmy37SqXjgXK1%J5a zd(ITFC=W+p3F6aY^jq@aW{2{GIk{)fyC4!Ru}G; z9%##a8E8WAmz`}^JwY#u1^)Jh`9OS_7RfN*mxwQZ_P~09GVk2=i#BPS>_@Q#45&ZP zOVl3692^^*E8{>6s2%l6 z8t@+mY{)9CzKav)5az$j>={WHuKlo`AK(2$y02dCFG(qP_Ah+#4lGlbzk~-p-97Qb zIu*8^f&d^DPb|zB`uw;X-W58~po59?YEarOKw#29+V9ON&%nSy6o{-9zPV?^E<;r zrAg|+4$*#ipd>2+OT2t&wl#0+ye*{{|J7vVy{>12%W6V+k-TP>3gF75ROIu12SFpv zrhOyP^*@{Gq~ufMXmfW|6n8HB4(^qO7}$#2esJG>?;eR+6i(VhI?Siv;s!VYi4b*W zw9m5q9^q7KD0XUV7Qx-`fI3Z?i3-kjWq)EvdB1zJc?{$|`Wh9XF*l}hadt0UjbLPZ zd$h|ScL1I;sFBbR9N<#w0{&kMF-tM=KXZ)D2b6?%+(ehV`Nr*-VOCS69gzp5N;w}O4kbehg za#$az@g?(ArJ^H2F#`;FR_PG)FmF8%FbdbtD}|`ZdO&{JvfA34ug9QYT8$f6zd)i< z!APME)&gm=Ji2_9l3dJZ1Adv{xS$3;q{iYtqrIiwsQ_#ihr2E|4F8c_p6UnAKT6o6Iu_ zO-M*GZetQEp%5aZ3>h-dbLL8>D5)fKDw3H>nG+Hv3Khwe$n>rKJKz6*UElS&&UKC> z_I}@YKhOQ#_qx|w_p*4yBvb4Od^lq2jyKGT`^!`I&ypX_q}^%48k=nYHNq!{2K7@P~sE!>g|lmQ(dcVlO9ispt1{CupCg zAEmO3k>_G)+PcB1Eroo)?eW(8=e&OWL`*}RXmYBxtVO9OclH8XT+6qfCQGvyyg5NT zbafj==W<20X^HaR#|~7BbsYKl#H;twYg2kJkHAlX4)sbyAK!1a5)L@<;EY`|sDIqelMpRNdcWR^C>B()yhL$=8XhsC<6C^PCe! zejb3lJKA}3HZDIs6`xyd#96I1qUBIxVrx0|0(MG_qe6)pe}hwk)#vv3+hge%~!4*T%FgULNl$1<*_P6Z9G z+Rv;#-<(z5$VEOrQJ5(o$lEN9*1Ys>ZFe?7`a^ZaG^5`x%49$bOAx;DB3m`C(odO# zl_c-{5kxb<;d=KK1OGkgab$4b)Y$U+vV0b!OpMx_ckD+8OpCl(VsJz_FFzYaVSZ&O zsYgPk;_xGlkypKqA`^1&*xFXvIQ&x3OS=B#hDn@j%|+<2|J=(!ZI z@baK{eSMpBJXj>3mf<}>AWJ&qIQD&fa-sLn)BxJ^r54Yd1mUoNZZd=}XjCnjJna2} z9=yuNvMo;3`5V$}QEwx%BoQt2urxBXwnJWTSfOtZB!aOu;k3Zi?BCH-thE z1VJ)?V>bkH7BBDi>JvcR!B&Tm?~#e+)&%Vx$9mCNz%|!>lD1K=ddcwmGQtHC;QkV; zDG}m@3r+>9roq2Em171F?ab0SLaA72GMFTwpbrjJdyrJmh%!ibVRH35gE0&s_N#;d zG@dZhJ@35V{0xt|i)B4Kr(pikfyScD^c!Zz7B?zn9X#zk7aJw^x&X9d(~5mL#wFPB zWr+DJ#csB%M@GJ&-25id(R@O7G$$m?J z@AFcfH{S*^+#V`=wuiGcV1875;AcrfLi}Yuq@%{W*=b}yIkVJI&#~kbQ^&a8*NwT% zIKrU3{H@urGh5NLy!fv5DN6m)Z5|vm{uf5pU;OO~y8JA!HvI1mBeuv3#kR>1Lw)>s zh7|M3rL?fnADvZ8GA*d&%u~JmUixgoCQ^8#+eABJ(aF=Nf4C_Cig#F~TjNtnlz%OHzEFH+BfCOQtO_C0VVFaYiIFtF9#zuVrQN3JQ+& z_IiAMmB7d=k86#2a3NK13soPa-;W>5%sZYJsxnmKKoG_M+hF#!wEWb*sU2m5Gaq{> z%Y-{W0O;UO!<#LI!5?2of31St$xF_wi9tkys7|C6~|+jyRA3H zn?@$4p)-J?`TIN1sXYV8UFfrAL9VuM?l{_)#t=?hbsNw!Dc4O*N_wS9g|aTVc#;SjKmx}nm)eqEme@jqJZ7y!EEZQCw{JICUn)=+$i8xmVYE@r7>D8utnwPFE14=qF z)<{GaZgg6db)Qvj03hrCGX1E3(UK-sH9cXCsP#k-IL%ZeW{`*n7x_NZLiaxIH>95+-ctg%f&(d}c0 zrA5h5!(L_~+Kb6$!YR6-+-4i(qc5zOjcQ?Iy(75mbmqchVm_zj8DMX5FOdaXl-s}T zXgheZ?h!XHw}%2BG~SYFtJlY=GLG;u>3qxDQD!erp;qwIO@S*aS7NuMN@rl{;V2sy z%NJgA(=QwA)Fnbsy?7ThJjJ9g!U%se=^ZJ$Ae*}07+&9)Syip6R|vj5ejqjDbVdZy zmYRQaeR~ETDvZdh3HtJnBS&fHTQ46!EO(W(;+2QQ6=> z_uJA;HZ0F;+tUh1Tr+xU$X4nX(ARg=^rGQF<}rJKp_N08C&OHCr%>2An-& zM1EBe+%&8EKMC-!E35Z*IXAS1D55l=5lP-AzHf)~%E$2q&Q8u)&F1xHqY*oqr?Sri zM%HStUtuKyx`0?l@BZf^yTq>n-a@UwVyzTHdV7LC1Pd|UN!kjC@~eVMR$`L25&{IP zdP5C!p(|I^O5Mup@2q+$vr|Up#Z;_Epm|5uC3pTJ%AA1JKb0nN?D4F#HzHoF5)iYY z=KFB9^{=Sh4>k_7EoW;-SBH;05B#(AtFZD@o7b*tA#?5ILCX45q5;|0j)(d8llOH) zGK0|Gy8tg3n%(#uxTG!W+*U$I)&JFa68cWEJw=OR?)u9U8Bgk8j2wX;R{UPmB7Z*f zDsz0~VWayeR+4$q+KUL!$6a|Abq`DN5BrK_wBU~z|>@{GZX7HLK ztdg3eC&c&>GN@5fc;C6~tV5<^X)Gi=_rbB6mlxam6&+uzby4VY`Whbjw$?BhNC%4B zd$t_}DjfMY&+{`ye`jf#WpJw{egzW@ zea~=>T;oeVP8x4su||>_2V@zz9@Rgp3tV{ZMJ(&5eo{h7M*DkYu)dTNw8Ntm!h|HD ztDiqA7AP*p)UgN9PH5k&I^|j$t9J=UUpaJ8=!ssds#1t&XG zk-jxM?7m9LOG;S;-QUo&G8Gi2Id`z`AS3-(3ew$uK8e!bLWNqV)pn=KFf>1&zSn!k zXd)(QnhMJK#o?(R&j-Gq%}GhwzmQE&LC}JZ^zxA zo=Je4Qp8rb&!Xbu;WP{;3ehi42C?j0*Cg6ags#}UJ1wGSOOi}0G=B7EN|+$y1yTzl zDpKF7cY8gQW_{n`vGh37tQ6mdQ3uQiZf<^rosgxaGGn0UPF2LHv6%^;jSH#hbYZa`qgK-ZYA`hn6@4)#Qc53fSUJ$vf@377!E6oW8z=|frL zco7^&=<6X97VLcID7(g09Y2|IzyV^DS5^dd`*gK1xs;j<;$iJPCr+GDqndE|CO3~J z>;K1LY5kD$!GqpDKIxTkv*eUIOS4_Lg@64J-WpN(nR#+zf|L#VXxz(dw{HuM35E!5>_7h- zww|~7rJM+?qiSIRHnh19#lX`wKX=V{VFEO-#@**fAaNqp<0S~6DEybEM|+zxJM4XW zM(Ts>&z5d(4*CDRBhfj%xtS3slt!tZ5oeliEi2@{|Bgv~+Fsk2`&ai>go@r5)H>#A z>Zy9Fh2qbVpWEej1)usNIrx$NnxX!=_f1_T$t}f%dwlYP9k0Bow|AqnX=7v* zMHyi;@%Q!0zks2fj2!z1lT^0`1WgS8_8Q)HnV#{!?UA!!HfzQJ6q-p;0@H3DHr$l zaXz@&U~^=+U-bL)Bl?ib64_+)w#zZeCF%wiHH0MZq;8=H=MsFZ{8I zi4z}q*xAoZb5;32EG&$Riwifcs`c{$Ed|v^jf?P0nx^evxbC=r`;mX!<>JL1Nlyu< z4xl_qiixq_qE7Mj%%_hZ^$5=cR@|`&&Yip4#<>*5$op`=RQ=lHW_kVW#rZZVS1hKp zDM(d4y}ZgzP#Uz~Ru%lFYi9?g2YB96(feTo1kRlPDH>5Ru~#iE0e?Svdv<_CblZRC z5Hs}&!kB=9B&|AC8d|3ngZhD@8}hFM!7%un#&G-hEwE>T=HG~8O-V}5JfF-7k2se3 zZx$Myp0w}H(agdNQA+*9F9vsPI7zy=NGQC1ts$|-9$0d>19#iT#wJb5D$33LV;M%~|EB?88#=C6Pil06MOFUotY+kSde zrf�wG!}jYm$xm!)N7NQHYBVjbNo=j`^} zre)&po8@*%AC1Co_AV1@)YXd`Okx;AqtapgR%~AB_+iVtW8W#ZGYQ#5*5!b z!{AJNSQ*n$H|_nubWVh!B2g9Tt1YA%3r*2fwGH^@dZV| ztEXXAHivkjFW`&v3ed*-0~b+PdjSNG%Im&UKYyO}|0NgSDfC&RJw)X!pNhu4c13&1 zd}@9;pt(j2^=*;mIp*V2b&&S=AoIV!u7@aTF2{b`f3Doe&Mv~=fLoim=68y(w;giCkVSAPR`eS)a^Pn&t=~TAtmZdB=KoGDOf8POt0_fyT+nE zOi}j$RCRrGJic4Xb~Q5NsleO|`?}zTQFOY(zCAtHq>xk6qP6-j6^;ENO2dCE|E5Zh z#|eG59`@M@KcwK_pOvGtHr9U!FRWPfO#LAfKRsCIvD;ZmnORxtr@GS$1D71l&CO4o zU_CHRRolZWcjYVRcs6cZ8MTp;`y(>22LRTgr_=Dl02t zy613j0*4A}d3&({Z0eqxXuwb1YbLd4yWCYRp1bda;obfFk%O=n63dWzL1o{OskLx6 zBcybE(BXD1q8i?XOk(AvqE*WR_Y#GpT8vKVK*cL@YT|`3`A*t@B!QAEqE|)89ZWA5O7N)qA<$b`Ve1;z848x{G{? z?Xo?2CHLnNFSMg(+E#hX) zlN=M@8?Rd*Gd5=1yak0ON&|-NG7H1!o8}S6$OeZGIAWxdBK3(3wBi5JNhQPOn*4Q? zN+laYD^mWI)-Lfd?$%Hw#7}`3?f(4M>G_xy{)yLkFPRi>F0R>1dfO6Pf47l<|TZw{H^0{ArdF31*fJ0uDR=T)21rw1}dJ z2-Ul%>2H3ve_B6lK%q{IV6HoI@vG!1FzU6JtrHGD+EzsqLAV!RHm`e&C4HQRY~y1` zB~rsTS5qBw;8wzWh`AZIZOeO?Q!OGidFzJ?JtN~5Wj4b9ev^i>rJ2FHugqh4@w=}V zrGo-*0kf3a$O=YFKEX5Gs z(kwnc9(JA*>|N{0EJ1gqKyVx`k!-l-6-IhJsK}?$HoWC$O!u7B5_KSey2th%Joc12 zZ*dsrlJ+pnM5Y=UP|*DosdK1dTGS%ig*61o; ztTgzxr{VlDHoU(Oo`X@sC;N)nm**0r({HTMg!1SOpSl(y&#q3)htmuGDn(cO6m7>Z zl7mZ{y8q&Lmd&43a2w(SA<1m-k{-8gkz|OFO7rzYIRUR`XWfE>gN==E)HCA4V>sY+ zD2#mP4m3fQwx}!Rv*?9KAJ4LE8=<1c)$Y~ar&Cr<5w;u;XT{*GFJbtftPTz~J~rij z)otD5t{y}h?m}yBSsL}>*XPQKq-(v`U$W*mFOnQ0-?G}Z=m}W(%(gD;y77Vn+&wf@ zI51UFQK1SM0`jv)=1`%SN_E&*g&zGfnW(nqd+V+gYO&Xs@B409u(#{Dqv{%Tc|m)S zQII=YsPzNoI3bKi(lg$ofhxo9x`x={8#@f^*JV$}5lIu5a%}k@KMVXQR<#q>f^46nUwn7(2uIxg(c6sWm&=QeHD3LaQQG-rTQMOPxd=6MRMv8%7PKJI_@qg}j-}CX~+FHx>3N2#G z#XEfJ$qhSw+@*!>@cBzfw2nMe5R;H_1R;H<_=Rwzu*iH%`iqzq7A9{Ny14TTcC{U~ zMSsBZ=SiSlY`Mz5=jPti@7}-vHeH3_a0cMl$i~t^BjMUw!^NdStM#qV;O}Qmq7RFw zFVj`RLwL)r_b`Z~Oyg3^xyx`Q#k*Q3KYgP(E!+OBTkaijwl58o-Ha?OEHWAGgEQsj z0&jIY*J>OZSVyv@SKcOyO1XKwiA#zs(q zF;AJ+7#8_2Qx!))YyM|f#mnn`1-w*bEaeVOubI~WV}D4z*<`){{986s%z>}d;>oD+HJE*3O=@lU75c2}Bf}}$w8N5+ z4s)oPx4e4wtg=!Z3HVzGIj+BXa%u3fYu$>NygW$%KiY>`K2*Dnl3MrXhVsJV82h}n zMFcerF0nV?(0j?*xhr*#Y2C^Mui4r1b6P<`b;LtY5;TfX%AlCLzRNEi>3RN( zdjG+?^A)C@<{>tgn0=-27d1rZ`8@>rvp>JIO*a_8{jN4%Y>aU2(i5QGRVB_^-#4 zOHNK_9^28!#}qt#*xK1i%kDn@HWdP9lCGsCzw;Vs;gfie+p2(lET6ka_OV}rSw={S zbP~HdmEeDux_^K9^XC~vDyy!%PwTG1t3$|uO^eJLKYG8s${vxNq`Hw_va;A~I1;eX z0ae=SDA_kenOZsP=RvKPnPu;yNY=Ge*gUlyc@ zrkW7r4}l9E2aTCzq+ZlY*p*;9072#%S2ME$4fj6m2RtTMxDL3u1WjZ(3{QPS)LvTc zyme>+?SXbVyqWj!&)ISpZw4|euPe8ym}x&H5I2LzCLKEJy?+Z7UOwt^v};`;MOSu0 zL=NF=4Z%fU2+N$1xQ;EIq0hm~JEbkoEU5CtYS;BQE}yDjp7C8>`V~WQ9S zrgMGJm5FJcXW|b$pAYz!7SzQOWqjwq?f=D((_oZV(5;nbEW;rOIa*o3_*?c&o*%Zb zl571Ev+g&QoGRyDJQ>7HMs{c~k%J;pyFqQXU~P_W>OX&>I1`ygVB*j5xkMU9m( zHcWPc{}Qp~GS>~^2(sbYCcHY#_GD#c$?-5&c(k4jeR7(kSLDHZ)nA>ybE*Hd zyeH!mUPUa^*tZ?IE_Otebz?FHrk=-wxOBF*#$5eZ z|NK6((P4Bh`4gEe%g;sXi&Y3-qt2v#^^Xc`e}eoEoz<+c&*kJaEq`#=>15yi9ch^x z`9Cx#Vk=fVJCAoX%7o)M(nsu)5jMjsysb~1yt^}|!kN26n6NcVH;=jfNtv^YIYjE`^dLx&t; zDpt)f>du`z(4E25HA6oti3foQ2}EFEApSc{zpSBNw??;kBY5WAL+K8dhqz$ur`=2n zn#7Ur-d=M3QDCfi;hkvu(ImqZpBeKHN43Uq_A2|z&}`Y#k)wPVaoy?Z6;Ds%o=2W+fPs@ms3FXYZc?L5r6m z3fV-d027Wtp`aaT_MR(W!$#bY#~;(-H=jIv2H;|^S=pV^(yl@kR=(5WLNUR~Z|ts% z9aDs<0uFmIS(%KYd-pWqcXUAsx*$jhp~rz3U*Tez$D~2D&XGWdsqW!*NSljIjt=$G zLXO!-pL1JmTdUAzh4IJ`jOb7&ZW19s+r5vKpTE|5NS>r?YisM|^fOl@8b$L&*KI2k zlLWjVb`|}eX#8A#@XoT1AKtbF-BJ$>Vi-IA?gQ*-lezn{BScR)cx{)^H92YdSs*y>%l zbV)P#Fl@+?`2ScSYt^jPO9xC%Oki0pFE9Vr?lFwO!W)e&EY^lbHf|EBuHJ*8 z1jmV!Cl9&|6AtfY+gf`TGG&*3h>pBa2Mgu@xE2~3+Lk7z3bVqm#}ci*C{cjQ%Hn*7 z9Epbev1%fqbM3s##xFEAHPw37;Ysb|oFx80KZCT{6<=SmO%@b;e2zLguiD z>@XZ?bXBKEE+3o*Ye$%7#P@{|-*tDZ=``abfZQ4Pd`M{qcxCYXw>Ka>OiWH9Z-QWp z)QaN{ZUvR_-y9n|cmKp8?-`3oQaP%t-JFYl%8ob2+tO73ygvy@hGR|EmlcI08{4=0 z46IgdEGAx|sJ4)<#*1Tj@$vI3Bla#YFR%Xn&C5V5$dJq%hnqm(c-FNZ(Hqzu3(4KX z1In=>EM`EF`*~|-o3=s2L6(KmjAr3icOJb73nO6vBwa8tV3eZ6r7lQn?JG3J{-wY8 zb80FV9xZ+(f@M@KFw*SV@36`3tYi80BJ7ighX)j^hh0A_DTuUveDXxrt&Ed{LqO5f zap;+|Vz$fD^dQWB!Yc`~4<9~kDDMtaJD}QbPd=?6I;9teQW*P|?L24C+XwaWNQ`CBJ286SlWwqz7qWa5Uwav}5-q`s;BZbJ7m6?g*?u%%Tt*tNF)G$ ztN6K)bV=>rO?6=&rq``hp&N&a`W$83UGTi0U+8^+w^^TUX68xI&hH()ep2H5;jEQ; zd&y?ytZUaG+b&=;D(pLaW!n0c!e_s!V%#NijcV=xn7Sr+_xy3V0R;^ko7yt11+4IQ z%4TYqHqH3e=17+PgSxTk05-P$#$IIKmo9zt&0K>CFCd`5*QN%%wZ_V;DP(e?I--YwZ)h_u|DWUZTRGE9xd7kSiDV z#X+#m!NCz`xaENPmU~d!78t_@=vB~32M6(Ed!3z}&ehd;Pk+dT;d+J6t}nmf*BAd=|g= zU_Go!uy!$MJFCfCm)wwBa^z6@p6LBmG;?z`*P5;H}+C zCWh0KBk|&xc95qRvJ)CPbU*ndw8nC)DV$hF78>aU1F$nBHNvquENlJbHd(A%ABMy) zYQ=GUst*=i&w??Aq6WO)v2jb+~OR1p7p9$QJl+)C*0py!uKKVd@?rcjp zr+Y-O7iEerj5+0GWije!b8z9S%f}}LFgppz9MJu6t-m*b9-JAop*FTuAl=>JL)5v3u?1!fcCqu^w;ltI z&sjexCwD(TUn7K8#-OS~54t;Ace;B?L;VnCP9-y#!wsHafHd52N&bNC{)l8mkw}ty zFwR0eMSpRf3m1OigDN}U#!-C%A20yI_WD1t3;2s-!-{Mv_4H!dgZlblq}cyO6tY7V z{#BqTnpQt)n{dl%U)i@fjLC7>x0I*_W1A~i$Pp2z@&*iHs>(SHYb14iOgJfo=ilxr z)z4NWn}d9Q_Lk1h-d;d8=+l;Hdo&{M_`pDXE8ly{tO$85VcR7AOs$ zgct%Jg}x5;g^i6cWa3pP3j1)7CTK%^hu!)M$YOU67=52^{t6m}^&EMyL{X9FEecj#CsCR$u_GT?$gWml?9;Z+44 z>_ext)Aws@Yl(4}KU~Xy3Z@t=H|r7Uc@id2i>7z{JTTFj{Q}1rQs~3OXDloN(C>qA z`1+fBeNF?VRwMoWv^-Q>(=s|H`uqD~xt}bgj}_~w$iHylX;BeC_CRv12~=_O*MZ-Q zfkEd=0S-*BJ=?DgT!dk&zzAA9H!muq9b4q3=EL+sW-uL&vqrf8<`*t>4C9oO} zttHPZQr*tOmULl$9=Qn&YZ^1|4NSeh$%lFb@J@i{oBxa(7tWsem!4NzYFJ>Lxch#T z_QCJR*3oro9LjvkfF>fkS__97;;(hndhV&XUvWOoqQ)nMJ(}GbM!(sW z9*AAP(3PXyDZm5^U2I+W=LaFZLfD66z_W)BCFJCuA|K8*f}0Xfc;}=1ddfW_ubLuQm_#?eo%G%8u%ZN0Kt3@sx6vBO1^2=B;)vrjMhz@$vXUBq z=mN$k$H#FXl?&Lsr=)P9Hail|`97EG$~(Pa*@wr?%m{m698njv?q^REp;XeSLqIWi zT!PJ0^dHs`4*JGex=o$azbVd?F@Nzn|HydvBPom;Y2>c=5uua0cUU*{zn-fSb z_fIqysQStk>hK|b1mmt(onM$He{Ua)N|HcM zlp*h&DHP312Oa8DG}y8a|HV2~_{cP;j1H*&K#_(*0DF6}Y*t^+=Em>MXRg$TA5)FB z=SxmCcKXGp>x)HyqM{8Cb=kRQaGe>1NMue6KL>6MjK?(pq{^F*(bUm_)UbQI6jCs_ zA}G|cIzD7KChUpbd+;FVvj_L@)2ABUHQtRCqb6xY2dIV@J}M}VPB&&YH8o)bOVj4Q zYy)o~DX(5>`xG}d6eG#(na*!-*Z1+p(TOyO#e#sGYF71WP0jFUu7`Mtr)&MZ;6t|V zT8&@E91fYy0Vv24+nt$o1ow;)-fg|*qdz^6{5s|EP3dT+(HjLZqs>2h^RaQ9KhiIv zSJ&u^D1hD!Vc-#hDjfbEKGZE8AX|fMNWAyhAvAm!E|(6k;<=jyF6*PegApDuL3=f} z*RTDNqdu^Sul}~^m}+WiMMg!PtqT||)R`;cyKH7Dq^ul>8B7k5;oPi$&xnX^E6X#w9=JR^in!<@E6Ki-Qc74f{mlKguh5RH z3EcbpElYIY7OV_>!OID*G%ruyXwXb+Ih9Qz;q_L7EC$~IyQ zW(;^T`e+iq*2HV?@NPQ5c%$3WaWOj4gB;mA>^)puW=yRDy>Oaf?`|N{bG&`?O7V2r z>y6)Ivr=33oPMr3(?>^*ISv?W)Gl>imZ$gKJDibf<@-yOiZ6WV?oLii%QxN)kg+Ch za3Lvp?JK185t(Q7Q3FwVsT1GAc4CdrtjxL*Zy;^BU7ac#={z~rc;VeO6vWc-;?(Iw zkG2Kia2*NWR2BqL|E9Khwf@ng9$Z@}(jz%7UB*wwL@~ej8YMp9q>C-A+x z?UrGq$=f|;Ey!8)@Zs{(kgHX|+)J{ZE6?g^opg6x#`mA@7Tn!EJ?hS|^_Z2UlwxW* z*C3s#b-o@3Vb}E};f#5N(yR9D?Hf1l8A$*dQ;UN;`|N*?`nPWJJ?auVM#B)|!)Uz6 zeYL97O-xfF!0C|;cPWM{;VcCvfOUD^-o8o61TIPGMqxsAbW!t2n(zfnyYFCE2n4;Q zH9BQ7^OI^#iymu{)r`U!-Ac3?3wRuzo#it0$+e~L{5`+|1;%8I69(M&UtVe8V){$1 zn7>+{USRvs++KC>EM@;P8(~=C9?Y$!q?lpaVgSHAcyo0G+v}l`1U5PFyw^Pc;QlK- z(i>zoQxRhDB>>!#3tQvPlBWcD+1l>7SL{~W-rlYhuv`W!Ug>i7X3G0wK{QzbvkxZ99_d=mKvOaZd6HU=)>>-yw?_U8<7qav+7&( zxEZ3t!or~8LaGG{@XYZJvfYTU?;%kKP->Ie@FWrR%jTEozVb!-7Zw)6Sm4giW8u~~ zJb45K>oLm#sJ1FHl$S4G!X4}IVSk)Tu#SB4vhSC_q!MF_nvJofJ3}= ztYQRT3GQE7dE3EjN1mP`-|Lrm$Gx}hH2qawwcuXv1?sR0882 z>6DAUS%@^!PH%&3#g{tx5I}jt%j+C=0xx+I&Loha8h>UX18JXAT3cI>D|5rwj6iNp z+hb5xqK{=Fk#z6G4;D`2n+4 z4122fHZ|n?9rHgR4**F+17>qEtL!AUSRp^v&yd+kHoV;p>Az)t(5LNbN34By2qv6r zVgjga=;`TEq~s*s;2j|&i0Eab1?A9rtI|Qmz|BS9tU(6|^>N7ktA) zQMgU$D_Q-oXosh$^RUtVD?4Cou%|zMyn0)L!0Lz!UoCI}!3iJ7ryAeh!svwpK4Ek# zxqp-E+@&R$&j~7v1R%nKf}*lKi~f!>ynZ`xt8NZ^KuYSJ^lR55AGzyCgBlduWbG+v zHs8R4iBwd~p7MWtV;AYz!&spl>XRu4+I*XDa7uR&`0dp4ga6VL7(11lS;)gL(9y*O z22FpRp8EXeh}A*s^ws4Q9~f&G@AmBsJ`5NEeQ0klFM0``>0GKUUi4!+2DeqRad>BD z>K&!NcmF;M2>}EP4I#+J?>!FhzW48aWIB(Fq^!*)1%q#oum7ds53s=S*m|NaQ9UFB zt1br;<64l6H!JYW*b8Z5@6!ugT3Y5Q(inapu90pN3tQ$i5=LcioY)}CiSiRG#8ZGF zJ*Z8C?g;Lb*R=*CgkI|0fP?l|a1Ho>2`_WTawa{y2!OK-{E*QrB>gOT@GACU)ZUXA z4Kj-&osEj)y^|MC7=1`W!0kX%S>}(_an!cI*!=Z0O(wQ?vd0Nn`K0Xjq7Z z-VmV&B`hNpcfSR1{tf283g5gOG@7OOdB}iu-Tzvl$xqz@6{1KURh_^6gXMBYKs7+E znD;O5uM3Au;P^y%Uk%+l>=6H1ZLQ`c=e4Wh?A#XNpDq-QqpzD9_-!68k zfz}`BZf< zR8h?qwSMd-xEvsL>?Uy3>AmTB-ylB=O^HjFiUQYzW@9c5J!8~MlKl?_Ks{-mzFj-{ zIMrMxR!grOssggD^Ob2X;cqzJs0>OvM_Fvp4zn|S(EIh3&Vhd-gfZH$KH66B>-U&w z2P)EY{%C>m*1O-ZtI-N^|3FKUCMJ%$h9AYOobuf~GG_rlk&9MCVG?;0fz!9S@wZ;b z>qbac4=0!uv1qbqb-*4Cv1@qWK8|fZdLV7PUZMsvtL>D0S>Ms|JMiTd0;p^tF<_97 zZ>1j4g&9?T(;eSB?eze-xAbgcKBf8pF1rr@|utiy9#WAJCa6jy<69e(Dv;YjSnh1QP=j^I2*n7_rR0%BSs@jDFVadwv$=p)NmUEyXBIeEyUXvbx2_m8 zlcKMVTGC{>!H4Q)PMwY%Vc-_q;kD_<*j+DuTY3 zxzQrz9IK0x-7oLUS9+|KE3ebup8v`as|jS(M?Oj|UsHtp)G3q7sS>{vsqFkdgs>0P z%@@`!QswTv>iM8^gl{8aK(*pg>iRC}b#BR8p`{E7Cf<{VvFn!$j;t}NQE?Nj#=avi z5yd`V+hH`-el$C2E0cP|V%&$xC+?mE^MdU!&r@A(mUlC9Xyi%Dj7_4{wWj)`J6jqh zL?F%GJV#L5cTQED$>SPJb^A4`_C==td!kS7Ylss+>Ag5F7cRlG=WMeg$DaBKmoZxk z)>fKU%>N*;3i*s@a0!*L09g2Q)-@fA9+)=ta1)-9$3JjisbCdts%C+005x!;Z27Dgn#5#BPmq=f>ctnURn4AO~Th zsas;ZQCiLF%nKMGiDEDljD*RatGFexFT!kFwNO~(f&+OX^daCN4 zJ#E&~mp)bhNjUhKI19z^yV1v!)&Zc9(GVX@N-;6r%3s?Gm3SLbj~-( z79mgg1P{+0GYrt-5{VbyU1(AaM_XWru-w9p0IdWcWgH(T@+%7qJ_(FJTN+QZszX!H z*f7K@g4^yiuw;C&>N)y5$)=x|$7qBj6c44pdDKnjxB&4&MFQ6@yxlver+Z9_Fd+kz zS8#keySUVO{$Sm6z`LpLUHTasOosUYhkw9cz&vej^NfyXZR3>07y)3xy+3Fe`jhF` zD!+z*KeFb`6c+Dr$#9I5_ny(Kl-PA1G091a?Msa;JUstU$CC$=peX>9_V2yI^mMM6 znugxuk`m}N;2VNWkYB;=4vcQ`6LInA3z#8nzu`Q+&h4qZH{@ANLIUHqZKxD?82ULv z*UHBOT8ck#TO0&vcVH%F#$YP`Lhj1<&khMK+@%|=sd`#zI8Wcc6Zu`YI)iC}@D5-4 zJ+c|Cx<<}!sl-x3bo3s6mXLJiDD2y+HSU}=wZ~&5hr*lJ@R_?T(dmks(Gumg;3%Q( zG&jFjo+&2vX+JjG+sz(O>7mpDg7r+Y*<{h{m$-`UVZ$I6x zg*G)f-DbNU>ie6e5g4j`7an|n)1!#dDc39{H!&w}>kBM;5Ub!{PMXwHvig??1FM2vp@7PP`d9}Epiu!#-b#JbkZojnY@TBQ zpgVB)ZWs|3hu(-)w{J6>>-^Cf&4UaE+&%JAaB_M)!p6aI^VY5D6XYrM5KP|TyBvF> zAmX?@GG?H6Wz)Wutg+wFr$zq@+{Y|==lO9ruBvDl7wZ50;@-*gA0s}v<23blbDwwA zVM0wLtypnpWh3E5;fdRaF4i=ztt^m18Wb37Gv4(cFSe(3bzpS)a|>kgb?4h4NL(V1 z9gB=)XJdOmFkotGI&|q7+U=ES&(?2D)lX}^0)?^N{=hMr^M`Z^N$n@OJ|WT|0Li<4 zW=P~FwPJ5K*t%8>Vv)eY+5{f&`Ks%0Hq?e^dqmVUH6tpHp+ghkH@hCtne6ltXjsJ2 z!^gk|Nl4fkfu$iPF79tDG^a@`a4H=GoI>fp$0EUY&8Lhn?@EE!_S$6dM*h)#BJRc| zdBw#Ehc+Usvol=i=gnk6B|)S?pw=}40ud7zw+91-kp8>6Zhp^8I%jKp+joL{RRcc5 z7=Xi~vmxP7;(n7dq{k7&9%B!W1sp3s#4`86af|uL(-O&aYT&rO&!&I<@_P2hhWJ`w zy##hgfiGU-+`L;(A1OENUcdeO=8cNA5 zDJg-1Bl)%PEXd3tWY|j_6=gjI5(o^BU(7p7x859+-iE9Z5519SWp8VuceOcH_ToK& z9#Ac9l!Cm(^z8V_QU6b$8Pa&hL0AZ2qyLtuFii7&hopbVFiTnoO^da(^twpd^dXe^ zSQY;;xAWIm-Hs}{2mF64*_qsK?(1Cp(I=%<&l@7Nr1s=x-1@f9d6UQO{K>~gmrg&P z2v{7y@28zbsHS#(joDE2!T>bR6fczquTLSn#m0u|!6o@23(2)ry z(;EKcWtbQCL3-Gz+BiL8Ur}uFd>#s6V|w(z9o*XDxaJ&bYw=l!FVu9yP}Sl2FO>rn>e_Ld zB6B%xOrGT2;P;hJEEBWbPyi1P2VklJ8FEHNs-0K<b}!%&G~6#;r7 z(>!u_UqmZDo)JJjyq@tx4|qU#*cajv04>fq)fs|P+2quz%abu5tZw1Rt^N125HA9U z4s=CwuDcoKT{1|KUu55`&!rbAd}AQ$4UozU0q#xqPM{KD(v$l2VVr5&aJJxF?V)T7A$>XtZ{# znm)As*Y7Pf?-h6wYxX48GgZ>vSXEm)P#Ymfp&mgcQXD4!Zc7Q@oKala=fQSPrHdK& z8O&5z{iU`Q2Af6gmnw3gSvc`Q^~ZFew8$%l+9Lj6pTsHJd<;uIaE(`Vsf0Tf>)oFR zI|`Cbj@{v@=+9+jWovJEKt^)u_7q(1Q}XZ)?0rLW-xxn zx}>4}nSxo-`mz`M_x_{)968J!hs5R19ibTm>ioMY8tbEF8MU5Yn7)4yd|9{CcfP#o z%pKN<=Q;Q9DoopC^Qic};tROC4Lk|*7%ixtMN(!j@1A`gSxa+UoSHs8nz~CoocGgX z7dMqdc2u8hP`D78M^6hB*VcY=dfKEB&tdNsbA$V3Xu|<-$Bd*j?ADY|7B1kQjY!*qGWFp_iovfM5`M$tKH?I_Em~6 z@tD~A3!@D7M^!4THFmZr-ukt_-Q~27-SaDllxlUq?5Mi!p-&V#6Cv51)j4W@OUXCw z5JO7$`Tm0ar^437qU%ohX*M!`$zx+_VT`EFtCj1QbmJ2ZIjXLfvvy|OgL#2PWT~ya zi^or>-(h>8(fryi(ks5uyn(agd@*S@24K(J`T8_t#K@`KcQ(Rnw>Q;y?b>sOxX09Z!7LhxAqQgdA5w)Bx8CDUh zOZ?ZwRgY7=wYH$MFOie|UyteWn zzj6|@KmE`Ngue?gDnRr-Oc%tFEi1$RpG2Zcc=EVz4}veM7>q?beNrV>^DcsopGyM< zv|vKs8(9i9x+5VA&?z?6aFD&LSCKA`k~8=vBKqNA5TGzY61z2$)R41ft#tfl61!k~ z)z?GM#4lH|pA~Sq6}N7>&9ZEByE`8#`E*VSMWcPiv39)$lJc|92ye>%ONfd-=jaz) z^z`*n9yxe<{ek~EPFyne!@7~G=c_Ddl{v&=HTVJU;nuBN_(xPY5AyR5tj5R2hWfo- zB(@wU{X@SP#Rxo$GlU0#tYiKIDg*k9A+^!$^lFZgRHRlUy%moiW0)b}*2q($?JJs* zndWRf3d0luV9K$)v$t zCeP8PE!hS-1Q67kI;j=yzr{GqJ{<~R?zL3UX&0+teyqU8c=l<6mx3T~h!3?CyQ#o- z!s@0x=tI9MNB$@N{#!<_Gtz^%x|CE`E6&=Clhu~ zF7C%dfLlMx)^sHk;l25UH83=IIXGzVhW^M!MS0=k#qIqReplq%h3G|%8K$2Sz9o_` z8u78fxG8{>R1Ts&CWRP$xF8vG=kL>j`~->t&xX#_NFfzl{RpYr=!15PP?UfPw+7K~ zQ`U!_OTXkf2ooo(ws=me!Uh%*@PPZ{FkBxNBH$dMGX&)Uet>f*b&0dSE*T4_o1QcLr zz$=r2J18tFs&N~|v@h1Oq4)2NyZQcqHGt3^GxAS2#1P7!TBN#WQ@S)|z z9K76&42rt{un?52_Qj8_npX8m5j4ze%3Fv z&`{*D^Ir6ukBaJU1_|AO%KIv-T|jtqMIi0wFiSMlrX3jvqLqSdkl15tNH5=&f35}w z2H-=htA`H$KaHJxJe2t!$IZkT*B!C$Wf)2!j9X5Vm=NWJT$f!D)wEfmB&24LOXU(J zDq^iJ>z)X?Yh9K?y^#vnK`wC^nwIcm&mtW5apHn8;pRiY#YvsY4D&J$9RCD=PTS**|Fx7+M^57jm~(cXIpFW8Zadmw^*);EiGoCGs% zHS*mo%XN2fpbcdsuFiw6LL~UxE9zkVx|3}2} zYyWE3xsrw$w11hoUUb)*%p1tJAT-BvH?&b!D}PAf_{-tt>vA8d2^$m++@)-$vwm!i z5D>u^3(Hl<%FxLT)PBSIFFu#}MzLb{Rt3F-nbO{})@{?(DWE@IkeToxLv>FR(=X*twdei#wIG1Rtw|Ov!B}cGbT8nk`MI=C@0)nP_wn%IT_BdnKjFF z{HE9&NfPqh73H+}_ZE?C5|tzBaJ;LL1>Rl8aG$r1tRj-iQ{*1#bh0Ubl#FFd2i)OH zV>S#sABsA+i9AjZveM~mDz@x-kh4<4a#4I!>BU#7VOuE6{mEo`QV(S#GU_6zNQMLj zmD$PVUw-oB3G@YN-49Aj9qKdfd9v08w$c5%0&4!E{@lmT*BvIAje&)B>r54o5W@)O ziW6%IV$?ep&E%{d)Wqj|02`UjDAzWoo-~JUGd7V-F=oof$>&dczX|>Di)G2#qHP^> z)RX^Um*td>U*tu2+8iicl5~Z{yq5YX%p28}8M(p5r#JRbhl#KAyxBYK&5QW?wGR*F zsLZ0D(6FSjR%vS>|B%jg1I^zC+PrRlYM6`dH~Hom($t9hX9jDb*^_`;Y^2EQw5hRG zh1YKmlX`o`Lo@f)52wlRmuFL%4TA=2D|mmEKGemH`C=F)a%*QqlcVz=Lc!k4X1a>+ zYrOcNZhw4e&N4S-Rq$~Q$x2)h>44uC7CuGEpCF2)$FqnV zJvHGZKJ!E-8MFhj1@!#+fR5R> zl)e;luby<=O;h-A^Je6=!oF+_8YYh1Sqttb+LE^f&KmzUJZkUlNP%04+11q;G$qM2 z^y_Eo$dZ-a%YDDAJF{Kcyy%5Pcha?M0AWSjF_EI|*ebItk*^64%-y>W6!?^Pwzuy= zb5valq;Wo}e|aqva{N>%X!oeQm3^4!`$C0+mhY78Eas^LA76d#8m_4sBhYD{|H%nhW9@Y-k z!nymt!!mDY2lO7!Xl8P#&7oK-;fD7>^`UY1IVM2^eT)ha(YHVq`|8N^I;e-O#p=gu zM@y0dS*l84rg##Mn6L)h3KPJ6F((k2KZI;_YZ6+|ZEc*qOINQ7x-_x$KQQ#s-J$!L z8gn;*J_n}Gb6SIw6hz9W$KHJmt%CHO%oCZ+U#P3DR#aSbv?pIz9%b+Ns3h911PGE4 z7NLHUo-T9`3srp&NJ@%ER1!X=KU{{^d<|bG-xf2{lGyYq^(yQbD!ROGg6~{k1Hq9G zMHyslz1jf}u|Lesnx)ZZ|6)imOJYabRw=L>|LV7wwmJoB$d` z1!CH|c_hU>I`9z<6@xt%NSmz(pPgW0{(@Dw(99bWQX%Rjq-NNNFJ&`D&kR=*jQW#3+fli z6#Tj~O?L#KieTqLYj1?{pE3b$zgVX@=erWRH8LSG0-*+gG~w4N@}kycd=+gpAUz2S zE|dZ3HqG4_fiQ9%r9C(XUHFZ7Fa=Fu$ABn#mMc$1s7)N)PPsQRJ$;jYP&j5Ylj8F| zox$*>r(={ZOK6vRWb;N3JTW9x{pEIAaG|B(H^|1oT$mGCM0_eB)2E=9N;v4Tr&vZ1jvA(#mRgh6CI6lGPZvFsI`0d48=c`>p0^ znQ`BRu)Mco-%p9Vzn*Gt?|FEBPMw)LZ64DMmvsFttEVt;~sW9_0!@pMa^WWE}E0^H6BftOd@32GzO$VW2 z|2EYKYCl_A&PhO-x7a_vQ$thX4V~~9LE4UQ!+{dEFvosb-5g5h#R|#%>{|J}=b;#S zgOpK9%BS=ttE*KG-P1fVV)og_iNkY6tJff_l%(Tx*5AMD$Fir5W;5DVmOO&6V!v-q zuO}ZL!Ohy-%<;+<6DN3{cR1-X7@{=48&`JEEhqUY(i3EKJhiFoOa#>|b!;N3tx?R? za$3{agH0mWY1Ieb$a>wEp?Gm*>_*979*zjiQ>xkPBV@XEZulUe806&KE6K~NMEN@weEjR!+DXVCLG(R|o_yH9o$~9wgo&@5Q-oO}S782y!2Ai} z?T3O0u_upjwfb!Gi$cuB5dqOr5%slktUw_W3+G`+*2~deNgi3y5^VhU0e6Lo@lPn7 zL-$)UDNZ(vi9Rt!HOmx%7IwwnzhBG(m1`twXT=Qt_snL9p{?e%>`dvnw{_12 v#Xb-0@G#}w8MDNypEBS6{j=Znm#=Gn-yWvGezz-x51X~cUh_hxXT-k%(Aa~9 literal 0 HcmV?d00001 diff --git a/images/2025-06-03_20-50-29.png b/images/2025-06-03_20-50-29.png new file mode 100644 index 0000000000000000000000000000000000000000..304f8365a486308beaee21767d7134c4a2fd7657 GIT binary patch literal 43045 zcmb5WbySsI^ev2ZbLj3ADFFdV0qK+m>F!jzIdqG3NsDxMcMFJgmvksyck}+nH^w){ zz4woMhJro^p4fZswdR_0?h~%8D20Jaf(iozgCQgRUIhjQ)(-~eMIkaQI1+gA`8W86 z;wY`{0t18D`~3Gs;wwy2a1i^Ggw`ij2lG#EM$Tq1s`gfnMlPSs^a#h7VPL3WWZsLa zxn~@<`RS^etvvaSVG)@dniG8`euwyC5)ugWS5v1P)joHH^lI4pOH)$(&hf(#Yi-%U zvCgvBNuk+Y8*g#x*f1$?jmj)k;rs7iX>JJ=nnX&}a}(uIBj49tqiD#w&>od3C0#@$^ zNW*v`4pb|hMpzD8822k8EAI#~8rc-ZA@HJ*Uea&3G{}KeVHKhm63+(=g7zb+z=3we z8A4HTz#BuVSdG6Vk0}Duy`3wDh1(iH3-7Ro+(pVKc8VASX$@doYY9sVF+4MpGKoDh zbaGStKU`4P3Hp4}7w|vW1HnJ8Nz@F$wcyX*d~XMbkvD9VeXyVJX7>MjH!gY}*#G;b zJ9RZvb`c!DHIQ7l`LzbP!<``_9%A%r;{3mzJ>;Y|a3ZuoN04Gdn>4L3RPP`N(f5QB zcI*rh^X}iIFWi_pm%_B;=(?lszaXkX_Z|(O8}D{yrmer3KR5%a>}leqEz^!qPEL-F zf^mKs!i)M-dSoHM257NJ?nywDy4D>70|Oc4Q#!YuUq<9IMO1}g!xK;33ry!r)1jiF z{XMaSn4;%RTB0Mu?eYjW((K%jQoF-RA%mGhH%-GshNp=iTn!S9tBO)%enS#1ND(`- z9SCERB_jVdjMs5#r87ViCrA0uhBv&ZpqQy?{){!k6Ib&$Y*WD=dMGkHEIOhngtBw< z%<&S;DeOKe4$j7j5qJ(!I-Hy%Fx`qFv|&ovsKHS;k+4xzn*?lKZ^Do_#Q!W_dp*w< zF54q2RamU(3rUEnWtK&}G+pvYUT&@_=X_j}G%EX5)dwtdGqWKK29w0fN~$o}Bw84E$=!DTrqy5}vj z7t#8uYAET$Y4Y>c^$G;if!%e#)gLCN4C8#52>M}nF745tO`ZbuiNMu z7IP7P9+}PvwrFhVh6nO{sZ35P0#!A z6`QrIM-b#$R#OPNY_p&FW8&!!QUu(}^(&CnX`x^mn7FyQ-@H+0PCTBk3B?|-ujkqM z3WY)|+HX-~N`CzK=P9IGpvEjNE`D%ufI>|YUD(p%wNPtSjs#Q0fp|=4;JyAx6?*pH z>V`Jrenv=Fa}Teq8PJHhq(0 z>AVh0jq8>B3w1WMP70dp>cJr)oHDHE=jR~W^z_IrN%Ml`lG)4@t?cd1g~u}ml08^c zcu zk_t_?w=j=}keYYJ_VfuJstykiAGe$cGjPtE_r)s~*;ra?fpON>CMFuN1uM3tr3H(e zAHF2&+v%@j)xSGqE{z*ZtgINuWlnwA8yL5@w`$Dz*%P5D7CTtO)D+=-_{i}3`+jOt z8O<)uk))Qn3jD*b672PZPwd;H|7N%yeKSEz`HdT4@D|N4kFkmDRXX}Hgvq4PhZ)hNE;_Q`CeRI zNyIt9XHQ_6?$JZT-qR2ks!+Ydj-b)k4c&ud;r3*Ei2az#fDM0!_c+G=5)XBmv`_>E z6Vn`4_C8DMXM1$833Z;g+@}cT(d2IVgcq4sIHC}x^;P+V@}g8Y_!KWjQH>Nm&`vayBQ)954KLiwhcOG7wue zlFX`IZ@!(@IM>LSQ3)!X?hBmf>JoY%lo#g$nS9^`&CCV z*_3wJ#5;fspb)Wq6)Q2ZZX`-?L(Wtc6_uUwOs}m$Y}cT#U-2UQV1pwG*~bS5`Fw7< zsp*)QzPGk6F-YBR^hHi&yoLLiHMs38=?Utg8gt$3MO9Zd{RneMvAmp&oSaNaS%q10 z$K&_`)hN%Af#2?hnOqXakM_&_f_y!jmcqS!vmTCbk8hocxq@IKs$-D z2%PhCb40|%R-1lVf*$8}HnVHj`!glB$<@*S80hJ>SqS3v#gOyrz^y$_Jl#(;x$FvG zuLWV&G{?B|2vzURotY%As+DUug}nSKn{T60KL0fNXk#}!r??01!OHW^oybB5m?FaQ z-5ZyktJ;4MF;5-OH7Zv6Bh$U1jpRtBq^$etlPCDGlvu)7?EQWH7q$OJv$>w8v4n%+ z9DIyS;lDi>S86Cc5)#+jMbV&b=eGhLx{i(<2~NLFIfFx=7O|V%&o)3h%v99W*0$nI z1sLI*MoFTqnY?_|ylr_!1#hanbCV!7wb5$3kK>{7Zq)}_U0Yop*@ddjf$}=NSM#dp zi-f*t+lCM>u@Sz4b*<@=x32doXnz~d(}z#59qV_SuD3U4S3SrVvRQ=Q`i*-lRG5zq zze$*8wDEjnh#NW(nz#G5a~(&lRO3}NRhZK0eo|(;6ct3b`P1Ive9hDCYzH#+D~(cr z=TvzfyM@}z%geAZWTs!1+ys&I^z;%`kSLi6uDsHEL~7il~h^Q`>L%U-6Zfmg=^+ zvOVrK`*)_oB}0zR0NDcOSFaLe3*NqcYhH76fAe?chS?WpX?#SJJv7$prByn` z{68>DEw)3|3%5G`Xc!0aM4DVC-H&@$ek3Y~_%75T3%3h_-GN|nei5Qi(;quHX*~|F z2zJK?P)0;y`jG-LFp7{V;@qVC6`}7*e}8`}m&Ne5bE0fwe7w`c?PsQJAum@iFE1OL z1CS7QCbA7wRZYM`RaK>ijVul}Y16MzDdr}@m`>DX!@|PSELKw~R-=ipt*^J{CaKdA z`g4h6?<=}mWL@oCdf8O|PBm5L_pk%AQ2@IovugR?jQ@NyxcTjpWG;Q<6dc#Gb}-Vj z3*}vS=Uhs{aENSpAo~0I`qh0^J<5)B?0i^pd+-TKdH*DL>uNrdzyNS1%AA-BRztgJ znT8EFiQkbENNRfd4RSIvsc&5h^((MYQJGm;gFC-}KKNZ%R~JC)akkO7ro}^qUipvd z~n+xox;8LCWD77&iVd4*Gm~s#b$won z6_;5PT6l!le&trE2*EyYoa0L;dgUeyW1b+YwmMncLlsD6LX+2LVTkgPm z_P{9GQF9mn6Fnv5y@P|lR}e(0PEaBPZ*T9I%BMM;BXS#cHBpa_5D#_O*3bC_|Gh=R z1kD_03`hEyWZnlRyjw3S#Q^MEFFgA$`XwrUrmzD+^8st851_RQ%8I$hRx>bVQC3s? zwX&kGqSDRwlAXPxxVVb_B@DUU6m`>COv9NO1Xtty(gZ;4|{ zN=lS6_|}bygoK0|{x$s55lSAwK*NYckHrgzwGSLKEd0Zg5qogE0ERapO<6e}F!}N*sQUW)z3HOG z3{q0k5z~+GzpOudd9|n%n+nt-{XpNpVINPXADRo6d5Ztq%6TKXjZvSsXkB;vdYX<3 zCmi-Ceav+Tpdr9b*Eck{@R6}4N2W47ynzwUo{)NiFXD2lBtzXe2F#=xaEG_Wof=Eb6;^nU>O-ZV)Xd%eT zp4h=+PMiH0gY`Sc$E&1D-;nMoXPU}?{*8*VFI39b|0oa#_dO?vnw52oy=Tq2>9;VF zoYhM%v!|BdtE&~W2M^ovm|Ol9y%wN3%27sUv-FEK%Lx3SImpByEUeE$u)DikgGQYp zTQalIaI&}OG#3kCj6=nYHFeSJ^trjY75zo@J7^q-(_CVnCZ0BG`nepX9?rRMzH35} zJSHm2zV!(wNwh9o^2Bb7{?9#ac-R-W&5(C21RxJakhqKXBJ~Ua+9NNeww60x{hfq_ z1Tir&7#5xdVoFM-3`XKCVV~QpRX+z+Elthm2K$(7C*^2)5*io&B|LHQRi1?u<4X1* zUe?2i_Kb(RtO{z2StZr=%Awb19X&IN0>zSpqa%FOU}3a>7$_)bW5IEqp*`!$cnJFk zo_e?&7|#P`{3RiwLceX8&C1eJT0wyx`fFxpEbWaP11WM)8n1&snD(9vX%PA#elFK_ zsMah0AkS4AiG$-lgZR{REG#V4)az?gK?4-Rm&;zOGp*WxjV?pgCo~mevJQXdhslL` z{Tq^+HqW2rw&NhG)hUb2bi?kIdqzfQ6Mq6PUt8YMsdc3KqO!l2F8}x*vV(`ePAJ2y}p+m*d-_H#y*pH zDf9AjYSDsBI*$ntMYGIc=30DH>cq#Cn+TN1p6+g4hbLa=pDkgP0OcPa9Kh}MS@aC7 zJ+>onsH!n1%CmjUXyK#bq>(2hU_s{jgF)|*#BE^D0ELExRJ+|@93KDRRv0nmypNY4 zvR`Sn;wHKA7Vf2-6CrKvVDA^C9h_Tte%Kbj!@9UcfA0_+-os-%&*9Z&D;5T-TpY^) zAdA?Nr6eSVOslE;f&qPkM?k<|%FM!&0s8JxL_T-`wT+B&YCd29dO=@N5tI6Tu^IsZ z!NR}>^&rO@__eq=NQSc%WcBClY|-pNw5Q2Y)57;TCXZUOFBI}h=@i1$tvU5w##NpC zhR6|4XAOJJNlPnAaYthWsKG%C&;1st$Hi)(IEBBbNf_F4ek<|8vu!1lozHCe>)P5{ zemGU=&w4)x-c#!*FuP$&>&{JLIdLP~X$m`6k*gxUTYR`j z&R0TiBDY(4qFtiqHQGhfH?L3`UcL+u*MM^u1bl}YlkKeP+a5y>h}axYuHR7~NT%cC z<2yU%q1s~DV#p!7Hx#4X;Z(v1zuf`PRWf7ElP)51d2?YiOX+z}EgI$z;oOdSNgB!1 zezE`2v#h=dwu}{RTY&VKYpy4$jS}_7-icV z>x6J{;mN?j#?G!(G)+%O2N4615aFPGjW(r>KeN2GH3O2X#{AsFTfE`r70(fpE#Tzl zu5EAEr-}DhSY22cMu%ZDdYatHmDdCmfRN)F42vFMoFb#5pv;Aizg~DKS*4j1JTCaB zGPL~?F$bVHyL;Q&1F$p$K@Y*B=6|?cQ(m+QY-cvwd%0e)K^@L$k55$>~~ZasmIm zI-G}7M?pb}N&!I#TD0$J0&ey?I%$eUv<<`wG7CTwb1m_n8D>KE~2IJm+_5ILC`9YsHC zUZYV`r&ESmqZtFCh-HkI{=~26pmbv6=Xn(Wj9XWaUTypj!Omy4Q?nmVVsd7 zYr6&R?lePlR3ss6(bh5Prxbit33+*Wz~-u|CP*Pdsa{A+Ng+|cXTnoV;|9EyOqqsw zc+r|8A2_dM-lQpK+tck`359mzTY&@!4hu5dy~27+%c`BN6ABs{=~M|yV3g$wp$0<+|7X+!LSihU=>AulDcpBEc^TUS&cCgFyXeptVf{=({k~-8^6JX#vYBV8)#EP!c968$ z(NJ3o9Fg^Uk%~MbfoZQath4~{iB0}gpqCe@sX94m6(_qqd8ZO z3DiJ~o-84+xtk(e7PXM`b7uw*4Zya#x$&asfyS$})OK&G0MLFW{IzcZyBs}#mnxb! z`N_?#sGtCh13CYvFyfwK)uI%Cmb=IWtV56-JUl!`ucfRyxi+LiQg;Ey9_a7a(9rO- z(n6-D)x=A&4~a*v6l2`d6r!NkUHkFr#TuX3)`IS~|Ku z6%R<`vjipKw|RF#-D@U-Nbr_{ni_mgQFL@Pd+TX8Xd3L;mw{uPL z&JJrP*mN6zW+qAIRkgK|Z&pmoa}e*6z}uxk3bE$W3zL z+2&V}oJqp}<_-S2`L&ZE^2W=Mgac-N{^pt*m!F;Hn4vY!+r!yU_q(_Z^6p1nVVKu-)fQ&`Sw6+9KP>b$d9G(&yh~vVrej zvze3>sBcg$7K$lZx`jpBB zY&P%i-xcb8eQMNSg+9KksqYcY+Nu1CAd=F3EdboA%kbBiYi!4V{8S73>&}^DgH_XHAP!sVy*qy+|2AXcI@`{_U?FQ5YEpcfSK;^ z?;BP;4}O<@#Ky*s6}lvP&Uj*-2rWsFYP&_AKf7yPL3wPq3pxKlz#cxoYzxWY9l$Si z+%2CVQ?s{UZ`_@9_}!duGiJFP-(4OZnmvLC@V(hiPG-^W=COXRzktP5SD#s0LO)}$ zva$kgT__f%u*W%FN)6~}P=hzNlMO;tlUTHI2na5R7_&Ko)QZ)>Pc=0)9hW%B$b{>% zg}M;GqEsV`0qbm}9qv>KWug=0VGx*WjiA_pS8eUDouxm2{*Vi}a%9>r)+^*F``Z|8 zHRg3$*LiOAhEp_Uhy#Ugqo}vVo=a4?D;22QIF;y9K7f6GAjlW5cB7@CadvioCrrqy z!)-C10Rr)@O{?d{K9|L~_gNnyRr4VkH#81BC9HK#b@kjAvg=a4mfyluswFc(2zs4V zqF7Xe{oxe91- zye*5X565m-%WejWii#E%MY08+YXL|kq@<}3R*p0g!!WHs`1Z6AJ2VbL)Y4e;p#AS7j0yJT4=6bRC+GbCYtv_sd1sCc0wJ z1yV`FaGGA=NA>}Thb^y|k4`_~;!IN`kv`C$si#<^N7h;OW|t*f=y zy2^zBO3Qfb0-`--t0Z6)q_nQv+KRw%ww+BthNhZuc%x8*U}0lsp0cj{@Zp2}jwj#X z;GndeTus#zCl^@9soT%rSuUgzWkt_kee`wY5*vHcpo!B2X(sKuQLr zVRCMrXb>O_8^MTWWlnK=fu`G4%BGpiqyGYNszuez7rsxV^CXUFyWt28{!;2?iTn zHU0dJ&F@P31Z`nDobW>V)Qv-()YMdYf-FLP_nfM#=L+|dnR$PIzuEhS(?n|(Bv{Y3 z$8g51>#YPOHiLG6Lw0DTVxl<8ob2t{=6abEBT(=R2khq4AC*%Q6AP4?tZi)6nbkp| z)zKkY!U)~X0`Pq#iS?Ag*l${}Z`Q!?;cS4?pw)e{P!K?%MX$1}kGD$Md_r#Px}kMA z#dI!;J2wvFr09rgh0Ci5FGL|vVD8_fj{$(skT5?W=m!=zSm6Z%BpHc`^aPPzJU}{( zh>3CijdK_UM1W9OTU7z(>Wr2o{cj~DBP(Mq09juS08G@Ke<*ad)Fb~J_OKx?DhgeT z3`jLv8X8H7iFH=fg`md*5;BmyLF@+hQV6=AK3AiMtJMylJ6ArZff(|h$C!7bqFs*< zck7WHEe+)#8_MhJlfrN7dHtw=$mJ+^S^|}OYI+)snvlgVC^3P5-wFua6&GcZtfw3~ z6}F%YhcP|w*Hq9jNG`>@hWg-#_T2Qb`(cxjdEb^yLl7A7P}_tk;6)|J9d~@1bj!sr z5SC^Q4Z&M!_U3#2`ZYg)n$QVa-I|@bR}Ng# z($bP>!7O>Nv6GI@!in!y%lGf!%gfpNP0#^o0Aw`?%WMZ*TT?afp)d3tCm%n2Z~_?= z6oY`1w6}64GeEcE6cWuMU?(sTugGa;)Hv1(3zrhko2q#p!a+*2orZD*E`e+ zGKwL9)y1L^Ov%p1%Dn?X2*4s-F)%tk+-z)YplmIIKeakX03jJpTgt^tjZ?W;7MR16vMG1UMec$Rb?JI{xU zS-WAk!?`Y^F>qwdwUqO&7eszYaIjvZV;95DTM?0`03?b(`}<%N_(8q=r~uqGhQf{Y z^_L*?+1hBa5I8J1*VWWa7dE^et(g$M+bha$&TE0j{rdF_sE4z~U58GL({w>KeSuV+ z8YSu!tbX^!1N7@~0-ITHKksZ_i7-In5fFecL5x7&wQvaZjydHgQvDnNkOfNYrdUl~ zJz#N7aAi+q^MBrNSoO8(_=hao5678m+aE>Tb2{qv-Uo&H;o*TKI%H&9)@*7Z+gH$d zW%%>Cbn)%UZ<{I0pZtC=LF1||E#-?Q67IBY!eL4&sUr|>#L!QCq0rof!+zZIH z*w`O5HF+(^E#Qf{-b+ap6&C8Zd46^mMdWo@0Se~mNoHiWU|OM7+KwaNIEgZz#B{BG zMUfT%dm@!!3j9vGvgg zqT}Z%r+oQBgA=av@ndKk(SmB0MPX7dQ<(-@joZPKjn77HZvfh|+KGpmlA9(ka4=wC z07^h|g3B1rGfQfpbmY>hi| zCISrR9!f3J%4N#8j+{#l%cm9lwVB7vDT_*lfW%h`dMo^ej@zTvqy=;NWWoy()j{dKl>R3DU|4NKp;irl8}--op&2N{>^#% zH0Qi9Jv}!ihn#vpGW0do+P7tXGtb(}Y95q*<%wYc>xPD`92_XK_q_ld$7NJ(Ty3w* zQy12JtcmqQq-2!Oj%(yTi!5rly#4mlNhQ~?9p45kte%4I5c`M27;~!rn5zKGF zj=F#|S5<|hrz!!O(#_2cNHxHfm-L^INCgGzf>uReIXx5)A74}i35&wk%F5|XZ^ie{ zP7L|q4xkC6-n#7QcK8Ybq472YC?E3j=q*n`ml#cZgBJ3ypVR>&rctUTiVWz`5FGPS z;9-Qu0aKi^u=)4X#PyWd1EeG*y6EWTuz7(eK#F&%?x-74RT1V?k;jqeMia~W$ znL65+i~v9la1%`CNr6X*&(1agAzl#QQba*)>VLN3%@hWT^?)RD~wK(Gg)_U?37?sPd|U0t9)P&^jQh>6GyZ z3HdOs4!xwBX=`f(k?}Ufsj#q+2bfCM05=Y@>AT6)s`=Q6mjIYRD^^!lFw@lm+dVro z(-l55GgBVgJD9J<;u`;#4j{LZnZQ*06Fc)+fWjmCHmWVM4DXPF= z_QM^bhvpdzv;o0Nw0=3N-Mn?)v@)F`URrkv)wQgK6b!MoEw^jNURW0K7%*=BS!lPa z@#q7t+Qzj$10W|3HuhUr@^nlJZqKUP4yRx8t?<&*(u%VX1mV7+rG+zBCx;!TOBf>C zhboj!bQgC995zSye0cjQsz;#r+i12(2OyhTmpGOWO3fS`PV!|EmWn^Ghib~Jd0)<1 zlyqp3<6w8=J-YOW;J-$_`#OTR{p%*c9Un0MgYLWg?Ip?j?WWB}fwBEOCJ%ov$fb~0 z<&FObT%+nMpAO@Q>FDSH{0Q2^oMFkL90K{*^&PV%Ky{>NWDK%r)vEMI$#>>scAu!c zp@GKb=F+6Z+gVx~9>ctM#)yz-Fkv|X*?cGKc}Zb{rWg=wT%4SSP@W=j)+LL2!i5l^ zV^1ku$}g+lgbv?6T&&*jrSopC{?h%~s}SL05f|q0ZV`B)r!I{?E{59Wx?|^rt*3QK zjHIQdfdt@hPcG=r0fc;E;dTHQb8=o7JI!-_1B$!khY#3L+(jt_an?V-yMmB>Diz!n z6r$Cb%`!S6Qg%H7a7aA1a~z->yu5Vn9~<$;7LyhihZ`NF5b~`3_6^2f;`8S>dEYM8 zG&JnzDqqml43CUZ=ih)#IbEdG@K&!(qel-oe1QyGc_zkHcRJa(P}V5pL=wF-yjrfs zgqA&$&6L5EAW)z9djGe9yJ4s(yy(&ZsQ@~EkGUW3(yH5c+oi^{bcc2)>|a6)z;L?b z+|=0f&EWPb;%MaN^-3drpgaEUz_k}&Sgijklhzbm)^3Sr#jTgJK6IpSNUCF9<^rVL zIF?^QJ~Uwll^#y{Ou(`J)u^I-oVF_@bGdu*`^-LIjm?Pw!6bejx~N2ld$rMekB~gl z_x+V$*9YcAKrzwEKZZ9)Mj;92b^)k$2vqa<)y|+!tXFqfE?dv&Dfd70)HD6ZF*z99 z_z9i{onr7a!jo+l2_z7nJe=1ck;^^l$%)35s$zmD|RynT}98GlHl#mMU32?cZo(k16BX z_8Nw+lNjF*4>=GXPrSw$j=4LP8090m=;b%)q)0kqZ}_I$PqeCzmKuuI=N}DdB3fd` z7lm9A2i}UwVw_;~`DCD^h}(R@7Q15(wcZ~k9VPJUcQ$JPW#KqB8QdagC8qno$DZH` z>o6!*92KRhBLtLyeYY=%W$Kg!GNS6wMYCCSB= zS7E;DlcJD)%9@dcJ&HEp=ryEgqb;-bJe`7xg9#nkj#2rm*{a8{lUI0BKeTO}NIFK=5U=s)gvqu0G|NE2<$lK(9re73LLPun7{bVdCgt;tt z#le#6>uYz%1mg4POY6Gkwl=+DwcJ}zRCf}qv_hug9IN$t#-E!HUM&Gvw{E=CrvB3G zE{1t6mQ1|MLFiR0x?s#`*xA%YLOu z5V;`{LV)D;U~X#KiCMRGNr0>jOM;?rTwg4U3@GkOb=H=o%6fW#yI@^!LLN{q?Cakn zB5*0(+Q1xJTT^{!H+q@iLv3^<2bX6y(Os=n`$4!c`eC$OT!6IFLq65U!x1K{#V#+l zw3J0`&;{_qOD+>#RRc{29ql}((mzaHsB$6>PUI*p*nLK(hM%G4+81e*Vid2DOGI; zl$?r=2Ls)zvX#JDYLtC2imF>{IeAdg;d{E)wXv}gj7GTdhmvm;@L*cHx}&v5o+iGp zTwGj4?oWGwazz?ox>yZR+x?%sSTS*NJH*q3?0eHKI4wXzN>TdsQowKsn_V~UW{r%F zLN@{b(eZdu5i5NA-GV(wnJFSNlG0-ZTc{@b(e z2%O414lDP-@16zHI`|gC^UUQ;%b1$dd1-#|QpJB10Tkc{M_L%|a zc(@7p7^MrclqYL~NQt6pY_UecMM8U@gvb7HQd~m$+LsO@OZJm|SrrS7PPAd=(ccyz z6cmccP6Ug#s%XAevjCMEb#ifYv1C|$%!R2?w$vZ22FJ9ffaNl~2;(zBkPY-0~j@y|E;Di7XA@nD(-Em#a!{aS= zerxM}cQ9JkeiHmvM<&=^w1+R(JDHGsMC$#ty1EGEb<6%^f1QmO+Gb{I8rHV~Lic#7 zsiwSqyzetGS0Gc5-w*NBvDsR}0A<$XS_kx_H1U4`iLp{=qp9-6E`M7YNTGm*XQmAW z(*gFka4Rq}FaT!I@$zu~XJJ7>@N21W7=siF)4=(e2Hf5&ZC>G-yeW3q{R0D`At8a1 z)zRso_W;I_^l>RZnDqHo818&QX>K&HJ)h&IU1VzTXnyyblz>?IU14z2CFcMP)hn$Y ziwzD}hS_wdzmf~B;#LerPeWKq0|UwBw{^Ia{-%%nT^+j3UK+Gq`pvQl`&89`{|&@> z8H2bffb_(q0q=@eUt0?ZDu8Aa7XQ{Ay5tbFj@h!;iW`Oy70hk|A7YQ_H=%~j7ulX! z^P`}@1CIkb5+3AzOl+t%@X-OU#9OCo3WND$?;bUPrTo0uL zgc}b@w0!o~CC~dPbQivi?UCfgl%h*G!_7w+py)JsUD<=^rGU9Pi#qv6l98GDFlQ1g z0lXlVT+Y9&F90Nop%gieN%*M%xOP!vavXg83&0Em%2h<AH5^s=A04oR5PEt}*NC;xe zJEJFwXJZzyuK^Q;PFa0U*YA{&1??JhQO$AhUxs!Qpn=lC5d;23Iaygma^?-pfXm(( z%KKJ>?hp)8Lx$@`hYoTX5uT0w={-y`zITXTm0;=WYHGUejIKqJSd69h5y>1AkO7h*qkhm z9268V%8xH_zFTzlbd;ytm)Nh>Nou0B1iGG==_(`EAMK5eFf=PW2JBp@WL(LCU{ zy#Yz{eZ?%7QPOcMbF+y5a*JE+Qo|wK9)c-n>WN!pAaZd`4CeW!WpTXYte#syI=8U! zN=;-8EK*t_uqh)`PpcHM>hn}C`7@Y!(7+Mn0NLBTjCMorzC+VcTs%mizQV^CHLWl` zJv~!M=$jv&MAfd@JZm!RM*ZWtvrY1D6iA z+MNO|8Sukbhb}K8!2YOAVB_$LJ52TmzXJ9-?R1e3!C+DdUWba@=YU*4bY=e3y&eT5 z&Jl~h?aVW4szDU_H8lx#7u{!rpeF($P|3UFgVe{H%J4E`>fizzcep*DtBg$zE!_LU zEB8q!v(NTLi^xwRNgP?zxqp|JB`P7!RhbNA`90p4jU<8la|Hzmn5hh9Bi`peAK`P_0F5ljwIIa+8`cddPUfR2FYMk; zH0qYOsiWqOkg@AFp8ngL=CxlkT&S#Z?1XW4aRD1BPe*p-frl|8Fq6UfH30641T0Ol+SLUl| z^sBxPb|Z-dz|e$;3O0|bF_RybP?-?qP0j%mN{uuyMtBQnk8Jbbk9El>tQ@SePSjY; z%o=jw5fW1Jf3hUnoe~)j0PFB-!KUJ2gRn!Z!?z9Wcwl1-K?rkh@}Bvj4yreE;*a)r zUr>hu;+-ZCFo&b6W(EQuZsnEvFtyzYjYjkvU(giBB`xp)fv>kK6wE{As z@S(0IPnOo@>qQH(8WpC+R^Cb%=_W1R5%=)tx(-r_!Q**yxu1$?rIm)xFg9lA<46qU zGhmaV@Yu?9TQSPNnzzUv8y+^673>DS2X$Rt^Z_NoGbmD#FCebHEHJ$Q9V?l){`bc4 z4#)#7mV&K>e}T~}Z;}Zm`TlaEBN*v0a*92M^6mJxGd&JUQzlYK&pIIJQG&bUAV@7N zR3_+=eYrE%pZWCwGZxwtVDLGu5GdGaFD}E54roeA>{dljzrx%C&1GWU;sx82uRh>E5r;P&}9Eb|at8=#PDR+CW;B@9Ed?j$XSr+&((YpIFmOaTD_ za*9~8j;Eg2Ipcq8c6)*>ev-DcJbZ!B;Gw2=kffXjWt+IW)lxQrTX|RjW`D3L+8ABW z9a(ZkWV8uKET3~PGu1653~>dD?%5N}Mfj*JB&r^6<1_@V`DhLVP@;v>%lAbkst7L^*|#>nXT;+I{$t zfJjjNaWrW{vnnW@L&AH4N!8%oWOHf-|4@vCcp!4@>1eU0 zO#QOph}9t!?(Z5*++^_TOmZ%hW72q|1>#q4w?~0Yje&K?L&j_((n0f$pn$?X(Y~RQ z=g$YY55jCg3av9d4?I**N`LH3LxS%Pu$qkd9TR)yv*c0&+~dHvjTYPu#%sb?)S^Qq z?8HgahuE)FQ)-g|GkADC!PLR5dcyhIWpg8Ul=dL`1=S18!;JOS*C}J>Vqt0w(Q;uy zUw*KcAXCEzbj~Z4jlD-9t#_HC22}jjBQaKh)NPT~f+=TLskgH7&+2=#njI|D@<*7 zM5Ub)250bk`INq4hSx*`(fhDz#-(5#HG6`RJGr99O=`B|*NQy^`2jcCdE58tKcPhW zC^zhb;J<5eTzoh$<|}K@tCM-Zx-;fy%hB`h-T<|g$G|X3(5@qXy*VJco7YI!DO^(6 zu5oj^bBB@k$3+fdr(>mo=C5XA((fu|GeY07A^64EE}8J8V<5r_QeW*X-AyC;4)Z}t ztw}LkT{7cR5pJ+yuvFs!I*5|6!5}(xv;P+g!3^EZO>j!zofw;znhTn@eYUq}L$zppEjPu( zr*Tq$utHQCjd>sKM*C?M}P1BK<%#aM;IQ^ky7ov%gS9goj z^g7(N_Ys7lGbC0iv^+6OJ3KhpJadm)Y-*`#dC(D{eou$fB)u+R??T?_a*K>he+git zJ|VD}zZLSBxm6dN?0h81oE5lgt5TZD**J3t-aT&aU+W!{&lkNzA*+s=;uf+b%Azz< z8++_*A4?&F*)X|pQ9^;cH^LD+Zv2bMR4k+^ha7+)@kln`2>%;z@51wW4?9`y2H`97 zWr4`z+x=_JZ1P8RSOkvQxkfWS8dP7N*sZMhaUU=ez7L-GyS-xKvjpZ{W#{5!94C{v zEE`Ux8YN(dd#rR6XW`meAkcWe$(BqewW?1!HwA;wxp4mAIl$F(KLRDfz3-9i=Xzm$ z0c&`zD3^+uR6cKNZfyA=SnU)tWXef|?%3*nvyrj=R`9V?r0a^XU8M5rU3OLM5@zN3 zgZ;ep8i)4{yb@vTfVc=V=L*b~(wa1TlieM|%4^7^%gx;gMBjh^fQu;toaqz`n*#fB zEd+^J1X?`tZ?8*fLM@9L3jBTG^9E)Uf(P9V-}k?w?c}S1Mp32&3kATHBGN&7mBK^6 z^iz_cP+vPAa)A#X(60z3tR}u1)jEsBr|y1{d=9+uaw{xNpEn7pdnNqwIaf~o4~Xj3 zNV#p$1EX9)a)y3xyX5$I_Wo5~4*b$MAMX~NWg6%>AWNh)=-PF@Gyhr&#qlLRFbWGe z-BS)&MKBN<2Hz7++&cx*V!k0FE`cvShW*xlCUbT=Yf+_TSccu4*%l38BoX?Wd&(C< zvkTgiK9$#M#STB@F2B~+)nd1xYaojW7PP}yawNnDbp<8AgRH)LBJ#tsfV}fzWW*Td z)VFCxH0jqgI(Z?J{+XY=bY=X(<5(#2*vR*I+{fkk^sz@WD1Knm2=P@fwfM;AZHBxJ zBs9_cKwf9P)G_n4!o)LKrq+c|_|+So>KxNQr4`-g5MfIVZ$&tn&vT-&UgqK}n{g9~ zEZwcU?RL;Y9h;WGo{CW7Qo%QmqNLYtPuQ0nA=#bOyUk)?ld8MmY$E7+NqL1|QVcIv z)B&5jxq;RwH}~N*d7-uOZ#+=t((}j0lGEf?m?tmx*B(JVYFf#hJu!f|=tURmYCh9| zh5<5WDh&UNtM`t_`u+d@?M||j?46eE5VAu>yfsK>$!HkavZ<7WBqSsW*-7?Fp=>22 zGdrtn!u2@&ey{6x`~A*8`n*47oagIxypH2J9*_I8;oZztPX6H5@sypP3yO`-Smhfj zDg3=~<@nx`gttert8B9X$B^=-qTXcOFyRggW*on85>mXzIK1Ma(s5Q z%BZD$HeLSgMt^r#UyrV%F4M3K(SKJ;dM_+2^mh2awk}^Q)$3fX=XT%WQdK(sLjS+5 zQt#!bFQP{Gh4UVzpHopw*5_j!G~pTkXWc6e81S)9ox^&e&N=dj3+WP6V0o`>ji=t4 z|NXPa@{xG@`yZ=L=IB)emd7sd+?mgs>yd1*;KC~-%Uu=)BrF=`wiZVFK62;AIBM4VFpQ^`JTK z>&4wF1QQzZx(AT7^1pr+XXMaa#PAMkp3+Nt1fpeDg)a4X#A z=}SAWOJ6lsyW78!X5@C;SSD7fUh{i=)PQNNt?n*cRM&Th&1v2wp9|}I`GaeRAdkWk ztFE@8!d5k~1LhE)1iL>-C)AHos@Cz;b_@0e#R=;Q?=rG@Fmp@YHRzL*ws>pOt<3@{ zEzx2>)t38Q)(4BqGB+yv=K3Z|Nw=+*uic-gy7cM<(Jb?m-YA%Zg^PuUiGFV%cOS(* zpT5BscK3d#ac&7GiHc(th8#K1)`wZb>o0CjOm%-kCTII}YrQcyoJa2az(DKxf(q6;tQr&YUL%vobZ__*25;Kk$935tbjkiVO|aq4tuhQNfI8@=wAp3?Ojd=HzG=bs)V&>2j8D_`Fx}_x?lduW2pZG(1ysCIWW5@C?vy& zl-t$2@vMiSMsX=4_FYFU;fhv7a^TdpQz*D`l|#XV)X~wzz(63Tk(wWh#hK}8q2Mg& z%AtA+j`;rFSW^(PM#zdt>!G2Q89#saZ24lxDSe{4k?15;bK5w4@!iYR)B`N{QI?FS z%4yvj4As5smfl6)A9v6gi z_Rh)Fn+qKw8!FY#9vX5fX0m5x8yIhxqpi-vBg%YFjcX|D#|}CTfylE5>EJb4Glk=g zaxwwF%>QNu_Jp0|KDgK`s6@_N=ob|icOa&*@iWp>yiME}i`vd_-z0a?+zw>KqH6AT z1~b88`LJo}*;IoBWxqtmKM<~c8vWDMqysa5LIT~v#Hc9p!k(U<=gq+pkdfA1b=k4f z>ME!0-d*LV>^<=4(Ib3-yFL)pNg9eBK1|uUFBn&>`1o{LO>9HR4$@xt7nk@N&sC+G zcV!By=lcLub@t#=$GoDNkx{G)AKUUI?Ad8w{$tsV(|xJowQGDA(vc4*td&DQ-h4g$ zH81xzpZXQKIN^F3KE)guxuLpq{L$V{y5nX^Ib@!%q999#O5A5-xkjswyW&bCE3uz@ z=PF03XHKmu{qFfV@_G6i7pR6*Y*B8F6>Bhw^pRzhN&GBL&E(ni&OKo!!Gi7#%SX9X z(-l4zn!9pt%8XoGSzjhiq#ajh5Sk%7-G4#9 zJun1GV_#LU)>WUje`KY*65ix@7&U88EmJRf@wS;@vpvH zjt2vautw+LEd$T=;{SQG?@DWLpz=T!`ef3(uhg}PRzj}sQ}`U!`?@o6!j?M~txMW< zJ9Uv!j^6BZj#$ilraa;17ZH@$IFVYb`eqVp#?=hKC*(-@M(#5j z=<8cJKl!CZywt|UoQ~u-gwvS9#Rol~FnA<7xDE)I5j>tWj^_AOolNc6Vu7*F`H4Zs zTd_qsmg9DX#t~*$%Kv1{=;fytZK-V@s`48}eyP?lTH2VzyZJY$NO`q>VB6`>>WQHK z!Oj5wkYj8Me+6l1Xi%I{^Y^+O+R0?GzqcldW3Q z1#jyug#LWb#3~j$a0jl_^jWnzi4+gN$x@T*aLtvI4;k!VS+y#=_!1q=fpwCxzGh+FIp} z>4Ic(S#m!rYH&)E7?#JUcU}$CB&R=j|Ak4JUqVSX-{Y>Ugc~&akQ(y4&Yr>b`RKq-41j$zz1zr`a=;`C~#0|9oJ8ch{$soP>T8l1l`?z6%m@` zWbu249_`@u5w|oZJJb8U10mT~e_tovqy|E=J23*Jlu3-2Wa1?XILK)2m^s*~5Z4OE zkVbG>moSaQ3T>S?-(z*<3XIS&>b=ftw-sSUf8wN%&*uCkGCt>*A1@94_K0(q^n15% zLrka23I~jxvWCsCY`?JM&@ub#qfW-Cxm1+!LH9bEdl zx4&&Ie{)mQZo8qld4ur-`TS8XdXa73jtA{|UEu-^9-z}0i#&-Gd$VIHLENc&`P{(S zD3We-)&GUo@VC`j+pj`igWm?rHTYAh2GP;cCsGZ%8ylM&j%0Se?)p`;n1uAKQ!rD= zfck8^^ybnY^B5lkP!aS9s0{#hqq|ecV(6j>>(f`zAwg9dKOWmWM)Z!j8WALoCnhJ^ z4K53J?am5gH+#XgS`e+vwHjZMTxwZpoUk-D%`9^u|A!?yUTE#ot3Rjahvf~13JRy@ zhK6F_O?iGRAYR~|4m^Xt)M3`plc|qRrZV4qSy*^3r5!30Rr0E#>gzHqM5TK3_<~`U zu~X~&Td^`6g|DpiPArUCs}aKMciq!y>Z?AhyEWq%eC)vxXTx0&pMSMe*z#&MuWxNH z_bX*(9O~OJ_lzVt;s1Q><9)UJ_w`xwjM3o8-CPsV2qnch#=1 z4Ab1ko*oVN$sdI(5Esqq3`FmPqtSSfx{CjDfLq*Argg$MTItnD60~V`$v1 z@{am)zJs1S!w1%U8fP1h;aH(3+Cs2XI{rlmDVUM0@jlLbE6EENFJ8QKX&5?uT1vDp z9i%yk+Z?>MP`2+rbpC&CO3lrAfPo+*?U=jychxD~56+)O!%4LM$0jI@o^}olIOH1z zs|~#^w!0>il}w#O&0je1jpwL`m8~O0x@Z{UPq8i*%h$rt+N>XXh;o!mlq`sf4caAW zC%3glLcd23NXb6f4B)$JfT3JsD(8y@wXJ9qBH9+ln?*?Yk12pkhJC#0as@>rTW zH+HZejs@k-Nj-RT0Q*anW}|J{*QeUWLQN5P*HH1gl)Go#3VMDF4DjySwQC~Sks#>=eLU!f1;*KYr4b`NNuK7vg?{Uu-?D0dbNub2 zl7+GUa$hL|X=}d--6hfa(aQG5+nR&dL+^!s^5K^i-QvA>IKI+BzIdEZ=$=(-yGlgc zIPW_#TGO&V-;L!P0V<5FtkwlRA1+DMM~wB$-(o2mqfc0;PCFjRE%#8N?P?)Cq}L1# zUuGVtcx_61r{0__HP5|tA4=U;wQ zyZa8eT%|&reNj-{PKTncjc3&SJ(&d;I(B~eSbuoq)0;Ef&Yw44tCV_Q(`;iTQ2dg~ zj#uy#J(g0Hp`5p&l9n9BD2YP}5rVX5WPXEk9S`06U~`&~rk7KvOsMlz_y$)ehblRO z$x_Ue7uo6=xZDQiW>v7G7aChSG3~bSYWi>Aj{Gy13!UH2<|q}9)G%et&Cfedv`VBN zd#st-WnYve?>5zKQ$Ouiro!jHcy5111KVigz>#%#TE8y>ks%qCqC;-09xgj5ORFco zF$@ps(RBy-eiJX{Ce-e@v%8bp@2lDo-H)A*@=oB_@GsnvrDb4t{utaS3R;F=A+ZareL z<6#iJm2n-vZ7O;~WqqPRvMR)d1&d#P*t>SvqNsN$O;d>Ve;$+br8Vr>1pm&D`SauOvBc0%S%pf(eaj z^`8OSR5@? zlt~;Q+nzYFqvVU{`04S4^@62En267mWU589Qx2z6QBe)1)GKc>Hh+4^7b0a^morvp z?QU45u#(`=c@i~N6P|S@BLeehxF=9kz`dOxCgB>TjFnsk9#q6XW#xgC3m`gvU#REpgW+v3;RxvS?RiR(q`-D zAJJq26H-gu-`&$EH-rJK{IMH9@iBQKlhrhg5MatySNr$byBDRgw`>SB7?J()L*0y& zi4n?y?_!ycc3Mj7~EZ}qd(GS0PvDm4!exo|pl#^8Hc zn(b5eecazs6?;O`E9-Vr#CycYT@c?$CNrPPD*o#ry)ld-Z4z7xOFT=XrwN1fE((I$ zDnn-O!GIuNrV{t2BqLo?RDV{$@J?*yqy~kmM}X)<$I9RZ{77qmJE18z;b52kS)qUx z#oG5m$@+3AZGZVfE=Fp`3{!iPRBKx}dZWm*No@t=VQboZ>Itv}wRagd95+zS0Yas( ztu60rCwixqk?)R5Nu3dnI(PQ$$k-TY1PVp`qN4U6@9&1t5UC!uugBwN|OX2kb`thIRCU zTof(Kx;i>mC&h>jf{!0RW}G!CJNzFGvj>`j(PnB5XvL~9dD>4@z?K6S_!3Zq)&-vJ zFOHM?Jdt8hTD1T-brX7+tj_)dF)I*x>$_ss*S(6b;Y;2!01gFtxPbxlV>@Wn=jQZI zoETc$$)~D1?G!3HOSWY~rxx&E?m#O?9{FaLQ zv6Y^cwA4*f+JtTd09&3zmNO6AzJ05c{)0ZeT^&O~53ThN+$NB1Gx7-!FBBg_!3D1R zmLQw%QT3>(sAg+t9L&znQit_szYl-C*to9EdB%J@ypP)`aU5ogH^z-*!AVynR#sN( zKYna4iU74U2}%Zj{+b)P5`%7DvNAF)iN|_PmHAS3>g`<=K_o*voKtiJI(l@7n|VHB3L&)!j5Xx1R;n*s?na zvC#JRzjQQo6j&A@rPFunFa+i9Zc=((Szyy@Z)eBvo*Hw11)mH#qY;m(N@k~wtn4l| zoY}F^KR*p@)6d@2B*?=+b?`z6;x&ffaq5Z4?=NK!DF~6;6l<#x{_R}i&bH8dk-lK)$FVOcAL92=Z(ReJ7YWKvA#H& zUU``wv~qUNHtox)#?(8C6v)ym1!=C2-I-MAgN)C0IncsiruLkpOOO>LdKKB(Ve1i= zuVe3Q-6V->J{tyq_c7zUP~@JbFglX14)d~zV}*|X^mEpBPNoiyEeD*ACa zpp$;KK+Ls(ed>9pCzn5Q&m(`*+hbqk?(P<<@uHeikuyxwed=1iQE6#6Cx^VcVY8%O zU^WWn2@w(1CJ#P%vld^)GZCj`HGDgCH<|whw-GEk3xT*Jy(bTM;XR256_}j7Y;K+e zXk20>(%Uo#(>t$gC=T;~NbQXyKi?fwP#DH}F6=+Yk4%9N=&F|!UOXk^4O{mYlbxEL zF8A4Z*LVsQzLmNorsn2VR-Yaf7t47GOG!mBcC@vT2%IeZaI`HWHFfWiBhi!UcfO}f ztNqyZOO)fq$Cvr}P@-c!sGfry&5_91p$B%Q$y(}0ZA!S()L&D0EngaBoK8#8lZ|g~ z`SJ+>tkm^Dg=65DVb!)69tCkkM_c>3Px190$+53ui*?$+G}K7nqUwEpK4|8a>|n{4 zRrM#C{;#f8y3Kqa9DHt&LA!HjkQ8*nNO;Uj6TmF8|55yCukG;v)dJK!%Yp*OrR5J) zKhi9jP%d3D0p$7s_&fRekr&Vys#BN9B!f)XHiUIo-b-HZ`5MH}AW0+2_fOqP=P!x+ zAs)rST(Dtn4nHQVzCnNGN48vGghWi@5W;`B>c^&RkX+hM%*>?vG@N9m;{P^rn4V=FA$lXb&FAFbClzvx&8juD8d_PzhH+~%H>Iv%Q7x}ou4lucg-wfw`7qfz#^mX^~VF-z%zd3x-YUWp11gd+G2 zMGy!DE5E=?vmc}G^ibwP>nwwcW(o_kOObC0UgmqEpK$(>_l=5p= zDpf6^kcd~*%@MEFuZo8Cse^H$$a{0uEuZ1tSnB2sz1zqEC{S#=?e=Z5}^L-#muW&F(5exD zi-A`8Uc2q$jFU?d>-bZqU~~8}r3RV48u}!0D5)jVjeo2WyyD@(R(p_1$g%CHV9aV} zC47sJ%%L*ARu1JMJ1v#+wXp=;cFwRmLdbZ z%FNfE&{}=kY^1-r94w|u*2^RPo9}=IQ&S@&ZryKQpFDjUyD?T7x=~Fx-XeyAMesKW zC;ms8)??%2cPbcmb#;BYXq0FZ6Q&UnDZofTGeKMEskGfA{r$nQTH2&=;tLHuI`Ely z*|2TkYp8}mcS%UE$7qZ6J$oChue^r?rms#^DYoY4=W`hX=4h6?Qt}5O#<#An+J}jG zVKTP8Qmu;Q3%Mh+XYEZ)OfFuGKb>~wV06T)^uM4bx`Z<0eC7>%dtrC?0i6>keiut< z+uP^VSdc`!_OHj6MH1fcnO|2t<}1J1?UWhxgS$JWN0LJIZ}rNN<7&y`@TaAatemq) zp$4Nc(mD7?KqvuG7`u_Y{5lM+Y41IeW_X>#k&T--SPc3JPTNyNGD^lGIN-q%IB!zg zp=`cBOI3aNYxObTzr`zIi$eedDp*JjPD)`-$9LHWn$y%qGg;~UESuqMTe1KD9G13k zS(d1GsX?HJdrjcSKKoX2VHvPOP;=&PT)%Ge#){f~x)XbA-1^L$mt~q>m=*BhQ1x=E zr;NQ>)d@mP!aF9bte;(MxLaq*5-jC?J^u|3xZqqp>;K6@UUjF?zI|s3cB`uGYVbWy zh)Yf&wGnRxdYR#`fq6>%#A)!#cDm;skwsq|#VSwDgVDfYzk2x+8pG1ey+@=GmktUZ zF+fbDN*o~Yo#2ef30mmaukYK5N*yY9`bi}vcSv6T`-bMj#YTMc>iKA`#@$hVMqj(W z{m>*nl=RmTr%Oo1Iul#>6bJ6Z^{bauq&+^7w*LOS0c19a5x_f$G|<=<&Ma zb=GyC-%4If>a-sF-1Uz&!;R$E{;@Rh&S55oD z31FeQy>f^Q6ro@>{$^T#zJ?GsE-sX{06ENAah2o3fdp^EcKNa&UuRvrx>nWe)vGYQ z=C$-+%J(P$p^mBmXD2+iqwj-{-gch7=)=3|G4u19VQB99+Kdq~_gh}ZV?91Hg42Ho zbkl#oh4Uq=;EDctiJF07+^<3`Dj#I_EN{Ri5ND=#8dl4Rf-N|+f+%$ z9{`9ZuC*D7(nTc;y@6FdWJwntDn8ySBuEO+%YG3G`{88X(!MV*rI@3w*1vp7Xl*r-Em-<_{%+^-tul0#Io9>j!8J78VWV;$_MHsZ zO#S^RNCN4OqD$00*Jxrpie%~WlP3sUyn3uDs4tL#EUd~{X>o-($A)qDk$qR_S&IMm zUD3ZXCA!?PFfID!}6J+7@3(httMh%Ejg+ux^KGO383a;+ctYwbO8>!yumm6GqM*chj zx{gUBW^cO4rAwCx{Ig9LzMY#tCD^b^b;F#38WsE<>2u{f{KQE)#boFg86rqBbF&A& zz3Dyfa&BRxS+o&|J@kT!q%OSAVY5l!@fL2ikg}z)tV}K$>A(k^&|CUp zMSXhfGHFmO4HVy4h2!q>W*ss7Zaz@!F9*CU|uWcj_)*Sa?Zg{E{~PE}#KN}Jxi-kAV5XMO@%TUMj^&qGm-JPjTs zEoYN*gjJmsuneS&zq%uula*AD7*r|v7_h!I(au{l@vF3NX>)B3ZY);S2nm{)bwspW z^8{ut6_>kr?{d@M^?2mbu6#If!Q9YSmjLqM#9qya-xT1oBH^(w>b?FBQBZ(N#G1D7 zhl@aQoKMJl({mlXPtm)tmE(jZlDMJtB5G_dlO*fg7WLj%v-HqdO*!P-t+(EkAw^#X z1Gbi?ro7(zdYm1zO(c`hbGQpA*vaL%=*10U6iYRjMfrnK4c*S3b|4PL2|NkC4z0;t zC?9`7=T*dZiSnMRV`Op*MXlT42D-#pLrrJja|!@BcS7h9g8veehxAfbS(o#T00luS zi?JgCJbLYP^x0jvPwiu&rshwmYso!WDG-baM8U6yAbgU!T=xKIld?W+;YxCHAqONQ z@=tqstb#p*1Rv2%X<1os?=c(C1$HFD)Vn))PKMrpD1dEsd3$THYR0V$Q>;$=W=6g+ z$atk2{|y?8AJg)bSXKY2IANn?6)}3QzaNzai(_PS$Yl|$W`~@97DcG959g`!+!W>) z0-%b5ZUBJ1d?WAmMY~c>cGWs|a&mI)^(fO|33ydhW@d)2X#M;4=U?{^4SFps2*B9`zX%$q&vSFb-FM23)X4`f}kWOH4TCPKt@PET#tZLQ8A&IR@&v|?T0I{B2l(@ruO7lg z3-;8U@p@ecJ40(8y>YI$Tk6yb0uwi+Jz0-%ouiWs>1k;(ng<2h(JoqHyc=Blg-Y-1 z*QepmBJiWH58531%qLErLVpzQ4|*fcpwGD7U_WA)2EOtPPKGmQq`n{KAcv3fina9? z*l|u7w?SQiui`XZ@@BdFf*VHvfP~5B&=_=TjT7BIerDu|TkZBbFY z&MSy_AQ(8cD4KnKqeWp3z^a;hSk)18d}*W_DEE|-mzS53d1AgTg?R>bhtU?TrVYaVqGLBeS1gi~jLY*j%39JpC3A2k8=Ob5Zzx{rC z{HYy-uEO6D%Mz282k9I<97@aE+?>KNW;F;?1PEA3U`s}azQpMy*v zvMF@+adiZ}OeRt~ua4eagPwt!A5OTGzGo8tCZb|6^8NaCX#f6hDAahj=I^3?FOeH~ z7)02=U(e|IEHujoNAPY=Xd!=q$}pwf|G!&6Uqa-zn}r2_ey6#99Oz&@RRPC`gsBkYKgUj$k$@HHo^ilw=ZEPmqj=# zeBC>i_6FleUM9wFx6<^Id>6)a$Kr4;kc6R7TcSU(HJe&xdNOs7_FwaRCo-e#{e0pl zC+x4;adNQXSwqyXsK_O=GSXtbd&I;L?VhK5%wQ?(gt3nR-~d%LhaQNN`oC`c^-<_Q zp{M6J8f{p)HpVF~PsBwI{2h~(Pq4@8vKV{aIR%ZLS_p_$8D~*mR_50bj>3R9IU*!; zVjjr7^1pu?gCoF(8t~iR%*e>F7z5Ktw%;QKG&sgfl5s%b_4f9<&-LqWyTdjOry2>R z=6wwTA`-ZLTFgD?5Qr2%qtLyqVV!k#b(K$`ck|)3+l$E&E&DKzC-$ojN;DFvACABG zsqm5Fi5v3v)%^CLz&znfVgNT4b}u#OdrbU72M?mI(Z2DAh!mExq35^4$T}kO^7LsG zaBsy@674~-Rx_*V2=7{b0Tu;h5kG)uZ?`4z!*W|6&byD6L56<>4=J%tDuut(R|g7$ zKbWW16i||V`*zHJFR%$$sQp-Su-FGb9rm54_)UtDo!>#wOE|4X04@+;(u(I1`xd-p z5;iSdtJ!zSAQUX#e+VB5l=4&qSLD7cD;Qc(yM_`M77fl}U^#d1*<)UE^U6Q99iVDl za*ia@%(J&)8JN<;;2n`ehk8q#_Z3t_jWeM>c?RTh_^LNHHmLbQQbdU4e+rkczj400 zTE*QR{OTczWtz@C>`9nGw-PE=>HiZ)JRFeg&nveCS!iwq$&4>7TP~0FkOlAw2nal0 zSp*kN(D;%Im~KS#-0|Z9<8iJakz58_k?%e(D?pkvusc)pgZDV+x4nV?R;^GWB>5~~ zh&@NCihAbllSuvEISkP66*I5Y2@mI0=HcXwhg{AbuH)<7c>%qO*sH%+`Y7!GjkMi@ zYR}ZTZ8qlfoyVxEND$X@OiKQsEC6sjR79zJ(t(;heAqS7XTO*jYJhlx&Fa(xl-Kg> zdEj4z1n*$fD>0& zNElC#-kgV{w*Ja7G`w3zb%oEMSMqBFIcd$u&uYRkM>Y#jL?>TLa8Ufp#(&=;kty^e zfAQNGh&+ioN7g(zaK+$3cBkNyTU*j(d)X zJN@;mSDI;G@bMirj@OhK!$yp~*akMky~L4p`Gq1xg%)tbtWs2#r1$`M0e^n8lA@qw z-fzd%lZ5{=&JuC36X{^>L!*@gn=~XU_%cUo1Y~LT(2O+Q#dC`Kqnt9!6#HZm8ylM= z&*~Pc#ShdmpX5TOKh$7+2k)mQAcgu6b~lbu5@+OG)pnxmo8)9v-6Rg(J!);^F3oRs z^6TT=BA;we?Uc=NKQ=#`?Vnl6-y)vHPB*%9?nTDup(lF`Yg#NQI(^qIjph3$qAQ2l zu+~4XyyrzrSB8zq>Ubho!xma zQR4dRNuyjpE+p>hxMMcxv)E3nR)<5Sr#*3c;K95}Ujq4=)f>pPJ9eisj2shRsF%tP z`c~Ib_t2SJ>WGwjPlv0uhevT*TKz`h#eR~7j3%jh$qxcDGH$52iwAsuMzTyuIL&iF zw2!U7$PLc`vL~-Jrq9sF8L2fs%b+}dHnVg4kJvU@O;d0L6ra&2k1Ic#;xXZury& zv~Cs)WheS?dG80WmOq8dEFCd~c+3V|L+H6iTtBZys z?GdXBLv2~(W|iLT*1r!OJcs~=m6vz!k6(nVA&zTsur!2k(fShwy!CGyADO*#!&V#2 z7{Zd7p58X;f|XlSeTXKZIW;dB{2%koPD$r$#hAZ=V+Y4Ebi0q^WbhWnD{%Op+0TMX zB_uP7b;^R;>4=|_+ihXotOSx~pr2N;yHc@mmudtxu{S^*BrrnU_dFMby{<(%-BU{@ zx`kl{_bKqIAhPHpn7&{)?d!|d-w&5R(NSAgwu6eQ?u(VUBx%5r(kQF9i7|N zBt~jDfMn*L1~WT5q}WaaLRN|rii+2PvXr@K+hRz`As$$>CKB}trQEgV+Pi!;^_TKp z+}x=4?eiw-lob&n5-*X^xs{>z!)+Epz9L6KEs5)j&x1b^7^A?0KI_1!|7Eh(&tPu+ zwO4$|_cKu?Bll$L(|=<w#=*qo1l$Nlh`PDEhZhv)AukE>Pj~o-PeyH7 zx`mv77xI(^@Hf}d}jR9 zr)07bxJU2%jzDF}8m^NFu`aaFbIqbZ5FPB>XM^7lJTymP!^eg#T?9O1+fD!;BGEx~*CU28Na@;9WP!WaAkmI{GL6mUUfU0PY$rG^9; zgkU1QE-_IFP;JB*)fI?RFm!kJO^3txw;8Wr6KA%z!~iz|aQ-;LOk`C7D$m<(ky%@DscCk43QCWuDkFa4Z`Ptq+Ce&4UdBvhn5+`JCNO?(( zFbORJRe;hm5&&Cid9H_GBVfA7!n;bWM7OepSKkS!6(Bw6`gLq;fMJAwxQGleRN2U5OEndSi-L*SbI*oz4L^5x4~Pv5%fSUnXY zW`NlBY11|8WRZCykq zld6+h?fL1_lp_Qw=b&pOZ=W$fN=&4WcSF29-B%{NZ{Ik=*wt~rt#SMhb~a`+YK&sQ zeuFx+>R`PDBJX9*AZ$Ba^mlcHIV0BtJ29Zdqg8tH%>xWpz7cMjYS7cxCax;{E|Qv_ z!dc|Po0qUl+`Y01tMc^$f19T$J7VFwcz(Lo2L-JErW`dtq(MlJ9jO+n|Gb>*DcJFj z`kljmh0EiO%w|>=uukYT z0-29@nZ^08EC&X<9J}(F#f)c<0cHe2WmB=Jsin2DxH!M$B3})zVyqxHyKJ?J_15D6 z^BK$=fcL1mwe{|R{OZJb0dygtDjQw5uy~FNn}kH<%Ht?Co?qy^w(QCgui8jc-n4^k zgC`Y|ne%C1X3*;ZEE`!A#2#b&|6)o8SXt`n-(kW@Z)T00J0mO+hw zqS(QMrH(^^r_;Vnxyjt-)ot--T*Loyde~Uo)YL~#Ao2mBe=a z!QH*^)hlb%gb`*T-Cf!G^?;S{NsGZrLO{=j5|b^Ia2VD4vEt;g>0zTCnZ8+dbMz!8 z$YE@8fpr)#&TE!;7_esrrtbVZp{0Dumm+<94R;Em@zH~*oYc+S_k^n>F5uf)f4ei5 zfBjd=P)uXJBc(GqszMwq`V{mc6}d~AnpEYlipz`MykX87Lg)-QlJur0h4d<}S`W@V za^&YLw=9e%6?V?DmU@U+iR`Fd^7>mFawTu{SI!n{<*v;>Dt8&^4kQ!VARAK2QIz{ z<^|omH~{P2%9j7rAVUDAmhAY@Q_1NcT)4b_5X%}1 zP<_9kU~Ny54yl5mjLf&uR?=Uuu}81%p^?Fk0pKgxIYjywl!oUF3?3yV0d1sAj4+8% zs{1wG;vsDStO1p@_4zX-9dYsT8<~2_+k-L{Kb?eJQF6o5|dfcyvwM`5+A<_(r;G_pBLkN#0EYhO5EbqMt0a{ zJC7p0K>B|+8xunwMMk3LN=`byzv42Bb@u@&L--&sSw}i=-J9RxARW81d&awE1M%_0 z&f0yapLHYXLv7`jJ9#W+!Xo7X3l_cNb@Lhl;o`RwywRu2e6#NRritBx}FfB#`9?9J0X zq=q&1pz(1G=SMQGG}7wgYGh?+r%F`dLto-+ytV7A;@)6PuO@y1jE*|% zaq`ruLgO2jmO}suQn9~5DFs3eNXE=iRje>eN=)4Pd?+`|q~s=Ci#P>vkn6Zwwybts z9$2mtIdA~u4;rz3WAHRy{03fFSv%;t5dVp@(Gnk};#UP2_vGxH>^vA+-I!}ee9FIm zn*L*^rj3CTxYy~PGs-+OruWs7i^1J9Z>6Ke7=~y23efAsVO`!T@rm~RA2}fu=8mE! zGi=9vA55VgiBm)$8WOpMHz%|SWOL1k#SlOax(U12EA4gc`HJs-1ZiwkRDzcL3RzX; z`?Pg9_Ek+D)VHf0={_p-C?}o&!93U4V+JXY&qu$8cv{nDZ(bSw=BSk0Zi~{ma@ztP zf1M1}B(37y^G0I!h5?{OEX9;~W7P#mV4IolEeh`^R89auhd7lO7o1G}0{|idb1b4; z2zF{BRR~8t=J}4|xiJd80}&P+ z{`4qm6KPgMFncCZ!5KOi9K0WgHZDsRO9@p_K#pDTYPgDGDbfuLOB2=d($Ju&qCc$U zYzjvSQ3Cw*$S-Ib`bWGLW2Femc_a6CTZ}Tl4sEhCmiy$i0n}mgU;I1uPdZ^0{Qlbm z+gmGUPtNXV;Rvg{J4>@^Vpvg8A)3$%EA}b6oXpIN#g6=PZzmME5{$3f+rK(b%4lDGtq|_rSG^@UIS)mpPwMJk5mp?er0=8jcC@7w>)NgO zDl_xY(W7f9Xe#}9$ancEGCQc)@qF^m(&5)4R}rl}Xm>dW7YO$TUEk6#gNP1d{}Tvq z>k)`Ka|-G4qSf)^A2Hnu{QHE3>p>M|80bnUCv&ZWlsu$z4|AaZ0{fF~GAwc6*n>Oi z@aKp`Y;DRc8L~h@S6ip?Qf3wDC@K28>Y_0l5fXeNB09;EYIb(ndb=k7p;@k; zj?T!uvJ`T~S=s@WJhvI+FiiwziO-KQv6sc_2#aCU20JX*w5;wH*WT?!8&;Gvh`2-< zVFNS1kQF15CkU}aKfVvG;9O6%fdU5!=U_RrBol6LF(f{aFZog7#f$lOd=(u8fT_UVL8E02z9PlTW!7S|f#D4jx`j|d+;{5}0iKIaBg5plC%nO7- zK7ypB>e6vW3e^Y;v}O75H#Jna68MWiy4Pa(_dp6x;>Tg6pq5c>cHLVXW1+;M>QUn) zmeR*Xp;~vaM&G>jn;fB4EyDl3m(yDmaLTx^u&F{F_|8ZqAd}m55BorMlB^iV^`hPz zGMw6IvcKSHJ*HN{UN!r9@Dw?O2HNiAYr>`@r8i>^)o(}Q4ePr{-XrO>V7snl2rte5 zcm!5IyHc_znm(xkhKAX;*x8iw!-5DSAy#Y-GFt!kD+N8mS-STExEqYD-Uc3BH8c-U z85DI#^WI@FoR>>CrSHjLz%HGJSZn;N$CDHDiN7!I6O-HMnP9gT!;tm!X$wm<%2RYI zfp?5V-t}DHWx1{N)zs>5&1LZ&vfi)7U#6+GmbSY|p5RilvSEN*?wV-e18xpzjU|STLdFrJW}9Q1OxOWvt;+Oc&EITQv{g zmfOz8G@NsB8G&4{>zHra(^;Q9P{5W|KyN}Gp-=VdVO{+;2bG(JMZLzI6$^Tvo2Iv) zFTN3q=;hganr6y-PoYUdJzKrM)v>)QbSi3G&@zRsaSoT{i01_r^ZpFDM#60HKsr`1 zuOvt9!~B&`Y-il#l+14CmQWwIx%aF<;L$Bzy|Ibm`J)A&Dbh)`Cw`{>`);*wQ&)x+ zRZO--Jnk`wUh?Ji59SkyM4`{iT+SaqO2ccC^ptzw0p4rhqca{iNxFqFs=w6V|HCv&TZqR;H+k^EFr$$C z5DhtrlBA7g9C?uE9alk1TA_8hn7M+iNY)UlV(&Hz40A}Ss31=6G<|nH4uhZ_9L6`9 zkj3~7-`h9sb_-=)bJzD9yWHa-0mp`u=RfK9w$FZCT)qUAu!myqLFgm@;0!&_DEXx# z%kbQor&o09;~%xbuANsbW2#7+$mlgKwD>sugE{)E4lXS(2hQxgBlc^%OKfK3m%}K$ za=*a|luha#d?QHgzCw8PZ)h;$D!oKS>harZ{M^SO0PtI%OtXs5(YY>x3)~ugFnNSU(pS&e*T$=0d(hb ziA6F-H3Y|aDX&kYfGrwPua$oGHK^BW4XwV*1;D~SGAHFp*}j%MI3U2*9T3c1?osgf zjnCRi$rZeB2~p8ml+B+@2nJRzOKw}C97?lur}F0F7x01M*}U%&mNykYEZo)5VL#)CUQR z*u{6#Nn4Xi6lAIxuxze(LPy6N(=LaIvLMCRVy>r31*LBcFiLpGNeVkEr>V0=DFjPc z#PJBy&qY?-9j){aXbs@r!UL>Z?tNnTozG^+X`bzqJh!IvgLW|F7#H4;`Of3__cv(B z%b@O31tnwNMr!I6%tw28>r+b$V2{X&TNUKxiG$Hlo+REB;<>hqnOSLT^(VAF_)B7D z3Ng)MyxQsm|Dj(P4Nesw-BUuG#ENJH)lQ7e1=4rZt~y!$F5&H*99zVZ-^e;rn@6C&e-N0szw!V zGaS*iAadijIm%)31>O&j*98CmFWnl>Zt+tU2e+>TZLDrv$Gz~Z1C%RtN zPMG)vgS3Fjb*+~Zwiv=9Vpp#GX;Wu7w%Ddl)RZ6`tvE2JC_+i53IY6nffDz57Z(?A zEHI?1n8gY&OKIuT9~;cN1Utxv(TZqkA$rb&6;Z1OcEjqS`dE=K_K1m~A4mM}tmTas~E5qEdy|uZ$WWHVKw~c{~ zb6lpY1_MO#3s@oRsM~DXQ9Bb5w_firU&H#xoH%)}#R+IGCKnlZ=b>zw1U@&CD{NSI z{E$v1SEB+3U!C#UvuBNr(_b>G9h=*st)EEv&K>8qfUc_WS#Ho8~_VgwQBP zi+&4I?W*mrq?5&)bswUw03kt&h-Nwd#0y1H=;%tPM{!ZRugjd7%a`5WC3qEc8;LI{ zjxn`xxYy5_@#P+EZEeEVcPTdQyor!p)g!2O{o|*5dn;D?zRo8Qk5a9{Y76@0k97TM z3j{_$A=V)tTjSmS&HLlyvuz!2`W>s+KUFZ@nU56a3Xo)Z9fRCBCH+bix=Hz zzVGJc1<%}MT#Lw6#x)g5lfB_z_RXuu9}|XiQ8K{`{xN=j(*CDfC!L)O7~I!XAZMH3 zw&44E6y*$_jaICmA=lT1h0y~6Q%nugb_G2ai#6gx!Nr`(Jh>edQL|LVT6bxxsCn{h* zj9B@Hstdj4t+O|QH-~oWCziM}lzgb$VK2p+*ZQ|%;VHzq>4`%Tg=X}+kHOiCl862n z&OhPcv#4d_fWk6ehwd83I~YGe6EASHdW=MBRcp4^Y*dQFDjNPhFL@>Z)ytQJyEL7P8a6&nnE|d3-W-u8SLn$tjjN`C zWnOx`zZuF8B075eh59mTcfaT8V?0{WHqnaDK1O)i!!Ec|=C%1et*_*@#RtpaDGwvJ zFvc0w_|X%gKgYzl-&kYU%=w==wnmrI~=5Cz8Yrj+)(GQXGLj#tZ2iM;IA~2 zDIH~Ge(|@>C~VCSSC5Q$1r0WKv-}4-#gEe6{?>42C$r?O@0Zs(*yKP6V?l##*vRjn zfH(b93M$kZpdXHW*DduS2#t+Tefnnglj$DzLOA6+Y9~CXjtGh=i8E4;Ber2mWhVWh zWY_T76TEo}H7yKfBgqF=sXt!w!xAoD9DQM}HfnUe_jxkErC&^a&5dF7yNTWG?djaY z%5nU5H?PCNw3NiTFhf(LS-LUJOQP`0OC_XR)$0i)ugpxzT~`s5nr}jyuF- zC%)vII0;MxJ42c318M(qJM=GgZjqsk-aNXUp0XUJSJq4^cT~_h8^0wzZ!vp}IZix?a%uo803UcbXOnlEn{~ zDg~(18dv{(6A{zp*V_fB7vxiNrSn~8(IHrbMoVh!t*vW7XHmr(jN|^dLha<(EfYNf zhs4XusH)_(|5O5J`|DVoa%COLu<$<6+4t!3Hfp|~gqT;lZbQu_v(I5}18VmY>ys6D z9>3?M2{sodB=CNVtxEnDBbBAqt=j_jN=u7|{JEeDLH@>PRjAgWuT2hxoF4}9Lyk#a z>#Ry&vn(+)ovN~i@-Dss>fxb+&JJX@1CSFDE(iB0>BoZR@{ zG*v}-f=$gK>bMx{V1h0gD-wp=zK7}xH6;x8`SoHz!odVa$^B`_Q_H}*FziCiBa-Uw z_r*5rg^fz8P8gV#E34L=_>U4MKymB~e_DdV+4j17kx!;=gXsas9~VpB3bLN0S{OCq z=%wZ@-xifJU9ouCu*b>D{+5hT>KD45u3GO(453|@qW0)B{oB`Zc9OoUokU7V;apqd z)(>}txYCiAY2M0YVcVBH#fI|zxrA!TB7GC17X2T1eHzlp_dosp?(s`2u$jZl@JwQ< z?tKeh_-I2?tIRxcj2o{HU4}T7(nFgM6jUslI`qeH{(+q&1GhAu?G#Ro4X>K)nPoSu zWUHCRW2dWVUtDN;$emXMY6dkGJ`Ui_Mbv?-z!_*wvrh>y@QKWuW z6ua?;d7kMcI3Y9eV;MyoRxVj;2x_ok1_G_L%%O~!D4U611M)g%!k3DIn!jkU54=OjZb zBO_ICP%g*_@ljO}auC>ilVshCWBgR*xN+0M_-gE$Vv66$ z@+)1QmFwzco_OS)zt-6jZxCK|>zC#Qz5MkQ|G?$9iOPWaluK1vUZ%t|iCvNk=A5#^ zoZ#gOugkM2SiKE2Bzn7|!jmYAerK9r6Eld4xM-0U}@n29S&CC4w9zW$lxo9E@zI;p5sGMiaEj3=k=BYJ?n65w(fxehef_`k&jw& zNZ-IUiT=n2=WX|WvS(Ovtk}g(ETuwo@rqPJB|B;G_L-sxsvt?*D7uLjM{=A4`K>B) z-(hBKBHU6&7mME5K-92g33p!9s5FuL zy114Cg>K+q*COa26;U0OAw{CDCHE&Qd%`+C24>gCc!M;;`Zq@G3&}g*#aaLlhxPqX znb+uwBo^#(Lg;MjixG!~kV}G9)E)Z4zU~A(!w9|Hc}wXhG$^}qf<-ygStrUep;5B? zN5hFF3nT#MgdSYZiCqg}gVm1pwME%t>eT4{ES4FH@jc&URwEaUihr@!gges_G4N)3 zulJoDDe0KRsz2j+hQ43M%U((aZdJ>Em(b^oaH{qpfE0DVah132#d;p^%;Lo*W|sYz zp%m(vRswr514meKX4$5^p&VzB(clq7IT|s3tn(bvUn=J)v=0r1Tm*8`qmXzc z&VSa}*pbVY{F^q8vlM`k&Yb-y7}ucXr`|}No!iR4dp8i6I2=a2R!Yo^=BX?QElP?G%_5&!;i8}i)}hlvRffy|E197{(>BF7NP9RlEb7dy_a zi7q@f@y`4X?@@~Jhyt_MGxEiKwdwL?CE@m8n1%lIJkl0^`3+C9*@TJ>${s*W85`q| zdbuwyYh@WB5~j!3qy9P3;yn{ZQDz0qpYk3wnoxv;9zKljY0_JcH>LO#& zr>fn37f=sGZo8Z8bGD}M;otYymqQYT@OuSRL5P*P+tLpSD<~WYa1~bvHjQmdu3g4G zJ|9XBO|HAACx2HSc-NrBfN>0e#fpZ);em*ejdTjD>Ia}kg8+?6%;OlyXP^cW)sYB3 zh{;_ftH7JIi3SKivIF>@D=Nv-?$^tAr`P$~0{3*t)X~#h%I%v7K2!`A*Qu3GOy+!f zS4T(r{Lg3+8j=gQR#3PVq@rLP%&5hB4bB@R*#yT9Ao^Gx|lyWydEDCLVwHvSAwfB5hfOFzpwM|eF5L}kV4tGbS#-apU< zpJ^xU1pPdl*=$@hWqSs2N){EtuMQVcxg_Nl=vtiEWM(~q-F=|fqOmN-w!Hr@$;X@V!LBZ27CW;_KLixw(j}=ts|P; z5d#}42ary7P$vq$@4w0kS^X}4ehQIN8~U+ska9_1UERj^)p&^lPNLBUw`>5B7e%p|)l%lw zCMb=$UO9pQc_k&l9q^17z8Q4UP0(&<UjwpH5rwQ%(WJ{ zR;_AbN(d2u_uqPHlrLU4hGeu5?_==!>IscMVw;{wa_Jd+5%~ zgX%C|{Iuu9LNN+8njEIYW!!4_%J_#4KI|_Xm8>v)mF$8GWb9p#GHpF8xleSuiS$4G zlzi5N#Bjx^4)7^C*Ec0uos!y*TSbPb!>k7|+H0S_#hb|#THEwoiBUv$bGFw|xI^5f zMr~R-Uj1D+$2q*|DlWS0dBHIyf>(v`h+@O6!eQQNz5VVkjYM_ qC6@@?ZR3OgHxNhchyHulGcy+~-CUmVv?8B`&!!D_me~vz=YIiMi^j+Y?8fKR%B*pJ(utI z`ThQQUXNF)I8L8)f3Eww-|IRd8fx?>e)aoW>>UCc_#(0E13gzQM@v_CQ)de_Er;h$rY^1)PbhxOprJ9MDc+aU z_DI=ic;!Jl)^O}yWteP1#Z~r~d0WKI96|8@iNZA(-YM=2Op%KqK ziHv2vu3dLzjOqGp#B*=a$&V4y@KpT*uLzg&&GiIhcN0ni69Qh7&ob{Q@hqQaXmDVl z`>W*z*2b!TP|DFQ%GpGpa;txmH}t3{{9#tJ31TgQKj9Tl+%!BoxAJexH@H_WnOd2kxVQ74rYoe;W7G;@?ggI>5K?pdvgJO1B1yM3|=O+Q+t2^ zDk&+crQKDu9U2+25J=Q5)|2C2+V!ZeuGVfdV2fd8WoeoRE^Z=(xj^EP zXmhp|85;%P-5Y!nTir_D_n0T$% zddnpF@NV!0QxF;U@Kg4N&F`H&d-kjxzl<{oxuC8+gf{hg!miQlMBww6(|`XOcf%d0 zzr4J|&0Ug{gXyWPtSsrg^mfMKF$dZBNqF<|`TKSgzJ#AlM2N}BN$Jy#%BJ&8M?_|c zVJ-4xex4Bt%NX`y8Y{4Q&2?pYSq@`*8lNf@co?`47JqMkY5e|No1SZz?$p5d-nzg1~fQgx;}pV z81^trg)J}+D^RX%EXLH9viBpVe7-4P0+ng)l;ia0YcZ9t?#@nBT4mm4XP0M=vaX&C z3k#E{e}#dQ{(7q}Dk>_5UH$$0_i?eYQ3I9fKL2hb^Xuxwd3d&x=g!zrQO0%7Z6guwx0!8NjeGefex-t8o549;w918&|;VnF)Npycd z>eSMqwTt5pnLz_HzECQzI(|wOqKsT#O!UOY<;w6Ghoi#}i(Mv^c-V(7&tS>11k+9U zQITBEj5D|Ht#jZBLOn;8fAJGYRN)}|alpam&xk0TWzeN?k`CaV$HSh%{63+69w2zR z0Hw${@om|ezY34kPK4%%5B2wpT}-?w+_y5he3ZRQVzlo=LPF~3F-aF>ka%o*^9W%) zEK(xfqLF21Yz%aQ;Ctks^tgOCrRWLoZT8YcR%6JCrK;7>sL@gW(;UmZIp~?-w#g-w zuy^&W1DGjwn7CIrjFf{QxOKtFsm>@A zb`P@#7I!ny(lDv@oSy|m#FjIoXlZXCJJrU|g>UI$B=7twGcCo}z0T9TT;YLE?z48| zoh&0!$bA(R?Er<%*63M!qnl};{--N&$_~~+0-J2R1&c7GopxanNZX8wR5Az zRKr+h{xi>NnTt@|3l#|gs(gr0dpx%Mos+X1<@j}136bPJ%S!eGhA5M+J`6p~^R|6b zjAo*(ej~{ktoae#Ry|{M;)g9_V_69V7|gvW{r0Ge7rnW@<>T!p$mVEP<*m7%_Bq&ZiGpNFF?R(09ObW>#&`rSiixAd#>CfGC#`v8~{?WJJi~g?)K4P1);(YF$7C zwSBf@Ke`sL+%T(wj9(UOse zMp&VLvfIZsL=##sRv)y;@Wik62U{MYf6;dky3`SL zXbrW)ripg)8tuiQ)*J?N-A$aJW)r_NHzDYv=H@KOw>3*q$V?&x#>1?rnjPYMk9$!A znEQ-xIgS)33O@yW4XE`!5-IT-JA8tbwa|MaINY+7z4T&JGhKPH3B1Vx9u&4aO~i;>sjEnmNW zjlJ{ueMH2b$Jbzb&)>rZT01yb-Sj>)6*V>Iojdj3r*1ATd)ZI8DW6^r{Q0CXk+^5& zdFt7P)X(0C-EIgV`lz@?k@m`T?@eEWcSFZ}hG$!DUtizld}*_QaQdEpzHX&8X&XhF zfk_zH+>7Dy6FrgU4eOEXXWz?$}&cFnO9*(tlsWY>|(4DbsSgGsfW*AC+o|wP61!hRV&{ZTpVuC z&VDuB%Z{E}tsp`UZ(m1gz(V#s=Ix)yCalH6^q0K4kFs-Mhl{I2xnqP%AjEfm=O5?_K<76S2;g`ja))d+E=}h{L0F-{l z(|Wh?X*Y>fFKGsE;iWzkKH1Es($NybJPnRw-691RiPwi4UcTc`JywDQSNn|`Jx^ev zk+IXfclTQQeGUWz2g+?vOBosu?{Q3jHLYy67IQnJ%wM~}Td^qWR`YeDW)hl}dQbFD zDLnc?H^JGNtlU%oW@|o5{8*jXpL>5~Jf92|E2=47EZE9;EhJKO>&7yno}SF!kvSdd zFJGZE5~Ed9QhH~^!>&B-aP;k)tRF*$_6Y`SY;5e!n>UAIaMY>7Z+St_C zO(5wJf@O&}Ug+&nM=1zNNW3E&TyypI_MSU#7yY2Le2-9ScyRESBSjE0DlYDDRp#*u_3FJzZukFXCaAf5W@b&@S8j5gRBiXJ_Y(aNVM6 zgOZbz6FKQ?y+(d7w}>>gwU=BarlzJAR_EtUEG;ed^{LYPWnTP3H8tR21rDwC_Vm~~ zI6Org{R`!;pN$tbq8%u%V2vUg?@|mrS;0?+SAY~Dm^NvIA}O=~M{Nfyg>^QOWZ4Jb z6z-!=I;SOd{k6ErN)pmvn4O)yySpok&iBmBEMKPDfLo`oxVV@sw5{B}LQmGaxUg{0 zS|~djr`>$K2p*e}f!5uew7Hk@>eVaq&<8~$Zf(r-}p*vrdAuf-Skr z3JX7m$6#m(Kft&{c5F5l7Cwx_%MuR9&z9)hdxUXW%1 z7Z+D?L4kkz|AM0X2&gzfMB)2EL*-MO!otFLbc;E-xa^)k&zziHy;WbcjYRlc(Cphq zPQ%}!+%t^GZ(skyFA~R>rau{YaR0u zJ8fPM4gUSx`i()Ik+hJ|_vzMyKVw6KgR+yaa3|lr!%0=U5sf{)V8Vd)GUmgyzP|pG zCr^GWSE%A#6c!eCb}A&@q}C|VPEAckWUj2N1Ox<}ot-IK7+P4ciYLI2r`zYJlfjFC z%`@8~$;;~q)uTH;IvN)fb3NLSEhgkXb5_*tf^rqMI~*JtKC);~3;g{26g)V?fW-x-sn@iBgUZ*UiAzCw9Wk+dCewIBOjxE5w>gE>BVKNi=dNvxIR&K`TO8@ z!HkU8X7I3rC@QkqdVFt?{+L>9XYV?6jArpWy^l>RCwdDE&=r@=_$UeJ@9I5$nidzQ zYhZx$LN7b{uq6mL`zt>s9(gEcjofrVw9;e)F%lCKbIDy0nO|C}R-*e_*&#>sHhCzM z(Xa_0z(ke8*cP5_x7Q0Tm^RvU8k(AB6?&bm-e8U9xf7(>z1eq4XWY&@>Z{L!OF7mZn_JfC%ogembn8`8ook|X>ERl=2SHH;BeVg(U>)rE!0 zb=LeBfSaQ4%+Agl$a*%u`>f1sJ64K!lb;g*lL8eI!94t)PR5dtn#Z^SZ&K!;zy&So z{aiBHA9y3PhB`VAsPpB<{fvSQ_+u*sT zCdtW+Un5zt5DQ7jh)qr$9MGM+vaN>Bw@?osK9ub((k>jzQ-3X*@gRS|&yf+O@#>Yh z>R8@;T0!4t@nNsl9X4Q*>UVcB>{{(gTZ0L2p`ew(bLke%e135&qT}^3oH;<1tlfa~ zK~aAG{MpfGFPVtnXSw@kVDo(C9e-I3UEPQ8H~(7XuBj}zT*JQa(IL4t*%&JF_!N37 zk>takZs(P=#K*>7BPK!`u`6&6@yRDJNcwC*72!W@jY~E3Ii4gWJUjcA+T+d`?bWaR zTBtAS3Y7e9cntVJ@#ZBwQJ##L_rJiQ__C*m1e&*uP^9OJ*(+xR>86C0iTC=&I?CA%3QiE_w#=T9Spv-$sMbh4z)D8e)}GG0j1}$0RIY5>r}(We_qc|+&B5xyU~%+dW#1`*Xeb$%Gkkz)aLOGdmTh5RYq zY4L|16o-cg0_+A8uqm9WDUtb~3Q4{$W{Wggtdbjq&-u5R`GN+x$~imC!nhf4$<{S0hbLWcBMTwB8mXYtRi=uWhHf9ix?90;=PVU1A3NT%Y+HAq8T&Fw8av?UpL3;d@vBip)^Nu*@ix$f@H19=pZDY?&0jMk%|La`t}-`dtTIaPeL z@)cLFHXv~4I-_d_JrzGTX@XmcFQO&!TJKN1UTU@4n2y6T0fEsUYS;1DhOMg&Qq>;A9bT{2VGMqy zKYXHl(MlLQc!Zt8k)bn}9 zmr#z`ziNr(B7auV;PLj|yI)RX7;-<0e}f7@s#~7_ijYqmh0N6EO#So7#4;Jwkc8-K zy$cGAud!V`&WediNdq^DC{=#{-rjbXNF{2xZz*$drGf4;=8=I{2 zBB9u~O*cXHidGZ8N-U7?;v*z*-0%nBF49sJbk-O$AS6iHMsX0?7ez zUgxs&r|+9?gm`$rUBAe>-DuX*(z?gf3~WdL(W6};OTp%!hnT0a6qRgIk-9}~1qmRo zF=H6^gB_%3~Sd*C<#q;C~87b+ln67}kKdJb5u|hx$ z3&kmRQI9!3_V6t*fS%Ld(ZLjsB?uU3I9;3d)5niy*47F+njvjwsK|#|1!ZOIhPpaB z&2QSw#DQ+;KYjE_?od(qR{Vz#WMpLD+@3t4t9$4Om8!qhztPL5;42kqY4rH9PfAdDGNUUB^ffpwe4T;zf`ZsuiYdz8^X{N*aqRh@UnFCp>8ZsOBTK&W~Z zEtZwxVPP7!E-o&BB@GM>)zj`y%+5BtZ5l|Y&y9@*69IYzWDUaK*H?SF_n9ZEo)7gU zixMN@`Vg5}@)5INUuoqH0rpkR8-nX1D!oBS4-d6`{rVI%+~&0O#KhZC3bWovGs&Wk z59Q^7)IS$Wx{V(_xd+b!=ZQ(AKq8l*prE)oRpH01tg$j_aQh5Xg_n?4=ZK|C< zfI124m6W@5dJ_QB!$Y_FnNn@enCNI%2|A$ znpLskJnCu=yaE3=Y_xOn&h2!Vwg$&$vB;Mp>60A zo)#7s8dwq7n9Ml}N$d|WsGFo5##HHN`E@`X0_)*n)c=Z(EN_9lTnu3w7veZpE4u$wW6Y;ib^j~$?JFJ!_?tK ztTO_A6pl(2w)mdq8MZ$;T!rHg5jl5bqYqOe{6Z3kW@g}ueTto)@ge?93ufl!by!y3 zCc}|VCuDt?^}@-CPu9Myv9YnNEP}HE3M7AJtF)9yfFrE%IG`cP^?A&fH9)~e&<^8b zNW12J4a9XQ)n7;c6nws$8*ZF8)GVhcDE+-JY7^ua00pU$k+MT;3tn9=W7a~JmHL?) zvV`QJ%h1{3-@|C4lXbxy6_YJk_p5uBo$Fy6Pc829l@YSNy~!dKxAY|KZ^?%(_j)NF z2GcZ~>ZG&Z#U&;7c6O-f^V6eSC`~Ob#n<3#`EJ#%5+T`Qy37RzY>1JskG6t*3X6*B z>+7vNw!oJNSRUGcNA7J00G9A>f-VOcNZWA!xJq1~w$L>3*_&RWN)=)1Q8lH zUEG}?|2qMa2;eBcW{h{R2G9ZHdmL(qwAY8jhCl3oz_36XnHkx&P!ym^0mj*>5~&&d z$CTpp1Sx&fq2VbZ2{jX#JGWz%&NtJNk~m|Pwx{a?iPpEa=qV|89#4&rf3_JZgjN|! z0XSL!tMc2oXqrI%rjNhxu6bEG?HI#yRo;^MEx`zfv`By~+Q*OL20vgy`0*Xih} za&+oSwo2W<3^paJ-PykTTk=7wK}noaPKI02$T|vz;d_IQfk7V8wd|bUlC0a4QU46o z91vDB`#@VSUhcfS{@x}31g&iIb7x|c`0#(AAqb#GJYu-89~SpC2eo7k*xA_str0Sz z!WP>vW-po>n;W;_x7_(}icbIp9&wNjQXkQ-y_lN0t)7F3!y!VFdXqkB+N72c{Qgs> z+-8Jx9F0FEDQR+MMwRW`_STk2q6)OFJhfCMIJy)RGAs}L4ay_6xL3j1#9Gy8H6f=k z!2)C#c|}8uDcgC}9Mz63m zA!L7P*65cW8y{Qtqu(HoHSYL_s8zL|q=Cew;iTbq&HIS(Tc8Oyl8t~bu+ErKDkh+u z7u;-ujs|WM8*Nv2e>Z2RlA`*HaoQ*LPsT2M>^)2pa-YichCi+G(pQsHGq6nR7q((A zCQY?ZgUa_6C|o*QNQv5RrC1PU#9AT`=oo`?jq5GrJkWg^z-Wl4+HCfp8 z_-f_Cnscnz8u*D2iKv^|dO!L;w>?)c}>{ z!4xW(EQh;Y$(46(+lF`gNw3(2A-&#nx6r=jO=DFsRNHJksxI?w|0*s9I)M9Fr59f% zEE^YHCBS7HT1z=6`)ZURxK8?}aN;>%@cE5T-$X1k(FMz(?H1>@F%n$=dL8`6m-wom z;(8V>d3%0v_x5r3#y(U_Ya3c{I@>JsbrFAZMIT#N_e@DBXj>MXR2)70g%TD>n^*T@ zVq#9bq~(YL0|P~p@$=LHM=&!p$8qTcs{rosw2Xl^`vyu4yR#Z^8%N+Ms)_x>A3*}R zZytB)$A?yJL+$3bbK0lTq-zstdb2NOsWr;pi<*|!0~UgNV^8_DZK&}9c7Rj;OjYlK zorok4KMxdzn~$4;B{g?7)yedUdA{S@))mBebZ7dp?4zb85iRck%fI)(x3{ygu(*Qk z?DkLX`TOQxXdk#iEr7Z^m;VA1-6s5v#xy6sbQ-)syz(yv3yv28VXis~LBU=_H8y3Y z_yO|@J>r(PVbft7XB*J>@_%#8(MhCia&N*{t9pePX1+{6mTn-SqEe*uYg&yr^%%0| zi(zNJbt|xEd30oCq0NaOHx~R}0MNiyzzx8|#|N=rHH(yQTh0eF!XM>) z`ss1h_=d&AM9(KJLCy0%!QqH?Lm}>~J#9GoM2#Z3&u6_}-D25dASv~9uP*I<8d2Iq zL}zCwpoOv0C&_>OP53e`%*;MkR%(@v@fz0j+KH$e?hz1y*`2U9zw6P-3P&eJ^}CMM z9IHGlxXHIWcWc9Pr#817wjFG3Y#XMO`W)KLDn1)+1qaLYa`&qCsl;|!IMf>Y8qYaZ zH=8F|;XYE%(S!r6O~g|3-Me>0NPK+!aw#z}F`*^*wz|vXh)a(@CAE~)jHqnQHFe*A zLkyS49T#RDBKDK6+tA>Sw)*VmUB69%hX)CbfGI)ztBaFm%IanO85UQ{{*>yfBPGTs zMyLJiZ?~Wotr4!Sp0=+W^Xj*$h_1h9BiI>_&I-MxuQoaA@}_)CI>^JYwpg>$5)%JT zR@3DX!a_nIq0>pYDwOE1E-lfDIWd|)OVK&pn79=S0+eB8Wu>jHt>erW;jvpd6los6 zLe!;CFj(BZyv|5LCJkgb^DCaIdah+>$91B`Mc#Dww}(3X4cBu!J3DLZ&fZ?Hg&>Mf zWUSIbsPS2gok(jxq!wytJ#oEnSAyx&W>#^Z*+1(R3I_)#qrY~}C;7u~kf-I})86Lg zB|J0wcY10epzrFszi+p4Edbiz;`O)v=DrP_wWpKQ&iT>Y;{xpr6gLad4Q)>E z#c)v~?ek=O^e!1%(#!v6mLE&2D1 zC=t7H4P#@%AR;yqAtBeLPGYc|G&p{w%MhL_v8+)r>K2dLc;RcJzuOv46W`#MH|emBK_U za5x@jJ<1x`7_V4_s8Oi-Ulki0&J;&eQ&R_rEy?Y#g5hdQ&L=MYQni3{--sM-&oQJ^ z)+Uu0Nk67$3J>3nQW;)zMGBDT4Jn2;t~iM)7AO`_72ICC4ZkkPSC9y*{WqiaCH8&x zM@icg_HH(Wj3rX^{pdeFK<{#unEBuq5*8*z9gfHZ?iuNX8s^TT2Cds`dO zy7T=!nH+M68C%M*MtLU$K-%J;7_m?x4=%d-!!~=mlfd1)hqZ-?*j2kmS!lL}E%$c|(k;WcVcp z&VpBRWrT9_vXyg;6O~nmN=TzgNn%K!Ej}aFBJqCc9rIiMK#ESPA+-Sx;Ctuyvc>7X zjtL!1uDPnM-d)88@(v8vzjm(1-q6iW1Zmx$CIum)DWGVX_k#2BM@yg7J1;Atz(q$t zJ^8!b^SH#oLL?caGh!roCZB=ke$<;GMuK3fL!qZl*>>J<`S40X z`Tmf4D=0lDKinIyGtB)ND{FFDCD0ZY5rNKIX57@6oqZqFhe{7*Ng8>Lhvc9`f zzR_2YUEl*+Gq!(5OT@*+uV23oKEuT%9wFiDk9XCLeUsS#A%}EIbvfUWy(}ADR@}?y zj{nZ3sG_W*prp{RsNSbwQDb@JcIZ~?e&lulpOeEaZEl(q1RG_C+NWhg{A5t5oRnS7 zZmNbG#E9bMs>$V^_%fq95{z}I!ZfeHT}N}i3`}s?mpyIxcX7HMxOAQAi_5Co4s#$M zbfq_Ha1PCREaF>FOin5}7=LzRl92H1cu!Y81~)U^_u`dhwHQ15YPjeu+HGlRX##rN z-|41RKx>B`)KkQEmtxc%rfN0^bClsr5YG!$f|YH>ES1Sufu+mnGe2*GuZj+PV= zacg~ppYgP#z1_{#)o#4pfBn+I&Mw>jXAP+CG}P4J5SOq~qHA`6=C)pDd($)RpUz$S zu;p1+PC&-uw{wwc7ad1{p4{LwX=kN8uRA%XGY%0w@|^P32C8pFhDS$5L`0C-Z_Nj# zi^u*bdJmWjG&3X{gc40cJ?v{ec``FILl-TGM}Nh1xJS(e{AXUnoq-~kkDz~mDABKc zQxzQe{Q;u7s_I+g@T9QqC>Zd!P*9R4#=ex{?LbDsk-+MbvZ0^Z|G}ndcl`k`*^Ib{ zm8+|3S2U}+siuvlm-AV40_%WsR>LWOaxZKw*j})=CV99^;@ZsQYy^4Y|G3ptluM;t zE6}E%)-jCZHXNOt#PQrom2}VlOPRfLd@P1Uv;=g%t$~Jf%wPjV?_SMU$OpXStv0Y1 zNX#fQtaSiWme;s3g(+ZTy3PfWSy53jIy$#E+wSjElm zczSuwTln4XVDqZ`{Eb;(xA|lN%3r_lh0D0luMl=2>ORQr5FsCz8a;P(4E_slFKBMz zG@_mi)*c=SAG#&oHXycCexADR6}?3GrCs9*>~heweVR|xYkJ8OAA0yB zyfzOk^f?T9;XU!+q3x9 zHq{bmsdMnkHn+EBUapfNtpN?#R8P)@+*{)F>d@BJ^&BIopt!fWHB(>fFoWa$&s*k8U-(>BzumYcU2=`Gl_?0Fiibd7&B`OV zA5y2%a#~7)AVSP15{HOZkyPkKGY+D`(TF`WalZWQife=C;tLiZ1{T+B64|pXclI6QlE0coSa_r!a0p;y5wWlWf^9{nnN_Wz!&H0> z=}9P-Bo(&7kBwsb4W2Kptl0O+?1}n8U-dd!j)$;Gy4PPtqA$=9hAqK~FVWRf27|5c ziMF;jfCsm}&8gbJ7@F=t$?KAwoHm7xgg_fCE1QjiUGi0_^uBl1dwqS8;zVLcKQs57 zp(feDTT)OVi(i#;2rW6S$(^>u?T{iTkuy8Ouj=gd9%pPc*25Kpa+@C*BFWrZos$>S zH5*+PRfX@HT))44me2a+=nEd!btGaiFz{yp?Mk*o-(NZTJ1kl=B|)>X zKgtT4q{O!i8X6kd2H51zJMRT=gaQm{j=$!`s`9!@(tQg>S?hgz$Un78!MF6C`xA?_ zbefR0)@N~>)vft`Tui-av482_Fcf<+<7)6~Rc*4au1~UFLtEKi@A6>W`=Tr2S>T<%oxaUW0@F zxI^^#^6S0~m*gEKx$SfSpA&W<0Zu%k91MKsB_JX~PDe*q9C_7JARifpBgDOt;x;`x zs#`t>{S>HCc5beKSYc-a=xpvAyrb}B(YjG$?>b7QqD>Ns_b@4VHGk^Hz zj)wY_F80D~C1k|p(>(>!j9*8@bWZB;SeTPzOJDu}r5ebdU(+oyz@%Y(16~qHfQmAz z?(Vf+=Aa`>zFgO~v9UoS(MK+@s!f>k41TmS9J~43XAK~ttPz>h(?;Xg048~Po4igw zJuTzq=FZt9`0oSrR9#)GW*mHLf6pw>J)v(Bm=&?K^(^a}bW{yj&Z%D7J7JDOFp&`x zS0Gwj0s|)O-iL;UMnn)!PpBltcb!Z_(gv;h;S`p`%$En81;e-JwFQMI{EBmG_G_hX zgHe)6)%rZnZDpXvKc8Bp+HJS|`A}|mclX@ciYO5uc*zm8;s5~azkHc}y)CtUI7@v9 z@MB`+AhtlNcl2*>Z!b`!(Xp|>8tzyn-`!B-=|=C#9WU;dTd1TA#59}EXX0EcIJr_k2e;GO@GoaE)nY{k8P+|v`IQuT)H!>?av z2K%48%-6TKnc~62* zZ47NZ4f5g^OO9=5=2sABRL05uGu#_@2JrUND|kkOr)H zUaO(@zl7PGz~WkPtFJcrR~(}_U{g&W!7eK!8aM3%B} z_Y6(>DzEE>!CEIJCAGZb%o4?G-|LCz?f+}Sw-BMh5f%{vxGv>QjTC&<{?N#U1ygWN_G-G!A662wXye3S%4TI{HBjgzbN&nj zD)}-BMm55R)`XJ#nn}h1<6y|iw!c+0mJ|z4BK&T~pN6PP@Xx-3VDfmkB7(G+UeVaQ z(d$G91Z{1w4ZKKmCGp{wUoO(qAOJw%8=~!OcbmZumc1y33_4b2WMbVWU0~upWD>Le zmEMIv3WR*VWpyu}OCL#u(&HjnLw`%zy8)vcn{M@B(kH4^ZlJY*GS!Sq0rrSOp_-fJ zKm!&Nduz38N{)ui;~U(1uh-HAAl92!nzYZm-~SnxUB(k$EK6RWBVG-8jfK4M z|0)&|5#e4x;~%;?|E8osyAV#62lDbUG;$lXqy!sNwPiImaRyhOprPR@Fqd1XG>EHC zP8$4O#anQ`KIqKO#s;Nq$<#y@H=VHa>xydVe1_Aj>j^at4KeceZRug_RJo&0nusOo zH03zNM~tb9sy-7TFh6XR;b!f6}6zqmMi zh0=KRh*HGOB2zr90sNc5fEx={T- z2qp9L^Vv+#;o?=*)UcjQ!1d?c!$8!nKL)tL@vo!zCP+{SrfLIx8bl`aFI%A`Nb=>S z`%sGzA*sVHA>Y?@dA@Ds%$2Be2oV>DxyJM3-NlXw2KPE6yzr|zA+Me00{WIsw|gzm zB0BCtdgBQ7Ya>q@p9cRU+t$Xp|Y0U`dVBK)@v?rV@k+l!k>xj=TMf z7yQFpAa4QR_{lD4J6bGVz?dmOh)fC22VhI~X6km(A&=;dpsNJ`35m5?D zdp_SL)Tuz%>=iUB5SU`Hg*C5pW<~x6)Beum(qf1vy?X}-*$qWJWorfrS8EWZ{;P8Z zH(bR8QpDuk(EFtS{l2B=cna#53>-l+mwPfXp&nZkm8ar{DJYBL;ug# z=HcKtUgq@;A%c0Las@ghFaW(w4UA={Yu(#``~77ZW0mrjHrprE`CYmSwDM%X2UI3D zze-F`S1+7=`SK-R_2aUqm?J5P!^6YBe*FTnvCRoVoJa(gJ!JK74p7`#v zv;FR@Jcb-{x3oVS4a0_?iz7$HZnLiW-(aAmqmuId`?bxSKe_KNy*dlYLe=MMB*erE z_MFC_%oSmd%`a^QmH`@If|}dfV9v&d-JA9s1hjtmd?D&Kl#m=yR8wnw`G=gAHFDRa zg{V9RXx$lHEyP^`M0XX_Q7;{3MM(e4lR57)yd3>=KoY{@O1yF`jKUTu(k=PobN=t* z{FEJENH9?a?+aPA+aSemwiyjmwu~eo%J7^lCpN3FMQnSYQ0=leUd_Nlb96 zAcG7_-iQgKA_uHh6cqe6r>A*8)+Z+=LDKYF{!;@3r71ca(_|a*qlY~4>YO$HcouJV z1s54~&lf0&n>@76R;n3{dldThzPR2lnecP_W>e8pBg^5jvDR^MFm=gE5(=hBmwD1u zEY=DpK`_udBnO!$uDyt%ynF{}B7mnDH~grw1{53wi`HN!4H%!vV;S#%u8g-(RJBlS z5fKsT`!&rdLGbo7GW={fPH5`#z(GK+@R&bDN(Ln_4BYqyQz{I~oSj?=*oLQaOo z&Ign%CosA|+7JLRRD0ZiKC-h}P(Ry3DIl+Nv^5h-1h+{7u?@=xEoR!KUykYaFb5Spv%9< zO|o5|mU})t@1DE-cLbUpOyg|24Sgqh^Jc($|2ZG{kO1i$8XCI(^Ko&3fyiucZwJO! z_@4|C^ueHcQGZD4e2KWuRONYK<4ZT_#ly|b4H^#kGaeN?ulisN@2V&IUY^e7C3Rn& zDP1N|k(cLlmvqyCfIk^ze5MUeVgUSrj5JUG0Y*g=7>CBke+wq0*xY4rTZb24Fg436 zjmd{Ez{ca05TAe&TbE3qp`xdpJvb}IB->sum5l&+&5HyO5!=h3aaAsA5F4R|IEankP2PLuLPtrvAh7i3!+ zJ0J&wka~3rjlO@I*p#4>i`FKBgtXnk{ys|LeXKBU-N5QC*nPdF8olV29v+@yu65Zz7`V3S2gEci43BitO>e5HGH7g^d6rc+h9c)h?Ud?(gIg3-a z1N8O7oy5ZZkI4p88JY#d)==6bPPOPT)lmLsjjK~>GiE5E@tHR>29uGADl=Xf(_}}BSONk zy$(1@0CtdvhCf^^AXH~9ie{2pPx3Vwni`oJ{)r*j?S#4ovzx?vtL?q7O-jmP@?jgi z9B^DBWG+58#T|KWD?V1bJm_v}H1}OQ&D}0L8|XfKe+UMH`eKF`^4bWzt-ycZ7Xa3N zl@w!l>+-U6;<)XjY=qQ-p1C#)veT?WW!nfCl_e0Tb`%8#@6(jRmXfhQ;875Ix~Z#2 zXmcKjk5!bFIj#jNUygNM=4jj@LV960{SNvluP!w?HK%O&atgZPj03{o5{*qM5ij?M z;Oh;2Eezs-jxA{)GluR_Kw0e76>|sju>tIvESU2^@o}j_OX2?(Ao;>%oOE%1zW96hMerV06}kRUEo&eNv@ria6{ zQ9tA843iv1LHGoZv3IDRy}_T|UeH#Z$cLFPA@UcXP1jX0yP-#R!nXB-aYG}@cqYZ{Egy1qVZ%y zl<;!Byk{92lJUPmw90g~Qz~qSL#?y>CVWhogRwUf&dE;e_~jm!=xzhh0zLeyFuTmn ztg-b~(F-6bMY!`DSh0!8x7T!ZM29-Os%{znRGgJ5$mjQzYWNti&x+nPO=D3 zye%L-7%)-G+XL7Qxle$Ld3kx|4To>T|S|`NBG* zVT-y~chhDZ+$B;T23DB7x`yPCzI`#fdW7d0IJNm^SutN;pT#x^wykW!pn<7}w*v#D zZ9?uNy>CAwzX=nFZuNAd-4+#knu+@%GwRvdzL`n&Gh03BMy2PYI_DYdJG_LFscMxIT(L@WoS`s0IUpu**Lk@m_S1m{ zBJ*#_a_zb!WCP&jCBq36{&7O^r>OV9i7&x#$1Apu8mlJm6lsm-Rk1;->>p{sP&z2H-$hs~GCFFvQ2=j76VJ7eo$1I{==7dSj0laA6u z5&fI0Yf3#mrbQ~iRCAp^d0sVb7PG9$$>m-{UY&8`X#juc2IYd`{XYBLtH@5g{ zc|*(2U;_j&j@yZ#3hRtYQ!K<0OcsTIC+Es0*z92FZ4$%zdg~uFETVl$uR?qjasjVo z*`ytCt6pn67v=l|;tz%fWn+PXVP2bzm+Bm&b>7$YD_~eekIOQ_B}`vo|LCZdAr&;# zI->wa>$K;!Q~!sj^N#1TZ{z-HWMpQ`R@r1_W<)5Pgv%D$d+#kfNwQZ$2uYGnRvJ=IRjSE`&DCk z#iz?{y>IPjeWBs=Tdp99Q1{vBeZg6tC1M#wQTCji&0cmSXTUKjgQfTXorc{d(%%G+avS0clqD=Rw z0~g7U(cnP?kfK88zw5{@|28gmkdX%+&j`k7z_;X`<61~)Xs@ES@8>s<%}i{65p6_W zEziutHG0JyPMG3GkWDLbIzV)TRi}d;`zlyz;*25Wph=M_esJ~S@luoj*<3>_Qww7V zt;bH_+~pjO`F#i=-)u=`_pQ22*E@P2yc&+^h``E@iM8{&YCKcUC;sO4Dv*;x(x&%- zg*)j#R6CEvPSUm62j{9hDB`&LFTYiFo84iVmRma7{%F69PyrxBbsl8iw0;Y7KU#!7 zDfK+5KVN-0m4(mHvglNI@vrH2<2QFq=^M>a-fmp<&S*)eGO|L?NbUS*dcghYR5-z8 zgE{l?hxts#Jst%@srB!lt?Q>A4oRT@Og--@I&toHp7F+Kcy}aepr<~mI@8zEg3dX! z7KF~Vq!;@Wa~kKimOqm3eRjL@=lcxmD(zsuq44GNAS@AL4j$Sd_XJ zn!O@K<;zlea%T>g3e=lF2p})8*D6)LtMRhvF=L!IV-IWBBU=84+_<;!=-GnA11z`k z|D8Ofz5nE9L*4`+3M%P--it;hV<$YnHWnsfTdyBZm$g=>Mie{?WQ0#~=`_$FbD!@v{-8Bwi4U0K;H z5+rzeUcvuPd>kw)9#p^@Na~zt4B_451FL>@=jyJe58NhG}!vY@a4E#mzg zugZz;G-LM+c{^nxC1md;@!+y!_ms=r_xWIM0ylN|pWp_y6KbS6AX&5}gn35`M|H35 zt*FpN*symzx`x%SGsp5vrp$J&mj$qSV|$;G7wOw+sd&f2W&4@Fi*4#}ZNMWsXn?s2 zQM=Xnf=*#<_azp?qY!#Fk-~!Pj#ZbN=%_p45Vdx!b`W-wk05?<=J|Vd|_KHp~<~J$uR?GF{us`QuFw@pJhW!`sQZUCU@} zYG|Y-$j_Zzt#>rgdM<|#&OOE|8{2@$z2P9;{@?LKhy?{w|MBnt2ewEkAcUySAGAS@ zCh+Hnut;^jXSbvvex}C9{M#=se1JWaXP(bo^tN9rZGwhrTp)^wWs8&EcPagqAV6lB7~ zMrxz{>hiNs2l*25)SIm-!v;~_he3oyH$$Bh8Xt7sH*$b%S~>q2A%n-sm9H=_8xh0M z#PHer)%gP{toJBjXvWI6Z`j|APQwjRU7z~X+#nK72tvk7@uKE6RHy{2c-<-cd51ej z1{+6?Btjv_BZyI{dXP?U9MFa;vEox z$l2eyrg<6F3$@elHa@MLeOz=s+pD@(0VSzB67t3FxVKq%=cqb$8@+4IET6^x;zq)^ zK5i;tif0R0hBj#*#&yy z`b*Qhemq}%PIjUFre)~s3PCej)JrfOt%P~Z24Y_GS_oTqb{xK(n%*EZeSI+*g;O|D zbEvWDZC|(i1so3?WY$yfU3uRF2a zxX9}R_+O9=&q7gN#9_AaPV7S|(qZ7ll|fJs%I4Q}63Y0|O{w*&o(IEPQIU~G-h+8c z4VMR{nZAc(v$Tu?0*8%(XuS{jx&qNKUznH#k2*VCX=^VZDrj3i?7}npNzo-HgyS%W>Xs19%*f692`gAy!`;D?z&?rRpssEp!iMf?u?;Od#m z^BOrhS!cRmG4Es;LqZtmre4($nr>Yfg$>_G(^$sP56pfPa6U)iugaDH+FQ*c*TS=b z$wo;(zst`14d?LnK@|-}3(!3R8)d4&KeHNK+SBmg^z`+8fxJ^Rtn>W&@Z{v@!BuUJ zKSibWw}Jg+cy(Y1eNVviaB*?fd;DWOa2UD+ZNfj$L$CGdmyY14 zLQ=aAX(gXM8{FC9j%o$&k;F-d3NIfYB4()y&mNL#n-?bb?RBdCk?4JB}6!V+eSsyUh69qC>HGqt9Xa{FE^uDqmU7zcBa8| z3z9kIOz~nM$q{(l(~=LnUR^b8Z=8Mk^#zL&>k~cW5Be7lt)F7>ZYMr2Dk$c$8vm$d zb#eaIkZRCxU`B9hf?wf5#8WFJE27koR<``FETWzN z>uL;GJ_(CW2Kn<*SDg3EQEl(pg$UHe+C&gSI=W0?9D$Y+`~~pG_jEPCprGK6Y&7C- z{9?4HBc7SEyxV0KFe?@=P%n2@p#i0CkToXYA0kQFL6;9Z2MorZ2W!s{Nv@`=;`(jM z6`-eltv7*uoYSCIh@HLDkh~h&!p6o72niXPLA}qIVRul&@??t>0XxKLicC2yzpcu) zJWYDZG3y;%>>qshnqQ;iyt*YTX2G9wxp7Bf2etlT0oAbLtI;63`?F3}AOf?h_^|(b zxKm3vMF$CJp6jaiCqqDtn|(=gxE=hIF%DER4W5e$THrT59n%1!8hDc_pXuo7eO529 zNKsZ)JcmX(==aNd;6;29mr(;b4k-7KfGg+{{H+4Tkv3%=j?KhVwIPrS=c=!U;*igV z?Oj4Ept@RQsVEQH%QWfcX=_&bbN+Ec^*y_PBFcV-EC0BhB=n^*duMpl)Wh;=R)H_^e7w zNd;-J#l*(WR5|C%=Yr~MqRN@sWPa+>R0YUDy{UpKT!}~86yxCHOqnFl%XW%~3RVZm%nsPgJ@$rb527cW&v z)c=|6tI&ymleROooq1c>>aNltFbFL8@ec|N2!n{NU2{&I91|;Tdp-#Jpv=rUK+5`1 z$F2Tzc^xt0@o?Jrnwl8M zCGWHz*Ae(9{Gkaw?SFMauZHG=Q#RR5jB5ivDx|Of&%d@zD1nxBMzXw%?4A%XUw(-e z&|wPvF|j;$La!UjH7K*AO(sI$)stO*N3$YE0;$PjE%m~GNLSwRiOSKF^zp}FPoir< zOaJI}z4U5Lt6@#$PR7?lFAh^W*VHvUq};q0I=?oF-u7qnxjT!AhT{I0J~A?`J?Wx= zK@u7D9(G2sLdIla>t@|WhlPoPFuj^vz69V{;OqDOIdaegInM2tl;k*akYyqHz33GA z!A5gJZrf*nJNBwOg#EsL`Tks{MAxFDtw5?~(LIrZ%Nxvp+;z8j2kj>9pnWpHA0ro4 z&&s=khtQK?n|CFnK-8TKol6ik?hb1i9vE~#=u634gzA7LTT(=)jN>G6BoVS%kMiAT^ou>`0J?Gw<%qh|IpMK+$Xtd znettC)BJU2$CaZvve45l=DG24*b4BK%1xN?TY`_}=gvebb#-)f0CJXm2||yG48b!37Xax%C{gFH$j808jc)e^gsbjd0ylk7dxE1nMrk?Vb=Iptji<``&Q(zYVuD z0NtVjw52n*Uq-I{qsPOk?1w5m^3n?4yo8%M>tB00(T5R89&-9E1XAsXr@b}_N{gpz z$=qwhBceRL&Iak~_vQ|YMtCN40W=Xy!5eUKeagTU`Dy}hjj4i7{suC3^pfuM7z(w^ zdna%u=MSb|oNlDaID7bH=&@$IDH!sttko|7~s&6M!>gU||YmH>>^W(c9n#PuLkV>9RqeOmyd)NPR zFEjIUKi{iv+K9U=<$*%6Cdhrhlx~^4(E*km{PCA{qOSrx1FWKdd(c+*KP=BIiv@b@ z#cpB)dH~SlK7Zz5s%^YKx_LM-pe`X0zJE*?^0(>n#XGEsbIH{B-eu?jgWjJS2Y^Q#H~iXrCwKhZJgK1kZtS#e1`21f;V*5{a7*uk7Z>2L?W97D32a z?X>tCAb=2)zW8|bEAGxUr*OOetORsR6&25mHOWaxjMA?g^uX?Ac?{QI8r=7TnV0+$ z64(8!YnxU5{8#@ClUOmg-GvUY-3W!I&y(Dw+P1|^+V~2>A(n*Spa|g7{?cf~y)j-< zl$+bS%1qFJ%R>_dLMkWu3UekE4;d|T@#*`EBfu%s>_wpV*wQ}Fd zmyg-v#gBHKEutm-w&%a%y!_FL^@=Y@4f7AjGml&$G}90|S}Um?$16bjLFN$<9>iJ5Gsst-Ug zV;SqX+V$LP0_E`CZJaUJG&WoI@LN=sX=|p+xLud@>#mf6-zoX$~n#d zg&vvzs}l~WD?|{`fLoQlj;5o$MT&wjMts`WW3?j-f2lha5S^>R`>cPh_L7_`U2BC-(qD_nWb#M=aB30Kh1s6<#_xkW(wH^* z_+J{THxZNXfe+?r56!3$hou<03;-lAXLxgg@(W(iInXUvXK;#jDsD<#Uy6!0Mpyr~ zU*BnQ^1k~iBh_yMkx`?FfKq0Rmr2Li+fQ+7x8iOg^zXy-yOIX{D(hC2EMKfSq4d&@vgGkb6a3FuwM z97axGR9)~cACg3DikAFI5$k>_Kl**pQWl5HZTx^pJAO*zD5 z2ZzQpC$pNeeJM^aYK98JZkx!2ji@W~(1v|$m3->Xii-#$mLR#i02j4tfg@F0A#KL1g=hD%0{ zveiYm*FzF2y0i6Ve=!kDe#A)jrl;&a9I)IipN}WFo?FR;qe^%83*@jVwB|M!@#9!;id4H#7Ii?9;J4?$w;YPOv$=TvVd6Kz5Q~uo#sR| zsh#=GkeW)uPTx{#wRak9t<*1+!j`%dr%lbmAS9aSqoRCnTdYa-uy>IGy`G|izjCtP zkxGF{IT;IK5e<#H#jlhp?1vGnTw{Mbu|3z)i{<9TjS0i>iT_x=WnYUwt)@COUvkNJ zwb)iM&bu`u+E_0iOS6+p5R%16D0qXEoP7C6HVji}(hob4hG=xtEx0M5!E4-i@Fo;B zu(20&7wPKG;)Y-3vdA;W9e+~uDaFeoSEc2(e>D9zK>?A>AFkV3Cr2c?5_61zP_}?` z+;FwGE%V|Q;&a$!zFaTdKBqQK6ZQYJ01hA3E#Kf8X=^7HwPfh3sxtOn{KGLYD=D!a z5c{jqohV~*RrHeq9hW&M0t4*h@^R6uGDpgw`}SIjcpL zyTMMi@jA+!>qC*x;m@(w{3xDUI`^pClb1&nMXhaim7-PQ!G!S0My9`_wfoq_@4jhy zU_?WYPQ|Zu#+5zJj#rWW&q|Sk0)ll#rsjkZ(aNp!#n1)~EjMfx*~c;uPbT9nlPwV= z{@aklBx4@qPy4MgJ>W7!YaGQk$@VTt{SW8ru*kk&?H28=jJNMm>sgyI8PGV6Cv`{H zXOykmY_|E!Gd9;s4@2^kzM`xn44<$jV zJqcK@-TJ@EkjOyiKV6x8fna4euw%NURz2e0{6IU3f+jYnySexBQDQ^;O}^!Bbfjqb zwheoGy(H!lW1aPmz*%rU!Xwe+i&=*jg3@SKeq-xpl(t@#fT^9bEcFzw%Ym+HSI1vKCs9A|P{m zwCW(SnvnTMlEMt4iGb55;Y3k#TaFC3B0o4xwc1ry@>je|e9s)KB5Oi?n>IyQtb9pF zNl~wBBeb0S;SYLtW_CT&l95%lE1ar!%)SCt88vWhmDlanpBy=Q>P3D?+2VgNWf`=E zE%>U?b#n7bX4-4kDacCJ7K`|mpleyVS8Vj`g(E6Wt@J&e7|9)S-&9*-ya!j`RB!`fQ^CU;Uj}dk% ztx$d+8t=HT?00+^X27=0`HSk4A07|9C3oBuZARG$kv1pyh>nR3^7^`fdsWWP|AxK8 zF_QVRq{Pc+LX5G;Hv(}t=-;Q#60=mis%iy2Nz~r6l6O(Le^EBZ-=J_-8swYbHRJi2 zR`#1j##ny3%DyK%4hI|OY4?-Rt%YL-JQ1RZ3npARyh}Kwktbf@%%4; z5~7k?{i7k-{{83#ZOPL!ar@r`i9S+{x(mPW+Px0OSpm#wwy9L7d7mf(9TkOA zGM^~5TT@Mw$G^h|36G3;`#U#%GaVy1@Rowq=EuQym3bTQu2G)*wUWq?k zxHfGdp{m!xn|;d~S=6S4-svk224Um#mVc3|q1{%`1Pc@*d_CVqOxwW?7tqOYbH&59>QV z0XcFzDa(!g^ZB`BtLGOpg zxG{7}gI(XK%ttqj3kpB&4*aaxF1hhFc*{|U8Wc>kr3&=8-p~#ov&pOew@C7$3fm7Y z_YW{Nr|cxYY&X_Vo!R{1vgY@m+fdRKeEL#7C9gtf^UeFc4GDdPsbT&1q71?)em@kN z5~8$fO^x;IJ%o3BU4o$oV^tCCWQl701ExM)`>54Oh@WpoT`rtrymOH;YgRZze^TwazW5FIcdNm~d9; zd9Qc2HJ*h_BcoJT&yBxW>8L|u-M+dR(`h`-!tdN#$=Sk;);rd4=kVzbXBOX z{wq_`exKAwn5Oy$7wXVu)|Di0fV3T=CNK<^if6Xp#|{T>Q?YSz^|Tir-4?qXCaMia z3-`}K-Xy*K;kKuftoKT8SZS@H;os5wB{GEYW~fy5PKG>%w14xiYiIf1SCm=!Gz8P< z2-=M#5th(TBkPta2A30L;VR5Z+CGpIj1IRXEb;Y7Oun;^ zMcL^0*Ahvla0;ob-JG|02F@Bg0$i;d>@zBI{#l91-Lr@OeWJ5x$cvv8N`p?JKATGw zieJCB+Ke@r`?u!aL}2SFW1BR4BQWTr$tcNf<{N2j34+uqU$vWGVj*BxqeQ2c1oAC! zvPOSlex*drqW{(U40|X0iBG*M3-$3OpIb_vWmN0WMJLL6Jc(~JVrFp%06dN5tV6#4&26yr6bl4#sv-i_S)I~qkk8m zW*cqhRT$D;oI+JGi5qx$7%ypscV>G{zs}|I7Vv6}jZK5(CVdjR+C zV*lRV#RVUN98)}q|KdAbR_3MQr)icLcFgY~Mo^Y@#CD|Hy6npm1gI~Rx^~!qpb^v; z7Euj%S$n(K@ax#+pQVAm7abK^`G@P}7FEyGpD})5)Jf3+G2qs+*baB^?Ip41VZZr- z^hDXy|FYW$lR2sX|2lF<^tZDN9&rd72V}9xbC5wzrp~}WcOkO+zl)Xka?;JxDD;+k z1J4?~*Dmz8XR2J=jkuvm44-850*?c@iVQs=2%XZEM>J;ke;;c7vC~O~h!^>-y0~PUO0X^Ab2;P* z-Sx_eu!}CL?d7(KnCQh!Ah!cSVdX?GwM4nYsuYq0Wwx^5b{C!WF&o-E60Wo_S3P`aF>+HC^8yLh@XgBcVB_#7)yUEB z)4`z|43z=At9lC{Lt5mmYvT!sO_s^p_IA><3RnVpt0;aAAn$3iCl9-@!kMED)3*DW#XxdBb4f-G zQgi>!*wP2nTVIX)9s5--wpjN+z^9rkj30lnTtD3S?s4q?3U@xsO+%M@&T}$3>5?}p zX}q^E^i*s=*^mqDs2!pI47gqv+|X$G(X1_}+irN(YpBqh@S;ka@$%%EL}s}MJ_1oe zj4a=|i@Ia_1RR@%x@94bNj1f>I9ImCw$UjY5P@GO19%IE*3vSGCZG1>C0EcMOLyOS zEM1&=EFQqG1X^ma-y&n4V>gO$l9J|%5=;G{sAU#UDLoiA_E^4XScLjDba$S%y~9O> z&JM=V&hWEX@P1H9!eI>J$pLlRvjY?D-L@$E$-DL*gjCbK!b&3BL{?~<>m3hEk2v1o12?IhNSG~#HBL=FYY>^t*CVvd$4Pt zIEjdBo=&1tHX7j3=W(nboI=7UFm0z-`7)hX0~-O_QG?&*i1+m9;P_;u?kFI;)0FW} z2{!!CPy^u#5*QZDUVKeS8vK*VaxdlS0h~{zywnfqK>L?9djW5Oy+ka^ii)aQE&4lo+9y z2@><)-QC3i8jXnoWMRG6L1E&0_0QV#INcw;`SIpPefadA$&BbYUHB&Mc2l0PEQU3Q zUwC>vK927dOMah>G1WP3a9BdCljUk0H1y(y$xbkHm@p6mMNcM5>s#OMcdSMd-oS0! zGPXj&_uPy@iZm;n@j_!_{i zNE2~o6IFQQaTEJU2=4A6EJX@X)j|eZoH!$Lu*k|Gmp*icvxN!0>%3trOsAKE4np@{ zVTT0mggflmRAqZt)a>gOJhF#*f?>`x=VwTxjR2-JwIMLU^z}#T=fN@d^vM&4LLkID z7H+ z-Dorm5BaRQzT+WF?1N6+1_!c1sl@JBTxx|N-Q`$w64&C#_%+{#7zPv_JEgET3+~8! zFRt4i^%G7bQHx7^Cy>C|m1l}Mx?R3KLEeb5*X%fCx7tv0JSWTF;^~|Kk~JIK?;~e; zr7+4!$2wbew7fna+9kNpFaP@c#U-a>BWC54&E=RC`4>TB zUg!<4Rbv9|!s3}7bR~0|JpOISa3l-+Glexn94f8jYPaw!4&Ky1gJrW@07#wQa7 ziX+t@sW{tJ7IIJy@*+lkG{3rn_^%&TULF&jN=tdt2FT<<`PxLixOW2fWpfP)!%F&_UNZ-oGs%qjox7!X?*GB-tcs~w^AF|{5#S6?t6c_okG*{ zEQ>WoAN%H0to08}1{wD2;)^NO{L|N*`e>%{-x5a}8UkuFhx{p!Xb>|gHqwH$-sszH zvmj52eoNyTXqLlF13o@t1k8&c)mcmYO+?MgtIEpM=n2uc!U(aN6no1Slg8&>{-J_O z+PjZ%f|vDe{x_|AhfwcU#N$u#I{z7*oL<#W4-CCrmc94dYwbuShWY`~tBtRM^$ENU z;qA-DjNY+>cC`^nJKoQ~u?at*#1IvO?u#N@0{G3(VYo=i=g%z`6=3QD%hGiK2ym3q zFUDxdB(z(PQLqUJ+`X1r*w|8Fy9N`83&AVmGw+#Q1%qSk!`)xk|H^f32(Yy%7q;k9 zeSOU#*OPJJT|&wTd^MmuxsuB0)cXie+k@{DL=u3C1?yY?2btqjY9gY)E}F0hy}U&3 z{FJ0W#^Ul`cd7pV=lh7EkMHp5bKmETCN`%JL}eJ`nv6ab7ux=qA|xh9Lp*VIOAPDm;EZ=WGpZ`+GTQ_1 z`S15n&;kJ6#VrTDL;Xr-k$awK0XoINFtgXwqu%=r$k8x-8mSn8U4&t>Qx)(B8EMKa zVf(N^QoFAm#S04y(qZ^z>|bz}yHi2&1qi-OK-P|rNB$83#yML8t(X_dI*fjS6-*l>7Jw6e|!&aZN-`)Du*3=X7^mhpdV^n&k|UTj+* z2(F>Ay&BIswR|Hvq;f{!iE$W@6lQu^@&kIeSaUwgSxKoonvxF&RZNhR)6w~_|KU9M z*<0#<8qbt2;h({&4oyrL9GP4SO+4st0b*IP1Arl5l>|J|le+*){P#h5Bc)2zC3GDl z?`KveT^0%n7CHo{7W^6Ke@&Z`YnL$L?VM1as7XG ztEsD~BIR@klNM+onI;W%VAVm$S2M6Zt~((u7lPV2DdT*!IUP3W9@fmWvzFh4HmKr+ zb9Caei;FvsvVX`$|G%M~qhcU0Q6>eeeez8P1_otL7~gexSfV%yM(+w8nC;ZfjkpKPaCS9NcaxUQ_*^Aw3KsD(3 z2=iKj<-w$s@(#?42-tOj31L=#0{R;8kP}FpbH69Xs)s}r{DweA(fRD{z9r_iqpkH_ zFt!pTL71B2l4|c@EXQSCUHZ~w_FT3eExAvs8)%CE7aTWmZj1-twf1_i!F2!r>^zAb%l=Ekr?iK$H-*>KY}$Uy2k-J^v^uO#|adt zu9V7rN!1D+-~AOe2)Z&e?aQ}82VEyru~zx&Y_G9vF)9my`rb3 zIl#xyv)o2m^KFAhPK>OceEU=j<-L(dn4$5QHRR}01bzIQ4nj5psml2#TQqd`jP}l3 zfbxKQRBs}K-;(H#Vp4iKXhHxo3HhqU5X^kw*1_v`^Y#{<5cJEqk>LTMR<3#n7IQ^q zCFW*u0^tJp*fhk{=J&ms^X`hDT!}}FGyT{Q`!uzWm117r5lw54goj5WGfR_wb(j&Y zHmo9J2{R9&TotGNc+e&5-Y`_|q_RPTwedy-HEyv74&LUv(c&zA680CjBql6bNQnt^jre!!JRNh$pD_d5U&6%BjsE<8oFzZeC| z=ibD_>t4&t&6UaN=k5lD4UCk92a~HV3~f)x`L^7=yvWwS5L|ffT2=dE90=X{X8>tn z8J_|F(&?CA7CWxj8JcjwkFj*9K&;@A_ZmdU#Nug%q6IwRNO@@D^i2AhwQ~{lEGIU7 z#52oyK(#?4Gnbx_5_g-33>Aqvh!HHAK@gafF^{N|$f&&=wt~Z%+oT5BX;ia}l!k3t3d*P%*h#YS@eS4>h1T^C?1q{zZ*nxv zO@Gi}^Nv<^7NS;+7HMuo-}jF5zC(GdNdWQ&Ai8xOW?VI%dlGatdk5sL&S%nHt?g(b zFqeh<8z&*n%JUnCYI@Oz-`&Q(B@ZH53pO3h-V)iCI&2B2yzm3u-P{KFFB&5;&3ZB+ zjEO(-ue>ZBegUwXjPI8lDp!k6CvOOqnFwDIy473&6%T~k`Geqpx&NPW_?V3wZ;`TW zHQeqJ*!tk*y89()vHiYJBLu*?3#crM67HuQ?`- zFZU(DH$S95SsE2g!n;R;-5&ypS$$}u$0HUJqH-nrV;TrZi6S=Kykn2VZ?!Yv3ObBY zAtmqir%tXmH!Rl2+ZkQe*NCrriD5ItgK(*SG@}`XSO3C&a&ygUG@KjR@$JV!`q)c` zE$(Nmj-T?~mr%J@L2VvQ+W81F5@Pwu!TCk~ns%hN0J{0;SH4e%RKgg8MFU_mh!OX- zrbgrIW#YOVb4;64@bnk=z2#5jVfc#u%&Par*=Wb_Y+9?arBk22lw>(FMd9e1bO%C^ z1tXAMTy8k2laLB9e$K6eBVk4ts!8zIv-sYTB+Po9q=h=e#29lPUC5W$t=wl$V}*^- zQ5htmT9VTbWGJ5Xf#QElvz?cX?mg~74Pha(=!tb|lV^i$uH0jPCAq@cPC#SsPjgG0(P89+h&c1q|g4D@g1Ps!$#?aYKGk0tAIX>r_|N%S|IYrui8d0nCVf|jisp1hCd zqmOF*Dfn<=(|Nuj9yeR{$z3#TbcvveXzl<(ggP=e%dR^LM5B_i)H1LyO99C!@+4Tg<5{GW=$cl2ZMx!1Or3h zNj2c2P)3y@d?*2-1_H+IsjxvvObZ8BZ{pz0|NTqGsWUVo-rm{@Tn=!R0;G;pQ1A$@ z)spId(CFLowz#lIvHySU!G?j(q#}ZI)GA8*UCYwf-==ClnN3J};O5 zUXHcc5Mu?RIrA55vgf=;=eqe)h$rTCjPmV7bc89V;Mj!+@u=dNlIJj<7f9VajG2Hs zwJd=KQvln*z&k+1fWQ|1 zFvVL!Yz=cfirXl#zJC1*TUmqcL9$|(EN(rY-?>LrD@?jA>bHQ(fLChl5r8s5mk$Xu zM5}5!vaKyGFb1WZ9U|OM8f>6A|L;F+1VZIn5M2XxSiTxM5FicXWMxGOI6wC>!v#{4 z1pL&8MPI)#U&+;LcFD=C1^SDQPQw8f+Jd4`u}MG09Opx;vse2_wNEQr?kL(dd~zb z0SG-Or&56iBx2kq*N9-?kPOwJ2LM3kTH$trNr*v6OQ?SX)mRdKWus-9Zqd@+HX4@C6R${8cf0b`zvCj)}|h0H+$q{9OoOsz1(E?v%KcP^=jZo0EHy+o>V}N zkJZ0u8^b|X6wT^0gq`z1y|^j+%C13BB(2{)W)X-@p`oF6r4;1k;1&V~W}zwz5oASw z|0dzvO#Zl4`u8;9-&*Aj>)Fs3uFbv~G;?T&5N-}R~l?4dUw!f0WHDf@1 z3GAm<8I+%dq4O`LQ6ZF&t7Z0vYGK1f1(Bkf1|3l|1+*_Are49zuixOsh&NYkvmHbg z+dRzJ;Y{rGw#kM&3ntNsiTT1422hrx2;j*QQ&VeTzx-bpd`(FSe({gNZ@dp4JTD#L zGdqJlp>4<%pB>}=a`J~Gyc_r=U^jr6x?b_$-rnAGHq!4fX0Ij5T);#4^D_g3dYJK% z7r#MwFF`(CvjV0>bH+VM`hETTEG$;aB#_zaSLum~?HPS=2SaTpA6v{AtLfi2DwIIU z9Z`TF#5YrOR_00X&Ai*XmEBjjsAF&JR0Cmp!Qjf53z<$blz?F?yiYbYN7!|-dm%8D z1<1i@0UB)3vw>eiMd=KSA2xh91w&g>b`>Z+TT9`YLV}KPL-e$}>(s^qrKup>*|EEK z?}8a)ZW*v|Za!(u@LrN4oWK*XbbFuX0o)Bd6hLmtr<2`be>7rYq4*K3`U)HZ<2FoK)J=G}fYs0W@F8IF2m@51 zZ-K~{kkXGhfENp05?p21C@o9h|T1YAutY=C(~f(HqY;$)Rjb#|X2QxK6eg-@*z( znqW>L=eP84GM*1=cK@JPIV#R9<3_Kd-b|>--KEIB`Ee?N!Tm} zWwt$mud2kRV@z#M4qA70PSxO$XBnTCtQL@$9K!8Ncn)vG+1{svv+u0qud;*9-I~ky}^b#wR z3pT-a0^(#?PcgFLG@qxT!~w$ml$76x-e-Tw+AQ46IMe=%H#=Xm(Gr+flPv$e6=3}q z-z|0X8ryEST$4fMGh}rDKLh|dUN2qX<`J-)TyfENyHUC4_fs{S{Jkz?93Xfe%gBVj zdUePq@IXRh=Gl`c68`~=OV*V8V{UF)VvAQNEbM;ErE%UsLW}C3of9?9oz2o&{_m-( z`Q)tFtbZ%i@CNTCF0e}{^ij3D+QZ~M_XpEZA@>7wKfkd+AQYca;{ zQ*6s(>G{ISuUrpt`1LlN2y98QelIH1hpyFqoa50rn>cZ ziXNL3$?S8Ke)1oBB~R`G`||2!*}qMh0xGu0sF$%?7stVs;$L8!M71hlX!%3++#Y7a zu_Zv|x6&0@r}dfjpppiq;8)*~=qznWnP0qMbhZHA2GHFiV`5s*>OOp66ckLXyXyl0 zHkjncrSmP^v9KmjAvZr?)G|Zz2v=N8tnSmN70`)Npkg5AJ!!^4BM?LIpyML&u_wnz zAX50^A2soroBx|#;_yFJI=w~sS$i(}(P;D;d)YXU9uLn;8v`TDPM_Uw=CgiBZ*O9U zY2cU8=s)y+FL{L^$A?LM`Ja{U3%#@@!n$mQw&Jjr%L_;zn2Ri7c#y1#yw&gCj8jTv z)b`Le)*g!I^0{9L&ff6KS*C1lY=E@8=dbXQgogy}#mZEyd~X7jHbSN5o_0+Ra%H0M z9v%q+5{5VVGl5DmeFIr`CASJTvh!_3ve$VFrfI9NGs2n&HUGuCJ`)4%1Fs9f(e0E! zn`O~`*i^iQ^7BWnm9_PlI6w1q^!(E3!~18s6RYWsw#?Y*S3#8)=5iw~;hppHABiUQ zYa~QNL1Py81nLDm3UWO;l6a>`PdK~bdF6dp_5kvjHS;0_@m;#G$($wcNfU1V z9FF-KmIDl_CTKiEVQ|G9Nd5PlW)#Nk;8JaGmmb1!!}4 z%lS}10FKVmlhy|#9@#J&bKPefQMOrULuT_aJ8AmP{$q+4jv#<(2j8WBLZ}HRL%5wY zyf^)$M%Li+!ouJ2>O{fuc5xGlsw(OGo6JO5bpa7V#{8;_^1{cf0t*g86mRv9x20T7 zv}{&`V;{04C|IVz#q2ve9s*a-O|epENR;nOwV)9FZK^QWn|9OLJ4EO56UAl)^7PwE z1*x6JPj^tRUDMB5q-}>lXmYFY7LtXl_V+L=qot=dDodCqmjxz0K}E~I_9E8s;!+FR z)pyB?{cL~?cw`%`>FAn$@JKpFl71gElYhv|25ybk_{Hx!#W35#cPkSd#swN1L;zsz zw*X5M)WGAbniL%#WNg8_$j8vc8&g3e^U zC;=c==Fs`R0!@5?QHajgg$8W#$`U9;TC^UE|~qVulX@tt+_o zlnbS!`%paoh{0{+NO}MR1>ef&!eME#N;&6ewydxC&J>-FVi?s>WNg_`iQWTcTiw=N zYptBFZc*{fV}yj3>vKat=_6kA{xr~dAXy{p9YKPDgy#+*^!M&shTZBSM7?3_3xh}W z_0wYqZkn`3QwlpREYPXz>w`ZCX1Xl(X2#w_0{SU$$1rhl*erD=Yx{wr446^)Vb5TM zj;<~h8QH6R5R-srqD=dKrt*yID;~>;! z(nW(545svc1Bj_sL}(1~8dXNH(1S#i^sE1|dv#{#ue#c2v&b{WySlgxX%72&u5^Yf zkfTbv*z7k`(O#~k$9~o-K^X-pA5$dJ!3v=d(Y8Zn2sirK$Da<0F5Xbw+AX~C-cFeG$dx!zXg787+b$=Y z{mo)_1;NW}dtK`bhPG5mEP?RF>u(Z0feqRW@GMa2#-?)%SecR_Ai943UQ5cI zRsh@;6G8Xo#z?u#CtsAgd~+SrE9ywgh;ZD%?!Y#_WenPJm`feuzUfHLh#J^Ohl>&1 zZ}GVJ#f}4c{1akSfy=68Ec_sr5{~ z`*HFJJ(RfhSZB^g>YzYm$_sSS`!y7VSRt3e@=6!aCGHVop<0$S2)TZXiXiUR$Zo2t83mn%!`HL8ezh0G~J5SNf>9(rZX0m}sB10&L^mB&$^1 zmn|=v*mj^x)}8dDqh^!C+)+>n>tDPxFVnt$2N)Rnb<5T$yQbwlmbEwl=}Uk92EPqi=uakkB!suyF*0RT9@72Ap$8*h<8XcY=xY756og>3;Y6h(04?ZYq7-|Ms1^ zdJE6rb^j`H)r)(CTMYku24E8VJJq?kz=`>O5#M9k4-16+*Bt_b(qVs)jDVk4%)8&F z@&PfOifNaC^yD4KHR83N&UC+RiZRPXhgfM2!9eaTa3}y~tJPyZZ30kCc(P@Wai``M zgFl1@WjI4IjGMx4c*@IvxO;F@L*4S)5+qI8KK~!i{yUz__WuKi z-(-_Sgosp9W)!luG_0bsGBdLG$ljHeR5rO7&Q0>@#c!OO( z&OoIX_3N#3b}-?2PDiPrFH9EN>q3Hp2!2R^avnlQeM3`obc%Mrkh@g>@UXtNcKowv z-OIv<7((P^c$Ad6=~#_3;@YO5eT2`~J{p)s4OpI1!6|K)qn4p;Su}t@vGP_ETyy+j z$2@4eUipThra)NL;)cATxtfRJ-i?5g{dIGdWqs{Ma8 z-nW8c&zGM0eW+(bz>j@&u4eP>P^rtW@n4IB_7wp+m1%ZTUw<@dV?cgK&Z?jv#<>J4 zBnm!wfPi*_y0-QiJR|2UwM6ljyU7P+f4DmXG6DYMvv~od^RpmSea?qxT}TLp>+5#- z0GuRzDF~vzVCIfsm!(~;JN82NkA`dP_oBK@tjjx#eiq!Lk3Agoh372K5yq)YLKF*T z#a1ucZ0*imOef!|c~+sCljBsG+d|Rv@>Zr62?0L}KD-E9sKk9nA6CMo-_OWGne`ts zpWz9QpM8_~Ue;jM_NkoVJUVta$de4zv~!7=4faxGJ(^TnLh0Ya+`ARdEBr{)=|@&YoCy{z;2xpi*d zym{@~*bgOmajxI%aRQD926L?t@|h4Pb3ZK1$tG>6U2Zu_>|D&*zTnI9s_aExdhkUq zRUdj;f7@@c-s7aRY4FcK&9_bsS1*Vh!vR77UGY#e5F-c04`_siVxtbqe%uq`?D}!< zKiB~SGjlzWiN^duRc7b(#TK>+9XZQwJ~11!SPQ9BgwYEj}7sbM*B0#WX zbP0&=8X&7pJuJMG6gj@3QF}?(n4y9}BZp$4tC*`TgblXKPKmRJHy=45C=pLQe7W~p z2BkFtw?ly(!Y5`p7^5-I@cnZ8xeU#Z-N0e8YxL3Y#l8Izq%8DKPLDI5`KPA--@8xa z_|0O$nlxD1Fjg*oeeMO_jk63?&l#zLM*S_)_lBJK`SYi0ybe4^lTY2fj_ZjTx*+^f zH%F2X4ss3lLbnr{>FJ-_+RP;)t$u|w9!uQ&aPOO<)X3IIRlT^+`}Dp3{E>h$ra zx0yb5*k5VE1n5#+-<&I<{jQ5e$*1clzoefLe6*)5B!SC~otcUZ1U}xFfA17=)WZQD z(ogvN1ix5e`!DSxYwL6R{iQ)26AGoC6JMB{nVMQ!Y_NkQiiulw3zX>1C#1t)K-_!% z4?^Wmj!l5`0+SQ_#FGLsjez;&v^N`rQkCzg?za=`dn=b=DReh7D$?y^C9BI3xzq)F z$+GitFD{+P|AzQQJ=1RiO6Rg}!HpZHpM%>?cSJpY2QyHeBTZ9JSLZu>?wrF_coM<4 zqBoV|K?u81uP81d>t@rX*W4H`VB^LPabe@jXG^J{AHokvM0VUFxmjPxg#Ekw@zJa5 z6T9wRH)cC&9Hb{?Cp+BlUKUMyr6%Sj@2i!m0@gNDfz@2?DWcmjEiLfV^tW$+pWI1N zU}e1j>HDKck4~D@1OK0#ob=NVOE`7+@#?BAP+{TzfdOf0X+UwUh`22c26;H9@lVwk zO0yAp%o6p=eve6WfqS@eqUw2b$;Rvm3l$j*wkZPBpa$LZh6(qUrzHb1`SkSk%F0TG zN=5t|GIx55qj?i#XGH>v`13oa)Ki^k$pSDCL2wg8z6$FwBHVuc`UN4i5E4LRgGfn; zC9lZW3Bev$&!*5~#;baIdguvlY~~Ezf*58qwem=Jr|1c_H8m~kI{~Mc6)(0KC~R(4 z7G@?!U<|vySHf+;wq}S%;vYG2XC^x=IVC<#xArtz$W4+%_H`iT0pwMZ`ai#AR{H$8 zApjP%D?+jRSy&K(hkyS&RIPWdtZ4P*DZ{~t!Z?FqarZ9%MB06E)Nqwg3vr_0TvmP7 z$hx_)HXely9c|!ArM};_ATR;@jgH=FskgSZm2><-MJW6Ig)K5G+wV?o56@28B~ZYsahvmA0a=;Yf}(73q>H3 z06(UZSlJ$WhC2IV5}7*N&);d1bCFR|3t2H&Sx36OJW8iNC1UVi~Mr4SbDJ^FN2u8d2`$nY%F|ms9?W@QwI7u0^)HZ zkNGTCQ^Fy9d8%y(ks@*3;&#JcFdi!8H9FFM@eI|ZTojI&e`4bzrCvBzM=4yFP4=L+4u+DG|=csgL$ zmeU;GLrWWIJCPKte~+5L$i(C+X@2QYOygot`Ofx6;7}|#G4aI0*7jn|10q%Q*fH~_ zXM$_y?OJvpWc_t# z-uK}#rS5D_;W3@uktQH9WWRvc^Uj}$1jbKIdR7Z7Y>-mQ^((FWL=j0qz|IXvDvMEv z;mt4nIY;mVcSse0vbA;~ePF+!&JXcWQr+cb9L^8UGrn}P{dN}W9mf){&n?tcWJI^j ztearcZ}}{EgbLn{Og$<}O{ZvAqIqKOhOL#Z?f~*)mwQd3(vF&#$4|l#C7XctbR=mN(=DkK>9@SqjzB#nWr~zvLuWl>{RrkGa@jWr+{E!E>lzBEzJK1 zbQM!oVXRH72u^917G*81UYIAfZ%M5Hsrf`t$b7y!^x!=z0#Nz5lO|u9ngE6%r!!uV zj<`aE913k2H}X~A1kQKa>Pa_rS$&wXb^HA(0iU4tYN_ytqM~ERj%~=6C-$`6a`#sd zvHZs8uYfZd22OAcx^|13?$~*rYtScOI)VZ0N6%GT(0zSR(R0#=wwS%R3#|&61mcOZ zU3UaN6l%Pqx<>HR;wkKdFQoh3w1yYn@7f2}Bun&sK-tqz{vKRccfLd~lD@03un;j5 z2)zB?1Ahs_Lj+h25eQ8Gg6G`?$o!y-%6#4V69rRT8%=PH&4#{!8`xthKHz8m?0h3O zHy(=m4#@&uxA`zB#R)xp5D9YM&s$s1Vu$&lGbhWM>xy_93|GESAqEZam#E?$-_3C@ z1R&azv;Uj3^9|!NYmVE?%>0Sf)2z2cHqxw@+4X)Os4>JmU5F$f-h;zidD|NV*)8EQ zh>khR!6!KAtWl1tQ3@jwpvYS@*gAy(Azt2)Bc2Q)5^Lub{DX1g$k7ULC)x5ntnn@H z>6!UsQDREN)~%#{{?uHGYD4TUnO}wU8fgtIdgOYZ+tC8|yv|JTnpRaBoF@qsIW(w~ zRd`E?iEF^E#^!f6RJ$iX&;R+^iBp&8BIQ*!Xn*wh@dvW!pa_^9n8P+xl`BfPq^yip zinJfo$nRfH`RJG5J8d$~hHb7|YaXMJ{mjf`Qy0Qmy1KjXMYEm|5SYDTc;;Lb?2*a2 z_O4aWsr0)wB|mvmH@*Z&ndv#RJL#cx+zAds{5eGj&eBLs> z>__7^&3vJywhA@83hb8ULk=*3!N6y|hk5GB!4M70Td&npTLoC{`rf~4o7=cQ5WCCF zyIhGyNH4w15~Y2$KA2^g8#H+2>H;qe>|8=b)PH4VUhs57YIYr}ubeJR1CI&jW1TaN z3lSLaW79r-{5WjAMj1C?(?Y^6E4zj4MAOs(a|lzw9$Af*5AwL;XFl{>jyHt&=utGF z#)Z_Y>>zXeFRB;D%arRnw;vewa~%oUUE*R3CKEgRubgfyD(KRgqCa5&!J=JpM=|{@ z%JOEp4OeX^!EpW0PmDZNmC%U(aP+_`Ct$-t%1w*LBQHfof7Lew1y;9AOb*u<4<+K1 zKKrIKPxdRe>0*HvGt>cK<4Psl;$_6NM!0M-V4(Ypo*~!P(1OpKE=!aH2m?jvH7pIn z?cU1&zgz%Mw8&T>NMpxF0CMh!k003yu)^sf-Ybrw?5zcP?+5BJO%q;nJDdNOeHL0o z4&xx;jVVz%gg`NDTG&+gFQ?61J1i4xR9`%}KcopQ3Q`NN`G(b`z49_l02( z{=C$6P6v{Ok^a@||N0B8^erJYBsQaXSry)R8yTey1YdZP>@k(>Mi`Yx&^HWbLh>9d z{zCnq~w4qYb(mPq8{5=w}Ve{gwN`9we0 z_F^bfg)YDPDi2daq9q~@kQ*nupws}1Ac7fNU;Jrz0~Dvb>*f!8_cl^3&rTB@lKwBlHY6zFx})Q7qGh3`=8wkJ;K!cIKR&?}2PA+}QMtYP zIO;w$On?3N>s`(w-(jjM{k3R>Mm~MGi|NnWQ@c}O=ZMN9r*gYL_6tsN*z2WdWc=(p>NVm(MgSugyy%*M z;N{q6O24-c-&U4yOFaB9?$WzMdE-Dd##WakY-(l(Z=dI>{QFT$p>D=MIM=0JF)R7iswqOX1|@K-=jji9du0EpySuL$ z?5y{Awm4#@F1x)pjwmgILTjk~Oxj+idD4TF0sEo5^=-?W%WcRYD9y`@mW?N#jyRol zrm+>9Ha**#cdK;z$WmAcwmET6ZoYvk;wHubSa-K?&m9#v2RR5MN0_itEmvzCT4t~I zTGagf8ov3$zW>4&YRJn;M;O};5yI_*qG-$Og8NFJ`BZbf#*G_)f4)gwv8eoH?0{h= z7z8X3B6#6S*pb0(O)< zPYa{vnJwyB76Spdobm}w`#q;+2Fz9q9SqX5vUX2W(U<5|VBEqcW@ZhSBRmA+362w8 zZf!IWyc01Bx-N&Mr3U)&x`OP41onEz@dFlk(cs;4O_%UO1nh+6?+>2h5#p$3OKpQ6 z7Q)3<7o6JBFFd^xTn;PC9E?qTe|+9s6ig6-fg9boO}wW#jQn@VZE4whb2$BA0za#&?hnTGw zg?beM9iec3x6g?Bh4bg;w`__>vu=jXY|g%@`k0|!lvX6)5H%{*61Zo3XSUp1PqU|B zt=uf6$B2UUi4jF0X?3h0-}ANWS?s%fPVlZKXg(H~>xerg-tlMqZN$3FQcV4a9-o6M z=TZwhe(rmKp#;(s;Tf2{v z%d6_FSlk{jq@CckANuIR|ITl}@#$v)pa==-6*DO9f(XJ;Boi^|Gm?ob#HO90yRT0h z+eIeB)FXQvifz;UD`jV2OJd9DoOuz@y&THrb3f+t^tf>YqTBe6AJ@h{iBtGm3U~&~ zH*ZW%OrUaE=n}7G-Woz*i?YRFs3QffsZxsjong1Bw3NsP5gzNG-VEproY>LxC><}{ znV;FSzP=8U87g5fR6)74R=Z6#{HYY-2}=~^T!+0}SgP$<;Up5a3Vjy`-{;3Xe*7>j z?Al+v3SQL#HR6|(yPJ%?n~O{w2Cy!aleUxLM+vjl?pOm4ee zWWUj!0-Y?gk4_U`p0Y>OXJwLOvX6A|moM+22Lk_QmR0E>A2rK+S%RNf{`_~oYQdnt zZgzGHFE7V<`DoOy9^4o%TWB3;e*OB@Gh05{#T`!8gN}z^S@5-Ybfl^CVR+by==PP# zGHTLm|4W8J|GBL(Fq%F2tZ8E9_Rg6m4dAmDk1e7|`b3 ze+n0yx24Kg8de9Cm6eUQp5c_Vhg}aU@xIgqbZY}%6sPm9XDDC3Z1lhfTn@y|`cpU~ zm(g2`Evyj2-)4dOeVYPm!ckwoT;WPcv%RLN>AbdRR3ud0L9rdX=b#Lg>x%5QH3pR3 z*|*{8TV9b-O-q+ zEZx`@)A$Xz8{MXX*?WH5X4Q&T*^9<_TF+RmU zCsuUH$VRj3>YWk_m#!pJ#zHur>9+{ zgUR;ANKAh}jsvRmjoSFHbM>^sW$Fe7!p}=Q+!{LkYR_)?#?<<9@YfY^kY?{&7>m%9 z>8)>YQB9Wm3->U#7%Rui%F50Xig8_*1zG~rThA^X0sgPu0z+wkS6*|KdMk1r*1wtN z%!4g;HMMs{Q8@@oN$G%p{qzgo`?WQ9JN6^@pPLZn590{vEE@5dDQ`8iqlPeBRP=k` z>>J*gjLOOpql+(LJWM>UoiF&B+$8&uiuRC9RA=-eDUL1rrxFX9rAyh{fpcsPqpiv> zcq#m#Gld83*umd~M~R6js;-63o0jP5YiibY2NLUzy=wgCW@f+daSI8lYPF(YbXMGD z&N`OMU>DSCzv-K&D=h#-WpOSuUv^uI=Wdi(atrH?<)x}`mtd@7(nNl?G)ChwdH>~v zI7CmdJ?VAy&$JOjn_$l=V@D|x!0osEjKDRi&IC~~TQm&mnt*H&Ejt}qKGg?KOgEUb zGSQ(_prtMO7YS8Lcem4w#Uz* zI%|@=>e|`Mm#GQ4(f+k(3GEiAP;)e-o8C+a1Y;1 z@9pK1gI#e8u6CZVoF=chpZfmrA!~EnP&&36fzM&wR9xRz7TLzjgLf$h&DkZz)W_K? ziv;QZMvy4E#QUtQEYqjja)%KRR!O(3&6uhg)F6(~sXTU8$)Iq3<0%3WqQ1Mf{~-TpnRt;y0Yx_RSSn*b+-Z7@86-hoQY~ll93xV6u5O7F4av9P;GJ zliXQ88M}WuzFSr|O8}}tMg2y=!=;9uPPAy`ZC??LMVoM}#D@(BfJ=sXahM=0a}R#p z`HM0IKaaoCx$FDIvS52besiJwb+>(E-S3n~NgIi_xE4qjDS%bcUwQ$V;u1{kc z1p!=JeEPc^T;dOrx`nydFq6J0xlMUX!b9YQBY&t)8P9fKs@wtQmK>))c1km14@0&jymNg7g;0TU=H4W?pCDV0%}`--Utgt>^}>#>O^esdAGB z4lh>gat&I4jDPfKbOHO&pWfZ{^p2dD$WBry9CJo@ja~YEe!i!t=WXY!o*~D?B?kXG zgTozBu2)BPi11n)l+_S(MP}Q+!HX~UI%WD4t(Cs(&Ls7FW+iA>xh+26e5A~J9xXBi z%U5d8hI@c54eBSN&CKhIDy?mc6E#T)X01sckon!)_u%B7CJR=-Q?zu^m+s{E(XC52 zk-c=dP5`6BC>C*ru|z(xvW>A!`@n!F4#hK0lSnI;|D@tsaEeKc=lH&dAj64UU5FW} z(}I)wO&lNlXkK*m4h+ongomsw7EM1J(tg3^njkGG7{S|ff2p>;T^(XVumTYfGfQ6a zshq=dB{w_-C>jxEuW_b`Zwr+4x&v_YG#Iyh2=I1jp$*t4e`a>*30ETf1%Kd)leD8;~_u+SB#1)y8T z-WODao2VkYy9>?nN1iK~u>=H=Zb%W)Z^V1UL{;wJj~F0cxe<&oL)eP8OQj-MTpq7P zaB<#kI`>4H-;CEbf(8c%@#$w}X3k2F{EOVumd{GyZBb!JqwO4QHWG?yB5;RW+rk8> zGTOt|HXEPLyzTj*!oQH#1#?E>XHm1a;7QoK2^*3K>6gVNBs~0!(V3wAZiBd5j@1)% z_TSCFja068<+#h2K&y|@Z0blP?s)IlLG7zq+P>&uWn!x3hx!io#7Qz9 zqHRYUsTOYVmC5TWDw3ErcKvkDRu=m2Z7Z4sWV_(qj{69ZZiM zX+y`fg=xfGtRQd`obdQrrz8}EPQj7cI4@U4zP432yJrRy%ef{*V}nPAwoOMTA$NN+ zIDyvOUN_fJ(Z)uQeHs)6#|t$;4l&04)X-q?(Yf*I`gw~09VUrQQ(daUq9U<99ys@6 z**Bc-!+Q$q&I}_aBACh98|RBj(jk7rS0EFi5@l0%F35t9q~gV7r4?`%>V$GkvvOuHwk12*;}%5yAX9 zySIn#%t+S*K#UR z1uPQW+&D-GAc&k|pYnDaHreShA+&GXY}on!(pi+Zgm)T*F zbYoPAVwoAeQmOM##=gP2bRymWjkUwl8_-P;$21NO-D-cusZ{fM?jpljWNDvT-%$aQ zx5`2$)an-iBj1t7{3Kb#C_pF6)pX=s(vIsYi#}Y80bcAIO=KMVjQ;-WeybDvXdq2f zyze>sK9!MdjAq_Up4<<^%KSc=Q|=920%5Uf^G_}vsSylPUD1H5Y_7imM>nFl)d!+2 zUHyVN2G?Sku5m&;|ItNwY~!+F_XOaA5E$TCCsNTX*iZ}XvBub*3wBZ zH=_YiQIU0<*Gp#`Asds$5E@lGcqRbDh%L#U7Qv7EQ;XCXw(DJmtC>Sy330KpDXJA{ z@&=0x{&!&hV+%9Qr~fX5K`<5y@_1eo6fZ!B0Rxo-mK<>SoOHD(l&Yr4-9jkr{D1eUbZAw>l}LBSm5Nvr&mK51>=?i;Q_s{Q$Py;|xt@VA zJ7PNcm=Humd7HpvDB1LAbPq@WAaUI~gbnnmbzGi5cDIboO?2{UFPHS@m+VCKS;vj5 zq=yN6d0-ym7HWL(jy%$R+`ANZlIOCtNHr5%SXlbhr~~XdN6h6f$fqqF`6IjLGJBdY zhyJG1QZoPZa9Jk5KFd44q0%@X5#XG?!zJ6~W0|Lg$g^#P4iAx~gYjwD_PD*#=L$p6 zO!4EbZ?bL%T69I%tfV?~248*54%ZpHK0r!pFX%$@E}2T2bWwfrYx?E!m+lIO5~nB#9swLU z|LWX7-Drt;+m!QPG1XhC4GZDP*JimiE2oE4TE8UzS6KOst#9y|$JyUAr6U}Mf)-1(I>rt`&m5qWljThxGO zYsom>*cnHjQz%-to7w}^Qa>59v2#3pT3>^k{v&yqc`Y+n!0s#s=@&iZAT z+giyu>1qA_J?oxwT3CP}R$Y?fG^^&x=4VpDWE5o3V%z;*(YL|q;IS7+hsaL)I9%om zz29#}DU;ly;Vv_SHa2486#aSQj3C@(Yg_A~s@{=MB z9=5-qG1aXdu|6fyn{mhXko~YNpWwD>$oul{4|buP`R}S)D_c(3%R4TZ3KMQQrkK_z z?xU3cyP)D0ER21ckPDe#vb5Xxyta6pyx2meJy)!`3V-%A{7`lOSDw+-h-JLgIK4br z@s$f$X8TmC?|ffDt6m?#Vdy0xv%u`7tZWlxA}t}7u=vD#&<74mVLz|!hBQZ2wG~Db zC{9c6cJii-H5Hgyiw~e&AkM2XiakI_kxc-^;faDNnc8nReA&NM(5GNd16PAZ%y!`~ z1WtUfcf+#gDlSx7N(wFOc8t-~LCiuN!x;{<|gB2zgrJF2$tIE-sLhUOvJIWFGAo z#`;fVW8GX_9LguE*v^H=7v=NM4wf+8I8B%xF5e(VV=T8wqmQ0MV`gD>B0v^k-e>p6bb%=R9Z5>Y`i4N^z)2A0aja^ei60)vH&Tw$*A>V zb&m%I(4wTogZQNKqPi?mn*p`_2Lu$`p#0XKPEbjJIFI##>16|?7SDF2_NabyLW>#y z@#AM)XXECEkM=`6^A>*%s+eEDqyReKB2X|XyzBapfPwslKIE)I=7hN-&f}BcR-4u2 zvOk>O)g@j8-WJi|es+2qDBn`rz$Q9ONNEY`H*UC$)sP#UrWKMAi0f%L&NIAuEuY(+ z`rycAzkW9*r>60VVuSd`JB-C%-&SPPMSyX{L}Ehi-@K*Duub9iXHjm6+Vd)*>htIO zwdPKL{Io`-auC;n0v$jNQVAwZHDG5$B>lZb`H2q^!4|;QQOo0j8W9D4nTJ`z053ne za+H7+cVr)7@jZ;sXhY8cG!Bq-aZmacRaI2sW?Utt#3~;o3iGq?>q|&wfFFVAW6|K0 zwYfAdsOgk@YXGxAGCxpffNeu}i~N+e#ctyG=YvHD=0IM(yv2*T_W7x#3NLY+8+_51 zwo86~#+HH&mNYi1zk-svtj*_&R3dAw$kv<+?cnSi`qY*xkEW7l?)z`*Q{=843_#MQZh2IsQcoY`=^E=2Ep%usJ3)7l=P zMZPVbUwR7Xf7dg%VUF*+voZ3&OI)7vKJb<|`jS~&iZ`C_ zh)~VSNP3t=Tf|;V23w6G3(Kg~yyGWrs}1ishyQ!!?R)AMkNkQ^6k-F)@v^%2(dSs9 zxJ0TIpuI*xx1x>WzcAyu1=ij@X+FQ3@0b}xg@ot{n0wJ5lvxA*@9N@GzS3`ns^Z|K z8Fiao3SM5D-!e3A+bPB25u&efW+kITGec8aQE_m-KFKrcJkJ@J#w_5H;r0`J;e=1_ z)!T@=udy|X;~U*bU^>sRG_yzUGyOa4W1ASlY~Z8)rE+-1>cgyb)A&5y$SQshw1 z8S28pQn9^01AW+C3yWS%dEhi8)qA_-QP{bHh`NS`DBjEib)P6EQw|V*ERDS^=?G@EKF(n%f5q%*S+0>hIU_AqRe45^ ztr8ilyb2xpKW$5E>p>L4n7{_T>!Z{euonLl8W=-1s>L>yR-)0f;xKNXW0$o02mw+D z&o~B6rJIvV#ymUyIq1i5@KcdnE2E>_1j!E;O5PTp_wFSfw_|xY%LI?zu7xxI_a=YK4A5q}*oW0{}_MS4GWRJ3L4eN-QARJ}ht z95W*{cq?Lt)IN%H`$ePwUKUgxse38|?mF!t8;v-5Qo1qFik$;(KFv&YP0d-&n89n7 z$d7|b6o0&g-9eo;qzYsu)pxtA@g986={4ER=BhD2UHa}F|KY=df@deF{UV+dX8W;r z5u`>!czDuoSf(sm^8`78WucyeGW$<0(wYDPn;1kc56=HdXOL`bNhDpA0Q9YAbs(y| z!8k8dw`l$tqR}wg+>EszEXp-7>wR_YT&hv-D~5mzxl&mjfobyyt{9A_^Q>Y1q z@1^%8i}~pGI(yBFb8S6fjGX!%4jwQLpB>b%!>2%(i$)h})AjZSR-Pc(LZ3fNx-W7Q zF1`AS9heg;daN(ds<02-`w^{VX=ypxnjGe0?K+cFe6;H1Qmmk^E8u!e2*)|&Z`Oa| zQ$SM%4j4`!GBWqIu~3JOGu>`l+B$4FJ*ls ze)T7&(6Io(0KLd!$}8V>H_b`Jlp?bY-Xy#X zSX_D*npx2DDecytL5_1`WY!ivbM^7IH zv>;yyZU&7WEi+HqGJRA0aAa(;&sAdP#I4Uu;uStv7sNyU*}P`fO(JHG2UMZrCxx1< z*@uRJUjFx(X3;0(z2X5t6VdlOM8GOB;^9Nabn8>>B-~O`?D7!P=l5-vD^(WFdNvWB zr)7^tZEUl$u@TY9MMu!{H!VDGf_bRJ8$+%4i6J;V)k~8(Z1Ld5F^aja%KZA0tR;Q9 zGAsjf`}EHZUHm+Q!06fj?tB_%BrtGt&WThB8h^X#^*4X*Uz_i|!?0tWK3i?Myk55c zj|`8|a2xGx4f{Lh`IUGsGd($!N>Q?Q7U931MRan~j4Bs6eX4bJAL&UvNGo0ZD}KVP zuiUf9?)C)MBz;u5`L~_@W=b_X@6YoXJ@`sy=(5;fF)X)ve|a#@`0h6=Z>{RT4qo?v z=!|LS+2;f}yHYPOS8Kc)%%A-^Sx|4KxF(cMRVDrDba%s8DUY&Z8|Dcs&lD(2Ye(9q zJ0~mF72Pum=T39=%4*6C-Ss`QTGVPk8Re)I;b~xg){$9EO?>|LY^qcCAvs&FPsnsp z%euMC@5-xtJ6o#|ZhA4t)lbjP4no9@&KEeV zmiN}$H8759=G#4FN`e9=Lp%lV>L0RCk$b|j)1TO+_bM{iuAwV|=h%wuyT+~0^ApMh z-ZOTm*;h?@3>m{IMKZn!x8@|zcHI%(bDrHMyd8_;(%Fx@cq8ss!xh) zNB9>!oLnF9OMUezo6RPa_4scMR9C3*gi}?``>YXGR{!KbUaMOhOB-X@tX{j~EAyxQ z+`+c@XT%i1qtVyVd8>H>dnVM7aE!}KB9D1^{y(ruiMBOokphek+(Ko7YcNEtk=wM966Mg^H zYpQC4fSukFCv(uLvgZVjAFm$r_hxONA)2|depzo8Ym)(^u_9S}neoNHLlrPPa5H=hUjO z&r%rDU{9Yp(|6!Eo4Y3anNV@<8CK1-kWR(~Tc?ZWY`^BUHw??y|Gl(q>DcuqcOdDmUJ)6C!zj(T@(~^mYQO5;Nh8Oc;>I=I zsN~}0Hf?gi?XKM;?*+o4_t*ZnH{OAV=LG6U)=gdb@(d1^A?|3P+|c$_o7{%-yDp5k z-)K=1gW~|YEqCkMoyL4#Lc$kCJn$!&-;kBM+Yzc<0V$f5x+Qqw1V>$glNt_#M6ixm zK}98^93sbp4z`;(9hZEd&jzPIa7bh-eFB*PM=jUK7Gb?&bT#Az zv^Qzbc>8u!7Vn4)EFY5Db?bR#gxHoqrc#N9@qZn>X-dq5Hj2sHYGm-fWr^GOcJ|hn z6;8HgaS5KMS3n$o1%n?XsR<{d_o#3frdsQEkAmf9tot<4kl zhjE)#iUN;5v9$1n!1Dh3als$fLdKy*&B?_tUOge>M;hZjiQNqOhK!pC3geb9TS)L+R zfJpcCxVE=fEC26m+BU>BRd}MCQ)D;LiKZ1+`+F*yizf;$mR=~K?>XEe zP4$r=Gd_I5DV54WukrqmbMs^we}{?XQOle>F^jJpXNA*b^U{0HVPBP;9$&nVWxHL1 zcJgO4Yb{ZunEVXX`xjiyqkvw3fThJDjA4ah0F?u&ysxv8(@64^s`8Z} z2k{1Kf|yzJ8Yq&h$f^65c+tJqH1vh}*r9^yVN+f`Os&jBSc`?++$Q$&d2RjeX-hiO z+1{ROHzGIK+8`_#71o0yDmZ*3?IamKZfI7OHYhVJLM zYkM=M%ipt0>=x5NTY$r{Nx=5lm5k*%L(q@7YLeszWt}|0!4XuNzaVpQ%k-DCl>BkK zz-oqlm2h|FB>b3~$~Rdku^&5&WKrO)_%HY2Z(cE5-wsWS@5zYpqv8>HKF}y8){{8< z#@#BvueofZ<7CIugC&Y&MV{#1sF#6>Ti?`*<=W;RWoKs}*La<~a~qVY@8R>B@ zd?ZLZ$Dw5j$%OwM!EniTW$6^D*Y>}u?Zm0cf1A4$u<7^ouiU7(6ZYvn2yj(xP8$Ln z8Rz1D(?`<@lI&ZNbbgo1QbiYeX+oSQ3ykv&E|M9?Dh51@9i}+{YXb^@^KQB2>D$)@^$7f zF;E*7tBnXN1tlfXIjzeQ>|g8E$p?-{FJaIL>4e=q(MR99y0WzMDWWj8E%qeYrQq!? zYj#D5kNCAXH-DZbj69}1h@BB&Uc^RNOJ4MnO1O=>xhEUFxQq-6R|jPTAO&&Ba_}r_ zzVER3pp!-WOz{50NK(IdJH>Bpdj8vMC!A})k#YlGT@1rX`;yGn?e{BZo|KeaBW4U1 zRc_g+skNoY+MA9Ze0~LFb!3vj=X)AWNJ>J&J!o6P|G@L}E2hQ502sTwi8ktJO+fC< zTyBuxCMJq9C4uNY`QwKS;4h$YptT`Q)YnQh-rZmY!GDnNXrXrzp5^d8(_36hn%iXx zQW<=1+;M(@nvHAuJaDRS?0glgLR(sPHd19y=17tfVB$WqJvRcK0hXmS*Zj2)=s^dWir{rm0h`N=4;zcO0eFplz3zM zl0;#`;6&6;+Y-nUbTX5Sbz3D47Cd?l%&I9e9-!%@_&9Q=>_+=hkx@+h!p6;wq~2tn z;_Tr5^!qVRPfSCgFu?mX2v82sRMt{Ysx+*WbkS`7f}0!VWb6H?fA>ET!-a|T*Pp6g zmVb#1Uv>4-0kU+Y;zYDHUSs^_&#N@ESfoc4g< z6ZlQtiJzB``wvUVzqg&lFi6i5X^~rp`$N5}%Lho8zRC33Y**_Ta<(d4W{zM3^4Yp) ze`=d>Z3K?QzvdI;+WS24+qhhxwWh zoW0|wn6fhQ(=I4JOYdFajNugf;L6+Rr;Gi0m9b*XtuAiT`v|iiB-?x$EO)l5{QiIH z@nhrGUkAzdsy>&RotdwVy37=_$o*}2=+J*9WBPMe3l)*N9d84_n9ngpp)wuJhWu7mu!qDoj$%uM)ZWJ>Oo3>mDG8fLiPUJkF;OO zY?&LxoJmy_F*(^8+e+tPo969pKuuV+bl~~3bR(0$k9Yx+x?ee^EFbTcGLztWZV)e@ zG!P+Ed#g%K$X;oq#L|ewo!-faL@(N(>bTe6qQG-_E}$bw(s6yNyBf8j;XW?LG0qO z5;KlRH}101g;EEM21Ge8%t>nurIox)+-u3SC{w&VQg+7V0Ht_p3u}StM%;S>{?fh* zZw8B$MDCjKT&=GypuJ#Nb?E8MWLk}MQc8OHNH&uDgwek40MbWb2N=~S2OG({v4l9p zI#X&0Ocl=`J90lSv_Y4aW^`CVi#NuopFM#45y7jaH{i-zVG4p%s)l zp~B!(Gu18reI*csMxqTsYLg~xy~;%ed)-SO8C*ZR(gd@z6tfi_p8Rd=}(iLHn^p<-zVvL zlHUTY{>12{`T^!m76Dya2_-NN8tD11U1>enmCRLaK9gGJeEpEh`3n$$I8cNd$`e{faK;y zdOB5<_WqNp@{wn$MgqiFpBve`TS)SXQxG2CXq+1MQfN_%iMqeAq$Vfc^WJm(S#1CAs0Fl!mqbfTn z7#Iq!n8_-xJsle}a-CHUj+v}D=i^gx9mYeMWUz3EL<02(&eQC>rT=H!2ZoE8(m^>d z)|M6!GC(zlbteJv==19tP*p;de&NCeNm4!`q56FMD!qr~`Nxk5+V8SQ*9(XjikToo zcYFeo{i#}9E^b{XdtGcu}S z|DoHnXMFi+S!+Pq#H!zwt8rk{MO+MQ0Jp;Zx#kH>|2-l&Kx)p=B}p8ZI>{%_4I)W$eKE3j(g1hu^kpz`|b>|=n@!iYm0pFi6c%Qre zz|hc;wa1Ybr_zU4Ua{ZsfXAii)A|(y7K*iwlAM@YZ%N+d`UXZwlE&wpX0*! zveaIQqF2zefL-uz_ZraAzHmvY<+0&m?oBe`OS?PMNeEXT{F29@=Fh!yW!~kP+f>G7 zuv$O`MKtggU@0tV)b$C)XNua!apKA*UvAmMnb&6YKR~(o^JrJYExnsJ>t8>_t#;65 zWoMhsn`%yF)1LWNpxh(1@9|Z0^JDc18JZ_#6hvfp6Y#P$m0l@U!Hu>W+Zj^+<{hUt z@E3QxHO1M&$#meb+M~do7F(}0e5?@vRu1E~nLz`DGP#ECfwGgd#_%1dofveDSjuU? zh-({IL-C0S2)eL~1YDbDkDyTbg;#_~@HcM4-@At;nhDRlYd?+6nDx`b!tW6k0NU0i z<&X4M=7%<_DY>wiAp^*!M+sm5PU}ZWZK=r)p$aNfZRyH?Eq06ZnG(=&fwHEeXt`bG zW2AM(@?o~Y{y1+XxbjpRLbw4Vy_NYRRCY*zkV*z~U(HLH)QJB-u%j+0DV4hKidPng z%PiP~|1KHwd>9pCK78@0s3_6&R?>Tv2hSC@6R2`D4tvB@wutJHhdl}>Hr52PSF zspC+v5l2rFGwVaJ@#?ZS_SixRQFOBN-7aWfM zL9$6}&uL`*rdM5`9UC;uzi|FMoUT^7UB3;+@T#F81hs-E`nlPsdd}g18}__@_T;t| zU%^7dgn47OMVVmku5I+KspHO)d-ur5?6t8!kynFJNQ~nkqS+?e72{n)X zs2JPcz5Z8j^{QTI5gZMxZz$awSm{GT6ZdIB=BxHq^KZ%~=rq4dUV)$DTCC?6<2))m zweP0E*Mu9oPLmo`H?0J2lV|K|P@ohoXbJhfwvSf6W-o8dgF!vA@7*t@$4Ij8C^>RI zIFB$u2r(94T+agKsCeThVKSw!ltOS9e7)^-!`E$>-Ulpdb11lX22@V$`AGQYUp=P~ zeW}is+R>`CjOxl;1qPD9M9xM}`_%TrM7L9rq1{xuKJ?LpUnJqc*u7=AIft_sZnrxa-BFl3T6Cog}{hVe}H7c3Jn| z_FoHfC#b*4xaPF+(tH0{g>&&2&uXt<6b_=oo+-i# z70KLYI`{D3T{o>FBvZ{gVR3?@xa_CCO-@YZJj@g9TYc1cslik30FBTJzj z`blIbLrTOZLgbdKtE(X#?qSMjAUg!6VU&Ijh*U<@EbX%^H#THiIGH(L#idV|4_W1D zk>-zToOpWi!Ugiti?&92T1Y+Wu%IjCCL7z&Giv*bg@CsKJtE74^H{wvgv2>rOm+$L z2c6yEF1{~U{2LskUkd&-sIx%YHWV=oN35D+!JKa30ZHi)(5b#X&$yeBUb4lhkGU$Z zavVc~)s>@+mo+u_T@_PeFbVg~rZGAG$iDc?*F)6Sr^E}Zg;nD%Wz3S?o`bF?a5R(B zp08HW!Bpk7J?FcwhuZ4M4|y{vPfoagT?2u;=s_K51{3Rp`7vM(ecf*c7r+-Ce4Cu* zXAbz$DB6uQv83-Rx32E%qgt6gDH~DpsM$=c!mZf6giKaeh3{o$+NL7L?NjvUhh#)W z8QItxTO#WU3YAohLR$6xrFzeTB>u_xu9eNX7TGf%65HIK1sR)C+i>j^RcHj zdbbx9;P(m-EL{oPv4%gH&N5WZuZ<2xHk>J<=6N#K7L+GJtJ>u6WOqijpZe6CY4G!+ zD05lc>XHBZv~S(>yRkTusR-xQ4h!p&;Q%fZW0NPAuL^G;__O#M2qnS7#>VBQVut2L zC8a%dbbnS?Y4^ho8z0euh>QIvpK(|9Z4c@drQ}P{hgms=#4?{v^JdVLseURyO{z}$ z@%kY`=cYyX$b6Yi`+@Iw90(#4+s74q28n~jpN|3J`2tQ|5t@OD{!)hK_j}JE>&LWC zn!mZV6_m@_nVFa?uXM6jpwZj2*>mXrBOZmu2o!Ei7Ja%|-Dn=}w6#+B9bZ+;*m^5| z|DP84@hxV{Xyzpd?$c?Z;A9YuI^R84=iB}rTM1W@pdd2ksrg)yLx<8}hXwboEMhtb zI~g^}hUgmtp+1T7LJFXs!7>k~DA+Um+d2sW>;3!p zZ@P2*ii?&cd?Y3(%_fhSuX{p{2zx6p_&h*C$d#URfgz2Q*ez}PT*1;rAA?fm!e z$!K5{aU!;9puZoV5{$A)39_=XS6(r?h=0o5JI?SP`}(Y6i%o%!o%m35TmlKr^>$q{M08D()bJUU~{qrY@oG#2xVGwZHgK2zsn!wd3 zcvb(mV*I7r|FWFt=|FeURKW6W@GcwD$cgmG7zMA=hTpUe&B()pF}t%HMB?BsEu9!` ztx12W9dJD~_+EL6jmhWR$B46RU=D!qa`Z}mk5&gB^detqqj-L{ePH!H`7Z@wT~^u9 zge~$X+Z24h3@(O?{ci7e!6eg0e4ypEw1ZKM7;o57AR$TXSf@pU9ZZt4dz{UHVUp=LOF~qSr`PVi5wQ(i<7PRjsVv-F)Xw^!_lk zx6dWgAs}dYt&#}*Xl{FZ8#e}tzcl?SITgt`fd!moQD~99!)j?|b5-Rkx!=!{&TCE$b#-L=DQ)sQ zGKUV4ND4u1udA=$DhG}4?-P>zneX0V;NW=nENNn!oCDF9{1}WcP>*I!@zx*Or>n|f zLPBi%Hom;#*ZN(RNl8?=9MO9B5u@S~8rJ!)26}!;e&@K=eVmhWjL*1hsEL^Qz2F_vZ|5P#B&Vok}3vMJInR$YI; z62rlB|N9QqLn=mQu5EEYg#`;;tAj(G|5S}DW{AHHlx!(K}jsplT zGWmbn`tEqH+W-G!lak8HPDNxkkWEEHB_cv335l|^XOSc+*|VY~BH635DH$O$l0;-= zX8oS0&*%5Y_xt#sKf3R`+c@uYuIqKZ#&h&#BX`mI90D(u;$nFt4=U;bwtu&O>mOiw zIV@xig*&KjB61=Zt<_s)PI#T;dfNK+Xy^xL$ExIm&6Sn6H3g|Eg>Q`QL?9uTQy9d| zM^%(v_?g7T*cff^X@36n|FHmfg^Ut6Z4TS2%B)0@RuuZK8oT-N;SXT69`I$fW2`p_`pob$c_T3uqL7tyG6P%JxrsbZ? zY&ag7HwJ6q0NGV5Lp?!D5)4 zB)7bzw6rrg;dn{uq*W{015}<=b0-$RGv5CqNY4lQ28J#`Z@f2qRb_TT=d?mCuRxF# z&FGiAhCH5%p8EF{<++x(T!7e(rgbd8M+j6#G&Ns?L+Io{Z)NMpp@W!}p816woP>*4DSp<$c)>!t_{V> z=#|k|=9M`#1Q2=xk`C(3hmKsx?fM7RRo$^UxyyJ-S1fXuZv9amLEni++xipg%wC5@ z*}8uZCFE9a2A6Ja6A#Y@0_LIPndAD}1?kKL5a*Ir0`zZr;x>dI+%>)lP--=SaMPmJ zm@Mnn*;{Mth+M|Y-g9Xm@$1!%pNtzm*0$2t-u%=TTGNP?NV;E|Bt%5yT7D9kn*vk= z*o3l2Eq76zR998$>gjZC~o|J(EBG{E(ucN3E~Hck~df( z#}S1Y(+BwpLIwB}b#>Y-%K4}ITf?R{i$16`$FR@V>wv(uq%Y3tB}V~J?xc2z>WF`~ z+dv1o#n#=SO~SQ>S`L-wduTKrp#F~lI(J$y;>qL3Jw;9#sj2TCHWdD4AOj64fQ1$1 ze&UBAb~#`xSQ1cQ8Swq_<86RTJV0?X_$)a77#J9cd(qL0 zeYT@~^`YSfv}ll~f`o`u;KfFEaU<}})D<8XF_Rjm6cZ7#B`TbGz)*mi?e?9boA3L` z^yQJ57M)ucrBjlO*dcXwri~l9W@wuZY^aW8_ez^w9{GbpPgE+RX1aLK$9Rkrb-V9C zS|;?)ogVNl+t@IbM9II@+1Ho#$rAsul357NqEEP?CYe6bbFX*d5C<_eD}V5QWTfE3 zV})N{gOo4h11X-Hn6KvNZpF>obcVwp;_oB~N$7=W8YU{bJHuw|apA$OJ*K@%5eF_Q zJFVL}@tA4-Jky=vU}(b)pG-X1>wTyvFs)EnEtJ1nlq1c#bjt1HI4=QHPY3oU8nPttrn}WJWPiub7^x4{%37tOmWjC{8N8KxwOwlm_X&F^pG_$j z>85`AJ~TA-`>wnU)B5$3FR$9z#6fOFF9EOyKt~1#2MbxRdAt)FsQUa$MOlz4rQ?_J z))?kwS~5^J971xuVMBYu3$fKlX5mLzghC7pYzF!=R+aYc+wt(&ii=4|m4Fe2(4>*? zqPx_jHP#6_brudE(>ta(W({z64ic$-Tm)DnrEY^G$^_g%j+{KH9|KgcWD7Z*)>_zY zftlZ>OFKuiS698-x@1e6;w{vj8XUQDue^FXDTefs!bc`+?Ax6j6DwQWGH8^btkrOO z$fyH)*P}zJXi5uvm}%FEBd5COVxe! z?QD0%mkNiO_!{!^@!3goaM$-s#*}7eUd6Y8352{R)r(QQgl0&gYwYmOx0JWhs+MB1 z9c};GZtU`u>3F24oxvgVfGV@8E64Hxu*T9@n#LbzKRXPz2-?ob5~3z$B?uek<$Sub zjD~{-bwK=pGy6l`DqhJ`PF(zDK>>#{j47`66od~u{J5EWy-Jc}y(|-Kz#CVqld0yN zQ4PEgn~Ufnl0I2p3*FkgOGYNYtgLr+xBr%_Uz(fm0W1TvR$FdY2Zy~8LRi->CefNd zo@_crO+y}rcK{^^ok&x#^;0=dQ|u$5@FUG*N^^ZRU@RF2v;2ty*8mGvGfI*obP1*U z@87@6%9Pk8NGRp)x`X-*dxNk*(07hLpY`H}QI7dhEDFv?i1yeivO61JdLj%+YQ`JC zZS@0(qw1P;0tZ4EJ~KzbOYLXbf!Oxa;v*9iYk>dbBw@HC0zK@xfuUW}TROX*XXAfg zZXIp(?q@5|zUz6V*AvkWnskbmk@VsMmw9)bI}ws$b=^2r`OC(??2nnam3iQ_-F(E; zaXgjO+abVu1`qnHMIQ9SH{O@)w0ZOR@eYashL1RZNGXg8ficNa_5ljq6S9xjn8#EL zBTsaiWM?%o)pCgw{Vl)Ii;e(;%TJWMGC42L>DsmC0d2d)yeIHfYz4L-g!UJ1L49*rr zOB{!P_x}?4P_HxE%3NbGr!yTX9~f8K=6mr471}OcHjyWa{H5sHDoAypjAG4b+C<6R5s39Wwd-f~Gz$p|Aiqzn+N+ zPNGT2G?)X9lUdgFfFnE95*MFIoHDj5@C)p7pK-4fu2WHR@T1>CrcVn3A|mSsKdk=x zy2iP0Jtg`F?<2Spp@{u`uTW674!Ir^XkFQ z*`25`8KZU$(3C}uwdyd7t08Y*{jCCT?~Y%Yu(Pd;@XGlEB+;BoUq$TNL~LokJ6wG= z1s$AS@HKiFD?2z;nNcd>{M#J7Ljc;x;aW`l&q}1RnBoPNQB|2q{XVqk%7+(x%RlMT z)0W)+_I_E?^R_E*vOt~Xm|lVk5^;ovQ;7a8mzIML(|$kx1RBX3>oqkt+DwY zd^A|uYoun{wXoe9wA}BkLP1*otvL&s|TEa~JzET*tJN?uW zuS57>HY+48%{IR-W99jr-P){-Xk65uslaiG_jTX2Zfs;?YFRa<-XwtcN>2&Di<7q# zm7M?9)@Wo)9@x3)T{>N!V_{AK?MZG94*tPdel-d_WL1DJFz`7zw1TC$_EBife+U2; zu9QPku`uf0-OA$@drn*2nzq$Wy99poL`pzAs@Oqva+&B zKQRLW-~03M_?AP>P;EtQ1M}tIB;(I#2c0<`yfM?&TR6%HS?|9dtq-3HNh^d)iBSUN)_Cz%H+>b*u;j`fc0MEKY| z!vyK@ckkW-_N**^t}Olg($3SLPNQna80yMz4yVSq<>D;ChfAibzlL7*zVav86bQeJ z=g*&KWzAoRrQW1Xe_1*7S|ne%{i$!72{lNNwdwBqGC^irCbkLrFk8qFqLN8^={6M7_Um1GO$J(Wd^tHp|nhwg4B`C-XKAGmeG@eh9EcxM;?g zj<;eO+B%q+kwT*HuM}QxZEXe4>DBUVhY9}O3B+K`k(^-EN zdSNMFs%5*-mAmYcxc^Gz$+Rn0Run!k5n+s50SyP5e{ODn)BA$>-?HO4^t;A@zQkO9 zyR}n+$k^PgZk^93UP*N5S@B5Ma`(fyG0j1e#T3m+#xwm1n8>kO-r}#O1qYmZfEPX? zGQtC&yq}dXeziD%VSeOy^e-mJri|tc=h|zP4te+$5^Hk;_*?(Tf)MnN7Q%$3_jL?AqiHo+&#y=LouS`9HeF_nZq$Us2Z zc{OB)K`QvJ{Wqq+qhErm8C@I{99m`9|1q$kfR78-27bq$j3S{K8=Gk@MX03Sr1eh& zQV*mB?3Cl_>$Acoml1JnS9)Mr!YEyBfITA+L{P<*z%Kun5Xu+ z4u)ox{T+o)BUtV3*!1Vb!=|qQ_ z{JB0>$nYv%{=f|lch%R6wYBsFoMG#ee~b<#$5o23P$w~Qf8_b;`$KOR3s8ynzcxo} ze(va?BcN4jKkuA;3Zy?o1Oka5JF>yU*;Ndzk)j4UfRCO@IijIa0}KI=4Zr=Z5dNk9+wE~(M^>AeZ$X?J^IM}VdW z{Vr&XH-G+o6C9)=(3q<~cj}USY*A5aBL1XMfBhRfl9~%SbaceArob)UvHSA+XjaFdB+c%E-w%1XNEIq*RN&AX6fT4?2jo;_dR?g=%COZB;apOP z*2ExR*T|^EqKVT&pQ`4W_`_p+_wJQ}xx^47o2aF=_1iEAul231&f~AK(kCT|I3-Vb zHjRBPW?wV8apQ)jsNqp+mmFFHZ4XG%(h^rG5p-P6yePyj-FZARDdthb5VTef?S$^bIk1R- zHgEf_3pI-u&+vg5I#)OpO2EZ~Y#+9}OP?n@RG;^|{H%yzo8aQ(3+uI?dkBHZ(^^^; zWkH6DY`?Lu+0VNUpzp*~bCfgm2Y?vRF17j%UQNAu4)o8LCdgsKQMd~0zw&F>NR#7D zlykcM&>^q5(v)1zMep(Hn}2-x_z|-9IvMTwOV(pyTP~NPm|W40%$g55R3v^N7D69_OWDC%%5oLSTPW+sX)j8gWdQ zzaHctxi3Kk(nx3)y7BkA{SsTYd)_{Zp6Ke|j`C`Iv@FiyT^-zST#QB^_*e*BiO__6M`wg9!9Pt^;A!v3((q(23QV~Z>js<80hK_-F8Eb%5W2RBbRP$lXxPrIOO1jUq}lZ3k&3p z@N}2g1|vvOxW=7nj~{~|K?f2g9Cr+$kc7w#oHOJFvHMADm7+Mq=*!Odvh3>>=pRRi zhn}``jRTv1<*yT;S1Me9er%lNt$(k^9MC7l8H&?R$z~E?h)q*5NJiuw8}NaGtDc3{ zgMCEq_>jCv==?sP3Uz45!DL_9bN*oQuB$jk&~64>$$M?y8}VhKc1nQ}>J-se6>nfL`?D!S$coXjPF-9ji1uNzfA zdcv*<&-wZJXZ7?bRSXQaE^R28X*mu}Eir$?WG$alGF!ICl56$ip6EjY;cCf~x^w#) z>-{8j20AB@wkXW^IA9?_E~V$0&IK&M#Hgs+p0QNg5NHmiN-|w35)gBn#R$+ZcMwgl z8_?;gHbe^vXU7ZU;<>zz^MfCR8)No|3{TM#(V91%4^)RE9^uv+{N!Wh+t|pJHtvpaWg~j{H{^<$9*3RQ+?*{i)C*Pi>!~FxKb_3o=k|W3d0|=z(hlr3D z6vg!O@HDR-Y-E0TEDr=J-1gc+*c!yssWYNuV_`L1u>V!SkbxnVhxh8R_qUy3!;>e1 zt}d(iI*lh~E)(Y^wl{$3+uheUp;wD>A$UCM2mD6u z5lkpffp(z`tb&8dih856OO&mwN%6+C`7L(Slru#C=QmmXlVjeT<(N@PTGF|d$QI*d zP1Zu1l9rj-l=)SIUZ4b~9JnA7F{yHOI~a8T@-+|4fi7Q8JDGO;)G79mDmay7WrdB| znKd$Vam7YOMa9H`p#gq^N5SShjA*>@Q#N6=jU^7jJOsz0EszZIOfV1Sr-TfkBt*4R zuVbtWM!UC+0GUlPs|<2Em?8U}+H-r$p-$hut-giqhQj`&zTVo-uH0>Ql;il#V|N@npJ$`tjjU?XN;irW>*pK4OYvgO+kKd; zwQaU~%As3?-k082tvz_(ar5P^xBW_*Etxg-_I2?&O=w+^u=r$toFyXM#)d({JXlcY z*Zp7%LBpJ9i(=84!Repa1zXg7x1HQVn;xrjddISEQfWE6P_L1#yWN$EtL}zniR=wu zONW)_M=Iys6NlseaEIMuPWfaiqH-^Z>Qc%NCUXPn0#56Idw1^KfiXuy0fF~tJDj53>>plUppQk zd|Nv@WMpLvz!q0K(j4vZiqqQKBc6_UzXKHjLN3gh!d`DD0{mg3f>tk9nb6Li!dJG| zZy@OD93OqGm>&ij7M1kbX5XBgJz)MQ6H?U>6SD`*Vs$G-VOv^q5bV@yv4xNz6s$db z2TG);?dKsJeT^u}Qy0?rJ>Vjknbwm`dQ<+3-5_Nmf7sW=q0K)aMNVLlaV=cNA258> z(707+4yg^mJY-P^!C&jUazVtSMeDEG>36njL3QTqw91$M{zb(9t{hI4 z@s=P4NUc?~Q`v|-P%m3z9Ydxdm-cnw8Vfr+A4CN}gwzB%&~-{++hbG>LPEEm!^8YE z5M8AiSl}a$_Ue|o$Y*g9Z&3rh0TUi%^6sQ!0JT$@GAT(uPrZJR||5Cl;6KUFD2z0Nc^0TCPW1%CN{KN26qK@oj4Ii-7C*~klA?w z|Cz+MJ{|T*whtaXY8`|Qt)q(zI#`?3X519Z4b;_d=6syZ=)*AxjTv3*8!|aVgwM6f z(e~Yy(ReaAOlcChO!BNBz8grYfhuH#s%I9i7lj>X2>$O- z@3^sMQwrLr(!Z-BWoX#~GJGweO%{7u$B0KQN^Ucjs8i46uH3DmYFFFx1LDOL53h-L ze6ufHX!<#qv(26)!}eZq@$e||oAq=;Qf$_-Lp(4|_5FCmYagZuRunwDTGf6odv_lb zg@Xw)`*|g0w9I5>EhRa?1zjGU$AHddtd&y(?--G-DFbd5G}^z%F%*OKZooYxW{X$X zw>6r0ujp!N!G5;z%zL1z}1mld`JTrz0W zVw`c_e>Pj&pXz5tRgbS-Mh_I@BlY8WlDwd}SDupMcUK*Iw>1KzHAJW*x|~*= zV^F^4A{URI0_qtQWz+LkKFC|GcQOGaxt#z*;`dhppsEvV9z zUc8?zBl#v00{-s+kih^6>C}-I!*dyZjwbz!&<_lRR; zWMoSOe%BWi$Z&dvKTyh9!R02q@*WL z0-3eXo=t1ZjXODvCghHNvHpot_W=**+o$^ekKniPC>@`Bn|KN=<6@G;q(*YKalotTk zN51d9yK9b7FOY(GTZ!X<6jFR?wAEGWS(3{RcSA&6JQ&Dyhl|kp^6Oo%;_|PA1rFxr zI+u7<8WA10 z#X(qGbmd&ap@-lJKmv9Vj=tpG)pSF&L_jRMJLnZ*2nP%zvQjiW_hH?%oTT7?)PeQo zI_1Z_IYld$44Faeao@PSenn9d)RK-B46d*`r{Bi8;b|WogPfM2^>3BPOH{(b>j-+L zq69-=C>()!4;fAK?S1*-jbKj@ zL4x=|1fjBz0yjtyDGYc71i^el1+R$kv82SE5X7M^h--im#i-2coNhC6-y@AzcVmkIGGHOJ1%hG zd8a#Q5kxut`F)DKVKsvcc>xFU3&Sa5HkfVvdrGeD?*`n?AtFfN@OBccPuTJkYPrhu*Jw~lq*<19Z+_;mUU)@>gTwBxwB>8Akuj;Yk0 zUj%wkw^O{{-YvqrgmKT*3&8)iG9sg@S z6I9Cic0YiTn_Cm)*{8Go~Z@KCT+uMH^5h)3t1F zKjmLy+>|HO-cQI~lXB@dth@Q#;{MW|*k}6o+q;a6b}tlu7r&7>^_w}^qxe*oha7*l zpeJoQpPU8Dg0|EdV1W-2=WKVjoM#pgcQT5JSi3p*hnR4GJHtU67$-b-nvi&lGW#)#WA>#L(S7}AeG8#PORM^VQ7q|z0 z*xs_y;CZbUBFcEIkdh!M&j@t)_^Ra6*f-E*#eHs(*?3avKuO|T8G(g1CDwCl3oC+D z3vWo5P;CRf{hF`BLaE=bMaT(SNiCF zyf^y^`CdNn3QG?K!~3J@{&)%RZH^|I~8sUq2Od% z1T(ZUF%U;H19NRw4TpQr z@>cR_=3zB%>*$Ep6m;w9>?}YYg?2RhxKM6M)5b>mhj+E>{I!WM%o?S?5dT6cjXh zI^SI7Dn%nq_v~305S?>^Y6=%~PHg4oR)EeKkPCI@+LBj$p?cfWqDGIt(a7kicD-o$ zJk2NA!vz~h6tE=FZpxQvrIgcpO5>QM$(Z~{nYmI+dg1tyBeD><0vAk_?YNy?7Pk&$ z%b0HYJA53t*6a$j2s+oE=p#56Bi99-BUIXyquMqf*}@hwf!hYkL|DCpySw~;2m>>& zn%n*D15XU->Ef>^*_a}bIKwgc<;GGTVAqG_>s)*;L`6q$O-$VVX3LM|7*E_WWfv$r zb~ZNL+}t)cHbB$1p`T&7&Uh1C&u~4+soqq}9>8XFvGg`&^39lC)-O$;(9$*gh7?l0 z%j$Um{|-P*MnLOF4nqI9biQZb%?m)qJd3#>$cOZ30cqdFuU{7pq~+ziVcs6v><>@t zNxPYZ)DLTIkfiy}yW%?8t$Y=Z8r6yF8V^q1WM^T4p^YMxi-r9kJbNab*4owE zt0X0roYV!4OI#twC^bkKV$2r4Ig;b99G&*p=b$oq0qT>DhdU`L7jTT+kOXE^$4r82eQf$IT}V=)#Z+n`n>MN(s=r zc~q;k_=cO-u_a9f)jK}sO@0VDELGMDd zYiqZ3bZF2g7dobXI$}tUaG0kTrY^3gNEH1j#$+ci@1grH8SxJ}V}P+bJ4^qe^`&~f z)5%9(M_XIXgi}BANq715;dpA+n7)!5I$g<@>v)FQ*2@L>LLSoV^@*^dLFZzOiJAD6 z0497gCY#UA>_A1RsqOmAmjC8WeWIal-nnvU1z~q%U@#dYe7sjmKP0((NXKw zc8Hq7t9^?a4i8)fBQ&@Bor_uHo_*MMnU;zyh&Mmb{XH^b6;d@_zP2J^c}jMCY-HpB z+SK66ZiMcneIeoi+SJ<)LG}bQW}r;L0fLu{Ec8%FCHING3Hw42IWWjRIDZ-LD|=#_ zu)0JW@F^jbRrc)K2Zm9!?=tmv-`M_yH|*BUjv0hvx)!DaBk>Qu=(H6Yy+q!N!XvRE z(tUXGqN^fDAm71O5i5_Wr$Ssuqn{Z$|xl!rI|buao;(G{j;j zwu5o-OqnF3d4j0X@uvKWl`;G4l2`j$TXUh&vKksE1F!wZq|op=wmdH%8X^(h+e9rQKE?7% z_#)+CFFoZvq0AVht-u>LgRLnBqAy}#M*F(lp>!{in=w~TJ+Gv2>^J*eL0t0ZS_zdk z3(x2bkoa~y?a6Q;gLaQnW^k$0LYq_B+>%W${oe0V{_p8!_+E{s)kb2c(Zu&Y;hhpZ zPm%GFBZg0oaZgJxgY}1`Q=5jm89q=18}fIbzFjjIzL@WiyjU&B?Ln--)2x8BN6W6Y zY3G*q@3QQVzx~MBNBQZ=k~U*gRX1v$^m_!AEdyEaFsD4vHsqY>8uajsViJ54)R0A` z#3y25kg`>4+ccFeC7BU(=68=PM!t}yKG@dVi!oM(a!|iw#l(!g72WcAn$cUA3=SVl zS3}g@Rx|FbVg4tbm@1@gbYGW0Pc2ivuHKISlm1%@sVn@Pr=L$3x|JU%e4>0WbuBL5 zXuq9rDjxNbDshdAmQJNnQdv2x?eW`P93GU!d*av-14WQ}k8Qr+6$9&ch4tlGQWV;dS*3vprZM< zuu!0$I&eFxCG*e@TbJUm=Q5>#(t6n?CMF66{B%qyx8E~K4;^P44=)S1M~IHoV;2i|k4Glg2WJq39#K`4)Ai2U zYz{Ei8Gm)u`~tak>sByHD2_Z@K0OZGhf(|L?|xmL?`Lt>f82j!l>V`KFLNs4#J{w~ zEVfxQXHLh_FDNJdnRInYNkVhU#AC^liDxCewWej}=$-m#%Xf(6wTO1R@0w=Iyvi2O z4ky}G3f(xx%t;P7nDkqH-aIJrq@-Cyvrrc$wymvge%_oCrN?%rjBj)K{QMlDC&Ov} zaq%HYt|7z&bq(V?zK4tWjBG>5HCNPQf(&}BQ22+E{hbjJnj8_@f~rdGe_v`(e(nUH z%MpdM-kQ1k5qd05`3Cg=zLXvAf&1T=q_Ed0u6|3-n`E{#hE&-P@4wIDXhP9YuF)El z=$-W~?RZONJ3c5HU7s&j=FsP&z_h*A>L7xO2--{?Tr;1TjrzE>a&&cD@2FHnpF7cC z-I^3FFB=tshs)&ieix2VY2-d6;>}!=r*ZTu@RSXoy%fp%*}Y4-WSY0iFCG~ zFe88b__4XU+35B6S(jzJ(q0Fl4ANTpsPCWe`TRUR;^F1>MCfL{@n?t0O86NRSw+S0 znfF+sSc=*@R|g{l!{^Hjzk~yXioqzpfP^kM%01esQ+k;!?x*(?6-R-=G z`JEpw!=@b;ERN76K2Hx&{Kk+NktbqB9WTGpNL@_xkRv7T-fnk1RigCC2Q4=1Og4DV z$%%fQ(awwx4*%l`UEM6q4@`T1Ez6BFkYFSYS6X?SX7 zA#C3WZb=UEEgU5UEwrP1K7sAem*tSuYt7Eio)d6zBkDvF(HHs-ONl44|J+}PNF$H><%)GK<#Nu(*Y6RyZGyyhWu{*ScBiX%y-p|P=| z8eXkJ6v6)CVP4f(SQsl-`yAYl*PrQ-1?#@e$*Lz$o}|0IAVK3_KayDWa&xQH<#2rR z%7_00XLECz>~+G0rkexkH5cLU$;koudUsnrdGdqugjZ< zE-{N9%j`vZy11B__xZ`b#S@JTp~9LPibPd4qHvk{Co^w9f4*psz~$!Ve)Q;(PJsq` z@WsUiy#0fJSI61E5lv!ZVs+|BEjHsC2ZK9z&_DbCJ3OS52{g{tDmHEQ9n2D!rWKbX zl4j)Laq5n{u^EknDU6AML3xU%3R@IYYK?T*zPh)!*E{Y>hnU>F#A^+Xl*I}YBjzb8 z1~QzSz>~F~KMiX(6!V6u;iN3d%PT7cczC*cdcH1aXmUrWZuS1K8)Pny7_Gcr=#5y^ z)6DrOCsLDN(nsI#N%oAT5_RI!i;9bjGpcp6IL;DxUtC;7-jtSV6zaaU`_kVJyT>}G z%c|M5Y}ih#;`g6FHkh8h)&j+oBO@a-Gx2h!XmWAaD8+*}hB!GmCY)ImdMOYhoCwOq zi03cqv4UFR67WWzJ;PDggB@gSX!vXyVu2*)9YhZtH8r05>!WFND9J|C_3pN9q3G%k z927hTl}rp*$70uYvrZ-0gQt)lies1|3X(A~HHpFr;TliIOS`O$BTw2e>5#i>Tt+oq zVq!z%1{N;;Ewc&tM4rHb{38{ST-V3SC^_M$KJ(3!~pEOp& zE33Fo@0pt2XDz^LRhCuwxI%xTOki_oWcQX!T{22Z%Ui17fu{oL4(}efh- z&T29DFENLCpF5e$PRkChp4qVODTy(3)Py?HV19cm5K8GMCRn!hj3CvB5v$#AJ53*x zHRjo~-Jt95yDW2;Z9Ut-ICk?fWqPKePy55=-Gj#(C>#A8WTWh28uL*ZK3@%A>aHcYo_6f2$Y zU*7y9rqA^X)~T&c&c?hZmGJ3TQF5i`03qx8X_Oh+zcaGQz2W4MuhFlyX=%6bkYQ*> zWA%{t+f?;humAn)bx$pB!R(FmRhIHVjrurmK>TY_A}Qp*W40S zZqhltBr%E%NlQvt?J^aHC1GJ=-7Mpp6Oa_WU*bPCQx>)Ag03QZwWRl`-$<$YeP{F?L$dz;e4& zOP>UVObK?`G=bKBD1;4~5M0Jx8Lj@Uk(Ml~+)?oF-@jWQ@T#a0DatYuVZp;pIvga; z$g`1Q_(tE5LqmhoE1Smy-|wW9$)}s9(+?gAq%Tomg>DCVOU<^nB0MFFd&isCBP}9Yqum1=WsF1V zvzWNJ`t5`&S3Tdb;;~>c1Y_aj3ug|ng?dZTCid*<4LS=vB?%vLalbKPCkM~ijlmO3 z{}VTX{wyO{hZn`}1DYHj?%;3Y9wTpCJ_NC>&}T^6K0brx&(5YbX5YZ8udgSG`PIkU z^6%)mUa@}e$lBMhU(;TE`t<2|Z`I4wvti7{!eXJ`eN~qu#jy`RgSxT>UkzR?Vw1|kvi496Z>9;gd=WIm=c1}*CDx3H=^Ku#mW8-P4 z#|d;Yv){jmDlF@o&#@K9^>||oei@RvY$_^_DOc!AInOrjtqxUf0CITlK6?7MW#~d*UqAXGB)An?w?C^87&*S{ z?C%)?7@F?HFflV%+`&Mw&_e6#>eA9Eojq0s1k$-ABu*B)Vtt#XSy^pAMJW&++(=%J z$kb+FX3npwO1{s9%tJ{!YC7DOrL*>k=!Q>C?FCAwLfGmSMMXsgNlfRTLe*rJnP$K1dFm$b z%gc!f|M}j|(~~SuSXo(NVPQd$wY6m}DE#q5x7l>=`}gnPzCF**9( zXIe(IZ?+b-BlpQ}ei(Bprojli#lw>@T=8DxweQa2otNGnixV_D45%#nZ_Akg7Z(>d_dS;6W++enc+QTFk9bto)usFnxM*`^f=tWC1q+~< zwzs#Jnl|Ggw{PEm{rWXPgr`zSQ~3RZ$vu||xrXfkY)kb;V)o&Yk-dx4zj-!%K^JH6 zD{$1+iAkLdwY)c+tfG)#fH_-9P6(k>FRp=`eJZycsVsULwb@~?zx?zW4H6r))V+a>eTJuxWM?>=qP^CJIxHCn3C5MCE`Em zU!Oih<3HGW^U~aC9~T$*hA7(Ly>2=jc%fP_$#N7!-)@Wk>#jLpw$oT1)P28TVk*9C;LM$k< zCSr4Cb32a2k|3fRBeg;)$&2-hCLOx-3(~F=o=LCP_FB97i+d}pHNiqeL`1-|t*@=g zmB^QkFX^~L{HRw0-UEmOdtGoAzcZaV^@3e!eSICkP%!U(SQsAVs#mG0W$>=D=+UUE zCgTKu+E-VO1SZN4{v8|~%zf7l&E&&}xWY2RwCrJ_t{8T@YF~{bTFg_`siK6@+Ig4OcdkNc$99XnX5sHgxL zPR`;x#3$XNyw_WA|05#Ou%xEQ=uI9gKYtR9{R-aGr%&M#`)@kD8!ik|T21REw#p5n&BDTOG=|+RQB~9-f{8?)a-*8dDh=Hxn%0 zz_o9YNTu80i-|Elq#rI(aw{9M-m-xSB9( z`Ohjc_A=?CgIg2b0!h&p*t_iqo#Ff_GGCAHU6CT4l&$vF4W0|pb9{gXW$9f?1VibS z!ZB+Z51Ic%q_&+6U9GLbdz1dyp)Qzt!(u-bT+pvNGZDVbfl?DO%PZI<;n_n1C&$3R zQ1ZhnmHiL0n3$N9*56^9{r{(2UJ$P?d-zlX{lIty11(K8{`dTKB(obn;*THSkUPSY z&Y_F;5r?*|a{}-sB=oO+s{cI!o5KIGIiBPUv=TK=Ge5`1Jdd_5JXIm69y~~K?9C8< z0=c5~v;X0r8E>K+H;xx$d0)MJDI_dx(L16k^ZLb$(4OVxWoVmS`Qm^`@LX&BXL#GJ zH8ouIE#q6bE7s3BW{e4>cXjg3LmEfDPKD2Dor54VH8(dy9Y#k-r}N*I^YW_gX{5)p z)CcN`mPbH9;OEbua9OzC+OM#Jnwo=5=g)0RcVa0VXBtxO^G-DV`+m3eyA<58#N+Qw zH5eT$VLyT$4yHJh)2mO3I9gsPu2}8$42V8+ zj$8Bwq)GCN^cF1-00r1aMn+R{Hw2CA&ic~@0bs>aa4){J>iryWn)ZTF`pCRJ9Q($N z8#VDjoOn?+j++>ZZsgeB8H!FAh%q@@X`c%`pL)N-qVeLd>~v8aHrFcOzF9#fIWCfR zHvOyfI%zY1@Q5x{&}ggAG6m6H4J#Rr488R4YQHNoca3^ zR82~WiP;IHpPZf^`v-b^ixq_ODTzH^W}3ofEz6ny=bAifZ98DEq@)BXgElV+=k7Ow zm(QQKdf)ygj*b|X=oO8+4WhKd+pM6eL5-!y`YY#&yLihI`t$1QYDDY71LhPpQW6ql zyLSZzJvCY!DS(??aEy$M9CIp>Hyfi}ya~13iDozFoCE05mZ`??Sd3V|53j5KIySuya4ucM7yt3vg5%0D9)NKa{RXI61YwLG2O0cB#Kw}aTCra(A z+`ETQJ3cn{zMvrSE8mg-=lY?+!LC==a9ITvn*;*?EV-kjg!Ec}U-P&~PR0?=o}D#) ztzj#Wj?gpTezY8!a_6$(cf{pRsBZUz>s(<997>+?6f-k3aXO4rQpOzBc|r8p*x1^I z|8|%AkZ(Y?jg5)?nEd#WqeO2bgEx*onpjSnc+uh-ogl>craazvMFoi{*dRY0m06M} z$Hwie1=8bm#9*c2!c!DX+P(YEr9IYNE?+ki#$wPIG%Wn({r%<7YMJK<2=H#kD+ReE z;Njs>;$i4|gy*(T?%B%J{HuumfQ6L9vc=MvEI5st^u392PSz@D$usjC7#zS1PBrz# zet?tE(0Yr30R@S~DVfizK22o7YG2YCbY8T$1;DI|$75rx1VZ~rqHhfE+e6Fo!)pk< zzAu3D+Szprb)hBN?B=gg$k(2%;gj}1)4U_oH5l{Y!myXj^i1Zua7xCHce+z2_m}c$ z`vQtLTfX99D0q7xv8|kS9}!kf;vtnPlW$qCFH5~*#3Oh6n2vF88W(@1J0M;sP&7hk zb~`X^CINQ*J8kwzJ!@<03110HEp%%8QNqP|C=cFJA)h=&1@@6Uy>~ z2jun`sk~<0tlt19z#iF@E~4NzWJ*zU43mW(V`cF5>G!+5{CDq|`tNi>^_r5^K%T2G zLGfkwVIH}3J^97Tl6j*3l=&%*jF)=C>;81J22&c^{tVHghNgxM&s0EGSr{FD;w|w? zR#2`c9VlWHV^g_J%RS^=zMu;K^_*$;-*Fol5`HBYtI{dlDnw^5a+t{~BWfUKiiLFR zSuI7yuUpw;hRLOwF6Zu6`7Qqw~5aACqIAAXVH#UQ-U=#n8v>Z ziO1m*OO9x8-fhr%H-?B19g@&@`|2!Vlr#%=0Ra+W#P|tYkEE`|W&WsqzTA24 zb3smEhQW(@F|V}ZZj zl{m!>2@D?t75T#OsK`wJb=uk74R$==j^mNlzDw=dk@LBWyl>yM>WyhhNCY-(foU27 zXat-}jVEsI^9}4r^o{6pXowu4hPIgmGdD%7X_@j})%E@#DO7rq&*XPrj@mbS|FK*P zdUbPOi3|2IHa>o}KG&x!oB;VF+>BJ12Zzwmxh9nGDQ?wd@u>Yy%JSL+%^?G^DDlzZ zvHQEKs!wz{1M#r2iDMM~rRh)vli1{lXrn+C*)~(OOj6-OL1MeK{pu-@&`^JV+jNhc z7B9-LUo1ypyB@xJtY0#w)1#RZpc9FE4``q}3`w%|+ilN*ey6_@UoO#;m5|o>pHvbh z2}7%=^KwoC>3oEWKaM`cmpQ~yd+{dan-<8NUDA|Z=F=+lbXoF8m=dE|T>s9E)gF`- zes}v#!`qvPyu5tV7Ab$aIwW9(&vm!+`b7`!V!7XjiL-Iz&gyy4*yE>H($F0SV)O@B zxa#W3TSf~1IDB{l9Fu(Rx1wwKHdT}6d_a*DKGqp`GVDuZ<`{h7@4AhPz)x6KOt^;P zhtfw64|BuCKsNy@O)dZB?_|omkg6aE@WTJ8gz1%MUt9xa5F5Kf zpKEJt3m)j@*GS4MB1!4aoruUt7G~y|dUxd#JwiOZx)Jk^>d2{#DSv4hF)_;3_o8lr zOKv)`T>?H|e+Buk(WT#^Q@FktHG_VW5b0{X;nQIrEqs zNVD)vJ_b7wLI4>L%^)4s;UB*Eh_3uZFaAfIv|11S(0NEF?PyBMvRz9y1Kg?T#jWg5d_}2 zuk(G3KTmE_Q=hfBG_`WwJs!F|($wVwE|8ZT7x3qTT%3}XfNz%f_rOZWrd zEI<~YTuOZdQ;cu*w2#ca{_OpS0f|;*5VAT#UXXMu^fmtdyb<@t-GKJ3kq8gZV8j!X z?(5P?@7^)7-oZ%h=^GlNqoh1K*(jU&>owEx6p$)Z8Xh*biI$+2L~i-P=UG`?Nh;q* zMmpNtnMitJE#QBC*b5_9G90K~&dvK2hzipSuUVhfqO#8%?Ci&VW*3!^9b3+pd4cmy ziA%yjt`pF@n0KhS~gc@OX!J%Xcv0GHFe&rt7>YF+X=C1W}^kO>%524y?giW zUZzO%$$HU*m4QL}Qi7Yid-~GTjWOVkAUqa(pviZyk<`@CNQjBCkX~6{PI>jpvW4-) zGoiP?f4cg~b=oqM##hHEfk>q+NEN ztr?6|l%H7cdzl4d9v))uXph&3@iIUE@Zm!|l_-iVXg)SJD;MXdf&v2C+S<=zW1B^k zaw>iHYzlRKw|-VX9(kXVl48=}@%WRnqIWFtOpVJ%H4fcF40K z-5ABUBWut8JJvTgfG2pUqZ2l&cK7b(ZfeV~)m2*?8yjnDpmZFkYJ`P^xJ(+Za2l0} zgP6;FaPn=n{O_W(-b>pH9k9{gNw*F)gt5(FRxcYxrikOBE->v!AFQRb{?`l8EFdU2H8nMA zzyCcSc+I=%{DK^*l5AwpOd36N)haAHA|X6%p%HSbNzcxf_SyXfTm^w-6M$oQ3x&E8 z3?|?FC~%8LKaOvd=shvEoZdZK)}NUDd9ik(>C=jfgR}dH1eY}*EfeS+_@yvRLXMQL z{r&2H;~jMYmunZ2nQ=R8`E)9@w}oL&E-u2(x$W^=>(Qfp+hZI^lE>o*9szUAHg zyQy#QxB0x&UX%RWq+?5_ z*FOSiwlE^f%d1&1;u=Q*0RF?G!_QBJd3j4g=Vg}>%Oo+@Bk%9I)i>qlE~Gp(y)m3x zeWyfE@C5pg(W)WoA}zM*hX(B(9R->>v~+ZT^*i`(#_JV5U`UEnsogeBy8|-%qys^6 zWXW6;t#I1K=L^jPe@lKNC{jT7@s2nVa56h!6B7R2+Csm~p64MYCT=^^=$@@*oOshB zDIzj9hLSsKFX}MS>K_*qLoQ zHhb7k$hm0;W9sE0+{fEkv?+FOZrG-4v(0{1R#pY->5y)VbvX#>q^T(=B3WKlKN+uV ziI&e5^Z5M@fabG~Q&4u(b3uJAz$Bpl($@!i#=vr4YATP3y`(wv<;$0Ec0zwRDewmE zD4|^fqV`ZX@sGJkCco(}o#_!KOW@uR-NmufxCw3Is#O;XwHkJ#)Ly?JtT$@#K&I$b zZzt9^N&bq<;Kl3L_|TJ5G&8|_bzdEXl18a(bpQSiP{SyFwJ6AVsmbYjQCa!Bw{KC5 zATuGz3T2smVdz8g?b@Fr5vd?@)R&Yv{`~ZKg38m^eE`)WE_7A zz4X>^qhq36kp54?zRFY2f zu48*6N+lrR9124NB&UM|&(|7J1n3|PruN+z7Wdg@T9S6@FEwd|XtA`k)QVTp(15Po zutY>qFxkiZp`PAbV}*blqt{o&#KjGYMk{k(r>CcvwOpKlh;w;5cX zy95`r#eG#nXjwI1J3>BWU7R3Wu26Sue4JOmT$U%*`R6Cqyy5wX-;EViBlM|#?(Xh; zySssKq8uro7vkOgeami0t9&;(%Hci3*3{Jg?sYdU&>&>)Rel8us7#*Mj@S z(^V6hKpLYPT#zp6cUt@0S^CnY^`Tk7e!t*QcvMuLX1F`N7fwk@D%(TBA*m{-nFh#N zD8afM=TmbRp4;o}%VD$0#y6Ir+Ew z(A2^rH(BF(X=!QgtS@d#FvtX$Pw>B~#(XtVP(W2KvC3)oSbG7cjkvpcca)ScKFdB4 z%J}+K&Hwkx>Z+)S$i;}vIqBX#CjBEnw8Bx{AVlQziY=UC8wlB zMEt&C(CE1ZK{CPVB!y*92_!()KYbsr<}Q9~ zF*^5}=yM#{a!&BKy6O?H*<5-QoF}a;f8u1GwmA8>2HR30#`So2d169e$*Q&2?eLEY z(%QbbXvq~99c_NG7@>a8Ki9hSE946MYH#w3M}a1u$B>SYH$_DoKdYbY?Cj`I{QN1g zX(vuCZpbA);G0#ZU={lQ;``Db*~OC(KQ`lUP3!P@0#ARf1c6`mrRzpxRDW@(Vjc;No z@%>$PnUmww9aqaIDM88Vqx0M3&({{W>c%U+%I!dH=WEvkI-(JOd+QsvfqlQp^AflUetv$~*>DHy<>(|-R2%JNl@TM+h9#wig@6s| z3K#$4fvhI#Jo_nL9o}mzdd^_jhT?Y8(CXgIP~YVExTu&I8xfYCOp75C#kJs3`}56? zAq1>(iX15!>MWvrfeAvq1t#nZERcKM1<+o&mc@N(=SopC!L7g}B&_B32>kTh^cWJX z@$Zan1%Nve5)y#6G_$4ZJ+}8jg^)^LODxAl zCZ(iwg{#{>dL;fuuCqyh;sLLi?d`_8Vc|~83S*w0{A{Qo&>BD(`$5*PHr!&(@xOw} zdCG~4OFn=iqX=j$&#)YFu%zlpVrVLU2LrwVKIRE^vRb}dJ@svYoAHr(8T0hjhTIyF zxwQRYtJ&Y*uZbVZ6dfwnFD@?T;^6q%@U*BTT@(a*@b#oj&=}v171xgZ&OwP0HHFhT z^=9K%fS#|KFV3s1(*XGn>i1@EHYXtAY`^>3P8<}q1YO68S)&245@d%$w2#q&C zmpWyS%7XV};#K9$*RM-P?QgVT6Ms|5|K8p%Z!8j4LSW5N|q+}65UsNIpfH1r2pAop%1NTQCs^tra zh`hR;yyF0AMahcqBI!Qbo~Q~F$ypGzg*4Lsl=$WL)fqm2hx+?psxYb3v%V@eX&j5l zCxtE<0=JW#;c|!WvI3-#?(XipsXY8LBPS=mx;kms1Hd|$i~V66Qq#5S*B!;f01%ouYVX_2h|S_Fy9F}tu*@`0y@8<<8t@r*QcABr0R`L z!QW)@kG+wj*T=y25QsvBKz zR=nA<2p*r8oDWNXt*k5>p+%}n>@68UZ8Ncxa<5p ziLbC(pqNV@{P`7(Era%O3wjNH?SQ|)7JL1LOciJMfQ<}D=h7@M$C>IOx6M#O4_xEM z1^tI^#TLfUlmMY@C&&df8XKEz$|9s3%8&6O(<-j6JMhr8&U2-s_RuSJKgJodRPj_* z?fqaoBLCWB?sI^Jg$3?)o*(wr@Paq#;x!72%F2{)w}zennX3K!t2fzpD7&P%xPNFU zX!W(1)j|FM-V-A0N*Fe2_w}{^;6O z`v(Xj2)YnLTB)?09KypAwGVGp^0js8T=%2)RM~&IiL)HZm$Xz??rH{|kmhMl@X)&A z+kV1f5)g2ES6?L+J!&sZiHDSnIaet94%r<}M&$tK3WP-J<4OltB{i`jfbC!gqGUv~ z#l|i$Q;K_;l5#JN4_FI}iH8ZLr95PccJ$OD#g(8a^5a~?(+x;rfKtC%HV2-c3k9A1s z^Z?d!mmBtlvQFN2x?4!R$Cm@db_(2%C4W)B9S_3h27q4oU}=!GrSnUgc#mX~Yg z4U=o#heTpI_9<3 zh^p(KrOucUSg+~RA8pGWHA7kWI5;nQtxZ!%LQqZ&>?+vWz+eU8=gyY?9_S6nse{pn zX7E6O1-JIc%?^K5CZOX|e5$EQ_*4yAX3?lUmGgr6p&}Wv={Pq+4RC(Fk;}Y$?m>SvvlXz7ZV!ieyvQ&|&bk&80!%f)X0S*!ki-V6gn&S0OG`^*V`E|AV~-wyJNyDthkyUv$(e5O z+1TysL+D9jl$diPtgg=8_a-)(ST66#k`#=@-(1&aW4jhLf{yG%F*E_*gPsi@Eqs!2 zGk!6#iK!{S)j^s>^~bSYq3oYZ^f+%aS=8P5T4*kG^B#+ip^{;LxF7c2&)mmj)&kgQ zC=dpH4L@+M96R3VpDJbXkJ6v41Ms>ejcpIl4}!{x)u; zSda76SA=a|YWHmzuQsf0UbaOj{Ta11E;L`>k?mnRl(mctd&im==gIo3V*^IxLG^>H zdq8J7BtPPJ_y6!%+5tTQal^dQmDlG7BOHjWMKZ6Mf6(<~i}02*C8{c=+1U(aK$ELO z9kdOR4Gz&|j9>n-vF|S(^z+|i(xty!EhjUna|?f%6z&*$GrdZ_@}lBv<(MlH_K_>ZL{DJ0=Z(C76qt$>AQ{MTtHBFPN31e z#a6|R8(!=jXGM_I`L zvvW_U0g$N>W1r^=uhap`F!RsFK8Zsee9mFBS|<1L398oOCA15L*x3tnb3-a^h{I1Q zmxiS&y|A?>!`fl4E|J<{59nzs-b*f{bi)uBD}@2;AuQVB7T`g?8oh$vWqI;$qKVUy?#YQIH9cHoWEn4uRvmNpO{;?v@`n>Q1s|IDI#Pogfh!wm_v|`LDCJJtP?C>j#1XVj%7T z9C&I10N?=*fRVAI!V=DLKgS!o_sFNIFb~jBT)YXd9?0@^5yuS2KJcc2!ibayU0%R! z0w~-zj(E|GU7;@h(BJ|HYLJ441bOr66Bz<~cw$tHZU#*lug8?c?ESN=aiwm8G$z~b z%^SDci4a(VlOY8f8%8ib)z&WEOoq7yiBj0VaVOU#q$8gO zfvDNt(<9i7CrP<@3$yvwheWUu03-6<)+8@-XI18MvNUpdW`;XS#b5Bn@Q`zxALJ^}jmK$_ZYB5kaZh#bGu2*|Q80M?=V4KyOHHw_k(8@tYzh zJ|5U$=hsIXogEtY?hOFVGzoVGY#+3!-qfU|FuSr8UpfQ^(0{-08v*tQNl2q$PpP0L zu08K{vqjNgKUOT~uw2dlN`jn_hOVYNd*x<%ZQdm-H;-zEQP#G$svG-y!ITbBaz!tK zUvqlWhwbCXk5K+o)R+-#>*-kGHcA4nEPkuVtp|X^sXxkToGF7P09YPqSlFB{6w!!2 zXr5ghw%AH4!fyf?BFBMvRngRx2|TN_l>{r`8BlL0KcAFMKHx&=1q8m^HbelVp!8T@ z(!g#%d}BaxJ?Z3)B-G`d1$TVT=B(9_lB6Haf3+zGNF#2GlICPl#31}{DO?p?UG%Bm`` z&D`FLh>1cxV$BQA|iA%!cM z!XIFuLs46vBLxO})_~EyS!1K6MPyG24T&>o=th{gX@2j!1mbm62Aq2S5R{r=k*kZ7 zr9T3GChG?WB3q~n174t2Bz#EyAAH(~F)HUR_6&=z`5RhiA1U0^*-qp0`}T_F98@;r zBU3cBQ-(k3iKrQnH$Z{;1{fM*wm)65g7bV{at92dspa0@LD& z$v)!T6u6(&`}@;lm`BM+ZK@3DiOkK}8!PEnmeG)E!`Clg{%RF$U$bP(ixAbXbuw{q z*aSv6V2n`?FBbes+7$=_Mi;=Rp$eTrdv0$ztrb&`@GyrQ4#qnr{>Ti9e~dFZjwN>x zrcJ~PrZhoAX;ZmgD3P98)zV@H9{mg{c4JTHlFpQmoRH9kggg{Ur_XUpmzU9kVI~B8 zC_*N6=?eug_NAJ)Mod5}^}B5ht35SZ@hxb4SZFBY#ZG}4sUw#rx9WU}oYPH0z7(g> z6xMzE+4?x-_4f`rU&2&O*`r&fx4oj!&6MBU&bUWUZ#7KKekZuLNMr#ls1tZ~|E-OP z%b5cvQ?j=H1@;(V@tmjY7M4nzPv;MfV$rESnU*_JI!J~*k=`Y)=Szzaq@_j{YT@!d zMmX_gMWfH~ZGKG~`58PgMGxGJ^^jFoRt6~?P8N!e3przWe0;T!s|%crAa>{G=0dTF zjgPUCvlI$W3>!yF!Ia5nz-`wD)1UDowO;9Ug>nbv|nH`lIy#uE{v=VQp-ZnO# zg5H}ajOV;YVb^=8y~Jqwp+J(gG9=I1W!+tfq`ZI#Z**Vlso0_IJ5=O59|}dk-$LQ# z&t`IlLmGw|uLh$`qqeBtfM&Ue53e^wDO~v$EM$YO!%!&X5$U^@3kaM!@PodK2MQ+6 zH6!QXTr0jCRZ%kqk_9yH3oXb|YW73dh9j;KUEt*m9Y(LU+gHDJN_v$P$($PW8Sl_N ztYPlrgCHN@JYnbyaDgq5tO z&1aN|CuMgz?=SgY_|jE|cyPXO-Z<_PQB({$DY?E|^$7=Bu|ha~bVbNRAw=drR~l-9 zUfX@Fw8Ovy_YC55DTdAge zvHkE7auL$qpoHZT1y-Yw!L(6s^WUxLHQ$j#<=7xc6T+HA$CWA9O@^9g_lA`;y0n(f zQb%rz&1-z9HUtwUQlFfXfL22$Dy(`EXe75Mmin{A_ad&Frir`1$DI22#))6pkt47e zj446f*J|aj?m2w*c4fA&Umy%n1|BV7ruLX0$~6L#0Yp-7TsmwAtT*VvVarF)yD^X7 z-ig&KHSYZ(9&jR50|UHZ1Nct|0+VT1SC?;db_vh?XRV1rKyF}3Z45m31^r}T(V+MS z1w{??_h4Fo`<9{3dCy$;#vlSbWW;qyzfy3@l*m-y`f}z#E5N$cWr>q=RVCcDb8Nt2><4K>^M#|r5 zJis*Vdwb<9X38Eq-eS)^=%#Y!(l37lGcr|GBh?gQp^`A-0x%JH&i{dIvz3ov#AS7~ z=;34%5j5IhvfJ&U3ov;Jlcn4L>3q;g+Dlx6;_xWq{#M4rgI=X z7bqFCdqXmW2(2zJ84mjCL~29CQq~*{h(nqbf6%M=@WV&liQ+JW%^dL-n7~vp9y}f( z3?xFZXm7g6Ki!_s`_s#sb2ZrcuvmW~Y3voRS>Qr@1jNB(p$uROfMEoOt^pkrFK^k~ zw~JHGEu(fqbN+kyW7mR#SuDuU-#SY+PczO^AqcBsS^4Wg{ab7b=+R=*= z7?&WZeK($bP{c^lzPyrqtm~y_{2Lb&a=rdJ?Fa)=< zV#6W=+@?HFs!@qvzP8W5-^1eWE9S4?zh@~PTH5*Ace=zw^1ogH(9pz@`8gQyX*z&- z>i(*J#zI=qyR-wYKp1!V+Siv}I(cXfEE@z{Jq&>JsH>>V;|1T>n?Nly zlz_PZ@^8`tF%*Qz%Wk9cbU|=yi@Qr7&4;045{T){iR=dVEHSgN2n_ih6ec7jFe)bi z3k@DMm^Fs+RZm85Xv14SfAeg{J~iSs`E5;HS_P=(IzIm?c zXfYOc>q47OUu80QA>LE*96MTw|{(41TuR^y<*X5 z;KlK3Li5qpU@!vH?^l=`wiWP^UN%{t>9_qva)!L-2!y!6OJ(P=a|0&=<-s#>L)5hd zb_PJb2RFs#`P}9Cd|TK>)dkA=>#v2E%_R>zsxg37qsNk=f9`>cfG~Mn8+2M5betOG zT0lJYG`nN0;&z{iNx8u%yCl;x+f&{&```GFot##TEyqI#?%xF0EHvz3ZG}~IAG#gY zP2R06$l(8&@9^B6V2KBK`Gq!0buV}?EH~pJZloIVMrpn|@iRZd32buRitM#sJU$=n zfImWjhn7b9@jq8jw+klb1kV>}FyrIn6Wmo0LNF4rbURV(t$-Jr5v~9IQA- zzJGtI29xK;)%N5%`e5kM)%95ZN^V1r?W?pISO{AitVb3Du(lifzBZ@@#1Sq)L*x`* z&z{*Hx$$3@4Dz(NXAW#W@VNu=2OD9i+pPhwlL12;v9~l@Jh}#X>0&xbG zb^e8lh+fhD*A&j<<6{@Is@|cY9O(dXY_$yD*KtAmKcrow|DYe;wHO*2y1VOk@MWP* zp9mAkX5uAor9fO|jBX?E#_J*=oj6vsS{=ejB{120N=*@cZ@yEo;8R_{O>p9GiI`|q|uomwfOHWTvOUp$Nu25i{)Eh)k*j&J* zctW*-mDJI^9U;z_CINTPk+PnjY6^UF*{D5`)|bnvW_TE=Qbz&Tc}@Kxj{@N$S%+#T zV!pcgiKZ12qBv^nS2)WDeM-E@M45}ULvu?lR(IQg%k#szpvz`SFSk!0ii;J`@et^Q z`4Z6;z#Q{8@K6%@0xZqfuN>Um`{v`|-=xYd<5>-nov}3h+5FPrz1+w^WF|sHO&u3I z3==a-#6jcS5PTqgq?!e;?W&{|l&R(WLa~4{3`QV~cqJWU<6`YX1l$5JC-$+h5L@Rl zxSYN3(VPzoGYwPEuYD1tPz!6q*D0*Q&_J+ssQ#4JNxUM<| zrVRpaBTpo)9`^|aIr(Fun<;AhJ3Vv|Q(V7_wQqEkZxN;4j4d%WS9NgUDwVzsEgNPd zcwxQ{3~+OD?rjR_rar~R`(+)#?71e2! z%-JnY&JFMs?7m62Sm{p(nFM(bBMh*}zI?yht1nZ#EZgQVtZFHz+9<@j*dAlbxtMcK z9+@u4s70>22l}n=96(E2-;{&Z9>_7JWY-=h8B&omP_xE4!OeFmUL%Nt=yH(r;6Hj; z^FSc@X{=4&Mndw<&RKPl5-^pIR1pwY8#*3N1VoJqY_&W#_C1RUWhcESAbt=ruR^};5#0n-fod(${E%~uftw!e^RKFIYil|{UhP$_( zRiBcJF;+C6SuDX{9EppMM@N9y15gECL|=~Zq>MB;s+4 zs~QOUsIBGl-JD9gW~FW(h_X!i(+8~qdx%sWv&5cI<@9@7SK|#@qYay0qup>EMR^2Q;+OmuMP_g#N&%ruwV9WEVI0<$)cYIR*5!I z)bQ?9ccs%c!gb7=A%{W6oW1hZmsK)9B?Qc)sb8ym)hyT8PSQ-$PW_Z}z*ImZDqW+{ zDrE*F@G)biS}rcsinb zQh99Kc~P77rm1fdAPz4H8$T0jbD@yMHMlLv>vGA5-|81(_+S)wEe~BHSjmqJjsHE^ zu)$fYJ`p~&t!G|;`I{ozM0_yNaPLo9pSi|lHK~}>x0-~K>yaUO3VbQzuH@9GIP9cW zsHv;s>l8a*y=?-8$3*4lc}Z?{h0-@+YVeqSygvIvIq*?~3!dMt$Ss4^oDGMru8`(% z(z_N$J*6`d@COw{3|r65W5e08k1Cq01KtE7C}y?WM*k0E?;S{G|HhBoj(zNragLqr zWD5tSlF_iULS&CbHpfbKM%fez8QFVgg(TV8QJE>(zSns^-|v6FKYq_2Jq?a??)!be z-`DlJUZcCix#{_aUsbj4lm7P?zu<+RkR>=oSu8Cr%a}dCTIV%195}~*yIprUps|qs zzRaJ`H4Z}+v;AI5VgUj7qjD_>_)R+b`19RV!#$PMm`Ng<8^hPY*UcwSv!x! zdDUcUvt;G)jAgIL zlH8U!Xp>uflz?TjXnj-wCIkRFI`{j35#i_4>WK6I(4z2CVZNuGymQguX;|pIP~(}b z?RE{mNo?n<^_EWAQzyz5Cd;qYZXYuk3NG_k2P00618F&jo}=!c(OVNxYsjcFq>*^e3tIayzA`gKvqW}2{oV2Mgqsn5$M z!zn{{R4g=^cr;-)AVS`PW@4{x6E8A?HS3=Y)81d-&`C#rs!(qhmC}w-^yI`B7!zuF zYB+e{{bD-Z&bZ&BH1U?2JrwIS!Jl&RdZLqm$p?KgS%O*pWc-v6tH*faRVi%@l_1F{ z1*QLyfCRbpH7VMpaMfpbSxP(J^V@w9S)^Os&NnOE`mq@tG$wM~g_bO-4Z+`R-Eh-l z@bX7ytZDxWrI23_*LO7)a`X>T!mEkFHur~@ewuh`COb%%#dqNX1)kSBt{?dc z+HWC@8Hm9@ib$H3@UPw!$_;N{9=96i3PA{$&@8?-b4}>&>Vx4;+GK~0A_Y&ke?2I$ zQ(wFw!+ThL2%Dy-t}KIEH?Dqr|KLn^hav;ofpvL2)c;3& zJS#gLE)j6N55t^N^VY0;5j{&H##MGJ{mE)nc*Uir^^!olrJtR`xw>6{y#qR;Q_Z3{ySRBj|)FJh=~La3oy9wyrFu=5qV)G;Qh*|Oq7N7 zM255M>HQFvb2*|zOFR4@W=BPwXPb!e8xOzu``bZ*y&n=OIjg!*&SN={4YF|&lN!I{ zogOHudi(l(4pvfOkePq0Xo5ciXx1WTP2ND201(`$(z*|fI2^@Upg+R50OBjuS5T%Y za=HLt30TRX_$Vy(>r8}!8(hG1lq44%GBqDQ{0ZOWH{TWvb>B3;MJCLI#mt)aq4&LA zbO)S|0^kS!eR2YoZ1mHo6Dd((bg?zt$V__0JBmZWD@PI^4}7k^p zZh)w*g>MN)gOx@R5J7>b+F>E?2R$478`QmEat3S$5s^{=44^A6D*6F80{Y^Skr9if z3;1}&V=q%vA4AnqW7SSM4@>O7>dA1ms4(jg+FtVElx_UV>^8 ziZzgv%k5+Z+pYc9`Tx@*n}xw>5`iZ^!Obsy1?j2t!p_A$0{5Zzc9>v7{^bkNKb~`N z8-U({heD$lpnZq8uU}jZs76y0JJ$q^AT7k#JmsUifyRoT12rNeqAK$lmvR=zqhM0_ zoWL#-7N(3LcbvWkE9d_G{7`)G)(f)lhj07)BvuCy_WaGo4TU{j+JO%J3izy~c(}&? z;CniGm=%oNIM*BAys-l+9RMAij-$n+u7CVt?o3JXlj&)lZ^6jr`*XAJ8jwKKbQNiBebAf0B82r04F`_!Bt1a=&N}c{Vb+uWuO`8S zD&1i{OYm-n@ZcH(QW>=_E<19p|G79?0ov_Aksg#X#$+=_i zfvPB~dlAQ&h7||Ye`gBnsbR2D`KFp#e=h@nep}p=fwav>yTr8q^ShCbzDVsrT^M*M zDk{p!)xUk)ALam3L3$m<1*2IjLr{U~>wkrJh8s*&R&Gz6*un@NT3yik!o>p!YBhcE z12S+JCYM+WaUVH+P&$bly`M6fcDm@!jsHA#G@Etr0$>KfW5LJaPr4$Me;ESdIsf}a zN#2}&97Z*YAMpKP_t?$}0YpzEbpTWm?PGWp@V4TCQkzM;4L_a*KCsQ(3ig|O(u|=4NHRAW{JMT{rfyi`W8YH z{?aa!l^ViK%#4gb{ysS-&Fu`y7o51u1|t`ET0Xe-h!a{|wn}+Cea!R`5TFt08=a-t z7TU*3p{Y}T^z*qkU?)baF4_hLsk_O|*TF4i00g_nG_hURaJ??xIA*Eh=#GIwKL6C0 zFX|O=*Stxjg8R8`a`SYB+f|0}sA{$!$SWwkC?|twI4NoL)2Ft8cRa&N|4gB~g;pbG z*aZxWR(CJ4QQ^UGX#Rcx$i84&N*Y4rU16A!l;i;>as?Sia#q#GVDLjvPyeES9lzYD zHNYQU0B}#einc?$#t$n(c2z;7RN_MpEJy#g{*nGASQ0U9WLgqYRLHW_D;{%DGeu45IB99dbvT<3jm+e>HZQtgRsXvn+oP9O}-9VAAT*5D)*Is^MHeH3-Pv4 zAL&$KBlrLHwD;q*DEal<@P*A=ngEy;5Qty})5L(EML3S=BcQ|q{OPx|h{G=@5Yu>} zH%#sdF)q2fi8y?PrxUoQKN3vjQ=riWPF>FuPz^ucZwDF3-^RHUYCQ4JA$nViAz(sP zccH7;P47PiW2lRedlH8$Zz1?xw%#RJ=L~-cHdznPXn;;&?9+# zQ4&l2Fw^EIjHKnzDG%X+qXpZ!bKu%Di{AwO91fNQoLZukNkEeD$|e)q(Q+Cseu>GX zxh)a6(Wd@Sfc8M+bq~CP($h`!^}n~H-k_y&DYg8;lKU?RSTG1FoF&5Yb^)ft_W^vGCO$dW}~`0mL}dZ!{HOS-^m3}{^i3p9-5b|P``iz zp8^m1Sh%S+Dp*tRS?BiE-CsX{HqBCi(HC&VMNMsR)T#haN2!22wMFQ74(|D=RZ|d8 z7?*~VcCM^5$%pHmpD-p+I8RKu^TFjY4@vB0%juBouZ>#+nw^-b-*G&|dxj=rCY9{T za8dPU^s;i{PyY&54?m0|5T|dQ;%0e+Hkf+PhQCc;)p(S7GUq!K%xas)Ok!=TtA|o2 zlV|X&rArdJ&n*#CV@zKpZBPGl-*tvmtD=+s(+=z zzX)p^%Un8Km`m`QrhKpAx9*X&?*4IFzo3h7!|NgxbIjbrw-GTm&$OWQ5 zxZiyJ`W2qL$F8pBFfz>=ZBq~QEY)uWW_T1;`1M2);FG zSsM5dmw`vH=&h6^t|{jC2T0#*(MZ7Vir%~l8s2BH9H#D*`beC`Bw0@xP1YHN&liXi zAQ2nNSdZH8k)iBxvA`PmTFj6t)~dSRrL`k4BjrmB5f`QG?7C{gyT+Hv0JcBD7p-9X zhHWxr(EiKAzYb;11tWHz@*L|yrbb5fPSescSMo>fo?o{CBMuBOVmH1uAH~Rg1HuTP zq1$O?MMc1mXR9iKca-@D6;4QgbChTv%$wp5#0|)|O;|nOF^= zs;$lQ7c#Qp_rbfbVYhnqiirBv=nqsJs0e_>&746su^olicAtmmB4LX>$56$Hgj84+ zF{XEEhqvSbZ{1QT^K)6Pwqid_tY%tAx$wE#(xJ$j$at6+g<BY#B23BFK}2CM7gaWPE#`ny?TLdZt@+XP>6)>T(tpLF&=v1T9uS4oS+~ zBAco1@d7lMmx3={Gq=7JM zrJ$u85$>g|D4V+$PGQj4W)FfRdQQmGC%CsD3DzoWD&EGR_h39}Ld z*@DG?%u6x4-M?<6zXaBD#qAb`tpQ4YRF!E-zn--@hrpCzI}{ar)dHydveuUH^UYS^ zM6003C%U?p#9k|=PZ%K3X9ItqCkb_dA08^69t~8fm9Y!^X+p{5;~kSh^lOX^7!`z# z?W{Skq?9bSAVZC)Q~cS$%Jqo}*gsh)?k2Na`T#Aut`Wn>)JQOhN`t%vDqi~R^p;;JO(s>T#mJ5%#TauP5 z;fjEjqC>TDXnWgTGFZ@`D9X5Zd_85A5xKyyV~%_8GIv>4{_;7vKM$Rq`4=)w!F^Nv zw#Vab(U(8P{yEp4`}|jHPaT1M6=9hs+lUcwnwgm?mj_1V?}KCHIiL;$`_muFJw1SZbO*H88fm4`kKF}#jGT{1XpuPiYX)^_!w{d%uM zj6s7>`-&?8`bQ9v(Mb9Qa1w$|?b zErLi@w3E00fP<(`gf2`Kg&j~YB1eHWU6%j`#VchM#j=CMtYs~5V(g5S94b0AmyeB! zFd=w9zAbSXFQk@NhmHQAJT+EJyNqQ#R*l3<9pFp2hQvV2J|X_Bz!~)bIdAr3$GiL; zcTpaKsB6+i(2bXv-_c28^K^iSz%ma=z;rK&{&c~D+h9skD!RrqvYAChfJRz6P(&(F zU3f0M=toA>;R)LAt`&Ei-isEu|1RX(Kdc4c)N5CHr+|YBG92WzLgo;TJL}6pV0m3q zUYXx0 zsM{}I5V)E4VRiTXQ^{Bsy+k_-x?~NM7L~<{0TBmJy-vFjLsB1usyRWhxF`(h!F>dQ zKA@}zbMi?x)C*61=E0Dek&i=hOk4_mlI=(RqUVXLpBg z6_AYrTYige^NAwo8;)@yWIrE!BT2bHFMtT9qBNJOgH<#QK{+?}Fw8fhXUWni#X*YW zA3xH|O#4mPrRtA_8QKEERbkC|A?wl%$4<60k2$=vDHczo+>Z@qA_M$tr#HMB7uu6v z_Zsg_;0z{Br~Qi?|K7TATvk|6aCmBvNS{{Ts`6j^qr|BLc8zg8q3m@RJsyBPuY4G1 zy#i)biuZG@q>F9It~(49JN7jBlg>YXBTNt+Z&zpf4*ZZc{|?^lk3v3mGSlXaBNZa~ z_6*Izf>%4M@;jtwiMW23q^!G>r2hR?xqy;$>XWlD%E{F3ZEIK`v7ulVdFA~t?A5IE z*-=sJjRJ9v zwUS@1D0b8>^)#qj&)ySF9eZeFl#*A}lkIn0A;8(W)#8KFBBY13(J?dwJO3}jULJ6+?R-~gVcFfm-D?+|me$-~N6xcv&e)_XIIH(5TLWx6t5oY| zjZ&cI(k`S&OR32%)tf8IcnoQo`W}7X$Pc{ykX@o91}sPFXPw;%l1`Z6u^94eIX0L^Y9Q8n#)+gXd^ zuYiC^-^W|I)XOkJ2!jd7q-kz~9p_P6AhQU?lC*}Rm z74H{F#~&-0B;VU@b;wz+J^SZU%2ce$`_xel1D-&U7tc>bQP>)0AV1ZfPkR{3qLt^; zCa&cg`tih|0WbE!KK%!!Cppr3L&}x(z2eUXZGoM|pfPwp?e^V!Va@A^+yRkPWyQnX z&dnS_n21vmxb~7|Otq?Dks5kNlSaC0?hWhG3L1LxSEDmjep9F~6T$YhEg;_g>!u*B zG!No)W@C`ZCq%o$efcxDbKYa}?9>2+RXPIB-077$~se z(ryP{)1H5RM%?kzy+j1oZD$$5))ro0#wd-~4L$7O8vU9C(ClGDX&;5!GF* z;0*k|^R)0ndGGt_b%E0-i@#S>X)z2MuL}$5Ph@7dKQrb3yP}s!>i$`3aCLVetZ^A`23o;-aY(DK@kSY*d(H?IP@SiM~DV&4k!R zd$(1V%sNMie1FCIIa~28>40K&$@qlBe>S)QeS*XYUb+M>AG|+8u}CHFq$@@rPL`SS zJFk_9By!V6tDsO*Au4HxP|?f9Gf-GLaGm$DhEG%wJ(~N}GR-49V>IAZn2j=bLZpn@ z^-CRJNUFlLL_0UdC+;a-3uL@TEAqUl?+Y0g%SZO^_!aChp)#pr^Jm+WT%rf3bf%Zw zU6y;NuRf9YMPg(lWJ=i4S|)Aikc;k56tkewcwRNd%>a%^BX)nP*+I$rBpDi6_hR69 zY`)eNERX40a0Ma<%TSsM>KW+>Q$>g+OLRAZ-kJybcg25;9y6~DnTO34cgE?pBk0k* z4CHR}#Fj6Ysx^tQ#4q!v@?bV#4s9GV;`u}E zLNGhQcy&A8^}72g`C$tAc#Z$8*ZS;z{pjQZ`_^j}@Kk?qRY|e2e>wEAp_TCG#2U-Z zQ7HDz`W_L4rkh}@L>ZWwd9W&Kt|uJjDd(0h2!DNd4}4@mIH$?`(9!XaQ_#YYT&{MG z*fJfzfcSZB)J~KZFFYn?k|O3Nr(FdC+kru|&dQUnzoR|OKS|w8w4x=z*K3~RU(x!m zG8DDwJ9qRA9EbdJq&Z51D&pIMD@*~7kB^~o=v+m0guoUl zySyVU0o(QaNp@<|PG|-XJX*a)1I;e~_XH-?#?ecS7&O>W&M~RZzf6#CcZHVs4}e`v8R!kZxNkwtJ!mAks21IY-utLM~E{ zFwx^ZPX5tR8D^ zg#eOOOFte)0>>)4a3PLeZqZft^3nRmH}asS0KqiywA(H09zHY%H81oSpu_+w5$jeF zJz|t4Pb4qI#%N-I0nMvP?>JLa;poHq?h+--PwFpJdo6k zxhn`j*TJ^9Laz)l4XlkdO zs*M*;e(Me9>p1!mi1AfVuXAx8Y{LO#nOtNj%zwmy>>vm*kac6?U0L4o9=)CwUTOHn zCH)fxywMW0kDQtlo@Kg4R*-yt1ywr!-@X7oBf9@o}T4o-_*di%j{$@m$o4ScNKE7-UV85H=S>&t z#d3wqPtNkoE}xO3)%PU%w1odL=UjG;EEiU=0bEWKg#_sfu6j^GkUh-<-7S>QBX(UK z+`?h1oH29l3$AUSUrJd{ePyn>{JZ(8>sV%%!lS^q=h)cu>|d_iTu*b|Bz3o)p;@j< z6Va6<4zLAm7SV@e0f3h?9K9NP3>q=NT$eFb-To*?`iwFkn$^3_cL0@e9jn(F$~;|@ z8Yoxf)PYbk^E@^9^!u_q+_d7xP>7l!Emcvxmq48gr<+EXGcCTXV^ZFmzskm~Pi7zb z(Lov8K=?>0Q|Z5TcR5=JON?fk_ompFxA9WBxvaqrfQG5vOD6I#>GYTad#hRmqw|Dq z3iq$V)?@pjb#~LE!DJV?+Jkf`68U4FGdSXJd8k_G_9I8nk4rnc42+wM^3I{& zrvx^VM)4_txSR)K8TkDd*f)76FCSAIQcyTTPuL(kd%^L2ZDZ}G&ZGR!{1(gfQQmtF z&kC6R3Exyp>Yr0s`OWbfWk$X=bKo)l&fNDPIrZlK0J?zuP?0xLJ zXt4**Hv%&K>h~X#Q&2->mdnXbDZ4%O^?+=ITO<;3n=nO9a6>@F1hh;QhU_P@5}?j; z_c#nlcS>3;Jz8k!$?~S0Yk^yLcCdBEy2hbGBTag3PvPiGXXmb#lO;ZQ>B)Yie!e`# zZ6qxO2{*aOuZ$d}H9+USW^Eaag!~@LPS`0@YydY(yRdBZ`>nbDV3xu;nb7i^#ogCO zrM|HP@GmYXx3?b488HxCFzVY1fKYTdgN7VCb^Zv;*YE3~zXKY}z5@)G^^0TL*_(~P z4H)?8p!vpe@(~Sy7}r)+nZm3Wh%SIh&B&FR*549la;TmH_!4sT6e0jC1Uy;>0KvVz zjyuh4t~v}-;lrPZb$$amScWae^2`TYq`CH++t?|AVb~&A1tO228dT4J!(1%0H6g6o zsO-@=cseo?_s-8v+3~hea^j#W9g}i<`fUc9rCen47q4h!uKg1jN`cQ#Dm0^jx`ehP zuRq?TXSfa|0dUS>nB2WAj5|G~q@Ke(KdCRz=kSX77`(NRkPt9D+83gW0uHecOtPKR zLGj}IS020r|2aH*bm39+%Ub#xG*|A2_ReK#OP;P4)Jy8`bW*g7Ii0}#Mt^+ebl%;wy| zw&J+@{rmShIYHAFP^m6Gy_l&@+MFU{5_f#K)rQ zCBgZTAh-?Y-sHrsu*4 z-|JjHCaQG8Oz>YbPxx~D-Jn&@=kV3Lo6Jv`A5|Q7jnK3Dup)9#TIa-)`rp1SKCYRm z@tMw;^k^cVJMK)k9Nd1~dNP|H=)HT)ysBo;?$`RMNL5=~8&&udiU<(wh0e%`!vue( z3<{VT=09-D0PEOytT3EsY-(sQ26J6ze^*wOG9bl(MWG;RcGz4f6Em}ka0+-K01i@PLOEpIK#?_bbK1NpRp(Vh!<0jfSy0JNr57+gW@b$Nm~sA7a4C0pcL}pDQMb=Z zfTISK6cnDE2%^?5eG^>L8&DfaLM_4S^5)H(!JdDhfq9XciPIAEU_t!>a~rU~zzFj2 zrPcgGcY&MhqeNXnHu8Y~OA9asV(VFH!;*rDglOo-f%A3vh1!6czELkmV$k;ZJLvC+Gvo+`tbqjxs}Ka-1zP;+O1^twY8#CN`Gs+#m&jKpfKK?6 zh-**DJ`?>4tl|uG8n13WTm9#xf9^@VlSjy@RfqL&xNN8UtIDUa_Q)!l%V;|cOx{F~ zO_1LTEaAKyR5L%tX?^R3meB6qbHV-fOaK1-W$8Jdt1N8#_F7PAe~h`m8VLL}?6Nr{ zb`Zb_EDB=UXj#+$5}%@gmj>I7ABifS*x^KDF!EOM7~FjCZNNCef6m+~0fszTS=5iw zkc$CYqs??B&vvE|V1>Ic=>)@)1aXO99mFo=z>+hlh5^m{$@Q^+(9+5pzptxf@r+}h z;3jYtk%^?In;skMPNv=V0eO$Q2@)d^K9!e~gQsu_{00A>Yz2ZU*V{-Ijx|tY0Kchx zV`#zEB6r?Xet}TN>WiJ!?}LBuAAsqEey*-zseXdymx&3MwZbzeA&J7`9hVs9(*S(5 zy54grdPf*<6TY}pNmeJ9W&H2U>D<=Vk6(Js#b>zdeiV4>o3pfec5?cgaV3Wao^p#e zl@*rl?zJgT58L4)W&N%3TFI4a4Z~>Fkk0C#a4y`JA)29A@lg?}vrs{P1W^@uH2KP) z)Ct8>5d=g&^XIg$UhS!K*@G+jp`S#_+rk&k6y0te4Wgdoh>3FV&S4Dd07rQ_7j*d1^_G6JQb!{ z?;yww3`#w|Q(}58#Q*&Ls{{s~K-mNtj*E*+#0V@0@DE*gm4(NR9%2Przi=BN)=74sGd^W)2^Ju7X&gz!{vXHmU{aZ%gt_iukrTfB^or7pi_ zu|b&>N_LK%_nJJB1#M76U`^6?aBvX4utY z&e5K%JTMP9KJB57;Nj!b;H9%1h+t02qJ051*EiIsE1ZTp8H)byrnS8_gVSE8PXr7Z zj!6QXr<*q5l3A)OrR~H8o-V<^9Y2hAO0RYD_9CPt=6nI`X|+|B>ka$kFV#O_;W6x+ z5)fw}@*iKh_LqXv=KL`nW+!eJt`RfwI;}JTV2ZCRSjPuoXFw} zl88Et6Tg{b|I%g>dNOY`*FS!7`zrB5TSZx5oeBpHsa#6u!hX&sVeK^4j|&836qrQr zq<&QVE}WRnyS2&;4JE4Iy&l zE_cB?Oi-IXQ@a*Y**1^*Ku|`D!C68_1!X^PSx5ro3s469nhhG+jzQ#|Z+^1SrXva` z9DHnmdBnm0M&dcRAaq24bp{wltd`pjvdy)wfO{u=1!T8z&01CLhr0p8wRMVsT?saJ zxPUFys({t>0-bFY;$_EX;wfa$EyP|+``#t~t;1SU3HJMKRs#MecJBEp1P1ch;IIOX zCr(Z?XJKq&!u4bg?OFy{A`H)8XaVN}Xe=;VpL0`mrojk`h{P29_~F``<*Zn}dH!BH zb%eUBnkYY3W%%@y$)QN#E)DRTBV=$7mOyM)nu+9iZM&zTKnSwI|EOlx*+7uN@K0P! z3>76Mb`4qYx_~h^=aSL1va>_x*ypE2P*=l871?9qF`&SW`v}#UnP%Oj7EyO?$d<$daCt#Z;2w0*|U|=9r zWdJmh;UG8MY5oYnHn=&iPGD$PZz>wvo-_UnNCQ%kD+Y8msKme^Unm)Tm4HSKmS6K< z5mD-iiHibtSaD+ShqJe5!p?n=Sc=LQKw=uCXD=uR5|=O>1-5e4D*GHRM)Epx_g1*p8=TuDsxuC5m5 zv)!~B+jrvY=E%>srATzv2r4jx09t5F)SMPvWkv5bRDlFOx;ydZ%PAn2tN>9F(kSCA zH=+0t6T3>|Ctd!rp|teo>sv}7fiLG>1_e0u2jpRSz}ub+AIhdli;D#Hug0k^hA4;> z+Zd}hUT`s%wcT@MblhX(_!4X}gUU@&9DHY|*=J^tb2j&?r?>6?al9`qd$3yeI7{|P zUSS#OSGRqtntAH6(eci~M#eWyg=MR_`m}FUV8Gz{*4atGy>#t+CDhO3?;qMx1T_|- z(6T^ugWB*B`)?x%?u7wTqHHu$RIyB}Vu2(C43*~NoaT<{bFo~IPTTwa$ED|6*XdhD znk`sxv;VE*=CnVC1N&X>B$LGa?f2ydo!^Jb1ms;24R+Tg8WzESUM5KTn(i10$7sSe zn-*2ZfPs3|%=QnD^MzGYT-@kR2#iva2UbSA-qQ!7gn7O;R^Bx*@Cdae8)a%X+wy8e zr+~!>ai0>QOmLb*w0e*og;u2lMZ|z*0k=_Vny0SzbFN~?+bjGwg}uKn!!XAz z&tC>sgev7aS;(w&_WF4wKP9;y_x0!DpG`{!x}S&%TCnhYBdoVcmb^$MIC$^+5iN!0 z|4i6Yk_2~SpMs{c{H^FLF+r=_yug9DQ6<-UPn20hFDj#Y(7D!we)^_+|4$+o<$hYa zRiaywbf~}vFVow`+a#(E=yZC+>3mUO9om$-Oy zG=M%J-#U7QQW8f&;O-baP?FfO`VaYsNEyheByN=;{>Odv)jj#?tZnUd%hGI)H$z_Y zbfC!%J#bsuxhIXa*9N0h%$uFE7?^|I7`-RWthqc78T8vGWS>AJGX zNmNKg5^?PE8utHj0d9`M3oq@&=V$`8!Bo9i1duh!7HwL@wP1B_OO|j;7DJIslcPWt z!Ny1Gn-E3N)rt;3v z04q6^FTzZ>8CG0npIzbk=wb?eJh)b$=yS-rwiQDk*Z{2>RO7TdxA>DFlWlBkKDQib z;QvMCKp|+PtE&t4Dmz*TB2o1odIXl2n{z6p;0H)``d8dd7k9k>%lGN5^M{?!6##;L zpbd`!-@NsZ?geW8mZ_=Yf1pA$G~#wrmt=7iAnl@KBU8ggWw9=O}hBRq|O|e{aO}qYSOH%;9lUimzT6( z07$pNzPKw;5|F*Lvk%8c=?@mlYR^{pN>6I}-7dqjvx=&(f$VY0TM!KkUP)3*7Z}J} zja5ta|A+O68kSp(y#;K8{3Y1hgL6BC?97S6rVrk_IJTvfv2Y5o>`O!la!2g8iBM8q zj?%u>@bl@PpzPo#)Lu-8c9(_Q`=H@2!ky`uZfd9!> zj4?525bL~S^#553a6sz{T$UV4XjYLJ*tx^qAr}2s(Xkh<0lN^8mgA@A2rM7MfEWBQ>P?|SGTT`ucx|z#V|(QF;^=+- z?AhpG!wI$on#6}vZ2OS9RA$yp&WBTxb`q;_X<@oujCYp359#0_R|4hglrZiA`A|p;9R$gKn=( z8d5UQUDdbJ2EVIla}%H^Em=N1tcH0E^q(+soK`XB{X3mNr<-^{HwSMSylTD$9Bsfw z!ui%hqOWRf3ux9rRkk_#eiaz`pjY|x)*lpRxnSAvJXsk=WQjyn@O-~a3!`3Dnt+|Z zC@mbkpoW0s#+UKRdtDNgPWu`>XYATeMp3H~j?xC_+)37X&?4mk;7y8-~V>cUE5VPhkI zB?5FM(ASg*Aju)J8T|JbFS??;bO4J$w9ZS1(|j@%O165B)#Uws>xRWxkZO*)Fq1%R z9yc;C!jdI@kVKz6#fEE4@o+yWF8Z^EaOd8)Ke2{v&GP+qEJ9<`RzF!h)11?eW6xj! z9QtZIk-`j8+)&uIe-h3j-mn2+o<&Np{;Qj5d<-c2aGFGC?c2ga0c!0eT>wge_F|Th z4#rCmKU7z@4}~*~v8TmxpxdsIcv20>A^d4<2NLQOEZ{ zI?K0sa;>x&;elHW$#!zB9MloB&Q}pTg5N&hy>Us$K~nOf?!2`*XK~@L(svQ38|bp% zLxiV3jg<4;z)z19~*Bf`tO2R)5FjSWr(>Ji%Q!CcJs7XTskv-2u5j-#X{n64vz-aW01E-uyxYe|kWtZR{eR7I zz`h$7OA`|oV!R#ddcp$_GXtEZgS*C4P^N&PW3CG;q<;Oc4rNnDVC^9t?Q!F5Q~@j$ zuqm)K(if&Z`?He$o$>|M+8{YoCt6Q{Dz5&-ODS85vferLlQBY;=mQfnjXL<;DygN z>1fMh8*X)FOsv>;xH+33+A{5;0Dx(3BrPFt!I9m1OP|1U3R4BsC>AaxY>x0OX8G>O zsZO0xebBmJ(AxAjb-LD>2P^pWf1H^8Yh+k^>JcW`x7o|f4G;GFA7uHGq|PPts&;N;k&s?LD}y6VJ^1en z@pmo=D{>Auo}D=N{H4YsZ;nE~^5-8gM019B=3oizp}226I!huSb@w1)aG3Rv{&8i=C!_q4flE# z!xFPvHVTT=(MPUx?e78~p|StX;er7L14@{D_ zxq&oz;ld66B&gnFpFM-TH}#9g$L;~XikvWv-0{YJ3@g%C(UBgT!gUSyFU^|VcSeHoAC?;>Mh@xA#fevTY{qoTbyt-`4T z5amMVOMXSAPOg>bPgJ=ug_$8!Mfq)7V2BZO#t95vwPqoC_6E2#$KE}0xu2ZhbItf} zYY%f^by1cQ!C(RoPaL$b6OJ`$oRKPNzu`CF0x2XQPzWyJW#c^$j&=|@_VOQZML+z& zy_w0!e;*5YX~V$bqM{=EFp4n6X3+NHKqW9VcF)FlEzYWpoB|I|vj05(gFq)}GLTv> zcs1hmUTK_Ru%*MxgAo}tnPBAqa`1a(DRVCUHBXsN@omY98g~&EM_@63X|sLrqsXcH za-R8YSWNu~H#WX!$!E3o8@?rgcK4gV-r-Jw>~>rZfP@lxusBuln`juW$g*Fcb6NoW zvidu$Bl^o|;2#@JYBS4y6>JY5ofAGj(~84@ zz!OP$IzGGBn_9&AJsh}Ayu0ZVH{)Kj($8#w#O(CNCMPB3kN@g4ro3frEH5suYhvOH z2js*}$4am|hv_H8&O+LC_If>17v2kW1Hb;Fh#`dnM0Qv5Cu?2IpZ~TxIs7Z1shvL} zB`vM}#qq@I{Uo6D<%&-^EPG5~IKF+FQYskT?uQ`i;B6D9RDU35R87<>joaFQZiPTY z65lTtil72}7;xz`L-dxkbU5Z83P&Uqx)p)bB!d z0r_ci-rK0!cDQLN;0ud{3J6r6R}naYKU5FjC-Y0eFbng0+v(gP!I{g$ks1J0ol>c)kR^!K?pKgpIdO0!?YX%_+)*-AQ@;b zh+MErTN^0#2U7&l@Ps2FZ`aTR@+5!+<)Lg**U(ifwl4-;KnrI$F{kCE*r>9|q`X0d z0Z5ly8QJ)A>2xP+YoLmC-7&##&<9tS64T7dyqGGr#uo~Xho^zMcKYV%ADzI1+mSKhasr3Aum^IOHJBHUJ%q(q4WAQkLJze)1ZJ4HEA!wkE0_`zG*=mmr0@sB zhzHEJl)Sg+;WgcsiJtj@rgLu6tlphCO9j5&u$o;tz|X?M0&W7B4sSPY zZD*LPGru9MZD|j9x0haMeWEy};&kx&`Q<$J)Asf@nEb+SPZ$P(=+e;FXh`>S0dYUh z?-2fFzK#eY=b66!Uf9{Xz$OmsXu@>CCV2a8X>l=OJ^_)*GX>ICe_9OkO)wIYC?VwO zLw0*&FnBt7Ub{wiOa`89lq~eIgSHehe&lnJ^S|n3g3EW^e|sywEVxJW+duJD1`w*+ ze{5TUaNSxG==bT_M^N9Y^@X*X+l^O~UTSp2(g zSd_p=cP(3bfERYAXq8G=Ytg>xg}C}{_El#A$1L&Q8^3* zoM&ENe;wl9>aRxHr4FnTd;9ZW1q0N8?yj{66q`X?55s#_S*d_KRR}w5NeM5gp<v3XCzp9ohL@wQ-CC4_L1Ae0^6{z9gbj>i1~jLb+GL?~~}%1DB8H zJGG1yr*p&7s~^Q2;T(h6cZ?PJzisKB+v3sYt>gq`Bd;zdnP7Fhm_|exPW*qp!b6dJN($ z#WaAP12E78L)=u*FKcKf%w}_$E{Dbu8i&{rpsT>E*N`QI#z0T60)E@--!w~q!c(dK(l)KEvsCfjC1fT?%`DVggFigkNGQ#-j@*F&6fphq=x-AM2yQpNUsg_a zlf38hSVMNO4Sb-tCf`5kh~PtZEP|vMMEexY{$(wXzfKvQJFUcTv1PPTs#?g^F*|*= zrEdQ1V^f;ePUE~tFlWHBr5`#lU~d;{+)x?oTQiveU%{3||M$uxu#bUL&=I?Bj9Gj% z10vnFr!;UHfO|z#Q&X`)S**e|&{x3OrR`y?Tc=RZhYuAb_O-kI7VxlL_EAAeR)30--P0Wk18>G4MbT$?9u&G;HjF{Wh*uE% z2m2^IUZ!$y#(zOLPXI_C4QWD6aLQO(S_(P4zew972;)F465yVI|Mj_H z;mLO&vJKsgB8W0`t}P@Rg%RsiSq;s%gV+n2ISeI--pMZJAKEjH+U34(2-E<4#E^Zy zRNohIfo7l2p_kW#F`-6QA zrISCSfG9jGj#JuW*?Tf??&$05tEZQcszVe|As%bK@XNjaomumtOXwN@Ex#V=c`1dn z3%jjMmn(Hq$oUR<6kZn;XnQvin|5OowyV$lZVJ=#U^l5|r~h#su~SEA@{(IsAufEw z@F#^aN`f+tf`S4HW7d^`Pf;ZMM!i;gn8zv~a}?ZbB_RrF3hujBj$5+$3S>!ULa1#PASCOPqjPgK z-yK@p#A1(3x5Dn!>$###H4QDBkg9}WC8u6CMz3KlrR6)dD`<54&J8(bL&S)Kqn_>R70|7WM zMG)X)KtvcBfz^FOxTe0e0)12Dpki?hKTqi2(r^)M`aIgp%3t7fMCk6!TRVqUC_+pw zxBZ9~=b1u^ijg}l&5k>$mxIo~)l|On`(jdB^P$poH7$EHWPIxH;LpJp>!5qNvk~0S zE1VwoeBvfb!>GTWea_wT>h3D}ecR*i3(Ht;<(bj+U3M>snatG~HNqa-?mpH)Z;1pX0|;iv^rz1l!U1U^_`9(%f_++~ zMV_lwpx)#%VRb2#{@U@*Cqwbv)cn6?8Ur^E7Lp=KWZmi{PRY->dII~+hbJBhkRQ2= zO8nct6Wb+FJA>p={{%)l4v^0ZRwONt4FUFbjPF_gqFeMvB^NL!NAHc8Zm}Q#|1p+R zN3jJX0wlhXY9~oOz-;UZ`Stav2+E&5J!xRVdPI$=oWq$vRm2T00Z;`XSfg(MH?|GF zj}>>C^+gf?A$1L8eX!(0sdo3*i7I!8mQNMyQL!LfGm|(sL^c04{a6ZdGJRX>>TgZM z%WbclVZT_5<;?5nHVAB5@k@!D*>EC*aF_DaW1E^L)vsF{RsT1k%jUMy($NL)Zh=K_ z-Lxb>Ka4%CKXBC=f4K2(y&P$%B5<{AWRrP`5t4FvQCYlG?3*J_ zViH%`wO)M963WenK6J7SuMc1P%dyAvw)&`P%r5&IislS(M16yTcTv zTB-{XpvQ88wFhT4s(yej{HSx9+Yr0Up2`ONJEdU z@5d~RUY%H5I$ZXi#K!mqBi|^olHWgR=xiV&sYff+XscYh1bpV2-E3nYA7SGaCVp>V z9a8XEJ+cIJV(v|lV2`ARJr?L<_D{Ha6q0Ar4aJbMY1Q_jJtKDInfDw+b2!0W`nvS4 zSaZ#_oN<&!j+Kl2ev6%n11%ou0bnupA%N{DNrz&G%>i|pB+Nh>*8;fi=bk5;PJfoD z$L>qW$co6l(vtyrH{5li&yfiPPfDm_NFoby_a*`X{~;JI_zlKA7p|W`sPpt^MHZ%# zB?$mHnUKcFr|RV;iU-h)Y!_^gQL~dR|GUtzutgsilt?JJbMfC)qbDuaU0Lif^CxQaS@Dn4#Q41F%$n2-Y z#e_u{bG&?}XPk73!G92;Tde`vo5E1@C$DpuDbU6&_x>9T0E!$QoesNUHmWQQq$c3r zej3yLjM=&bFCAp;e0lUm_Zl%V$F`w48CGV0rho-eX1@g$c6~v6u8NWyn4^PP1@KNG zlQ)=(2@74uY(81!*8<=-R;_2ru_!7iQ~Sz36OMXLd_n@e`m$11{O@`#4`!n8nHZIq zLf$E?y_>_1aau_7u>Ut%$Au7sze5-?!`I?sDe(H(R1QT37C*dkn~kkw%Pn@Y_eJ?L z@YwOAfR#l0)L6|!K2daID$jPx1H6Z~u+>_+%AMtdpb>+IF?d@30 z24$J!KkF1A-LJjBW7-McgV6A>Yw8~d4r7y02bYMzpam5VrWk8$5amCuz-59^8er-C zKbwx=f%&$uwA9pt)4ij1Tuikg;4@16&m^()5fzN#@Lxh_8eM6@+Oh29dHp0AGe3GP zW6zuimQw%{}8YuE`u8IZc{XB$g-kY;07ygX^Y?#;UT+f zP^LcohRaKAHClP(cJxb-n<8=^o8;qJ&0?%RZOcW=;vVK>BZEgI;m|=rU_JSNKn~1U z5DW?%6<|~j;2Vg^!GF0oW7bd zfHCkQl@*!en{Rw?G&45`Sn6{{grj~216Xm zG3@eyMZjlm@#2LFD=93wTVSZK0RP2&9o%dH9D{*T%_6Imuj7>F=^IU|Nd-?9DpVVz zBcO}vt0Sr1osf_aU~m!Vvc+~yLazo<(_|Yru}r|R3&xi8!VhHipY!uQ0gl=oh0oi< zIIo(mbLah?oRl=*ih2_8Yvt}^s|`VfcBdS%1i8v0TuNXC%=LHRVKQ?AExq9Gm#n4C z(aZdup%3!W6$p5{edw99ZUG7^d(RstW_Z4lt%vY6vFz&s4H>|hJEj`F7@V0QN)u~+ z*!iB&I2B_CIq=~nBKxHL|F{5^-=J%^3&Rd^1iR@bd{e-@xG1_@(Dz0tz@6T!!Ovh2 zvtR-IAz?$EO{lM}jdA5aD9ppj3Ew7SQXl}yjC6@^g3^o1Pn)dP05nuscyxH!v$D7l z^4ybP0E5!kba`0p$cvi|qth9@t8>20G=emJ`t}Wvw6z7kww8>8vd(sp;f{Q>W$E9& zr2O-Beop*(Ufh3NCH;N5^{8F_Lsztv=O z|DB%G?B+@A(3NzAZ3O~=?(=W=VX%Vh2iyn1lwj$d&KT*0T0sL80@@GQd;rOtGnk5q z6C!r^wko|2*QGJMI{d-V6U=O1y+SKvO+w;3d_q|^jWVH>7dFC=z6^uCM8;TX$j1+Th2*Z6wW#<6mYYWMifiA0_fi_b#)Gog|c*JaI@jx;qU_l$3ucD zR3FB{mU=Y#lM>)(D-w&>pyG2c)8Spz)MSiTVsx;9A<4)n>*L2@%g7WhU0qa?j_{1f zhsw(9tpc?tJM8aay8%v(=Wu6G{&m$;FUmW-J~)K6y+WooIrvra)2c|?o{}>qrlzV4 zxdd7hd>Ix65MXXJ25`1pgtOQpcd@{XEN){;-1}dlJ(J&fK;x8HHvngj0yXRK?9lw1 zRY>^?L+`0%0TI0@xv!`?4^shtuqMQC?0yf%WBNMLI`437SBqM2zz*YL4o7ON-qV>A ze-Q*PfVvc`99>AY-AE9@aaz2Yy(8+2S86o<%~kY0QELe?y=Vm(lI_OMVH+Bf@;z?n z2>!9<)^_SN>l0t<=FCsH=u~T&YS^&d81HZ6vN^GA%~{IA%?F5&Nht*S7)|Ax=^4Wd z({5oWEhB>lQZrKgc%=hy><`Jgg_MQH9X!d~b5jSAqxI>qqp}n{t@K1?*k$^iT7~0d zG}-=V5FJA=vVg_}>j!4Q9Re|AzshDBCdL&u)NhtcMl#>7xfQn;=U$^M_yHHg_B#@M`$il{9Auup6Kq0K>pe|gyT;} zoW%f0U2v;w!opJkpM~F?PTYgfPLUPa*X!pVh{1t=4sw3ue5H>;laZobBM8pXYD z48JAD!flc+@ww4g4JIARpz^y6X$)$Twyuq;!|+i@g5{1Sk0W zJzQJ_6X`Xj0JIe8eJvf1fXTIzkQMiTI9jQ`@48zZSVL&9n#b}4y(#APv%RAa2%?s} z<0;^8I5;?9ETobP+D7CznJP7Qq^vy8+aPXdGqaBnKJ!yCwi>i1wbR@4^YaS}3*FOS z3GI_tvGBTCRYd?@WF_r5_!~Mr9jVLJI+vLB<# za>^PO8vgcDDsOT>q0~McdkV^o$W{}l3M@S_5?NM*NWPF)$N6yNTwZN9cG$+`u})s+M}(y;bhFrIJuF`!z{p;;UbR#)QV? zq1BgsTT_$9F)CYAQeNfHci3+|)Rf2MeW)qT6NHW-&}Ptoa#7m#sjK!*Gob)szcqx} z_j~?y&Tvk@x8nP6rhg_|NGWH~Wc?sNLF-wMoyH9bJV=2TY)$ad=1ly|Dn5a}2FJ$Q znzE9TUab>b=S!gGYx>_ZR}K223hvAUPhX z<$y%p7R{ZIYLx=afN7%He^bZbOwdw2xu6ggrJUk*GX=GP%)TwTu;Ia*GrTg>22v+aFT*(`2&=uV4K?!j%P8o`V%r zmBHs)qt&~7kQ`Y!I_ng?at;CKBuXDw&To*vQ>B2jbh#8 z?$98zN4j_JU@I@NxCo<2*C&xRgxkUpCMV;Dqd?urA?(6R)J4IjW_VAah#92^~m z{q75Uq=A;TEBSUXr@Uw%3ifAiS$6|q>h5WzRCat+P5@sC$V}p4`to2o{7RLfh3Cn1xGE;LlR#ry+s%#bsWRM zd!qjme97${H{j2d7ENO%LZe}WG4}lvYdoge3DDj?7Z-z4hT(*SL2Jz$HMC(#% z(_Ve8$bnnjY-t@iHZ}&Q45H^Y=p<2#NN5TP4+|NMmSyl*h^5X;ju+vwyY0+8gMnB} z3+6J4>Yo6_rV3hVfmt|MQ`a$E9e0hTq^Coj`~%eqL{khi^_~LpH`{_1Yfz!ZEjSE> z^2SD=|0dqlR&bI>$0x{Dd6(PMnylTWj)wTEeeDMmoS^1oQEv73Arcr|NU~be;FyDC zuddNP$ODS{@!MeF^m9DkODxA8_;p|w@;_N`a#~iI@>(6pX1G{Y1VtsxQ?SN>Mf1k$ zDk1==&d}HaF(SrHmnQlT4wP>I31EbV!&8z=VLcN3Lm+oAddUIvHwD%aMb;ajc&#EM zPC3>8*tJT~`*@wlai5T`utT!tnTc9TB8vgT6ffDUq$7|H`fgy;R2k5VpEt_RaSzomH_c@ zds-jTix*K*20`~hUzL%W8M|l?WP)(YAbw20gAWW15S9j3z@k>LjkD;_I2?XnD1PAO z3!kZ`77g+7==F$!@o}(|4dSmxT<-*~mjMq5f#{77LD&`%rdMG`Qe`0G-SV;&JwWHy zVX=KDXUdDjR5MO;sTw@Hj1|ue;PI)iIE5dH^?#C}OcyV1fylueYWr5;6 z0{q<2yuji5Sx)~#-O&BckXdeso>Cb%UfZy;n( zOETe%WfaJc3PvUNK`3Ul{$jQqbWfte{?G$38SRb(v2VM;|QmMDw6$tucWmqL8 zCBecTsNk?NvP)A)@qs=bH;^$&d1bl3ExBOv5u(}AKyZ1%j4E_dgyx}4HYS#!CkVwJ zI>$#9RKYk385vNLZN9CpuftNJ8rUz@kTr{{8Sp1TF^kkw(yEju;b<%_e-}Ve2$U$| z!8vkycxv0YW;|EuHHgDGfK=GNoIU(<@Fn;Pxr)*YLHcq2*RMR}WM2ylr>gBUKNTP# zscN2Mn*%vOevj8bW9et4%whM-@UNWep~mP2QxCE=(J}pXkDd7O!)R^2O`ku{e0*pm zNP}9Ggx&Ch9E_=e$G>oo!hSEgb9E2=q{QuU*KId6>`3pJ--#tglz1dPRZVS#UhCo?rdXbnj$X3D7xsHf?_bz|X5{Ok{E{Qg2w(Fy@Nd>$TU$+Z z?9qJBvZN4f9(69y8_bkUh5!MK8oM?)Hu(BBTxGO}G8vMqWXl5y7Ig=SV?Y~GbK}+$aQFDuC^g542Ln)mBnvMp@++& zAz~I~QwcZ-?eRbj2}9b^A~lJrAPuT&7|}mXcX>egq6c^6A?IQ+ti2H-SO_?eW@;Qw zmx$umo(wGtI*tWiAO67di#bO(#KYHbaVRxd1bJsZx#as08`JjYhV(01ai7}z4w%B8 zXetTGB1ZjORVTI;J8h<3E!2|Fq1-eM;yRb5;uo@DHq3!l!<5KVB zQ?<*!UGe!$*ZqMufqVx(iE7;S?I(z1KkNh!VLul#YP*YRa=a<}`}2y%;YmNTrp05~ zE|pHh+2V`wG&?y`j+o_V!*j#b|Q7(0F{b45ASrw}J>=05>=q6QFdxP=Fpql|ohv*x0W~vvj zuKH8c*b;gW(#YVU=NX;ivn=OOZre1S{S?Co1M3Is-d&rTVsNM~Lbuo+?-~`6m#eWN z`lTq&ra>aG%Ha2hkeN-sh?1PuVdy@z%@n5Pu|=1l~`bL$nq0w?atg;cEgz z0@*F)il;Y{W+`EAp{w78(73o$q));t7wa zj^hQt)ylT}mb|yb4K@DGuy8qN{B@UxdOcrg?q{wu!Jv`+?I%-tE5iq}c+7BWz{zz}mCWeK2)21MqOwC1uc?X=c-3<=Fv6@N-o7^P@9(sT7+NbV zDhTwMuXmb$x2}sjOhdehR7YFBt2czEgc-g(Aa7J61Y)x?zOmMq`YGu(vXJaKYYE(W!6&iug^1P&? zgV%2_5}F;Tx4@pd=FPq^X$>hbz^$x01Q?xa@2hI(n45%9L||!{Lbj zCyh*45j0Ce*p?C1A7HG&CLjg*4+5mTU<2ZJxu5Ah(^zT_`77}M1on(cF+Ncvlw-Pc zh)Age!+oeuV=ViS0mh?WwHfPwZtClM0o{9s_x8PnZk=b(E`HoD4yV_G%QXcn1ysg} zyE*5pUAO{-9hivo^~X(Kun2(GUP=;jV-QGr&-nCou1m2umG|ZKSZ+cp(tts})|X!b zze<;<)|`>tMDdIntsoj}f_nyCZdOzkpvT&u)ak`NfTzzO>H___6+nQk(}fU5T2QbK zb`B8h4bhvek_bNYABlkbLW2OiWcY!9;1v_|UTBY)KwO!)n1>chSrdV>5tf?zyx&j4=*BOF{*(T7V(!TmJY_v`bfm<#Gq)Cx@Z z3l`$sAC_O+-hXmYEa#-8gkYh4|&rd;tia74&9oKc8y%;;=G`s#w zSb2JQ-TsbQB6S%_5ZJsg-uVdjn;3|JtH28g0fIoJasbvG$vN8iK%4oZ`~(un+UkA- z0~UDi&~LSk35Tbs<^k;=jvwRA4F{0}uM4G-SKehtE$HM(LuGaKOVsKG6@mnP2~?YF zIn(F}kr-I=8bVD&V(_sH+RZoJ5Dk%~#>0kiGE#Qdb`BW6o2C2KXu>Hz+*k61{Cc=+ zP23<^4PLA{$p!@&Fx{Zp&6I3Z*qtbXK-__BxmQs^@S8v>C@wG8<}%=OIe;fL1Irrf z#_F&oNmir;X)qQ7mg`XBuGv42kPl4$ILLbM9yMDB&}a~hdVd2KYha0YF;DD2R%!h> zcyyLJzKZcn0l+z&%^?^=FVxyPmh=3yp{WmleVzG(G$eOjUG7|6?uJRxbu-!(yBN;y zo;UrwbECEh_h{NZv)hbROgB`NjgBrz0lqzkJ3%5$Ohvkp6XqUh6!DB)$Rw4!o4#!( zOKmkh|Ngnvh+#Xk_NECF7nz`5ys9x8`oOriWJA{2{x8XY9D^|~Y!cIqQ{LSZxKn*J zt}^Q6nXz*5{j@act$>TcHy10Yqg2~-IF7QoHYAwY`T5=453jGOdcBc!s186{W0e*M z6RQ9F(cdrSv74197OEQHLtgCV`oIMZ(Jy&cD4Ts;e05q3Ya)e2BROO+Scfr4c!5>V zxA!)4qa{sEQnE`S908b5RdJn@`XWt1O z@eKjuO5gY#%|H|w)KiwA#m>ggredy@Y}%iBj~D^|xnu8yS@?J@!g0lCT5p5WRXb0W zDRv&NQrRV$K2_-2`Z{TtE(LMcrf=T88wa-q1gvfl(#cj?l5?Jh`8pKjbgHw-!0`}BH41ga*(5gKlRp|r#BX9jKdbg$Of&jc1PEJlUUK`xMcrttf@E2jP z8$UrZ4*+jTgtd;l$kA9#zv z69dH^M{p+nKwV-nw@rw7gK5qSDgczkpnH5unNas2#Y74M!2s}}#54w_4NylQD#5`$ zqYvhCRI)PsS03`DK@s%dF`*tq1QvWdn8HKI^vYlK4SrCx2Mh|}8z6~pCOtN6k(ugc zg;TMhxR^sBao)VI0FJFMUue6NqEueKd^uA5Jk}p#rt$ankO1!=WnP_RqLP&1=fQ$i zU?Aj6czx^F9HUe$;NJD_-LHqp84)_LxNS+f9>VVjK1Z;`yIoMuCC1^d9>uHx(pGYg zMu}kW7#WtDiB=mSz(ls-cwzXb`hBROvGEdBjA1vx;JtaL!LSLIM1%=IX$4cTZ+;h| zdB0AFGA}EP+d_Bzug>6@(eOXj_VtxuKk)+^aHi*FoW3x}*Q& z0z3zGF`QW`x;;a>B(LR_7q6AlqidT+EMrxHYIJUW2*fiudjn9}Rp9i1oj3ArcId)c zbIqfB452bfAD!WuflYO`=J#c%=gx=zx2X>l7H_lO3SSJXkYkM^Hr+>G=%cLev|2E^ zTk7!XZQFLuwCki@dE5J(R3-@-++*~GSlL?%>MHMpxo!Jw#rsW9?o`tcviU$hr&fCJ znv42-N!R^^wCgJ1pMAFFe^GU&Tdi{oR=9pIs2)m^Kw^AE)ieONHD^{UBq|MQvLWn?xCq!ryws}zSvdxtz z8&iE5{!zABU+;;Z3(V}w>MHlyb?hd)pQTDAH#~qN{kLQzg3)AfjyfAJhf`gH3H`?1 z2i>?!$!a?2|B8!?{SO8eJH+0+3%{VTOKtul&lrl7(P8h~;!<#eR*I1V$(lA&?05KO zPE=l0RJ8t5LY8@uBQS1zcni|UUr(PelfO|;QN04f0q4R zJuiBD^|Z?5Z#hdC*y>J}YQ#*}xM8>hFeT$kdyjHp-l2pmEipY&t&aU9B~}6o-nedax~})dlCdm`338Z$?%K53ASV=|tZtrOXyz7@JrA1Ay%z7yW!1jd&1gb4q4brQ4T(B8b3cAWNINqmgc;dAfi7>ID-uJtVaXdt*Ycj;KhAV@kjiQ?wum771s@{(VSAxLIlska z(*v<8q3RnRI{~Hr-yPFIuF6TiBt&zMGR~MPW>ITMi$gUXfp|v$jFanp=cgkbHmVWR z&)3RH5kUb5fasv_kq1)>`_AR@A^$E>G>JqoT$y|bd5jS>>L?z&+;#Zpo`#M4#^y~; z&JT#!5ED*F0uqFlXTCH!tdClHIa|XK49_iHgwAMM;8|Kcqx**&6vMDiWX#}SXdo?v z1bJUxox1Xa@q@9vW4yhOZ7U4G7ytAefoS8Snk6g5!#qM?FrCY|dDQ;W1g+VmEUTxp zf7@~dc#?!|TzVAot5zbJc?JDVnxD#3{DMpf!bz_xp5A)} zY*9=ZcofQ4`Z_aUU4i5d(U547{{B6la?Ww*uh)17JAQT8w-tezp^14nP5zinJdfz^ zUGa*{22OHeK50pEmN0R$g2$g0<4%by`*FL7UBgOZrCLd3CH&L+LXFebMGliWs?ETp zG41E$%-M41*r9n$1lTzJ3&M0nZ}}ELW%xr`gs=(`u>I8zvHO7=sp|2zoDt{x_5S_O z+&RCP0_9_>qAk|hREVR8;*(pb57(H zCgaX9kDHz0KgPl5#+${6`=XrM_Og8Cr{8Ks7`Y0TU6~%&Sg8Ie$r;Q<&8;A|;_e_u_^l4ob`WLJp_X1OJIH@~ zEdS6aw%p3cxo`t@*tsVSlAot*2u<5GKDk`^yRAQc+{r*mxw67U{@?rSK=(<_?YdFL z(?X0tXb9k<`~9&O0#pA-oYw`MUzqfq2&~+TN$57StfrEiZDwlV{6N6fn(1$-hGj>8 z9&GH@p_YF<`%D~7DlDsoB2k6}(^%7Mp|Yvlvz<|L(*+ATYO=s3G)Di*k>O z8!{7WT7}N8UF_8FsXLnrAWFEt=zh5uL(pza!ZfE$^4{fvKfyT<9~!Ja`Pe$A>TAcG z*3Q{L{z;gO;Oq}Dv%e21K~Ur4zD9) zC(B%W60Nkq=S5ah9&JvUQkMkGmh>rgPTOTSVWClRW6EfD6@2PT4dq@b_$hPn6}gAG zRUy-j4${|p*o7+^A0me-68Ye97`IViksp~V#9{E~Q<(puZeOWKyeXmST91#Q3VoQb zV|u6OZs}_dHI>%n$ zz-L}jI=Kag?QKF$`*ZGpU;q7{m)0ljZaZnddIte3%RSZScXXx0!gbiABi$cicX|*= zePY2ye_d2d@Qf(_)cKI8&NN*_zCFv?Y@p*C>{lHII7MEbsCSM^M3Nfv*^p({HMR=T zyWw;>;)k?qyl?o9(;uHGg%$8GZENnae%7FDLRNRmyvo|EyYeLneI*OpS4hu03LR`A zqz`w+8P@02gg;khMUhffS(T5A#MJHRA0_#m&5on)+_L{o=y6;TMo)xzcpgt|Ztk2w+wY|eTn>5n;vedAk4s)@WX#txj-{MK~Ekb;A5BTtJOKE;EP4m;} zaV5h+bpgAW-8}P1HtykgMY&e+OJ4#9kc(S>sHX%6OjlH z#67r<9m22EntLnG(RVT@3SGP(*rh%Wf4IJn@2+B#R#}zm5pCYn^ev;2H%>FPOv-1N z+N5~){@L89BN&70@OI2Bf9?!!!`rGT z(i)d%@gEny=S zD0{nNUnHC?oJ6LiT77OBkP(wN`nl5_5uC;z!cUZaugMDPM@<0hBzsSpmG-Zy^RDDz zJiM{6l@n72hO>hY8d!$Agp*N$1m7L$zB)3xMe5U6^VwIA`VQK4JJJnOxLT~2WZY8u z%GAf_#KXuPb3a@Dn&65s+c^;b;S88Zzdc%LQ1y~}FOk-Za9Qlfok?~I<=DVgd|@R8 zb*GU_U*=q<T3&>+*btot&O5ZaX6QEQ1IWfgR=BY5lhN)vRz(X<*-r z$iQ-_A>gX7Ebb5?GuZU+aO8)3v8~H%9c~ z_P&KT$0elQrB%V>P}j8S_hz29blf%-Q>`gm3ST!|UeSyvg#YK-&Q8}+bMq{A*Lhzg zDU*>S#5MrdBqUK~^{;;1Y$)1V`0{eJ3CP%}#sA)ORtbD*w0Iv4e!t(*_up&U`>+49i@e-!v>*fo0bK&86v!hJ$w*Rx_lZ%#Fsb=|6 z*tWsXL+YwEyqA$5m6hdHU8WA~st593UPLYWZ%{#^jX}+WWvmeXDqu;%4`Ca4=r3|} zW}95*oXm2a<<*3kKY5c+gW$qQZm{1eD$d72gc|>T=}O+tiXGF#6lJB{O&mb2XV6>ZT#YV|Hk$t&GbZXMqCG3^VnubA?#2 z?|FPwR*W{Uer4~dE?O|{WBdC3PO)P_n3nNF0uzadeL_u7(M=AR^I4ttJJou1qHlib zIIF6xh+r|CwzRiBa!tJBj)o=E{0kdaCtN)_^;u{c|mbv4585 zy&$oMcz4{hW0v}WHR?6~+J1aW_77_s#TR(jzrqgZDFMqmFC$N+lCryb%eu|Rl88F9 znL0Dtm;yY?*uA^6tN;uVw>} zII>X%rc^MNL7%Z9-~c_w!6!nVIiXuqKb(cTgsnO3y9mFownF&uOEp!MgCUuG9dtxq zarkxzW23Zdzat%t!El%gG9?;YaHCI$~X(UO&baZ92#9Yr#qvlef%%jbpM zb9SCBd{LRSnO8+024E&yto*xg`H?ctu$d$#mNLvTa{dv{M<~Q1W=Z*1{13semYrQ8 zS1UT9L``F0gDGA`DXHvd+Ixpj-(sn_3NBsSMmzmJi0Vdjhu)C$wmw@4liU$Lh}~|N zBRWwj;fn0gKKFThPOFaH_l;tmXr7*-cv49ri_#&OMg9Vdbpm~xlMs0kj1deEB_o)d zK)I6o9vp$^=jB%E-o5N9uQfyi2f9aP^B1ORHLK9FFeL7Uz!p%>Cz2Uq`Ghw=KMSayq|} z)a0d(!}~u&Ug$}koi^ErenSn}ehkqE=)VZ4e(j;Y#cOxqg+j8$=ecyOd$c#Eksrd* zdPF0#hJP*!US7jp=4@~3{J=VsOh9Md>Dk1B&!Ok0g4Uy$Ndnvh4pm+#C4)|A zY<55AaJdv1JClwE+`sA8O>g^eh!k@$HwK9J(?%*8xuj-l_aV8v?3=Qci2zP|K$0PVt-%y zPIt#Rrg{7&)e#l*ubJi>hq}9(M{afRN$xue!J2bZ82wrmG!KJV!xW1|GDt{S?NnuD zFKlb{r-6c(v68E0 z_4Az0Ccmfv3z4s*!V6i@5$vC!}M5(2Ogzp9^{xcw4@6C7{on0p0uXuz{(y3p2-?-#fBFmJhB~JCgQ~VH8 z{M9zodPdy94ig!va8ccclkUDD3k?o^d!bn$i!-sB7g{`T!SC^uonD0MBlHR#eH+3p zi%em6%ANP0-7NnqXh6+JES;DKRGuXK5ay}!-9Uv8AWWx-WJ5I#In)K*omi|kWQe@} zjLU|iSr+3o@qPvgpM&Q(mu$P}ZG0-ZpnVT@Ei)yqFX8HW^O3P{G;My{h#;(u2{%xc zCr!d#s7k(6j@2XtD@QAzDzv5WAB+p+nzket;cY^rHU`@DIVm>$2UI`Y6NQC^+TNF_ zrBXUk&E0>Xtifcm&^b)&P(6)~Vc_&TJIm}7B{fxAg>ZT?7|t((oImxW zA$rQ-6etuYW+u~`xF%I4Sk*k8J3Ppgqj~E^-@6cQ*4F~3bj+s9QJp`;Zzh(8w`5s!N(WHw{-ZXg|zrYzJd z5I9e7HpUVK>3DQq#$)VVj&|cho*7r8ulyhR=(RY~DLnQdy3aeO+4Kqmh!$(~)NERV zF>S2uUj7l`VQ0s#!wU>pQ;U#WY8@vCWQ35K{sMbrsg^7sjEhY$x&%c&5Ffp~y`_%l zFsh557F7Yh3TXp~g~dfM_m4U?h0WX3qJI$B0WlRoS_Jom!(l2BP~URpV@S0qond3U zpF{D3TH}b8ufbXXJt2F7G7>XpQKd=$X~j7Vh~9QbiC4Afrg7x%`PH92!yLV%Ox8(uf;pe1?1RIju2#_#JRI8_p#IHuMfz`M{&F=1e+kf0( zSZ7~VJT2msgpJ2~WqedZM#jvqU;HmQ?q_RGWlb=vKz)ZHGG>uoJ@t53n^;5Tz5IWZ zOGEWc+gwXZN(#_ecl!h<9IPU(wD%TaK-jfSsc$qTUpWsv`OMnOz7Gcu6V2mr%)sH7 zwpLakj65vhrlFx}mjJpH0&Hu*MQL&JjbX9yJ5`W4z#=Z>jjr-OwHB(by!ynrJu zOB!AQMxsfB3bVoWYhSvZt((Pn=j#|o(hJZKy;aI>LP5CQsdvr4eUTqO{4y$I#>YBH zwZFe&`e6EbJ=5mAlM@^`tQ|Ao; zLj-BCWR%iVjyE+j8UPa^In?z3dg(58MUPJqA`1lZ2O;;vk(e!N`htdbmqz%ycw%Noy;vZG zlxMYPhzcX})432;eYcP#9uUOW}>XB`ZR-DLDn4?l|IVz977{a z=@m>|bsKK_&a_>Qi{rXYX5DW<%(pPBA-*U7?AbIh2=Y|lkuosMg2|`P$=~DuemF8b z(9e88WbD--DJ)zKVJrwpee;r7a;hEeq!xGqd*gl^AycoY-E`Iv{%zy%e+*=ETH%go z%n%F3#v5Ry&eZL)ts#D_JMpAXN;F?-SS@$lS%vhA{)v1I^0t=ff?&&1ozdzIUoAAy!VE zOFI``N2KiRaYbFRbW#;N$o_r8w)T#WlY8wo4FN4Q&wbD!)uzjFs61~mZ-lm}J^=?o z?pEXabVTqC<)O{;6I7!S?-Q8)vu7l0<%pXHA= z$WO(3y5Ba_W>45T9H$%A6o%^jeUCOGpJM2c z-;4D_n2Z6VB>{fYdnF z+Ecszv9i>K+%zm-DzxTkkZ&3T&ByH}c8k0p&pa>@uy>#KTZwiTeJX~KzLf-$Xv4<0 zl=U7lA24nlqZtQ~iQFTT>A9^b(fi|F2t)C+Z{Q_Q9XcMc&!OsL7Qqbr|z+N^t zqS;1>ATUcuA(@VqDq2Bmtr<&ZmqqN{p58M|+6||P=>6_bIPkM5`Q#Ug?;e^6S`*Y7 zNd7_p3wfy1c94xBZSUI}9ITk($I2!V;v%B$`QmE(!Yz#+k*Jbuz$5%u;YNvuf+Q14hI)9Uxx->Uf}fWf7ZYH?j=}&5zzNy z@@!-V`XWcVE=Rh2(HE4P#_xAHTQH=;1(J;PUi;QioC z%)Wle_Y3W|4m}I0lQ6eUA*PT2fT4v!t0>uSW?RU1xlcb%E~*0LHKR`#NsdIM!HTXexp055zjl$R$3QV(+6yI#7mnynt7)EZVibV713yXH)Q=nz-w9fg_V1^VwZvXCwUG)4yY^xm1JPi} z`*RYE)`5-(v@oCRmLm`_SZh#ESv5BcMyct&IOSw9A4kgdPw|sfLgF*fwCZd0-WE+> z{`tp5e=z*^xy0r0-l*SIDv!SV+%$&u4Fl~_&6ROmr->zK_UrE6xLQBn`53P(lu>)o?^%1g27rKNAlt4kx1%cH9)W^uzufcG0 z=+`fxGPIjh#Y1NW6haAp` zNE+pfZ}C_hqu;-vD5YMV7;gK<$@rNl*HK7gm@I5V6~9Fzw=qv83iv-M!Jya zQR8%*NPAqf|4@lP#+F;%&{}%DM9{~w8Xb4MD=XN|qzXDU#YevH*U11v15DuJzd3vT z2Z%oK`T6+NUy%TSImX7slnu!c+?7;u^CK;G)rz)1?WMe40_{Man(IlK88|=b4RK0I zT>)Ft+R6$X@uU9a0GhV<|y{ zewQYCRKCMbC-bx9$%%3eS)$4e`ZNy-vG_5cnW}nPRIAD1#TN|0XXLy*6#DdpzM-hw zTNA{t{vl#0Yr@jbwzEj;WM1PnC1u@Vdb-f~fCb+{?WB#%xM;C1lK&DO(ne7*v-v5L z>==4VYYDlPM6ii*{{NWz>ZquF$z}22nw}5s?t3yO9=@2I)`<5kb1U zl$I0(B&DT!_uOaw)_Uh3*OluqbH3;7v*Q!w1Fht5Vg(>mugVx4$SR3+CmRzr_03uq zeY57B!Aej#kHEWC2d&)B&sd!ObyCmw*bvowZRNq;$H{4Xv7-EU^d_|0p@hj1JITgl<6oPocwTQah(JG&Z(e>ZiU{=9M0+tl~; z;V@mzrOixoc#{vj*D!;Oj6EvDJ%i9VaFyf@km&By=+9L?px=-C9IlBRo{? zp{lk9U3_Stu;Nq2{gjV>@@Z#%9Tf)0u_nYFjx|~-ymGS-*b+e^eR-jnm)8Ov1>^zt zGXMJ6LjB@LUIck9n11?0>421JPhe3-!)?>P^A>8xsoS}a7n;u&i?DC{c{#uMX>}d9 z(fc$JKIwS&)l$*FvO0IiJ)`o>?~VL59y_y3-)0tOO`7s%C+%RC66gQQ$8m4=wrdgP z836|iTagC$g3i@IT=qBj@x}+Mng+ht_BHy1#$O#_`H*il82s$3!yLaP`w+)sDHA{F zDCwd>)9c9A_$S)k-d@Y!o(nzgL(A;LB7jhQ>`WU@l-blK)_ugK^U< ziw5A{%WmCt^@UGUet9Ky=;Gk-TF<6=Z;Qc#_jswO`Q)~r&-TCaoUo||zk|KUZJc<0Hvu<8dtMbK(quk>H!s zwE!-aQ>Oow&Mp1zW`p-)OFcH6CvWaQ2smUOSV|}CE#1xKj#=Ab((Mo>;V4SO-qZ3s zNo3Nn8EXkiN8uH5;{H08S%Qvi@yH7uL>;({LNRxKeimpS)e7Z8J4Zi^bD1&9_boEY zND}*3fJ6Y=p2P+OKtlski-I^@L*O5=#|;fr$a5L%(NVB$5gmcqCf|WWev@IsPVm;P zTf*s|KtB67`%)CFtRE~{g7OLMcnB1<3d;8a?%(!#b@0<&WOR$q#pfml3eHBy7@UZ87Q;H>fv8l5_eeq%KcJr>^0w@sVJmoZt;jH9Co` zUmq@O(VP*O9O#1(tB30|f%VfwhARwXuKIu5Y~dpvPqX|ko_@9=3zlLp(&&m5su7+P z8cV<#=?TS63P>I7l{&UJECr1IQr2fZ5v6P+IPjvyxlss!*8C}jXvS!Dp*56E_?EdXkd zii+wjZ2+pnq6w1Sdg}@Tx_Bm%TnUBK6o~|_jC2UbgL~F+nklDcMNigSix~X9;Ix{a zUp|BAq+^SG1kyvB)EjJv!X2Kg1_T?T~(IjIOFt{Xp$guDuQ-|sg0Mw0PDrC z0Zf#5)q+}&kVNNiQN~ZXN@E}620=-!!L)+g&I=gZy^SLvL4r6mus*MpN9+XQd(^(9 zEh#V07IU?E^yuB1w~ur0zyRo(fIlV3a5JokoP#19INw#$K6P*a-anBRdUDn2?E*am zgObs8K-EGT1Nhn=p^w1s08X-JD2#X|x|=5ZOS@x3{?80s7{4fQ#Oay(pIjPKEX zM+|-%bd6ejBhwJ8wV2}UwPLI|zBZuST?fn(f)G)K)!##rIvtCJ<*~k)V|KrS;o!@- zNr2pL@mE>s$#uv)wD^II@{%e(j$JbYsM!<}_=~ab=yP~~a$1gIX-tRpzSx1H7F4d* zR#p*z$v_;UGta}1wTVX1BR;1?nrJB{%8EDu>;yqq-q+o@8Sl86xLK|bHIveQIH5B7eJY>Y7<1^|2XlrOkA7``FjikA zqG<}-Kg^HlK9p%nN)Y$8#>409VESC)%=8sFNeZ;g0ANw@Y=Yu+VrC9s&+w$hA1xZ% zZF0(xga9NKiC7&4JH!3``bP?21qo&%3GATeeP2gM2S_M0A3WXMzV=!%w*_@@!zqPV zb(Fblt4#I2)Jd|#5=>B65cd>VAr3(>?WS;GlI_^I? z65Me0yZL0l>$)@^<(?^oJaR&pcyFqjxbnq|wY_QGvU@=zR6hcb1Ia z)TOf?&{I+_ik8`Z>qnC@qc1KmSrgSP(et_?ME}C?X5IUjW@ar~8PLqxYnG667*|e| z1#R~GEn4I+8H>rNX=tEzDEE5~1*JfD{S+lOD#~?~oU-A57E*AZvUYK8J*u+6fw z+%`!lD6riP1wom7G5~kj7xjI0^()ZigFOJ!Xs~y(=pg0M_B=sTBrxHwH0&O2lFvba z=80(RvS^9k#=B#%gVNa~fli_2By-Go1=`d8=k>t>hs z?|eC@YQ3KLzRr90zMAt9NxImm^@#L?b=BI)bQro6kCjmdJuuBcgZx)>^vMa>iGube zP@Q`>3xTyHr_90038(%geih?tAgJWHe8mzZV5}XJl#npon#1Wh(Rx%E!9qWxA4E=6 z*&d{CT~RfKk*jxSZVOfPc_qFQXk%AX)pBv)`g3#^6chx3shp3wX;;1Cmz$C4=zDE6 zrnCb?Fw+7p-@_ylk7k1M;L-IjGf>;j{c`T$umrgIZyC>MS{8QN8_J9%V2%p{2NW2A z2jF+NSN(M4lU*}N^S0BQkL(4{$RUWzoDvvuI2iY%jNakiEC00_R{l$qkO&1p+6a+q zBd69SE6Yh6VR2B%f}>L>bbNFK#2K#mIna><_4VHF07!H|yWs{3xY&6TKvbz0lK@tI z5SN0w-s8u%R#x#oWKj7*e1xPdzD3K0rA`l8+o;g^$umbs0WO0|HU!kqXayA&BLcCY zPFnEc!!uA^$k)=Cu;4L@f0=N^yD_H>C z-8QllYv@@V%P8n^n!^75OmY9T9s zJpL%E8KwTLRK@9b`NMHPBSn;JQz*|CI3a`Mw zO^^YC9dTuN5G5I})f)#04a1Pkz}2yf36zJX;$jurw2>b_UI1wvgxZ$G_lz>&?%ddr zBp#X&BHkOwQkEZ4{S&3 zuzUGHf1h;WqYph(=4cOdi88$5&8W{SM^A4BcChs}Hqq+3#n#{$YbO8#fK7yjht6 ziPM)5RSww7z~0VF9JUvXQ>2R0=LF|Yo&*)2&KF9jGh}4sI>9g#D$&#b;XY=>3Zw8C zk_G)^LWgf2R~G+1Iii?RDj_gR^y+qL%AzTXnkBbiiu!Wx=4dOa!0n)<)b%HSX=43l zmLC=J=@}Smn-f`sx=5%zU)ny}wpAnviS2;f+Y$-C-=Zk+_0$XAf{x4IKS7KcCtDaJ zrA0~FFP1uRiBpNhA&-OTq?EHubL(~Fsz+31txCvJc5SWbK-6ot+IS_ic#a`&wVlp) zGsZRLW2UG&E}xg)cGj-%hxUijqia8x!@UJcFnI`czWgIL($|k+E6GuyD&6Aj2TNQd zN0JT894`Q7cWk;`n0BMeGAF#xvwAY|1@}IQ^MjSOx3bNz;A5kq=wuwEljFobfzY0Otp&`-MfLtKl zo>*}DpvqziNQ>wXIpKx!Pf!W2xsRPG6h9Szjr&TTOAb*{ zs@U3ync_UU{?bop4>Mxln#-h)(q9v5s|^LR5g!BiMlZtujFDS5YEyN}4dT@f;0z4*cE}1`M!>re+n!!G_Kbmp^@^2vZUm0(`S5g@{ctxD2xU)1XszHmg zE9rkYqJsom!)O;J{$Gq_fof!C^T!inJJO;E9h|BHZH`~oBiX056Xdkr8mauyrwaT6 z(pyLa!WBwb@{*4q2cGvn0#Bn_=kM`$PoM7HdE^s#i>!wm?lidZUwkMQOoK#+Dn@CH z0J)?GK!O5cdpV9dRO-+8FK?r)hzkZNg(t(5& zKc9$`BXUwK<~3#gr6qebV^$28#MtX_ioF|2NBEA$jG_LR8rhOha_0A|R%@$cUgGBY)18KCC?5+;=;_(uK`M0Lz7$H+Po&_AOMf z<_xAou^+|W)1q}^GTO@npBo3s*tXntefJvbF?u}EjS$}96{+XK;3bcj}-*TF++@YKH zYOG%0WJC5SaYubvh>ZA2{`(3D{y#HgCykg9#!MzPV7r9DuX*oYJE%xTAbmI`oflEQ zllx79fu`pnNW{KPN~(Q`MSLzXGW~1S`v1#ZiRffo*hRh*Lh!KorvXf0$G+ZPra3tL zpw^*@UZ7=e&Xn~;SC=^Txw$#GL*7c91{yWo6blebIoCklcTTh8IUC$WWZ7Gwsp(P- z*(|VPYco6TfF#8&#nn*moa@Fc{K!Wi6$N0J#Xqv&`ebg9l}? zkhP(dJh)ZX4aD@u)?eC@L7UF*2f`gv$9B2h`}bF~Gcwe>D9$A!>=2s-TkjGR@A*(& zSw4nKcnT*Z9R4>Fe^!qTb}f}vWux00Utf7F$Y!nadD*O_y{*OSFn9Oa-N6Rh9Cvd1 zpiR-Jn9GljEzdF^gKYo(Z|%Ppk@KM%)NW4`P4x(96y;K8t#we}vuL`Aj7D z4zFZxe!jwo@mS~8duUQ;APn0)x>SBt(HscZIc;~Ry+zccT8z<3t+><}n@9yf=71h9 z#96iEn@3EhEXH1$E`gzlfPesqZ;0hiJ7@yBp@6GH0LatA z)G?qQ$HH=tc5;35Y#>|4G^4S-`5uNX^G?&mE{5@#T zMh^fP%NHAWs_k2sFsH5oS9qg7*xI0u5#Wc~l2pz3Yw+1X(TS@(9jUUCc3o6i$p;t; zfeVtia&BMy<25ZLg{TK1FoU(kE{hwYKLYwMurF5njn#;uvsL3b2SqD4eZ%L^x&^xC zJ?uHf#eYD72L;?XGZPZzZh)eAEzv-ly;m&HEG5EJAs0x z!KZ>xfRzVIuPVW=UUv!p zU)P6HQl5vu13UxFDfg=Z0!jLsGVEkEp+e#rdSuORB6>oHA?S%OBw{zrZLZfO)5@>K26JftMR0*Lc6*F_+kOP@#Z zN5a&1LWH0}Lo4O;C9gWzoHA5iE}A@_e|pcKP9%LmwW|s>(!T{KQ`mejdo zLQA*DkgEagwTs=Yp6)RKK?ilJV@=epP;t#nDIRalos8S54|vtYktrQ;$Avd3?|HKd4HR%Qn%SJl%E|(=H|VTFlAaSy z3!<3Bp#&0ZHHnaLA2VFj+1Ymu2pNcTA9M*!=jF}v+ko0 z4E29aDv)@f!d}8x{~S> z)uA8&W~{OHBHF~n~%$4(n6849BOgx^qog9c15a_wYrbG;^`S-O{UUWyU zvwC%#d3}23K2|M$S4zB!|6kXv=5S_Y$9hW2gs_y(DBryf{0FB# z2`U&_0{)oH`d|Xfl9M0!BjX|if}w)KOwY*AiJy=BKQ&dBRW7&2&0|(gJ2>S#O;7%C z{mSleKZRFCG@o{(bdjy6=Dwew^nn4WJNque`lO^#a%7Hzg>-Tq+nsGrpbtQYcIO0#w91_uK z5it@Hj|Pg#hX-20F`w@n_*Z914te}amQD4rnZuS8MRHp$_DG(lKJ{shSnF}%7g=~( z*8Ir7>bOuzidjPXM1_uXgwCFd+P;#`tQE_G1;pNPm_Qt(E1C*wZq!$b{glEMj=EMywgITd$HBmar)a!oUV}ZJEn14GJy9)i;Z| z*Fv4o%)H%||0&nY{MbAMn_{*^9a6>7mw@St>oz<%5W0)^xW!%Or0Awsy|*GWkzVMl z{Ts9kCO0XakL49YTg}X9$v!h0%ztixwGRmit!Rc(*x#{*lIBOI{?+q>4i;bU@#&xZ zUEK2+`hB5%AkDm-l>N?PF#0-y41hOTUz<~ zh0Y3|=ae zD@?t4tM>UtRm!V;g2~b{U;mx+bFP1>T)Sy#w`Cd?LYogrbjiUrOgkYhJUHRT+)ZfGT`;TLKe}%2fxs4Uw^S^ zRH~;KntnMsOexs7xcJk%U15Kw?54)|-Y>)VWZ}U2W;7I?0HQiZmplz;mft^Tm1U_d zm@tPxbtcoZODHy!^BX&RSnIwZMCe^T$hY9U&-sG>H?RxBE+L6s8NHf_I z`PzaE-0RNPKD4v6nJ*_}H|0O2o}ZNRqjy9n58!<`4e|jtjy6{hP80YN90W2SxvzxZ zr5HBRJ1UU{%V4xheF6j3#xZwTV!o?)2Q+U;)biiR8Zg@9H-{K^re_||VCh60lD-ZZ z{^ZYXd;iA4+r9^H$xI&w7~$aHROoY>;Z`w$(X&n4K*5LPAm8nRpQ{ps$ZchDU~_Pp zu^^}cWDB-}LP701I3f)zo*5`>OoQ4hC4GN~{hdZ=aG1bp;N5cDLu3-h&5L~;mRI1t*}GgyFQG3K-wcYXcFAlkk)>gnVH$Q5GxR3lXFfao0+K!a<8J)*z?j-jEq9*4{ ziAi})@(&t01UpewCyd>gn3yV7jXsPcJG4I*llo5y(bbJ~jb<8!o&I5_>YH6fc?z+I z3`C^w9bE4n)aHwJ6@IGT)?&E2FICJI0#LSq0F+}15doUd$KR^WN88Po{kP9~g6Cy* z#_|o_@tR)4zC+xXPvE^SxnRf7vf@F>d3pvrKqlk**Ky2D1!igSiKc=g(n8114m4Du zAG|freh*>g1Mj`KxCr{0QJtW|E9HW{FY+Aquf1HWzG#b+`5P5t`IHU=+}pVQlib*1_J*<7toU&~PzlokEC9K#*aSg9V$tFocs_ zEr8W*?{%jd{yVHb^l9P8ezh--Po}Tt{Z1ZmlAr5k6soX-f(r=pfn<=70=KK0f z5VZokUvCJ?fLJ}Qs(=jFyBfUD6%d>(l^TIk{csoHltj5t~* zR<66xPt#T$Ofi{$`}JRzU+=$yOp~d@L>fzGkAf2qJ~fdCy1^ zor5b=vl}B#^4nIg^|N=qzhyc(fS_G>_tonco$Wi}5G*-JgGzUA?}0f5)M@|e_oO$@ zQDE%3z7#hG|6^0r1*p@6s0oYk;tY3T-S4Rwewf2en4Ym6_VB|xu-&x%^sjHnGuO7q z4rJ_M9WIPddqaCAQMEjkJb$@UFjpsh|Nfm;EkQpiJ|jLYL0rV!QQw0y?mE%&&-#|w z>3FZiU2vc~asI#RI-u7vVzwdZ?+MRuqCfq#MELZc#Cn(!(2yskXOuzVi z|18x1Tkb_yK;e4b&pLq>TQ70%o@H$fxB(uNq5(8e(L9i`R>~I}) zQTtY2q%*&-sE9}C>ge!I4MZs@doIz72`oCPXO0K{0>9`uTel%{}TsNNk@@pIjaK94El1hNF}cxUHd ztt~CGZm+f?{zP8zsoE3&|zXeEr{L-zg-NcayWirt<3ltI3q7#Ci+e)jChwr}`lwR>QU0&`$dJN7WQ zpr9Tram;;vQVI%4bqzl~0y9BqeSOqs{XXH zG{67o5xL~A5XA0jJ2|BtFfdXK;vlmE18HLUF?%5uwPO=`96bxx5L&Sp3efA{^kGNa zbY=QzaMOl|DIaXX8X7VVUZvaCfi(lpc{!M9gBmj^8jCJ?P5tAo7fdH2msPOoSX$s} zGImS<#HCTEzZJp2;175$K3{9FOI41ged_2q zUuoHmanu(zkSS49S9bKx!9m#ZSpsO;VP5J#Xn@}$BnaK={2=gjsc?XHI;SlbH<=KJNZdVCGmeU9z1-ywPG zaB6nRXzLAH!SEy_g2A&AbT`E!*gf_-DL#NC;oZA(x&$|3?Vf|WbA`y=yZ!%yd1f^^!4%@0ES$Aoy*Zn<;y379s2a+ zb#@)(6!b7=3UkVSbv$OTo2l<*)gcRs9~sdG*}6h@WfAEpD<_Fm$O0`ky9kgn8vv>FuRJCVCyTAY(FV|w6}`jE!U7$95OW{e0*$t~ zo%zPi3nHevQK?4TUrs-RB;AblooZ6HCWgPPe*q7&0R>rYZA63H3@Z~8cx*1zx&$L& zo*-t)?)?{NF+iFEKGb6agZ0^I59;nUd7NqR7W2Wmaf2;HJzY44ifk+MDfG+ba~MM) zN>l-;-f*(oDZGB#3Ox%am6D=%Rq>O#$^#x13tIpTk7_dpt?d*Dbd(9b0e}>~`LU{}zH2`5$gp1Mr z(S^Goe7>}`N8pkL-Vi<>UW*p`8kz~=8V@!9?G$KkpB>4FKQNyFoz;02RaGzeQ;=E% zAUvRb2Q2~+#|C{UIr|2Pwjktwes%sgRmqBf8veeEl?N!0035GcXdB*t@uqv)`{N`g zfe~E=fuV=|ha*xFU4!usc>wEwAjg6VP9L2prXkrR}H|>C>@$~O; zfU0pwJQOp!tNATmACk+nWL&ZkP}|Gt*RI^4{bv%3N;+?;r)Z=*Tg4+ylPzhns^Y2T zxiM8zIHO^eUvA-JAY?2;`OIVp`3OH%0Y&N>s7oTxe_~j22|Vl725k1!l*iT-A7vQ$ zYo^Q~m1TotU?UHLvV3PJNE09ikubllEe8Z140mn9P63q0n+@X@&z)JZ&0{)+*8o@J zH3eo;nB`!W6pfWCBtI~|#YqA|W9aL{8+ya2f>AX^ECEC_;R(U+h51G!m=PJ<%i}00A`g-UCrFek;6U;M!=os2u8p0YFvH6Fdhd@v zEkHid(u<(T$VAgFu<+@#=jIb>hR0u7ukWN*HJ-+O^{0TX^81BYEjRoL+iHuB)bAU} zT|3<9&1k%F!O#@2>6+?B!yb!D0z$&Q(~ED@uE5^9K{J!|Tv=KI?E#Z+ z$yn5 zG#a==qaJ?@bXqzwn)5*7SCeb$u|fr;MY|Xc z?-IG(o9X~|G!U5{I5uZ_=iqszK3u&TqtHI#X*?!&@X7Z&TyOTbhmk&7pca>Ss83$} zmvjU%Ep@}{+B1Gvf$5^z$tLH zTIB0+>#BFC5w|-(?GgAc!R!!(VPEb$_v8nE!v*sMu(!p`h6)6*dxIC>z$yZa{6KrJ zKO`P74yo!OBWxd|bO4}+Y-!&!h&U+!rcy}5mMN?E_^Q8PGUabO8_xU^ej1ddA936? z5qL|1prte%AeOM?0r_xx0#3ifb=c(rmRF!}3a~|xI|QgwMxmLLqa(NAN`GKR$FFa) zEf^VqK1$)UR00sQ_jJ^t0L;`Dz{tetzV{w^Nc{Wv?+;rJvuLD*ChhIyv9(cMM_21i z!p0ngp9xY&brRE4U7cNf88eb1bNT#Z-j}`d zwrPrJoQ^@g8xVTAZ{Id=la4RxUFwY5+}P+@rf7hm0kX@1C&eE=Kwdkq-+zvYfg#G1 zkZINT6kqnk%ltc(DYc4is=L}ICPSPJR^e7pYy-@kQjNZG(`MkGAEt&1_oz@9x5YVpl&zIjp(zG6w2qrjWvQbOyRV+ZzQOs@%!m?q-D?_FjjFfla8} z@z5#)DztPq&kcmkw~gM5AfMy&5H}S=&(hEPFK>%Aom31x$D2XZ$Ql@8SZvVC_7=$1w1K+1*9&Fw>YhnS-4V~H+NgRw<@`%V+5)enh z&jqA;A-^(P5=Qv1fZh85s|r9rK*f66l^NXYz|spWbwc7n9+gBMrl`!!%$nKK8v2R1 zhe)WY@oxLi_K=Q~N3c(WXAPio0d-`qi7^rFfs)=9glJQc2KNx2*uhcxA}NF2-@`xr z*+_AbhD6WmbB$bzt~8a?Xa6orRM*m*r~jlfP9+=K`s} zgx{eklfs?1PZm^m>d=+v(od>7OdU=_Urpti6y1!!`2B%5eQ=zjlx*cS(-DFR)Q98; z3WMprE{WdtT3Np6eXWpNTU$FnZ*r}6GRXOF*G)OT7vtkco2!e(;F~EgGYAa7Isg(F zIlzz@uQB2d$BA~I_v}7xS5#JxW7)1zE$}-0L&Q1JuCLi4Q6UiO5%%ksIfn;15nZL{ zj7?!rEv)(3=BSUfDRY7cpT;{;S)&u5oWUc~=!*AG&y)sZ5=}Fq<%7e7aK3|>(ifDT zhNtrH?}C;l)FoJ47<+1>{iV~lMX4NYj$p@G#&iO%9KgdBE#r;vzsg?Vr4>RC#)Cq% zE%zd7KWdGeSRQm+6gm_o7TS}(kQ&5aig>5ZK5O_lnBc@1CC0&Y?bb#W(*-vO48`t0 zF`iqMj{~=;;eQx%T?zmPRaVBHkW)k`%M#nw zjhtuIKX-eav2CDdxGdwN4&hUkIYB-5Fs+Q2mlr`wnv|W5EHEN_WF>PJ+X$_GFeRt} z3=I6|Fve@-p9h}c^4_!%PFJ+}4v9*=^Sb<$=I9}EtW?4X%#9j(#+(RB4b<9u!GX zEU~xiH*Z+MhJii`zd3dv+GDa$eFME;4j#(mtUGLmB$26?{;fP7sgK*OWs#3ovUGQc z4FIV=$eIEH5XR;}Tnpw!bi4vTV`t|I1!nj`4o4|SQNJxgM0KJyx5s0qACOz|Qevpz zwpT$@85LyT%|d-rSu@Fn1#+>iP&krCJ8Mg)BhXB)Rn|s?|ZQ#Pgywof&6|B_M0@;8dF`sS4}uT~0*DzrYtX(Jb78k}u~LKK1tu znGqa{U;1B9`cIma-U`9N2!bAkR+oMSMJYmlb#2h}a%&^L0`YZlo2Ev~$5Wnz`0BE- z`DWBf-@Osj0bY^WQnLeqWPihmDoKclZ3sSm$t zt>WoFee&~f4a3EsMcHeVy^}YHVx!V=d%3=RoBXodCL~g>3U~+Sga6_fq~^WK-y-C7 z#On%`M)^w0j5f&y_O{>H&kEY4{?$LbS6ltq! zl-@(K`(*g5Ta8<{W?bmT8UYy8d%C@_SF%65I9!Z3>tg>tIpYq!ytF8x#rOCG1cxbx zvBuw$3fHGPrgAA>FTGis5(WXe+OObi1VFh8;t*l#YV zx^=WNMt1FvS+x@Gf0?s-nKqWM=eA{aIPj-7yE8N_eBV>Xk@gRRsQ;2=5>~;}Cmf_O znD$~@#JZEbapa|+Oe(};Kj`{w)oQ?erua2LPjM>mm&G z5%T1hZaTwnv&z(rfh~3N^sbtK9hOkhP-Y$Uj<3Mq(V{YCc`b#Jk?5Uf5&AxQ_D)}n990cV=hQd^^gb> zSJ;cwQJ<&Fa2ieg_qIBr->=jjr>4kCjGI0Dui2i_krf6h{QRH)GYe$u>Bt#f2e)G2 z3qvcEi@YC0j29Np?KZgZ^{;5#I6!Bm{~Z_lcL>-o|BNuPgUN_(FXJ4x=I%e z=r53yBY{n@Y+Hu^z!G9^ZVuU?gCYf7O&sM_0Gu?KyJ5MM={_$8X(sf2Wl9TF3jikk zIwqzO67Q57|Dh=a8YTnrQybkqJoH=ck1ZeN`^7fjaEi(M8US+Ym>s&&Ge2yC@X|3b zF+*>J`86FLJiM;FvocL8ud+EkabP%%or2!R{OzawE+rJj*ONPq=&g@YOXb;;xr`fM zet)f)kwH~cwjd4-rK}-(`3PA?1YRgq$Mf4F-?c`Hv6@no0SqT0K!aAw47>sKQ*o+l zzVF!bg*aqnNKVmUU!TPnMR3$sP{8>3%^R%QkQNm1-?ZS;Qc+O>RO-%l=pg`-ZwLH3 z^6o?f2Vrs13`7zNt4rdO;PHbH%4P%5;t*>Jh6ZsHx$| zq{cvp1|}rV&b6RV0`HOKbuS-KoF|*?Xvx{|4r4+)tV! z9yA{geUgBOYVY)%J%snITF&c|wR1^NTtdR_jScE?EWJ(zcdQQuEF{W+h*Rp)q6z_1 zb7dUBL#D|1`@xs`6A4reu$k$51@^SPJ;)d3L-Ri`#nTF{`BS|QOdwisA*U^1M+fZ^ zB6(W&p3hHv=cm4ZM|F8`U;T4coX8Aj1b`U9POg8fYbsz+VMeI(p3`}KqrkI&zXO$p zJXTmpF!Q!do`Wv&yHeTveA3W64yICPf2asx(U0;$La)G{1l$AfLbGj(m~k4g|GxJX z36z`(GRD{Y|&6DL@N9r!#hec9&8Va^L{QBVUl_ zTD^(v-Oi3UZD@sxz{!AEo|~Hs))Fv9P{E0ejtL?V{GeYXtot3-*{U?BLqoNUlN9ha$M_aP^_rI*Gely{##eaeV|MaW<< z!e+g7i%7mjM#s=F9Z0e*DSP4}VPVAV@u#;c#41QfP<|7$-ZqYMGO;JlmfCo-j4D&=s%v3E$FH#h}aqpB(@b`B1- zE$u*(qZ5$^#i{yT{gNpO$SUHX!x>QK>3Sr@i%C<#L z8W*D(P9C0{Hd;7LKz$iHtN_m=FkXPpE~v)7?4p<3LS6QPKDq!Cgap~c0?70^%~b;RP?VK#v=BP88V(t2XgG5AMd<(D47|T z0`w%@KHGyrAUO^%(MOcf?gSn_6fm&=)l^l3L^b3il=4!=T-lhAH>|ig^~NGEC?pBI zo2ZC!g5?Oz#lAsHx!lXtg98+1h^N4`>&rp}-mEx$30_Ud@0MfYbRJ?N|Mgg9jT0@h zaAWwI2eu#wrx&#ywJ?3Bnx>K`5^+3yGG1NPJ$&qT^d_OSjfO6p_&?wxbAk%YXQgPg zDR0=^hZ`EXeG?N$r>A=dUP7HM0{!)$faZMi$)-?%+qh0oTRR?p3^Ye@i&<(<18YhO z=(|%;fZGbem#7(kpM;WLh-oQg%mheB*+%bw0Ct6D%vX z8B6h>%vST^TmgVMY^DiPu#y$kAOX2&==Aj{uBQTETHh zNuvAW$+1^Ui^#zX(CdM_*eC-jVqscBDKY%+(}0I^vzrX=>r zN*?NZ;0!`034@;-HcAa`ASws=;*?TvoFx&!1VBAr>BX6W0Tn{zf8vUrK*(2Tix~N$ z&k5}VY7UL;?5uaG@7|=fMHU0V0}@4}(uOgQss!APkT<1hfiXXPBWwY83Z!PFlops2 zMv1YCmIxdFhH>6brimQFl<|Kxb9ME2V@VSgWP{S!IGEnC?rDYK_?}<4q#mK5pqxLX zrZKD@r}nxki8}|E#@rxD?vgvik-UCw6yVGW94GiLzz9H^?h0XJ1)X3M zN3wEpY20}$EF`26U%?2cFp>pyQu6aVOx6+UFiw}&lcC*U$o0?*kYrS+CnxLt^w35) zovAozcvb)Sukid@!Lwf^yk>ib^rUT;+*$s3Y$$PqpR&iBw*sau?EBw z0Pyh3aGlix8UY23E7f(Ag}2l#g{jB8v_@qJNx~}eAKkZl6EYSuvO7|-%X=#LO`mgh z&z~6YURuuChocw~8k$WSjdnI0@N6KB>XgqJ5pZ9uo%wghvg&oth(r4YK7f^Sfv%L- zp$a2nQJkp50LB{7S_gq0Ksw+u!DJVdXu!amzy~NSz@vls887Jw1Q2NW2W}LCR6>|S zIP{WXHjp7k#PL-V2MkRn%Z!(4U&;!^A8gWog}Fz>@7&O4V~-WnvXF%AFPMh+c_p_z zJvM)S0-VF#vxe@0x7wt)5IAN#=##80I6)o+u<(SxUz`SG-|Xw}hmI^6t#Bd}ZmvjgK&hv&~%Fw7|j!g@}H_qmj;mDj%r zTb!x47h?Bemxh-P9S;dx&#;WM*ia;qkN|Eu5akwT5M#ue-1ERGEF*kK+Cb6zRMHYl zB=c4~!0PBJnE^D;L1BrXl5MG@VWm_T$CEwK(|j{rV4tSfx% z5SdSyA5v1rj_29ReL|Kzm0hnzS7*M{dy~|J5hxd;AgDF(*vj5`2Y87-exQyg;sgjM3(#@ zd@~IsBKU2G;?S?bw+tH_Th7q=wE^h*RzHz)ns4aqQ09h30lzr2G#@! z+@4Sa_>tpv5XU1YcMKN+xYoTIvn+mX(?gqw(dgmWH5s3Bk=s-v$?SDfHcFM-4V^u6*@ z<+3gq2hvNVYyu=yKH|VL(Jt+)WLWodWT!=-Y`}U8I}s!w9v+lRr1L5b{9VyUH!^|% z@rHq2&(4Q?kiGM60)Y|uedI{4qP*P6%nXGXRe$Y*E#$;@$Z1upHYmCQ(w^NNwJlOw zzVyKi&iEanU9WTP&DKEt?)u9b;eg!20ZPkCeYlDVgkk<%J{P#NowbRTcxz3h6b~a6 zCxtZ#@@Dt*VLr1l^k~7| zB6Rb>6YP{xIyp8{C3AJ7LM)j{@(>{$K5GHGD4}ki=Vde=HNK+mkDsLcxtb9)WBUnu zhNXBiSV_08C|Dw}11N;6rX`77(5}k4P;8uo{xNwczZj!nqabXUm)@dt$0wQdPseij ziy15KN1&)f=^VqDg+VV4{;+PIeAQW2HqM_#6!@0fz_r!E{F=7?@b1ijvG|k;wh|qIb_9ziK+Ql$5~H;KlZMFGP(~d78%i;4 zd(1Z!(7kuD(p_$@4KRH)tYBx^x=oKkg$<*n#wa}4IlqIAuet|&sVI6C&W%S zCE3`!i*p0lQMfCVyJEcAl}Apa86!mY4B{|p5ahJVmbabxsd)_HkYdwKY%gSpjhp9I`CfJA$ z>K^Z3XLCzy5L9k>pc-TFEEbDqAIvDfu!{MsxbRcUY!tPfU?a9&=wtdTd|H33KKu@< zjNTHVaaX_k8gF=bjlK;59;m*oYc+AATvL`Zl`ANVJ}BsH8vn1WD-VZy-^13B?ieAZ z!9>w=Iz-97R#Kw0a8mYTi&E5Bvrfixs8rWJ9GyC%a3s~BAx27CCXDP$t1ZbGg!@kC z-sidZ`h$6%G4uQVzU$}ne&5ggrc{)cb`WHixds~1+!w_xGdbV;@?ONfdu$pKtsAR- zC+R@o`LM+faH5wEsMf{IFmuPrM|#9PT`n|9@|Z*UHys?5somlqiuDBZt-W*)$c_{c zwAgr`7(G+!w6jjBdlfy}NLj+& zr$pGVqLvr-=cRXDocuMUYPZp3Gehvibk+=wR`#Bs8?Edw@5DJyIQ;YYm1dk9!>Q`5 zGXR=>y0+CBp-KfAt0G0&FEj%2%rD4+rKS1Or$bLJVL9Xw)?IN8a)wxuq(&8!6OONY znF)D<$6+m8@}9nYNo3fMu%KJKOtH^94`y@u zrKNmou+G+f<8LcA85!jS>O;cGIGm?xLkp(jP8rC`?%lgaw8MmRaNvEqNk0#vo|A?< zKqQ4m(wd=)cBeTw^S`hHE%o`IS5+P7lltjbFsh+z2>+k3;yVI@~nlRK&a=T>{Z##w*7a$9?Qi9^}I z^XCz^WL5) zkv8vIuYr`CLyxd@Zo}EECfyv zu*?EWezU&xK}RE_c~ofm;*ouP^g_RRiWGk-+<%j%BOl z3dV_s?e+5d0C%~L*4D!g4j)-fm#}3+Pu-kmo`3)R{6j|OS;aTlXo3}{38(A|zum6v znZ@BE;E7{?zJO`)UCl1ekX@0MHe(0b*}cY|2-rzn2Ws~t8xk?N5@|GY9P<(K(l6bc z?C(Tv(e5cP*aqUEkg#xVA(NY9O*gLo;}5DK;Wi#4F*$i{9E$5mMxnBvj_1a@nu1l= zC!*gpC+li%*%iv(O-hm%IepoCMO)nfW`|l_(7AK)g=9eTrle-})@rG0_&Y_SJTo8T76&8O-yR`qM&l(%|TmKEO;fu))L;tJ#B3|DZ(M5X-#j~?1Cdj z-JJJeB$}CW)^6PR`5BxK8IR@|pcjE%HofHR*O~9o9zmBd*3oY;OdnywEIfwwrpe4u z8kEHDa|r6Y%?nbbm;YMe=x81>mi}SsU^RQ?| zVbG$^*skK^3T+mD4W53h&x&z{n3six#Ylpj7c1=Es_+YWub`O z@-F>Vr2-fsjiKV>16N-&h!m8@TkT@GJcQS?nr<`lB`7nx?n@Ml>dFZ?@ymyIBqb$Q zRkjj8UK`x}NuzZ^Q8Q`PpIh}K8o*cv;AAq-HUYU@SB1mgI*(~fH;HsAcoksNHgn?7BSnpC zH8{__)kU2SFoABbi$jSoeP%=-A+P<$tBBI~{P8-iEIa+5GFS!bp||Z6IX`uPN;Se7 zheL!6CN(>ow14@?{tGz4Kj*#k2hFjzAA-40?s+q!l(ukaShPREv zxbXS42)WtKpg@%oG}x>g$lyQYW3#8pRIf}?+=-=F!p&bPfzofQ|7yU zdR*k!4;Bno;Tjes1-12w@(OTy=r8lCz zT1Ev!S;r*o-+L6rM^c_jecqp4O-+^+%jSqTCYh~FpCP1g?;lZ7{dFljC3dV#cyGV( z-4|Qywo+{*#}S{r}0p z3N$v=P84eTr&L6UxPn7f;G6U}Mu}aaL+|$Z7G&fHgi9(@<%t@3<3r!M8``Pv#r83! zt1PcPc0e<1>E}Hysk^BsgZ@9@|4oc`IZ_JOx%wM2Kho`^1C8=}Wk;_j@J{Uh{_!rgBd5vi#CY zm(~nb6`FgO!=-wPvJawap{@JiOh~^f-?qst``GG%o&p1J<#nP1E3zfZ19Gi2*$HG? z^P^V&0{H1**Ma-&F8{B|V_KbHMG8Zdf1{fdJ)Yt|*Nit@-yIQ6P9r2W3|t$^BAEI|lW8vm zU*-G>22FLa>0yEN$uQq&`IkGt2&E)W>OIPdS4aq3ZAmn_pp?MKY{>2%8$sfVbX0@v zWZK&_GivD2DPtC%h@4|^SUv`T0C@~>>W>%htXh$E{9Esm@Z{_uK5F#P(n|>k7l}+W zs2!0NADI3)75ct>-=%1NJ5q3eK(wp7@d34^IW}T1bbkavdeFS8WB_>E@v5`a;B#gG z{~7PEg0#$6A+ztxQ9jc?ydiO5BKXGwN&FI6&_3MiTTwX&e98c|5mHKq0X7}4q@8MMqDz+O+pwr)+PJo%p&^D@Z?CO*|3vV`OYG=jd?54< zb9i)+A0=dP%^Y#Rv9&lb29sxKIPO~}!JO%xsWgqMRiwZo{3emQSJLV$u?DMR95H{t zw@y}wVeam=Zg)j*it?nHM}Ng!67TC4q~?Rk;%~aoOYC#~j}wWB(W$8D_Re@-cvaNk zg1!@TqC#q?g{hTk-KoYg#~+WZocOzU<^^Z`$@`}AeJ}g&V6U+~A4zA7J}xFXF;9;- zNb0t6&1_38Hj#T`bb}W8Qd~7yYu(4WRqmTF2 z>yW5nUBNZgUFeu6B6Z)_N}I2=y%5{3meim%R;{G@@5m&yD1uvVxRKw=84l^ns$Rzm zbqXG5cMS^)JA2cA1X2^+oGY@+{@$okB5xk+z>?f#^r`@L&Ap)=o1A;#+ z{`bW;ch<5~)5yy4#-bUsSSR}MqtfNwp6kWdR{s}Q3+D)G+dyTwX}(dC zhRKbl=cmgPLUyw2qSh`hGqy`+Dg_+LHV!-*P(?{R6j& None: + super().__init__() + self.EPS = 1e-8 + if json_dir == None: + raise ValueError("JSON DIR is None!") + if n_src not in [1, 2]: + raise ValueError("{} is not in [1, 2]".format(n_src)) + self.json_dir = json_dir + self.sample_rate = sample_rate + self.normalize_audio = normalize_audio + + if segment is None: + self.seg_len = None + self.fps_len = None + else: + self.seg_len = int(segment * sample_rate) + + self.n_src = n_src + self.test = self.seg_len is None + mix_json = os.path.join(json_dir, "mix_both.json") + sources_json = [ + os.path.join(json_dir, source + ".json") for source in ["s1", "s2"] + ] + + with open(mix_json, "r") as f: + mix_infos = json.load(f) + sources_infos = [] + for src_json in sources_json: + with open(src_json, "r") as f: + sources_infos.append(json.load(f)) + + self.mix = [] + self.sources = [] + if self.n_src == 1: + orig_len = len(mix_infos) * 2 + drop_utt, drop_len = 0, 0 + if not self.test: + for i in range(len(mix_infos) - 1, -1, -1): + if mix_infos[i][1] < self.seg_len: + drop_utt = drop_utt + 1 + drop_len = drop_len + mix_infos[i][1] + del mix_infos[i] + for src_inf in sources_infos: + del src_inf[i] + else: + for src_inf in sources_infos: + self.mix.append(mix_infos[i]) + self.sources.append(src_inf[i]) + else: + for i in range(len(mix_infos)): + for src_inf in sources_infos: + self.mix.append(mix_infos[i]) + self.sources.append(src_inf[i]) + + print_( + "Drop {} utts({:.2f} h) from {} (shorter than {} samples)".format( + drop_utt, drop_len / sample_rate / 3600, orig_len, self.seg_len + ) + ) + self.length = len(self.mix) + + elif self.n_src == 2: + orig_len = len(mix_infos) + drop_utt, drop_len = 0, 0 + if not self.test: + for i in range(len(mix_infos) - 1, -1, -1): # Go backward + if mix_infos[i][1] < self.seg_len: + drop_utt = drop_utt + 1 + drop_len = drop_len + mix_infos[i][1] + del mix_infos[i] + for src_inf in sources_infos: + del src_inf[i] + + print_( + "Drop {} utts({:.2f} h) from {} (shorter than {} samples)".format( + drop_utt, drop_len / sample_rate / 36000, orig_len, self.seg_len + ) + ) + self.mix = mix_infos + self.sources = sources_infos + self.length = len(self.mix) + + def __len__(self): + return self.length + + def preprocess_audio_only(self, idx: int): + if self.n_src == 1: + if self.mix[idx][1] == self.seg_len or self.test: + rand_start = 0 + else: + rand_start = np.random.randint(0, self.mix[idx][1] - self.seg_len) + if self.test: + stop = None + else: + stop = rand_start + self.seg_len + # Load mixture + x, _ = sf.read( + self.mix[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + # Load sources + s, _ = sf.read( + self.sources[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + # torch from numpy + target = torch.from_numpy(s) + mixture = torch.from_numpy(x) + if self.normalize_audio: + m_std = mixture.std(-1, keepdim=True) + mixture = normalize_tensor_wav(mixture, eps=self.EPS, std=m_std) + target = normalize_tensor_wav(target, eps=self.EPS, std=m_std) + return mixture, target.unsqueeze(0), self.mix[idx][0].split("/")[-1] + # import pdb; pdb.set_trace() + if self.n_src == 2: + if self.mix[idx][1] == self.seg_len or self.test: + rand_start = 0 + else: + rand_start = np.random.randint(0, self.mix[idx][1] - self.seg_len) + if self.test: + stop = None + else: + stop = rand_start + self.seg_len + # Load mixture + x, _ = sf.read( + self.mix[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + # Load sources + source_arrays = [] + for src in self.sources: + s, _ = sf.read( + src[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + source_arrays.append(s) + sources = torch.from_numpy(np.vstack(source_arrays)) + mixture = torch.sum(sources, dim=0) + + if self.normalize_audio: + m_std = mixture.std(-1, keepdim=True) + mixture = normalize_tensor_wav(mixture, eps=self.EPS, std=m_std) + sources = normalize_tensor_wav(sources, eps=self.EPS, std=m_std) + + return mixture, sources, self.mix[idx][0].split("/")[-1] + + def __getitem__(self, index: int): + return self.preprocess_audio_only(index) + +class Libri2MixModuleRemix(LightningDataModule): + def __init__( + self, + train_dir: str, + valid_dir: str, + test_dir: str, + n_src: int = 2, + sample_rate: int = 8000, + segment: float = 4.0, + normalize_audio: bool = False, + batch_size: int = 64, + num_workers: int = 0, + pin_memory: bool = False, + persistent_workers: bool = False, + ) -> None: + super().__init__() + self.save_hyperparameters(logger=False) + + self.data_train: Optional[Dataset] = None + self.data_val: Optional[Dataset] = None + self.data_test: Optional[Dataset] = None + + def setup(self, stage: Optional[str] = None) -> None: + """Load data. Set variables: `self.data_train`, `self.data_val`, `self.data_test`. + + This method is called by Lightning before `trainer.fit()`, `trainer.validate()`, `trainer.test()`, and + `trainer.predict()`, so be careful not to execute things like random split twice! Also, it is called after + `self.prepare_data()` and there is a barrier in between which ensures that all the processes proceed to + `self.setup()` once the data is prepared and available for use. + + :param stage: The stage to setup. Either `"fit"`, `"validate"`, `"test"`, or `"predict"`. Defaults to ``None``. + """ + # load and split datasets only if not loaded already + if not self.data_train and not self.data_val and not self.data_test: + self.data_train = Libri2MixDataset( + json_dir=self.hparams.train_dir, + n_src=self.hparams.n_src, + sample_rate=self.hparams.sample_rate, + segment=self.hparams.segment, + normalize_audio=self.hparams.normalize_audio, + ) + self.data_val = Libri2MixDataset( + json_dir=self.hparams.valid_dir, + n_src=self.hparams.n_src, + sample_rate=self.hparams.sample_rate, + segment=None, + normalize_audio=self.hparams.normalize_audio, + ) + self.data_test = Libri2MixDataset( + json_dir=self.hparams.test_dir, + n_src=self.hparams.n_src, + sample_rate=self.hparams.sample_rate, + segment=None, + normalize_audio=self.hparams.normalize_audio, + ) + + def train_dataloader(self) -> DataLoader: + return DataLoader( + self.data_train, + batch_size=self.hparams.batch_size, + num_workers=self.hparams.num_workers, + shuffle=True, + pin_memory=True, + ) + + def val_dataloader(self) -> DataLoader: + return DataLoader( + self.data_val, + batch_size=1, + num_workers=self.hparams.num_workers, + shuffle=False, + pin_memory=True, + ) + + def test_dataloader(self) -> DataLoader: + return DataLoader( + self.data_test, + batch_size=1, + num_workers=self.hparams.num_workers, + shuffle=False, + pin_memory=True, + ) + + @property + def make_loader(self): + return self.train_dataloader(), self.val_dataloader(), self.test_dataloader() + \ No newline at end of file diff --git a/look2hear/datas/__init__.py b/look2hear/datas/__init__.py new file mode 100644 index 0000000..4d518ab --- /dev/null +++ b/look2hear/datas/__init__.py @@ -0,0 +1,9 @@ +from .echosetdatamodule import EchoSetDataModule +from .Libri2Mix16 import Libri2MixModuleRemix +from .lrs2datamodule import LRS2DataModule + +__all__ = [ + "EchoSetDataModule", + "Libri2MixModuleRemix", + "LRS2DataModule", +] diff --git a/look2hear/datas/echosetdatamodule.py b/look2hear/datas/echosetdatamodule.py new file mode 100644 index 0000000..568c28d --- /dev/null +++ b/look2hear/datas/echosetdatamodule.py @@ -0,0 +1,283 @@ +import os +import json +import numpy as np +from typing import Any, Tuple +import soundfile as sf +import torch +from pytorch_lightning import LightningDataModule +# from pytorch_lightning.core.mixins import HyperparametersMixin +from torch.utils.data import ConcatDataset, DataLoader, Dataset +from typing import Dict, Iterable, List, Iterator +from rich import print +from pytorch_lightning.utilities import rank_zero_only + + +@rank_zero_only +def print_(message: str): + print(message) + + +def normalize_tensor_wav(wav_tensor, eps=1e-8, std=None): + mean = wav_tensor.mean(-1, keepdim=True) + if std is None: + std = wav_tensor.std(-1, keepdim=True) + return (wav_tensor - mean) / (std + eps) + + +class MP3DDataset(Dataset): + def __init__( + self, + json_dir: str = "", + n_src: int = 2, + sample_rate: int = 8000, + segment: float = 4.0, + normalize_audio: bool = False, + ) -> None: + super().__init__() + self.EPS = 1e-8 + if json_dir == None: + raise ValueError("JSON DIR is None!") + if n_src not in [1, 2]: + raise ValueError("{} is not in [1, 2]".format(n_src)) + self.json_dir = json_dir + self.sample_rate = sample_rate + self.normalize_audio = normalize_audio + + if segment is None: + self.seg_len = None + self.fps_len = None + else: + self.seg_len = int(segment * sample_rate) + + self.n_src = n_src + self.test = self.seg_len is None + mix_json = os.path.join(json_dir, "mix.json") + sources_json = [ + os.path.join(json_dir, source + ".json") for source in ["s1", "s2"] + ] + + with open(mix_json, "r") as f: + mix_infos = json.load(f) + sources_infos = [] + for src_json in sources_json: + with open(src_json, "r") as f: + sources_infos.append(json.load(f)) + + self.mix = [] + self.sources = [] + if self.n_src == 1: + orig_len = len(mix_infos) * 2 + drop_utt, drop_len = 0, 0 + if not self.test: + for i in range(len(mix_infos) - 1, -1, -1): + if mix_infos[i][1] < self.seg_len: + drop_utt = drop_utt + 1 + drop_len = drop_len + mix_infos[i][1] + del mix_infos[i] + for src_inf in sources_infos: + del src_inf[i] + else: + for src_inf in sources_infos: + self.mix.append(mix_infos[i]) + self.sources.append(src_inf[i]) + else: + for i in range(len(mix_infos)): + for src_inf in sources_infos: + self.mix.append(mix_infos[i]) + self.sources.append(src_inf[i]) + + print_( + "Drop {} utts({:.2f} h) from {} (shorter than {} samples)".format( + drop_utt, drop_len / sample_rate / 3600, orig_len, self.seg_len + ) + ) + self.length = len(self.mix) + + elif self.n_src == 2: + orig_len = len(mix_infos) + drop_utt, drop_len = 0, 0 + if not self.test: + for i in range(len(mix_infos) - 1, -1, -1): # Go backward + if mix_infos[i][1] < self.seg_len: + drop_utt = drop_utt + 1 + drop_len = drop_len + mix_infos[i][1] + del mix_infos[i] + for src_inf in sources_infos: + del src_inf[i] + + print_( + "Drop {} utts({:.2f} h) from {} (shorter than {} samples)".format( + drop_utt, drop_len / sample_rate / 36000, orig_len, self.seg_len + ) + ) + self.mix = mix_infos + self.sources = sources_infos + self.length = len(self.mix) + + def __len__(self): + return self.length + + def preprocess_audio_only(self, idx: int): + if self.n_src == 1: + if self.mix[idx][1] == self.seg_len or self.test: + rand_start = 0 + else: + rand_start = np.random.randint(0, self.mix[idx][1] - self.seg_len) + if self.test: + stop = None + else: + stop = rand_start + self.seg_len + # Load mixture + x, _ = sf.read( + self.mix[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + # Load sources + s, _ = sf.read( + self.sources[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + # torch from numpy + target = torch.from_numpy(s) + mixture = torch.from_numpy(x) + if self.normalize_audio: + m_std = mixture.std(-1, keepdim=True) + mixture = normalize_tensor_wav(mixture, eps=self.EPS, std=m_std) + target = normalize_tensor_wav(target, eps=self.EPS, std=m_std) + # return mixture, target.unsqueeze(0), self.mix[idx][0].split("/")[-1] + return mixture, target.unsqueeze(0), self.mix[idx][0] + # import pdb; pdb.set_trace() + if self.n_src == 2: + if self.mix[idx][1] == self.seg_len or self.test: + rand_start = 0 + else: + rand_start = np.random.randint(0, self.mix[idx][1] - self.seg_len) + if self.test: + stop = None + else: + stop = rand_start + self.seg_len + # Load mixture + x, _ = sf.read( + self.mix[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + # Load sources + source_arrays = [] + for src in self.sources: + s, _ = sf.read( + src[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + source_arrays.append(s) + sources = torch.from_numpy(np.vstack(source_arrays)) + mixture = torch.from_numpy(x) + + if self.normalize_audio: + m_std = mixture.std(-1, keepdim=True) + mixture = normalize_tensor_wav(mixture, eps=self.EPS, std=m_std) + sources = normalize_tensor_wav(sources, eps=self.EPS, std=m_std) + + # return mixture, sources, self.mix[idx][0].split("/")[-1] + return mixture, sources, self.mix[idx][0] + + def __getitem__(self, index: int): + return self.preprocess_audio_only(index) + + +class EchoSetDataModule(object): + def __init__( + self, + train_dir: str, + valid_dir: str, + test_dir: str, + n_src: int = 2, + sample_rate: int = 8000, + segment: float = 4.0, + normalize_audio: bool = False, + batch_size: int = 64, + num_workers: int = 0, + pin_memory: bool = False, + persistent_workers: bool = False, + ) -> None: + super().__init__() + if train_dir == None or valid_dir == None or test_dir == None: + raise ValueError("JSON DIR is None!") + if n_src not in [1, 2]: + raise ValueError("{} is not in [1, 2]".format(n_src)) + + # this line allows to access init params with 'self.hparams' attribute + self.train_dir = train_dir + self.valid_dir = valid_dir + self.test_dir = test_dir + self.n_src = n_src + self.sample_rate = sample_rate + self.segment = segment + self.normalize_audio = normalize_audio + self.batch_size = batch_size + self.num_workers = num_workers + self.pin_memory = pin_memory + self.persistent_workers = persistent_workers + + self.data_train: Dataset = None + self.data_val: Dataset = None + self.data_test: Dataset = None + + def setup(self) -> None: + self.data_train = MP3DDataset( + json_dir=self.train_dir, + n_src=self.n_src, + sample_rate=self.sample_rate, + segment=self.segment, + normalize_audio=self.normalize_audio, + ) + self.data_val = MP3DDataset( + json_dir=self.valid_dir, + n_src=self.n_src, + sample_rate=self.sample_rate, + segment=None, + normalize_audio=self.normalize_audio, + ) + self.data_test = MP3DDataset( + json_dir=self.test_dir, + n_src=self.n_src, + sample_rate=self.sample_rate, + segment=None, + normalize_audio=self.normalize_audio, + ) + + def train_dataloader(self) -> DataLoader: + return DataLoader( + dataset=self.data_train, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers, + persistent_workers=self.persistent_workers, + pin_memory=self.pin_memory, + drop_last=True, + ) + + def val_dataloader(self) -> DataLoader: + return DataLoader( + dataset=self.data_val, + shuffle=False, + batch_size=self.batch_size, + num_workers=self.num_workers, + persistent_workers=self.persistent_workers, + pin_memory=self.pin_memory, + drop_last=False, + ) + + def test_dataloader(self) -> DataLoader: + return DataLoader( + dataset=self.data_test, + shuffle=False, + batch_size=self.batch_size, + num_workers=self.num_workers, + persistent_workers=self.persistent_workers, + pin_memory=self.pin_memory, + drop_last=False, + ) + + @property + def make_loader(self): + return self.train_dataloader(), self.val_dataloader(), self.test_dataloader() + + @property + def make_sets(self): + return self.data_train, self.data_val, self.data_test diff --git a/look2hear/datas/lrs2datamodule.py b/look2hear/datas/lrs2datamodule.py new file mode 100644 index 0000000..e1abdf1 --- /dev/null +++ b/look2hear/datas/lrs2datamodule.py @@ -0,0 +1,358 @@ +import os +import json +from tkinter.tix import Tree +import numpy as np +from typing import Any, Tuple +import soundfile as sf +import torch +from pytorch_lightning import LightningDataModule +from pytorch_lightning.core.mixins import HyperparametersMixin +from torch.utils.data import ConcatDataset, DataLoader, Dataset +from typing import Dict, Iterable, List, Iterator +from rich import print +from pytorch_lightning.utilities import rank_zero_only + + +@rank_zero_only +def print_(message: str): + print(message) + + +def normalize_tensor_wav(wav_tensor, eps=1e-8, std=None): + mean = wav_tensor.mean(-1, keepdim=True) + if std is None: + std = wav_tensor.std(-1, keepdim=True) + return (wav_tensor - mean) / (std + eps) + + +class LRS2Dataset(Dataset): + def __init__( + self, + json_dir: str = "", + n_src: int = 2, + sample_rate: int = 8000, + fps: int = 25, + segment: float = 4.0, + normalize_audio: bool = False, + audio_only: bool = True, + ) -> None: + super().__init__() + self.EPS = 1e-8 + if json_dir == None: + raise ValueError("JSON DIR is None!") + if n_src not in [1, 2]: + raise ValueError("{} is not in [1, 2]".format(n_src)) + self.json_dir = json_dir + self.sample_rate = sample_rate + self.normalize_audio = normalize_audio + self.audio_only = audio_only + + if segment is None: + self.seg_len = None + self.fps_len = None + else: + self.seg_len = int(segment * sample_rate) + self.fps_len = int(segment * fps) + self.n_src = n_src + self.test = self.seg_len is None + mix_json = os.path.join(json_dir, "mix.json") + sources_json = [ + os.path.join(json_dir, source + ".json") for source in ["s1", "s2"] + ] + + with open(mix_json, "r") as f: + mix_infos = json.load(f) + sources_infos = [] + for src_json in sources_json: + with open(src_json, "r") as f: + sources_infos.append(json.load(f)) + + self.mix = [] + self.sources = [] + if self.n_src == 1: + orig_len = len(mix_infos) * 2 + drop_utt, drop_len = 0, 0 + if not self.test: + for i in range(len(mix_infos) - 1, -1, -1): + if mix_infos[i][1] < self.seg_len: + drop_utt = drop_utt + 1 + drop_len = drop_len + mix_infos[i][1] + del mix_infos[i] + for src_inf in sources_infos: + del src_inf[i] + else: + for src_inf in sources_infos: + self.mix.append(mix_infos[i]) + self.sources.append(src_inf[i]) + else: + for i in range(len(mix_infos)): + for src_inf in sources_infos: + self.mix.append(mix_infos[i]) + self.sources.append(src_inf[i]) + + print_( + "Drop {} utts({:.2f} h) from {} (shorter than {} samples)".format( + drop_utt, drop_len / sample_rate / 3600, orig_len, self.seg_len + ) + ) + self.length = len(self.mix) + + elif self.n_src == 2: + orig_len = len(mix_infos) + drop_utt, drop_len = 0, 0 + if not self.test: + for i in range(len(mix_infos) - 1, -1, -1): # Go backward + if mix_infos[i][1] < self.seg_len: + drop_utt = drop_utt + 1 + drop_len = drop_len + mix_infos[i][1] + del mix_infos[i] + for src_inf in sources_infos: + del src_inf[i] + + print_( + "Drop {} utts({:.2f} h) from {} (shorter than {} samples)".format( + drop_utt, drop_len / sample_rate / 36000, orig_len, self.seg_len + ) + ) + self.mix = mix_infos + self.sources = sources_infos + self.length = len(self.mix) + + def __len__(self): + return self.length + + def preprocess_audio_only(self, idx: int): + if self.n_src == 1: + if self.mix[idx][1] == self.seg_len or self.test: + rand_start = 0 + else: + rand_start = np.random.randint(0, self.mix[idx][1] - self.seg_len) + if self.test: + stop = None + else: + stop = rand_start + self.seg_len + # Load mixture + x, _ = sf.read( + self.mix[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + # Load sources + s, _ = sf.read( + self.sources[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + # torch from numpy + target = torch.from_numpy(s) + mixture = torch.from_numpy(x) + if self.normalize_audio: + m_std = mixture.std(-1, keepdim=True) + mixture = normalize_tensor_wav(mixture, eps=self.EPS, std=m_std) + target = normalize_tensor_wav(target, eps=self.EPS, std=m_std) + return mixture, target.unsqueeze(0), self.mix[idx][0].split("/")[-1] + # import pdb; pdb.set_trace() + if self.n_src == 2: + if self.mix[idx][1] == self.seg_len or self.test: + rand_start = 0 + else: + rand_start = np.random.randint(0, self.mix[idx][1] - self.seg_len) + if self.test: + stop = None + else: + stop = rand_start + self.seg_len + # Load mixture + x, _ = sf.read( + self.mix[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + # Load sources + source_arrays = [] + for src in self.sources: + s, _ = sf.read( + src[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + source_arrays.append(s) + sources = torch.from_numpy(np.vstack(source_arrays)) + mixture = torch.from_numpy(x) + + if self.normalize_audio: + m_std = mixture.std(-1, keepdim=True) + mixture = normalize_tensor_wav(mixture, eps=self.EPS, std=m_std) + sources = normalize_tensor_wav(sources, eps=self.EPS, std=m_std) + + return mixture, sources, self.mix[idx][0].split("/")[-1] + + def preprocess_audio_visual(self, idx: int): + if self.n_src == 1: + if self.mix[idx][1] == self.seg_len or self.test: + rand_start = 0 + else: + rand_start = np.random.randint(0, self.mix[idx][1] - self.seg_len) + if self.test: + stop = None + else: + stop = rand_start + self.seg_len + + mix_source, _ = sf.read( + self.mix[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + source = sf.read( + self.sources[idx][0], start=rand_start, stop=stop, dtype="float32" + )[0] + source_mouth = None + + source = torch.from_numpy(source) + mixture = torch.from_numpy(mix_source) + + if self.normalize_audio: + m_std = mixture.std(-1, keepdim=True) + mixture = normalize_tensor_wav(mixture, eps=self.EPS, std=m_std) + source = normalize_tensor_wav(source, eps=self.EPS, std=m_std) + return mixture, source, source_mouth, self.mix[idx][0].split("/")[-1] + + if self.n_src == 2: + if self.mix[idx][1] == self.seg_len or self.test: + rand_start = 0 + else: + rand_start = np.random.randint(0, self.mix[idx][1] - self.seg_len) + if self.test: + stop = None + else: + stop = rand_start + self.seg_len + + mix_source, _ = sf.read( + self.mix[idx][0], start=rand_start, stop=stop, dtype="float32" + ) + sources = [] + for src in self.sources[idx]: + # import pdb; pdb.set_trace() + sources.append( + sf.read(src[0], start=rand_start, stop=stop, dtype="float32")[0] + ) + # import pdb; pdb.set_trace() + sources_mouths = None + # import pdb; pdb.set_trace() + sources = torch.stack([torch.from_numpy(source) for source in sources]) + mixture = torch.from_numpy(mix_source) + + if self.normalize_audio: + m_std = mixture.std(-1, keepdim=True) + mixture = normalize_tensor_wav(mixture, eps=self.EPS, std=m_std) + sources = normalize_tensor_wav(sources, eps=self.EPS, std=m_std) + + return mixture, sources, sources_mouths, self.mix[idx][0].split("/")[-1] + + def __getitem__(self, index: int): + if self.audio_only: + return self.preprocess_audio_only(index) + else: + return self.preprocess_audio_visual(index) + + +class LRS2DataModule(object): + def __init__( + self, + train_dir: str, + valid_dir: str, + test_dir: str, + n_src: int = 2, + sample_rate: int = 8000, + fps: int = 25, + segment: float = 4.0, + normalize_audio: bool = False, + batch_size: int = 64, + num_workers: int = 0, + pin_memory: bool = False, + persistent_workers: bool = False, + audio_only: bool = True, + ) -> None: + super().__init__() + if train_dir == None or valid_dir == None or test_dir == None: + raise ValueError("JSON DIR is None!") + if n_src not in [1, 2]: + raise ValueError("{} is not in [1, 2]".format(n_src)) + + # this line allows to access init params with 'self.hparams' attribute + self.train_dir = train_dir + self.valid_dir = valid_dir + self.test_dir = test_dir + self.n_src = n_src + self.sample_rate = sample_rate + self.fps = fps + self.segment = segment + self.normalize_audio = normalize_audio + self.batch_size = batch_size + self.num_workers = num_workers + self.pin_memory = pin_memory + self.persistent_workers = persistent_workers + self.audio_only = audio_only + + self.data_train: Dataset = None + self.data_val: Dataset = None + self.data_test: Dataset = None + + def setup(self) -> None: + self.data_train = LRS2Dataset( + json_dir=self.train_dir, + n_src=self.n_src, + sample_rate=self.sample_rate, + fps=self.fps, + segment=self.segment, + normalize_audio=self.normalize_audio, + audio_only=self.audio_only, + ) + self.data_val = LRS2Dataset( + json_dir=self.valid_dir, + n_src=self.n_src, + sample_rate=self.sample_rate, + fps=self.fps, + segment=self.segment, + normalize_audio=self.normalize_audio, + audio_only=self.audio_only, + ) + self.data_test = LRS2Dataset( + json_dir=self.test_dir, + n_src=self.n_src, + sample_rate=self.sample_rate, + fps=self.fps, + segment=self.segment, + normalize_audio=self.normalize_audio, + audio_only=self.audio_only, + ) + + def train_dataloader(self) -> DataLoader: + return DataLoader( + dataset=self.data_train, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers, + persistent_workers=self.persistent_workers, + pin_memory=self.pin_memory, + drop_last=True, + ) + + def val_dataloader(self) -> DataLoader: + return DataLoader( + dataset=self.data_val, + shuffle=False, + batch_size=self.batch_size, + num_workers=self.num_workers, + persistent_workers=self.persistent_workers, + pin_memory=self.pin_memory, + drop_last=True, + ) + + def test_dataloader(self) -> DataLoader: + return DataLoader( + dataset=self.data_test, + shuffle=False, + batch_size=self.batch_size, + num_workers=self.num_workers, + persistent_workers=self.persistent_workers, + pin_memory=self.pin_memory, + drop_last=True, + ) + + @property + def make_loader(self): + return self.train_dataloader(), self.val_dataloader(), self.test_dataloader() + + @property + def make_sets(self): + return self.data_train, self.data_val, self.data_test diff --git a/look2hear/layers/__init__.py b/look2hear/layers/__init__.py new file mode 100644 index 0000000..3b85d70 --- /dev/null +++ b/look2hear/layers/__init__.py @@ -0,0 +1,50 @@ +from .cnnlayers import ( + TAC, + Conv1DBlock, + ConvNormAct, + ConvNorm, + NormAct, + Video1DConv, + Concat, + FRCNNBlock, + FRCNNBlockTCN, + Bottomup, + BottomupTCN, + Bottomup_Concat_Topdown, + Bottomup_Concat_Topdown_TCN, +) +from .rnnlayers import DPRNN, DPRNNBlock, DPRNNLinear, LSTMBlockTF, TransformerBlockTF +from .enc_dec import make_enc_dec, FreeFB +from .normalizations import gLN, cLN, LN, bN +from .stft import forward_stft, inverse_stft +from .stft_tfgn import Stft + +__all__ = [ + "TAC", + "DPRNN", + "DPRNNBlock", + "DPRNNLinear", + "LSTMBlockTF", + "TransformerBlockTF", + "Conv1DBlock", + "ConvNormAct", + "ConvNorm", + "NormAct", + "Video1DConv", + "Concat", + "FRCNNBlock", + "FRCNNBlockTCN", + "Bottomup", + "Bottomup", + "Bottomup_Concat_Topdown", + "Bottomup_Concat_Topdown_TCN", + "make_enc_dec", + "FreeFB", + "gLN", + "cLN", + "LN", + "bN", + "forward_stft", + "inverse_stft", + "Stft", +] diff --git a/look2hear/layers/activations.py b/look2hear/layers/activations.py new file mode 100644 index 0000000..b2e875b --- /dev/null +++ b/look2hear/layers/activations.py @@ -0,0 +1,68 @@ +import torch +from torch import nn + + +def linear(): + return nn.Identity() + + +def relu(): + return nn.ReLU() + + +def prelu(): + return nn.PReLU() + + +def leaky_relu(): + return nn.LeakyReLU() + + +def sigmoid(): + return nn.Sigmoid() + + +def softmax(dim=None): + return nn.Softmax(dim=dim) + + +def tanh(): + return nn.Tanh() + + +def gelu(): + return nn.GELU() + + +def register_activation(custom_act): + if ( + custom_act.__name__ in globals().keys() + or custom_act.__name__.lower() in globals().keys() + ): + raise ValueError( + f"Activation {custom_act.__name__} already exists. Choose another name." + ) + globals().update({custom_act.__name__: custom_act}) + + +def get(identifier): + if identifier is None: + return None + elif callable(identifier): + return identifier + elif isinstance(identifier, str): + cls = globals().get(identifier) + if cls is None: + raise ValueError( + "Could not interpret activation identifier: " + str(identifier) + ) + return cls + else: + raise ValueError( + "Could not interpret activation identifier: " + str(identifier) + ) + + +if __name__ == "__main__": + print(globals().keys()) + print(globals().get("tanh")) diff --git a/look2hear/layers/cnnlayers.py b/look2hear/layers/cnnlayers.py new file mode 100644 index 0000000..3351b66 --- /dev/null +++ b/look2hear/layers/cnnlayers.py @@ -0,0 +1,887 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from . import normalizations, activations + + +class _Chop1d(nn.Module): + """To ensure the output length is the same as the input.""" + + def __init__(self, chop_size): + super().__init__() + self.chop_size = chop_size + + def forward(self, x): + return x[..., : -self.chop_size].contiguous() + + +class Conv1DBlock(nn.Module): + def __init__( + self, + in_chan, + hid_chan, + skip_out_chan, + kernel_size, + padding, + dilation, + norm_type="gLN", + causal=False, + ): + super(Conv1DBlock, self).__init__() + self.skip_out_chan = skip_out_chan + conv_norm = normalizations.get(norm_type) + in_conv1d = nn.Conv1d(in_chan, hid_chan, 1) + depth_conv1d = nn.Conv1d( + hid_chan, + hid_chan, + kernel_size, + padding=padding, + dilation=dilation, + groups=hid_chan, + ) + if causal: + depth_conv1d = nn.Sequential(depth_conv1d, _Chop1d(padding)) + self.shared_block = nn.Sequential( + in_conv1d, + nn.PReLU(), + conv_norm(hid_chan), + depth_conv1d, + nn.PReLU(), + conv_norm(hid_chan), + ) + self.res_conv = nn.Conv1d(hid_chan, in_chan, 1) + if skip_out_chan: + self.skip_conv = nn.Conv1d(hid_chan, skip_out_chan, 1) + + def forward(self, x): + r"""Input shape $(batch, feats, seq)$.""" + shared_out = self.shared_block(x) + res_out = self.res_conv(shared_out) + if not self.skip_out_chan: + return res_out + skip_out = self.skip_conv(shared_out) + return res_out, skip_out + + +class ConvNormAct(nn.Module): + """ + This class defines the convolution layer with normalization and a PReLU + activation + """ + + def __init__( + self, + in_chan, + out_chan, + kernel_size, + stride=1, + groups=1, + dilation=1, + padding=0, + norm_type="gLN", + act_type="prelu", + ): + super(ConvNormAct, self).__init__() + self.conv = nn.Conv1d( + in_chan, + out_chan, + kernel_size, + stride=stride, + dilation=dilation, + padding=padding, + bias=True, + groups=groups, + ) + self.norm = normalizations.get(norm_type)(out_chan) + self.act = activations.get(act_type)() + + def forward(self, x): + output = self.conv(x) + output = self.norm(output) + return self.act(output) + + +class ConvNorm(nn.Module): + def __init__( + self, + in_chan, + out_chan, + kernel_size, + stride=1, + groups=1, + dilation=1, + padding=0, + norm_type="gLN", + ): + super(ConvNorm, self).__init__() + self.conv = nn.Conv1d( + in_chan, + out_chan, + kernel_size, + stride, + padding, + dilation, + bias=True, + groups=groups, + ) + self.norm = normalizations.get(norm_type)(out_chan) + + def forward(self, x): + output = self.conv(x) + return self.norm(output) + + +class NormAct(nn.Module): + """ + This class defines a normalization and PReLU activation + """ + + def __init__( + self, out_chan, norm_type="gLN", act_type="prelu", + ): + """ + :param nOut: number of output channels + """ + super(NormAct, self).__init__() + # self.norm = nn.GroupNorm(1, nOut, eps=1e-08) + self.norm = normalizations.get(norm_type)(out_chan) + self.act = activations.get(act_type)() + + def forward(self, input): + output = self.norm(input) + return self.act(output) + + +class Video1DConv(nn.Module): + """ + video part 1-D Conv Block + in_chan: video Encoder output channels + out_chan: dconv channels + kernel_size: the depthwise conv kernel size + dilation: the depthwise conv dilation + residual: Whether to use residual connection + skip_con: Whether to use skip connection + first_block: first block, not residual + """ + + def __init__( + self, + in_chan, + out_chan, + kernel_size, + dilation=1, + residual=True, + skip_con=True, + first_block=True, + ): + super(Video1DConv, self).__init__() + self.first_block = first_block + # first block, not residual + self.residual = residual and not first_block + self.bn = nn.BatchNorm1d(in_chan) if not first_block else None + self.relu = nn.ReLU() if not first_block else None + self.dconv = nn.Conv1d( + in_chan, + in_chan, + kernel_size, + groups=in_chan, + dilation=dilation, + padding=(dilation * (kernel_size - 1)) // 2, + bias=True, + ) + self.bconv = nn.Conv1d(in_chan, out_chan, 1) + self.sconv = nn.Conv1d(in_chan, out_chan, 1) + self.skip_con = skip_con + + def forward(self, x): + """ + x: [B, N, T] + out: [B, N, T] + """ + if not self.first_block: + y = self.bn(self.relu(x)) + y = self.dconv(y) + else: + y = self.dconv(x) + # skip connection + if self.skip_con: + skip = self.sconv(y) + if self.residual: + y = y + x + return skip, y + else: + return skip, y + else: + y = self.bconv(y) + if self.residual: + y = y + x + return y + else: + return y + + +class Concat(nn.Module): + def __init__(self, ain_chan, vin_chan, out_chan): + super(Concat, self).__init__() + self.ain_chan = ain_chan + self.vin_chan = vin_chan + # project + self.conv1d = nn.Sequential( + nn.Conv1d(ain_chan + vin_chan, out_chan, 1), nn.PReLU() + ) + + def forward(self, a, v): + # up-sample video features + v = torch.nn.functional.interpolate(v, size=a.size(-1)) + # concat: n x (A+V) x Ta + y = torch.cat([a, v], dim=1) + # conv1d + return self.conv1d(y) + + +class FRCNNBlock(nn.Module): + def __init__( + self, + in_chan=128, + out_chan=512, + upsampling_depth=4, + norm_type="gLN", + act_type="prelu", + ): + super().__init__() + self.proj_1x1 = ConvNormAct( + in_chan, + out_chan, + kernel_size=1, + stride=1, + groups=1, + dilation=1, + padding=0, + norm_type=norm_type, + act_type=act_type, + ) + self.depth = upsampling_depth + self.spp_dw = nn.ModuleList([]) + self.spp_dw.append( + ConvNorm( + out_chan, + out_chan, + kernel_size=5, + stride=1, + groups=out_chan, + dilation=1, + padding=((5 - 1) // 2) * 1, + norm_type=norm_type, + ) + ) + # ----------Down Sample Layer---------- + for i in range(1, upsampling_depth): + self.spp_dw.append( + ConvNorm( + out_chan, + out_chan, + kernel_size=5, + stride=2, + groups=out_chan, + dilation=1, + padding=((5 - 1) // 2) * 1, + norm_type=norm_type, + ) + ) + # ----------Fusion Layer---------- + self.fuse_layers = nn.ModuleList([]) + for i in range(upsampling_depth): + fuse_layer = nn.ModuleList([]) + for j in range(upsampling_depth): + if i == j: + fuse_layer.append(None) + elif j - i == 1: + fuse_layer.append(None) + elif i - j == 1: + fuse_layer.append( + ConvNorm( + out_chan, + out_chan, + kernel_size=5, + stride=2, + groups=out_chan, + dilation=1, + padding=((5 - 1) // 2) * 1, + norm_type=norm_type, + ) + ) + self.fuse_layers.append(fuse_layer) + self.concat_layer = nn.ModuleList([]) + # ----------Concat Layer---------- + for i in range(upsampling_depth): + if i == 0 or i == upsampling_depth - 1: + self.concat_layer.append( + ConvNormAct( + out_chan * 2, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + else: + self.concat_layer.append( + ConvNormAct( + out_chan * 3, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + self.last_layer = nn.Sequential( + ConvNormAct( + out_chan * upsampling_depth, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + self.res_conv = nn.Conv1d(out_chan, in_chan, 1) + # ----------parameters------------- + self.depth = upsampling_depth + + def forward(self, x): + """ + :param x: input feature map + :return: transformed feature map + """ + residual = x.clone() + # Reduce --> project high-dimensional feature maps to low-dimensional space + output1 = self.proj_1x1(x) + output = [self.spp_dw[0](output1)] + for k in range(1, self.depth): + out_k = self.spp_dw[k](output[-1]) + output.append(out_k) + + x_fuse = [] + for i in range(len(self.fuse_layers)): + wav_length = output[i].shape[-1] + y = torch.cat( + ( + self.fuse_layers[i][0](output[i - 1]) + if i - 1 >= 0 + else torch.Tensor().to(output1.device), + output[i], + F.interpolate(output[i + 1], size=wav_length, mode="nearest") + if i + 1 < self.depth + else torch.Tensor().to(output1.device), + ), + dim=1, + ) + x_fuse.append(self.concat_layer[i](y)) + + wav_length = output[0].shape[-1] + for i in range(1, len(x_fuse)): + x_fuse[i] = F.interpolate(x_fuse[i], size=wav_length, mode="nearest") + + concat = self.last_layer(torch.cat(x_fuse, dim=1)) + expanded = self.res_conv(concat) + return expanded + residual + + +class Bottomup(nn.Module): + def __init__( + self, + in_chan=128, + out_chan=512, + upsampling_depth=4, + norm_type="gLN", + act_type="prelu", + ): + super().__init__() + self.proj_1x1 = ConvNormAct( + in_chan, + out_chan, + kernel_size=1, + stride=1, + groups=1, + dilation=1, + padding=0, + norm_type=norm_type, + act_type=act_type, + ) + self.depth = upsampling_depth + self.spp_dw = nn.ModuleList([]) + self.spp_dw.append( + ConvNorm( + out_chan, + out_chan, + kernel_size=5, + stride=1, + groups=out_chan, + dilation=1, + padding=((5 - 1) // 2) * 1, + norm_type=norm_type, + ) + ) + # ----------Down Sample Layer---------- + for i in range(1, upsampling_depth): + self.spp_dw.append( + ConvNorm( + out_chan, + out_chan, + kernel_size=5, + stride=2, + groups=out_chan, + dilation=1, + padding=((5 - 1) // 2) * 1, + norm_type=norm_type, + ) + ) + + def forward(self, x): + residual = x.clone() + # Reduce --> project high-dimensional feature maps to low-dimensional space + output1 = self.proj_1x1(x) + output = [self.spp_dw[0](output1)] + for k in range(1, self.depth): + out_k = self.spp_dw[k](output[-1]) + output.append(out_k) + + return residual, output[-1], output + + +class BottomupTCN(nn.Module): + def __init__( + self, + in_chan=128, + out_chan=512, + upsampling_depth=4, + norm_type="gLN", + act_type="prelu", + ): + super().__init__() + self.proj_1x1 = ConvNormAct( + in_chan, + out_chan, + kernel_size=1, + stride=1, + groups=1, + dilation=1, + padding=0, + norm_type=norm_type, + act_type=act_type, + ) + self.depth = upsampling_depth + self.spp_dw = nn.ModuleList([]) + self.spp_dw.append( + Video1DConv(out_chan, out_chan, 3, skip_con=False, first_block=True) + ) + # ----------Down Sample Layer---------- + for i in range(1, upsampling_depth): + self.spp_dw.append( + Video1DConv(out_chan, out_chan, 3, skip_con=False, first_block=False) + ) + + def forward(self, x): + residual = x.clone() + # Reduce --> project high-dimensional feature maps to low-dimensional space + output1 = self.proj_1x1(x) + output = [self.spp_dw[0](output1)] + for k in range(1, self.depth): + out_k = self.spp_dw[k](output[-1]) + output.append(out_k) + + return residual, output[-1], output + + +class Bottomup_Concat_Topdown(nn.Module): + def __init__( + self, + in_chan=128, + out_chan=512, + upsampling_depth=4, + norm_type="gLN", + act_type="prelu", + ): + super().__init__() + # ----------Fusion Layer---------- + self.fuse_layers = nn.ModuleList([]) + for i in range(upsampling_depth): + fuse_layer = nn.ModuleList([]) + for j in range(upsampling_depth): + if i == j: + fuse_layer.append(None) + elif j - i == 1: + fuse_layer.append(None) + elif i - j == 1: + fuse_layer.append( + ConvNorm( + out_chan, + out_chan, + kernel_size=5, + stride=2, + groups=out_chan, + dilation=1, + padding=((5 - 1) // 2) * 1, + norm_type=norm_type, + ) + ) + self.fuse_layers.append(fuse_layer) + self.concat_layer = nn.ModuleList([]) + # ----------Concat Layer---------- + for i in range(upsampling_depth): + if i == 0 or i == upsampling_depth - 1: + self.concat_layer.append( + ConvNormAct( + out_chan * 3, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + else: + self.concat_layer.append( + ConvNormAct( + out_chan * 4, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + self.last_layer = nn.Sequential( + ConvNormAct( + out_chan * upsampling_depth, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + self.res_conv = nn.Conv1d(out_chan, in_chan, 1) + # ----------parameters------------- + self.depth = upsampling_depth + + def forward(self, residual, bottomup, topdown): + x_fuse = [] + for i in range(len(self.fuse_layers)): + wav_length = bottomup[i].shape[-1] + y = torch.cat( + ( + self.fuse_layers[i][0](bottomup[i - 1]) + if i - 1 >= 0 + else torch.Tensor().to(bottomup[i].device), + bottomup[i], + F.interpolate(bottomup[i + 1], size=wav_length, mode="nearest") + if i + 1 < self.depth + else torch.Tensor().to(bottomup[i].device), + F.interpolate(topdown, size=wav_length, mode="nearest"), + ), + dim=1, + ) + x_fuse.append(self.concat_layer[i](y)) + + wav_length = bottomup[0].shape[-1] + for i in range(1, len(x_fuse)): + x_fuse[i] = F.interpolate(x_fuse[i], size=wav_length, mode="nearest") + + concat = self.last_layer(torch.cat(x_fuse, dim=1)) + expanded = self.res_conv(concat) + return expanded + residual + + +class Bottomup_Concat_Topdown_TCN(nn.Module): + def __init__( + self, + in_chan=128, + out_chan=512, + upsampling_depth=4, + norm_type="gLN", + act_type="prelu", + ): + super().__init__() + # ----------Fusion Layer---------- + self.fuse_layers = nn.ModuleList([]) + for i in range(upsampling_depth): + fuse_layer = nn.ModuleList([]) + for j in range(upsampling_depth): + if i == j: + fuse_layer.append(None) + elif j - i == 1: + fuse_layer.append(None) + elif i - j == 1: + fuse_layer.append(None) + self.fuse_layers.append(fuse_layer) + self.concat_layer = nn.ModuleList([]) + # ----------Concat Layer---------- + for i in range(upsampling_depth): + if i == 0 or i == upsampling_depth - 1: + self.concat_layer.append( + ConvNormAct( + out_chan * 3, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + else: + self.concat_layer.append( + ConvNormAct( + out_chan * 4, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + self.last_layer = nn.Sequential( + ConvNormAct( + out_chan * upsampling_depth, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + self.res_conv = nn.Conv1d(out_chan, in_chan, 1) + # ----------parameters------------- + self.depth = upsampling_depth + + def forward(self, residual, bottomup, topdown): + x_fuse = [] + for i in range(len(self.fuse_layers)): + wav_length = bottomup[i].shape[-1] + y = torch.cat( + ( + bottomup[i - 1] + if i - 1 >= 0 + else torch.Tensor().to(bottomup[i].device), + bottomup[i], + bottomup[i + 1] + if i + 1 < self.depth + else torch.Tensor().to(bottomup[i].device), + F.interpolate(topdown, size=wav_length, mode="nearest"), + ), + dim=1, + ) + x_fuse.append(self.concat_layer[i](y)) + + concat = self.last_layer(torch.cat(x_fuse, dim=1)) + expanded = self.res_conv(concat) + return expanded + residual + + +class FRCNNBlockTCN(nn.Module): + def __init__( + self, + in_chan=128, + out_chan=512, + upsampling_depth=4, + norm_type="gLN", + act_type="prelu", + ): + super().__init__() + self.proj_1x1 = ConvNormAct( + in_chan, + out_chan, + kernel_size=1, + stride=1, + groups=1, + dilation=1, + padding=0, + norm_type=norm_type, + act_type=act_type, + ) + self.depth = upsampling_depth + self.spp_dw = nn.ModuleList([]) + self.spp_dw.append( + Video1DConv(out_chan, out_chan, 3, skip_con=False, first_block=True) + ) + # ----------Down Sample Layer---------- + for i in range(1, upsampling_depth): + self.spp_dw.append( + Video1DConv(out_chan, out_chan, 3, skip_con=False, first_block=False) + ) + # ----------Fusion Layer---------- + self.fuse_layers = nn.ModuleList([]) + for i in range(upsampling_depth): + fuse_layer = nn.ModuleList([]) + for j in range(upsampling_depth): + if i == j: + fuse_layer.append(None) + elif j - i == 1: + fuse_layer.append(None) + elif i - j == 1: + fuse_layer.append(None) + self.fuse_layers.append(fuse_layer) + self.concat_layer = nn.ModuleList([]) + # ----------Concat Layer---------- + for i in range(upsampling_depth): + if i == 0 or i == upsampling_depth - 1: + self.concat_layer.append( + ConvNormAct( + out_chan * 2, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + else: + self.concat_layer.append( + ConvNormAct( + out_chan * 3, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + self.last_layer = nn.Sequential( + ConvNormAct( + out_chan * upsampling_depth, + out_chan, + 1, + 1, + norm_type=norm_type, + act_type=act_type, + ) + ) + self.res_conv = nn.Conv1d(out_chan, in_chan, 1) + # ----------parameters------------- + self.depth = upsampling_depth + + def forward(self, x): + """ + :param x: input feature map + :return: transformed feature map + """ + residual = x.clone() + # Reduce --> project high-dimensional feature maps to low-dimensional space + output1 = self.proj_1x1(x) + output = [self.spp_dw[0](output1)] + for k in range(1, self.depth): + out_k = self.spp_dw[k](output[-1]) + output.append(out_k) + + x_fuse = [] + for i in range(len(self.fuse_layers)): + wav_length = output[i].shape[-1] + y = torch.cat( + ( + output[i - 1] if i - 1 >= 0 else torch.Tensor().to(output1.device), + output[i], + output[i + 1] + if i + 1 < self.depth + else torch.Tensor().to(output1.device), + ), + dim=1, + ) + x_fuse.append(self.concat_layer[i](y)) + + concat = self.last_layer(torch.cat(x_fuse, dim=1)) + expanded = self.res_conv(concat) + return expanded + residual + + +class TAC(nn.Module): + """Transform-Average-Concatenate inter-microphone-channel permutation invariant communication block [1]. + Args: + input_dim (int): Number of features of input representation. + hidden_dim (int, optional): size of hidden layers in TAC operations. + activation (str, optional): type of activation used. See asteroid.masknn.activations. + norm_type (str, optional): type of normalization layer used. See asteroid.masknn.norms. + .. note:: Supports inputs of shape :math:`(batch, mic\_channels, features, chunk\_size, n\_chunks)` + as in FasNet-TAC. The operations are applied for each element in ``chunk_size`` and ``n_chunks``. + Output is of same shape as input. + References + [1] : Luo, Yi, et al. "End-to-end microphone permutation and number invariant multi-channel + speech separation." ICASSP 2020. + """ + + def __init__(self, input_dim, hidden_dim=384, activation="prelu", norm_type="gLN"): + super().__init__() + self.hidden_dim = hidden_dim + self.input_tf = nn.Sequential( + nn.Linear(input_dim, hidden_dim), activations.get(activation)() + ) + self.avg_tf = nn.Sequential( + nn.Linear(hidden_dim, hidden_dim), activations.get(activation)() + ) + self.concat_tf = nn.Sequential( + nn.Linear(2 * hidden_dim, input_dim), activations.get(activation)() + ) + self.norm = normalizations.get(norm_type)(input_dim) + + def forward(self, x, valid_mics=None): + """ + Args: + x: (:class:`torch.Tensor`): Input multi-channel DPRNN features. + Shape: :math:`(batch, mic\_channels, features, chunk\_size, n\_chunks)`. + valid_mics: (:class:`torch.LongTensor`): tensor containing effective number of microphones on each batch. + Batches can be composed of examples coming from arrays with a different + number of microphones and thus the ``mic_channels`` dimension is padded. + E.g. torch.tensor([4, 3]) means first example has 4 channels and the second 3. + Shape: :math`(batch)`. + Returns: + output (:class:`torch.Tensor`): features for each mic_channel after TAC inter-channel processing. + Shape :math:`(batch, mic\_channels, features, chunk\_size, n\_chunks)`. + """ + # Input is 5D because it is multi-channel DPRNN. DPRNN single channel is 4D. + batch_size, nmics, channels, chunk_size, n_chunks = x.size() + if valid_mics is None: + valid_mics = torch.LongTensor([nmics] * batch_size) + # First operation: transform the input for each frame and independently on each mic channel. + output = self.input_tf( + x.permute(0, 3, 4, 1, 2).reshape( + batch_size * nmics * chunk_size * n_chunks, channels + ) + ).reshape(batch_size, chunk_size, n_chunks, nmics, self.hidden_dim) + + # Mean pooling across channels + if valid_mics.max() == 0: + # Fixed geometry array + mics_mean = output.mean(1) + else: + # Only consider valid channels in each batch element: each example can have different number of microphones. + mics_mean = [ + output[b, :, :, : valid_mics[b]].mean(2).unsqueeze(0) + for b in range(batch_size) + ] # 1, dim1*dim2, H + mics_mean = torch.cat(mics_mean, 0) # B*dim1*dim2, H + + # The average is processed by a non-linear transform + mics_mean = self.avg_tf( + mics_mean.reshape(batch_size * chunk_size * n_chunks, self.hidden_dim) + ) + mics_mean = ( + mics_mean.reshape(batch_size, chunk_size, n_chunks, self.hidden_dim) + .unsqueeze(3) + .expand_as(output) + ) + + # Concatenate the transformed average in each channel with the original feats and + # project back to same number of features + output = torch.cat([output, mics_mean], -1) + output = self.concat_tf( + output.reshape(batch_size * chunk_size * n_chunks * nmics, -1) + ).reshape(batch_size, chunk_size, n_chunks, nmics, -1) + output = self.norm( + output.permute(0, 3, 4, 1, 2).reshape( + batch_size * nmics, -1, chunk_size, n_chunks + ) + ).reshape(batch_size, nmics, -1, chunk_size, n_chunks) + + output += x + return output diff --git a/look2hear/layers/enc_dec.py b/look2hear/layers/enc_dec.py new file mode 100644 index 0000000..d831548 --- /dev/null +++ b/look2hear/layers/enc_dec.py @@ -0,0 +1,463 @@ +import warnings +from typing import Optional +import torch +from torch import nn +from torch.nn import functional as F + + +def make_enc_dec( + fb_name, + n_filters, + kernel_size, + stride=None, + sample_rate=8000.0, + who_is_pinv=None, + padding=0, + output_padding=0, + **kwargs, +): + """Creates congruent encoder and decoder from the same filterbank family. + Args: + fb_name (str, className): Filterbank family from which to make encoder + and decoder. To choose among [``'free'``, ``'analytic_free'``, + ``'param_sinc'``, ``'stft'``]. Can also be a class defined in a + submodule in this subpackade (e.g. :class:`~.FreeFB`). + n_filters (int): Number of filters. + kernel_size (int): Length of the filters. + stride (int, optional): Stride of the convolution. + If None (default), set to ``kernel_size // 2``. + sample_rate (float): Sample rate of the expected audio. + Defaults to 8000.0. + who_is_pinv (str, optional): If `None`, no pseudo-inverse filters will + be used. If string (among [``'encoder'``, ``'decoder'``]), decides + which of ``Encoder`` or ``Decoder`` will be the pseudo inverse of + the other one. + padding (int): Zero-padding added to both sides of the input. + Passed to Encoder and Decoder. + output_padding (int): Additional size added to one side of the output shape. + Passed to Decoder. + **kwargs: Arguments which will be passed to the filterbank class + additionally to the usual `n_filters`, `kernel_size` and `stride`. + Depends on the filterbank family. + Returns: + :class:`.Encoder`, :class:`.Decoder` + """ + fb_class = get(fb_name) + + if who_is_pinv in ["dec", "decoder"]: + fb = fb_class( + n_filters, kernel_size, stride=stride, sample_rate=sample_rate, **kwargs + ) + enc = Encoder(fb, padding=padding) + # Decoder filterbank is pseudo inverse of encoder filterbank. + dec = Decoder.pinv_of(fb) + elif who_is_pinv in ["enc", "encoder"]: + fb = fb_class( + n_filters, kernel_size, stride=stride, sample_rate=sample_rate, **kwargs + ) + dec = Decoder(fb, padding=padding, output_padding=output_padding) + # Encoder filterbank is pseudo inverse of decoder filterbank. + enc = Encoder.pinv_of(fb) + else: + fb = fb_class( + n_filters, kernel_size, stride=stride, sample_rate=sample_rate, **kwargs + ) + enc = Encoder(fb, padding=padding) + # Filters between encoder and decoder should not be shared. + fb = fb_class( + n_filters, kernel_size, stride=stride, sample_rate=sample_rate, **kwargs + ) + dec = Decoder(fb, padding=padding, output_padding=output_padding) + return enc, dec + + +def register_filterbank(custom_fb): + """Register a custom filterbank, gettable with `filterbanks.get`. + Args: + custom_fb: Custom filterbank to register. + """ + if ( + custom_fb.__name__ in globals().keys() + or custom_fb.__name__.lower() in globals().keys() + ): + raise ValueError( + f"Filterbank {custom_fb.__name__} already exists. Choose another name." + ) + globals().update({custom_fb.__name__: custom_fb}) + + +def get(identifier): + """Returns a filterbank class from a string. Returns its input if it + is callable (already a :class:`.Filterbank` for example). + Args: + identifier (str or Callable or None): the filterbank identifier. + Returns: + :class:`.Filterbank` or None + """ + if identifier is None: + return None + elif callable(identifier): + return identifier + elif isinstance(identifier, str): + cls = globals().get(identifier) + if cls is None: + raise ValueError( + "Could not interpret filterbank identifier: " + str(identifier) + ) + return cls + else: + raise ValueError( + "Could not interpret filterbank identifier: " + str(identifier) + ) + + +class Filterbank(nn.Module): + """Base Filterbank class. + Each subclass has to implement a ``filters`` method. + Args: + n_filters (int): Number of filters. + kernel_size (int): Length of the filters. + stride (int, optional): Stride of the conv or transposed conv. (Hop size). + If None (default), set to ``kernel_size // 2``. + sample_rate (float): Sample rate of the expected audio. + Defaults to 8000. + Attributes: + n_feats_out (int): Number of output filters. + """ + + def __init__(self, n_filters, kernel_size, stride=None, sample_rate=8000.0): + super(Filterbank, self).__init__() + self.n_filters = n_filters + self.kernel_size = kernel_size + self.stride = stride if stride else self.kernel_size // 2 + # If not specified otherwise in the filterbank's init, output + # number of features is equal to number of required filters. + self.n_feats_out = n_filters + self.sample_rate = sample_rate + + def filters(self): + """Abstract method for filters.""" + raise NotImplementedError + + def pre_analysis(self, wav: torch.Tensor): + """Apply transform before encoder convolution.""" + return wav + + def post_analysis(self, spec: torch.Tensor): + """Apply transform to encoder convolution.""" + return spec + + def pre_synthesis(self, spec: torch.Tensor): + """Apply transform before decoder transposed convolution.""" + return spec + + def post_synthesis(self, wav: torch.Tensor): + """Apply transform after decoder transposed convolution.""" + return wav + + def get_config(self): + """Returns dictionary of arguments to re-instantiate the class. + Needs to be subclassed if the filterbanks takes additional arguments + than ``n_filters`` ``kernel_size`` ``stride`` and ``sample_rate``. + """ + config = { + "fb_name": self.__class__.__name__, + "n_filters": self.n_filters, + "kernel_size": self.kernel_size, + "stride": self.stride, + "sample_rate": self.sample_rate, + } + return config + + def forward(self, waveform): + raise NotImplementedError( + "Filterbanks must be wrapped with an Encoder or a Decoder." + ) + + +class _EncDec(nn.Module): + """Base private class for Encoder and Decoder. + Common parameters and methods. + Args: + filterbank (:class:`Filterbank`): Filterbank instance. The filterbank + to use as an encoder or a decoder. + is_pinv (bool): Whether to be the pseudo inverse of filterbank. + Attributes: + filterbank (:class:`Filterbank`) + stride (int) + is_pinv (bool) + """ + + def __init__(self, filterbank, is_pinv=False): + super(_EncDec, self).__init__() + self.filterbank = filterbank + self.sample_rate = getattr(filterbank, "sample_rate", None) + self.stride = self.filterbank.stride + self.is_pinv = is_pinv + + def filters(self): + return self.filterbank.filters() + + def compute_filter_pinv(self, filters): + """Computes pseudo inverse filterbank of given filters.""" + scale = self.filterbank.stride / self.filterbank.kernel_size + shape = filters.shape + ifilt = torch.pinverse(filters.squeeze()).transpose(-1, -2).view(shape) + # Compensate for the overlap-add. + return ifilt * scale + + def get_filters(self): + """Returns filters or pinv filters depending on `is_pinv` attribute""" + if self.is_pinv: + return self.compute_filter_pinv(self.filters()) + else: + return self.filters() + + def get_config(self): + """Returns dictionary of arguments to re-instantiate the class.""" + config = {"is_pinv": self.is_pinv} + base_config = self.filterbank.get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class Encoder(_EncDec): + r"""Encoder class. + Add encoding methods to Filterbank classes. + Not intended to be subclassed. + Args: + filterbank (:class:`Filterbank`): The filterbank to use + as an encoder. + is_pinv (bool): Whether to be the pseudo inverse of filterbank. + as_conv1d (bool): Whether to behave like nn.Conv1d. + If True (default), forwarding input with shape :math:`(batch, 1, time)` + will output a tensor of shape :math:`(batch, freq, conv\_time)`. + If False, will output a tensor of shape :math:`(batch, 1, freq, conv\_time)`. + padding (int): Zero-padding added to both sides of the input. + """ + + def __init__(self, filterbank, is_pinv=False, as_conv1d=True, padding=0): + super(Encoder, self).__init__(filterbank, is_pinv=is_pinv) + self.as_conv1d = as_conv1d + self.n_feats_out = self.filterbank.n_feats_out + self.kernel_size = self.filterbank.kernel_size + self.padding = padding + + @classmethod + def pinv_of(cls, filterbank, **kwargs): + """Returns an :class:`~.Encoder`, pseudo inverse of a + :class:`~.Filterbank` or :class:`~.Decoder`.""" + if isinstance(filterbank, Filterbank): + return cls(filterbank, is_pinv=True, **kwargs) + elif isinstance(filterbank, Decoder): + return cls(filterbank.filterbank, is_pinv=True, **kwargs) + + def forward(self, waveform): + """Convolve input waveform with the filters from a filterbank. + Args: + waveform (:class:`torch.Tensor`): any tensor with samples along the + last dimension. The waveform representation with and + batch/channel etc.. dimension. + Returns: + :class:`torch.Tensor`: The corresponding TF domain signal. + Shapes + >>> (time, ) -> (freq, conv_time) + >>> (batch, time) -> (batch, freq, conv_time) # Avoid + >>> if as_conv1d: + >>> (batch, 1, time) -> (batch, freq, conv_time) + >>> (batch, chan, time) -> (batch, chan, freq, conv_time) + >>> else: + >>> (batch, chan, time) -> (batch, chan, freq, conv_time) + >>> (batch, any, dim, time) -> (batch, any, dim, freq, conv_time) + """ + filters = self.get_filters() + waveform = self.filterbank.pre_analysis(waveform) + spec = multishape_conv1d( + waveform, + filters=filters, + stride=self.stride, + padding=self.padding, + as_conv1d=self.as_conv1d, + ) + return self.filterbank.post_analysis(spec) + + +def multishape_conv1d( + waveform: torch.Tensor, + filters: torch.Tensor, + stride: int, + padding: int = 0, + as_conv1d: bool = True, +) -> torch.Tensor: + if waveform.ndim == 1: + # Assumes 1D input with shape (time,) + # Output will be (freq, conv_time) + return F.conv1d( + waveform[None, None], filters, stride=stride, padding=padding + ).squeeze() + elif waveform.ndim == 2: + # Assume 2D input with shape (batch or channels, time) + # Output will be (batch or channels, freq, conv_time) + warnings.warn( + "Input tensor was 2D. Applying the corresponding " + "Decoder to the current output will result in a 3D " + "tensor. This behaviours was introduced to match " + "Conv1D and ConvTranspose1D, please use 3D inputs " + "to avoid it. For example, this can be done with " + "input_tensor.unsqueeze(1)." + ) + return F.conv1d(waveform.unsqueeze(1), filters, stride=stride, padding=padding) + elif waveform.ndim == 3: + batch, channels, time_len = waveform.shape + if channels == 1 and as_conv1d: + # That's the common single channel case (batch, 1, time) + # Output will be (batch, freq, stft_time), behaves as Conv1D + return F.conv1d(waveform, filters, stride=stride, padding=padding) + else: + # Return batched convolution, input is (batch, 3, time), output will be + # (b, 3, f, conv_t). Useful for multichannel transforms. If as_conv1d is + # false, (batch, 1, time) will output (batch, 1, freq, conv_time), useful for + # consistency. + return batch_packed_1d_conv( + waveform, filters, stride=stride, padding=padding + ) + else: # waveform.ndim > 3 + # This is to compute "multi"multichannel convolution. + # Input can be (*, time), output will be (*, freq, conv_time) + return batch_packed_1d_conv(waveform, filters, stride=stride, padding=padding) + + +def batch_packed_1d_conv( + inp: torch.Tensor, filters: torch.Tensor, stride: int = 1, padding: int = 0 +): + # Here we perform multichannel / multi-source convolution. + # Output should be (batch, channels, freq, conv_time) + batched_conv = F.conv1d( + inp.view(-1, 1, inp.shape[-1]), filters, stride=stride, padding=padding + ) + output_shape = inp.shape[:-1] + batched_conv.shape[-2:] + return batched_conv.view(output_shape) + + +class Decoder(_EncDec): + """Decoder class. + Add decoding methods to Filterbank classes. + Not intended to be subclassed. + Args: + filterbank (:class:`Filterbank`): The filterbank to use as an decoder. + is_pinv (bool): Whether to be the pseudo inverse of filterbank. + padding (int): Zero-padding added to both sides of the input. + output_padding (int): Additional size added to one side of the + output shape. + .. note:: + ``padding`` and ``output_padding`` arguments are directly passed to + ``F.conv_transpose1d``. + """ + + def __init__(self, filterbank, is_pinv=False, padding=0, output_padding=0): + super().__init__(filterbank, is_pinv=is_pinv) + self.padding = padding + self.output_padding = output_padding + + @classmethod + def pinv_of(cls, filterbank): + """Returns an Decoder, pseudo inverse of a filterbank or Encoder.""" + if isinstance(filterbank, Filterbank): + return cls(filterbank, is_pinv=True) + elif isinstance(filterbank, Encoder): + return cls(filterbank.filterbank, is_pinv=True) + + def forward(self, spec, length: Optional[int] = None) -> torch.Tensor: + """Applies transposed convolution to a TF representation. + This is equivalent to overlap-add. + Args: + spec (:class:`torch.Tensor`): 3D or 4D Tensor. The TF + representation. (Output of :func:`Encoder.forward`). + length: desired output length. + Returns: + :class:`torch.Tensor`: The corresponding time domain signal. + """ + filters = self.get_filters() + spec = self.filterbank.pre_synthesis(spec) + wav = multishape_conv_transpose1d( + spec, + filters, + stride=self.stride, + padding=self.padding, + output_padding=self.output_padding, + ) + wav = self.filterbank.post_synthesis(wav) + if length is not None: + length = min(length, wav.shape[-1]) + return wav[..., :length] + return wav + + +def multishape_conv_transpose1d( + spec: torch.Tensor, + filters: torch.Tensor, + stride: int = 1, + padding: int = 0, + output_padding: int = 0, +) -> torch.Tensor: + if spec.ndim == 2: + # Input is (freq, conv_time), output is (time) + return F.conv_transpose1d( + spec.unsqueeze(0), + filters, + stride=stride, + padding=padding, + output_padding=output_padding, + ).squeeze() + if spec.ndim == 3: + # Input is (batch, freq, conv_time), output is (batch, 1, time) + return F.conv_transpose1d( + spec, + filters, + stride=stride, + padding=padding, + output_padding=output_padding, + ) + else: + # Multiply all the left dimensions together and group them in the + # batch. Make the convolution and restore. + view_as = (-1,) + spec.shape[-2:] + out = F.conv_transpose1d( + spec.reshape(view_as), + filters, + stride=stride, + padding=padding, + output_padding=output_padding, + ) + return out.view(spec.shape[:-2] + (-1,)) + + +class FreeFB(Filterbank): + """Free filterbank without any constraints. Equivalent to + :class:`nn.Conv1d`. + Args: + n_filters (int): Number of filters. + kernel_size (int): Length of the filters. + stride (int, optional): Stride of the convolution. + If None (default), set to ``kernel_size // 2``. + sample_rate (float): Sample rate of the expected audio. + Defaults to 8000. + Attributes: + n_feats_out (int): Number of output filters. + References + [1] : "Filterbank design for end-to-end speech separation". ICASSP 2020. + Manuel Pariente, Samuele Cornell, Antoine Deleforge, Emmanuel Vincent. + """ + + def __init__( + self, n_filters, kernel_size, stride=None, sample_rate=8000.0, **kwargs + ): + super().__init__(n_filters, kernel_size, stride=stride, sample_rate=sample_rate) + self._filters = nn.Parameter(torch.ones(n_filters, 1, kernel_size)) + for p in self.parameters(): + nn.init.xavier_normal_(p) + + def filters(self): + return self._filters + + +free = FreeFB diff --git a/look2hear/layers/normalizations.py b/look2hear/layers/normalizations.py new file mode 100644 index 0000000..e83d95d --- /dev/null +++ b/look2hear/layers/normalizations.py @@ -0,0 +1,183 @@ +import torch +from torch.autograd import Variable +import torch.nn as nn +import numpy as np +from typing import List +from torch.nn.modules.batchnorm import _BatchNorm +from collections.abc import Iterable + + +def norm(x, dims: List[int], EPS: float = 1e-8): + mean = x.mean(dim=dims, keepdim=True) + var2 = torch.var(x, dim=dims, keepdim=True, unbiased=False) + value = (x - mean) / torch.sqrt(var2 + EPS) + return value + + +def glob_norm(x, ESP: float = 1e-8): + dims: List[int] = torch.arange(1, len(x.shape)).tolist() + return norm(x, dims, ESP) + + +class MLayerNorm(nn.Module): + def __init__(self, channel_size): + super().__init__() + self.channel_size = channel_size + self.gamma = nn.Parameter(torch.ones(channel_size), requires_grad=True) + self.beta = nn.Parameter(torch.ones(channel_size), requires_grad=True) + + def apply_gain_and_bias(self, normed_x): + """Assumes input of size `[batch, chanel, *]`.""" + return (self.gamma * normed_x.transpose(1, -1) + self.beta).transpose(1, -1) + + def forward(self, x, EPS: float = 1e-8): + pass + + +class GlobalLN(MLayerNorm): + def forward(self, x, EPS: float = 1e-8): + value = glob_norm(x, EPS) + return self.apply_gain_and_bias(value) + + +class ChannelLN(MLayerNorm): + def forward(self, x, EPS: float = 1e-8): + mean = torch.mean(x, dim=1, keepdim=True) + var = torch.var(x, dim=1, keepdim=True, unbiased=False) + return self.apply_gain_and_bias((x - mean) / (var + EPS).sqrt()) + + +# class CumulateLN(MLayerNorm): +# def forward(self, x, EPS: float = 1e-8): +# batch, channels, time = x.size() +# cum_sum = torch.cumsum(x.sum(1, keepdim=True), dim=1) +# cum_pow_sum = torch.cumsum(x.pow(2).sum(1, keepdim=True), dim=1) +# cnt = torch.arange( +# start=channels, end=channels * (time + 1), step=channels, dtype=x.dtype, device=x.device +# ).view(1, 1, -1) +# cum_mean = cum_sum / cnt +# cum_var = (cum_pow_sum / cnt) - cum_mean.pow(2) +# return self.apply_gain_and_bias((x - cum_mean) / (cum_var + EPS).sqrt()) + + +class BatchNorm(_BatchNorm): + """Wrapper class for pytorch BatchNorm1D and BatchNorm2D""" + + def _check_input_dim(self, input): + if input.dim() < 2 or input.dim() > 4: + raise ValueError( + "expected 4D or 3D input (got {}D input)".format(input.dim()) + ) + + +class CumulativeLayerNorm(nn.LayerNorm): + def __init__(self, dim, elementwise_affine=True): + super(CumulativeLayerNorm, self).__init__( + dim, elementwise_affine=elementwise_affine + ) + + def forward(self, x): + # x: N x C x L + # N x L x C + x = torch.transpose(x, 1, -1) + # N x L x C == only channel norm + x = super().forward(x) + # N x C x L + x = torch.transpose(x, 1, -1) + return x + + +class CumulateLN(nn.Module): + def __init__(self, dimension, eps=1e-8, trainable=True): + super(CumulateLN, self).__init__() + + self.eps = eps + if trainable: + self.gain = nn.Parameter(torch.ones(1, dimension, 1)) + self.bias = nn.Parameter(torch.zeros(1, dimension, 1)) + else: + self.gain = Variable(torch.ones(1, dimension, 1), requires_grad=False) + self.bias = Variable(torch.zeros(1, dimension, 1), requires_grad=False) + + def forward(self, input): + # input size: (Batch, Freq, Time) + # cumulative mean for each time step + + batch_size = input.size(0) + channel = input.size(1) + time_step = input.size(2) + + step_sum = input.sum(1) # B, T + step_pow_sum = input.pow(2).sum(1) # B, T + cum_sum = torch.cumsum(step_sum, dim=1) # B, T + cum_pow_sum = torch.cumsum(step_pow_sum, dim=1) # B, T + + entry_cnt = np.arange(channel, channel * (time_step + 1), channel) + entry_cnt = torch.from_numpy(entry_cnt).type(input.type()) + entry_cnt = entry_cnt.view(1, -1).expand_as(cum_sum) + + cum_mean = cum_sum / entry_cnt # B, T + cum_var = (cum_pow_sum - 2 * cum_mean * cum_sum) / entry_cnt + cum_mean.pow( + 2 + ) # B, T + cum_std = (cum_var + self.eps).sqrt() # B, T + + cum_mean = cum_mean.unsqueeze(1) + cum_std = cum_std.unsqueeze(1) + + x = (input - cum_mean.expand_as(input)) / cum_std.expand_as(input) + return x * self.gain.expand_as(x).type(x.type()) + self.bias.expand_as(x).type( + x.type() + ) + +class LayerNormalization4D(nn.Module): + def __init__(self, input_dimension: Iterable, eps: float = 1e-5): + super(LayerNormalization4D, self).__init__() + assert len(input_dimension) == 2 + param_size = [1, input_dimension[0], 1, input_dimension[1]] + + self.dim = (1, 3) if param_size[-1] > 1 else (1,) + self.gamma = nn.Parameter(torch.Tensor(*param_size).to(torch.float32)) + self.beta = nn.Parameter(torch.Tensor(*param_size).to(torch.float32)) + nn.init.ones_(self.gamma) + nn.init.zeros_(self.beta) + self.eps = eps + + def forward(self, x: torch.Tensor): + mu_ = x.mean(dim=self.dim, keepdim=True) + std_ = torch.sqrt(x.var(dim=self.dim, unbiased=False, keepdim=True) + self.eps) + x_hat = ((x - mu_) / std_) * self.gamma + self.beta + return x_hat + +# Aliases. +gLN = GlobalLN +cLN = CumulateLN +LN = CumulativeLayerNorm +bN = BatchNorm +LN4D = LayerNormalization4D + +def get(identifier): + """Returns a norm class from a string. Returns its input if it + is callable (already a :class:`._LayerNorm` for example). + + Args: + identifier (str or Callable or None): the norm identifier. + + Returns: + :class:`._LayerNorm` or None + """ + if identifier is None: + return None + elif callable(identifier): + return identifier + elif isinstance(identifier, str): + cls = globals().get(identifier) + if cls is None: + raise ValueError( + "Could not interpret normalization identifier: " + str(identifier) + ) + return cls + else: + raise ValueError( + "Could not interpret normalization identifier: " + str(identifier) + ) diff --git a/look2hear/layers/rnnlayers.py b/look2hear/layers/rnnlayers.py new file mode 100644 index 0000000..c657e7b --- /dev/null +++ b/look2hear/layers/rnnlayers.py @@ -0,0 +1,921 @@ +import torch +import math +import inspect +from torch import nn +from torch import Tensor +from typing import Tuple +from typing import Optional +from torch.nn.functional import fold, unfold +import numpy as np + +from . import activations, normalizations +from .normalizations import gLN + + +def has_arg(fn, name): + """Checks if a callable accepts a given keyword argument. + Args: + fn (callable): Callable to inspect. + name (str): Check if ``fn`` can be called with ``name`` as a keyword + argument. + Returns: + bool: whether ``fn`` accepts a ``name`` keyword argument. + """ + signature = inspect.signature(fn) + parameter = signature.parameters.get(name) + if parameter is None: + return False + return parameter.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + + +class SingleRNN(nn.Module): + """Module for a RNN block. + Inspired from https://github.com/yluo42/TAC/blob/master/utility/models.py + Licensed under CC BY-NC-SA 3.0 US. + Args: + rnn_type (str): Select from ``'RNN'``, ``'LSTM'``, ``'GRU'``. Can + also be passed in lowercase letters. + input_size (int): Dimension of the input feature. The input should have + shape [batch, seq_len, input_size]. + hidden_size (int): Dimension of the hidden state. + n_layers (int, optional): Number of layers used in RNN. Default is 1. + dropout (float, optional): Dropout ratio. Default is 0. + bidirectional (bool, optional): Whether the RNN layers are + bidirectional. Default is ``False``. + """ + + def __init__( + self, + rnn_type, + input_size, + hidden_size, + n_layers=1, + dropout=0, + bidirectional=False, + ): + super(SingleRNN, self).__init__() + assert rnn_type.upper() in ["RNN", "LSTM", "GRU"] + rnn_type = rnn_type.upper() + self.rnn_type = rnn_type + self.input_size = input_size + self.hidden_size = hidden_size + self.n_layers = n_layers + self.dropout = dropout + self.bidirectional = bidirectional + self.rnn = getattr(nn, rnn_type)( + input_size, + hidden_size, + num_layers=n_layers, + dropout=dropout, + batch_first=True, + bidirectional=bool(bidirectional), + ) + + @property + def output_size(self): + return self.hidden_size * (2 if self.bidirectional else 1) + + def forward(self, inp): + """ Input shape [batch, seq, feats] """ + self.rnn.flatten_parameters() # Enables faster multi-GPU training. + output = inp + rnn_output, _ = self.rnn(output) + return rnn_output + + +class LSTMBlockTF(nn.Module): + def __init__( + self, + in_chan, + hid_size, + norm_type="gLN", + bidirectional=True, + rnn_type="LSTM", + num_layers=1, + dropout=0, + ): + super(LSTMBlockTF, self).__init__() + self.RNN = SingleRNN( + rnn_type, + in_chan, + hid_size, + num_layers, + dropout=dropout, + bidirectional=bidirectional, + ) + self.linear = nn.Linear(self.RNN.output_size, in_chan) + self.norm = normalizations.get(norm_type)(in_chan) + + def forward(self, x): + B, F, T = x.size() + output = self.RNN(x.transpose(1, 2)) # B, T, N + output = self.linear(output) + output = output.transpose(1, -1) # B, N, T + output = self.norm(output) + return output + x + + +# ===================Transformer====================== +class Linear(nn.Module): + """ + Wrapper class of torch.nn.Linear + Weight initialize by xavier initialization and bias initialize to zeros. + """ + + def __init__(self, in_features: int, out_features: int, bias: bool = True) -> None: + super(Linear, self).__init__() + self.linear = nn.Linear(in_features, out_features, bias=bias) + nn.init.xavier_uniform_(self.linear.weight) + if bias: + nn.init.zeros_(self.linear.bias) + + def forward(self, x): + return self.linear(x) + + +class Swish(nn.Module): + """ + Swish is a smooth, non-monotonic function that consistently matches or outperforms ReLU on deep networks applied + to a variety of challenging domains such as Image classification and Machine translation. + """ + + def __init__(self): + super(Swish, self).__init__() + + def forward(self, inputs): + return inputs * inputs.sigmoid() + + +class Transpose(nn.Module): + """ Wrapper class of torch.transpose() for Sequential module. """ + + def __init__(self, shape: tuple): + super(Transpose, self).__init__() + self.shape = shape + + def forward(self, x: Tensor) -> Tensor: + return x.transpose(*self.shape) + + +class GLU(nn.Module): + """ + The gating mechanism is called Gated Linear Units (GLU), which was first introduced for natural language processing + in the paper “Language Modeling with Gated Convolutional Networks” + """ + + def __init__(self, dim: int) -> None: + super(GLU, self).__init__() + self.dim = dim + + def forward(self, inputs: Tensor) -> Tensor: + outputs, gate = inputs.chunk(2, dim=self.dim) + return outputs * gate.sigmoid() + + +class FeedForwardModule(nn.Module): + def __init__( + self, encoder_dim: int = 512, expansion_factor: int = 4, dropout_p: float = 0.1, + ) -> None: + super(FeedForwardModule, self).__init__() + self.sequential = nn.Sequential( + nn.LayerNorm(encoder_dim), + Linear(encoder_dim, encoder_dim * expansion_factor, bias=True), + Swish(), + nn.Dropout(p=dropout_p), + Linear(encoder_dim * expansion_factor, encoder_dim, bias=True), + nn.Dropout(p=dropout_p), + ) + + def forward(self, inputs): + return self.sequential(inputs) + + +class PositionalEncoding(nn.Module): + """ + Positional Encoding proposed in "Attention Is All You Need". + Since transformer contains no recurrence and no convolution, in order for the model to make + use of the order of the sequence, we must add some positional information. + "Attention Is All You Need" use sine and cosine functions of different frequencies: + PE_(pos, 2i) = sin(pos / power(10000, 2i / d_model)) + PE_(pos, 2i+1) = cos(pos / power(10000, 2i / d_model)) + """ + + def __init__(self, d_model: int = 512, max_len: int = 10000) -> None: + super(PositionalEncoding, self).__init__() + pe = torch.zeros(max_len, d_model, requires_grad=False) + position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div_term = torch.exp( + torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model) + ) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + self.register_buffer("pe", pe) + + def forward(self, length: int) -> Tensor: + return self.pe[:, :length] + + +class RelativeMultiHeadAttention(nn.Module): + """ + Multi-head attention with relative positional encoding. + This concept was proposed in the "Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context" + Args: + d_model (int): The dimension of model + num_heads (int): The number of attention heads. + dropout_p (float): probability of dropout + Inputs: query, key, value, pos_embedding, mask + - **query** (batch, time, dim): Tensor containing query vector + - **key** (batch, time, dim): Tensor containing key vector + - **value** (batch, time, dim): Tensor containing value vector + - **pos_embedding** (batch, time, dim): Positional embedding tensor + - **mask** (batch, 1, time2) or (batch, time1, time2): Tensor containing indices to be masked + Returns: + - **outputs**: Tensor produces by relative multi head attention module. + """ + + def __init__( + self, d_model: int = 512, num_heads: int = 16, dropout_p: float = 0.1, + ): + super(RelativeMultiHeadAttention, self).__init__() + assert d_model % num_heads == 0, "d_model % num_heads should be zero." + self.d_model = d_model + self.d_head = int(d_model / num_heads) + self.num_heads = num_heads + self.sqrt_dim = math.sqrt(d_model) + + self.query_proj = Linear(d_model, d_model) + self.key_proj = Linear(d_model, d_model) + self.value_proj = Linear(d_model, d_model) + self.pos_proj = Linear(d_model, d_model, bias=False) + + self.dropout = nn.Dropout(p=dropout_p) + self.u_bias = nn.Parameter(torch.Tensor(self.num_heads, self.d_head)) + self.v_bias = nn.Parameter(torch.Tensor(self.num_heads, self.d_head)) + torch.nn.init.xavier_uniform_(self.u_bias) + torch.nn.init.xavier_uniform_(self.v_bias) + + self.out_proj = Linear(d_model, d_model) + + def forward( + self, + query: Tensor, + key: Tensor, + value: Tensor, + pos_embedding: Tensor, + mask: Optional[Tensor] = None, + ) -> Tensor: + batch_size = value.size(0) + + query = self.query_proj(query).view(batch_size, -1, self.num_heads, self.d_head) + key = ( + self.key_proj(key) + .view(batch_size, -1, self.num_heads, self.d_head) + .permute(0, 2, 1, 3) + ) + value = ( + self.value_proj(value) + .view(batch_size, -1, self.num_heads, self.d_head) + .permute(0, 2, 1, 3) + ) + pos_embedding = self.pos_proj(pos_embedding).view( + batch_size, -1, self.num_heads, self.d_head + ) + + content_score = torch.matmul( + (query + self.u_bias).transpose(1, 2), key.transpose(2, 3) + ) + pos_score = torch.matmul( + (query + self.v_bias).transpose(1, 2), pos_embedding.permute(0, 2, 3, 1) + ) + pos_score = self._relative_shift(pos_score) + + score = (content_score + pos_score) / self.sqrt_dim + + if mask is not None: + mask = mask.unsqueeze(1) + score.masked_fill_(mask, -1e9) + + attn = torch.nn.functional.softmax(score, -1) + attn = self.dropout(attn) + + context = torch.matmul(attn, value).transpose(1, 2) + context = context.contiguous().view(batch_size, -1, self.d_model) + + return self.out_proj(context) + + def _relative_shift(self, pos_score: Tensor) -> Tensor: + batch_size, num_heads, seq_length1, seq_length2 = pos_score.size() + zeros = pos_score.new_zeros(batch_size, num_heads, seq_length1, 1) + padded_pos_score = torch.cat([zeros, pos_score], dim=-1) + + padded_pos_score = padded_pos_score.view( + batch_size, num_heads, seq_length2 + 1, seq_length1 + ) + pos_score = padded_pos_score[:, :, 1:].view_as(pos_score) + + return pos_score + + +class MultiHeadedSelfAttentionModule(nn.Module): + """ + Conformer employ multi-headed self-attention (MHSA) while integrating an important technique from Transformer-XL, + the relative sinusoidal positional encoding scheme. The relative positional encoding allows the self-attention + module to generalize better on different input length and the resulting encoder is more robust to the variance of + the utterance length. Conformer use prenorm residual units with dropout which helps training + and regularizing deeper models. + Args: + d_model (int): The dimension of model + num_heads (int): The number of attention heads. + dropout_p (float): probability of dropout + device (torch.device): torch device (cuda or cpu) + Inputs: inputs, mask + - **inputs** (batch, time, dim): Tensor containing input vector + - **mask** (batch, 1, time2) or (batch, time1, time2): Tensor containing indices to be masked + Returns: + - **outputs** (batch, time, dim): Tensor produces by relative multi headed self attention module. + """ + + def __init__( + self, d_model: int, num_heads: int, dropout_p: float = 0.1, is_casual=True + ): + super(MultiHeadedSelfAttentionModule, self).__init__() + self.positional_encoding = PositionalEncoding(d_model) + self.layer_norm = nn.LayerNorm(d_model) + self.attention = RelativeMultiHeadAttention(d_model, num_heads, dropout_p) + self.dropout = nn.Dropout(p=dropout_p) + self.is_casual = is_casual + + def forward(self, inputs: Tensor): + batch_size, seq_length, _ = inputs.size() + pos_embedding = self.positional_encoding(seq_length) + pos_embedding = pos_embedding.repeat(batch_size, 1, 1) + + mask = None + if self.is_casual: + mask = torch.triu( + torch.ones((seq_length, seq_length), dtype=torch.uint8).to( + inputs.device + ), + diagonal=1, + ) + mask = mask.unsqueeze(0).expand(batch_size, -1, -1).bool() # [B, L, L] + + inputs = self.layer_norm(inputs) + outputs = self.attention( + inputs, inputs, inputs, pos_embedding=pos_embedding, mask=mask + ) + + return self.dropout(outputs) + + +class ResidualConnectionModule(nn.Module): + """ + Residual Connection Module. + outputs = (module(inputs) x module_factor + inputs x input_factor) + """ + + def __init__( + self, module: nn.Module, module_factor: float = 1.0, input_factor: float = 1.0 + ): + super(ResidualConnectionModule, self).__init__() + self.module = module + self.module_factor = module_factor + self.input_factor = input_factor + + def forward(self, inputs): + return (self.module(inputs) * self.module_factor) + (inputs * self.input_factor) + + +class DepthwiseConv1d(nn.Module): + """ + When groups == in_channels and out_channels == K * in_channels, where K is a positive integer, + this operation is termed in literature as depthwise convolution. + Args: + in_channels (int): Number of channels in the input + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + bias (bool, optional): If True, adds a learnable bias to the output. Default: True + Inputs: inputs + - **inputs** (batch, in_channels, time): Tensor containing input vector + Returns: outputs + - **outputs** (batch, out_channels, time): Tensor produces by depthwise 1-D convolution. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + padding: int = 0, + bias: bool = False, + is_casual: bool = True, + ) -> None: + super(DepthwiseConv1d, self).__init__() + assert ( + out_channels % in_channels == 0 + ), "out_channels should be constant multiple of in_channels" + if is_casual: + padding = kernel_size - 1 + else: + padding = (kernel_size - 1) // 2 + self.conv = nn.Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + groups=in_channels, + stride=stride, + padding=padding, + bias=bias, + ) + self.is_casual = is_casual + self.kernel_size = kernel_size + + def forward(self, inputs: Tensor) -> Tensor: + if self.is_casual: + return self.conv(inputs)[:, :, : -(self.kernel_size - 1)] + return self.conv(inputs) + + +class PointwiseConv1d(nn.Module): + """ + When kernel size == 1 conv1d, this operation is termed in literature as pointwise convolution. + This operation often used to match dimensions. + Args: + in_channels (int): Number of channels in the input + out_channels (int): Number of channels produced by the convolution + stride (int, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + bias (bool, optional): If True, adds a learnable bias to the output. Default: True + Inputs: inputs + - **inputs** (batch, in_channels, time): Tensor containing input vector + Returns: outputs + - **outputs** (batch, out_channels, time): Tensor produces by pointwise 1-D convolution. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + stride: int = 1, + padding: int = 0, + bias: bool = True, + ) -> None: + super(PointwiseConv1d, self).__init__() + self.conv = nn.Conv1d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=1, + stride=stride, + padding=padding, + bias=bias, + ) + + def forward(self, inputs: Tensor) -> Tensor: + return self.conv(inputs) + + +class ConformerConvModule(nn.Module): + """ + Conformer convolution module starts with a pointwise convolution and a gated linear unit (GLU). + This is followed by a single 1-D depthwise convolution layer. Batchnorm is deployed just after the convolution + to aid training deep models. + Args: + in_channels (int): Number of channels in the input + kernel_size (int or tuple, optional): Size of the convolving kernel Default: 31 + dropout_p (float, optional): probability of dropout + device (torch.device): torch device (cuda or cpu) + Inputs: inputs + inputs (batch, time, dim): Tensor contains input sequences + Outputs: outputs + outputs (batch, time, dim): Tensor produces by conformer convolution module. + """ + + def __init__( + self, + in_channels: int, + kernel_size: int = 31, + expansion_factor: int = 2, + dropout_p: float = 0.1, + is_casual: bool = True, + ) -> None: + super(ConformerConvModule, self).__init__() + assert ( + kernel_size - 1 + ) % 2 == 0, "kernel_size should be a odd number for 'SAME' padding" + assert expansion_factor == 2, "Currently, Only Supports expansion_factor 2" + + self.sequential = nn.Sequential( + nn.LayerNorm(in_channels), + Transpose(shape=(1, 2)), + PointwiseConv1d( + in_channels, + in_channels * expansion_factor, + stride=1, + padding=0, + bias=True, + ), + GLU(dim=1), + DepthwiseConv1d( + in_channels, in_channels, kernel_size, stride=1, is_casual=is_casual + ), + nn.BatchNorm1d(in_channels), + Swish(), + PointwiseConv1d(in_channels, in_channels, stride=1, padding=0, bias=True), + nn.Dropout(p=dropout_p), + ) + + def forward(self, inputs: Tensor) -> Tensor: + return self.sequential(inputs).transpose(1, 2) + + +class TransformerLayer(nn.Module): + def __init__( + self, in_chan=128, n_head=8, n_att=1, dropout=0.1, max_len=500, is_casual=True + ): + super(TransformerLayer, self).__init__() + self.in_chan = in_chan + self.n_head = n_head + self.dropout = dropout + self.max_len = max_len + self.n_att = n_att + + self.seq = nn.Sequential( + ResidualConnectionModule( + FeedForwardModule(in_chan, expansion_factor=4, dropout_p=dropout), + module_factor=0.5, + ), + ResidualConnectionModule( + MultiHeadedSelfAttentionModule(in_chan, n_head, dropout, is_casual) + ), + ResidualConnectionModule( + ConformerConvModule(in_chan, 31, 2, dropout, is_casual=is_casual) + ), + ResidualConnectionModule( + FeedForwardModule(in_chan, expansion_factor=4, dropout_p=dropout), + module_factor=0.5, + ), + nn.LayerNorm(in_chan), + ) + + def forward(self, x): + return self.seq(x) + + +class TransformerBlockTF(nn.Module): + def __init__( + self, + in_chan, + n_head=8, + n_att=1, + dropout=0.1, + max_len=500, + norm_type="cLN", + is_casual=True, + ): + super(TransformerBlockTF, self).__init__() + self.transformer = TransformerLayer( + in_chan, n_head, n_att, dropout, max_len, is_casual + ) + self.norm = normalizations.get(norm_type)(in_chan) + + def forward(self, x): + B, F, T = x.size() + output = self.transformer(x.permute(0, 2, 1).contiguous()) # B, T, N + output = output.permute(0, 2, 1).contiguous() # B, N, T + output = self.norm(output) + return output + x + + +# ==================================================== + + +class DPRNNBlock(nn.Module): + def __init__( + self, + in_chan, + hid_size, + norm_type="gLN", + bidirectional=True, + rnn_type="LSTM", + num_layers=1, + dropout=0, + ): + super(DPRNNBlock, self).__init__() + self.intra_RNN = SingleRNN( + rnn_type, + in_chan, + hid_size, + num_layers, + dropout=dropout, + bidirectional=True, + ) + self.inter_RNN = SingleRNN( + rnn_type, + in_chan, + hid_size, + num_layers, + dropout=dropout, + bidirectional=bidirectional, + ) + self.intra_linear = nn.Linear(self.intra_RNN.output_size, in_chan) + self.intra_norm = normalizations.get(norm_type)(in_chan) + + self.inter_linear = nn.Linear(self.inter_RNN.output_size, in_chan) + self.inter_norm = normalizations.get(norm_type)(in_chan) + + def forward(self, x): + """ Input shape : [batch, feats, chunk_size, num_chunks] """ + B, N, K, L = x.size() + output = x # for skip connection + # Intra-chunk processing + x = x.transpose(1, -1).reshape(B * L, K, N) + x = self.intra_RNN(x) + x = self.intra_linear(x) + x = x.reshape(B, L, K, N).transpose(1, -1) + x = self.intra_norm(x) + output = output + x + # Inter-chunk processing + x = output.transpose(1, 2).transpose(2, -1).reshape(B * K, L, N) + x = self.inter_RNN(x) + x = self.inter_linear(x) + x = x.reshape(B, K, L, N).transpose(1, -1).transpose(2, -1).contiguous() + x = self.inter_norm(x) + return output + x + + +class DPRNN(nn.Module): + def __init__( + self, + in_chan, + n_src, + out_chan=None, + bn_chan=128, + hid_size=128, + chunk_size=100, + hop_size=None, + n_repeats=6, + norm_type="gLN", + mask_act="relu", + bidirectional=True, + rnn_type="LSTM", + num_layers=1, + dropout=0, + ): + super(DPRNN, self).__init__() + self.in_chan = in_chan + out_chan = out_chan if out_chan is not None else in_chan + self.out_chan = out_chan + self.bn_chan = bn_chan + self.hid_size = hid_size + self.chunk_size = chunk_size + hop_size = hop_size if hop_size is not None else chunk_size // 2 + self.hop_size = hop_size + self.n_repeats = n_repeats + self.n_src = n_src + self.norm_type = norm_type + self.mask_act = mask_act + self.bidirectional = bidirectional + self.rnn_type = rnn_type + self.num_layers = num_layers + self.dropout = dropout + + layer_norm = normalizations.get(norm_type)(in_chan) + bottleneck_conv = nn.Conv1d(in_chan, bn_chan, 1) + self.bottleneck = nn.Sequential(layer_norm, bottleneck_conv) + + # Succession of DPRNNBlocks. + net = [] + for x in range(self.n_repeats): + net += [ + DPRNNBlock( + bn_chan, + hid_size, + norm_type=norm_type, + bidirectional=bidirectional, + rnn_type=rnn_type, + num_layers=num_layers, + dropout=dropout, + ) + ] + self.net = nn.Sequential(*net) + # Masking in 3D space + net_out_conv = nn.Conv2d(bn_chan, n_src * bn_chan, 1) + self.first_out = nn.Sequential(nn.PReLU(), net_out_conv) + # Gating and masking in 2D space (after fold) + self.net_out = nn.Sequential(nn.Conv1d(bn_chan, bn_chan, 1), nn.Tanh()) + self.net_gate = nn.Sequential(nn.Conv1d(bn_chan, bn_chan, 1), nn.Sigmoid()) + self.mask_net = nn.Conv1d(bn_chan, out_chan, 1, bias=False) + + # Get activation function. + mask_nl_class = activations.get(mask_act) + # For softmax, feed the source dimension. + if has_arg(mask_nl_class, "dim"): + self.output_act = mask_nl_class(dim=1) + else: + self.output_act = mask_nl_class() + + def forward(self, mixture_w): + r"""Forward. + Args: + mixture_w (:class:`torch.Tensor`): Tensor of shape $(batch, nfilters, nframes)$ + Returns: + :class:`torch.Tensor`: estimated mask of shape $(batch, nsrc, nfilters, nframes)$ + """ + batch, n_filters, n_frames = mixture_w.size() + output = self.bottleneck(mixture_w) # [batch, bn_chan, n_frames] + output = unfold( + output.unsqueeze(-1), + kernel_size=(self.chunk_size, 1), + padding=(self.chunk_size, 0), + stride=(self.hop_size, 1), + ) + n_chunks = output.shape[-1] + output = output.reshape(batch, self.bn_chan, self.chunk_size, n_chunks) + # Apply stacked DPRNN Blocks sequentially + output = self.net(output) + # Map to sources with kind of 2D masks + output = self.first_out(output) + output = output.reshape( + batch * self.n_src, self.bn_chan, self.chunk_size, n_chunks + ) + # Overlap and add: + # [batch, out_chan, chunk_size, n_chunks] -> [batch, out_chan, n_frames] + to_unfold = self.bn_chan * self.chunk_size + output = fold( + output.reshape(batch * self.n_src, to_unfold, n_chunks), + (n_frames, 1), + kernel_size=(self.chunk_size, 1), + padding=(self.chunk_size, 0), + stride=(self.hop_size, 1), + ) + # Apply gating + output = output.reshape(batch * self.n_src, self.bn_chan, -1) + # output = self.net_out(output) * self.net_gate(output) + # Compute mask + score = self.mask_net(output) + est_mask = self.output_act(score) + est_mask = est_mask.view(batch, self.n_src, self.out_chan, n_frames) + return est_mask + + def get_config(self): + config = { + "in_chan": self.in_chan, + "out_chan": self.out_chan, + "bn_chan": self.bn_chan, + "hid_size": self.hid_size, + "chunk_size": self.chunk_size, + "hop_size": self.hop_size, + "n_repeats": self.n_repeats, + "n_src": self.n_src, + "norm_type": self.norm_type, + "mask_act": self.mask_act, + "bidirectional": self.bidirectional, + "rnn_type": self.rnn_type, + "num_layers": self.num_layers, + "dropout": self.dropout, + } + return config + + +class DPRNNLinear(nn.Module): + def __init__( + self, + in_chan, + n_src, + out_chan=None, + bn_chan=128, + hid_size=128, + chunk_size=100, + hop_size=None, + n_repeats=6, + norm_type="gLN", + mask_act="relu", + bidirectional=True, + rnn_type="LSTM", + num_layers=1, + dropout=0, + ): + super(DPRNNLinear, self).__init__() + self.in_chan = in_chan + out_chan = out_chan if out_chan is not None else in_chan + self.out_chan = out_chan + self.bn_chan = bn_chan + self.hid_size = hid_size + self.chunk_size = chunk_size + hop_size = hop_size if hop_size is not None else chunk_size // 2 + self.hop_size = hop_size + self.n_repeats = n_repeats + self.n_src = n_src + self.norm_type = norm_type + self.mask_act = mask_act + self.bidirectional = bidirectional + self.rnn_type = rnn_type + self.num_layers = num_layers + self.dropout = dropout + + layer_norm = normalizations.get(norm_type)(in_chan) + bottleneck_conv = nn.Conv1d(in_chan, bn_chan, 1) + self.bottleneck = nn.Sequential(layer_norm, bottleneck_conv) + + # Succession of DPRNNBlocks. + net = [] + for x in range(self.n_repeats): + net += [ + DPRNNBlock( + bn_chan, + hid_size, + norm_type=norm_type, + bidirectional=bidirectional, + rnn_type=rnn_type, + num_layers=num_layers, + dropout=dropout, + ) + ] + self.net = nn.Sequential(*net) + # Masking in 3D space + net_out_conv = nn.Conv2d(bn_chan, n_src * bn_chan, 1) + self.first_out = nn.Sequential(nn.PReLU(), net_out_conv) + # Gating and masking in 2D space (after fold) + # self.net_out = nn.Sequential(nn.Conv1d(bn_chan, bn_chan, 1), nn.Tanh()) + self.net_out = nn.Linear(bn_chan, out_chan) + self.net_gate = nn.Sequential(nn.Conv1d(bn_chan, bn_chan, 1), nn.Sigmoid()) + self.mask_net = nn.Conv1d(bn_chan, out_chan, 1, bias=False) + + # Get activation function. + mask_nl_class = activations.get(mask_act) + # For softmax, feed the source dimension. + if has_arg(mask_nl_class, "dim"): + self.output_act = mask_nl_class(dim=1) + else: + self.output_act = mask_nl_class() + + def forward(self, mixture_w): + r"""Forward. + Args: + mixture_w (:class:`torch.Tensor`): Tensor of shape $(batch, nfilters, nframes)$ + Returns: + :class:`torch.Tensor`: estimated mask of shape $(batch, nsrc, nfilters, nframes)$ + """ + batch, n_filters, n_frames = mixture_w.size() + output = self.bottleneck(mixture_w) # [batch, bn_chan, n_frames] + output = unfold( + output.unsqueeze(-1), + kernel_size=(self.chunk_size, 1), + padding=(self.chunk_size, 0), + stride=(self.hop_size, 1), + ) + n_chunks = output.shape[-1] + output = output.reshape(batch, self.bn_chan, self.chunk_size, n_chunks) + # Apply stacked DPRNN Blocks sequentially + output = self.net(output) + # Map to sources with kind of 2D masks + output = self.first_out(output) + output = output.reshape( + batch * self.n_src, self.bn_chan, self.chunk_size, n_chunks + ) + # Overlap and add: + # [batch, out_chan, chunk_size, n_chunks] -> [batch, out_chan, n_frames] + to_unfold = self.bn_chan * self.chunk_size + output = fold( + output.reshape(batch * self.n_src, to_unfold, n_chunks), + (n_frames, 1), + kernel_size=(self.chunk_size, 1), + padding=(self.chunk_size, 0), + stride=(self.hop_size, 1), + ) + # Apply gating + output = output.reshape(batch * self.n_src, self.bn_chan, -1) + output = self.net_out(output.transpose(1, 1)).transpose(1, 2) * self.net_gate( + output + ) + # Compute mask + score = self.mask_net(output) + est_mask = self.output_act(score) + est_mask = est_mask.view(batch, self.n_src, self.out_chan, n_frames) + return est_mask + + def get_config(self): + config = { + "in_chan": self.in_chan, + "out_chan": self.out_chan, + "bn_chan": self.bn_chan, + "hid_size": self.hid_size, + "chunk_size": self.chunk_size, + "hop_size": self.hop_size, + "n_repeats": self.n_repeats, + "n_src": self.n_src, + "norm_type": self.norm_type, + "mask_act": self.mask_act, + "bidirectional": self.bidirectional, + "rnn_type": self.rnn_type, + "num_layers": self.num_layers, + "dropout": self.dropout, + } + return config diff --git a/look2hear/layers/stft.py b/look2hear/layers/stft.py new file mode 100644 index 0000000..0bef618 --- /dev/null +++ b/look2hear/layers/stft.py @@ -0,0 +1,797 @@ +# Copyright 2019 Jian Wu +# License: Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +import math + +import numpy as np +import torch as th +import torch.nn as nn +import torch.nn.functional as tf +import librosa.filters as filters + +from typing import Optional, Tuple +from distutils.version import LooseVersion + +EPSILON = float(np.finfo(np.float32).eps) +TORCH_VERSION = th.__version__ + +if TORCH_VERSION >= LooseVersion("1.7"): + from torch.fft import fft as fft_func +else: + pass + + +def export_jit(transform: nn.Module) -> nn.Module: + """ + Export transform module for inference + """ + export_out = [module for module in transform if module.exportable()] + return nn.Sequential(*export_out) + + +def init_window(wnd: str, frame_len: int, device: th.device = "cpu") -> th.Tensor: + """ + Return window coefficient + Args: + wnd: window name + frame_len: length of the frame + """ + + def sqrthann(frame_len, periodic=True): + return th.hann_window(frame_len, periodic=periodic) ** 0.5 + + if wnd not in ["bartlett", "hann", "hamm", "blackman", "rect", "sqrthann"]: + raise RuntimeError(f"Unknown window type: {wnd}") + + wnd_tpl = { + "sqrthann": sqrthann, + "hann": th.hann_window, + "hamm": th.hamming_window, + "blackman": th.blackman_window, + "bartlett": th.bartlett_window, + "rect": th.ones, + } + if wnd != "rect": + # match with librosa + c = wnd_tpl[wnd](frame_len, periodic=True) + else: + c = wnd_tpl[wnd](frame_len) + return c.to(device) + + +def init_kernel( + frame_len: int, + frame_hop: int, + window: th.Tensor, + round_pow_of_two: bool = True, + normalized: bool = False, + inverse: bool = False, + mode: str = "librosa", +) -> Tuple[th.Tensor, th.Tensor]: + """ + Return STFT kernels + Args: + frame_len: length of the frame + frame_hop: hop size between frames + window: window tensor + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + normalized: return normalized DFT matrix + inverse: return iDFT matrix + mode: framing mode (librosa or kaldi) + """ + if mode not in ["librosa", "kaldi"]: + raise ValueError(f"Unsupported mode: {mode}") + # FFT size: B + if round_pow_of_two or mode == "kaldi": + fft_size = 2 ** math.ceil(math.log2(frame_len)) + else: + fft_size = frame_len + # center padding window if needed + if mode == "librosa" and fft_size != frame_len: + lpad = (fft_size - frame_len) // 2 + window = tf.pad(window, (lpad, fft_size - frame_len - lpad)) + if normalized: + # make K^H * K = I + S = fft_size ** 0.5 + else: + S = 1 + # W x B x 2 + if TORCH_VERSION >= LooseVersion("1.7"): + K = fft_func(th.eye(fft_size) / S, dim=-1) + K = th.stack([K.real, K.imag], dim=-1) + else: + I = th.stack([th.eye(fft_size), th.zeros(fft_size, fft_size)], dim=-1) + K = th.fft(I / S, 1) + if mode == "kaldi": + K = K[:frame_len] + if inverse and not normalized: + # to make K^H * K = I + K = K / fft_size + # 2 x B x W + K = th.transpose(K, 0, 2) + # 2B x 1 x W + K = th.reshape(K, (fft_size * 2, 1, K.shape[-1])) + return K.to(window.device), window + + +def mel_filter( + frame_len: int, + round_pow_of_two: bool = True, + num_bins: Optional[int] = None, + sr: int = 16000, + num_mels: int = 80, + fmin: float = 0.0, + fmax: Optional[float] = None, + norm: bool = False, +) -> th.Tensor: + """ + Return mel filter coefficients + Args: + frame_len: length of the frame + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + num_bins: number of the frequency bins produced by STFT + num_mels: number of the mel bands + fmin: lowest frequency (in Hz) + fmax: highest frequency (in Hz) + norm: normalize the mel filter coefficients + """ + # FFT points + if num_bins is None: + N = 2 ** math.ceil(math.log2(frame_len)) if round_pow_of_two else frame_len + else: + N = (num_bins - 1) * 2 + # fmin & fmax + freq_upper = sr // 2 + if fmax is None: + fmax = freq_upper + else: + fmax = min(fmax + freq_upper if fmax < 0 else fmax, freq_upper) + fmin = max(0, fmin) + # mel filter coefficients + mel = filters.mel( + sr, + N, + n_mels=num_mels, + fmax=fmax, + fmin=fmin, + htk=True, + norm="slaney" if norm else None, + ) + # num_mels x (N // 2 + 1) + return th.tensor(mel, dtype=th.float32) + + +def speed_perturb_filter( + src_sr: int, dst_sr: int, cutoff_ratio: float = 0.95, num_zeros: int = 64 +) -> th.Tensor: + """ + Return speed perturb filters, reference: + https://github.com/danpovey/filtering/blob/master/lilfilter/resampler.py + Args: + src_sr: sample rate of the source signal + dst_sr: sample rate of the target signal + Return: + weight (Tensor): coefficients of the filter + """ + if src_sr == dst_sr: + raise ValueError(f"src_sr should not be equal to dst_sr: {src_sr}/{dst_sr}") + gcd = math.gcd(src_sr, dst_sr) + src_sr = src_sr // gcd + dst_sr = dst_sr // gcd + if src_sr == 1 or dst_sr == 1: + raise ValueError("do not support integer downsample/upsample") + zeros_per_block = min(src_sr, dst_sr) * cutoff_ratio + padding = 1 + int(num_zeros / zeros_per_block) + # dst_sr x src_sr x K + times = ( + np.arange(dst_sr)[:, None, None] / float(dst_sr) + - np.arange(src_sr)[None, :, None] / float(src_sr) + - np.arange(2 * padding + 1)[None, None, :] + + padding + ) + window = np.heaviside(1 - np.abs(times / padding), 0.0) * ( + 0.5 + 0.5 * np.cos(times / padding * math.pi) + ) + weight = np.sinc(times * zeros_per_block) * window * zeros_per_block / float(src_sr) + return th.tensor(weight, dtype=th.float32) + + +def splice_feature( + feats: th.Tensor, lctx: int = 1, rctx: int = 1, op: str = "cat" +) -> th.Tensor: + """ + Splice feature + Args: + feats (Tensor): N x ... x T x F, original feature + lctx: left context + rctx: right context + op: operator on feature context + Return: + splice (Tensor): feature with context padded + """ + if lctx + rctx == 0: + return feats + if op not in ["cat", "stack"]: + raise ValueError(f"Unknown op for feature splicing: {op}") + # [N x ... x T x F, ...] + ctx = [] + T = feats.shape[-2] + for c in range(-lctx, rctx + 1): + idx = th.arange(c, c + T, device=feats.device, dtype=th.int64) + idx = th.clamp(idx, min=0, max=T - 1) + ctx.append(th.index_select(feats, -2, idx)) + if op == "cat": + # N x ... x T x FD + splice = th.cat(ctx, -1) + else: + # N x ... x T x F x D + splice = th.stack(ctx, -1) + return splice + + +def _forward_stft( + wav: th.Tensor, + kernel: th.Tensor, + window: th.Tensor, + return_polar: bool = False, + pre_emphasis: float = 0, + frame_hop: int = 256, + onesided: bool = False, + center: bool = False, + eps: float = EPSILON, +) -> th.Tensor: + """ + STFT function implemented by conv1d (not efficient, but we don't care during training) + Args: + wav (Tensor): N x (C) x S + kernel (Tensor): STFT transform kernels, from init_kernel(...) + return_polar: return [magnitude; phase] Tensor or [real; imag] Tensor + pre_emphasis: factor of preemphasis + frame_hop: frame hop size in number samples + onesided: return half FFT bins + center: if true, we assumed to have centered frames + Return: + transform (Tensor): STFT transform results + """ + wav_dim = wav.dim() + if wav_dim not in [2, 3]: + raise RuntimeError(f"STFT expect 2D/3D tensor, but got {wav_dim:d}D") + # if N x S, reshape N x 1 x S + # else: reshape NC x 1 x S + N, S = wav.shape[0], wav.shape[-1] + wav = wav.view(-1, 1, S) + # NC x 1 x S+2P + if center: + pad = kernel.shape[-1] // 2 + # NOTE: match with librosa + wav = tf.pad(wav, (pad, pad), mode="reflect") + # STFT + kernel = kernel * window + if pre_emphasis > 0: + # NC x W x T + frames = tf.unfold( + wav[:, None], (1, kernel.shape[-1]), stride=frame_hop, padding=0 + ) + # follow Kaldi's Preemphasize + frames[:, 1:] = frames[:, 1:] - pre_emphasis * frames[:, :-1] + frames[:, 0] *= 1 - pre_emphasis + # 1 x 2B x W, NC x W x T, NC x 2B x T + packed = th.matmul(kernel[:, 0][None, ...], frames) + else: + packed = tf.conv1d(wav, kernel, stride=frame_hop, padding=0) + # NC x 2B x T => N x C x 2B x T + if wav_dim == 3: + packed = packed.view(N, -1, packed.shape[-2], packed.shape[-1]) + # N x (C) x B x T + real, imag = th.chunk(packed, 2, dim=-2) + # N x (C) x B/2+1 x T + if onesided: + num_bins = kernel.shape[0] // 4 + 1 + real = real[..., :num_bins, :] + imag = imag[..., :num_bins, :] + if return_polar: + mag = (real ** 2 + imag ** 2 + eps) ** 0.5 + pha = th.atan2(imag, real) + return th.stack([mag, pha], dim=-1) + else: + return th.stack([real, imag], dim=-1) + + +def _inverse_stft( + transform: th.Tensor, + kernel: th.Tensor, + window: th.Tensor, + return_polar: bool = False, + frame_hop: int = 256, + onesided: bool = False, + center: bool = False, + eps: float = EPSILON, +) -> th.Tensor: + """ + iSTFT function implemented by conv1d + Args: + transform (Tensor): STFT transform results + kernel (Tensor): STFT transform kernels, from init_kernel(...) + return_polar (bool): keep same with the one in _forward_stft + frame_hop: frame hop size in number samples + onesided: return half FFT bins + center: used in _forward_stft + Return: + wav (Tensor), N x S + """ + # (N) x F x T x 2 + transform_dim = transform.dim() + # if F x T x 2, reshape 1 x F x T x 2 + if transform_dim == 3: + transform = th.unsqueeze(transform, 0) + if transform_dim != 4: + raise RuntimeError(f"Expect 4D tensor, but got {transform_dim}D") + + if return_polar: + real = transform[..., 0] * th.cos(transform[..., 1]) + imag = transform[..., 0] * th.sin(transform[..., 1]) + else: + real, imag = transform[..., 0], transform[..., 1] + + if onesided: + # [self.num_bins - 2, ..., 1] + reverse = range(kernel.shape[0] // 4 - 1, 0, -1) + # extend matrix: N x B x T + real = th.cat([real, real[:, reverse]], 1) + imag = th.cat([imag, -imag[:, reverse]], 1) + # pack: N x 2B x T + packed = th.cat([real, imag], dim=1) + # N x 1 x T + wav = tf.conv_transpose1d(packed, kernel * window, stride=frame_hop, padding=0) + # normalized audio samples + # refer: https://github.com/pytorch/audio/blob/2ebbbf511fb1e6c47b59fd32ad7e66023fa0dff1/torchaudio/functional.py#L171 + num_frames = packed.shape[-1] + win_length = window.shape[0] + # W x T + win = th.repeat_interleave(window[..., None] ** 2, num_frames, dim=-1) + # Do OLA on windows + # v1) + I = th.eye(win_length, device=win.device)[:, None] + denorm = tf.conv_transpose1d(win[None, ...], I, stride=frame_hop, padding=0) + # v2) + # num_samples = (num_frames - 1) * frame_hop + win_length + # denorm = tf.fold(win[None, ...], (num_samples, 1), (win_length, 1), + # stride=frame_hop)[..., 0] + if center: + pad = kernel.shape[-1] // 2 + wav = wav[..., pad:-pad] + denorm = denorm[..., pad:-pad] + wav = wav / (denorm + eps) + # N x S + return wav.squeeze(1) + + +def _pytorch_stft( + wav: th.Tensor, + frame_len: int, + frame_hop: int, + n_fft: int = 512, + return_polar: bool = False, + window: str = "sqrthann", + normalized: bool = False, + onesided: bool = True, + center: bool = False, + eps: float = EPSILON, +) -> th.Tensor: + """ + Wrapper of PyTorch STFT function + Args: + wav (Tensor): source audio signal + frame_len: length of the frame + frame_hop: hop size between frames + n_fft: number of the FFT size + return_polar: return the results in polar coordinate + window: window tensor + center: same definition with the parameter in librosa.stft + normalized: use normalized DFT kernel + onesided: output onesided STFT + Return: + transform (Tensor), STFT transform results + """ + if TORCH_VERSION < LooseVersion("1.7"): + raise RuntimeError("Can not use this function as TORCH_VERSION < 1.7") + wav_dim = wav.dim() + if wav_dim not in [2, 3]: + raise RuntimeError(f"STFT expect 2D/3D tensor, but got {wav_dim:d}D") + # if N x C x S, reshape NC x S + wav = wav.view(-1, wav.shape[-1]) + # STFT: N x F x T x 2 + stft = th.stft( + wav, + n_fft, + hop_length=frame_hop, + win_length=window.shape[-1], + window=window, + center=center, + normalized=normalized, + onesided=onesided, + return_complex=False, + ) + if wav_dim == 3: + N, F, T, _ = stft.shape + stft = stft.view(N, -1, F, T, 2) + # N x (C) x F x T x 2 + if not return_polar: + return stft + # N x (C) x F x T + real, imag = stft[..., 0], stft[..., 1] + mag = (real ** 2 + imag ** 2 + eps) ** 0.5 + pha = th.atan2(imag, real) + return th.stack([mag, pha], dim=-1) + + +def _pytorch_istft( + transform: th.Tensor, + frame_len: int, + frame_hop: int, + window: th.Tensor, + n_fft: int = 512, + return_polar: bool = False, + normalized: bool = False, + onesided: bool = True, + center: bool = False, + eps: float = EPSILON, +) -> th.Tensor: + """ + Wrapper of PyTorch iSTFT function + Args: + transform (Tensor): results of STFT + frame_len: length of the frame + frame_hop: hop size between frames + window: window tensor + n_fft: number of the FFT size + return_polar: keep same with _pytorch_stft + center: same definition with the parameter in librosa.stft + normalized: use normalized DFT kernel + onesided: output onesided STFT + Return: + wav (Tensor): synthetic audio + """ + if TORCH_VERSION < LooseVersion("1.7"): + raise RuntimeError("Can not use this function as TORCH_VERSION < 1.7") + + transform_dim = transform.dim() + # if F x T x 2, reshape 1 x F x T x 2 + if transform_dim == 3: + transform = th.unsqueeze(transform, 0) + if transform_dim != 4: + raise RuntimeError(f"Expect 4D tensor, but got {transform_dim}D") + + if return_polar: + real = transform[..., 0] * th.cos(transform[..., 1]) + imag = transform[..., 0] * th.sin(transform[..., 1]) + transform = th.stack([real, imag], -1) + # stft is a complex tensor of PyTorch + stft = th.view_as_complex(transform) + # (N) x S + wav = th.istft( + stft, + n_fft, + hop_length=frame_hop, + win_length=window.shape[-1], + window=window, + center=center, + normalized=normalized, + onesided=onesided, + return_complex=False, + ) + return wav + + +def forward_stft( + wav: th.Tensor, + frame_len: int, + frame_hop: int, + window: str = "sqrthann", + round_pow_of_two: bool = True, + return_polar: bool = False, + pre_emphasis: float = 0, + normalized: bool = False, + onesided: bool = True, + center: bool = False, + mode: str = "librosa", + eps: float = EPSILON, +) -> th.Tensor: + """ + STFT function implementation, equals to STFT layer + Args: + wav: source audio signal + frame_len: length of the frame + frame_hop: hop size between frames + return_polar: return [magnitude; phase] Tensor or [real; imag] Tensor + window: window name + center: center flag (similar with that in librosa.stft) + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + pre_emphasis: factor of preemphasis + normalized: use normalized DFT kernel + onesided: output onesided STFT + inverse: using iDFT kernel (for iSTFT) + mode: STFT mode, "kaldi" or "librosa" or "torch" + Return: + transform: results of STFT + """ + window = init_window(window, frame_len, device=wav.device) + if mode == "torch": + n_fft = 2 ** math.ceil(math.log2(frame_len)) if round_pow_of_two else frame_len + return _pytorch_stft( + wav, + frame_len, + frame_hop, + n_fft=n_fft, + return_polar=return_polar, + window=window, + normalized=normalized, + onesided=onesided, + center=center, + eps=eps, + ) + else: + kernel, window = init_kernel( + frame_len, + frame_hop, + window=window, + round_pow_of_two=round_pow_of_two, + normalized=normalized, + inverse=False, + mode=mode, + ) + return _forward_stft( + wav, + kernel, + window, + return_polar=return_polar, + frame_hop=frame_hop, + pre_emphasis=pre_emphasis, + onesided=onesided, + center=center, + eps=eps, + ) + + +def inverse_stft( + transform: th.Tensor, + frame_len: int, + frame_hop: int, + return_polar: bool = False, + window: str = "sqrthann", + round_pow_of_two: bool = True, + normalized: bool = False, + onesided: bool = True, + center: bool = False, + mode: str = "librosa", + eps: float = EPSILON, +) -> th.Tensor: + """ + iSTFT function implementation, equals to iSTFT layer + Args: + transform: results of STFT + frame_len: length of the frame + frame_hop: hop size between frames + return_polar: keep same with function forward_stft(...) + window: window name + center: center flag (similar with that in librosa.stft) + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + normalized: use normalized DFT kernel + onesided: output onesided STFT + mode: STFT mode, "kaldi" or "librosa" or "torch" + Return: + wav: synthetic signals + """ + window = init_window(window, frame_len, device=transform.device) + if mode == "torch": + n_fft = 2 ** math.ceil(math.log2(frame_len)) if round_pow_of_two else frame_len + return _pytorch_istft( + transform, + frame_len, + frame_hop, + n_fft=n_fft, + return_polar=return_polar, + window=window, + normalized=normalized, + onesided=onesided, + center=center, + eps=eps, + ) + else: + kernel, window = init_kernel( + frame_len, + frame_hop, + window, + round_pow_of_two=round_pow_of_two, + normalized=normalized, + inverse=True, + mode=mode, + ) + return _inverse_stft( + transform, + kernel, + window, + return_polar=return_polar, + frame_hop=frame_hop, + onesided=onesided, + center=center, + eps=eps, + ) + + +class STFTBase(nn.Module): + """ + Base layer for (i)STFT + Args: + frame_len: length of the frame + frame_hop: hop size between frames + window: window name + center: center flag (similar with that in librosa.stft) + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + normalized: use normalized DFT kernel + pre_emphasis: factor of preemphasis + mode: STFT mode, "kaldi" or "librosa" or "torch" + onesided: output onesided STFT + inverse: using iDFT kernel (for iSTFT) + """ + + def __init__( + self, + frame_len: int, + frame_hop: int, + window: str = "sqrthann", + round_pow_of_two: bool = True, + normalized: bool = False, + pre_emphasis: float = 0, + onesided: bool = True, + inverse: bool = False, + center: bool = False, + mode: str = "librosa", + ) -> None: + super(STFTBase, self).__init__() + if mode != "torch": + K, w = init_kernel( + frame_len, + frame_hop, + init_window(window, frame_len), + round_pow_of_two=round_pow_of_two, + normalized=normalized, + inverse=inverse, + mode=mode, + ) + self.K = nn.Parameter(K, requires_grad=False) + self.w = nn.Parameter(w, requires_grad=False) + self.num_bins = self.K.shape[0] // 4 + 1 + self.pre_emphasis = pre_emphasis + self.win_length = self.K.shape[2] + else: + self.K = None + w = init_window(window, frame_len) + self.w = nn.Parameter(w, requires_grad=False) + fft_size = ( + 2 ** math.ceil(math.log2(frame_len)) if round_pow_of_two else frame_len + ) + self.num_bins = fft_size // 2 + 1 + self.pre_emphasis = 0 + self.win_length = fft_size + self.frame_len = frame_len + self.frame_hop = frame_hop + self.window = window + self.normalized = normalized + self.onesided = onesided + self.center = center + self.mode = mode + + def num_frames(self, wav_len: th.Tensor) -> th.Tensor: + """ + Compute number of the frames + """ + assert th.sum(wav_len <= self.win_length) == 0 + if self.center: + wav_len += self.win_length + return ( + th.div(wav_len - self.win_length, self.frame_hop, rounding_mode="trunc") + 1 + ) + + def extra_repr(self) -> str: + str_repr = ( + f"num_bins={self.num_bins}, win_length={self.win_length}, " + + f"stride={self.frame_hop}, window={self.window}, " + + f"center={self.center}, mode={self.mode}" + ) + if not self.onesided: + str_repr += f", onesided={self.onesided}" + if self.pre_emphasis > 0: + str_repr += f", pre_emphasis={self.pre_emphasis}" + if self.normalized: + str_repr += f", normalized={self.normalized}" + return str_repr + + +class STFT(STFTBase): + """ + Short-time Fourier Transform as a Layer + """ + + def __init__(self, *args, **kwargs): + super(STFT, self).__init__(*args, inverse=False, **kwargs) + + def forward( + self, wav: th.Tensor, return_polar: bool = False, eps: float = EPSILON + ) -> th.Tensor: + """ + Accept (single or multiple channel) raw waveform and output magnitude and phase + Args + wav (Tensor) input signal, N x (C) x S + Return + transform (Tensor), N x (C) x F x T x 2 + """ + if self.mode == "torch": + return _pytorch_stft( + wav, + self.frame_len, + self.frame_hop, + n_fft=(self.num_bins - 1) * 2, + return_polar=return_polar, + window=self.w, + normalized=self.normalized, + onesided=self.onesided, + center=self.center, + eps=eps, + ) + else: + return _forward_stft( + wav, + self.K, + self.w, + return_polar=return_polar, + frame_hop=self.frame_hop, + pre_emphasis=self.pre_emphasis, + onesided=self.onesided, + center=self.center, + eps=eps, + ) + + +class iSTFT(STFTBase): + """ + Inverse Short-time Fourier Transform as a Layer + """ + + def __init__(self, *args, **kwargs): + super(iSTFT, self).__init__(*args, inverse=True, **kwargs) + + def forward( + self, transform: th.Tensor, return_polar: bool = False, eps: float = EPSILON + ) -> th.Tensor: + """ + Accept phase & magnitude and output raw waveform + Args + transform (Tensor): STFT output, N x F x T x 2 + Return + s (Tensor): N x S + """ + if self.mode == "torch": + return _pytorch_istft( + transform, + self.frame_len, + self.frame_hop, + n_fft=(self.num_bins - 1) * 2, + return_polar=return_polar, + window=self.w, + normalized=self.normalized, + onesided=self.onesided, + center=self.center, + eps=eps, + ) + else: + return _inverse_stft( + transform, + self.K, + self.w, + return_polar=return_polar, + frame_hop=self.frame_hop, + onesided=self.onesided, + center=self.center, + eps=eps, + ) diff --git a/look2hear/layers/stft_tfgn.py b/look2hear/layers/stft_tfgn.py new file mode 100644 index 0000000..98d659d --- /dev/null +++ b/look2hear/layers/stft_tfgn.py @@ -0,0 +1,246 @@ +from typing import Optional, Tuple, Union + +import librosa +import numpy as np +import torch +from packaging.version import parse as V +from torch_complex.tensor import ComplexTensor +from typeguard import typechecked + +from ..utils.complex_utils import is_complex +from ..utils.inversible_interface import InversibleInterface +from ..utils.nets_utils import make_pad_mask + +is_torch_1_10_plus = V(torch.__version__) >= V("1.10.0") + + +is_torch_1_9_plus = V(torch.__version__) >= V("1.9.0") + + +is_torch_1_7_plus = V(torch.__version__) >= V("1.7") + + +class Stft(torch.nn.Module, InversibleInterface): + @typechecked + def __init__( + self, + n_fft: int = 512, + win_length: int = None, + hop_length: int = 128, + window: Optional[str] = "hann", + center: bool = True, + normalized: bool = False, + onesided: bool = True, + ): + + super().__init__() + self.n_fft = n_fft + if win_length is None: + self.win_length = n_fft + else: + self.win_length = win_length + self.hop_length = hop_length + self.center = center + self.normalized = normalized + self.onesided = onesided + if window is not None and not hasattr(torch, f"{window}_window"): + raise ValueError(f"{window} window is not implemented") + self.window = window + + def extra_repr(self): + return ( + f"n_fft={self.n_fft}, " + f"win_length={self.win_length}, " + f"hop_length={self.hop_length}, " + f"center={self.center}, " + f"normalized={self.normalized}, " + f"onesided={self.onesided}" + ) + + def forward( + self, input: torch.Tensor, ilens: torch.Tensor = None + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """STFT forward function. + + Args: + input: (Batch, Nsamples) or (Batch, Nsample, Channels) + ilens: (Batch) + Returns: + output: (Batch, Frames, Freq, 2) or (Batch, Frames, Channels, Freq, 2) + + """ + bs = input.size(0) + if input.dim() == 3: + multi_channel = True + # input: (Batch, Nsample, Channels) -> (Batch * Channels, Nsample) + input = input.transpose(1, 2).reshape(-1, input.size(1)) + else: + multi_channel = False + + # NOTE(kamo): + # The default behaviour of torch.stft is compatible with librosa.stft + # about padding and scaling. + # Note that it's different from scipy.signal.stft + + # output: (Batch, Freq, Frames, 2=real_imag) + # or (Batch, Channel, Freq, Frames, 2=real_imag) + if self.window is not None: + window_func = getattr(torch, f"{self.window}_window") + window = window_func( + self.win_length, dtype=input.dtype, device=input.device + ) + else: + window = None + + # For the compatibility of ARM devices, which do not support + # torch.stft() due to the lack of MKL (on older pytorch versions), + # there is an alternative replacement implementation with librosa. + # Note: pytorch >= 1.10.0 now has native support for FFT and STFT + # on all cpu targets including ARM. + if is_torch_1_10_plus or input.is_cuda or torch.backends.mkl.is_available(): + stft_kwargs = dict( + n_fft=self.n_fft, + win_length=self.win_length, + hop_length=self.hop_length, + center=self.center, + window=window, + normalized=self.normalized, + onesided=self.onesided, + ) + if is_torch_1_7_plus: + stft_kwargs["return_complex"] = False + output = torch.stft(input, **stft_kwargs) + else: + if self.training: + raise NotImplementedError( + "stft is implemented with librosa on this device, which does not " + "support the training mode." + ) + + # use stft_kwargs to flexibly control different PyTorch versions' kwargs + # note: librosa does not support a win_length that is < n_ftt + # but the window can be manually padded (see below). + stft_kwargs = dict( + n_fft=self.n_fft, + win_length=self.n_fft, + hop_length=self.hop_length, + center=self.center, + window=window, + pad_mode="reflect", + ) + + if window is not None: + # pad the given window to n_fft + n_pad_left = (self.n_fft - window.shape[0]) // 2 + n_pad_right = self.n_fft - window.shape[0] - n_pad_left + stft_kwargs["window"] = torch.cat( + [torch.zeros(n_pad_left), window, torch.zeros(n_pad_right)], 0 + ).numpy() + else: + win_length = ( + self.win_length if self.win_length is not None else self.n_fft + ) + stft_kwargs["window"] = torch.ones(win_length) + + output = [] + # iterate over istances in a batch + for i, instance in enumerate(input): + stft = librosa.stft(input[i].numpy(), **stft_kwargs) + output.append(torch.tensor(np.stack([stft.real, stft.imag], -1))) + output = torch.stack(output, 0) + if not self.onesided: + len_conj = self.n_fft - output.shape[1] + conj = output[:, 1 : 1 + len_conj].flip(1) + conj[:, :, :, -1].data *= -1 + output = torch.cat([output, conj], 1) + if self.normalized: + output = output * (stft_kwargs["window"].shape[0] ** (-0.5)) + + # output: (Batch, Freq, Frames, 2=real_imag) + # -> (Batch, Frames, Freq, 2=real_imag) + output = output.transpose(1, 2) + if multi_channel: + # output: (Batch * Channel, Frames, Freq, 2=real_imag) + # -> (Batch, Frame, Channel, Freq, 2=real_imag) + output = output.view(bs, -1, output.size(1), output.size(2), 2).transpose( + 1, 2 + ) + + if ilens is not None: + if self.center: + pad = self.n_fft // 2 + ilens = ilens + 2 * pad + + if is_torch_1_9_plus: + olens = ( + torch.div( + ilens - self.n_fft, self.hop_length, rounding_mode="trunc" + ) + + 1 + ) + else: + olens = (ilens - self.n_fft) // self.hop_length + 1 + output.masked_fill_(make_pad_mask(olens, output, 1), 0.0) + else: + olens = None + + return output, olens + + def inverse( + self, input: Union[torch.Tensor, ComplexTensor], ilens: torch.Tensor = None + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Inverse STFT. + + Args: + input: Tensor(batch, T, F, 2) or ComplexTensor(batch, T, F) + ilens: (batch,) + Returns: + wavs: (batch, samples) + ilens: (batch,) + """ + if V(torch.__version__) >= V("1.6.0"): + istft = torch.functional.istft + else: + try: + import torchaudio + except ImportError: + raise ImportError( + "Please install torchaudio>=0.3.0 or use torch>=1.6.0" + ) + + if not hasattr(torchaudio.functional, "istft"): + raise ImportError( + "Please install torchaudio>=0.3.0 or use torch>=1.6.0" + ) + istft = torchaudio.functional.istft + + if self.window is not None: + window_func = getattr(torch, f"{self.window}_window") + if is_complex(input): + datatype = input.real.dtype + else: + datatype = input.dtype + window = window_func(self.win_length, dtype=datatype, device=input.device) + else: + window = None + + if is_complex(input): + input = torch.stack([input.real, input.imag], dim=-1) + elif input.shape[-1] != 2: + raise TypeError("Invalid input type") + input = input.transpose(1, 2) + input = torch.complex(input[:,:,:,0], input[:,:,:,1]) + + wavs = istft( + input, + n_fft=self.n_fft, + hop_length=self.hop_length, + win_length=self.win_length, + window=window, + center=self.center, + normalized=self.normalized, + onesided=self.onesided, + length=ilens.max() if ilens is not None else ilens, + ) + + return wavs, ilens \ No newline at end of file diff --git a/look2hear/losses/__init__.py b/look2hear/losses/__init__.py new file mode 100644 index 0000000..76c874d --- /dev/null +++ b/look2hear/losses/__init__.py @@ -0,0 +1,37 @@ +from .matrix import pairwise_neg_sisdr +from .matrix import pairwise_neg_sdsdr +from .matrix import pairwise_neg_snr +from .matrix import singlesrc_neg_sisdr +from .matrix import singlesrc_neg_sdsdr +from .matrix import singlesrc_neg_snr +from .matrix import multisrc_neg_sisdr +from .matrix import multisrc_neg_sdsdr +from .matrix import multisrc_neg_snr +from .matrix import freq_mae_wavl1loss +from .matrix import pairwise_neg_sisdr_freq_mse +from .matrix import pairwise_neg_snr_multidecoder +from .pit_wrapper import PITLossWrapper +from .mixit import MixITLossWrapper +from .matrix import PairwiseNegSDR +from .matrix import SingleSrcNegSDR +from .sisnri import SISNRi + +__all__ = [ + "SISNRi", + "MixITLossWrapper", + "PITLossWrapper", + "PairwiseNegSDR", + "SingleSrcNegSDR", + "singlesrc_neg_sisdr", + "pairwise_neg_sisdr", + "multisrc_neg_sisdr", + "pairwise_neg_sdsdr", + "singlesrc_neg_sdsdr", + "multisrc_neg_sdsdr", + "pairwise_neg_snr", + "singlesrc_neg_snr", + "multisrc_neg_snr", + "freq_mae_wavl1loss", + "pairwise_neg_sisdr_freq_mse", + "pairwise_neg_snr_multidecoder" +] diff --git a/look2hear/losses/matrix.py b/look2hear/losses/matrix.py new file mode 100644 index 0000000..07a7acc --- /dev/null +++ b/look2hear/losses/matrix.py @@ -0,0 +1,200 @@ +import torch +from torch.nn.modules.loss import _Loss + + +class PairwiseNegSDR(_Loss): + def __init__(self, sdr_type, zero_mean=True, take_log=True, EPS=1e-8): + super().__init__() + assert sdr_type in ["snr", "sisdr", "sdsdr"] + self.sdr_type = sdr_type + self.zero_mean = zero_mean + self.take_log = take_log + self.EPS = EPS + + def forward(self, ests, targets): + if targets.size() != ests.size() or targets.ndim != 3: + raise TypeError( + f"Inputs must be of shape [batch, n_src, time], got {targets.size()} and {ests.size()} instead" + ) + assert targets.size() == ests.size() + # Step 1. Zero-mean norm + if self.zero_mean: + mean_source = torch.mean(targets, dim=2, keepdim=True) + mean_estimate = torch.mean(ests, dim=2, keepdim=True) + targets = targets - mean_source + ests = ests - mean_estimate + # Step 2. Pair-wise SI-SDR. (Reshape to use broadcast) + s_target = torch.unsqueeze(targets, dim=1) + s_estimate = torch.unsqueeze(ests, dim=2) + if self.sdr_type in ["sisdr", "sdsdr"]: + # [batch, n_src, n_src, 1] + pair_wise_dot = torch.sum(s_estimate * s_target, dim=3, keepdim=True) + # [batch, 1, n_src, 1] + s_target_energy = torch.sum(s_target ** 2, dim=3, keepdim=True) + self.EPS + # [batch, n_src, n_src, time] + pair_wise_proj = pair_wise_dot * s_target / s_target_energy + else: + # [batch, n_src, n_src, time] + pair_wise_proj = s_target.repeat(1, s_target.shape[2], 1, 1) + if self.sdr_type in ["sdsdr", "snr"]: + e_noise = s_estimate - s_target + else: + e_noise = s_estimate - pair_wise_proj + # [batch, n_src, n_src] + pair_wise_sdr = torch.sum(pair_wise_proj ** 2, dim=3) / ( + torch.sum(e_noise ** 2, dim=3) + self.EPS + ) + if self.take_log: + pair_wise_sdr = 10 * torch.log10(pair_wise_sdr + self.EPS) + return -pair_wise_sdr + + +class SingleSrcNegSDR(_Loss): + def __init__( + self, sdr_type, zero_mean=True, take_log=True, reduction="none", EPS=1e-8 + ): + assert reduction != "sum", NotImplementedError + super().__init__(reduction=reduction) + + assert sdr_type in ["snr", "sisdr", "sdsdr"] + self.sdr_type = sdr_type + self.zero_mean = zero_mean + self.take_log = take_log + self.EPS = 1e-8 + + def forward(self, ests, targets): + if targets.size() != ests.size() or targets.ndim != 2: + raise TypeError( + f"Inputs must be of shape [batch, time], got {targets.size()} and {ests.size()} instead" + ) + # Step 1. Zero-mean norm + if self.zero_mean: + mean_source = torch.mean(targets, dim=1, keepdim=True) + mean_estimate = torch.mean(ests, dim=1, keepdim=True) + targets = targets - mean_source + ests = ests - mean_estimate + # Step 2. Pair-wise SI-SDR. + if self.sdr_type in ["sisdr", "sdsdr"]: + # [batch, 1] + dot = torch.sum(ests * targets, dim=1, keepdim=True) + # [batch, 1] + s_target_energy = torch.sum(targets ** 2, dim=1, keepdim=True) + self.EPS + # [batch, time] + scaled_target = dot * targets / s_target_energy + else: + # [batch, time] + scaled_target = targets + if self.sdr_type in ["sdsdr", "snr"]: + e_noise = ests - targets + else: + e_noise = ests - scaled_target + # [batch] + losses = torch.sum(scaled_target ** 2, dim=1) / ( + torch.sum(e_noise ** 2, dim=1) + self.EPS + ) + if self.take_log: + losses = 10 * torch.log10(losses + self.EPS) + losses = losses.mean() if self.reduction == "mean" else losses + return -losses + + +class MultiSrcNegSDR(_Loss): + def __init__(self, sdr_type, zero_mean=True, take_log=True, EPS=1e-8): + super().__init__() + + assert sdr_type in ["snr", "sisdr", "sdsdr"] + self.sdr_type = sdr_type + self.zero_mean = zero_mean + self.take_log = take_log + self.EPS = 1e-8 + + def forward(self, ests, targets): + if targets.size() != ests.size() or targets.ndim != 3: + raise TypeError( + f"Inputs must be of shape [batch, n_src, time], got {targets.size()} and {ests.size()} instead" + ) + # Step 1. Zero-mean norm + if self.zero_mean: + mean_source = torch.mean(targets, dim=2, keepdim=True) + mean_est = torch.mean(ests, dim=2, keepdim=True) + targets = targets - mean_source + ests = ests - mean_est + # Step 2. Pair-wise SI-SDR. + if self.sdr_type in ["sisdr", "sdsdr"]: + # [batch, n_src] + pair_wise_dot = torch.sum(ests * targets, dim=2, keepdim=True) + # [batch, n_src] + s_target_energy = torch.sum(targets ** 2, dim=2, keepdim=True) + self.EPS + # [batch, n_src, time] + scaled_targets = pair_wise_dot * targets / s_target_energy + else: + # [batch, n_src, time] + scaled_targets = targets + if self.sdr_type in ["sdsdr", "snr"]: + e_noise = ests - targets + else: + e_noise = ests - scaled_targets + # [batch, n_src] + pair_wise_sdr = torch.sum(scaled_targets ** 2, dim=2) / ( + torch.sum(e_noise ** 2, dim=2) + self.EPS + ) + if self.take_log: + pair_wise_sdr = 10 * torch.log10(pair_wise_sdr + self.EPS) + return -torch.mean(pair_wise_sdr, dim=-1) + +class freq_MAE_WavL1Loss(_Loss): + def __init__(self, win=2048, stride=512): + super().__init__() + self.EPS = 1e-8 + self.win = win + self.stride = stride + + def forward(self, ests, targets): + B, nsrc, _ = ests.shape + est_spec = torch.stft(ests.view(-1, ests.shape[-1]), n_fft=self.win, hop_length=self.stride, + window=torch.hann_window(self.win).to(ests.device).float(), + return_complex=True) + est_target = torch.stft(targets.view(-1, targets.shape[-1]), n_fft=self.win, hop_length=self.stride, + window=torch.hann_window(self.win).to(ests.device).float(), + return_complex=True) + freq_L1 = (est_spec.real - est_target.real).abs().mean((1,2)) + (est_spec.imag - est_target.imag).abs().mean((1,2)) + freq_L1 = freq_L1.view(B, nsrc).mean(-1) + + wave_l1 = (ests - targets).abs().mean(-1) + # print(freq_L1.shape, wave_l1.shape) + wave_l1 = wave_l1.view(B, nsrc).mean(-1) + return freq_L1 + wave_l1 + +class freq_MSE_Loss(_Loss): + def __init__(self, win=640, stride=160): + super().__init__() + self.EPS = 1e-8 + self.win = win + self.stride = stride + + def forward(self, ests, targets): + B, nsrc, _ = ests.shape + est_spec = torch.stft(ests.view(-1, ests.shape[-1]), n_fft=self.win, hop_length=self.stride, + window=torch.hann_window(self.win).to(ests.device).float(), + return_complex=True) + est_target = torch.stft(targets.view(-1, targets.shape[-1]), n_fft=self.win, hop_length=self.stride, + window=torch.hann_window(self.win).to(ests.device).float(), + return_complex=True) + freq_mse = (est_spec.real - est_target.real).square().mean((1,2)) + (est_spec.imag - est_target.imag).square().mean((1,2)) + freq_mse = freq_mse.view(B, nsrc).mean(-1) + + return freq_mse + +# aliases +pairwise_neg_sisdr = PairwiseNegSDR("sisdr") +pairwise_neg_sdsdr = PairwiseNegSDR("sdsdr") +pairwise_neg_snr = PairwiseNegSDR("snr") +singlesrc_neg_sisdr = SingleSrcNegSDR("sisdr") +singlesrc_neg_sdsdr = SingleSrcNegSDR("sdsdr") +singlesrc_neg_snr = SingleSrcNegSDR("snr") +multisrc_neg_sisdr = MultiSrcNegSDR("sisdr") +multisrc_neg_sdsdr = MultiSrcNegSDR("sdsdr") +multisrc_neg_snr = MultiSrcNegSDR("snr") +freq_mae_wavl1loss = freq_MAE_WavL1Loss() +pairwise_neg_sisdr_freq_mse = (PairwiseNegSDR("sisdr"), freq_MSE_Loss()) +pairwise_neg_snr_multidecoder = (PairwiseNegSDR("snr"), MultiSrcNegSDR("snr")) \ No newline at end of file diff --git a/look2hear/losses/mixit.py b/look2hear/losses/mixit.py new file mode 100644 index 0000000..750466f --- /dev/null +++ b/look2hear/losses/mixit.py @@ -0,0 +1,311 @@ +# import torch +# import torch.nn as nn +# from itertools import combinations +# from pprint import pprint + + +# def parts_mixgen(lists): +# parts = [] +# for k in range(len(lists) + 1): +# for sublist in combinations(lists, k): +# rest = [] +# if sublist != () and len(sublist) != len(lists): +# for item in lists: +# if item not in sublist: +# rest.append(item) +# parts.append([sublist, rest]) +# return parts + + +# def parts_mixgen_bisection(lists, srcmix, mix): +# if mix == 0: +# yield [] +# else: +# for c in combinations(lists, srcmix): +# rest = [x for x in lists if x not in c] +# for r in parts_mixgen_bisection(rest, srcmix, mix - 1): +# yield [list(c), *r] + + +# class MixIT(nn.Module): +# def __init__(self, loss_func, bisection=True): +# super().__init__() +# self.loss_func = loss_func +# self.bisection = bisection + +# def forward(self, ests, targets, return_ests=None, **kwargs): +# assert ests.shape[0] == targets.shape[0] +# assert ests.shape[-1] == targets.shape[-1] + +# if self.bisection: +# loss, min_loss_idx, parts = self.mixit_bisection(self.loss_func, ests, targets, **kwargs) +# else: +# loss, min_loss_idx, parts = self.mixit_non_bisection(self.loss_func, ests, targets, **kwargs) + +# mean_loss = torch.mean(loss) +# if not return_ests: +# return mean_loss + +# reordered = self.reorder_source(ests, targets, min_loss_idx, parts) +# return mean_loss, reordered + +# def mixit_bisection(self, loss_func, ests, targets, **kwargs): +# n_mix = targets.shape[1] +# n_src = ests.shape[1] +# srcmix = n_src // n_mix + +# parts = parts_mixgen_bisection(range(n_src), srcmix, n_mix) + +# loss_lists = [] +# for part in parts: +# ests_mix = torch.stack([ests[:, i, :].sum(1) for i in part], dim=1) +# loss = loss_func(ests_mix, targets, **kwargs) +# loss_lists.append(loss[:, None]) +# loss_lists = torch.cat(loss_lists, dim=1) +# min_loss, min_loss_indexes = torch.min(loss_lists, dim=1, keepdim=True) +# return min_loss, min_loss_indexes, parts + +# def mixit_non_bisection(self, loss_func, ests, targets, **kwargs): +# n_mix = targets.shape[1] +# n_src = ests.shape[1] + +# parts = parts_mixgen(range(n_src)) + +# loss_lists = [] +# for part in parts: +# ests_mix = torch.stack([ests[:, i, :].sum(1) for i in part], dim=1) +# loss = loss_func(ests_mix, targets, **kwargs) +# loss_lists.append(loss[:, None]) +# loss_lists = torch.cat(loss_lists, dim=1) +# min_loss, min_loss_indexes = torch.min(loss_lists, dim=1, keepdim=True) +# return min_loss, min_loss_indexes, parts + +# def reoder_source(self, ests, targets, min_loss_idx, parts, **kwargs): +# ordered = torch.zeros_like(targets) +# for b, idx in enumerate(min_loss_idx): +# right_partition = parts[idx] +# ordered[b, :, :] = torch.stack( +# [ests[b, idx, :][None, :, :].sum(1) for idx in right_partition], dim=1 +# ) +# return ordered + +# if __name__ == "__main__": +# print(parts_mixgen(range(4))) +# print([item for item in parts_mixgen_bisection(range(4), 2, 2)]) + +import warnings +from itertools import combinations +import torch +from torch import nn + + +class MixITLossWrapper(nn.Module): + r"""Mixture invariant loss wrapper. + Args: + loss_func: function with signature (est_targets, targets, **kwargs). + generalized (bool): Determines how MixIT is applied. If False , + apply MixIT for any number of mixtures as soon as they contain + the same number of sources (:meth:`~MixITLossWrapper.best_part_mixit`.) + If True (default), apply MixIT for two mixtures, but those mixtures do not + necessarly have to contain the same number of sources. + See :meth:`~MixITLossWrapper.best_part_mixit_generalized`. + For each of these modes, the best partition and reordering will be + automatically computed. + Examples: + >>> import torch + >>> from asteroid.losses import multisrc_mse + >>> mixtures = torch.randn(10, 2, 16000) + >>> est_sources = torch.randn(10, 4, 16000) + >>> # Compute MixIT loss based on pairwise losses + >>> loss_func = MixITLossWrapper(multisrc_mse) + >>> loss_val = loss_func(est_sources, mixtures) + References + [1] Scott Wisdom et al. "Unsupervised sound separation using + mixtures of mixtures." arXiv:2006.12701 (2020) + """ + + def __init__(self, loss_func, generalized=True): + super().__init__() + self.loss_func = loss_func + self.generalized = generalized + + def forward(self, est_targets, targets, return_est=False, **kwargs): + r"""Find the best partition and return the loss. + Args: + est_targets: torch.Tensor. Expected shape :math:`(batch, nsrc, *)`. + The batch of target estimates. + targets: torch.Tensor. Expected shape :math:`(batch, nmix, ...)`. + The batch of training targets + return_est: Boolean. Whether to return the estimated mixtures + estimates (To compute metrics or to save example). + **kwargs: additional keyword argument that will be passed to the + loss function. + Returns: + - Best partition loss for each batch sample, average over + the batch. torch.Tensor(loss_value) + - The estimated mixtures (estimated sources summed according to the partition) + if return_est is True. torch.Tensor of shape :math:`(batch, nmix, ...)`. + """ + # Check input dimensions + assert est_targets.shape[0] == targets.shape[0] + assert est_targets.shape[2] == targets.shape[2] + + if not self.generalized: + min_loss, min_loss_idx, parts = self.best_part_mixit( + self.loss_func, est_targets, targets, **kwargs + ) + else: + min_loss, min_loss_idx, parts = self.best_part_mixit_generalized( + self.loss_func, est_targets, targets, **kwargs + ) + # Take the mean over the batch + mean_loss = torch.mean(min_loss) + if not return_est: + return mean_loss + # Order and sum on the best partition to get the estimated mixtures + reordered = self.reorder_source(est_targets, targets, min_loss_idx, parts) + return mean_loss, reordered + + @staticmethod + def best_part_mixit(loss_func, est_targets, targets, **kwargs): + r"""Find best partition of the estimated sources that gives the minimum + loss for the MixIT training paradigm in [1]. Valid for any number of + mixtures as soon as they contain the same number of sources. + Args: + loss_func: function with signature ``(est_targets, targets, **kwargs)`` + The loss function to get batch losses from. + est_targets: torch.Tensor. Expected shape :math:`(batch, nsrc, ...)`. + The batch of target estimates. + targets: torch.Tensor. Expected shape :math:`(batch, nmix, ...)`. + The batch of training targets (mixtures). + **kwargs: additional keyword argument that will be passed to the + loss function. + Returns: + - :class:`torch.Tensor`: + The loss corresponding to the best permutation of size (batch,). + - :class:`torch.LongTensor`: + The indices of the best partition. + - :class:`list`: + list of the possible partitions of the sources. + """ + nmix = targets.shape[1] + nsrc = est_targets.shape[1] + if nsrc % nmix != 0: + raise ValueError( + "The mixtures are assumed to contain the same number of sources" + ) + nsrcmix = nsrc // nmix + + # Generate all unique partitions of size k from a list lst of + # length n, where l = n // k is the number of parts. The total + # number of such partitions is: NPK(n,k) = n! / ((k!)^l * l!) + # Algorithm recursively distributes items over parts + def parts_mixit(lst, k, l): + if l == 0: + yield [] + else: + for c in combinations(lst, k): + rest = [x for x in lst if x not in c] + for r in parts_mixit(rest, k, l - 1): + yield [list(c), *r] + + # Generate all the possible partitions + parts = list(parts_mixit(range(nsrc), nsrcmix, nmix)) + # Compute the loss corresponding to each partition + loss_set = MixITLossWrapper.loss_set_from_parts( + loss_func, est_targets=est_targets, targets=targets, parts=parts, **kwargs + ) + # Indexes and values of min losses for each batch element + min_loss, min_loss_indexes = torch.min(loss_set, dim=1, keepdim=True) + return min_loss, min_loss_indexes, parts + + @staticmethod + def best_part_mixit_generalized(loss_func, est_targets, targets, **kwargs): + r"""Find best partition of the estimated sources that gives the minimum + loss for the MixIT training paradigm in [1]. Valid only for two mixtures, + but those mixtures do not necessarly have to contain the same number of + sources e.g the case where one mixture is silent is allowed.. + Args: + loss_func: function with signature ``(est_targets, targets, **kwargs)`` + The loss function to get batch losses from. + est_targets: torch.Tensor. Expected shape :math:`(batch, nsrc, ...)`. + The batch of target estimates. + targets: torch.Tensor. Expected shape :math:`(batch, nmix, ...)`. + The batch of training targets (mixtures). + **kwargs: additional keyword argument that will be passed to the + loss function. + Returns: + - :class:`torch.Tensor`: + The loss corresponding to the best permutation of size (batch,). + - :class:`torch.LongTensor`: + The indexes of the best permutations. + - :class:`list`: + list of the possible partitions of the sources. + """ + nmix = targets.shape[1] # number of mixtures + nsrc = est_targets.shape[1] # number of estimated sources + if nmix != 2: + raise ValueError("Works only with two mixtures") + + # Generate all unique partitions of any size from a list lst of + # length n. Algorithm recursively distributes items over parts + def parts_mixit_gen(lst): + partitions = [] + for k in range(len(lst) + 1): + for c in combinations(lst, k): + rest = [] + if c != () and len(c) != len(lst): + for item in lst: + if item not in c: + rest.append(item) + partitions.append([c, rest]) + return partitions + + # Generate all the possible partitions + parts = parts_mixit_gen(range(nsrc)) + # Compute the loss corresponding to each partition + loss_set = MixITLossWrapper.loss_set_from_parts( + loss_func, est_targets=est_targets, targets=targets, parts=parts, **kwargs + ) + # Indexes and values of min losses for each batch element + min_loss, min_loss_indexes = torch.min(loss_set, dim=1, keepdim=True) + return min_loss, min_loss_indexes, parts + + @staticmethod + def loss_set_from_parts(loss_func, est_targets, targets, parts, **kwargs): + """Common loop between both best_part_mixit""" + loss_set = [] + for partition in parts: + # sum the sources according to the given partition + est_mixes = torch.stack( + [est_targets[:, idx, :].sum(1) for idx in partition], dim=1 + ) + # get loss for the given partition + loss_set.append(loss_func(est_mixes, targets, **kwargs)[:, None]) + loss_set = torch.cat(loss_set, dim=1) + return loss_set + + @staticmethod + def reorder_source(est_targets, targets, min_loss_idx, parts): + """Reorder sources according to the best partition. + Args: + est_targets: torch.Tensor. Expected shape :math:`(batch, nsrc, ...)`. + The batch of target estimates. + targets: torch.Tensor. Expected shape :math:`(batch, nmix, ...)`. + The batch of training targets. + min_loss_idx: torch.LongTensor. The indexes of the best permutations. + parts: list of the possible partitions of the sources. + Returns: + :class:`torch.Tensor`: Reordered sources of shape :math:`(batch, nmix, time)`. + """ + # For each batch there is a different min_loss_idx + ordered = torch.zeros_like(targets) + for b, idx in enumerate(min_loss_idx): + right_partition = parts[idx] + # Sum the estimated sources to get the estimated mixtures + ordered[b, :, :] = torch.stack( + [est_targets[b, idx, :][None, :, :].sum(1) for idx in right_partition], + dim=1, + ) + + return ordered diff --git a/look2hear/losses/pit_wrapper.py b/look2hear/losses/pit_wrapper.py new file mode 100644 index 0000000..3434e36 --- /dev/null +++ b/look2hear/losses/pit_wrapper.py @@ -0,0 +1,180 @@ +from itertools import permutations +import torch +from torch import nn +from scipy.optimize import linear_sum_assignment + + +class PITLossWrapper(nn.Module): + def __init__( + self, loss_func, pit_from="pw_mtx", equidistant_weight=False, perm_reduce=None, threshold_byloss=True + ): + super().__init__() + self.loss_func = loss_func + self.pit_from = pit_from + self.perm_reduce = perm_reduce + self.threshold_byloss = threshold_byloss + self.equidistant_weight = equidistant_weight + if self.pit_from not in ["pw_mtx", "pw_pt", "perm_avg", "pw_mtx_broadcast", "pw_mtx_multidecoder_keepmtx", "pw_mtx_multidecoder_batchmin"]: + raise ValueError( + "Unsupported loss function type {} for now. Expected" + "one of [`pw_mtx`, `pw_pt`, `perm_avg`, `pw_mtx_broadcast`]".format(self.pit_from) + ) + + def forward(self, ests, targets, return_ests=False, reduce_kwargs=None, **kwargs): + n_src = targets.shape[1] + if self.pit_from == "pw_mtx": + pw_loss = self.loss_func(ests, targets, **kwargs) + elif self.pit_from == "pw_mtx_broadcast": + pw_loss = self.loss_func[0](ests, targets, **kwargs) + elif self.pit_from == "pw_mtx_multidecoder_keepmtx": + ests_last_block = ests[-1] + pw_loss = self.loss_func[0](ests_last_block, targets, **kwargs) + elif self.pit_from == "pw_mtx_multidecoder_batchmin": + blocks_num = len(ests) + ests = torch.cat(ests, dim=0) + targets = torch.cat([targets] * blocks_num, dim=0) + pw_loss = self.loss_func(ests, targets, **kwargs) + elif self.pit_from == "pw_pt": + pw_loss = self.get_pw_losses(self.loss_func, ests, targets, **kwargs) + elif self.pit_from == "perm_avg": + min_loss, batch_indices = self.best_perm_from_perm_avg_loss( + self.loss_func, ests, targets, **kwargs + ) + # print(batch_indices) + mean_loss = torch.mean(min_loss) + if not return_ests: + return mean_loss + reordered = self.reordered_sources(ests, batch_indices) + return mean_loss, reordered + else: + return + # import pdb; pdb.set_trace() + assert pw_loss.ndim == 3, ( + "Something went wrong with the loss " "function, please read the docs." + ) + assert ( + pw_loss.shape[0] == targets.shape[0] + ), "PIT loss needs same batch dim as input" + + reduce_kwargs = reduce_kwargs if reduce_kwargs is not None else dict() + min_loss, batch_indices = self.find_best_perm( + pw_loss, perm_reduce=self.perm_reduce, **reduce_kwargs + ) + if self.pit_from == "pw_mtx_multidecoder_keepmtx": + reordered = [] + mean_loss = 0 + for i in range(len(ests)): + reordered_ests_each_block = self.reordered_sources(ests[i], batch_indices) + reordered.append(reordered_ests_each_block) + loss_each_block = self.loss_func[1](reordered_ests_each_block, targets, **kwargs) + if self.threshold_byloss: + if loss_each_block[loss_each_block > -30].nelement() > 0: + loss_each_block = loss_each_block[loss_each_block > -30] + if self.equidistant_weight: + mean_loss = mean_loss + (i + 1) * 1 / len(ests) * loss_each_block.mean() + else: + mean_loss = mean_loss + 1 / len(ests) * loss_each_block.mean() + reordered = torch.cat(reordered, dim=0) + if not return_ests: + return mean_loss + return mean_loss, reordered + else: + if self.threshold_byloss: + if min_loss[min_loss > -30].nelement() > 0: + min_loss = min_loss[min_loss > -30] + mean_loss = torch.mean(min_loss) + reordered = self.reordered_sources(ests, batch_indices) + # import pdb; pdb.set_trace() + if self.pit_from == "pw_mtx_broadcast": + mean_loss += 0.5 * self.loss_func[1](reordered, targets, **kwargs).mean() + if not return_ests: + return mean_loss + return mean_loss, reordered + + def get_pw_losses(self, loss_func, ests, targets, **kwargs): + B, n_src, _ = targets.shape + pair_wise_losses = targets.new_empty(B, n_src, n_src) + for est_idx, est_src in enumerate(ests.transpose(0, 1)): + for target_idx, target_src in enumerate(targets.transpose(0, 1)): + pair_wise_losses[:, est_idx, target_idx] = loss_func( + est_src, target_src, **kwargs + ) + return pair_wise_losses + + def best_perm_from_perm_avg_loss(self, loss_func, ests, targets, **kwargs): + n_src = targets.shape[1] + perms = torch.tensor(list(permutations(range(n_src))), dtype=torch.long) + # import pdb; pdb.set_trace() + loss_set = torch.stack( + [loss_func(ests[:, perm], targets) for perm in perms], dim=1 + ) + min_loss, min_loss_idx = torch.min(loss_set, dim=1) + batch_indices = torch.stack([perms[m] for m in min_loss_idx], dim=0) + return min_loss, batch_indices + + def reordered_sources(self, sources, batch_indices): + reordered_sources = torch.stack( + [torch.index_select(s, 0, b) for s, b in zip(sources, batch_indices)] + ) + return reordered_sources + + def find_best_perm(self, pair_wise_losses, perm_reduce=None, **kwargs): + n_src = pair_wise_losses.shape[-1] + if perm_reduce is not None or n_src <= 3: + min_loss, batch_indices = self.find_best_perm_factorial( + pair_wise_losses, perm_reduce=perm_reduce, **kwargs + ) + else: + min_loss, batch_indices = self.find_best_perm_hungarian(pair_wise_losses) + return min_loss, batch_indices + + def find_best_perm_factorial(self, pair_wise_losses, perm_reduce=None, **kwargs): + n_src = pair_wise_losses.shape[-1] + # After transposition, dim 1 corresp. to sources and dim 2 to estimates + pwl = pair_wise_losses.transpose(-1, -2) + perms = pwl.new_tensor(list(permutations(range(n_src))), dtype=torch.long) + # Column permutation indices + idx = torch.unsqueeze(perms, 2) + # Loss mean of each permutation + if perm_reduce is None: + # one-hot, [n_src!, n_src, n_src] + # import pdb; pdb.set_trace() + perms_one_hot = pwl.new_zeros((*perms.size(), n_src)).scatter_(2, idx, 1) + loss_set = torch.einsum("bij,pij->bp", [pwl, perms_one_hot]) + loss_set /= n_src + else: + # batch = pwl.shape[0]; n_perm = idx.shape[0] + # [batch, n_src!, n_src] : Pairwise losses for each permutation. + pwl_set = pwl[:, torch.arange(n_src), idx.squeeze(-1)] + # Apply reduce [batch, n_src!, n_src] --> [batch, n_src!] + loss_set = perm_reduce(pwl_set, **kwargs) + # Indexes and values of min losses for each batch element + min_loss, min_loss_idx = torch.min(loss_set, dim=1) + + # Permutation indices for each batch. + batch_indices = torch.stack([perms[m] for m in min_loss_idx], dim=0) + return min_loss, batch_indices + + def find_best_perm_hungarian(self, pair_wise_losses: torch.Tensor): + pwl = pair_wise_losses.transpose(-1, -2) + # Just bring the numbers to cpu(), not the graph + pwl_copy = pwl.detach().cpu() + # Loop over batch + row indices are always ordered for square matrices. + batch_indices = torch.tensor( + [linear_sum_assignment(pwl)[1] for pwl in pwl_copy] + ).to(pwl.device) + min_loss = torch.gather(pwl, 2, batch_indices[..., None]).mean([-1, -2]) + return min_loss, batch_indices + + +if __name__ == "__main__": + import torch + from matrix import pairwise_neg_sisdr, pairwise_neg_sisdr + + ests = torch.randn(10, 2, 32000) + targets = torch.randn(10, 2, 32000) + + pit_wrapper_1 = PITLossWrapper(pairwise_neg_sisdr, pit_from="pw_mtx") + pit_wrapper_2 = PITLossWrapper(pairwise_neg_sisdr, pit_from="pw_mtx") + print(pit_wrapper_1(ests, targets)) + print(pit_wrapper_2(ests, targets)) diff --git a/look2hear/losses/sisnri.py b/look2hear/losses/sisnri.py new file mode 100644 index 0000000..c879546 --- /dev/null +++ b/look2hear/losses/sisnri.py @@ -0,0 +1,42 @@ +import torch +from itertools import permutations + +class SISNRi(object): + def __init__(self): + super(Loss, self).__init__() + + def sisnr(self, mix, est, ref, eps = 1e-8): + """ + input: + mix: B x L + est: B x L + output: B + """ + est = est - torch.mean(est, dim = -1, keepdim = True) + ref = ref - torch.mean(ref, dim = -1, keepdim = True) + mix = mix - torch.mean(mix, dim = -1, keepdim = True) + est_p = (torch.sum(est * ref, dim = -1, keepdim = True) * ref) / torch.sum(ref * ref, dim = -1, keepdim = True) + est_v = est - est_p + mix_p = (torch.sum(mix * ref, dim = -1, keepdim = True) * ref) / torch.sum(ref * ref, dim = -1, keepdim = True) + mix_v = mix - mix_p + est_sisnr = 10 * torch.log10((torch.sum(est_p * est_p, dim = -1, keepdim = True) + eps) / (torch.sum(est_v * est_v, dim = -1, keepdim = True) + eps)) + mix_sisnr = 10 * torch.log10((torch.sum(mix_p * mix_p, dim = -1, keepdim = True) + eps) / (torch.sum(mix_v * mix_v, dim = -1, keepdim = True) + eps)) + return est_sisnr - mix_sisnr + + def compute_loss(self, mix, ests, refs): + """ + input: + mix: B x L + est: num_spk x B x L + output: 1 + """ + + def sisnr_loss(permute): + # B + return torch.mean(torch.stack([self.sisnr(mix, ests[s], refs[t]) for s, t in enumerate(permute)]), dim = 0, keepdim = True) + num_spks = len(ests) + # pmt_num x B + sisnr_mat = torch.stack([sisnr_loss(p) for p in permutations(range(num_spks))]) + # B + max_pmt, _ = torch.max(sisnr_mat, dim=0) + return -torch.mean(max_pmt) \ No newline at end of file diff --git a/look2hear/metrics/__init__.py b/look2hear/metrics/__init__.py new file mode 100644 index 0000000..fd65a55 --- /dev/null +++ b/look2hear/metrics/__init__.py @@ -0,0 +1,4 @@ +from .wrapper import MetricsTracker +from .splitwrapper import SPlitMetricsTracker + +__all__ = ["MetricsTracker", "SPlitMetricsTracker"] diff --git a/look2hear/metrics/splitwrapper.py b/look2hear/metrics/splitwrapper.py new file mode 100644 index 0000000..6d985b6 --- /dev/null +++ b/look2hear/metrics/splitwrapper.py @@ -0,0 +1,115 @@ +import csv +import torch +import numpy as np +import logging + +# from torch_mir_eval.separation import bss_eval_sources +from ..losses import ( + PITLossWrapper, + pairwise_neg_sisdr, + pairwise_neg_snr, + singlesrc_neg_sisdr, +) + +logger = logging.getLogger(__name__) + + +class SPlitMetricsTracker: + def __init__(self, save_file: str = ""): + self.one_all_snrs = [] + self.one_all_snrs_i = [] + self.one_all_sisnrs = [] + self.one_all_sisnrs_i = [] + self.two_all_snrs = [] + self.two_all_snrs_i = [] + self.two_all_sisnrs = [] + self.two_all_sisnrs_i = [] + csv_columns = [ + "snt_id", + "one_snr", + "one_snr_i", + "one_si-snr", + "one_si-snr_i", + "two_snr", + "two_snr_i", + "two_si-snr", + "two_si-snr_i", + ] + self.results_csv = open(save_file, "w") + self.writer = csv.DictWriter(self.results_csv, fieldnames=csv_columns) + self.writer.writeheader() + self.pit_sisnr = PITLossWrapper(pairwise_neg_sisdr, pit_from="pw_mtx") + self.pit_snr = PITLossWrapper(pairwise_neg_snr, pit_from="pw_mtx") + + def __call__(self, mix, clean, estimate, key): + _, ests_np = self.pit_snr( + estimate.unsqueeze(0), clean.unsqueeze(0), return_ests=True + ) + # sisnr + two_sisnr = self.pit_sisnr(ests_np[:, 0:2], clean.unsqueeze(0)[:, 0:2]) + one_sisnr = self.pit_sisnr( + ests_np[:, 2].unsqueeze(1), clean.unsqueeze(0)[:, 2].unsqueeze(1) + ) + mix = torch.stack([mix] * clean.shape[0], dim=0) + two_sisnr_baseline = self.pit_sisnr( + mix.unsqueeze(0)[:, 0:2], clean.unsqueeze(0)[:, 0:2] + ) + one_sisnr_baseline = self.pit_sisnr( + mix.unsqueeze(0)[:, 2].unsqueeze(1), clean.unsqueeze(0)[:, 2].unsqueeze(1) + ) + two_sisnr_i = two_sisnr - two_sisnr_baseline + one_sisnr_i = one_sisnr - one_sisnr_baseline + # sdr + two_snr = self.pit_snr(ests_np[:, 0:2], clean.unsqueeze(0)[:, 0:2]) + one_snr = self.pit_snr( + ests_np[:, 2].unsqueeze(1), clean.unsqueeze(0)[:, 2].unsqueeze(1) + ) + two_snr_baseline = self.pit_snr( + mix.unsqueeze(0)[:, 0:2], clean.unsqueeze(0)[:, 0:2] + ) + one_snr_baseline = self.pit_snr( + mix.unsqueeze(0)[:, 2].unsqueeze(1), clean.unsqueeze(0)[:, 2].unsqueeze(1) + ) + two_snr_i = two_snr - two_snr_baseline + one_snr_i = one_snr - one_snr_baseline + + row = { + "snt_id": key, + "one_snr": -one_snr.item(), + "one_snr_i": -one_snr_i.item(), + "one_si-snr": -one_sisnr.item(), + "one_si-snr_i": -one_sisnr_i.item(), + "two_snr": -two_snr.item(), + "two_snr_i": -two_snr_i.item(), + "two_si-snr": -two_sisnr.item(), + "two_si-snr_i": -two_sisnr_i.item(), + } + self.writer.writerow(row) + # Metric Accumulation + self.one_all_snrs.append(-one_snr.item()) + self.one_all_snrs_i.append(-one_snr_i.item()) + self.one_all_sisnrs.append(-one_sisnr.item()) + self.one_all_sisnrs_i.append(-one_sisnr_i.item()) + self.two_all_snrs.append(-two_snr.item()) + self.two_all_snrs_i.append(-two_snr_i.item()) + self.two_all_sisnrs.append(-two_sisnr.item()) + self.two_all_sisnrs_i.append(-two_sisnr_i.item()) + + def final(self,): + row = { + "snt_id": "avg", + "one_snr": np.array(self.one_all_snrs).mean(), + "one_snr_i": np.array(self.one_all_snrs_i).mean(), + "one_si-snr": np.array(self.one_all_sisnrs).mean(), + "one_si-snr_i": np.array(self.one_all_sisnrs_i).mean(), + "two_snr": np.array(self.two_all_snrs).mean(), + "two_snr_i": np.array(self.two_all_snrs_i).mean(), + "two_si-snr": np.array(self.two_all_sisnrs).mean(), + "two_si-snr_i": np.array(self.two_all_sisnrs_i).mean(), + } + self.writer.writerow(row) + # logger.info("Mean SISNR is {}".format(row["si-snr"])) + # logger.info("Mean SISNRi is {}".format(row["si-snr_i"])) + # logger.info("Mean SDR is {}".format(row["sdr"])) + # logger.info("Mean SDRi is {}".format(row["sdr_i"])) + self.results_csv.close() diff --git a/look2hear/metrics/wrapper.py b/look2hear/metrics/wrapper.py new file mode 100644 index 0000000..1c2a10c --- /dev/null +++ b/look2hear/metrics/wrapper.py @@ -0,0 +1,84 @@ +import csv +import torch +import numpy as np +import logging + +from torch_mir_eval.separation import bss_eval_sources +import fast_bss_eval +from ..losses import ( + PITLossWrapper, + pairwise_neg_sisdr, + pairwise_neg_snr, + singlesrc_neg_sisdr, + PairwiseNegSDR, +) + +logger = logging.getLogger(__name__) + + +class MetricsTracker: + def __init__(self, save_file: str = ""): + self.all_sdrs = [] + self.all_sdrs_i = [] + self.all_sisnrs = [] + self.all_sisnrs_i = [] + csv_columns = ["snt_id", "sdr", "sdr_i", "si-snr", "si-snr_i"] + self.results_csv = open(save_file, "w") + self.writer = csv.DictWriter(self.results_csv, fieldnames=csv_columns) + self.writer.writeheader() + self.pit_sisnr = PITLossWrapper( + PairwiseNegSDR("sisdr", zero_mean=False), pit_from="pw_mtx" + ) + self.pit_snr = PITLossWrapper( + PairwiseNegSDR("snr", zero_mean=False), pit_from="pw_mtx" + ) + + def __call__(self, mix, clean, estimate, key): + # sisnr + sisnr = self.pit_sisnr(estimate.unsqueeze(0), clean.unsqueeze(0)) + mix = torch.stack([mix] * clean.shape[0], dim=0) + sisnr_baseline = self.pit_sisnr(mix.unsqueeze(0), clean.unsqueeze(0)) + sisnr_i = sisnr - sisnr_baseline + + # sdr + sdr = -fast_bss_eval.sdr_pit_loss(estimate, clean).mean() + sdr_baseline = -fast_bss_eval.sdr_pit_loss(mix, clean).mean() + sdr_i = sdr - sdr_baseline + # import pdb; pdb.set_trace() + row = { + "snt_id": key, + "sdr": sdr.item(), + "sdr_i": sdr_i.item(), + "si-snr": -sisnr.item(), + "si-snr_i": -sisnr_i.item(), + } + self.writer.writerow(row) + # Metric Accumulation + self.all_sdrs.append(sdr.item()) + self.all_sdrs_i.append(sdr_i.item()) + self.all_sisnrs.append(-sisnr.item()) + self.all_sisnrs_i.append(-sisnr_i.item()) + + def update(self, ): + return {"sdr_i": np.array(self.all_sdrs_i).mean(), + "si-snr_i": np.array(self.all_sisnrs_i).mean() + } + + def final(self,): + row = { + "snt_id": "avg", + "sdr": np.array(self.all_sdrs).mean(), + "sdr_i": np.array(self.all_sdrs_i).mean(), + "si-snr": np.array(self.all_sisnrs).mean(), + "si-snr_i": np.array(self.all_sisnrs_i).mean(), + } + self.writer.writerow(row) + row = { + "snt_id": "std", + "sdr": np.array(self.all_sdrs).std(), + "sdr_i": np.array(self.all_sdrs_i).std(), + "si-snr": np.array(self.all_sisnrs).std(), + "si-snr_i": np.array(self.all_sisnrs_i).std(), + } + self.writer.writerow(row) + self.results_csv.close() diff --git a/look2hear/models/__init__.py b/look2hear/models/__init__.py new file mode 100644 index 0000000..00bc01f --- /dev/null +++ b/look2hear/models/__init__.py @@ -0,0 +1,42 @@ +from .tiger import TIGER +from .tiger_dnr import TIGERDNR +from .base_model import BaseModel + +__all__ = [ + "TIGER", +] + + +def register_model(custom_model): + """Register a custom model, gettable with `models.get`. + + Args: + custom_model: Custom model to register. + + """ + if ( + custom_model.__name__ in globals().keys() + or custom_model.__name__.lower() in globals().keys() + ): + raise ValueError( + f"Model {custom_model.__name__} already exists. Choose another name." + ) + globals().update({custom_model.__name__: custom_model}) + + +def get(identifier): + """Returns an model class from a string (case-insensitive). + + Args: + identifier (str): the model name. + + Returns: + :class:`torch.nn.Module` + """ + if isinstance(identifier, str): + to_get = {k.lower(): v for k, v in globals().items()} + cls = to_get.get(identifier.lower()) + if cls is None: + raise ValueError(f"Could not interpret model name : {str(identifier)}") + return cls + raise ValueError(f"Could not interpret model name : {str(identifier)}") diff --git a/look2hear/models/base_model.py b/look2hear/models/base_model.py new file mode 100644 index 0000000..d827640 --- /dev/null +++ b/look2hear/models/base_model.py @@ -0,0 +1,94 @@ +### +# Author: Kai Li +# Date: 2021-06-17 23:08:32 +# LastEditors: Please set LastEditors +# LastEditTime: 2022-05-26 18:06:22 +### +import torch +import torch.nn as nn +from huggingface_hub import PyTorchModelHubMixin + +def _unsqueeze_to_3d(x): + """Normalize shape of `x` to [batch, n_chan, time].""" + if x.ndim == 1: + return x.reshape(1, 1, -1) + elif x.ndim == 2: + return x.unsqueeze(1) + else: + return x + + +def pad_to_appropriate_length(x, lcm): + values_to_pad = int(x.shape[-1]) % lcm + if values_to_pad: + appropriate_shape = x.shape + padded_x = torch.zeros( + list(appropriate_shape[:-1]) + + [appropriate_shape[-1] + lcm - values_to_pad], + dtype=torch.float32, + ).to(x.device) + padded_x[..., : x.shape[-1]] = x + return padded_x + return x + + +class BaseModel(nn.Module, PyTorchModelHubMixin, repo_url="https://github.com/JusperLee/Apollo", pipeline_tag="audio-to-audio"): + def __init__(self, sample_rate, in_chan=1): + super().__init__() + self._sample_rate = sample_rate + self._in_chan = in_chan + + def forward(self, *args, **kwargs): + raise NotImplementedError + + def sample_rate(self,): + return self._sample_rate + + @staticmethod + def load_state_dict_in_audio(model, pretrained_dict): + model_dict = model.state_dict() + update_dict = {} + for k, v in pretrained_dict.items(): + if "audio_model" in k: + update_dict[k[12:]] = v + model_dict.update(update_dict) + model.load_state_dict(model_dict) + return model + + # @staticmethod + # def from_pretrain(pretrained_model_conf_or_path, *args, **kwargs): + # from . import get + + # conf = torch.load( + # pretrained_model_conf_or_path, map_location="cpu" + # ) # Attempt to find the model and instantiate it. + + # model_class = get(conf["model_name"]) + # # model_class = get("Conv_TasNet") + # model = model_class(*args, **kwargs) + # model.load_state_dict(conf["state_dict"]) + # return model + + def serialize(self): + import pytorch_lightning as pl # Not used in torch.hub + + model_conf = dict( + model_name=self.__class__.__name__, + state_dict=self.get_state_dict(), + model_args=self.get_model_args(), + ) + # Additional infos + infos = dict() + infos["software_versions"] = dict( + torch_version=torch.__version__, pytorch_lightning_version=pl.__version__, + ) + model_conf["infos"] = infos + return model_conf + + def get_state_dict(self): + """In case the state dict needs to be modified before sharing the model.""" + return self.state_dict() + + def get_model_args(self): + """Should return args to re-instantiate the class.""" + raise NotImplementedError \ No newline at end of file diff --git a/look2hear/models/tiger.py b/look2hear/models/tiger.py new file mode 100644 index 0000000..51fcbb0 --- /dev/null +++ b/look2hear/models/tiger.py @@ -0,0 +1,649 @@ +import inspect +import torch +import numpy as np +import torch.nn as nn +import torch.nn.functional as F +import math +from .base_model import BaseModel +from ..layers import activations, normalizations + + +def GlobLN(nOut): + return nn.GroupNorm(1, nOut, eps=1e-8) + + +class ConvNormAct(nn.Module): + """ + This class defines the convolution layer with normalization and a PReLU + activation + """ + + def __init__(self, nIn, nOut, kSize, stride=1, groups=1): + """ + :param nIn: number of input channels + :param nOut: number of output channels + :param kSize: kernel size + :param stride: stride rate for down-sampling. Default is 1 + """ + super().__init__() + padding = int((kSize - 1) / 2) + self.conv = nn.Conv1d( + nIn, nOut, kSize, stride=stride, padding=padding, bias=True, groups=groups + ) + self.norm = GlobLN(nOut) + self.act = nn.PReLU() + + def forward(self, input): + output = self.conv(input) + output = self.norm(output) + return self.act(output) + + +class ConvNorm(nn.Module): + """ + This class defines the convolution layer with normalization and PReLU activation + """ + + def __init__(self, nIn, nOut, kSize, stride=1, groups=1, bias=True): + """ + :param nIn: number of input channels + :param nOut: number of output channels + :param kSize: kernel size + :param stride: stride rate for down-sampling. Default is 1 + """ + super().__init__() + padding = int((kSize - 1) / 2) + self.conv = nn.Conv1d( + nIn, nOut, kSize, stride=stride, padding=padding, bias=bias, groups=groups + ) + self.norm = GlobLN(nOut) + + def forward(self, input): + output = self.conv(input) + return self.norm(output) + +class ATTConvActNorm(nn.Module): + def __init__( + self, + in_chan: int = 1, + out_chan: int = 1, + kernel_size: int = -1, + stride: int = 1, + groups: int = 1, + dilation: int = 1, + padding: int = None, + norm_type: str = None, + act_type: str = None, + n_freqs: int = -1, + xavier_init: bool = False, + bias: bool = True, + is2d: bool = False, + *args, + **kwargs, + ): + super(ATTConvActNorm, self).__init__() + self.in_chan = in_chan + self.out_chan = out_chan + self.kernel_size = kernel_size + self.stride = stride + self.groups = groups + self.dilation = dilation + self.padding = padding + self.norm_type = norm_type + self.act_type = act_type + self.n_freqs = n_freqs + self.xavier_init = xavier_init + self.bias = bias + + if self.padding is None: + self.padding = 0 if self.stride > 1 else "same" + + if kernel_size > 0: + conv = nn.Conv2d if is2d else nn.Conv1d + + self.conv = conv( + in_channels=self.in_chan, + out_channels=self.out_chan, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + groups=self.groups, + bias=self.bias, + ) + if self.xavier_init: + nn.init.xavier_uniform_(self.conv.weight) + else: + self.conv = nn.Identity() + + self.act = activations.get(self.act_type)() + self.norm = normalizations.get(self.norm_type)( + (self.out_chan, self.n_freqs) if self.norm_type == "LayerNormalization4D" else self.out_chan + ) + + def forward(self, x: torch.Tensor): + output = self.conv(x) + output = self.act(output) + output = self.norm(output) + return output + + def get_config(self): + encoder_args = {} + + for k, v in (self.__dict__).items(): + if not k.startswith("_") and k != "training": + if not inspect.ismethod(v): + encoder_args[k] = v + + return encoder_args + +class DilatedConvNorm(nn.Module): + """ + This class defines the dilated convolution with normalized output. + """ + + def __init__(self, nIn, nOut, kSize, stride=1, d=1, groups=1): + """ + :param nIn: number of input channels + :param nOut: number of output channels + :param kSize: kernel size + :param stride: optional stride rate for down-sampling + :param d: optional dilation rate + """ + super().__init__() + self.conv = nn.Conv1d( + nIn, + nOut, + kSize, + stride=stride, + dilation=d, + padding=((kSize - 1) // 2) * d, + groups=groups, + ) + # self.norm = nn.GroupNorm(1, nOut, eps=1e-08) + self.norm = GlobLN(nOut) + + def forward(self, input): + output = self.conv(input) + return self.norm(output) + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_size, drop=0.1): + super().__init__() + self.fc1 = ConvNorm(in_features, hidden_size, 1, bias=False) + self.dwconv = nn.Conv1d( + hidden_size, hidden_size, 5, 1, 2, bias=True, groups=hidden_size + ) + self.act = nn.ReLU() + self.fc2 = ConvNorm(hidden_size, in_features, 1, bias=False) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.dwconv(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class InjectionMultiSum(nn.Module): + def __init__(self, inp: int, oup: int, kernel: int = 1) -> None: + super().__init__() + groups = 1 + if inp == oup: + groups = inp + self.local_embedding = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.global_embedding = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.global_act = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.act = nn.Sigmoid() + + def forward(self, x_l, x_g): + """ + x_g: global features + x_l: local features + """ + B, N, T = x_l.shape + local_feat = self.local_embedding(x_l) + + global_act = self.global_act(x_g) + sig_act = F.interpolate(self.act(global_act), size=T, mode="nearest") + # sig_act = self.act(global_act) + + global_feat = self.global_embedding(x_g) + global_feat = F.interpolate(global_feat, size=T, mode="nearest") + + out = local_feat * sig_act + global_feat + return out + +class InjectionMulti(nn.Module): + def __init__(self, inp: int, oup: int, kernel: int = 1) -> None: + super().__init__() + groups = 1 + if inp == oup: + groups = inp + self.local_embedding = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.global_act = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.act = nn.Sigmoid() + + def forward(self, x_l, x_g): + """ + x_g: global features + x_l: local features + """ + B, N, T = x_l.shape + local_feat = self.local_embedding(x_l) + + global_act = self.global_act(x_g) + sig_act = F.interpolate(self.act(global_act), size=T, mode="nearest") + # sig_act = self.act(global_act) + + out = local_feat * sig_act + return out + +class UConvBlock(nn.Module): + """ + This class defines the block which performs successive downsampling and + upsampling in order to be able to analyze the input features in multiple + resolutions. + """ + + def __init__(self, out_channels=128, in_channels=512, upsampling_depth=4, model_T=True): + super().__init__() + self.proj_1x1 = ConvNormAct(out_channels, in_channels, 1, stride=1, groups=1) + self.depth = upsampling_depth + self.spp_dw = nn.ModuleList() + self.spp_dw.append( + DilatedConvNorm( + in_channels, in_channels, kSize=5, stride=1, groups=in_channels, d=1 + ) + ) + for i in range(1, upsampling_depth): + self.spp_dw.append( + DilatedConvNorm( + in_channels, + in_channels, + kSize=5, + stride=2, + groups=in_channels, + d=1, + ) + ) + + self.loc_glo_fus = nn.ModuleList([]) + for i in range(upsampling_depth): + self.loc_glo_fus.append(InjectionMultiSum(in_channels, in_channels)) + + self.res_conv = nn.Conv1d(in_channels, out_channels, 1) + + self.globalatt = Mlp(in_channels, in_channels, drop=0.1) + + self.last_layer = nn.ModuleList([]) + for i in range(self.depth - 1): + self.last_layer.append(InjectionMultiSum(in_channels, in_channels, 5)) + + def forward(self, x): + """ + :param x: input feature map + :return: transformed feature map + """ + residual = x.clone() + # Reduce --> project high-dimensional feature maps to low-dimensional space + output1 = self.proj_1x1(x) + output = [self.spp_dw[0](output1)] + + # Do the downsampling process from the previous level + for k in range(1, self.depth): + out_k = self.spp_dw[k](output[-1]) + output.append(out_k) + + # global features + global_f = torch.zeros( + output[-1].shape, requires_grad=True, device=output1.device + ) + for fea in output: + global_f = global_f + F.adaptive_avg_pool1d( + fea, output_size=output[-1].shape[-1] + ) + # global_f = global_f + fea + global_f = self.globalatt(global_f) # [B, N, T] + + x_fused = [] + # Gather them now in reverse order + for idx in range(self.depth): + local = output[idx] + x_fused.append(self.loc_glo_fus[idx](local, global_f)) + + expanded = None + for i in range(self.depth - 2, -1, -1): + if i == self.depth - 2: + expanded = self.last_layer[i](x_fused[i], x_fused[i - 1]) + else: + expanded = self.last_layer[i](x_fused[i], expanded) + # import pdb; pdb.set_trace() + return self.res_conv(expanded) + residual + +class MultiHeadSelfAttention2D(nn.Module): + def __init__( + self, + in_chan: int, + n_freqs: int, + n_head: int = 4, + hid_chan: int = 4, + act_type: str = "prelu", + norm_type: str = "LayerNormalization4D", + dim: int = 3, + *args, + **kwargs, + ): + super(MultiHeadSelfAttention2D, self).__init__() + self.in_chan = in_chan + self.n_freqs = n_freqs + self.n_head = n_head + self.hid_chan = hid_chan + self.act_type = act_type + self.norm_type = norm_type + self.dim = dim + + assert self.in_chan % self.n_head == 0 + + self.Queries = nn.ModuleList() + self.Keys = nn.ModuleList() + self.Values = nn.ModuleList() + + for _ in range(self.n_head): + self.Queries.append( + ATTConvActNorm( + in_chan=self.in_chan, + out_chan=self.hid_chan, + kernel_size=1, + act_type=self.act_type, + norm_type=self.norm_type, + n_freqs=self.n_freqs, + is2d=True, + ) + ) + self.Keys.append( + ATTConvActNorm( + in_chan=self.in_chan, + out_chan=self.hid_chan, + kernel_size=1, + act_type=self.act_type, + norm_type=self.norm_type, + n_freqs=self.n_freqs, + is2d=True, + ) + ) + self.Values.append( + ATTConvActNorm( + in_chan=self.in_chan, + out_chan=self.in_chan // self.n_head, + kernel_size=1, + act_type=self.act_type, + norm_type=self.norm_type, + n_freqs=self.n_freqs, + is2d=True, + ) + ) + + self.attn_concat_proj = ATTConvActNorm( + in_chan=self.in_chan, + out_chan=self.in_chan, + kernel_size=1, + act_type=self.act_type, + norm_type=self.norm_type, + n_freqs=self.n_freqs, + is2d=True, + ) + + def forward(self, x: torch.Tensor): + if self.dim == 4: + x = x.transpose(-2, -1).contiguous() + + batch_size, _, time, freq = x.size() + residual = x + + all_Q = [q(x) for q in self.Queries] # [B, E, T, F] + all_K = [k(x) for k in self.Keys] # [B, E, T, F] + all_V = [v(x) for v in self.Values] # [B, C/n_head, T, F] + + Q = torch.cat(all_Q, dim=0) # [B', E, T, F] B' = B*n_head + K = torch.cat(all_K, dim=0) # [B', E, T, F] + V = torch.cat(all_V, dim=0) # [B', C/n_head, T, F] + + Q = Q.transpose(1, 2).flatten(start_dim=2) # [B', T, E*F] + K = K.transpose(1, 2).flatten(start_dim=2) # [B', T, E*F] + V = V.transpose(1, 2) # [B', T, C/n_head, F] + old_shape = V.shape + V = V.flatten(start_dim=2) # [B', T, C*F/n_head] + emb_dim = Q.shape[-1] # C*F/n_head + + attn_mat = torch.matmul(Q, K.transpose(1, 2)) / (emb_dim**0.5) # [B', T, T] + attn_mat = F.softmax(attn_mat, dim=2) # [B', T, T] + V = torch.matmul(attn_mat, V) # [B', T, C*F/n_head] + V = V.reshape(old_shape) # [B', T, C/n_head, F] + V = V.transpose(1, 2) # [B', C/n_head, T, F] + emb_dim = V.shape[1] # C/n_head + + x = V.view([self.n_head, batch_size, emb_dim, time, freq]) # [n_head, B, C/n_head, T, F] + x = x.transpose(0, 1).contiguous() # [B, n_head, C/n_head, T, F] + + x = x.view([batch_size, self.n_head * emb_dim, time, freq]) # [B, C, T, F] + x = self.attn_concat_proj(x) # [B, C, T, F] + + x = x + residual + + if self.dim == 4: + x = x.transpose(-2, -1).contiguous() + + return x + + +class Recurrent(nn.Module): + def __init__( + self, + out_channels=128, + in_channels=512, + nband=8, + upsampling_depth=3, + n_head=4, + att_hid_chan=4, + kernel_size: int = 8, + stride: int = 1, + _iter=4 + ): + super().__init__() + self.nband = nband + + self.freq_path = nn.ModuleList([ + UConvBlock(out_channels, in_channels, upsampling_depth), + MultiHeadSelfAttention2D(out_channels, 1, n_head=n_head, hid_chan=att_hid_chan, act_type="prelu", norm_type="LayerNormalization4D", dim=4), + normalizations.get("LayerNormalization4D")((out_channels, 1)) + ]) + + self.frame_path = nn.ModuleList([ + UConvBlock(out_channels, in_channels, upsampling_depth), + MultiHeadSelfAttention2D(out_channels, 1, n_head=n_head, hid_chan=att_hid_chan, act_type="prelu", norm_type="LayerNormalization4D", dim=4), + normalizations.get("LayerNormalization4D")((out_channels, 1)) + ]) + + self.iter = _iter + self.concat_block = nn.Sequential( + nn.Conv2d(out_channels, out_channels, 1, 1, groups=out_channels), nn.PReLU() + ) + + def forward(self, x): + # B, nband, N, T + B, nband, N, T = x.shape + x = x.permute(0, 2, 1, 3).contiguous() # B, N, nband, T + mixture = x.clone() + for i in range(self.iter): + if i == 0: + x = self.freq_time_process(x, B, nband, N, T) # B, N, nband, T + else: + x = self.freq_time_process(self.concat_block(mixture + x), B, nband, N, T) # B, N, nband, T + + return x.permute(0, 2, 1, 3).contiguous() # B, nband, N, T + + def freq_time_process(self, x, B, nband, N, T): + # Process Frequency Path + residual_1 = x.clone() + x = x.permute(0, 3, 1, 2).contiguous() # B, T, N, nband + freq_fea = self.freq_path[0](x.view(B*T, N, nband)) # B*T, N, nband + freq_fea = freq_fea.view(B, T, N, nband).permute(0, 2, 1, 3).contiguous() # B, N, T, nband + freq_fea = self.freq_path[1](freq_fea) # B, N, T, nband + freq_fea = self.freq_path[2](freq_fea) # B, N, T, nband + freq_fea = freq_fea.permute(0, 1, 3, 2).contiguous() + x = freq_fea + residual_1 # B, N, nband, T + # Process Frame Path + residual_2 = x.clone() + x2 = x.permute(0, 2, 1, 3).contiguous() + frame_fea = self.frame_path[0](x2.view(B*nband, N, T)) # B*nband, N, T + frame_fea = frame_fea.view(B, nband, N, T).permute(0, 2, 1, 3).contiguous() + frame_fea = self.frame_path[1](frame_fea) # B, N, nband, T + frame_fea = self.frame_path[2](frame_fea) # B, N, nband, T + x = frame_fea + residual_2 # B, N, nband, T + return x + +class TIGER(BaseModel): + def __init__( + self, + out_channels=128, + in_channels=512, + num_blocks=16, + upsampling_depth=4, + att_n_head=4, + att_hid_chan=4, + att_kernel_size=8, + att_stride=1, + win=2048, + stride=512, + num_sources=2, + sample_rate=44100, + ): + super(TIGER, self).__init__(sample_rate=sample_rate) + + self.sample_rate = sample_rate + self.win = win + self.stride = stride + self.group = self.win // 2 + self.enc_dim = self.win // 2 + 1 + self.feature_dim = out_channels + self.num_output = num_sources + self.eps = torch.finfo(torch.float32).eps + + # 0-1k (25 hop), 1k-2k (100 hop), 2k-4k (250 hop), 4k-8k (500 hop) + bandwidth_25 = int(np.floor(25 / (sample_rate / 2.) * self.enc_dim)) + bandwidth_100 = int(np.floor(100 / (sample_rate / 2.) * self.enc_dim)) + bandwidth_250 = int(np.floor(250 / (sample_rate / 2.) * self.enc_dim)) + bandwidth_500 = int(np.floor(500 / (sample_rate / 2.) * self.enc_dim)) + self.band_width = [bandwidth_25]*40 + self.band_width += [bandwidth_100]*10 + self.band_width += [bandwidth_250]*8 + self.band_width += [bandwidth_500]*8 + self.band_width.append(self.enc_dim - np.sum(self.band_width)) + self.nband = len(self.band_width) + print(self.band_width) + + self.BN = nn.ModuleList([]) + for i in range(self.nband): + self.BN.append(nn.Sequential(nn.GroupNorm(1, self.band_width[i]*2, self.eps), + nn.Conv1d(self.band_width[i]*2, self.feature_dim, 1) + ) + ) + + self.separator = Recurrent(self.feature_dim, in_channels, self.nband, upsampling_depth, att_n_head, att_hid_chan, att_kernel_size, att_stride, num_blocks) + + self.mask = nn.ModuleList([]) + for i in range(self.nband): + self.mask.append(nn.Sequential( + nn.PReLU(), + nn.Conv1d(self.feature_dim, self.band_width[i]*4*num_sources, 1, groups=num_sources) + ) + ) + + def pad_input(self, input, window, stride): + """ + Zero-padding input according to window/stride size. + """ + batch_size, nsample = input.shape + + # pad the signals at the end for matching the window/stride size + rest = window - (stride + nsample % window) % window + if rest > 0: + pad = torch.zeros(batch_size, rest).type(input.type()) + input = torch.cat([input, pad], 1) + pad_aux = torch.zeros(batch_size, stride).type(input.type()) + input = torch.cat([pad_aux, input, pad_aux], 1) + + return input, rest + + def forward(self, input): + # input shape: (B, C, T) + was_one_d = False + if input.ndim == 1: + was_one_d = True + input = input.unsqueeze(0).unsqueeze(1) + if input.ndim == 2: + was_one_d = True + input = input.unsqueeze(1) + if input.ndim == 3: + input = input + batch_size, nch, nsample = input.shape + input = input.view(batch_size*nch, -1) + + # frequency-domain separation + spec = torch.stft(input, n_fft=self.win, hop_length=self.stride, + window=torch.hann_window(self.win).to(input.device).type(input.type()), + return_complex=True) + + # print(spec.shape) + + # concat real and imag, split to subbands + spec_RI = torch.stack([spec.real, spec.imag], 1) # B*nch, 2, F, T + subband_spec_RI = [] + subband_spec = [] + band_idx = 0 + for i in range(len(self.band_width)): + subband_spec_RI.append(spec_RI[:,:,band_idx:band_idx+self.band_width[i]].contiguous()) + subband_spec.append(spec[:,band_idx:band_idx+self.band_width[i]]) # B*nch, BW, T + band_idx += self.band_width[i] + + # normalization and bottleneck + subband_feature = [] + for i in range(len(self.band_width)): + subband_feature.append(self.BN[i](subband_spec_RI[i].view(batch_size*nch, self.band_width[i]*2, -1))) + subband_feature = torch.stack(subband_feature, 1) # B, nband, N, T + # import pdb; pdb.set_trace() + # separator + sep_output = self.separator(subband_feature.view(batch_size*nch, self.nband, self.feature_dim, -1)) # B, nband, N, T + sep_output = sep_output.view(batch_size*nch, self.nband, self.feature_dim, -1) + + sep_subband_spec = [] + for i in range(self.nband): + this_output = self.mask[i](sep_output[:,i]).view(batch_size*nch, 2, 2, self.num_output, self.band_width[i], -1) + this_mask = this_output[:,0] * torch.sigmoid(this_output[:,1]) # B*nch, 2, K, BW, T + this_mask_real = this_mask[:,0] # B*nch, K, BW, T + this_mask_imag = this_mask[:,1] # B*nch, K, BW, T + # force mask sum to 1 + this_mask_real_sum = this_mask_real.sum(1).unsqueeze(1) # B*nch, 1, BW, T + this_mask_imag_sum = this_mask_imag.sum(1).unsqueeze(1) # B*nch, 1, BW, T + this_mask_real = this_mask_real - (this_mask_real_sum - 1) / self.num_output + this_mask_imag = this_mask_imag - this_mask_imag_sum / self.num_output + est_spec_real = subband_spec[i].real.unsqueeze(1) * this_mask_real - subband_spec[i].imag.unsqueeze(1) * this_mask_imag # B*nch, K, BW, T + est_spec_imag = subband_spec[i].real.unsqueeze(1) * this_mask_imag + subband_spec[i].imag.unsqueeze(1) * this_mask_real # B*nch, K, BW, T + sep_subband_spec.append(torch.complex(est_spec_real, est_spec_imag)) + sep_subband_spec = torch.cat(sep_subband_spec, 2) + + output = torch.istft(sep_subband_spec.view(batch_size*nch*self.num_output, self.enc_dim, -1), + n_fft=self.win, hop_length=self.stride, + window=torch.hann_window(self.win).to(input.device).type(input.type()), length=nsample) + output = output.view(batch_size*nch, self.num_output, -1) + # if was_one_d: + # return output.squeeze(0) + return output + + def get_model_args(self): + model_args = {"n_sample_rate": 2} + return model_args \ No newline at end of file diff --git a/look2hear/models/tiger_dnr.py b/look2hear/models/tiger_dnr.py new file mode 100644 index 0000000..27f7ab1 --- /dev/null +++ b/look2hear/models/tiger_dnr.py @@ -0,0 +1,878 @@ +import inspect +import torch +import numpy as np +import torch.nn as nn +import torch.nn.functional as F +import math +from .base_model import BaseModel +from ..layers import activations, normalizations + + +def GlobLN(nOut): + return nn.GroupNorm(1, nOut, eps=1e-8) + + +class ConvNormAct(nn.Module): + """ + This class defines the convolution layer with normalization and a PReLU + activation + """ + + def __init__(self, nIn, nOut, kSize, stride=1, groups=1): + """ + :param nIn: number of input channels + :param nOut: number of output channels + :param kSize: kernel size + :param stride: stride rate for down-sampling. Default is 1 + """ + super().__init__() + padding = int((kSize - 1) / 2) + self.conv = nn.Conv1d( + nIn, nOut, kSize, stride=stride, padding=padding, bias=True, groups=groups + ) + self.norm = GlobLN(nOut) + self.act = nn.PReLU() + + def forward(self, input): + output = self.conv(input) + output = self.norm(output) + return self.act(output) + + +class ConvNorm(nn.Module): + """ + This class defines the convolution layer with normalization and PReLU activation + """ + + def __init__(self, nIn, nOut, kSize, stride=1, groups=1, bias=True): + """ + :param nIn: number of input channels + :param nOut: number of output channels + :param kSize: kernel size + :param stride: stride rate for down-sampling. Default is 1 + """ + super().__init__() + padding = int((kSize - 1) / 2) + self.conv = nn.Conv1d( + nIn, nOut, kSize, stride=stride, padding=padding, bias=bias, groups=groups + ) + self.norm = GlobLN(nOut) + + def forward(self, input): + output = self.conv(input) + return self.norm(output) + + +class ATTConvActNorm(nn.Module): + def __init__( + self, + in_chan: int = 1, + out_chan: int = 1, + kernel_size: int = -1, + stride: int = 1, + groups: int = 1, + dilation: int = 1, + padding: int = None, + norm_type: str = None, + act_type: str = None, + n_freqs: int = -1, + xavier_init: bool = False, + bias: bool = True, + is2d: bool = False, + *args, + **kwargs, + ): + super(ATTConvActNorm, self).__init__() + self.in_chan = in_chan + self.out_chan = out_chan + self.kernel_size = kernel_size + self.stride = stride + self.groups = groups + self.dilation = dilation + self.padding = padding + self.norm_type = norm_type + self.act_type = act_type + self.n_freqs = n_freqs + self.xavier_init = xavier_init + self.bias = bias + + if self.padding is None: + self.padding = 0 if self.stride > 1 else "same" + + if kernel_size > 0: + conv = nn.Conv2d if is2d else nn.Conv1d + + self.conv = conv( + in_channels=self.in_chan, + out_channels=self.out_chan, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + groups=self.groups, + bias=self.bias, + ) + if self.xavier_init: + nn.init.xavier_uniform_(self.conv.weight) + else: + self.conv = nn.Identity() + + self.act = activations.get(self.act_type)() + self.norm = normalizations.get(self.norm_type)( + (self.out_chan, self.n_freqs) + if self.norm_type == "LayerNormalization4D" + else self.out_chan + ) + + def forward(self, x: torch.Tensor): + output = self.conv(x) + output = self.act(output) + output = self.norm(output) + return output + + def get_config(self): + encoder_args = {} + + for k, v in (self.__dict__).items(): + if not k.startswith("_") and k != "training": + if not inspect.ismethod(v): + encoder_args[k] = v + + return encoder_args + + +class DilatedConvNorm(nn.Module): + """ + This class defines the dilated convolution with normalized output. + """ + + def __init__(self, nIn, nOut, kSize, stride=1, d=1, groups=1): + """ + :param nIn: number of input channels + :param nOut: number of output channels + :param kSize: kernel size + :param stride: optional stride rate for down-sampling + :param d: optional dilation rate + """ + super().__init__() + self.conv = nn.Conv1d( + nIn, + nOut, + kSize, + stride=stride, + dilation=d, + padding=((kSize - 1) // 2) * d, + groups=groups, + ) + # self.norm = nn.GroupNorm(1, nOut, eps=1e-08) + self.norm = GlobLN(nOut) + + def forward(self, input): + output = self.conv(input) + return self.norm(output) + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_size, drop=0.1): + super().__init__() + self.fc1 = ConvNorm(in_features, hidden_size, 1, bias=False) + self.dwconv = nn.Conv1d( + hidden_size, hidden_size, 5, 1, 2, bias=True, groups=hidden_size + ) + self.act = nn.ReLU() + self.fc2 = ConvNorm(hidden_size, in_features, 1, bias=False) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.dwconv(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class InjectionMultiSum(nn.Module): + def __init__(self, inp: int, oup: int, kernel: int = 1) -> None: + super().__init__() + groups = 1 + if inp == oup: + groups = inp + self.local_embedding = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.global_embedding = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.global_act = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.act = nn.Sigmoid() + + def forward(self, x_l, x_g): + """ + x_g: global features + x_l: local features + """ + B, N, T = x_l.shape + local_feat = self.local_embedding(x_l) + + global_act = self.global_act(x_g) + sig_act = F.interpolate(self.act(global_act), size=T, mode="nearest") + # sig_act = self.act(global_act) + + global_feat = self.global_embedding(x_g) + global_feat = F.interpolate(global_feat, size=T, mode="nearest") + + out = local_feat * sig_act + global_feat + return out + + +class InjectionMulti(nn.Module): + def __init__(self, inp: int, oup: int, kernel: int = 1) -> None: + super().__init__() + groups = 1 + if inp == oup: + groups = inp + self.local_embedding = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.global_act = ConvNorm(inp, oup, kernel, groups=groups, bias=False) + self.act = nn.Sigmoid() + + def forward(self, x_l, x_g): + """ + x_g: global features + x_l: local features + """ + B, N, T = x_l.shape + local_feat = self.local_embedding(x_l) + + global_act = self.global_act(x_g) + sig_act = F.interpolate(self.act(global_act), size=T, mode="nearest") + # sig_act = self.act(global_act) + + out = local_feat * sig_act + return out + + +class UConvBlock(nn.Module): + """ + This class defines the block which performs successive downsampling and + upsampling in order to be able to analyze the input features in multiple + resolutions. + """ + + def __init__( + self, out_channels=128, in_channels=512, upsampling_depth=4, model_T=True + ): + super().__init__() + self.proj_1x1 = ConvNormAct(out_channels, in_channels, 1, stride=1, groups=1) + self.depth = upsampling_depth + self.spp_dw = nn.ModuleList() + self.spp_dw.append( + DilatedConvNorm( + in_channels, in_channels, kSize=5, stride=1, groups=in_channels, d=1 + ) + ) + for i in range(1, upsampling_depth): + self.spp_dw.append( + DilatedConvNorm( + in_channels, + in_channels, + kSize=5, + stride=2, + groups=in_channels, + d=1, + ) + ) + + self.loc_glo_fus = nn.ModuleList([]) + for i in range(upsampling_depth): + self.loc_glo_fus.append(InjectionMultiSum(in_channels, in_channels)) + + self.res_conv = nn.Conv1d(in_channels, out_channels, 1) + + self.globalatt = Mlp(in_channels, in_channels, drop=0.1) + + self.last_layer = nn.ModuleList([]) + for i in range(self.depth - 1): + self.last_layer.append(InjectionMultiSum(in_channels, in_channels, 5)) + + def forward(self, x): + """ + :param x: input feature map + :return: transformed feature map + """ + residual = x.clone() + # Reduce --> project high-dimensional feature maps to low-dimensional space + output1 = self.proj_1x1(x) + output = [self.spp_dw[0](output1)] + + # Do the downsampling process from the previous level + for k in range(1, self.depth): + out_k = self.spp_dw[k](output[-1]) + output.append(out_k) + + # global features + global_f = torch.zeros( + output[-1].shape, requires_grad=True, device=output1.device + ) + for fea in output: + global_f = global_f + F.adaptive_avg_pool1d( + fea, output_size=output[-1].shape[-1] + ) + # global_f = global_f + fea + global_f = self.globalatt(global_f) # [B, N, T] + + x_fused = [] + # Gather them now in reverse order + for idx in range(self.depth): + local = output[idx] + x_fused.append(self.loc_glo_fus[idx](local, global_f)) + + expanded = None + for i in range(self.depth - 2, -1, -1): + if i == self.depth - 2: + expanded = self.last_layer[i](x_fused[i], x_fused[i - 1]) + else: + expanded = self.last_layer[i](x_fused[i], expanded) + # import pdb; pdb.set_trace() + return self.res_conv(expanded) + residual + + +class MultiHeadSelfAttention2D(nn.Module): + def __init__( + self, + in_chan: int, + n_freqs: int, + n_head: int = 4, + hid_chan: int = 4, + act_type: str = "prelu", + norm_type: str = "LayerNormalization4D", + dim: int = 3, + *args, + **kwargs, + ): + super(MultiHeadSelfAttention2D, self).__init__() + self.in_chan = in_chan + self.n_freqs = n_freqs + self.n_head = n_head + self.hid_chan = hid_chan + self.act_type = act_type + self.norm_type = norm_type + self.dim = dim + + assert self.in_chan % self.n_head == 0 + + self.Queries = nn.ModuleList() + self.Keys = nn.ModuleList() + self.Values = nn.ModuleList() + + for _ in range(self.n_head): + self.Queries.append( + ATTConvActNorm( + in_chan=self.in_chan, + out_chan=self.hid_chan, + kernel_size=1, + act_type=self.act_type, + norm_type=self.norm_type, + n_freqs=self.n_freqs, + is2d=True, + ) + ) + self.Keys.append( + ATTConvActNorm( + in_chan=self.in_chan, + out_chan=self.hid_chan, + kernel_size=1, + act_type=self.act_type, + norm_type=self.norm_type, + n_freqs=self.n_freqs, + is2d=True, + ) + ) + self.Values.append( + ATTConvActNorm( + in_chan=self.in_chan, + out_chan=self.in_chan // self.n_head, + kernel_size=1, + act_type=self.act_type, + norm_type=self.norm_type, + n_freqs=self.n_freqs, + is2d=True, + ) + ) + + self.attn_concat_proj = ATTConvActNorm( + in_chan=self.in_chan, + out_chan=self.in_chan, + kernel_size=1, + act_type=self.act_type, + norm_type=self.norm_type, + n_freqs=self.n_freqs, + is2d=True, + ) + + def forward(self, x: torch.Tensor): + if self.dim == 4: + x = x.transpose(-2, -1).contiguous() + + batch_size, _, time, freq = x.size() + residual = x + + all_Q = [q(x) for q in self.Queries] # [B, E, T, F] + all_K = [k(x) for k in self.Keys] # [B, E, T, F] + all_V = [v(x) for v in self.Values] # [B, C/n_head, T, F] + + Q = torch.cat(all_Q, dim=0) # [B', E, T, F] B' = B*n_head + K = torch.cat(all_K, dim=0) # [B', E, T, F] + V = torch.cat(all_V, dim=0) # [B', C/n_head, T, F] + + Q = Q.transpose(1, 2).flatten(start_dim=2) # [B', T, E*F] + K = K.transpose(1, 2).flatten(start_dim=2) # [B', T, E*F] + V = V.transpose(1, 2) # [B', T, C/n_head, F] + old_shape = V.shape + V = V.flatten(start_dim=2) # [B', T, C*F/n_head] + emb_dim = Q.shape[-1] # C*F/n_head + + attn_mat = torch.matmul(Q, K.transpose(1, 2)) / (emb_dim**0.5) # [B', T, T] + attn_mat = F.softmax(attn_mat, dim=2) # [B', T, T] + V = torch.matmul(attn_mat, V) # [B', T, C*F/n_head] + V = V.reshape(old_shape) # [B', T, C/n_head, F] + V = V.transpose(1, 2) # [B', C/n_head, T, F] + emb_dim = V.shape[1] # C/n_head + + x = V.view( + [self.n_head, batch_size, emb_dim, time, freq] + ) # [n_head, B, C/n_head, T, F] + x = x.transpose(0, 1).contiguous() # [B, n_head, C/n_head, T, F] + + x = x.view([batch_size, self.n_head * emb_dim, time, freq]) # [B, C, T, F] + x = self.attn_concat_proj(x) # [B, C, T, F] + + x = x + residual + + if self.dim == 4: + x = x.transpose(-2, -1).contiguous() + + return x + + +class Recurrent(nn.Module): + def __init__( + self, + out_channels=128, + in_channels=512, + nband=8, + upsampling_depth=3, + n_head=4, + att_hid_chan=4, + kernel_size: int = 8, + stride: int = 1, + _iter=4, + ): + super().__init__() + self.nband = nband + + self.freq_path = nn.ModuleList( + [ + UConvBlock(out_channels, in_channels, upsampling_depth), + MultiHeadSelfAttention2D( + out_channels, + 1, + n_head=n_head, + hid_chan=att_hid_chan, + act_type="prelu", + norm_type="LayerNormalization4D", + dim=4, + ), + normalizations.get("LayerNormalization4D")((out_channels, 1)), + ] + ) + + self.frame_path = nn.ModuleList( + [ + UConvBlock(out_channels, in_channels, upsampling_depth), + MultiHeadSelfAttention2D( + out_channels, + 1, + n_head=n_head, + hid_chan=att_hid_chan, + act_type="prelu", + norm_type="LayerNormalization4D", + dim=4, + ), + normalizations.get("LayerNormalization4D")((out_channels, 1)), + ] + ) + + self.iter = _iter + self.concat_block = nn.Sequential( + nn.Conv2d(out_channels, out_channels, 1, 1, groups=out_channels), nn.PReLU() + ) + + def forward(self, x): + # B, nband, N, T + B, nband, N, T = x.shape + x = x.permute(0, 2, 1, 3).contiguous() # B, N, nband, T + mixture = x.clone() + for i in range(self.iter): + if i == 0: + x = self.freq_time_process(x, B, nband, N, T) # B, N, nband, T + else: + x = self.freq_time_process( + self.concat_block(mixture + x), B, nband, N, T + ) # B, N, nband, T + + return x.permute(0, 2, 1, 3).contiguous() # B, nband, N, T + + def freq_time_process(self, x, B, nband, N, T): + # Process Frequency Path + residual_1 = x.clone() + x = x.permute(0, 3, 1, 2).contiguous() # B, T, N, nband + freq_fea = self.freq_path[0](x.view(B * T, N, nband)) # B*T, N, nband + freq_fea = ( + freq_fea.view(B, T, N, nband).permute(0, 2, 1, 3).contiguous() + ) # B, N, T, nband + freq_fea = self.freq_path[1](freq_fea) # B, N, T, nband + freq_fea = self.freq_path[2](freq_fea) # B, N, T, nband + freq_fea = freq_fea.permute(0, 1, 3, 2).contiguous() + x = freq_fea + residual_1 # B, N, nband, T + # Process Frame Path + residual_2 = x.clone() + x2 = x.permute(0, 2, 1, 3).contiguous() + frame_fea = self.frame_path[0](x2.view(B * nband, N, T)) # B*nband, N, T + frame_fea = frame_fea.view(B, nband, N, T).permute(0, 2, 1, 3).contiguous() + frame_fea = self.frame_path[1](frame_fea) # B, N, nband, T + frame_fea = self.frame_path[2](frame_fea) # B, N, nband, T + x = frame_fea + residual_2 # B, N, nband, T + return x + + +class TIGER(nn.Module): + def __init__( + self, + out_channels=128, + in_channels=512, + num_blocks=16, + upsampling_depth=4, + att_n_head=4, + att_hid_chan=4, + att_kernel_size=8, + att_stride=1, + win=2048, + stride=512, + num_sources=2, + sample_rate=44100, + ): + super(TIGER, self).__init__() + + self.sample_rate = sample_rate + self.win = win + self.stride = stride + self.group = self.win // 2 + self.enc_dim = self.win // 2 + 1 + self.feature_dim = out_channels + self.num_output = num_sources + self.eps = torch.finfo(torch.float32).eps + + # 0-1k (50 hop), 1k-2k (100 hop), 2k-4k (250 hop), 4k-8k (500 hop), 8k-16k (1k hop), 16k-20k (2k hop), 20k-inf + bandwidth_50 = int(np.floor(50 / (sample_rate / 2.0) * self.enc_dim)) + bandwidth_100 = int(np.floor(100 / (sample_rate / 2.0) * self.enc_dim)) + bandwidth_250 = int(np.floor(250 / (sample_rate / 2.0) * self.enc_dim)) + bandwidth_500 = int(np.floor(500 / (sample_rate / 2.0) * self.enc_dim)) + bandwidth_1k = int(np.floor(1000 / (sample_rate / 2.0) * self.enc_dim)) + bandwidth_2k = int(np.floor(2000 / (sample_rate / 2.0) * self.enc_dim)) + self.band_width = [bandwidth_50] * 20 + self.band_width += [bandwidth_100] * 10 + self.band_width += [bandwidth_250] * 8 + self.band_width += [bandwidth_500] * 8 + self.band_width += [bandwidth_1k] * 8 + self.band_width += [bandwidth_2k] * 2 + self.band_width.append(self.enc_dim - np.sum(self.band_width)) + self.nband = len(self.band_width) + print(self.band_width) + + self.BN = nn.ModuleList([]) + for i in range(self.nband): + self.BN.append( + nn.Sequential( + nn.GroupNorm(1, self.band_width[i] * 2, self.eps), + nn.Conv1d(self.band_width[i] * 2, self.feature_dim, 1), + ) + ) + + self.separator = Recurrent( + self.feature_dim, + in_channels, + self.nband, + upsampling_depth, + att_n_head, + att_hid_chan, + att_kernel_size, + att_stride, + num_blocks, + ) + + self.mask = nn.ModuleList([]) + for i in range(self.nband): + self.mask.append( + nn.Sequential( + nn.PReLU(), + nn.Conv1d( + self.feature_dim, + self.band_width[i] * 4 * num_sources, + 1, + groups=num_sources, + ), + ) + ) + + def pad_input(self, input, window, stride): + """ + Zero-padding input according to window/stride size. + """ + batch_size, nsample = input.shape + + # pad the signals at the end for matching the window/stride size + rest = window - (stride + nsample % window) % window + if rest > 0: + pad = torch.zeros(batch_size, rest).type(input.type()) + input = torch.cat([input, pad], 1) + pad_aux = torch.zeros(batch_size, stride).type(input.type()) + input = torch.cat([pad_aux, input, pad_aux], 1) + + return input, rest + + def forward(self, input): + # input shape: (B, C, T) + was_one_d = False + if input.ndim == 1: + was_one_d = True + input = input.unsqueeze(0).unsqueeze(1) + if input.ndim == 2: + was_one_d = True + input = input.unsqueeze(1) + if input.ndim == 3: + input = input + batch_size, nch, nsample = input.shape + input = input.view(batch_size * nch, -1) + + # frequency-domain separation + spec = torch.stft( + input, + n_fft=self.win, + hop_length=self.stride, + window=torch.hann_window(self.win).type(input.dtype).to(input.device), + return_complex=True, + ) + + # print(spec.shape) + + # concat real and imag, split to subbands + spec_RI = torch.stack([spec.real, spec.imag], 1) # B*nch, 2, F, T + subband_spec_RI = [] + subband_spec = [] + band_idx = 0 + for i in range(len(self.band_width)): + subband_spec_RI.append( + spec_RI[:, :, band_idx : band_idx + self.band_width[i]].contiguous() + ) + subband_spec.append( + spec[:, band_idx : band_idx + self.band_width[i]] + ) # B*nch, BW, T + band_idx += self.band_width[i] + + # normalization and bottleneck + subband_feature = [] + for i in range(len(self.band_width)): + subband_feature.append( + self.BN[i]( + subband_spec_RI[i].view( + batch_size * nch, self.band_width[i] * 2, -1 + ) + ) + ) + subband_feature = torch.stack(subband_feature, 1) # B, nband, N, T + # import pdb; pdb.set_trace() + # separator + sep_output = self.separator( + subband_feature.view(batch_size * nch, self.nband, self.feature_dim, -1) + ) # B, nband, N, T + sep_output = sep_output.view(batch_size * nch, self.nband, self.feature_dim, -1) + + sep_subband_spec = [] + for i in range(self.nband): + this_output = self.mask[i](sep_output[:, i]).view( + batch_size * nch, 2, 2, self.num_output, self.band_width[i], -1 + ) + this_mask = this_output[:, 0] * torch.sigmoid( + this_output[:, 1] + ) # B*nch, 2, K, BW, T + this_mask_real = this_mask[:, 0] # B*nch, K, BW, T + this_mask_imag = this_mask[:, 1] # B*nch, K, BW, T + # force mask sum to 1 + this_mask_real_sum = this_mask_real.sum(1).unsqueeze(1) # B*nch, 1, BW, T + this_mask_imag_sum = this_mask_imag.sum(1).unsqueeze(1) # B*nch, 1, BW, T + this_mask_real = this_mask_real - (this_mask_real_sum - 1) / self.num_output + this_mask_imag = this_mask_imag - this_mask_imag_sum / self.num_output + est_spec_real = ( + subband_spec[i].real.unsqueeze(1) * this_mask_real + - subband_spec[i].imag.unsqueeze(1) * this_mask_imag + ) # B*nch, K, BW, T + est_spec_imag = ( + subband_spec[i].real.unsqueeze(1) * this_mask_imag + + subband_spec[i].imag.unsqueeze(1) * this_mask_real + ) # B*nch, K, BW, T + sep_subband_spec.append(torch.complex(est_spec_real, est_spec_imag)) + sep_subband_spec = torch.cat(sep_subband_spec, 2) + + output = torch.istft( + sep_subband_spec.view(batch_size * nch * self.num_output, self.enc_dim, -1), + n_fft=self.win, + hop_length=self.stride, + window=torch.hann_window(self.win).type(input.dtype).to(input.device), + length=nsample, + ) + output = output.view(batch_size * nch, self.num_output, -1) + # if was_one_d: + # return output.squeeze(0) + return output + + def get_model_args(self): + model_args = {"n_sample_rate": 2} + return model_args + + +class TIGERDNR(BaseModel): + def __init__( + self, + out_channels=132, + in_channels=256, + num_blocks=8, + upsampling_depth=5, + att_n_head=4, + att_hid_chan=4, + att_kernel_size=8, + att_stride=1, + win=2048, + stride=512, + num_sources=3, + sample_rate=44100, + ): + super(TIGERDNR, self).__init__(sample_rate=sample_rate) + self.sr = sample_rate + + self.dialog = TIGER( + out_channels=out_channels, + in_channels=in_channels, + num_blocks=num_blocks, + upsampling_depth=upsampling_depth, + att_n_head=att_n_head, + att_hid_chan=att_hid_chan, + att_kernel_size=att_kernel_size, + att_stride=att_stride, + win=win, + stride=stride, + num_sources=num_sources, + sample_rate=sample_rate, + ) + self.effect = TIGER( + out_channels=out_channels, + in_channels=in_channels, + num_blocks=num_blocks, + upsampling_depth=upsampling_depth, + att_n_head=att_n_head, + att_hid_chan=att_hid_chan, + att_kernel_size=att_kernel_size, + att_stride=att_stride, + win=win, + stride=stride, + num_sources=num_sources, + sample_rate=sample_rate, + ) + self.music = TIGER( + out_channels=out_channels, + in_channels=in_channels, + num_blocks=num_blocks, + upsampling_depth=upsampling_depth, + att_n_head=att_n_head, + att_hid_chan=att_hid_chan, + att_kernel_size=att_kernel_size, + att_stride=att_stride, + win=win, + stride=stride, + num_sources=num_sources, + sample_rate=sample_rate, + ) + + def wav_chunk_inference(self, model, mixture_tensor, target_length=12.0, hop_length=4.0, batch_size=1, n_tracks=3): + """ + Input: + mixture_tensor: Tensor, [nch, input_length] + + Output: + all_target_tensor: Tensor, [nch, n_track, input_length] + """ + + batch_mixture = mixture_tensor # [1, nch, T] + # print(batch_mixture.shape, [:,:int(self.sr*24)]) + + # split data into segments + batch_length = batch_mixture.shape[-1] + + session = int(self.sr * target_length) + target = int(self.sr * target_length) + ignore = (session - target) // 2 + hop = int(self.sr * hop_length) + tr_ratio = target_length / hop_length + if ignore > 0: + zero_pad = torch.zeros(batch_mixture.shape[0], batch_mixture.shape[1], ignore).type(batch_mixture.dtype).to(batch_mixture.device) + batch_mixture_pad = torch.cat([zero_pad, batch_mixture, zero_pad], -1) + else: + batch_mixture_pad = batch_mixture + if target - hop > 0: + hop_pad = torch.zeros(batch_mixture.shape[0], batch_mixture.shape[1], target-hop).type(batch_mixture.dtype).to(batch_mixture.device) + batch_mixture_pad = torch.cat([hop_pad, batch_mixture_pad, hop_pad], -1) + + skip_idx = ignore + target - hop + zero_pad = torch.zeros(batch_mixture.shape[0], batch_mixture.shape[1], session).type(batch_mixture.dtype).to(batch_mixture.device) + num_session = (batch_mixture_pad.shape[-1] - session) // hop + 2 + all_target = torch.zeros(batch_mixture_pad.shape[0], n_tracks, batch_mixture_pad.shape[1], batch_mixture_pad.shape[2]).to(batch_mixture_pad.device) + all_input = [] + all_segment_length = [] + + for i in range(num_session): + this_input = batch_mixture_pad[:,:,i*hop:i*hop+session] + segment_length = this_input.shape[-1] + if segment_length < session: + this_input = torch.cat([this_input, zero_pad[:,:,:session-segment_length]], -1) + all_input.append(this_input) + all_segment_length.append(segment_length) + + all_input = torch.cat(all_input, 0) + num_batch = num_session // batch_size + if num_session % batch_size > 0: + num_batch += 1 + + for i in range(num_batch): + + this_input = all_input[i*batch_size:(i+1)*batch_size] + actual_batch_size = this_input.shape[0] + with torch.no_grad(): + est_target = model(this_input) + # batch, ntrack, nch, T = est_target.shape + # est_target = est_target.transpose(1, 2).view(batch*nch, ntrack, T) + est_target = est_target.unsqueeze(2) + + for j in range(actual_batch_size): + this_est_target = est_target[j,:,:,:all_segment_length[i*batch_size+j]][:,:,ignore:ignore+target].unsqueeze(0) + all_target[:,:,:,ignore+(i*batch_size+j)*hop:ignore+(i*batch_size+j)*hop+target] += this_est_target # [batch, ntrack, nch, T] + + all_target = all_target[:,:,:,skip_idx:skip_idx+batch_length].contiguous() / tr_ratio + + return all_target.squeeze(0) + + def forward(self, mixture_tensor): + all_target_dialog = self.wav_chunk_inference(self.dialog, mixture_tensor)[2] + all_target_effect = self.wav_chunk_inference(self.effect, mixture_tensor)[1] + all_target_music = self.wav_chunk_inference(self.music, mixture_tensor)[0] + return all_target_dialog, all_target_effect, all_target_music + + def get_model_args(self): + model_args = {"n_sample_rate": 2} + return model_args \ No newline at end of file diff --git a/look2hear/system/__init__.py b/look2hear/system/__init__.py new file mode 100644 index 0000000..3e85918 --- /dev/null +++ b/look2hear/system/__init__.py @@ -0,0 +1,11 @@ +from .optimizers import make_optimizer +from .audio_litmodule import AudioLightningModule +from .audio_litmodule_multidecoder import AudioLightningModuleMultiDecoder +from .schedulers import DPTNetScheduler + +__all__ = [ + "make_optimizer", + "AudioLightningModule", + "DPTNetScheduler", + "AudioLightningModuleMultiDecoder" +] diff --git a/look2hear/system/audio_litmodule.py b/look2hear/system/audio_litmodule.py new file mode 100644 index 0000000..dc5ec6b --- /dev/null +++ b/look2hear/system/audio_litmodule.py @@ -0,0 +1,247 @@ +import torch +import pytorch_lightning as pl +from torch.optim.lr_scheduler import ReduceLROnPlateau +from collections.abc import MutableMapping +from speechbrain.processing.speech_augmentation import SpeedPerturb + +def flatten_dict(d, parent_key="", sep="_"): + """Flattens a dictionary into a single-level dictionary while preserving + parent keys. Taken from + `SO `_ + + Args: + d (MutableMapping): Dictionary to be flattened. + parent_key (str): String to use as a prefix to all subsequent keys. + sep (str): String to use as a separator between two key levels. + + Returns: + dict: Single-level dictionary, flattened. + """ + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, MutableMapping): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +class AudioLightningModule(pl.LightningModule): + def __init__( + self, + audio_model=None, + video_model=None, + optimizer=None, + loss_func=None, + train_loader=None, + val_loader=None, + test_loader=None, + scheduler=None, + config=None, + ): + super().__init__() + self.audio_model = audio_model + self.video_model = video_model + self.optimizer = optimizer + self.loss_func = loss_func + self.train_loader = train_loader + self.val_loader = val_loader + self.test_loader = test_loader + self.scheduler = scheduler + self.config = {} if config is None else config + # Speed Aug + self.speedperturb = SpeedPerturb( + self.config["datamodule"]["data_config"]["sample_rate"], + speeds=[95, 100, 105], + perturb_prob=1.0 + ) + # Save lightning"s AttributeDict under self.hparams + self.default_monitor = "val_loss/dataloader_idx_0" + self.save_hyperparameters(self.config_to_hparams(self.config)) + # self.print(self.audio_model) + self.validation_step_outputs = [] + self.test_step_outputs = [] + + + def forward(self, wav, mouth=None): + """Applies forward pass of the model. + + Returns: + :class:`torch.Tensor` + """ + return self.audio_model(wav) + + def training_step(self, batch, batch_nb): + mixtures, targets, _ = batch + + new_targets = [] + min_len = -1 + if self.config["training"]["SpeedAug"] == True: + with torch.no_grad(): + for i in range(targets.shape[1]): + new_target = self.speedperturb(targets[:, i, :]) + new_targets.append(new_target) + if i == 0: + min_len = new_target.shape[-1] + else: + if new_target.shape[-1] < min_len: + min_len = new_target.shape[-1] + + targets = torch.zeros( + targets.shape[0], + targets.shape[1], + min_len, + device=targets.device, + dtype=torch.float, + ) + for i, new_target in enumerate(new_targets): + targets[:, i, :] = new_targets[i][:, 0:min_len] + + mixtures = targets.sum(1) + # print(mixtures.shape) + est_sources = self(mixtures) + loss = self.loss_func["train"](est_sources, targets) + + self.log( + "train_loss", + loss, + on_epoch=True, + prog_bar=True, + sync_dist=True, + logger=True, + ) + + return {"loss": loss} + + + def validation_step(self, batch, batch_nb, dataloader_idx): + # cal val loss + if dataloader_idx == 0: + mixtures, targets, _ = batch + # print(mixtures.shape) + est_sources = self(mixtures) + loss = self.loss_func["val"](est_sources, targets) + self.log( + "val_loss", + loss, + on_epoch=True, + prog_bar=True, + sync_dist=True, + logger=True, + ) + + self.validation_step_outputs.append(loss) + + return {"val_loss": loss} + + # cal test loss + if (self.trainer.current_epoch) % 10 == 0 and dataloader_idx == 1: + mixtures, targets, _ = batch + # print(mixtures.shape) + est_sources = self(mixtures) + tloss = self.loss_func["val"](est_sources, targets) + self.log( + "test_loss", + tloss, + on_epoch=True, + prog_bar=True, + sync_dist=True, + logger=True, + ) + self.test_step_outputs.append(tloss) + return {"test_loss": tloss} + + def on_validation_epoch_end(self): + # val + avg_loss = torch.stack(self.validation_step_outputs).mean() + val_loss = torch.mean(self.all_gather(avg_loss)) + self.log( + "lr", + self.optimizer.param_groups[0]["lr"], + on_epoch=True, + prog_bar=True, + sync_dist=True, + ) + self.logger.experiment.log( + {"learning_rate": self.optimizer.param_groups[0]["lr"], "epoch": self.current_epoch} + ) + self.logger.experiment.log( + {"val_pit_sisnr": -val_loss, "epoch": self.current_epoch} + ) + + # test + if (self.trainer.current_epoch) % 10 == 0: + avg_loss = torch.stack(self.test_step_outputs).mean() + test_loss = torch.mean(self.all_gather(avg_loss)) + self.logger.experiment.log( + {"test_pit_sisnr": -test_loss, "epoch": self.current_epoch} + ) + self.validation_step_outputs.clear() # free memory + self.test_step_outputs.clear() # free memory + + def configure_optimizers(self): + """Initialize optimizers, batch-wise and epoch-wise schedulers.""" + if self.scheduler is None: + return self.optimizer + + if not isinstance(self.scheduler, (list, tuple)): + self.scheduler = [self.scheduler] # support multiple schedulers + + epoch_schedulers = [] + for sched in self.scheduler: + if not isinstance(sched, dict): + if isinstance(sched, ReduceLROnPlateau): + sched = {"scheduler": sched, "monitor": self.default_monitor} + epoch_schedulers.append(sched) + else: + sched.setdefault("monitor", self.default_monitor) + sched.setdefault("frequency", 1) + # Backward compat + if sched["interval"] == "batch": + sched["interval"] = "step" + assert sched["interval"] in [ + "epoch", + "step", + ], "Scheduler interval should be either step or epoch" + epoch_schedulers.append(sched) + return [self.optimizer], epoch_schedulers + + # def lr_scheduler_step(self, scheduler, optimizer_idx, metric): + # if metric is None: + # scheduler.step() + # else: + # scheduler.step(metric) + + def train_dataloader(self): + """Training dataloader""" + return self.train_loader + + def val_dataloader(self): + """Validation dataloader""" + return [self.val_loader, self.test_loader] + + def on_save_checkpoint(self, checkpoint): + """Overwrite if you want to save more things in the checkpoint.""" + checkpoint["training_config"] = self.config + return checkpoint + + @staticmethod + def config_to_hparams(dic): + """Sanitizes the config dict to be handled correctly by torch + SummaryWriter. It flatten the config dict, converts ``None`` to + ``"None"`` and any list and tuple into torch.Tensors. + + Args: + dic (dict): Dictionary to be transformed. + + Returns: + dict: Transformed dictionary. + """ + dic = flatten_dict(dic) + for k, v in dic.items(): + if v is None: + dic[k] = str(v) + elif isinstance(v, (list, tuple)): + dic[k] = torch.tensor(v) + return dic diff --git a/look2hear/system/audio_litmodule_multidecoder.py b/look2hear/system/audio_litmodule_multidecoder.py new file mode 100644 index 0000000..6b89663 --- /dev/null +++ b/look2hear/system/audio_litmodule_multidecoder.py @@ -0,0 +1,258 @@ +import torch +import pytorch_lightning as pl +from torch.optim.lr_scheduler import ReduceLROnPlateau +from collections.abc import MutableMapping +# from speechbrain.processing.speech_augmentation import SpeedPerturb + +def flatten_dict(d, parent_key="", sep="_"): + """Flattens a dictionary into a single-level dictionary while preserving + parent keys. Taken from + `SO `_ + + Args: + d (MutableMapping): Dictionary to be flattened. + parent_key (str): String to use as a prefix to all subsequent keys. + sep (str): String to use as a separator between two key levels. + + Returns: + dict: Single-level dictionary, flattened. + """ + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, MutableMapping): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +class AudioLightningModuleMultiDecoder(pl.LightningModule): + def __init__( + self, + audio_model=None, + video_model=None, + optimizer=None, + loss_func=None, + train_loader=None, + val_loader=None, + test_loader=None, + scheduler=None, + config=None, + ): + super().__init__() + self.audio_model = audio_model + self.video_model = video_model + self.optimizer = optimizer + self.loss_func = loss_func + self.train_loader = train_loader + self.val_loader = val_loader + self.test_loader = test_loader + self.scheduler = scheduler + self.config = {} if config is None else config + # Speed Aug + # self.speedperturb = SpeedPerturb( + # self.config["datamodule"]["data_config"]["sample_rate"], + # speeds=[95, 100, 105], + # perturb_prob=1.0 + # ) + # Save lightning"s AttributeDict under self.hparams + self.default_monitor = "val_loss/dataloader_idx_0" + self.save_hyperparameters(self.config_to_hparams(self.config)) + # self.print(self.audio_model) + self.validation_step_outputs = [] + self.test_step_outputs = [] + + + def forward(self, wav, mouth=None): + """Applies forward pass of the model. + + Returns: + :class:`torch.Tensor` + """ + if self.training: + return self.audio_model(wav) + else: + ests, num_blocks = self.audio_model(wav) + assert ests.shape[0] % num_blocks == 0 + batch_size = int(ests.shape[0] / num_blocks) + return ests[-batch_size:] + + def training_step(self, batch, batch_nb): + mixtures, targets, _ = batch + + new_targets = [] + min_len = -1 + # if self.config["training"]["SpeedAug"] == True: + # with torch.no_grad(): + # for i in range(targets.shape[1]): + # new_target = self.speedperturb(targets[:, i, :]) + # new_targets.append(new_target) + # if i == 0: + # min_len = new_target.shape[-1] + # else: + # if new_target.shape[-1] < min_len: + # min_len = new_target.shape[-1] + + # targets = torch.zeros( + # targets.shape[0], + # targets.shape[1], + # min_len, + # device=targets.device, + # dtype=torch.float, + # ) + # for i, new_target in enumerate(new_targets): + # targets[:, i, :] = new_targets[i][:, 0:min_len] + + # mixtures = targets.sum(1) + # print(mixtures.shape) + est_sources, num_blocks = self(mixtures) + assert est_sources.shape[0] % num_blocks == 0 + batch_size = int(est_sources.shape[0] / num_blocks) + ests_sources_each_block = [] + for i in range(num_blocks): + ests_sources_each_block.append(est_sources[i * batch_size : (i + 1) * batch_size]) + loss = self.loss_func["train"](ests_sources_each_block, targets) + + self.log( + "train_loss", + loss, + on_epoch=True, + prog_bar=True, + sync_dist=True, + logger=True, + ) + + return {"loss": loss} + + + def validation_step(self, batch, batch_nb, dataloader_idx): + # cal val loss + if dataloader_idx == 0: + mixtures, targets, _ = batch + # print(mixtures.shape) + est_sources = self(mixtures) + loss = self.loss_func["val"](est_sources, targets) + self.log( + "val_loss", + loss, + on_epoch=True, + prog_bar=True, + sync_dist=True, + logger=True, + ) + + self.validation_step_outputs.append(loss) + + return {"val_loss": loss} + + # cal test loss + if (self.trainer.current_epoch) % 10 == 0 and dataloader_idx == 1: + mixtures, targets, _ = batch + # print(mixtures.shape) + est_sources = self(mixtures) + tloss = self.loss_func["val"](est_sources, targets) + self.log( + "test_loss", + tloss, + on_epoch=True, + prog_bar=True, + sync_dist=True, + logger=True, + ) + self.test_step_outputs.append(tloss) + return {"test_loss": tloss} + + def on_validation_epoch_end(self): + # val + avg_loss = torch.stack(self.validation_step_outputs).mean() + val_loss = torch.mean(self.all_gather(avg_loss)) + self.log( + "lr", + self.optimizer.param_groups[0]["lr"], + on_epoch=True, + prog_bar=True, + sync_dist=True, + ) + self.logger.experiment.log( + {"learning_rate": self.optimizer.param_groups[0]["lr"], "epoch": self.current_epoch} + ) + self.logger.experiment.log( + {"val_pit_sisnr": -val_loss, "epoch": self.current_epoch} + ) + + # test + if (self.trainer.current_epoch) % 10 == 0: + avg_loss = torch.stack(self.test_step_outputs).mean() + test_loss = torch.mean(self.all_gather(avg_loss)) + self.logger.experiment.log( + {"test_pit_sisnr": -test_loss, "epoch": self.current_epoch} + ) + self.validation_step_outputs.clear() # free memory + self.test_step_outputs.clear() # free memory + + def configure_optimizers(self): + """Initialize optimizers, batch-wise and epoch-wise schedulers.""" + if self.scheduler is None: + return self.optimizer + + if not isinstance(self.scheduler, (list, tuple)): + self.scheduler = [self.scheduler] # support multiple schedulers + + epoch_schedulers = [] + for sched in self.scheduler: + if not isinstance(sched, dict): + if isinstance(sched, ReduceLROnPlateau): + sched = {"scheduler": sched, "monitor": self.default_monitor} + epoch_schedulers.append(sched) + else: + sched.setdefault("monitor", self.default_monitor) + sched.setdefault("frequency", 1) + # Backward compat + if sched["interval"] == "batch": + sched["interval"] = "step" + assert sched["interval"] in [ + "epoch", + "step", + ], "Scheduler interval should be either step or epoch" + epoch_schedulers.append(sched) + return [self.optimizer], epoch_schedulers + + # def lr_scheduler_step(self, scheduler, optimizer_idx, metric): + # if metric is None: + # scheduler.step() + # else: + # scheduler.step(metric) + + def train_dataloader(self): + """Training dataloader""" + return self.train_loader + + def val_dataloader(self): + """Validation dataloader""" + return [self.val_loader, self.test_loader] + + def on_save_checkpoint(self, checkpoint): + """Overwrite if you want to save more things in the checkpoint.""" + checkpoint["training_config"] = self.config + return checkpoint + + @staticmethod + def config_to_hparams(dic): + """Sanitizes the config dict to be handled correctly by torch + SummaryWriter. It flatten the config dict, converts ``None`` to + ``"None"`` and any list and tuple into torch.Tensors. + + Args: + dic (dict): Dictionary to be transformed. + + Returns: + dict: Transformed dictionary. + """ + dic = flatten_dict(dic) + for k, v in dic.items(): + if v is None: + dic[k] = str(v) + elif isinstance(v, (list, tuple)): + dic[k] = torch.tensor(v) + return dic diff --git a/look2hear/system/optimizers.py b/look2hear/system/optimizers.py new file mode 100644 index 0000000..ca3293e --- /dev/null +++ b/look2hear/system/optimizers.py @@ -0,0 +1,106 @@ +from torch.optim.optimizer import Optimizer +from torch.optim import Adam, RMSprop, SGD, Adadelta, Adagrad, Adamax, AdamW, ASGD +from torch_optimizer import ( + AccSGD, + AdaBound, + AdaMod, + DiffGrad, + Lamb, + NovoGrad, + PID, + QHAdam, + QHM, + RAdam, + SGDW, + Yogi, + Ranger, + RangerQH, + RangerVA, +) + + +__all__ = [ + "AccSGD", + "AdaBound", + "AdaMod", + "DiffGrad", + "Lamb", + "NovoGrad", + "PID", + "QHAdam", + "QHM", + "RAdam", + "SGDW", + "Yogi", + "Ranger", + "RangerQH", + "RangerVA", + "Adam", + "RMSprop", + "SGD", + "Adadelta", + "Adagrad", + "Adamax", + "AdamW", + "ASGD", + "make_optimizer", + "get", +] + + +def make_optimizer(params, optim_name="adam", **kwargs): + """ + + Args: + params (iterable): Output of `nn.Module.parameters()`. + optimizer (str or :class:`torch.optim.Optimizer`): Identifier understood + by :func:`~.get`. + **kwargs (dict): keyword arguments for the optimizer. + + Returns: + torch.optim.Optimizer + Examples + >>> from torch import nn + >>> model = nn.Sequential(nn.Linear(10, 10)) + >>> optimizer = make_optimizer(model.parameters(), optimizer='sgd', + >>> lr=1e-3) + """ + return get(optim_name)(params, **kwargs) + + +def register_optimizer(custom_opt): + """Register a custom opt, gettable with `optimzers.get`. + + Args: + custom_opt: Custom optimizer to register. + + """ + if ( + custom_opt.__name__ in globals().keys() + or custom_opt.__name__.lower() in globals().keys() + ): + raise ValueError( + f"Activation {custom_opt.__name__} already exists. Choose another name." + ) + globals().update({custom_opt.__name__: custom_opt}) + + +def get(identifier): + """Returns an optimizer function from a string. Returns its input if it + is callable (already a :class:`torch.optim.Optimizer` for example). + + Args: + identifier (str or Callable): the optimizer identifier. + + Returns: + :class:`torch.optim.Optimizer` or None + """ + if isinstance(identifier, Optimizer): + return identifier + elif isinstance(identifier, str): + to_get = {k.lower(): v for k, v in globals().items()} + cls = to_get.get(identifier.lower()) + if cls is None: + raise ValueError(f"Could not interpret optimizer : {str(identifier)}") + return cls + raise ValueError(f"Could not interpret optimizer : {str(identifier)}") diff --git a/look2hear/system/schedulers.py b/look2hear/system/schedulers.py new file mode 100644 index 0000000..085dcb7 --- /dev/null +++ b/look2hear/system/schedulers.py @@ -0,0 +1,115 @@ +import torch +from torch.optim.optimizer import Optimizer +import pytorch_lightning as pl + + +class BaseScheduler(object): + """Base class for the step-wise scheduler logic. + + Args: + optimizer (Optimize): Optimizer instance to apply lr schedule on. + + Subclass this and overwrite ``_get_lr`` to write your own step-wise scheduler. + """ + + def __init__(self, optimizer): + self.optimizer = optimizer + self.step_num = 0 + + def zero_grad(self): + self.optimizer.zero_grad() + + def _get_lr(self): + raise NotImplementedError + + def _set_lr(self, lr): + for param_group in self.optimizer.param_groups: + param_group["lr"] = lr + + def step(self, metrics=None, epoch=None): + """Update step-wise learning rate before optimizer.step.""" + self.step_num += 1 + lr = self._get_lr() + self._set_lr(lr) + + def load_state_dict(self, state_dict): + self.__dict__.update(state_dict) + + def state_dict(self): + return {key: value for key, value in self.__dict__.items() if key != "optimizer"} + + def as_tensor(self, start=0, stop=100_000): + """Returns the scheduler values from start to stop.""" + lr_list = [] + for _ in range(start, stop): + self.step_num += 1 + lr_list.append(self._get_lr()) + self.step_num = 0 + return torch.tensor(lr_list) + + def plot(self, start=0, stop=100_000): # noqa + """Plot the scheduler values from start to stop.""" + import matplotlib.pyplot as plt + + all_lr = self.as_tensor(start=start, stop=stop) + plt.plot(all_lr.numpy()) + plt.show() + +class DPTNetScheduler(BaseScheduler): + """Dual Path Transformer Scheduler used in [1] + + Args: + optimizer (Optimizer): Optimizer instance to apply lr schedule on. + steps_per_epoch (int): Number of steps per epoch. + d_model(int): The number of units in the layer output. + warmup_steps (int): The number of steps in the warmup stage of training. + noam_scale (float): Linear increase rate in first phase. + exp_max (float): Max learning rate in second phase. + exp_base (float): Exp learning rate base in second phase. + + Schedule: + This scheduler increases the learning rate linearly for the first + ``warmup_steps``, and then decay it by 0.98 for every two epochs. + + References + [1]: Jingjing Chen et al. "Dual-Path Transformer Network: Direct Context- + Aware Modeling for End-to-End Monaural Speech Separation" Interspeech 2020. + """ + + def __init__( + self, + optimizer, + steps_per_epoch, + d_model, + warmup_steps=4000, + noam_scale=1.0, + exp_max=0.0004, + exp_base=0.98, + ): + super().__init__(optimizer) + self.noam_scale = noam_scale + self.d_model = d_model + self.warmup_steps = warmup_steps + self.exp_max = exp_max + self.exp_base = exp_base + self.steps_per_epoch = steps_per_epoch + self.epoch = 0 + + def _get_lr(self): + if self.step_num % self.steps_per_epoch == 0: + self.epoch += 1 + + if self.step_num > self.warmup_steps: + # exp decaying + lr = self.exp_max * (self.exp_base ** ((self.epoch - 1) // 2)) + else: + # noam + lr = ( + self.noam_scale + * self.d_model ** (-0.5) + * min(self.step_num ** (-0.5), self.step_num * self.warmup_steps ** (-1.5)) + ) + return lr + +# Backward compat +_BaseScheduler = BaseScheduler \ No newline at end of file diff --git a/look2hear/utils/__init__.py b/look2hear/utils/__init__.py new file mode 100644 index 0000000..f9d9019 --- /dev/null +++ b/look2hear/utils/__init__.py @@ -0,0 +1,41 @@ +from .stft import STFT +from .torch_utils import pad_x_to_y, shape_reconstructed, tensors_to_device +from .parser_utils import ( + prepare_parser_from_dict, + parse_args_as_dict, + str_int_float, + str2bool, + str2bool_arg, + isfloat, + isint, +) +from .lightning_utils import print_only, RichProgressBarTheme, MyRichProgressBar, BatchesProcessedColumn, MyMetricsTextColumn +from .complex_utils import is_complex, is_torch_complex_tensor, new_complex_like +from .get_layer_from_string import get_layer +from .inversible_interface import InversibleInterface +from .nets_utils import make_pad_mask + +__all__ = [ + "STFT", + "pad_x_to_y", + "shape_reconstructed", + "tensors_to_device", + "prepare_parser_from_dict", + "parse_args_as_dict", + "str_int_float", + "str2bool", + "str2bool_arg", + "isfloat", + "isint", + "print_only", + "RichProgressBarTheme", + "MyRichProgressBar", + "BatchesProcessedColumn", + "MyMetricsTextColumn", + "is_complex", + "is_torch_complex_tensor", + "new_complex_like", + "get_layer", + "InversibleInterface", + "make_pad_mask", +] diff --git a/look2hear/utils/complex_utils.py b/look2hear/utils/complex_utils.py new file mode 100644 index 0000000..2e35319 --- /dev/null +++ b/look2hear/utils/complex_utils.py @@ -0,0 +1,191 @@ +"""Beamformer module.""" +from typing import Sequence, Tuple, Union + +import torch +from packaging.version import parse as V +from torch_complex import functional as FC +from torch_complex.tensor import ComplexTensor + +EPS = torch.finfo(torch.double).eps +is_torch_1_8_plus = V(torch.__version__) >= V("1.8.0") +is_torch_1_9_plus = V(torch.__version__) >= V("1.9.0") + + +def new_complex_like( + ref: Union[torch.Tensor, ComplexTensor], + real_imag: Tuple[torch.Tensor, torch.Tensor], +): + if isinstance(ref, ComplexTensor): + return ComplexTensor(*real_imag) + elif is_torch_complex_tensor(ref): + return torch.complex(*real_imag) + else: + raise ValueError( + "Please update your PyTorch version to 1.9+ for complex support." + ) + + +def is_torch_complex_tensor(c): + return ( + not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c) + ) + + +def is_complex(c): + return isinstance(c, ComplexTensor) or is_torch_complex_tensor(c) + + +def to_double(c): + if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c): + return c.to(dtype=torch.complex128) + else: + return c.double() + + +def to_float(c): + if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c): + return c.to(dtype=torch.complex64) + else: + return c.float() + + +def cat(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs): + if not isinstance(seq, (list, tuple)): + raise TypeError( + "cat(): argument 'tensors' (position 1) must be tuple of Tensors, " + "not Tensor" + ) + if isinstance(seq[0], ComplexTensor): + return FC.cat(seq, *args, **kwargs) + else: + return torch.cat(seq, *args, **kwargs) + + +def complex_norm( + c: Union[torch.Tensor, ComplexTensor], dim=-1, keepdim=False +) -> torch.Tensor: + if not is_complex(c): + raise TypeError("Input is not a complex tensor.") + if is_torch_complex_tensor(c): + return torch.norm(c, dim=dim, keepdim=keepdim) + else: + if dim is None: + return torch.sqrt((c.real**2 + c.imag**2).sum() + EPS) + else: + return torch.sqrt( + (c.real**2 + c.imag**2).sum(dim=dim, keepdim=keepdim) + EPS + ) + + +def einsum(equation, *operands): + # NOTE: Do not mix ComplexTensor and torch.complex in the input! + # NOTE (wangyou): Until PyTorch 1.9.0, torch.einsum does not support + # mixed input with complex and real tensors. + if len(operands) == 1: + if isinstance(operands[0], (tuple, list)): + operands = operands[0] + complex_module = FC if isinstance(operands[0], ComplexTensor) else torch + return complex_module.einsum(equation, *operands) + elif len(operands) != 2: + op0 = operands[0] + same_type = all(op.dtype == op0.dtype for op in operands[1:]) + if same_type: + _einsum = FC.einsum if isinstance(op0, ComplexTensor) else torch.einsum + return _einsum(equation, *operands) + else: + raise ValueError("0 or More than 2 operands are not supported.") + a, b = operands + if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor): + return FC.einsum(equation, a, b) + elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)): + if not torch.is_complex(a): + o_real = torch.einsum(equation, a, b.real) + o_imag = torch.einsum(equation, a, b.imag) + return torch.complex(o_real, o_imag) + elif not torch.is_complex(b): + o_real = torch.einsum(equation, a.real, b) + o_imag = torch.einsum(equation, a.imag, b) + return torch.complex(o_real, o_imag) + else: + return torch.einsum(equation, a, b) + else: + return torch.einsum(equation, a, b) + + +def inverse( + c: Union[torch.Tensor, ComplexTensor] +) -> Union[torch.Tensor, ComplexTensor]: + if isinstance(c, ComplexTensor): + return c.inverse2() + else: + return c.inverse() + + +def matmul( + a: Union[torch.Tensor, ComplexTensor], b: Union[torch.Tensor, ComplexTensor] +) -> Union[torch.Tensor, ComplexTensor]: + # NOTE: Do not mix ComplexTensor and torch.complex in the input! + # NOTE (wangyou): Until PyTorch 1.9.0, torch.matmul does not support + # multiplication between complex and real tensors. + if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor): + return FC.matmul(a, b) + elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)): + if not torch.is_complex(a): + o_real = torch.matmul(a, b.real) + o_imag = torch.matmul(a, b.imag) + return torch.complex(o_real, o_imag) + elif not torch.is_complex(b): + o_real = torch.matmul(a.real, b) + o_imag = torch.matmul(a.imag, b) + return torch.complex(o_real, o_imag) + else: + return torch.matmul(a, b) + else: + return torch.matmul(a, b) + + +def trace(a: Union[torch.Tensor, ComplexTensor]): + # NOTE (wangyou): until PyTorch 1.9.0, torch.trace does not + # support bacth processing. Use FC.trace() as fallback. + return FC.trace(a) + + +def reverse(a: Union[torch.Tensor, ComplexTensor], dim=0): + if isinstance(a, ComplexTensor): + return FC.reverse(a, dim=dim) + else: + return torch.flip(a, dims=(dim,)) + + +def solve(b: Union[torch.Tensor, ComplexTensor], a: Union[torch.Tensor, ComplexTensor]): + """Solve the linear equation ax = b.""" + # NOTE: Do not mix ComplexTensor and torch.complex in the input! + # NOTE (wangyou): Until PyTorch 1.9.0, torch.solve does not support + # mixed input with complex and real tensors. + if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor): + if isinstance(a, ComplexTensor) and isinstance(b, ComplexTensor): + return FC.solve(b, a, return_LU=False) + else: + return matmul(inverse(a), b) + elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)): + if torch.is_complex(a) and torch.is_complex(b): + return torch.linalg.solve(a, b) + else: + return matmul(inverse(a), b) + else: + if is_torch_1_8_plus: + return torch.linalg.solve(a, b) + else: + return torch.solve(b, a)[0] + + +def stack(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs): + if not isinstance(seq, (list, tuple)): + raise TypeError( + "stack(): argument 'tensors' (position 1) must be tuple of Tensors, " + "not Tensor" + ) + if isinstance(seq[0], ComplexTensor): + return FC.stack(seq, *args, **kwargs) + else: + return torch.stack(seq, *args, **kwargs) \ No newline at end of file diff --git a/look2hear/utils/get_layer_from_string.py b/look2hear/utils/get_layer_from_string.py new file mode 100644 index 0000000..3987a8d --- /dev/null +++ b/look2hear/utils/get_layer_from_string.py @@ -0,0 +1,43 @@ +import difflib + +import torch + + +def get_layer(l_name, library=torch.nn): + """Return layer object handler from library e.g. from torch.nn + + E.g. if l_name=="elu", returns torch.nn.ELU. + + Args: + l_name (string): Case insensitive name for layer in library (e.g. .'elu'). + library (module): Name of library/module where to search for object handler + with l_name e.g. "torch.nn". + + Returns: + layer_handler (object): handler for the requested layer e.g. (torch.nn.ELU) + + """ + + all_torch_layers = [x for x in dir(torch.nn)] + match = [x for x in all_torch_layers if l_name.lower() == x.lower()] + if len(match) == 0: + close_matches = difflib.get_close_matches( + l_name, [x.lower() for x in all_torch_layers] + ) + raise NotImplementedError( + "Layer with name {} not found in {}.\n Closest matches: {}".format( + l_name, str(library), close_matches + ) + ) + elif len(match) > 1: + close_matches = difflib.get_close_matches( + l_name, [x.lower() for x in all_torch_layers] + ) + raise NotImplementedError( + "Multiple matchs for layer with name {} not found in {}.\n " + "All matches: {}".format(l_name, str(library), close_matches) + ) + else: + # valid + layer_handler = getattr(library, match[0]) + return layer_handler \ No newline at end of file diff --git a/look2hear/utils/inversible_interface.py b/look2hear/utils/inversible_interface.py new file mode 100644 index 0000000..919107e --- /dev/null +++ b/look2hear/utils/inversible_interface.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod +from typing import Tuple + +import torch + + +class InversibleInterface(ABC): + @abstractmethod + def inverse( + self, input: torch.Tensor, input_lengths: torch.Tensor = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + # return output, output_lengths + raise NotImplementedError \ No newline at end of file diff --git a/look2hear/utils/lightning_utils.py b/look2hear/utils/lightning_utils.py new file mode 100644 index 0000000..29f2f73 --- /dev/null +++ b/look2hear/utils/lightning_utils.py @@ -0,0 +1,104 @@ +from rich import print +from dataclasses import dataclass +from pytorch_lightning.utilities import rank_zero_only +from typing import Union +from pytorch_lightning.callbacks.progress.rich_progress import * +from rich.console import Console, RenderableType +from rich.progress_bar import ProgressBar +from rich.style import Style +from rich.text import Text +from rich.progress import ( + BarColumn, + DownloadColumn, + Progress, + TaskID, + TextColumn, + TimeRemainingColumn, + TransferSpeedColumn, + ProgressColumn +) +from rich import print, reconfigure + +@rank_zero_only +def print_only(message: str): + print(message) + +@dataclass +class RichProgressBarTheme: + """Styles to associate to different base components. + + Args: + description: Style for the progress bar description. For eg., Epoch x, Testing, etc. + progress_bar: Style for the bar in progress. + progress_bar_finished: Style for the finished progress bar. + progress_bar_pulse: Style for the progress bar when `IterableDataset` is being processed. + batch_progress: Style for the progress tracker (i.e 10/50 batches completed). + time: Style for the processed time and estimate time remaining. + processing_speed: Style for the speed of the batches being processed. + metrics: Style for the metrics + + https://rich.readthedocs.io/en/stable/style.html + """ + + description: Union[str, Style] = "#FF4500" + progress_bar: Union[str, Style] = "#f92672" + progress_bar_finished: Union[str, Style] = "#b7cc8a" + progress_bar_pulse: Union[str, Style] = "#f92672" + batch_progress: Union[str, Style] = "#fc608a" + time: Union[str, Style] = "#45ada2" + processing_speed: Union[str, Style] = "#DC143C" + metrics: Union[str, Style] = "#228B22" + +class BatchesProcessedColumn(ProgressColumn): + def __init__(self, style: Union[str, Style]): + self.style = style + super().__init__() + + def render(self, task) -> RenderableType: + total = task.total if task.total != float("inf") else "--" + return Text(f"{int(task.completed)}/{int(total)}", style=self.style) + +class MyMetricsTextColumn(ProgressColumn): + """A column containing text.""" + + def __init__(self, style): + self._tasks = {} + self._current_task_id = 0 + self._metrics = {} + self._style = style + super().__init__() + + def update(self, metrics): + # Called when metrics are ready to be rendered. + # This is to prevent render from causing deadlock issues by requesting metrics + # in separate threads. + self._metrics = metrics + + def render(self, task) -> Text: + text = "" + for k, v in self._metrics.items(): + text += f"{k}: {round(v, 3) if isinstance(v, float) else v} " + return Text(text, justify="left", style=self._style) + +class MyRichProgressBar(RichProgressBar): + """A progress bar prints metrics at the end of each epoch + """ + + def _init_progress(self, trainer): + if self.is_enabled and (self.progress is None or self._progress_stopped): + self._reset_progress_bar_ids() + reconfigure(**self._console_kwargs) + # file = open("Look2Hear/Experiments/run_logs/EdgeFRCNN-Noncausal.log", 'w') + self._console: Console = Console(force_terminal=True) + self._console.clear_live() + self._metric_component = MetricsTextColumn(trainer, self.theme.metrics) + self.progress = CustomProgress( + *self.configure_columns(trainer), + self._metric_component, + auto_refresh=False, + disable=self.is_disabled, + console=self._console, + ) + self.progress.start() + # progress has started + self._progress_stopped = False \ No newline at end of file diff --git a/look2hear/utils/nets_utils.py b/look2hear/utils/nets_utils.py new file mode 100644 index 0000000..e7a6770 --- /dev/null +++ b/look2hear/utils/nets_utils.py @@ -0,0 +1,503 @@ +# -*- coding: utf-8 -*- + +"""Network related utility tools.""" + +import logging +from typing import Dict + +import numpy as np +import torch + + +def to_device(m, x): + """Send tensor into the device of the module. + + Args: + m (torch.nn.Module): Torch module. + x (Tensor): Torch tensor. + + Returns: + Tensor: Torch tensor located in the same place as torch module. + + """ + if isinstance(m, torch.nn.Module): + device = next(m.parameters()).device + elif isinstance(m, torch.Tensor): + device = m.device + else: + raise TypeError( + "Expected torch.nn.Module or torch.tensor, " f"bot got: {type(m)}" + ) + return x.to(device) + + +def pad_list(xs, pad_value): + """Perform padding for the list of tensors. + + Args: + xs (List): List of Tensors [(T_1, `*`), (T_2, `*`), ..., (T_B, `*`)]. + pad_value (float): Value for padding. + + Returns: + Tensor: Padded tensor (B, Tmax, `*`). + + Examples: + >>> x = [torch.ones(4), torch.ones(2), torch.ones(1)] + >>> x + [tensor([1., 1., 1., 1.]), tensor([1., 1.]), tensor([1.])] + >>> pad_list(x, 0) + tensor([[1., 1., 1., 1.], + [1., 1., 0., 0.], + [1., 0., 0., 0.]]) + + """ + n_batch = len(xs) + max_len = max(x.size(0) for x in xs) + pad = xs[0].new(n_batch, max_len, *xs[0].size()[1:]).fill_(pad_value) + + for i in range(n_batch): + pad[i, : xs[i].size(0)] = xs[i] + + return pad + + +def make_pad_mask(lengths, xs=None, length_dim=-1, maxlen=None): + """Make mask tensor containing indices of padded part. + + Args: + lengths (LongTensor or List): Batch of lengths (B,). + xs (Tensor, optional): The reference tensor. + If set, masks will be the same shape as this tensor. + length_dim (int, optional): Dimension indicator of the above tensor. + See the example. + + Returns: + Tensor: Mask tensor containing indices of padded part. + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (including 1.2) + + Examples: + With only lengths. + + >>> lengths = [5, 3, 2] + >>> make_pad_mask(lengths) + masks = [[0, 0, 0, 0 ,0], + [0, 0, 0, 1, 1], + [0, 0, 1, 1, 1]] + + With the reference tensor. + + >>> xs = torch.zeros((3, 2, 4)) + >>> make_pad_mask(lengths, xs) + tensor([[[0, 0, 0, 0], + [0, 0, 0, 0]], + [[0, 0, 0, 1], + [0, 0, 0, 1]], + [[0, 0, 1, 1], + [0, 0, 1, 1]]], dtype=torch.uint8) + >>> xs = torch.zeros((3, 2, 6)) + >>> make_pad_mask(lengths, xs) + tensor([[[0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1]], + [[0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1]], + [[0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8) + + With the reference tensor and dimension indicator. + + >>> xs = torch.zeros((3, 6, 6)) + >>> make_pad_mask(lengths, xs, 1) + tensor([[[0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1]], + [[0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1]], + [[0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1]]], dtype=torch.uint8) + >>> make_pad_mask(lengths, xs, 2) + tensor([[[0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1]], + [[0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1]], + [[0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8) + + """ + if length_dim == 0: + raise ValueError("length_dim cannot be 0: {}".format(length_dim)) + + if not isinstance(lengths, list): + lengths = lengths.long().tolist() + + bs = int(len(lengths)) + if maxlen is None: + if xs is None: + maxlen = int(max(lengths)) + else: + maxlen = xs.size(length_dim) + else: + assert xs is None + assert maxlen >= int(max(lengths)) + + seq_range = torch.arange(0, maxlen, dtype=torch.int64) + seq_range_expand = seq_range.unsqueeze(0).expand(bs, maxlen) + seq_length_expand = seq_range_expand.new(lengths).unsqueeze(-1) + mask = seq_range_expand >= seq_length_expand + + if xs is not None: + assert xs.size(0) == bs, (xs.size(0), bs) + + if length_dim < 0: + length_dim = xs.dim() + length_dim + # ind = (:, None, ..., None, :, , None, ..., None) + ind = tuple( + slice(None) if i in (0, length_dim) else None for i in range(xs.dim()) + ) + mask = mask[ind].expand_as(xs).to(xs.device) + return mask + + +def make_non_pad_mask(lengths, xs=None, length_dim=-1): + """Make mask tensor containing indices of non-padded part. + + Args: + lengths (LongTensor or List): Batch of lengths (B,). + xs (Tensor, optional): The reference tensor. + If set, masks will be the same shape as this tensor. + length_dim (int, optional): Dimension indicator of the above tensor. + See the example. + + Returns: + ByteTensor: mask tensor containing indices of padded part. + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (including 1.2) + + Examples: + With only lengths. + + >>> lengths = [5, 3, 2] + >>> make_non_pad_mask(lengths) + masks = [[1, 1, 1, 1 ,1], + [1, 1, 1, 0, 0], + [1, 1, 0, 0, 0]] + + With the reference tensor. + + >>> xs = torch.zeros((3, 2, 4)) + >>> make_non_pad_mask(lengths, xs) + tensor([[[1, 1, 1, 1], + [1, 1, 1, 1]], + [[1, 1, 1, 0], + [1, 1, 1, 0]], + [[1, 1, 0, 0], + [1, 1, 0, 0]]], dtype=torch.uint8) + >>> xs = torch.zeros((3, 2, 6)) + >>> make_non_pad_mask(lengths, xs) + tensor([[[1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0]], + [[1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0]], + [[1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0]]], dtype=torch.uint8) + + With the reference tensor and dimension indicator. + + >>> xs = torch.zeros((3, 6, 6)) + >>> make_non_pad_mask(lengths, xs, 1) + tensor([[[1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0]], + [[1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0]], + [[1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0]]], dtype=torch.uint8) + >>> make_non_pad_mask(lengths, xs, 2) + tensor([[[1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0]], + [[1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0]], + [[1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0]]], dtype=torch.uint8) + + """ + return ~make_pad_mask(lengths, xs, length_dim) + + +def mask_by_length(xs, lengths, fill=0): + """Mask tensor according to length. + + Args: + xs (Tensor): Batch of input tensor (B, `*`). + lengths (LongTensor or List): Batch of lengths (B,). + fill (int or float): Value to fill masked part. + + Returns: + Tensor: Batch of masked input tensor (B, `*`). + + Examples: + >>> x = torch.arange(5).repeat(3, 1) + 1 + >>> x + tensor([[1, 2, 3, 4, 5], + [1, 2, 3, 4, 5], + [1, 2, 3, 4, 5]]) + >>> lengths = [5, 3, 2] + >>> mask_by_length(x, lengths) + tensor([[1, 2, 3, 4, 5], + [1, 2, 3, 0, 0], + [1, 2, 0, 0, 0]]) + + """ + assert xs.size(0) == len(lengths) + ret = xs.data.new(*xs.size()).fill_(fill) + for i, l in enumerate(lengths): + ret[i, :l] = xs[i, :l] + return ret + + +def th_accuracy(pad_outputs, pad_targets, ignore_label): + """Calculate accuracy. + + Args: + pad_outputs (Tensor): Prediction tensors (B * Lmax, D). + pad_targets (LongTensor): Target label tensors (B, Lmax, D). + ignore_label (int): Ignore label id. + + Returns: + float: Accuracy value (0.0 - 1.0). + + """ + pad_pred = pad_outputs.view( + pad_targets.size(0), pad_targets.size(1), pad_outputs.size(1) + ).argmax(2) + mask = pad_targets != ignore_label + numerator = torch.sum( + pad_pred.masked_select(mask) == pad_targets.masked_select(mask) + ) + denominator = torch.sum(mask) + return float(numerator) / float(denominator) + + +def to_torch_tensor(x): + """Change to torch.Tensor or ComplexTensor from numpy.ndarray. + + Args: + x: Inputs. It should be one of numpy.ndarray, Tensor, ComplexTensor, and dict. + + Returns: + Tensor or ComplexTensor: Type converted inputs. + + Examples: + >>> xs = np.ones(3, dtype=np.float32) + >>> xs = to_torch_tensor(xs) + tensor([1., 1., 1.]) + >>> xs = torch.ones(3, 4, 5) + >>> assert to_torch_tensor(xs) is xs + >>> xs = {'real': xs, 'imag': xs} + >>> to_torch_tensor(xs) + ComplexTensor( + Real: + tensor([1., 1., 1.]) + Imag; + tensor([1., 1., 1.]) + ) + + """ + # If numpy, change to torch tensor + if isinstance(x, np.ndarray): + if x.dtype.kind == "c": + # Dynamically importing because torch_complex requires python3 + from torch_complex.tensor import ComplexTensor + + return ComplexTensor(x) + else: + return torch.from_numpy(x) + + # If {'real': ..., 'imag': ...}, convert to ComplexTensor + elif isinstance(x, dict): + # Dynamically importing because torch_complex requires python3 + from torch_complex.tensor import ComplexTensor + + if "real" not in x or "imag" not in x: + raise ValueError("has 'real' and 'imag' keys: {}".format(list(x))) + # Relative importing because of using python3 syntax + return ComplexTensor(x["real"], x["imag"]) + + # If torch.Tensor, as it is + elif isinstance(x, torch.Tensor): + return x + + else: + error = ( + "x must be numpy.ndarray, torch.Tensor or a dict like " + "{{'real': torch.Tensor, 'imag': torch.Tensor}}, " + "but got {}".format(type(x)) + ) + try: + from torch_complex.tensor import ComplexTensor + except Exception: + # If PY2 + raise ValueError(error) + else: + # If PY3 + if isinstance(x, ComplexTensor): + return x + else: + raise ValueError(error) + + +def get_subsample(train_args, mode, arch): + """Parse the subsampling factors from the args for the specified `mode` and `arch`. + + Args: + train_args: argument Namespace containing options. + mode: one of ('asr', 'mt', 'st') + arch: one of ('rnn', 'rnn-t', 'rnn_mix', 'rnn_mulenc', 'transformer') + + Returns: + np.ndarray / List[np.ndarray]: subsampling factors. + """ + if arch == "transformer": + return np.array([1]) + + elif mode == "mt" and arch == "rnn": + # +1 means input (+1) and layers outputs (train_args.elayer) + subsample = np.ones(train_args.elayers + 1, dtype=np.int64) + logging.warning("Subsampling is not performed for machine translation.") + logging.info("subsample: " + " ".join([str(x) for x in subsample])) + return subsample + + elif ( + (mode == "asr" and arch in ("rnn", "rnn-t")) + or (mode == "mt" and arch == "rnn") + or (mode == "st" and arch == "rnn") + ): + subsample = np.ones(train_args.elayers + 1, dtype=np.int64) + if train_args.etype.endswith("p") and not train_args.etype.startswith("vgg"): + ss = train_args.subsample.split("_") + for j in range(min(train_args.elayers + 1, len(ss))): + subsample[j] = int(ss[j]) + else: + logging.warning( + "Subsampling is not performed for vgg*. " + "It is performed in max pooling layers at CNN." + ) + logging.info("subsample: " + " ".join([str(x) for x in subsample])) + return subsample + + elif mode == "asr" and arch == "rnn_mix": + subsample = np.ones( + train_args.elayers_sd + train_args.elayers + 1, dtype=np.int64 + ) + if train_args.etype.endswith("p") and not train_args.etype.startswith("vgg"): + ss = train_args.subsample.split("_") + for j in range( + min(train_args.elayers_sd + train_args.elayers + 1, len(ss)) + ): + subsample[j] = int(ss[j]) + else: + logging.warning( + "Subsampling is not performed for vgg*. " + "It is performed in max pooling layers at CNN." + ) + logging.info("subsample: " + " ".join([str(x) for x in subsample])) + return subsample + + elif mode == "asr" and arch == "rnn_mulenc": + subsample_list = [] + for idx in range(train_args.num_encs): + subsample = np.ones(train_args.elayers[idx] + 1, dtype=np.int64) + if train_args.etype[idx].endswith("p") and not train_args.etype[ + idx + ].startswith("vgg"): + ss = train_args.subsample[idx].split("_") + for j in range(min(train_args.elayers[idx] + 1, len(ss))): + subsample[j] = int(ss[j]) + else: + logging.warning( + "Encoder %d: Subsampling is not performed for vgg*. " + "It is performed in max pooling layers at CNN.", + idx + 1, + ) + logging.info("subsample: " + " ".join([str(x) for x in subsample])) + subsample_list.append(subsample) + return subsample_list + + else: + raise ValueError("Invalid options: mode={}, arch={}".format(mode, arch)) + + +def rename_state_dict( + old_prefix: str, new_prefix: str, state_dict: Dict[str, torch.Tensor] +): + """Replace keys of old prefix with new prefix in state dict.""" + # need this list not to break the dict iterator + old_keys = [k for k in state_dict if k.startswith(old_prefix)] + if len(old_keys) > 0: + logging.warning(f"Rename: {old_prefix} -> {new_prefix}") + for k in old_keys: + v = state_dict.pop(k) + new_k = k.replace(old_prefix, new_prefix) + state_dict[new_k] = v + + +def get_activation(act): + """Return activation function.""" + # Lazy load to avoid unused import + from espnet.nets.pytorch_backend.conformer.swish import Swish + + activation_funcs = { + "hardtanh": torch.nn.Hardtanh, + "tanh": torch.nn.Tanh, + "relu": torch.nn.ReLU, + "selu": torch.nn.SELU, + "swish": Swish, + } + + return activation_funcs[act]() \ No newline at end of file diff --git a/look2hear/utils/parser_utils.py b/look2hear/utils/parser_utils.py new file mode 100644 index 0000000..6cec1ea --- /dev/null +++ b/look2hear/utils/parser_utils.py @@ -0,0 +1,151 @@ +import sys +import argparse + + +def prepare_parser_from_dict(dic, parser=None): + """Prepare an argparser from a dictionary. + + Args: + dic (dict): Two-level config dictionary with unique bottom-level keys. + parser (argparse.ArgumentParser, optional): If a parser already + exists, add the keys from the dictionary on the top of it. + + Returns: + argparse.ArgumentParser: + Parser instance with groups corresponding to the first level keys + and arguments corresponding to the second level keys with default + values given by the values. + """ + + def standardized_entry_type(value): + """If the default value is None, replace NoneType by str_int_float. + If the default value is boolean, look for boolean strings.""" + if value is None: + return str_int_float + if isinstance(str2bool(value), bool): + return str2bool_arg + return type(value) + + if parser is None: + parser = argparse.ArgumentParser() + for k in dic.keys(): + group = parser.add_argument_group(k) + if isinstance(dic[k], list): + entry_type = standardized_entry_type(dic[k]) + group.add_argument("--" + k, default=dic[k], type=entry_type) + elif isinstance(dic[k], dict): + for kk in dic[k].keys(): + entry_type = standardized_entry_type(dic[k][kk]) + group.add_argument("--" + kk, default=dic[k][kk], type=entry_type) + elif isinstance(dic[k], str): + entry_type = standardized_entry_type(dic[k]) + group.add_argument("--" + k, default=dic[k], type=entry_type) + return parser + + +def str_int_float(value): + """Type to convert strings to int, float (in this order) if possible. + + Args: + value (str): Value to convert. + + Returns: + int, float, str: Converted value. + """ + if isint(value): + return int(value) + if isfloat(value): + return float(value) + elif isinstance(value, str): + return value + + +def str2bool(value): + """Type to convert strings to Boolean (returns input if not boolean)""" + if not isinstance(value, str): + return value + if value.lower() in ("yes", "true", "y", "1"): + return True + elif value.lower() in ("no", "false", "n", "0"): + return False + else: + return value + + +def str2bool_arg(value): + """Argparse type to convert strings to Boolean""" + value = str2bool(value) + if isinstance(value, bool): + return value + raise argparse.ArgumentTypeError("Boolean value expected.") + + +def isfloat(value): + """Computes whether `value` can be cast to a float. + + Args: + value (str): Value to check. + + Returns: + bool: Whether `value` can be cast to a float. + + """ + try: + float(value) + return True + except ValueError: + return False + + +def isint(value): + """Computes whether `value` can be cast to an int + + Args: + value (str): Value to check. + + Returns: + bool: Whether `value` can be cast to an int. + + """ + try: + int(value) + return True + except ValueError: + return False + + +def parse_args_as_dict(parser, return_plain_args=False, args=None): + """Get a dict of dicts out of process `parser.parse_args()` + + Top-level keys corresponding to groups and bottom-level keys corresponding + to arguments. Under `'main_args'`, the arguments which don't belong to a + argparse group (i.e main arguments defined before parsing from a dict) can + be found. + + Args: + parser (argparse.ArgumentParser): ArgumentParser instance containing + groups. Output of `prepare_parser_from_dict`. + return_plain_args (bool): Whether to return the output or + `parser.parse_args()`. + args (list): List of arguments as read from the command line. + Used for unit testing. + + Returns: + dict: + Dictionary of dictionaries containing the arguments. Optionally the + direct output `parser.parse_args()`. + """ + args = parser.parse_args(args=args) + args_dic = {} + for group in parser._action_groups: + group_dict = {a.dest: getattr(args, a.dest, None) for a in group._group_actions} + args_dic[group.title] = group_dict + if sys.version_info.minor == 10: + args_dic["main_args"] = args_dic["positional arguments"] + del args_dic["positional arguments"] + else: + args_dic["main_args"] = args_dic["optional arguments"] + del args_dic["optional arguments"] + if return_plain_args: + return args_dic, args + return args_dic diff --git a/look2hear/utils/separator.py b/look2hear/utils/separator.py new file mode 100644 index 0000000..f38077b --- /dev/null +++ b/look2hear/utils/separator.py @@ -0,0 +1,69 @@ +import os +import warnings +import torch +import numpy as np +import soundfile as sf + + +def get_device(tensor_or_module, default=None): + if hasattr(tensor_or_module, "device"): + return tensor_or_module.device + elif hasattr(tensor_or_module, "parameters"): + return next(tensor_or_module.parameters()).device + elif default is None: + raise TypeError( + f"Don't know how to get device of {type(tensor_or_module)} object" + ) + else: + return torch.device(default) + + +class Separator: + def forward_wav(self, wav, **kwargs): + raise NotImplementedError + + def sample_rate(self): + raise NotImplementedError + + +def separate(model, wav, **kwargs): + if isinstance(wav, np.ndarray): + return numpy_separate(model, wav, **kwargs) + elif isinstance(wav, torch.Tensor): + return torch_separate(model, wav, **kwargs) + else: + raise ValueError( + f"Only support filenames, numpy arrays and torch tensors, received {type(wav)}" + ) + + +@torch.no_grad() +def torch_separate(model: Separator, wav: torch.Tensor, **kwargs) -> torch.Tensor: + """Core logic of `separate`.""" + if model.in_channels is not None and wav.shape[-2] != model.in_channels: + raise RuntimeError( + f"Model supports {model.in_channels}-channel inputs but found audio with {wav.shape[-2]} channels." + f"Please match the number of channels." + ) + # Handle device placement + input_device = get_device(wav, default="cpu") + model_device = get_device(model, default="cpu") + wav = wav.to(model_device) + # Forward + separate_func = getattr(model, "forward_wav", model) + out_wavs = separate_func(wav, **kwargs) + + # FIXME: for now this is the best we can do. + out_wavs *= wav.abs().sum() / (out_wavs.abs().sum()) + + # Back to input device (and numpy if necessary) + out_wavs = out_wavs.to(input_device) + return out_wavs + + +def numpy_separate(model: Separator, wav: np.ndarray, **kwargs) -> np.ndarray: + """Numpy interface to `separate`.""" + wav = torch.from_numpy(wav) + out_wavs = torch_separate(model, wav, **kwargs) + out_wavs = out_wavs.data.numpy() + return out_wavs diff --git a/look2hear/utils/stft.py b/look2hear/utils/stft.py new file mode 100644 index 0000000..0bef618 --- /dev/null +++ b/look2hear/utils/stft.py @@ -0,0 +1,797 @@ +# Copyright 2019 Jian Wu +# License: Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +import math + +import numpy as np +import torch as th +import torch.nn as nn +import torch.nn.functional as tf +import librosa.filters as filters + +from typing import Optional, Tuple +from distutils.version import LooseVersion + +EPSILON = float(np.finfo(np.float32).eps) +TORCH_VERSION = th.__version__ + +if TORCH_VERSION >= LooseVersion("1.7"): + from torch.fft import fft as fft_func +else: + pass + + +def export_jit(transform: nn.Module) -> nn.Module: + """ + Export transform module for inference + """ + export_out = [module for module in transform if module.exportable()] + return nn.Sequential(*export_out) + + +def init_window(wnd: str, frame_len: int, device: th.device = "cpu") -> th.Tensor: + """ + Return window coefficient + Args: + wnd: window name + frame_len: length of the frame + """ + + def sqrthann(frame_len, periodic=True): + return th.hann_window(frame_len, periodic=periodic) ** 0.5 + + if wnd not in ["bartlett", "hann", "hamm", "blackman", "rect", "sqrthann"]: + raise RuntimeError(f"Unknown window type: {wnd}") + + wnd_tpl = { + "sqrthann": sqrthann, + "hann": th.hann_window, + "hamm": th.hamming_window, + "blackman": th.blackman_window, + "bartlett": th.bartlett_window, + "rect": th.ones, + } + if wnd != "rect": + # match with librosa + c = wnd_tpl[wnd](frame_len, periodic=True) + else: + c = wnd_tpl[wnd](frame_len) + return c.to(device) + + +def init_kernel( + frame_len: int, + frame_hop: int, + window: th.Tensor, + round_pow_of_two: bool = True, + normalized: bool = False, + inverse: bool = False, + mode: str = "librosa", +) -> Tuple[th.Tensor, th.Tensor]: + """ + Return STFT kernels + Args: + frame_len: length of the frame + frame_hop: hop size between frames + window: window tensor + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + normalized: return normalized DFT matrix + inverse: return iDFT matrix + mode: framing mode (librosa or kaldi) + """ + if mode not in ["librosa", "kaldi"]: + raise ValueError(f"Unsupported mode: {mode}") + # FFT size: B + if round_pow_of_two or mode == "kaldi": + fft_size = 2 ** math.ceil(math.log2(frame_len)) + else: + fft_size = frame_len + # center padding window if needed + if mode == "librosa" and fft_size != frame_len: + lpad = (fft_size - frame_len) // 2 + window = tf.pad(window, (lpad, fft_size - frame_len - lpad)) + if normalized: + # make K^H * K = I + S = fft_size ** 0.5 + else: + S = 1 + # W x B x 2 + if TORCH_VERSION >= LooseVersion("1.7"): + K = fft_func(th.eye(fft_size) / S, dim=-1) + K = th.stack([K.real, K.imag], dim=-1) + else: + I = th.stack([th.eye(fft_size), th.zeros(fft_size, fft_size)], dim=-1) + K = th.fft(I / S, 1) + if mode == "kaldi": + K = K[:frame_len] + if inverse and not normalized: + # to make K^H * K = I + K = K / fft_size + # 2 x B x W + K = th.transpose(K, 0, 2) + # 2B x 1 x W + K = th.reshape(K, (fft_size * 2, 1, K.shape[-1])) + return K.to(window.device), window + + +def mel_filter( + frame_len: int, + round_pow_of_two: bool = True, + num_bins: Optional[int] = None, + sr: int = 16000, + num_mels: int = 80, + fmin: float = 0.0, + fmax: Optional[float] = None, + norm: bool = False, +) -> th.Tensor: + """ + Return mel filter coefficients + Args: + frame_len: length of the frame + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + num_bins: number of the frequency bins produced by STFT + num_mels: number of the mel bands + fmin: lowest frequency (in Hz) + fmax: highest frequency (in Hz) + norm: normalize the mel filter coefficients + """ + # FFT points + if num_bins is None: + N = 2 ** math.ceil(math.log2(frame_len)) if round_pow_of_two else frame_len + else: + N = (num_bins - 1) * 2 + # fmin & fmax + freq_upper = sr // 2 + if fmax is None: + fmax = freq_upper + else: + fmax = min(fmax + freq_upper if fmax < 0 else fmax, freq_upper) + fmin = max(0, fmin) + # mel filter coefficients + mel = filters.mel( + sr, + N, + n_mels=num_mels, + fmax=fmax, + fmin=fmin, + htk=True, + norm="slaney" if norm else None, + ) + # num_mels x (N // 2 + 1) + return th.tensor(mel, dtype=th.float32) + + +def speed_perturb_filter( + src_sr: int, dst_sr: int, cutoff_ratio: float = 0.95, num_zeros: int = 64 +) -> th.Tensor: + """ + Return speed perturb filters, reference: + https://github.com/danpovey/filtering/blob/master/lilfilter/resampler.py + Args: + src_sr: sample rate of the source signal + dst_sr: sample rate of the target signal + Return: + weight (Tensor): coefficients of the filter + """ + if src_sr == dst_sr: + raise ValueError(f"src_sr should not be equal to dst_sr: {src_sr}/{dst_sr}") + gcd = math.gcd(src_sr, dst_sr) + src_sr = src_sr // gcd + dst_sr = dst_sr // gcd + if src_sr == 1 or dst_sr == 1: + raise ValueError("do not support integer downsample/upsample") + zeros_per_block = min(src_sr, dst_sr) * cutoff_ratio + padding = 1 + int(num_zeros / zeros_per_block) + # dst_sr x src_sr x K + times = ( + np.arange(dst_sr)[:, None, None] / float(dst_sr) + - np.arange(src_sr)[None, :, None] / float(src_sr) + - np.arange(2 * padding + 1)[None, None, :] + + padding + ) + window = np.heaviside(1 - np.abs(times / padding), 0.0) * ( + 0.5 + 0.5 * np.cos(times / padding * math.pi) + ) + weight = np.sinc(times * zeros_per_block) * window * zeros_per_block / float(src_sr) + return th.tensor(weight, dtype=th.float32) + + +def splice_feature( + feats: th.Tensor, lctx: int = 1, rctx: int = 1, op: str = "cat" +) -> th.Tensor: + """ + Splice feature + Args: + feats (Tensor): N x ... x T x F, original feature + lctx: left context + rctx: right context + op: operator on feature context + Return: + splice (Tensor): feature with context padded + """ + if lctx + rctx == 0: + return feats + if op not in ["cat", "stack"]: + raise ValueError(f"Unknown op for feature splicing: {op}") + # [N x ... x T x F, ...] + ctx = [] + T = feats.shape[-2] + for c in range(-lctx, rctx + 1): + idx = th.arange(c, c + T, device=feats.device, dtype=th.int64) + idx = th.clamp(idx, min=0, max=T - 1) + ctx.append(th.index_select(feats, -2, idx)) + if op == "cat": + # N x ... x T x FD + splice = th.cat(ctx, -1) + else: + # N x ... x T x F x D + splice = th.stack(ctx, -1) + return splice + + +def _forward_stft( + wav: th.Tensor, + kernel: th.Tensor, + window: th.Tensor, + return_polar: bool = False, + pre_emphasis: float = 0, + frame_hop: int = 256, + onesided: bool = False, + center: bool = False, + eps: float = EPSILON, +) -> th.Tensor: + """ + STFT function implemented by conv1d (not efficient, but we don't care during training) + Args: + wav (Tensor): N x (C) x S + kernel (Tensor): STFT transform kernels, from init_kernel(...) + return_polar: return [magnitude; phase] Tensor or [real; imag] Tensor + pre_emphasis: factor of preemphasis + frame_hop: frame hop size in number samples + onesided: return half FFT bins + center: if true, we assumed to have centered frames + Return: + transform (Tensor): STFT transform results + """ + wav_dim = wav.dim() + if wav_dim not in [2, 3]: + raise RuntimeError(f"STFT expect 2D/3D tensor, but got {wav_dim:d}D") + # if N x S, reshape N x 1 x S + # else: reshape NC x 1 x S + N, S = wav.shape[0], wav.shape[-1] + wav = wav.view(-1, 1, S) + # NC x 1 x S+2P + if center: + pad = kernel.shape[-1] // 2 + # NOTE: match with librosa + wav = tf.pad(wav, (pad, pad), mode="reflect") + # STFT + kernel = kernel * window + if pre_emphasis > 0: + # NC x W x T + frames = tf.unfold( + wav[:, None], (1, kernel.shape[-1]), stride=frame_hop, padding=0 + ) + # follow Kaldi's Preemphasize + frames[:, 1:] = frames[:, 1:] - pre_emphasis * frames[:, :-1] + frames[:, 0] *= 1 - pre_emphasis + # 1 x 2B x W, NC x W x T, NC x 2B x T + packed = th.matmul(kernel[:, 0][None, ...], frames) + else: + packed = tf.conv1d(wav, kernel, stride=frame_hop, padding=0) + # NC x 2B x T => N x C x 2B x T + if wav_dim == 3: + packed = packed.view(N, -1, packed.shape[-2], packed.shape[-1]) + # N x (C) x B x T + real, imag = th.chunk(packed, 2, dim=-2) + # N x (C) x B/2+1 x T + if onesided: + num_bins = kernel.shape[0] // 4 + 1 + real = real[..., :num_bins, :] + imag = imag[..., :num_bins, :] + if return_polar: + mag = (real ** 2 + imag ** 2 + eps) ** 0.5 + pha = th.atan2(imag, real) + return th.stack([mag, pha], dim=-1) + else: + return th.stack([real, imag], dim=-1) + + +def _inverse_stft( + transform: th.Tensor, + kernel: th.Tensor, + window: th.Tensor, + return_polar: bool = False, + frame_hop: int = 256, + onesided: bool = False, + center: bool = False, + eps: float = EPSILON, +) -> th.Tensor: + """ + iSTFT function implemented by conv1d + Args: + transform (Tensor): STFT transform results + kernel (Tensor): STFT transform kernels, from init_kernel(...) + return_polar (bool): keep same with the one in _forward_stft + frame_hop: frame hop size in number samples + onesided: return half FFT bins + center: used in _forward_stft + Return: + wav (Tensor), N x S + """ + # (N) x F x T x 2 + transform_dim = transform.dim() + # if F x T x 2, reshape 1 x F x T x 2 + if transform_dim == 3: + transform = th.unsqueeze(transform, 0) + if transform_dim != 4: + raise RuntimeError(f"Expect 4D tensor, but got {transform_dim}D") + + if return_polar: + real = transform[..., 0] * th.cos(transform[..., 1]) + imag = transform[..., 0] * th.sin(transform[..., 1]) + else: + real, imag = transform[..., 0], transform[..., 1] + + if onesided: + # [self.num_bins - 2, ..., 1] + reverse = range(kernel.shape[0] // 4 - 1, 0, -1) + # extend matrix: N x B x T + real = th.cat([real, real[:, reverse]], 1) + imag = th.cat([imag, -imag[:, reverse]], 1) + # pack: N x 2B x T + packed = th.cat([real, imag], dim=1) + # N x 1 x T + wav = tf.conv_transpose1d(packed, kernel * window, stride=frame_hop, padding=0) + # normalized audio samples + # refer: https://github.com/pytorch/audio/blob/2ebbbf511fb1e6c47b59fd32ad7e66023fa0dff1/torchaudio/functional.py#L171 + num_frames = packed.shape[-1] + win_length = window.shape[0] + # W x T + win = th.repeat_interleave(window[..., None] ** 2, num_frames, dim=-1) + # Do OLA on windows + # v1) + I = th.eye(win_length, device=win.device)[:, None] + denorm = tf.conv_transpose1d(win[None, ...], I, stride=frame_hop, padding=0) + # v2) + # num_samples = (num_frames - 1) * frame_hop + win_length + # denorm = tf.fold(win[None, ...], (num_samples, 1), (win_length, 1), + # stride=frame_hop)[..., 0] + if center: + pad = kernel.shape[-1] // 2 + wav = wav[..., pad:-pad] + denorm = denorm[..., pad:-pad] + wav = wav / (denorm + eps) + # N x S + return wav.squeeze(1) + + +def _pytorch_stft( + wav: th.Tensor, + frame_len: int, + frame_hop: int, + n_fft: int = 512, + return_polar: bool = False, + window: str = "sqrthann", + normalized: bool = False, + onesided: bool = True, + center: bool = False, + eps: float = EPSILON, +) -> th.Tensor: + """ + Wrapper of PyTorch STFT function + Args: + wav (Tensor): source audio signal + frame_len: length of the frame + frame_hop: hop size between frames + n_fft: number of the FFT size + return_polar: return the results in polar coordinate + window: window tensor + center: same definition with the parameter in librosa.stft + normalized: use normalized DFT kernel + onesided: output onesided STFT + Return: + transform (Tensor), STFT transform results + """ + if TORCH_VERSION < LooseVersion("1.7"): + raise RuntimeError("Can not use this function as TORCH_VERSION < 1.7") + wav_dim = wav.dim() + if wav_dim not in [2, 3]: + raise RuntimeError(f"STFT expect 2D/3D tensor, but got {wav_dim:d}D") + # if N x C x S, reshape NC x S + wav = wav.view(-1, wav.shape[-1]) + # STFT: N x F x T x 2 + stft = th.stft( + wav, + n_fft, + hop_length=frame_hop, + win_length=window.shape[-1], + window=window, + center=center, + normalized=normalized, + onesided=onesided, + return_complex=False, + ) + if wav_dim == 3: + N, F, T, _ = stft.shape + stft = stft.view(N, -1, F, T, 2) + # N x (C) x F x T x 2 + if not return_polar: + return stft + # N x (C) x F x T + real, imag = stft[..., 0], stft[..., 1] + mag = (real ** 2 + imag ** 2 + eps) ** 0.5 + pha = th.atan2(imag, real) + return th.stack([mag, pha], dim=-1) + + +def _pytorch_istft( + transform: th.Tensor, + frame_len: int, + frame_hop: int, + window: th.Tensor, + n_fft: int = 512, + return_polar: bool = False, + normalized: bool = False, + onesided: bool = True, + center: bool = False, + eps: float = EPSILON, +) -> th.Tensor: + """ + Wrapper of PyTorch iSTFT function + Args: + transform (Tensor): results of STFT + frame_len: length of the frame + frame_hop: hop size between frames + window: window tensor + n_fft: number of the FFT size + return_polar: keep same with _pytorch_stft + center: same definition with the parameter in librosa.stft + normalized: use normalized DFT kernel + onesided: output onesided STFT + Return: + wav (Tensor): synthetic audio + """ + if TORCH_VERSION < LooseVersion("1.7"): + raise RuntimeError("Can not use this function as TORCH_VERSION < 1.7") + + transform_dim = transform.dim() + # if F x T x 2, reshape 1 x F x T x 2 + if transform_dim == 3: + transform = th.unsqueeze(transform, 0) + if transform_dim != 4: + raise RuntimeError(f"Expect 4D tensor, but got {transform_dim}D") + + if return_polar: + real = transform[..., 0] * th.cos(transform[..., 1]) + imag = transform[..., 0] * th.sin(transform[..., 1]) + transform = th.stack([real, imag], -1) + # stft is a complex tensor of PyTorch + stft = th.view_as_complex(transform) + # (N) x S + wav = th.istft( + stft, + n_fft, + hop_length=frame_hop, + win_length=window.shape[-1], + window=window, + center=center, + normalized=normalized, + onesided=onesided, + return_complex=False, + ) + return wav + + +def forward_stft( + wav: th.Tensor, + frame_len: int, + frame_hop: int, + window: str = "sqrthann", + round_pow_of_two: bool = True, + return_polar: bool = False, + pre_emphasis: float = 0, + normalized: bool = False, + onesided: bool = True, + center: bool = False, + mode: str = "librosa", + eps: float = EPSILON, +) -> th.Tensor: + """ + STFT function implementation, equals to STFT layer + Args: + wav: source audio signal + frame_len: length of the frame + frame_hop: hop size between frames + return_polar: return [magnitude; phase] Tensor or [real; imag] Tensor + window: window name + center: center flag (similar with that in librosa.stft) + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + pre_emphasis: factor of preemphasis + normalized: use normalized DFT kernel + onesided: output onesided STFT + inverse: using iDFT kernel (for iSTFT) + mode: STFT mode, "kaldi" or "librosa" or "torch" + Return: + transform: results of STFT + """ + window = init_window(window, frame_len, device=wav.device) + if mode == "torch": + n_fft = 2 ** math.ceil(math.log2(frame_len)) if round_pow_of_two else frame_len + return _pytorch_stft( + wav, + frame_len, + frame_hop, + n_fft=n_fft, + return_polar=return_polar, + window=window, + normalized=normalized, + onesided=onesided, + center=center, + eps=eps, + ) + else: + kernel, window = init_kernel( + frame_len, + frame_hop, + window=window, + round_pow_of_two=round_pow_of_two, + normalized=normalized, + inverse=False, + mode=mode, + ) + return _forward_stft( + wav, + kernel, + window, + return_polar=return_polar, + frame_hop=frame_hop, + pre_emphasis=pre_emphasis, + onesided=onesided, + center=center, + eps=eps, + ) + + +def inverse_stft( + transform: th.Tensor, + frame_len: int, + frame_hop: int, + return_polar: bool = False, + window: str = "sqrthann", + round_pow_of_two: bool = True, + normalized: bool = False, + onesided: bool = True, + center: bool = False, + mode: str = "librosa", + eps: float = EPSILON, +) -> th.Tensor: + """ + iSTFT function implementation, equals to iSTFT layer + Args: + transform: results of STFT + frame_len: length of the frame + frame_hop: hop size between frames + return_polar: keep same with function forward_stft(...) + window: window name + center: center flag (similar with that in librosa.stft) + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + normalized: use normalized DFT kernel + onesided: output onesided STFT + mode: STFT mode, "kaldi" or "librosa" or "torch" + Return: + wav: synthetic signals + """ + window = init_window(window, frame_len, device=transform.device) + if mode == "torch": + n_fft = 2 ** math.ceil(math.log2(frame_len)) if round_pow_of_two else frame_len + return _pytorch_istft( + transform, + frame_len, + frame_hop, + n_fft=n_fft, + return_polar=return_polar, + window=window, + normalized=normalized, + onesided=onesided, + center=center, + eps=eps, + ) + else: + kernel, window = init_kernel( + frame_len, + frame_hop, + window, + round_pow_of_two=round_pow_of_two, + normalized=normalized, + inverse=True, + mode=mode, + ) + return _inverse_stft( + transform, + kernel, + window, + return_polar=return_polar, + frame_hop=frame_hop, + onesided=onesided, + center=center, + eps=eps, + ) + + +class STFTBase(nn.Module): + """ + Base layer for (i)STFT + Args: + frame_len: length of the frame + frame_hop: hop size between frames + window: window name + center: center flag (similar with that in librosa.stft) + round_pow_of_two: if true, choose round(#power_of_two) as the FFT size + normalized: use normalized DFT kernel + pre_emphasis: factor of preemphasis + mode: STFT mode, "kaldi" or "librosa" or "torch" + onesided: output onesided STFT + inverse: using iDFT kernel (for iSTFT) + """ + + def __init__( + self, + frame_len: int, + frame_hop: int, + window: str = "sqrthann", + round_pow_of_two: bool = True, + normalized: bool = False, + pre_emphasis: float = 0, + onesided: bool = True, + inverse: bool = False, + center: bool = False, + mode: str = "librosa", + ) -> None: + super(STFTBase, self).__init__() + if mode != "torch": + K, w = init_kernel( + frame_len, + frame_hop, + init_window(window, frame_len), + round_pow_of_two=round_pow_of_two, + normalized=normalized, + inverse=inverse, + mode=mode, + ) + self.K = nn.Parameter(K, requires_grad=False) + self.w = nn.Parameter(w, requires_grad=False) + self.num_bins = self.K.shape[0] // 4 + 1 + self.pre_emphasis = pre_emphasis + self.win_length = self.K.shape[2] + else: + self.K = None + w = init_window(window, frame_len) + self.w = nn.Parameter(w, requires_grad=False) + fft_size = ( + 2 ** math.ceil(math.log2(frame_len)) if round_pow_of_two else frame_len + ) + self.num_bins = fft_size // 2 + 1 + self.pre_emphasis = 0 + self.win_length = fft_size + self.frame_len = frame_len + self.frame_hop = frame_hop + self.window = window + self.normalized = normalized + self.onesided = onesided + self.center = center + self.mode = mode + + def num_frames(self, wav_len: th.Tensor) -> th.Tensor: + """ + Compute number of the frames + """ + assert th.sum(wav_len <= self.win_length) == 0 + if self.center: + wav_len += self.win_length + return ( + th.div(wav_len - self.win_length, self.frame_hop, rounding_mode="trunc") + 1 + ) + + def extra_repr(self) -> str: + str_repr = ( + f"num_bins={self.num_bins}, win_length={self.win_length}, " + + f"stride={self.frame_hop}, window={self.window}, " + + f"center={self.center}, mode={self.mode}" + ) + if not self.onesided: + str_repr += f", onesided={self.onesided}" + if self.pre_emphasis > 0: + str_repr += f", pre_emphasis={self.pre_emphasis}" + if self.normalized: + str_repr += f", normalized={self.normalized}" + return str_repr + + +class STFT(STFTBase): + """ + Short-time Fourier Transform as a Layer + """ + + def __init__(self, *args, **kwargs): + super(STFT, self).__init__(*args, inverse=False, **kwargs) + + def forward( + self, wav: th.Tensor, return_polar: bool = False, eps: float = EPSILON + ) -> th.Tensor: + """ + Accept (single or multiple channel) raw waveform and output magnitude and phase + Args + wav (Tensor) input signal, N x (C) x S + Return + transform (Tensor), N x (C) x F x T x 2 + """ + if self.mode == "torch": + return _pytorch_stft( + wav, + self.frame_len, + self.frame_hop, + n_fft=(self.num_bins - 1) * 2, + return_polar=return_polar, + window=self.w, + normalized=self.normalized, + onesided=self.onesided, + center=self.center, + eps=eps, + ) + else: + return _forward_stft( + wav, + self.K, + self.w, + return_polar=return_polar, + frame_hop=self.frame_hop, + pre_emphasis=self.pre_emphasis, + onesided=self.onesided, + center=self.center, + eps=eps, + ) + + +class iSTFT(STFTBase): + """ + Inverse Short-time Fourier Transform as a Layer + """ + + def __init__(self, *args, **kwargs): + super(iSTFT, self).__init__(*args, inverse=True, **kwargs) + + def forward( + self, transform: th.Tensor, return_polar: bool = False, eps: float = EPSILON + ) -> th.Tensor: + """ + Accept phase & magnitude and output raw waveform + Args + transform (Tensor): STFT output, N x F x T x 2 + Return + s (Tensor): N x S + """ + if self.mode == "torch": + return _pytorch_istft( + transform, + self.frame_len, + self.frame_hop, + n_fft=(self.num_bins - 1) * 2, + return_polar=return_polar, + window=self.w, + normalized=self.normalized, + onesided=self.onesided, + center=self.center, + eps=eps, + ) + else: + return _inverse_stft( + transform, + self.K, + self.w, + return_polar=return_polar, + frame_hop=self.frame_hop, + onesided=self.onesided, + center=self.center, + eps=eps, + ) diff --git a/look2hear/utils/torch_utils.py b/look2hear/utils/torch_utils.py new file mode 100644 index 0000000..6c3ec70 --- /dev/null +++ b/look2hear/utils/torch_utils.py @@ -0,0 +1,42 @@ +import torch +import torch.nn as nn + + +def pad_x_to_y(x, y, axis: int = -1): + if axis != -1: + raise NotImplementedError + inp_len = y.shape[axis] + output_len = x.shape[axis] + return nn.functional.pad(x, [0, inp_len - output_len]) + + +def shape_reconstructed(reconstructed, size): + if len(size) == 1: + return reconstructed.squeeze(0) + return reconstructed + + +def tensors_to_device(tensors, device): + """Transfer tensor, dict or list of tensors to device. + + Args: + tensors (:class:`torch.Tensor`): May be a single, a list or a + dictionary of tensors. + device (:class: `torch.device`): the device where to place the tensors. + + Returns: + Union [:class:`torch.Tensor`, list, tuple, dict]: + Same as input but transferred to device. + Goes through lists and dicts and transfers the torch.Tensor to + device. Leaves the rest untouched. + """ + if isinstance(tensors, torch.Tensor): + return tensors.to(device) + elif isinstance(tensors, (list, tuple)): + return [tensors_to_device(tens, device) for tens in tensors] + elif isinstance(tensors, dict): + for key in tensors.keys(): + tensors[key] = tensors_to_device(tensors[key], device) + return tensors + else: + return tensors diff --git a/pyproject.toml b/pyproject.toml index f4bd3e0..126065f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "audiotools" description = "A ComfyUI node containing multiple audio processing tools." -version = "1.1.6" +version = "1.2.0" license = {file = "LICENSE"} dependencies = ["sox", "librosa"] diff --git a/requirements.txt b/requirements.txt index 6067328..45be513 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ librosa pydub pyyaml rotary-embedding-torch +typeguard git+https://github.com/SesameAILabs/silentcipher \ No newline at end of file