From 008ee2099aed36ebb9d53677065a5db7fd2afba3 Mon Sep 17 00:00:00 2001 From: William Lin Date: Fri, 11 Apr 2025 15:34:54 -0700 Subject: [PATCH] V1 wan rebased (#335) Co-authored-by: JerryZhou54 --- .github/workflows/pr-test.yml | 6 +- .pre-commit-config.yaml | 10 +- docs/source/inference/wanvideo.md | 44 + fastvideo/v1/inference_args.py | 26 +- fastvideo/v1/inference_engine.py | 2 + fastvideo/v1/models/dits/base.py | 35 +- fastvideo/v1/models/dits/hunyuanvideo.py | 2 + fastvideo/v1/models/dits/wanvideo.py | 132 +- fastvideo/v1/models/encoders/clip.py | 5 + fastvideo/v1/models/encoders/t5.py | 60 +- fastvideo/v1/models/hf_transformer_utils.py | 5 +- .../v1/models/loader/component_loader.py | 96 +- fastvideo/v1/models/registry.py | 10 + fastvideo/v1/models/schedulers/base.py | 47 + .../scheduling_flow_match_euler_discrete.py | 10 +- .../schedulers/scheduling_unipc_multistep.py | 1162 +++++++++++++++++ fastvideo/v1/models/vaes/common.py | 26 +- fastvideo/v1/models/vaes/hunyuanvae.py | 3 + fastvideo/v1/models/vaes/wanvae.py | 30 +- fastvideo/v1/models/vision_utils.py | 220 ++++ fastvideo/v1/pipelines/pipeline_batch_info.py | 6 + fastvideo/v1/pipelines/stages/__init__.py | 7 + .../pipelines/stages/clip_image_encoding.py | 71 + .../v1/pipelines/stages/clip_text_encoding.py | 24 + fastvideo/v1/pipelines/stages/conditioning.py | 3 +- fastvideo/v1/pipelines/stages/decoding.py | 21 +- fastvideo/v1/pipelines/stages/denoising.py | 99 +- fastvideo/v1/pipelines/stages/encoding.py | 168 +++ .../v1/pipelines/stages/latent_preparation.py | 11 +- .../v1/pipelines/stages/llama_encoding.py | 33 + fastvideo/v1/pipelines/stages/t5_encoding.py | 115 ++ fastvideo/v1/pipelines/wan/__init__.py | 0 .../v1/pipelines/wan/wan_i2v_pipeline.py | 81 ++ fastvideo/v1/pipelines/wan/wan_pipeline.py | 72 + fastvideo/v1/tests/conftest.py | 2 + .../v1/tests/encoders/test_clip_encoder.py | 1 - .../v1/tests/encoders/test_llama_encoder.py | 1 - .../v1/tests/encoders/test_t5_encoder.py | 136 ++ fastvideo/v1/tests/ssim/README.md | 55 +- fastvideo/v1/tests/ssim/compute_ssim.py | 1 - ... the vast horizon. The rugged landscap.mp4 | Bin ...ting with the energetic background of .mp4 | Bin ... the vast horizon. The rugged landscap.mp4 | Bin ...ting with the energetic background of .mp4 | Bin ... the darkness and depth of space reali.mp4 | Bin 0 -> 205123 bytes ... the darkness and depth of space reali.mp4 | Bin 0 -> 201602 bytes ... the vast horizon. The rugged landscap.mp4 | Bin 0 -> 221305 bytes ...ting with the energetic background of .mp4 | Bin 0 -> 228926 bytes ... the vast horizon. The rugged landscap.mp4 | Bin 0 -> 221805 bytes ...ting with the energetic background of .mp4 | Bin 0 -> 234203 bytes .../tests/ssim/test_inference_similarity.py | 188 ++- .../v1/tests/transformers/test_wanvideo.py | 118 ++ fastvideo/v1/tests/vaes/test_wan_vae.py | 98 ++ pyproject.toml | 5 +- scripts/inference/v1_inference_wan.sh | 27 + scripts/inference/v1_inference_wan_i2v.sh | 29 + 56 files changed, 3079 insertions(+), 224 deletions(-) create mode 100644 docs/source/inference/wanvideo.md create mode 100644 fastvideo/v1/models/schedulers/base.py create mode 100644 fastvideo/v1/models/schedulers/scheduling_unipc_multistep.py create mode 100644 fastvideo/v1/models/vision_utils.py create mode 100644 fastvideo/v1/pipelines/stages/clip_image_encoding.py create mode 100644 fastvideo/v1/pipelines/stages/encoding.py create mode 100644 fastvideo/v1/pipelines/stages/t5_encoding.py create mode 100644 fastvideo/v1/pipelines/wan/__init__.py create mode 100644 fastvideo/v1/pipelines/wan/wan_i2v_pipeline.py create mode 100644 fastvideo/v1/pipelines/wan/wan_pipeline.py create mode 100644 fastvideo/v1/tests/encoders/test_t5_encoder.py rename fastvideo/v1/tests/ssim/reference_videos/{ => FastHunyuan-diffusers}/FLASH_ATTN/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 (100%) rename fastvideo/v1/tests/ssim/reference_videos/{ => FastHunyuan-diffusers}/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 (100%) rename fastvideo/v1/tests/ssim/reference_videos/{ => FastHunyuan-diffusers}/TORCH_SDPA/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 (100%) rename fastvideo/v1/tests/ssim/reference_videos/{ => FastHunyuan-diffusers}/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 (100%) create mode 100644 fastvideo/v1/tests/ssim/reference_videos/Wan2.1-I2V-14B-480P-Diffusers/FLASH_ATTN/An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space reali.mp4 create mode 100644 fastvideo/v1/tests/ssim/reference_videos/Wan2.1-I2V-14B-480P-Diffusers/TORCH_SDPA/An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space reali.mp4 create mode 100644 fastvideo/v1/tests/ssim/reference_videos/Wan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 create mode 100644 fastvideo/v1/tests/ssim/reference_videos/Wan2.1-T2V-1.3B-Diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 create mode 100644 fastvideo/v1/tests/ssim/reference_videos/Wan2.1-T2V-1.3B-Diffusers/TORCH_SDPA/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 create mode 100644 fastvideo/v1/tests/ssim/reference_videos/Wan2.1-T2V-1.3B-Diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 create mode 100644 fastvideo/v1/tests/transformers/test_wanvideo.py create mode 100644 fastvideo/v1/tests/vaes/test_wan_vae.py create mode 100755 scripts/inference/v1_inference_wan.sh create mode 100644 scripts/inference/v1_inference_wan_i2v.sh diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 6aaa0abf9..afb450e66 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -248,13 +248,13 @@ jobs: JOB_ID: "ssim-test" RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} GITHUB_RUN_ID: ${{ github.run_id }} - timeout-minutes: 30 + timeout-minutes: 45 run: >- python .github/scripts/runpod_api.py --gpu-type "NVIDIA A40" --gpu-count 2 - --disk-size 100 - --volume-size 100 + --disk-size 200 + --volume-size 200 --test-command "pip install -e .[test] && pip install flash-attn==2.7.0.post2 --no-build-isolation && pytest ./fastvideo/v1/tests/ssim -vs" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 310316204..40b86e93a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,10 +40,10 @@ repos: - id: codespell additional_dependencies: ['tomli'] args: ['--toml', 'pyproject.toml'] -- repo: https://github.com/PyCQA/isort - rev: 0a0b7a830386ba6a31c2ec8316849ae4d1b8240d # 6.0.0 - hooks: - - id: isort +# - repo: https://github.com/PyCQA/isort +# rev: 0a0b7a830386ba6a31c2ec8316849ae4d1b8240d # 6.0.0 +# hooks: +# - id: isort - repo: https://github.com/jackdewinter/pymarkdown rev: v0.9.29 hooks: @@ -66,7 +66,7 @@ repos: entry: bash args: - -c - - 'git ls-files | grep -v "^fastvideo/v1/tests/ssim/reference_videos/" | grep " " && echo "Filenames should not contain spaces!" && exit 1 || exit 0' + - 'git ls-files | grep -v "^fastvideo/v1/tests/ssim/" | grep " " && echo "Filenames should not contain spaces!" && exit 1 || exit 0' language: system always_run: true pass_filenames: false diff --git a/docs/source/inference/wanvideo.md b/docs/source/inference/wanvideo.md new file mode 100644 index 000000000..e57eab188 --- /dev/null +++ b/docs/source/inference/wanvideo.md @@ -0,0 +1,44 @@ +(wanvideo)= + +# WanVideo +## Inference T2V with WanVideo +First, download the model: + +```bash +python scripts/huggingface/download_hf.py --repo_id=Wan-AI/Wan2.1-T2V-1.3B-Diffusers --local_dir=YOUR_LOCAL_DIR --repo_type=model +``` + +or + +```bash +python scripts/huggingface/download_hf.py --repo_id=Wan-AI/Wan2.1-T2V-14B-Diffusers --local_dir=YOUR_LOCAL_DIR --repo_type=model +``` + +Then run the inference using: + +```bash +sh scripts/inference/v1_inference_wan.sh +``` + +Remember to set `MODEL_BASE` and `num_gpus` accordingly. + +## Inference I2V with WanVideo +First, download the model: + +```bash +python scripts/huggingface/download_hf.py --repo_id=Wan-AI/Wan2.1-I2V-14B-480P-Diffusers --local_dir=YOUR_LOCAL_DIR --repo_type=model +``` + +or + +```bash +python scripts/huggingface/download_hf.py --repo_id=Wan-AI/Wan2.1-I2V-14B-720P-Diffusers --local_dir=YOUR_LOCAL_DIR --repo_type=model +``` + +Then run the inference using: + +```bash +sh scripts/inference/v1_inference_wan_i2v.sh +``` + +Remember to set `MODEL_BASE` and `num_gpus` accordingly. diff --git a/fastvideo/v1/inference_args.py b/fastvideo/v1/inference_args.py index 7ce58ee8a..f48427b14 100644 --- a/fastvideo/v1/inference_args.py +++ b/fastvideo/v1/inference_args.py @@ -31,7 +31,7 @@ class InferenceArgs: guidance_scale: float = 1.0 guidance_rescale: float = 0.0 embedded_cfg_scale: float = 6.0 - flow_shift: int = 7 + flow_shift: Optional[float] = None output_type: str = "pil" @@ -43,6 +43,9 @@ class InferenceArgs: vae_tiling: bool = True vae_sp: bool = False + # Image encoder configuration + image_encoder_precision: str = "fp32" + # Text encoder configuration text_encoder_precision: str = "fp16" text_len: int = 256 @@ -54,14 +57,14 @@ class InferenceArgs: # Flow Matching parameters flow_solver: str = "euler" - denoise_type: str = "flow" + denoise_type: str = "flow" # Deprecated. Will use scheduler_config.json # STA (Spatial-Temporal Attention) parameters mask_strategy_file_path: Optional[str] = None enable_torch_compile: bool = False # Scheduler options - scheduler_type: str = "euler" + scheduler_type: str = "euler" # Deprecated. Will use the param in scheduler_config.json neg_prompt: Optional[str] = None num_videos: int = 1 @@ -73,6 +76,7 @@ class InferenceArgs: log_level: str = "info" # Inference parameters + image_path: Optional[str] = None prompt: Optional[str] = None prompt_path: Optional[str] = None output_path: str = "outputs/" @@ -187,7 +191,7 @@ class InferenceArgs: parser.add_argument( "--flow-shift", "--shift", - type=int, + type=float, default=InferenceArgs.flow_shift, help="Flow shift parameter", ) @@ -240,6 +244,16 @@ class InferenceArgs: default=InferenceArgs.text_len, help="Maximum text length", ) + + # Image encoder config + parser.add_argument( + "--image-encoder-precision", + type=str, + default=InferenceArgs.image_encoder_precision, + choices=["fp32", "fp16", "bf16"], + help="Precision for image encoder", + ) + # Secondary text encoder parser.add_argument( @@ -343,6 +357,10 @@ class InferenceArgs: help="Path to a text file containing the prompt", ) + parser.add_argument("--image-path", + type=str, + help="Path to the image for I2V generation") + parser.add_argument( "--output-path", type=str, diff --git a/fastvideo/v1/inference_engine.py b/fastvideo/v1/inference_engine.py index 28e59f677..882021ca3 100644 --- a/fastvideo/v1/inference_engine.py +++ b/fastvideo/v1/inference_engine.py @@ -106,6 +106,7 @@ class InferenceEngine: guidance_scale = inference_args.guidance_scale flow_shift = inference_args.flow_shift embedded_guidance_scale = inference_args.embedded_cfg_scale + image_path = inference_args.image_path # ======================================================================== # Arguments: target_width, target_height, target_video_length @@ -162,6 +163,7 @@ class InferenceEngine: # local_rank = sp_group.rank device = torch.device(inference_args.device_str) batch = ForwardBatch( + image_path=image_path, prompt=prompt, negative_prompt=negative_prompt, num_videos_per_prompt=num_videos_per_prompt, diff --git a/fastvideo/v1/models/dits/base.py b/fastvideo/v1/models/dits/base.py index 2e0bcb430..2cf98697d 100644 --- a/fastvideo/v1/models/dits/base.py +++ b/fastvideo/v1/models/dits/base.py @@ -1,15 +1,46 @@ # SPDX-License-Identifier: Apache-2.0 +from abc import ABC, abstractmethod +from typing import List, Union +import torch from torch import nn # TODO -class BaseDiT(nn.Module): +class BaseDiT(nn.Module, ABC): _fsdp_shard_conditions: list = [] attention_head_dim: int | None = None + _param_names_mapping: dict + hidden_size: int + num_attention_heads: int + + def __init_subclass__(cls): + required_class_attrs = [ + "_fsdp_shard_conditions", "_param_names_mapping" + ] + super().__init_subclass__() + for attr in required_class_attrs: + if not hasattr(cls, attr): + raise AttributeError( + f"Subclasses of BaseDiT must define '{attr}' class variable" + ) def __init__(self, *args, **kwargs) -> None: super().__init__() - def forward(self, *args, **kwargs): + @abstractmethod + def forward(self, + hidden_states: torch.Tensor, + encoder_hidden_states: Union[torch.Tensor, List[torch.Tensor]], + timestep: torch.LongTensor, + guidance=None, + **kwargs) -> torch.Tensor: pass + + def __post_init__(self): + required_attrs = ["hidden_size", "num_attention_heads"] + for attr in required_attrs: + if not hasattr(self, attr): + raise AttributeError( + f"Subclasses of BaseDiT must define '{attr}' instance variable" + ) diff --git a/fastvideo/v1/models/dits/hunyuanvideo.py b/fastvideo/v1/models/dits/hunyuanvideo.py index c3b264e9d..c7ed836c1 100644 --- a/fastvideo/v1/models/dits/hunyuanvideo.py +++ b/fastvideo/v1/models/dits/hunyuanvideo.py @@ -648,6 +648,8 @@ class HunyuanVideoTransformer3DModel(BaseDiT): self.out_channels, dtype=dtype) + self.__post_init__() + # TODO: change the input the FORWAD_BACTCH Dict # TODO: change output to a dict def forward( diff --git a/fastvideo/v1/models/dits/wanvideo.py b/fastvideo/v1/models/dits/wanvideo.py index 7750d470e..37a9c0dd1 100644 --- a/fastvideo/v1/models/dits/wanvideo.py +++ b/fastvideo/v1/models/dits/wanvideo.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 import math -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import torch import torch.nn as nn @@ -34,9 +34,10 @@ class WanImageEmbedding(torch.nn.Module): def forward(self, encoder_hidden_states_image: torch.Tensor) -> torch.Tensor: + dtype = encoder_hidden_states_image.dtype hidden_states = self.norm1(encoder_hidden_states_image) hidden_states = self.ff(hidden_states) - hidden_states = self.norm2(hidden_states) + hidden_states = self.norm2(hidden_states).to(dtype) return hidden_states @@ -52,10 +53,7 @@ class WanTimeTextImageEmbedding(nn.Module): super().__init__() self.time_embedder = TimestepEmbedder( - dim, - frequency_embedding_size=time_freq_dim, - act_layer="silu", - freq_dtype=torch.float64) + dim, frequency_embedding_size=time_freq_dim, act_layer="silu") self.time_modulation = ModulateProjection(dim, factor=6, act_layer="silu") @@ -75,9 +73,8 @@ class WanTimeTextImageEmbedding(nn.Module): encoder_hidden_states: torch.Tensor, encoder_hidden_states_image: Optional[torch.Tensor] = None, ): - with torch.cuda.amp.autocast(dtype=torch.float32): - temb = self.time_embedder(timestep.float()) - timestep_proj = self.time_modulation(temb) + temb = self.time_embedder(timestep) + timestep_proj = self.time_modulation(temb) encoder_hidden_states = self.text_embedder(encoder_hidden_states) if encoder_hidden_states_image is not None: @@ -116,7 +113,9 @@ class WanSelfAttention(nn.Module): self.norm_k = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() # Scaled dot product attention - self.attn = LocalAttention(dropout_rate=0, + self.attn = LocalAttention(num_heads=num_heads, + head_size=self.head_dim, + dropout_rate=0, softmax_scale=None, causal=False) @@ -284,16 +283,15 @@ class WanTransformerBlock(nn.Module): hidden_states = hidden_states.squeeze(1) bs, seq_length, _ = hidden_states.shape orig_dtype = hidden_states.dtype - assert temb.dtype == torch.float32 - with torch.cuda.amp.autocast(dtype=torch.float32): - e = self.scale_shift_table + temb - shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk( - 6, dim=1) + assert orig_dtype != torch.float32 + e = self.scale_shift_table + temb.float() + shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk( + 6, dim=1) assert shift_msa.dtype == torch.float32 # 1. Self-attention - norm_hidden_states = self.norm1(hidden_states.float()).to( - dtype=orig_dtype) * (1 + scale_msa) + shift_msa + norm_hidden_states = (self.norm1(hidden_states.float()) * + (1 + scale_msa) + shift_msa).to(orig_dtype) query, _ = self.to_q(norm_hidden_states) key, _ = self.to_k(norm_hidden_states) value, _ = self.to_v(norm_hidden_states) @@ -320,6 +318,8 @@ class WanTransformerBlock(nn.Module): null_shift = null_scale = torch.tensor([0], device=hidden_states.device) norm_hidden_states, hidden_states = self.self_attn_residual_norm( hidden_states, attn_output, gate_msa, null_shift, null_scale) + norm_hidden_states, hidden_states = norm_hidden_states.to( + orig_dtype), hidden_states.to(orig_dtype) # 2. Cross-attention attn_output = self.attn2(norm_hidden_states, @@ -327,10 +327,13 @@ class WanTransformerBlock(nn.Module): context_lens=None) norm_hidden_states, hidden_states = self.cross_attn_residual_norm( hidden_states, attn_output, 1, c_shift_msa, c_scale_msa) + norm_hidden_states, hidden_states = norm_hidden_states.to( + orig_dtype), hidden_states.to(orig_dtype) # 3. Feed-forward ff_output = self.ffn(norm_hidden_states) hidden_states = self.mlp_residual(hidden_states, ff_output, c_gate_msa) + hidden_states = hidden_states.to(orig_dtype) return hidden_states @@ -400,8 +403,9 @@ class WanTransformer3DModel(BaseDiT): super().__init__() inner_dim = num_attention_heads * attention_head_dim - self.inner_dim = inner_dim + self.hidden_size = inner_dim self.num_attention_heads = num_attention_heads + self.in_channels = in_channels self.out_channels = out_channels or in_channels self.patch_size = patch_size self.text_len = text_len @@ -440,19 +444,28 @@ class WanTransformer3DModel(BaseDiT): self.gradient_checkpointing = False + self.__post_init__() + def forward( self, hidden_states: torch.Tensor, + encoder_hidden_states: Union[torch.Tensor, List[torch.Tensor]], timestep: torch.LongTensor, - encoder_hidden_states: torch.Tensor, seq_len: Optional[int] = None, - encoder_hidden_states_image: Optional[torch.Tensor] = None, - y: Optional[torch.Tensor] = None, + encoder_hidden_states_image: Optional[Union[torch.Tensor, + List[torch.Tensor]]] = None, return_dict: bool = True, attention_kwargs: Optional[Dict[str, Any]] = None, - ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]: - if y is not None: - hidden_states = torch.cat([hidden_states, y], dim=1) + guidance=None, + ) -> torch.Tensor: + orig_dtype = hidden_states.dtype + if not isinstance(encoder_hidden_states, torch.Tensor): + encoder_hidden_states = encoder_hidden_states[0] + if isinstance(encoder_hidden_states_image, + list) and len(encoder_hidden_states_image) > 0: + encoder_hidden_states_image = encoder_hidden_states_image[0] + else: + encoder_hidden_states_image = None batch_size, num_channels, num_frames, height, width = hidden_states.shape p_t, p_h, p_w = self.patch_size @@ -461,40 +474,23 @@ class WanTransformer3DModel(BaseDiT): post_patch_width = width // p_w # Get rotary embeddings - d = self.inner_dim // self.num_attention_heads + d = self.hidden_size // self.num_attention_heads rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] freqs_cos, freqs_sin = get_rotary_pos_embed( (post_patch_num_frames * get_sequence_model_parallel_world_size(), post_patch_height, post_patch_width), - self.inner_dim, + self.hidden_size, self.num_attention_heads, rope_dim_list, dtype=torch.float64, rope_theta=10000) freqs_cos = freqs_cos.to(hidden_states.device) freqs_sin = freqs_sin.to(hidden_states.device) - freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None + freqs_cis = (freqs_cos.float(), + freqs_sin.float()) if freqs_cos is not None else None hidden_states = self.patch_embedding(hidden_states) - grid_sizes = torch.stack( - [torch.tensor(hidden_states[0].shape[1:], dtype=torch.long)]) hidden_states = hidden_states.flatten(2).transpose(1, 2) - if seq_len is None: - seq_len = hidden_states.size(1) - hidden_states = torch.cat([ - hidden_states, - hidden_states.new_zeros(1, seq_len - hidden_states.size(1), - hidden_states.size(2)) - ], - dim=1) - - encoder_hidden_states = torch.cat([ - encoder_hidden_states, - encoder_hidden_states.new_zeros( - 1, self.text_len - encoder_hidden_states.size(1), - encoder_hidden_states.size(2)) - ], - dim=1) temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( timestep, encoder_hidden_states, encoder_hidden_states_image) @@ -504,6 +500,7 @@ class WanTransformer3DModel(BaseDiT): encoder_hidden_states = torch.concat( [encoder_hidden_states_image, encoder_hidden_states], dim=1) + assert encoder_hidden_states.dtype == orig_dtype # 4. Transformer blocks if torch.is_grad_enabled() and self.gradient_checkpointing: for block in self.blocks: @@ -516,39 +513,16 @@ class WanTransformer3DModel(BaseDiT): timestep_proj, freqs_cis) # 5. Output norm, projection & unpatchify - with torch.cuda.amp.autocast(dtype=torch.float32): - shift, scale = (self.scale_shift_table + temb.unsqueeze(1)).chunk( - 2, dim=1) - hidden_states = self.norm_out(hidden_states.float(), shift, scale) - hidden_states = self.proj_out(hidden_states) + shift, scale = (self.scale_shift_table + temb.unsqueeze(1)).chunk(2, + dim=1) + hidden_states = self.norm_out(hidden_states.float(), shift, scale) + hidden_states = self.proj_out(hidden_states) - output = self.unpatchify(hidden_states, grid_sizes) + hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, + post_patch_height, + post_patch_width, p_t, p_h, p_w, + -1) + hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) + output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) - return output.float() - - def unpatchify(self, x, grid_sizes) -> torch.Tensor: - r""" - Reconstruct video tensors from patch embeddings. - - Args: - x (List[Tensor]): - List of patchified features, each with shape [L, C_out * prod(patch_size)] - grid_sizes (Tensor): - Original spatial-temporal grid dimensions before patching, - shape [B, 3] (3 dimensions correspond to F_patches, H_patches, W_patches) - - Returns: - Tensor: - Reconstructed video tensors with shape [B, C_out, F, H / 8, W / 8] - """ - - c = self.out_channels - out = [] - for u, v in zip(x, grid_sizes.tolist()): - u = u[:math.prod(v)].view(*v, *self.patch_size, c) - u = u.permute(6, 0, 3, 1, 4, 2, 5) - # u = torch.einsum('fhwpqrc->cfphqwr', u.contiguous()) - u = u.reshape(c, *[i * j for i, j in zip(v, self.patch_size)]) - out.append(u) - out = torch.cat(out, dim=0) - return out + return output diff --git a/fastvideo/v1/models/encoders/clip.py b/fastvideo/v1/models/encoders/clip.py index 5fbf77c16..a62bb64b9 100644 --- a/fastvideo/v1/models/encoders/clip.py +++ b/fastvideo/v1/models/encoders/clip.py @@ -598,6 +598,9 @@ class CLIPVisionTransformer(nn.Module): inputs_embeds=hidden_states, return_all_hidden_states=return_all_hidden_states) + if not return_all_hidden_states: + encoder_outputs = encoder_outputs[0] + # Handle post-norm (if applicable) and stacks feature layers if needed encoder_outputs = resolve_visual_encoder_outputs( encoder_outputs, feature_sample_layers, self.post_layernorm, @@ -654,6 +657,8 @@ class CLIPVisionModel(nn.Module, SupportsQuant): layer_count = len(self.vision_model.encoder.layers) for name, loaded_weight in weights: + if name.startswith("visual_projection"): + continue # post_layernorm is not needed in CLIPVisionModel if (name.startswith("vision_model.post_layernorm") and self.vision_model.post_layernorm is None): diff --git a/fastvideo/v1/models/encoders/t5.py b/fastvideo/v1/models/encoders/t5.py index 1422d16c3..85b6fb45a 100644 --- a/fastvideo/v1/models/encoders/t5.py +++ b/fastvideo/v1/models/encoders/t5.py @@ -28,11 +28,12 @@ import torch.nn.functional as F from torch import nn from transformers import T5Config -from fastvideo.v1.distributed import get_tensor_model_parallel_world_size +from fastvideo.v1.distributed import (get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) from fastvideo.v1.layers.activation import get_act_fn from fastvideo.v1.layers.layernorm import RMSNorm -from fastvideo.v1.layers.linear import (ColumnParallelLinear, QKVParallelLinear, - RowParallelLinear) +from fastvideo.v1.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, RowParallelLinear) from fastvideo.v1.layers.vocab_parallel_embedding import VocabParallelEmbedding from fastvideo.v1.models.loader.weight_utils import default_weight_loader @@ -67,7 +68,8 @@ class T5DenseActDense(nn.Module): config: T5Config, quant_config: Optional[QuantizationConfig] = None): super().__init__() - self.wi = ColumnParallelLinear(config.d_model, config.d_ff, bias=False) + self.wi = MergedColumnParallelLinear(config.d_model, [config.d_ff], + bias=False) self.wo = RowParallelLinear(config.d_ff, config.d_model, bias=False, @@ -87,14 +89,12 @@ class T5DenseGatedActDense(nn.Module): config: T5Config, quant_config: Optional[QuantizationConfig] = None): super().__init__() - self.wi_0 = ColumnParallelLinear(config.d_model, - config.d_ff, - bias=False, - quant_config=quant_config) - self.wi_1 = ColumnParallelLinear(config.d_model, - config.d_ff, - bias=False, - quant_config=quant_config) + self.wi_0 = MergedColumnParallelLinear(config.d_model, [config.d_ff], + bias=False, + quant_config=quant_config) + self.wi_1 = MergedColumnParallelLinear(config.d_model, [config.d_ff], + bias=False, + quant_config=quant_config) # Should not run in fp16 unless mixed-precision is used, # see https://github.com/huggingface/transformers/issues/20287. self.wo = RowParallelLinear(config.d_ff, @@ -170,6 +170,7 @@ class T5Attention(nn.Module): config.relative_attention_max_distance self.d_model = config.d_model self.key_value_proj_dim = config.d_kv + self.total_num_heads = self.total_num_kv_heads = config.num_heads # Partition heads across multiple tensor parallel GPUs. tp_world_size = get_tensor_model_parallel_world_size() @@ -178,29 +179,33 @@ class T5Attention(nn.Module): self.inner_dim = self.n_heads * self.key_value_proj_dim # No GQA in t5. - self.n_kv_heads = self.n_heads + # self.n_kv_heads = self.n_heads - self.qkv_proj = QKVParallelLinear(self.d_model, - self.d_model // self.n_heads, - self.n_heads, - self.n_kv_heads, - bias=False, - quant_config=quant_config) + self.qkv_proj = QKVParallelLinear( + self.d_model, + self.d_model // self.total_num_heads, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) self.attn = T5MultiHeadAttention() if self.has_relative_attention_bias: self.relative_attention_bias = \ VocabParallelEmbedding(self.relative_attention_num_buckets, - self.n_heads, + self.total_num_heads, org_num_embeddings=self.relative_attention_num_buckets, padding_size=self.relative_attention_num_buckets, quant_config=quant_config) self.o = RowParallelLinear( - self.inner_dim, + self.d_model, self.d_model, bias=False, quant_config=quant_config, + prefix=f"{prefix}.o_proj", ) @staticmethod @@ -295,13 +300,13 @@ class T5Attention(nn.Module): ) -> torch.Tensor: bs, seq_len, _ = hidden_states.shape num_seqs = bs - n, c = self.n_heads, self.d_model // self.n_heads + n, c = self.n_heads, self.d_model // self.total_num_heads qkv, _ = self.qkv_proj(hidden_states) # Projection of 'own' hidden state (self-attention). No GQA here. q, k, v = qkv.split(self.inner_dim, dim=-1) - q = q.view(bs, -1, n, c) - k = k.view(bs, -1, n, c) - v = v.view(bs, -1, n, c) + q = q.reshape(bs, seq_len, n, c) + k = k.reshape(bs, seq_len, n, c) + v = v.reshape(bs, seq_len, n, c) assert attn_metadata is not None attn_bias = attn_metadata.attn_bias @@ -325,6 +330,11 @@ class T5Attention(nn.Module): -1) if attention_mask.ndim == 2 else attention_mask.unsqueeze(1) attn_bias.masked_fill_(attention_mask == 0, torch.finfo(q.dtype).min) + + if get_tensor_model_parallel_world_size() > 1: + rank = get_tensor_model_parallel_rank() + attn_bias = attn_bias[:, rank * self.n_heads:(rank + 1) * + self.n_heads, :, :] attn_output = self.attn(q, k, v, attn_bias) output, _ = self.o(attn_output) return output diff --git a/fastvideo/v1/models/hf_transformer_utils.py b/fastvideo/v1/models/hf_transformer_utils.py index feda42212..e2f39ea57 100644 --- a/fastvideo/v1/models/hf_transformer_utils.py +++ b/fastvideo/v1/models/hf_transformer_utils.py @@ -95,9 +95,12 @@ def get_diffusers_config( Returns: The loaded configuration. """ + config_name = "config.json" + if "scheduler" in model: + config_name = "scheduler_config.json" # Check if the model path exists if os.path.exists(model): - config_file = os.path.join(model, "config.json") + config_file = os.path.join(model, config_name) if os.path.exists(config_file): try: # Load the config directly from the file diff --git a/fastvideo/v1/models/loader/component_loader.py b/fastvideo/v1/models/loader/component_loader.py index bf7e1d09e..47e5f2fb4 100644 --- a/fastvideo/v1/models/loader/component_loader.py +++ b/fastvideo/v1/models/loader/component_loader.py @@ -10,7 +10,7 @@ from typing import Any, Generator, Iterable, List, Optional, Tuple, cast import torch import torch.nn as nn from safetensors.torch import load_file as safetensors_load_file -from transformers import AutoTokenizer, PretrainedConfig +from transformers import AutoImageProcessor, AutoTokenizer, PretrainedConfig from transformers.utils import SAFE_WEIGHTS_INDEX_NAME from fastvideo.v1.inference_args import InferenceArgs @@ -23,6 +23,7 @@ from fastvideo.v1.models.loader.weight_utils import ( filter_duplicate_safetensors_files, filter_files_not_needed_for_inference, pt_weights_iterator, safetensors_weights_iterator) from fastvideo.v1.models.registry import ModelRegistry +from fastvideo.v1.utils import PRECISION_TO_TYPE logger = init_logger(__name__) @@ -71,6 +72,8 @@ class ComponentLoader(ABC): "text_encoder_2": (TextEncoderLoader, "transformers"), "tokenizer": (TokenizerLoader, "transformers"), "tokenizer_2": (TokenizerLoader, "transformers"), + "image_processor": (ImageProcessorLoader, "transformers"), + "image_encoder": (ImageEncoderLoader, "transformers"), } if module_type in module_loaders: @@ -209,11 +212,15 @@ class TextEncoderLoader(ComponentLoader): target_device = torch.device(inference_args.device_str) # TODO(will): add support for other dtypes - return self.load_model(model_path, model_config, target_device) + return self.load_model(model_path, model_config, target_device, + inference_args.text_encoder_precision) - def load_model(self, model_path: str, model_config, - target_device: torch.device): - with set_default_torch_dtype(torch.float16): + def load_model(self, + model_path: str, + model_config, + target_device: torch.device, + dtype: str = "fp16"): + with set_default_torch_dtype(PRECISION_TO_TYPE[dtype]): with target_device: architectures = getattr(model_config, "architectures", []) model_cls, _ = ModelRegistry.resolve_model_cls(architectures) @@ -240,6 +247,40 @@ class TextEncoderLoader(ComponentLoader): return model.eval() +class ImageEncoderLoader(TextEncoderLoader): + + def load(self, model_path: str, architecture: str, + inference_args: InferenceArgs): + """Load the text encoders based on the model path, architecture, and inference args.""" + model_config: PretrainedConfig = get_hf_config( + model=model_path, + trust_remote_code=inference_args.trust_remote_code, + revision=inference_args.revision, + model_override_args=None, + inference_args=inference_args, + ) + logger.info("HF Model config: %s", model_config) + + target_device = torch.device(inference_args.device_str) + # TODO(will): add support for other dtypes + return self.load_model(model_path, model_config, target_device, + inference_args.image_encoder_precision) + + +class ImageProcessorLoader(ComponentLoader): + """Loader for image processor.""" + + def load(self, model_path: str, architecture: str, + inference_args: InferenceArgs): + """Load the image processor based on the model path, architecture, and inference args.""" + logger.info("Loading image processor from %s", model_path) + + image_processor = AutoImageProcessor.from_pretrained(model_path, ) + logger.info("Loaded image processor: %s", + image_processor.__class__.__name__) + return image_processor + + class TokenizerLoader(ComponentLoader): """Loader for tokenizers.""" @@ -265,8 +306,6 @@ class VAELoader(ComponentLoader): inference_args: InferenceArgs): """Load the VAE based on the model path, architecture, and inference args.""" # TODO(will): move this to a constants file - from fastvideo.v1.utils import PRECISION_TO_TYPE - config = get_diffusers_config(model=model_path) class_name = config.pop("_class_name") @@ -289,14 +328,6 @@ class VAELoader(ComponentLoader): dtype = PRECISION_TO_TYPE[inference_args.vae_precision] vae = vae.eval().to(dtype) - # TODO(will): should we define hunyuan vae config class? - vae_kwargs = { - "s_ratio": config["spatial_compression_ratio"], - "t_ratio": config["temporal_compression_ratio"], - } - - vae.kwargs = vae_kwargs - return vae @@ -326,6 +357,7 @@ class TransformerLoader(ComponentLoader): len(safetensors_list), model_path) # initialize_sequence_parallel_group(inference_args.sp_size) + default_dtype = PRECISION_TO_TYPE[inference_args.precision] # Load the model using FSDP loader logger.info("Loading model from %s", cls_name) @@ -333,12 +365,16 @@ class TransformerLoader(ComponentLoader): init_params=model_config, weight_dir_list=safetensors_list, device=inference_args.device, - cpu_offload=inference_args.use_cpu_offload) + cpu_offload=inference_args.use_cpu_offload, + default_dtype=default_dtype) total_params = sum(p.numel() for p in model.parameters()) logger.info("Loaded model with %.2fB parameters", total_params / 1e9) - model.eval() + dtypes = set(param.dtype for param in model.parameters()) + if len(dtypes) > 1: + model = model.to(default_dtype) + model = model.eval() return model @@ -348,21 +384,17 @@ class SchedulerLoader(ComponentLoader): def load(self, model_path: str, architecture: str, inference_args: InferenceArgs): """Load the scheduler based on the model path, architecture, and inference args.""" - if hasattr(inference_args, - 'denoise_type') and inference_args.denoise_type == "flow": - # TODO(will): add schedulers to register or create a new scheduler registry - # TODO(will): default to config file but allow override through - # inference args. Currently only uses inference args. - from fastvideo.v1.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchDiscreteScheduler) - scheduler = FlowMatchDiscreteScheduler( - shift=inference_args.flow_shift, - solver=inference_args.flow_solver, - ) - logger.info("Scheduler loaded: %s", scheduler) - else: - raise ValueError( - f"Invalid denoise type: {inference_args.denoise_type}") + config = get_diffusers_config(model=model_path) + + class_name = config.pop("_class_name") + assert class_name is not None, "Model config does not contain a _class_name attribute. Only diffusers format is supported." + config.pop("_diffusers_version") + + scheduler_cls, _ = ModelRegistry.resolve_model_cls(class_name) + + scheduler = scheduler_cls(**config) + if inference_args.flow_shift is not None: + scheduler.set_shift(inference_args.flow_shift) return scheduler diff --git a/fastvideo/v1/models/registry.py b/fastvideo/v1/models/registry.py index e80e1e1c2..bc11f365b 100644 --- a/fastvideo/v1/models/registry.py +++ b/fastvideo/v1/models/registry.py @@ -38,6 +38,7 @@ _TEXT_ENCODER_MODELS = { _IMAGE_ENCODER_MODELS: dict[str, tuple] = { # "HunyuanVideoTransformer3DModel": ("image_encoder", "hunyuanvideo", "HunyuanVideoImageEncoder"), + "CLIPVisionModelWithProjection": ("encoders", "clip", "CLIPVisionModel"), } _VAE_MODELS = { @@ -46,12 +47,21 @@ _VAE_MODELS = { "AutoencoderKLWan": ("vaes", "wanvae", "AutoencoderKLWan"), } +_SCHEDULERS = { + "FlowMatchEulerDiscreteScheduler": + ("schedulers", "scheduling_flow_match_euler_discrete", + "FlowMatchDiscreteScheduler"), + "UniPCMultistepScheduler": + ("schedulers", "scheduling_unipc_multistep", "UniPCMultistepScheduler"), +} + _FAST_VIDEO_MODELS = { **_TEXT_TO_VIDEO_DIT_MODELS, **_IMAGE_TO_VIDEO_DIT_MODELS, **_TEXT_ENCODER_MODELS, **_IMAGE_ENCODER_MODELS, **_VAE_MODELS, + **_SCHEDULERS, } _SUBPROCESS_COMMAND = [ diff --git a/fastvideo/v1/models/schedulers/base.py b/fastvideo/v1/models/schedulers/base.py new file mode 100644 index 000000000..5333d07a9 --- /dev/null +++ b/fastvideo/v1/models/schedulers/base.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 + +from abc import ABC, abstractmethod +from typing import Optional, Tuple, Union + +import torch +from diffusers.utils import BaseOutput + + +class BaseScheduler(ABC): + timesteps: torch.tensor + order: int + + def __init__(self, *args, **kwargs) -> None: + # Check if subclass has defined all required properties + required_attributes = ['timesteps', 'order'] + + for attr in required_attributes: + if not hasattr(self, attr): + raise AttributeError( + f"Subclasses of BaseScheduler must define '{attr}' property" + ) + + @abstractmethod + def set_shift(self, shift: float) -> None: + pass + + @abstractmethod + def set_timesteps(self, *args, **kwargs) -> None: + pass + + @abstractmethod + def scale_model_input(self, + sample: torch.Tensor, + timestep: Optional[int] = None) -> torch.Tensor: + pass + + @abstractmethod + def step( + self, + model_output: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + sample: torch.FloatTensor, + return_dict: bool = True, + **kwargs, + ) -> Union[BaseOutput, Tuple]: + pass diff --git a/fastvideo/v1/models/schedulers/scheduling_flow_match_euler_discrete.py b/fastvideo/v1/models/schedulers/scheduling_flow_match_euler_discrete.py index 592af94e4..0e5f1e7d3 100644 --- a/fastvideo/v1/models/schedulers/scheduling_flow_match_euler_discrete.py +++ b/fastvideo/v1/models/schedulers/scheduling_flow_match_euler_discrete.py @@ -27,6 +27,8 @@ from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.schedulers.scheduling_utils import SchedulerMixin from diffusers.utils import BaseOutput, logging +from fastvideo.v1.models.schedulers.base import BaseScheduler + logger = logging.get_logger(__name__) # pylint: disable=invalid-name @@ -44,7 +46,7 @@ class FlowMatchDiscreteSchedulerOutput(BaseOutput): prev_sample: torch.FloatTensor -class FlowMatchDiscreteScheduler(SchedulerMixin, ConfigMixin): +class FlowMatchDiscreteScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): """ Euler scheduler. @@ -74,6 +76,7 @@ class FlowMatchDiscreteScheduler(SchedulerMixin, ConfigMixin): reverse: bool = True, solver: str = "euler", n_tokens: Optional[int] = None, + **kwargs, ): sigmas = torch.linspace(1, 0, num_train_timesteps + 1) @@ -94,6 +97,8 @@ class FlowMatchDiscreteScheduler(SchedulerMixin, ConfigMixin): f"Solver {solver} not supported. Supported solvers: {self.supported_solver}" ) + BaseScheduler.__init__(self) + @property def step_index(self): """ @@ -170,6 +175,9 @@ class FlowMatchDiscreteScheduler(SchedulerMixin, ConfigMixin): return idx + def set_shift(self, shift: float) -> None: + self.config.shift = shift + def _init_step_index(self, timestep) -> None: if self.begin_index is None: if isinstance(timestep, torch.Tensor): diff --git a/fastvideo/v1/models/schedulers/scheduling_unipc_multistep.py b/fastvideo/v1/models/schedulers/scheduling_unipc_multistep.py new file mode 100644 index 000000000..a81d95620 --- /dev/null +++ b/fastvideo/v1/models/schedulers/scheduling_unipc_multistep.py @@ -0,0 +1,1162 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Copyright 2024 TSAIL Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: check https://arxiv.org/abs/2302.04867 and https://github.com/wl-zhao/UniPC for more info +# The codebase is modified based on https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py +# ============================================================================== +# +# Modified from diffusers==0.33.0.dev0 +# +# ============================================================================== + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import (KarrasDiffusionSchedulers, + SchedulerMixin, + SchedulerOutput) +from diffusers.utils import deprecate, is_scipy_available + +from fastvideo.v1.models.schedulers.base import BaseScheduler + +if is_scipy_available(): + import scipy.stats + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +) -> torch.Tensor: + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t: float) -> float: + return math.cos((t + 0.008) / 1.008 * math.pi / 2)**2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t: float) -> float: + return math.exp(t * -12.0) + + else: + raise ValueError( + f"Unsupported alpha_transform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas: torch.Tensor) -> torch.Tensor: + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.Tensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.Tensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - + alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): + """ + `UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + solver_order (`int`, default `2`): + The UniPC order which can be any positive integer. The effective order of accuracy is `solver_order + 1` + due to the UniC. It is recommended to use `solver_order=2` for guided sampling, and `solver_order=3` for + unconditional sampling. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and `predict_x0=True`. + predict_x0 (`bool`, defaults to `True`): + Whether to use the updating algorithm on the predicted x0. + solver_type (`str`, default `bh2`): + Solver type for UniPC. It is recommended to use `bh1` for unconditional sampling when steps < 10, and `bh2` + otherwise. + lower_order_final (`bool`, default `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + disable_corrector (`list`, default `[]`): + Decides which step to disable the corrector to mitigate the misalignment between `epsilon_theta(x_t, c)` + and `epsilon_theta(x_t^c, c)` which can influence convergence for a large guidance scale. Corrector is + usually disabled during the first few steps. + solver_p (`SchedulerMixin`, default `None`): + Any other scheduler that if specified, the algorithm becomes `solver_p + UniC`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + use_exponential_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use exponential sigmas for step sizes in the noise schedule during the sampling process. + use_beta_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use beta sigmas for step sizes in the noise schedule during the sampling process. Refer to [Beta + Sampling is All You Need](https://huggingface.co/papers/2407.12173) for more information. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + final_sigmas_type (`str`, defaults to `"zero"`): + The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final + sigma is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + solver_order: int = 2, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + predict_x0: bool = True, + solver_type: str = "bh2", + lower_order_final: bool = True, + disable_corrector: Tuple[int, ...] = (), + solver_p: SchedulerMixin = None, + use_karras_sigmas: Optional[bool] = False, + use_exponential_sigmas: Optional[bool] = False, + use_beta_sigmas: Optional[bool] = False, + use_flow_sigmas: Optional[bool] = False, + flow_shift: Optional[float] = 1.0, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min" + rescale_betas_zero_snr: bool = False, + ): + if self.config.use_beta_sigmas and not is_scipy_available(): + raise ImportError( + "Make sure to install scipy if you want to use beta sigmas.") + if sum([ + self.config.use_beta_sigmas, self.config.use_exponential_sigmas, + self.config.use_karras_sigmas + ]) > 1: + raise ValueError( + "Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, `config.use_karras_sigmas` can be used." + ) + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, + beta_end, + num_train_timesteps, + dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, + beta_end**0.5, + num_train_timesteps, + dtype=torch.float32)**2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError( + f"{beta_schedule} is not implemented for {self.__class__}") + + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + if rescale_betas_zero_snr: + # Close to 0 without being 0 so first sigma is not inf + # FP16 smallest positive subnormal works well here + self.alphas_cumprod[-1] = 2**-24 + + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod)**0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + if solver_type not in ["bh1", "bh2"]: + if solver_type in ["midpoint", "heun", "logrho"]: + self.register_to_config(solver_type="bh2") + else: + raise NotImplementedError( + f"{solver_type} is not implemented for {self.__class__}") + + self.predict_x0 = predict_x0 + # setable values + self.num_inference_steps: Optional[int] = None + timesteps = np.linspace(0, + num_train_timesteps - 1, + num_train_timesteps, + dtype=np.float32)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.model_outputs = [None] * solver_order + self.timestep_list: List[Union[int, + torch.Tensor]] = [None] * solver_order + self.lower_order_nums = 0 + self.disable_corrector = list(disable_corrector) + self.solver_p = solver_p + self.last_sample = None + self._step_index: Optional[int] = None + self._begin_index: Optional[int] = None + self.sigmas = self.sigmas.to( + "cpu") # to avoid too much CPU/GPU communication + + BaseScheduler.__init__(self) + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + def set_shift(self, shift: float) -> None: + self.config.flow_shift = shift + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_timesteps(self, + num_inference_steps: int, + device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = (np.linspace( + 0, self.config.num_train_timesteps - 1, num_inference_steps + + 1).round()[::-1][:-1].copy().astype(np.int64)) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // ( + num_inference_steps + 1) + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps + 1) * + step_ratio).round()[::-1][:-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.arange(self.config.num_train_timesteps, 0, + -step_ratio).round().copy().astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array( + ((1 - self.alphas_cumprod) / self.alphas_cumprod)**0.5) + if self.config.use_karras_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_karras( + in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([ + self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas + ]).round() + if self.config.final_sigmas_type == "sigma_min": + sigma_last = sigmas[-1] + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + elif self.config.use_exponential_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_exponential( + in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array( + [self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]) + if self.config.final_sigmas_type == "sigma_min": + sigma_last = sigmas[-1] + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + elif self.config.use_beta_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_beta( + in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array( + [self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]) + if self.config.final_sigmas_type == "sigma_min": + sigma_last = sigmas[-1] + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + elif self.config.use_flow_sigmas: + alphas = np.linspace(1, 1 / self.config.num_train_timesteps, + num_inference_steps + 1) + sigmas = 1.0 - alphas + sigmas = np.flip( + self.config.flow_shift * sigmas / + (1 + (self.config.flow_shift - 1) * sigmas))[:-1].copy() + timesteps = (sigmas * self.config.num_train_timesteps).copy() + if self.config.final_sigmas_type == "sigma_min": + sigma_last = sigmas[-1] + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ((1 - self.alphas_cumprod[0]) / + self.alphas_cumprod[0])**0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps).to(device=device, + dtype=torch.int64) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + self.last_sample = None + if self.solver_p: + self.solver_p.set_timesteps(self.num_inference_steps, device=device) + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to( + "cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float( + ) # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, + self.config.dynamic_thresholding_ratio, + dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze( + 1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp( + sample, -s, s + ) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas) -> torch.Tensor: + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum( + (dists >= 0), + axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._sigma_to_alpha_sigma_t + def _sigma_to_alpha_sigma_t( + self, sigma: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + if self.config.use_flow_sigmas: + alpha_t = 1 - sigma + sigma_t = sigma + else: + alpha_t = 1 / ((sigma**2 + 1)**0.5) + sigma_t = sigma * alpha_t + + return alpha_t, sigma_t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.Tensor, + num_inference_steps) -> torch.Tensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min**(1 / rho) + max_inv_rho = sigma_max**(1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho))**rho + return sigmas + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential + def _convert_to_exponential(self, in_sigmas: torch.Tensor, + num_inference_steps: int) -> torch.Tensor: + """Constructs an exponential noise schedule.""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + sigmas = np.exp( + np.linspace(math.log(sigma_max), math.log(sigma_min), + num_inference_steps)) + return sigmas + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta + def _convert_to_beta(self, + in_sigmas: torch.Tensor, + num_inference_steps: int, + alpha: float = 0.6, + beta: float = 0.6) -> torch.Tensor: + """From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + sigmas = np.array([ + sigma_min + (ppf * (sigma_max - sigma_min)) for ppf in [ + scipy.stats.beta.ppf(timestep, alpha, beta) + for timestep in 1 - np.linspace(0, 1, num_inference_steps) + ] + ]) + return sigmas + + def convert_model_output( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + **kwargs, + ) -> torch.Tensor: + r""" + Convert the model output to the corresponding type the UniPC algorithm needs. + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.Tensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError( + "missing `sample` as a required keyword argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + + if self.predict_x0: + if self.config.prediction_type == "epsilon": + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + x0_pred = alpha_t * sample - sigma_t * model_output + elif self.config.prediction_type == "flow_prediction": + sigma_t = self.sigmas[self.step_index] + x0_pred = sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, " + "`v_prediction`, or `flow_prediction` for the UniPCMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + else: + if self.config.prediction_type == "epsilon": + return model_output + elif self.config.prediction_type == "sample": + epsilon = (sample - alpha_t * model_output) / sigma_t + return epsilon + elif self.config.prediction_type == "v_prediction": + epsilon = alpha_t * model_output + sigma_t * sample + return epsilon + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the UniPCMultistepScheduler.") + + def multistep_uni_p_bh_update( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + order: Optional[int] = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the UniP (B(h) version). Alternatively, `self.solver_p` is used if is specified. + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model at the current timestep. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + order (`int`): + The order of UniP at this timestep (corresponds to the *p* in UniPC-p). + + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + prev_timestep = args[0] if len(args) > 0 else kwargs.pop( + "prev_timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError( + " missing `sample` as a required keyword argument") + if order is None: + if len(args) > 2: + order = args[2] + else: + raise ValueError( + " missing `order` as a required keyword argument") + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + model_output_list = self.model_outputs + + s0 = self.timestep_list[-1] + m0 = model_output_list[-1] + x = sample + + if self.solver_p: + x_t = self.solver_p.step(model_output, s0, x).prev_sample + return x_t + + sigma_t, sigma_s0 = self.sigmas[self.step_index + + 1], self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = sample.device + + rks = [] + D1s = [] + for i in range(1, order): + si = self.step_index - i + mi: torch.Tensor = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks.append(rk) + D1s.append((mi - m0) / rk) + + rks.append(1.0) + rks = torch.tensor(rks, device=device) + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R_tensor: torch.Tensor = torch.stack(R) + b = torch.tensor(b, device=device) + + D1s_tensor: Optional[torch.Tensor] = None + if len(D1s) > 0: + D1s_tensor = torch.stack(D1s, dim=1) # (B, K) + # for order 2, we use a simplified version + if order == 2: + rhos_p = torch.tensor([0.5], dtype=x.dtype, device=device) + else: + rhos_p = torch.linalg.solve(R_tensor[:-1, :-1], + b[:-1]).to(device).to(x.dtype) + else: + D1s_tensor = None + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s_tensor is not None: + pred_res = torch.einsum("k,bkc...->bc...", rhos_p, D1s_tensor) + else: + pred_res = 0 + x_t = x_t_ - alpha_t * B_h * pred_res + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s_tensor is not None: + pred_res = torch.einsum("k,bkc...->bc...", rhos_p, D1s_tensor) + else: + pred_res = 0 + x_t = x_t_ - sigma_t * B_h * pred_res + + x_t = x_t.to(x.dtype) + return x_t + + def multistep_uni_c_bh_update( + self, + this_model_output: torch.Tensor, + *args, + last_sample: Optional[torch.Tensor] = None, + this_sample: Optional[torch.Tensor] = None, + order: Optional[int] = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the UniC (B(h) version). + + Args: + this_model_output (`torch.Tensor`): + The model outputs at `x_t`. + this_timestep (`int`): + The current timestep `t`. + last_sample (`torch.Tensor`): + The generated sample before the last predictor `x_{t-1}`. + this_sample (`torch.Tensor`): + The generated sample after the last predictor `x_{t}`. + order (`int`): + The `p` of UniC-p at this step. The effective order of accuracy should be `order + 1`. + + Returns: + `torch.Tensor`: + The corrected sample tensor at the current timestep. + """ + this_timestep = args[0] if len(args) > 0 else kwargs.pop( + "this_timestep", None) + if last_sample is None: + if len(args) > 1: + last_sample = args[1] + else: + raise ValueError( + " missing`last_sample` as a required keyword argument") + if this_sample is None: + if len(args) > 2: + this_sample = args[2] + else: + raise ValueError( + " missing`this_sample` as a required keyword argument") + if order is None: + if len(args) > 3: + order = args[3] + else: + raise ValueError( + " missing`order` as a required keyword argument") + if this_timestep is not None: + deprecate( + "this_timestep", + "1.0.0", + "Passing `this_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + model_output_list = self.model_outputs + + m0 = model_output_list[-1] + x = last_sample + x_t = this_sample + model_t = this_model_output + + sigma_t, sigma_s0 = self.sigmas[self.step_index], self.sigmas[ + self.step_index - 1] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = this_sample.device + + rks = [] + D1s = [] + for i in range(1, order): + si = self.step_index - (i + 1) + mi: torch.Tensor = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks.append(rk) + D1s.append((mi - m0) / rk) + + rks.append(1.0) + rks = torch.tensor(rks, device=device) + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R = torch.stack(R) + b = torch.tensor(b, device=device) + + D1s_tensor: Optional[torch.Tensor] = torch.stack( + D1s, dim=1) if len(D1s) > 0 else None + + # for order 1, we use a simplified version + if order == 1: + rhos_c = torch.tensor([0.5], dtype=x.dtype, device=device) + else: + rhos_c = torch.linalg.solve(R, b).to(device).to(x.dtype) + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s_tensor is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], + D1s_tensor) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - alpha_t * B_h * (corr_res + rhos_c[-1] * D1_t) + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s_tensor is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], + D1s_tensor) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - sigma_t * B_h * (corr_res + rhos_c[-1] * D1_t) + x_t = x_t.to(x.dtype) + return x_t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None) -> int: + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + index_candidates = (schedule_timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + return step_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index + def _init_step_index(self, timestep) -> None: + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.Tensor, + timestep: Union[int, torch.Tensor], + sample: torch.Tensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep UniPC. + + Args: + model_output (`torch.Tensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + assert self.step_index is not None + use_corrector = (self.step_index > 0 + and self.step_index - 1 not in self.disable_corrector + and self.last_sample is not None) + + model_output_convert = self.convert_model_output(model_output, + sample=sample) + if use_corrector: + sample = self.multistep_uni_c_bh_update( + this_model_output=model_output_convert, + last_sample=self.last_sample, + this_sample=sample, + order=self.this_order, + ) + + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.timestep_list[i] = self.timestep_list[i + 1] + + self.model_outputs[-1] = model_output_convert + self.timestep_list[-1] = timestep + + if self.config.lower_order_final: + this_order = min(self.config.solver_order, + len(self.timesteps) - self.step_index) + else: + this_order = self.config.solver_order + + self.this_order: int = min(this_order, self.lower_order_nums + + 1) # warmup for multistep + assert self.this_order > 0 + + self.last_sample = sample + prev_sample = self.multistep_uni_p_bh_update( + model_output= + model_output, # pass the original non-converted model output, in case solver-p is used + sample=sample, + order=self.this_order, + ) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # upon completion increase step index by one + assert self._step_index is not None + self._step_index += 1 + + if not return_dict: + return (prev_sample, ) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.Tensor, *args, + **kwargs) -> torch.Tensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.Tensor`): + The input sample. + + Returns: + `torch.Tensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise + def add_noise( + self, + original_samples: torch.Tensor, + noise: torch.Tensor, + timesteps: torch.IntTensor, + ) -> torch.Tensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, + dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point( + timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, + dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, + dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [ + self.index_for_timestep(t, schedule_timesteps) + for t in timesteps + ] + elif self.step_index is not None: + # add_noise is called after first denoising step (for inpainting) + step_indices = [self.step_index] * timesteps.shape[0] + else: + # add noise is called before first denoising step to create initial latent(img2img) + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/fastvideo/v1/models/vaes/common.py b/fastvideo/v1/models/vaes/common.py index 8e9cc5920..c56d1454d 100644 --- a/fastvideo/v1/models/vaes/common.py +++ b/fastvideo/v1/models/vaes/common.py @@ -2,11 +2,12 @@ from abc import ABC, abstractmethod from math import prod -from typing import Iterator, Optional, Tuple +from typing import Iterator, Optional, Tuple, Union import numpy as np import torch import torch.distributed as dist +from diffusers.utils.torch_utils import randn_tensor from fastvideo.v1.distributed import (get_sequence_model_parallel_rank, get_sequence_model_parallel_world_size) @@ -20,8 +21,11 @@ class ParallelTiledVAE(ABC): tile_sample_stride_width: int tile_sample_stride_num_frames: int use_tiling: bool + use_temporal_tiling: bool + use_parallel_tiling: bool temporal_compression_ratio: int spatial_compression_ratio: int + scaling_factor: Union[float, torch.tensor] def __init__(self, *args, **kwargs) -> None: # Check if subclass has defined all required properties @@ -30,7 +34,8 @@ class ParallelTiledVAE(ABC): 'tile_sample_min_num_frames', 'tile_sample_stride_height', 'tile_sample_stride_width', 'tile_sample_stride_num_frames', 'spatial_compression_ratio', 'temporal_compression_ratio', - 'use_tiling' + 'use_tiling', 'use_temporal_tiling', 'use_parallel_tiling', + 'scaling_factor' ] for attr in required_attributes: @@ -52,13 +57,13 @@ class ParallelTiledVAE(ABC): latent_num_frames = (num_frames - 1) // self.temporal_compression_ratio + 1 - if self.use_tiling and num_frames > self.tile_sample_min_num_frames: + if self.use_tiling and self.use_temporal_tiling and num_frames > self.tile_sample_min_num_frames: latents = self.tiled_encode(x)[:, :, :latent_num_frames] elif self.use_tiling and (width > self.tile_sample_min_width or height > self.tile_sample_min_height): - latents = self.spatial_tiled_encode(x) + latents = self.spatial_tiled_encode(x)[:, :, :latent_num_frames] else: - latents = self._encode(x) + latents = self._encode(x)[:, :, :latent_num_frames] return DiagonalGaussianDistribution(latents) def decode(self, z: torch.Tensor) -> torch.Tensor: @@ -69,16 +74,17 @@ class ParallelTiledVAE(ABC): num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1 - if self.use_tiling and get_sequence_model_parallel_world_size() > 1: + if self.use_tiling and self.use_parallel_tiling and get_sequence_model_parallel_world_size( + ) > 1: return self.parallel_tiled_decode(z)[:, :, :num_sample_frames] - if self.use_tiling and num_frames > tile_latent_min_num_frames: + if self.use_tiling and self.use_temporal_tiling and num_frames > tile_latent_min_num_frames: return self.tiled_decode(z)[:, :, :num_sample_frames] if self.use_tiling and (width > tile_latent_min_width or height > tile_latent_min_height): - return self.spatial_tiled_decode(z) + return self.spatial_tiled_decode(z)[:, :, :num_sample_frames] - return self._decode(z) + return self._decode(z)[:, :, :num_sample_frames] def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: @@ -462,7 +468,7 @@ class DiagonalGaussianDistribution: def sample(self, generator: Optional[torch.Generator] = None) -> torch.Tensor: # make sure sample is on the same device as the parameters and has same dtype - sample = torch.randn( + sample = randn_tensor( self.mean.shape, generator=generator, device=self.parameters.device, diff --git a/fastvideo/v1/models/vaes/hunyuanvae.py b/fastvideo/v1/models/vaes/hunyuanvae.py index 0d5fd4400..e68e5089b 100644 --- a/fastvideo/v1/models/vaes/hunyuanvae.py +++ b/fastvideo/v1/models/vaes/hunyuanvae.py @@ -847,6 +847,9 @@ class AutoencoderKLHunyuanVideo(nn.Module, ParallelTiledVAE): # frames spatially into smaller tiles and performing multiple forward passes for decoding, and then blending the # intermediate tiles together, the memory requirement can be lowered. self.use_tiling = True + self.use_temporal_tiling = True + self.use_parallel_tiling = True + self.scaling_factor = scaling_factor # The minimal tile height and width for spatial tiling to be used self.tile_sample_min_height = 256 diff --git a/fastvideo/v1/models/vaes/wanvae.py b/fastvideo/v1/models/vaes/wanvae.py index 094631875..691861837 100644 --- a/fastvideo/v1/models/vaes/wanvae.py +++ b/fastvideo/v1/models/vaes/wanvae.py @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 + # Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,8 +22,8 @@ import torch.nn.functional as F import torch.utils.checkpoint from fastvideo.v1.layers.activation import get_act_fn +from fastvideo.v1.models.utils import auto_attributes from fastvideo.v1.models.vaes.common import ParallelTiledVAE -from fastvideo.v1.utils import auto_attributes CACHE_T = 2 @@ -647,6 +649,10 @@ class AutoencoderKLWan(nn.Module, ParallelTiledVAE): self.temperal_upsample = list(temperal_downsample)[::-1] self.latents_mean = list(latents_mean) self.latents_std = list(latents_std) + self.scaling_factor = 1.0 / torch.tensor(self.config.latents_std).view( + 1, self.config.z_dim, 1, 1, 1) + self.shift_factor = torch.tensor(self.config.latents_mean).view( + 1, self.config.z_dim, 1, 1, 1) if load_encoder: self.encoder = WanEncoder3d(base_dim, z_dim * 2, dim_mult, @@ -661,6 +667,8 @@ class AutoencoderKLWan(nn.Module, ParallelTiledVAE): self.temperal_upsample, dropout) self.use_tiling = True + self.use_temporal_tiling = False + self.use_parallel_tiling = False self.spatial_compression_ratio = 8 self.temporal_compression_ratio = 4 @@ -691,12 +699,16 @@ class AutoencoderKLWan(nn.Module, ParallelTiledVAE): enc = torch.cat([first_frame, enc], dim=2) return enc + def spatial_tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + first_frame = x[:, :, 0, :, :].unsqueeze(2) + first_frame = self._encode(first_frame, first_frame=True) + + enc = ParallelTiledVAE.spatial_tiled_encode(self, x) + enc = enc[:, :, 1:] + enc = torch.cat([first_frame, enc], dim=2) + return enc + def _decode(self, z: torch.Tensor, first_frame=False) -> torch.Tensor: - latents_mean = (torch.tensor(self.latents_mean).view( - 1, self.z_dim, 1, 1, 1).to(z.device, z.dtype)) - latents_std = 1.0 / torch.tensor(self.latents_std).view( - 1, self.z_dim, 1, 1, 1).to(z.device, z.dtype) - z = z / latents_std + latents_mean x = self.post_quant_conv(z) out = self.decoder(x, first_frame=first_frame) @@ -711,6 +723,12 @@ class AutoencoderKLWan(nn.Module, ParallelTiledVAE): dec = dec[:, :, start_frame_idx:] return dec + def spatial_tiled_decode(self, z: torch.Tensor) -> torch.Tensor: + dec = ParallelTiledVAE.spatial_tiled_decode(self, z) + start_frame_idx = self.temporal_compression_ratio - 1 + dec = dec[:, :, start_frame_idx:] + return dec + def parallel_tiled_decode(self, z: torch.FloatTensor) -> torch.FloatTensor: self.blend_num_frames *= 2 dec = ParallelTiledVAE.parallel_tiled_decode(self, z) diff --git a/fastvideo/v1/models/vision_utils.py b/fastvideo/v1/models/vision_utils.py new file mode 100644 index 000000000..b86f7489a --- /dev/null +++ b/fastvideo/v1/models/vision_utils.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +from typing import Callable, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import PIL.ImageOps +import requests +import torch +from packaging import version + +if version.parse(version.parse( + PIL.__version__).base_version) >= version.parse("9.1.0"): + PIL_INTERPOLATION = { + "linear": PIL.Image.Resampling.BILINEAR, + "bilinear": PIL.Image.Resampling.BILINEAR, + "bicubic": PIL.Image.Resampling.BICUBIC, + "lanczos": PIL.Image.Resampling.LANCZOS, + "nearest": PIL.Image.Resampling.NEAREST, + } +else: + PIL_INTERPOLATION = { + "linear": PIL.Image.LINEAR, + "bilinear": PIL.Image.BILINEAR, + "bicubic": PIL.Image.BICUBIC, + "lanczos": PIL.Image.LANCZOS, + "nearest": PIL.Image.NEAREST, + } + + +def pil_to_numpy( + images: Union[List[PIL.Image.Image], PIL.Image.Image]) -> np.ndarray: + r""" + Convert a PIL image or a list of PIL images to NumPy arrays. + + Args: + images (`PIL.Image.Image` or `List[PIL.Image.Image]`): + The PIL image or list of images to convert to NumPy format. + + Returns: + `np.ndarray`: + A NumPy array representation of the images. + """ + if not isinstance(images, list): + images = [images] + images = [np.array(image).astype(np.float32) / 255.0 for image in images] + images_arr: np.ndarray = np.stack(images, axis=0) + + return images_arr + + +def numpy_to_pt(images: np.ndarray) -> torch.Tensor: + r""" + Convert a NumPy image to a PyTorch tensor. + + Args: + images (`np.ndarray`): + The NumPy image array to convert to PyTorch format. + + Returns: + `torch.Tensor`: + A PyTorch tensor representation of the images. + """ + if images.ndim == 3: + images = images[..., None] + + images = torch.from_numpy(images.transpose(0, 3, 1, 2)) + return images + + +def normalize( + images: Union[np.ndarray, + torch.Tensor]) -> Union[np.ndarray, torch.Tensor]: + r""" + Normalize an image array to [-1,1]. + + Args: + images (`np.ndarray` or `torch.Tensor`): + The image array to normalize. + + Returns: + `np.ndarray` or `torch.Tensor`: + The normalized image array. + """ + return 2.0 * images - 1.0 + + +def load_image( + image: Union[str, PIL.Image.Image], + convert_method: Optional[Callable[[PIL.Image.Image], + PIL.Image.Image]] = None +) -> PIL.Image.Image: + """ + Loads `image` to a PIL Image. + + Args: + image (`str` or `PIL.Image.Image`): + The image to convert to the PIL Image format. + convert_method (Callable[[PIL.Image.Image], PIL.Image.Image], *optional*): + A conversion method to apply to the image after loading it. When set to `None` the image will be converted + "RGB". + + Returns: + `PIL.Image.Image`: + A PIL Image. + """ + if isinstance(image, str): + if image.startswith("http://") or image.startswith("https://"): + image = PIL.Image.open(requests.get(image, stream=True).raw) + elif os.path.isfile(image): + image = PIL.Image.open(image) + else: + raise ValueError( + f"Incorrect path or URL. URLs must start with `http://` or `https://`, and {image} is not a valid path." + ) + elif isinstance(image, PIL.Image.Image): + image = image + else: + raise ValueError( + "Incorrect format used for the image. Should be a URL linking to an image, a local path, or a PIL image." + ) + + image = PIL.ImageOps.exif_transpose(image) + + if convert_method is not None: + image = convert_method(image) + else: + image = image.convert("RGB") + + return image + + +def get_default_height_width( + image: Union[PIL.Image.Image, np.ndarray, torch.Tensor], + vae_scale_factor: int, + height: Optional[int] = None, + width: Optional[int] = None, +) -> Tuple[int, int]: + r""" + Returns the height and width of the image, downscaled to the next integer multiple of `vae_scale_factor`. + + Args: + image (`Union[PIL.Image.Image, np.ndarray, torch.Tensor]`): + The image input, which can be a PIL image, NumPy array, or PyTorch tensor. If it is a NumPy array, it + should have shape `[batch, height, width]` or `[batch, height, width, channels]`. If it is a PyTorch + tensor, it should have shape `[batch, channels, height, width]`. + height (`Optional[int]`, *optional*, defaults to `None`): + The height of the preprocessed image. If `None`, the height of the `image` input will be used. + width (`Optional[int]`, *optional*, defaults to `None`): + The width of the preprocessed image. If `None`, the width of the `image` input will be used. + + Returns: + `Tuple[int, int]`: + A tuple containing the height and width, both resized to the nearest integer multiple of + `vae_scale_factor`. + """ + + if height is None: + if isinstance(image, PIL.Image.Image): + height = image.height + elif isinstance(image, torch.Tensor): + height = image.shape[2] + else: + height = image.shape[1] + + if width is None: + if isinstance(image, PIL.Image.Image): + width = image.width + elif isinstance(image, torch.Tensor): + width = image.shape[3] + else: + width = image.shape[2] + + width, height = (x - x % vae_scale_factor for x in (width, height) + ) # resize to integer multiple of vae_scale_factor + + return height, width + + +def resize( + image: Union[PIL.Image.Image, np.ndarray, torch.Tensor], + height: int, + width: int, + resize_mode: str = "default", # "default", "fill", "crop" + resample: str = "lanczos", +) -> Union[PIL.Image.Image, np.ndarray, torch.Tensor]: + """ + Resize image. + + Args: + image (`PIL.Image.Image`, `np.ndarray` or `torch.Tensor`): + The image input, can be a PIL image, numpy array or pytorch tensor. + height (`int`): + The height to resize to. + width (`int`): + The width to resize to. + resize_mode (`str`, *optional*, defaults to `default`): + The resize mode to use, can be one of `default` or `fill`. If `default`, will resize the image to fit + within the specified width and height, and it may not maintaining the original aspect ratio. If `fill`, + will resize the image to fit within the specified width and height, maintaining the aspect ratio, and + then center the image within the dimensions, filling empty with data from image. If `crop`, will resize + the image to fit within the specified width and height, maintaining the aspect ratio, and then center + the image within the dimensions, cropping the excess. Note that resize_mode `fill` and `crop` are only + supported for PIL image input. + + Returns: + `PIL.Image.Image`, `np.ndarray` or `torch.Tensor`: + The resized image. + """ + if resize_mode != "default" and not isinstance(image, PIL.Image.Image): + raise ValueError( + f"Only PIL image input is supported for resize_mode {resize_mode}") + assert isinstance(image, PIL.Image.Image) + if resize_mode == "default": + image = image.resize((width, height), + resample=PIL_INTERPOLATION[resample]) + else: + raise ValueError(f"resize_mode {resize_mode} is not supported") + return image diff --git a/fastvideo/v1/pipelines/pipeline_batch_info.py b/fastvideo/v1/pipelines/pipeline_batch_info.py index 2c448ccd7..e89bdf75c 100644 --- a/fastvideo/v1/pipelines/pipeline_batch_info.py +++ b/fastvideo/v1/pipelines/pipeline_batch_info.py @@ -29,6 +29,10 @@ class ForwardBatch: generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None + # Image inputs + image_path: Optional[str] = None + image_embeds: List[torch.Tensor] = field(default_factory=list) + # Text inputs prompt: Optional[Union[str, List[str]]] = None negative_prompt: Optional[Union[str, List[str]]] = None @@ -55,6 +59,7 @@ class ForwardBatch: # Latent tensors latents: Optional[torch.Tensor] = None noise_pred: Optional[torch.Tensor] = None + image_latent: Optional[torch.Tensor] = None # Latent dimensions num_channels_latents: Optional[int] = None @@ -100,3 +105,4 @@ class ForwardBatch: # Set do_classifier_free_guidance based on guidance scale and negative prompt if self.guidance_scale > 1.0: self.do_classifier_free_guidance = True + self.negative_prompt_embeds = [] diff --git a/fastvideo/v1/pipelines/stages/__init__.py b/fastvideo/v1/pipelines/stages/__init__.py index 613ae46b1..432161dc2 100644 --- a/fastvideo/v1/pipelines/stages/__init__.py +++ b/fastvideo/v1/pipelines/stages/__init__.py @@ -7,15 +7,19 @@ complete diffusion pipelines. """ from fastvideo.v1.pipelines.stages.base import PipelineStage +from fastvideo.v1.pipelines.stages.clip_image_encoding import ( + CLIPImageEncodingStage) from fastvideo.v1.pipelines.stages.clip_text_encoding import ( CLIPTextEncodingStage) from fastvideo.v1.pipelines.stages.conditioning import ConditioningStage from fastvideo.v1.pipelines.stages.decoding import DecodingStage from fastvideo.v1.pipelines.stages.denoising import DenoisingStage +from fastvideo.v1.pipelines.stages.encoding import EncodingStage from fastvideo.v1.pipelines.stages.input_validation import InputValidationStage from fastvideo.v1.pipelines.stages.latent_preparation import ( LatentPreparationStage) from fastvideo.v1.pipelines.stages.llama_encoding import LlamaEncodingStage +from fastvideo.v1.pipelines.stages.t5_encoding import T5EncodingStage from fastvideo.v1.pipelines.stages.timestep_preparation import ( TimestepPreparationStage) @@ -26,7 +30,10 @@ __all__ = [ "LatentPreparationStage", "ConditioningStage", "DenoisingStage", + "EncodingStage", "DecodingStage", "LlamaEncodingStage", + "T5EncodingStage", "CLIPTextEncodingStage", + "CLIPImageEncodingStage", ] diff --git a/fastvideo/v1/pipelines/stages/clip_image_encoding.py b/fastvideo/v1/pipelines/stages/clip_image_encoding.py new file mode 100644 index 000000000..5b9d77807 --- /dev/null +++ b/fastvideo/v1/pipelines/stages/clip_image_encoding.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Image encoding stages for I2V diffusion pipelines. + +This module contains implementations of image encoding stages for diffusion pipelines. +""" + +import torch + +from fastvideo.v1.forward_context import set_forward_context +from fastvideo.v1.inference_args import InferenceArgs +from fastvideo.v1.logger import init_logger +from fastvideo.v1.models.vision_utils import load_image +from fastvideo.v1.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.v1.pipelines.stages.base import PipelineStage + +logger = init_logger(__name__) + + +class CLIPImageEncodingStage(PipelineStage): + """ + Stage for encoding image prompts into embeddings for diffusion models. + + This stage handles the encoding of image prompts into the embedding space + expected by the diffusion model. + """ + + def __init__(self, image_encoder, image_processor) -> None: + """ + Initialize the prompt encoding stage. + + Args: + enable_logging: Whether to enable logging for this stage. + is_secondary: Whether this is a secondary image encoder. + """ + super().__init__() + self.image_processor = image_processor + self.image_encoder = image_encoder + + def forward( + self, + batch: ForwardBatch, + inference_args: InferenceArgs, + ) -> ForwardBatch: + """ + Encode the prompt into image encoder hidden states. + + Args: + batch: The current batch information. + inference_args: The inference arguments. + + Returns: + The batch with encoded prompt embeddings. + """ + if inference_args.use_cpu_offload: + self.image_encoder = self.image_encoder.to(batch.device) + + image = load_image(batch.image_path) + + image_inputs = self.image_processor( + images=image, return_tensors="pt").to(batch.device) + with set_forward_context(current_timestep=0, attn_metadata=None): + image_embeds = self.image_encoder(**image_inputs) + + batch.image_embeds.append(image_embeds) + + if inference_args.use_cpu_offload: + self.image_encoder.to('cpu') + torch.cuda.empty_cache() + + return batch diff --git a/fastvideo/v1/pipelines/stages/clip_text_encoding.py b/fastvideo/v1/pipelines/stages/clip_text_encoding.py index 490f43f01..50be5aab6 100644 --- a/fastvideo/v1/pipelines/stages/clip_text_encoding.py +++ b/fastvideo/v1/pipelines/stages/clip_text_encoding.py @@ -5,6 +5,8 @@ Prompt encoding stages for diffusion pipelines. This module contains implementations of prompt encoding stages for diffusion pipelines. """ +import torch + from fastvideo.v1.forward_context import set_forward_context from fastvideo.v1.inference_args import InferenceArgs from fastvideo.v1.logger import init_logger @@ -49,6 +51,8 @@ class CLIPTextEncodingStage(PipelineStage): Returns: The batch with encoded prompt embeddings. """ + if inference_args.use_cpu_offload: + self.text_encoder = self.text_encoder.to(batch.device) text_inputs = self.tokenizer( batch.prompt, @@ -64,4 +68,24 @@ class CLIPTextEncodingStage(PipelineStage): batch.prompt_embeds.append(prompt_embeds) + if batch.do_classifier_free_guidance: + negative_text_inputs = self.tokenizer( + batch.negative_prompt, + truncation=True, + # better way to handle this? + max_length=77, + return_tensors="pt", + ) + with set_forward_context(current_timestep=0, attn_metadata=None): + negative_outputs = self.text_encoder( + input_ids=negative_text_inputs["input_ids"].to( + batch.device), ) + negative_prompt_embeds = negative_outputs["pooler_output"] + + batch.negative_prompt_embeds.append(negative_prompt_embeds) + + if inference_args.use_cpu_offload: + self.text_encoder.to('cpu') + torch.cuda.empty_cache() + return batch diff --git a/fastvideo/v1/pipelines/stages/conditioning.py b/fastvideo/v1/pipelines/stages/conditioning.py index 66a8e0fbe..fb88d33de 100644 --- a/fastvideo/v1/pipelines/stages/conditioning.py +++ b/fastvideo/v1/pipelines/stages/conditioning.py @@ -39,8 +39,7 @@ class ConditioningStage(PipelineStage): if not batch.do_classifier_free_guidance: return batch else: - raise NotImplementedError( - "Classifier-free guidance is not supported yet") + return batch logger.info("batch.negative_prompt_embeds: %s", batch.negative_prompt_embeds) diff --git a/fastvideo/v1/pipelines/stages/decoding.py b/fastvideo/v1/pipelines/stages/decoding.py index fb52e6c37..432f67282 100644 --- a/fastvideo/v1/pipelines/stages/decoding.py +++ b/fastvideo/v1/pipelines/stages/decoding.py @@ -54,13 +54,20 @@ class DecodingStage(PipelineStage): vae_autocast_enabled = (vae_dtype != torch.float32 ) and not inference_args.disable_autocast - # Apply scaling/shifting if needed - if (hasattr(self.vae.config, "shift_factor") - and self.vae.config.shift_factor): - latents = (latents / self.vae.config.scaling_factor + - self.vae.config.shift_factor) + if isinstance(self.vae.scaling_factor, torch.Tensor): + latents = latents / self.vae.scaling_factor.to( + latents.device, latents.dtype) else: - latents = latents / self.vae.config.scaling_factor + latents = latents / self.vae.scaling_factor + + # Apply shifting if needed + if (hasattr(self.vae, "shift_factor") + and self.vae.shift_factor is not None): + if isinstance(self.vae.shift_factor, torch.Tensor): + latents += self.vae.shift_factor.to(latents.device, + latents.dtype) + else: + latents += self.vae.shift_factor # Decode latents with torch.autocast(device_type="cuda", @@ -70,6 +77,8 @@ class DecodingStage(PipelineStage): self.vae.enable_tiling() # if inference_args.vae_sp: # self.vae.enable_parallel() + if not vae_autocast_enabled: + latents = latents.to(vae_dtype) image = self.vae.decode(latents) # Normalize image to [0, 1] range diff --git a/fastvideo/v1/pipelines/stages/denoising.py b/fastvideo/v1/pipelines/stages/denoising.py index 990c062dc..23d4ede92 100644 --- a/fastvideo/v1/pipelines/stages/denoising.py +++ b/fastvideo/v1/pipelines/stages/denoising.py @@ -53,6 +53,9 @@ class DenoisingStage(PipelineStage): Returns: The batch with denoised latents. """ + # If use cpu offload, need to load the model back into gpu again + if inference_args.use_cpu_offload: + self.transformer = self.transformer.to(batch.device) # Prepare extra step kwargs for scheduler extra_step_kwargs = self.prepare_extra_func_kwargs( self.scheduler.step, @@ -77,6 +80,12 @@ class DenoisingStage(PipelineStage): n=world_size).contiguous() latents = latents[:, :, rank, :, :, :] batch.latents = latents + if batch.image_latent is not None: + image_latent = rearrange(batch.image_latent, + "b t (n s) h w -> b t n s h w", + n=world_size).contiguous() + image_latent = image_latent[:, :, rank, :, :, :] + batch.image_latent = image_latent # Get timesteps and calculate warmup steps timesteps = batch.timesteps @@ -98,9 +107,28 @@ class DenoisingStage(PipelineStage): result[t][layer][h] = value return result + # Prepare image latents and embeddings for I2V generation + image_embeds = batch.image_embeds + if len(image_embeds) > 0: + assert torch.isnan(image_embeds[0]).sum() == 0 + image_embeds = [ + image_embed.to(target_dtype) for image_embed in image_embeds + ] + + image_kwargs = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + "encoder_hidden_states_image": image_embeds, + }, + ) + # Get latents and embeddings latents = batch.latents prompt_embeds = batch.prompt_embeds + assert torch.isnan(prompt_embeds[0]).sum() == 0 + if batch.do_classifier_free_guidance: + neg_prompt_embeds = batch.negative_prompt_embeds + assert torch.isnan(neg_prompt_embeds[0]).sum() == 0 # Run denoising loop with self.progress_bar(total=num_inference_steps) as progress_bar: @@ -109,10 +137,13 @@ class DenoisingStage(PipelineStage): if hasattr(self, 'interrupt') and self.interrupt: continue - # Expand latents for classifier-free guidance - latent_model_input = (torch.cat( - [latents] * - 2) if batch.do_classifier_free_guidance else latents) + # Expand latents for I2V + latent_model_input = latents.to(target_dtype) + if batch.image_latent is not None: + latent_model_input = torch.cat( + [latent_model_input, batch.image_latent], + dim=1).to(target_dtype) + assert torch.isnan(latent_model_input).sum() == 0 latent_model_input = self.scheduler.scale_model_input( latent_model_input, t) @@ -160,8 +191,6 @@ class DenoisingStage(PipelineStage): inference_args=inference_args, ) assert attn_metadata is not None, "attn_metadata cannot be None" - else: - attn_metadata = None else: attn_metadata = None @@ -180,29 +209,43 @@ class DenoisingStage(PipelineStage): prompt_embeds, t_expand, guidance=guidance_expand, + **image_kwargs, ) - # Apply guidance - if batch.do_classifier_free_guidance: - noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) - noise_pred = noise_pred_uncond + batch.guidance_scale * ( - noise_pred_text - noise_pred_uncond) + # Apply guidance + if batch.do_classifier_free_guidance: + with set_forward_context( + current_timestep=i, + attn_metadata=attn_metadata, + # inference_args=inference_args + ): + # Run transformer + noise_pred_uncond = self.transformer( + latent_model_input, + neg_prompt_embeds, + t_expand, + guidance=guidance_expand, + **image_kwargs, + ) + noise_pred_text = noise_pred + noise_pred = noise_pred_uncond + batch.guidance_scale * ( + noise_pred_text - noise_pred_uncond) - # Apply guidance rescale if needed - if batch.guidance_rescale > 0.0: - # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf - noise_pred = self.rescale_noise_cfg( - noise_pred, - noise_pred_text, - guidance_rescale=batch.guidance_rescale, - ) + # Apply guidance rescale if needed + if batch.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = self.rescale_noise_cfg( + noise_pred, + noise_pred_text, + guidance_rescale=batch.guidance_rescale, + ) - # Compute the previous noisy sample - latents = self.scheduler.step(noise_pred, - t, - latents, - **extra_step_kwargs, - return_dict=False)[0] + # Compute the previous noisy sample + latents = self.scheduler.step(noise_pred, + t, + latents, + **extra_step_kwargs, + return_dict=False)[0] # Update progress bar if i == len(timesteps) - 1 or ( @@ -218,11 +261,15 @@ class DenoisingStage(PipelineStage): # Update batch with final latents batch.latents = latents + if inference_args.use_cpu_offload: + self.transformer.to('cpu') + torch.cuda.empty_cache() + return batch def prepare_extra_func_kwargs(self, func, kwargs) -> Dict[str, Any]: """ - Prepare extra kwargs for the scheduler step. + Prepare extra kwargs for the scheduler step / denoise step. Args: func: The function to prepare kwargs for. diff --git a/fastvideo/v1/pipelines/stages/encoding.py b/fastvideo/v1/pipelines/stages/encoding.py new file mode 100644 index 000000000..e2f2d8ee9 --- /dev/null +++ b/fastvideo/v1/pipelines/stages/encoding.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Encoding stage for diffusion pipelines. +""" +from typing import Optional + +import PIL.Image +import torch + +from fastvideo.v1.inference_args import InferenceArgs +from fastvideo.v1.logger import init_logger +from fastvideo.v1.models.vision_utils import (get_default_height_width, + load_image, normalize, + numpy_to_pt, pil_to_numpy, resize) +from fastvideo.v1.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.v1.pipelines.stages.base import PipelineStage +from fastvideo.v1.utils import PRECISION_TO_TYPE + +logger = init_logger(__name__) + + +class EncodingStage(PipelineStage): + """ + Stage for encoding pixel representations into latent space. + + This stage handles the encoding of pixel representations into the final + input format (e.g., latents). + """ + + def __init__(self, vae) -> None: + self.vae = vae + + def forward( + self, + batch: ForwardBatch, + inference_args: InferenceArgs, + ) -> ForwardBatch: + """ + Encode pixel representations into latent space. + + Args: + batch: The current batch information. + inference_args: The inference arguments. + + Returns: + The batch with encoded outputs. + """ + image_path = batch.image_path + # TODO(will): remove this once we add input/output validation for stages + if image_path is None: + raise ValueError("Image Path must be provided") + latent_height = batch.height // self.vae.spatial_compression_ratio + latent_width = batch.width // self.vae.spatial_compression_ratio + + image = load_image(image_path) + image = self.preprocess( + image, + vae_scale_factor=self.vae.spatial_compression_ratio, + height=batch.height, + width=batch.width).to(batch.device, dtype=torch.float32) + image = image.unsqueeze(2) + video_condition = torch.cat([ + image, + image.new_zeros(image.shape[0], image.shape[1], + inference_args.num_frames - 1, batch.height, + batch.width) + ], + dim=2) + video_condition = video_condition.to(device=batch.device, + dtype=torch.float32) + + # Setup VAE precision + vae_dtype = PRECISION_TO_TYPE[inference_args.vae_precision] + vae_autocast_enabled = ( + vae_dtype != torch.float32) and not inference_args.disable_autocast + + # Encode Image + with torch.autocast(device_type="cuda", + dtype=vae_dtype, + enabled=vae_autocast_enabled): + if inference_args.vae_tiling: + self.vae.enable_tiling() + # if inference_args.vae_sp: + # self.vae.enable_parallel() + if not vae_autocast_enabled: + video_condition = video_condition.to(vae_dtype) + encoder_output = self.vae.encode(video_condition) + + generator = batch.generator + latent_condition = self.retrieve_latents(encoder_output, generator[0]) + + # Apply shifting if needed + if (hasattr(self.vae, "shift_factor") + and self.vae.shift_factor is not None): + if isinstance(self.vae.shift_factor, torch.Tensor): + latent_condition -= self.vae.shift_factor.to( + latent_condition.device, latent_condition.dtype) + else: + latent_condition -= self.vae.shift_factor + + if isinstance(self.vae.scaling_factor, torch.Tensor): + latent_condition = latent_condition * self.vae.scaling_factor.to( + latent_condition.device, latent_condition.dtype) + else: + latent_condition = latent_condition * self.vae.scaling_factor + + mask_lat_size = torch.ones(1, 1, inference_args.num_frames, + latent_height, latent_width) + mask_lat_size[:, :, list(range(1, inference_args.num_frames))] = 0 + first_frame_mask = mask_lat_size[:, :, 0:1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, + dim=2, + repeats=self.vae.temporal_compression_ratio) + mask_lat_size = torch.concat( + [first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2) + mask_lat_size = mask_lat_size.view(1, -1, + self.vae.temporal_compression_ratio, + latent_height, latent_width) + mask_lat_size = mask_lat_size.transpose(1, 2) + mask_lat_size = mask_lat_size.to(latent_condition.device) + + batch.image_latent = torch.concat([mask_lat_size, latent_condition], + dim=1) + + # Offload models if needed + if hasattr(self, 'maybe_free_model_hooks'): + self.maybe_free_model_hooks() + + return batch + + def retrieve_latents(self, + encoder_output: torch.Tensor, + generator: Optional[torch.Generator] = None, + sample_mode: str = "sample"): + if sample_mode == "sample": + return encoder_output.sample(generator) + elif sample_mode == "argmax": + return encoder_output.mode() + else: + raise AttributeError( + "Could not access latents of provided encoder_output") + + def preprocess( + self, + image: PIL.Image.Image, + vae_scale_factor: int, + height: Optional[int] = None, + width: Optional[int] = None, + resize_mode: str = "default", # "default", "fill", "crop" + ) -> torch.Tensor: + image = [image] + + height, width = get_default_height_width(image[0], vae_scale_factor, + height, width) + image = [ + resize(i, height, width, resize_mode=resize_mode) for i in image + ] + image = pil_to_numpy(image) # to np + image = numpy_to_pt(image) # to pt + + do_normalize = True + if image.min() < 0: + do_normalize = False + if do_normalize: + image = normalize(image) + + return image diff --git a/fastvideo/v1/pipelines/stages/latent_preparation.py b/fastvideo/v1/pipelines/stages/latent_preparation.py index 73beeef02..86854649b 100644 --- a/fastvideo/v1/pipelines/stages/latent_preparation.py +++ b/fastvideo/v1/pipelines/stages/latent_preparation.py @@ -6,6 +6,7 @@ from diffusers.utils.torch_utils import randn_tensor from fastvideo.v1.inference_args import InferenceArgs from fastvideo.v1.logger import init_logger +from fastvideo.v1.models.vaes.common import ParallelTiledVAE from fastvideo.v1.pipelines.pipeline_batch_info import ForwardBatch from fastvideo.v1.pipelines.stages.base import PipelineStage @@ -20,9 +21,10 @@ class LatentPreparationStage(PipelineStage): denoised during the diffusion process. """ - def __init__(self, scheduler) -> None: + def __init__(self, scheduler, vae=None) -> None: super().__init__() self.scheduler = scheduler + self.vae = vae def forward( self, @@ -42,7 +44,7 @@ class LatentPreparationStage(PipelineStage): # Adjust video length based on VAE version if needed if hasattr(self, 'adjust_video_length'): - batch = self.adjust_video_length(batch, inference_args) + batch = self.adjust_video_length(self.vae, batch, inference_args) # Determine batch size if isinstance(batch.prompt, list): batch_size = len(batch.prompt) @@ -101,7 +103,7 @@ class LatentPreparationStage(PipelineStage): return batch - def adjust_video_length(self, batch: ForwardBatch, + def adjust_video_length(self, vae: ParallelTiledVAE, batch: ForwardBatch, inference_args: InferenceArgs) -> ForwardBatch: """ Adjust video length based on VAE version. @@ -114,6 +116,7 @@ class LatentPreparationStage(PipelineStage): The batch with adjusted video length. """ video_length = batch.num_frames + temporal_scale_factor = vae.temporal_compression_ratio if vae is not None else 4 # TODO - batch.num_frames = (video_length - 1) // 4 + 1 + batch.num_frames = (video_length - 1) // temporal_scale_factor + 1 return batch diff --git a/fastvideo/v1/pipelines/stages/llama_encoding.py b/fastvideo/v1/pipelines/stages/llama_encoding.py index ef88e8216..c72dab181 100644 --- a/fastvideo/v1/pipelines/stages/llama_encoding.py +++ b/fastvideo/v1/pipelines/stages/llama_encoding.py @@ -7,6 +7,8 @@ This module contains implementations of prompt encoding stages for diffusion pip from typing import TypedDict +import torch + from fastvideo.v1.forward_context import set_forward_context from fastvideo.v1.inference_args import InferenceArgs from fastvideo.v1.logger import init_logger @@ -71,6 +73,8 @@ class LlamaEncodingStage(PipelineStage): Returns: The batch with encoded prompt embeddings. """ + if inference_args.use_cpu_offload: + self.text_encoder = self.text_encoder.to(batch.device) text = prompt_template_video["template"].format(batch.prompt) text_inputs = self.tokenizer( @@ -93,4 +97,33 @@ class LlamaEncodingStage(PipelineStage): last_hidden_state = last_hidden_state[:, crop_start:] batch.prompt_embeds.append(last_hidden_state) + if batch.do_classifier_free_guidance: + negative_text = prompt_template_video["template"].format( + batch.negative_prompt) + negative_text_inputs = self.tokenizer( + negative_text, + truncation=True, + # better way to handle this? + max_length=256, + return_tensors="pt", + ) + hidden_state_skip_layer = 2 + with set_forward_context(current_timestep=0, attn_metadata=None): + negative_outputs = self.text_encoder( + input_ids=negative_text_inputs["input_ids"].to( + batch.device), + output_hidden_states=hidden_state_skip_layer is not None, + ) + + negative_last_hidden_state = negative_outputs.hidden_states[-( + hidden_state_skip_layer + 1)] + crop_start = prompt_template_video.get("crop_start", -1) + negative_last_hidden_state = negative_last_hidden_state[:, + crop_start:] + batch.negative_prompt_embeds.append(negative_last_hidden_state) + + if inference_args.use_cpu_offload: + self.text_encoder.to('cpu') + torch.cuda.empty_cache() + return batch diff --git a/fastvideo/v1/pipelines/stages/t5_encoding.py b/fastvideo/v1/pipelines/stages/t5_encoding.py new file mode 100644 index 000000000..51ebec271 --- /dev/null +++ b/fastvideo/v1/pipelines/stages/t5_encoding.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Prompt encoding stages for diffusion pipelines. + +This module contains implementations of prompt encoding stages for diffusion pipelines. +""" +import torch + +from fastvideo.v1.forward_context import set_forward_context +from fastvideo.v1.inference_args import InferenceArgs +from fastvideo.v1.logger import init_logger +from fastvideo.v1.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.v1.pipelines.stages.base import PipelineStage + +logger = init_logger(__name__) + + +class T5EncodingStage(PipelineStage): + """ + Stage for encoding text prompts into embeddings for diffusion models. + + This stage handles the encoding of text prompts into the embedding space + expected by the diffusion model. + """ + + def __init__(self, text_encoder, tokenizer) -> None: + """ + Initialize the prompt encoding stage. + + Args: + enable_logging: Whether to enable logging for this stage. + is_secondary: Whether this is a secondary text encoder. + """ + super().__init__() + self.text_encoder = text_encoder + self.tokenizer = tokenizer + + def forward( + self, + batch: ForwardBatch, + inference_args: InferenceArgs, + ) -> ForwardBatch: + """ + Encode the prompt into text encoder hidden states. + + Args: + batch: The current batch information. + inference_args: The inference arguments. + + Returns: + The batch with encoded prompt embeddings. + """ + if inference_args.use_cpu_offload: + self.text_encoder = self.text_encoder.to(batch.device) + + text = batch.prompt + text_inputs = self.tokenizer( + text, + padding="max_length", + truncation=True, + max_length=512, + add_special_tokens=True, + return_attention_mask=True, + return_tensors="pt", + ).to(batch.device) + text_input_ids, mask = text_inputs.input_ids, text_inputs.attention_mask + seq_lens = mask.gt(0).sum(dim=1).long() + with set_forward_context(current_timestep=0, attn_metadata=None): + outputs = self.text_encoder( + input_ids=text_input_ids, + attention_mask=mask, + ) + assert torch.isnan(outputs).sum() == 0 + prompt_embeds = [u[:v] for u, v in zip(outputs, seq_lens)] + prompt_embeds = torch.stack([ + torch.cat([u, u.new_zeros(512 - u.size(0), u.size(1))]) + for u in prompt_embeds + ], + dim=0) + batch.prompt_embeds.append(prompt_embeds) + + if batch.do_classifier_free_guidance: + negative_text = batch.negative_prompt + negative_text_inputs = self.tokenizer( + negative_text, + padding="max_length", + truncation=True, + max_length=512, + add_special_tokens=True, + return_attention_mask=True, + return_tensors="pt", + ).to(batch.device) + text_input_ids, mask = negative_text_inputs.input_ids, negative_text_inputs.attention_mask + seq_lens = mask.gt(0).sum(dim=1).long() + with set_forward_context(current_timestep=0, attn_metadata=None): + negative_outputs = self.text_encoder( + input_ids=text_input_ids, + attention_mask=mask, + ) + assert torch.isnan(negative_outputs).sum() == 0 + neg_prompt_embeds = [ + u[:v] for u, v in zip(negative_outputs, seq_lens) + ] + neg_prompt_embeds = torch.stack([ + torch.cat([u, u.new_zeros(512 - u.size(0), u.size(1))]) + for u in neg_prompt_embeds + ], + dim=0) + batch.negative_prompt_embeds.append(neg_prompt_embeds) + + if inference_args.use_cpu_offload: + self.text_encoder.to('cpu') + torch.cuda.empty_cache() + + return batch diff --git a/fastvideo/v1/pipelines/wan/__init__.py b/fastvideo/v1/pipelines/wan/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/fastvideo/v1/pipelines/wan/wan_i2v_pipeline.py b/fastvideo/v1/pipelines/wan/wan_i2v_pipeline.py new file mode 100644 index 000000000..5ccd9f95a --- /dev/null +++ b/fastvideo/v1/pipelines/wan/wan_i2v_pipeline.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Wan video diffusion pipeline implementation. + +This module contains an implementation of the Wan video diffusion pipeline +using the modular pipeline architecture. +""" + +from fastvideo.v1.inference_args import InferenceArgs +from fastvideo.v1.logger import init_logger +from fastvideo.v1.pipelines.composed_pipeline_base import ComposedPipelineBase +from fastvideo.v1.pipelines.stages import ( + CLIPImageEncodingStage, ConditioningStage, DecodingStage, DenoisingStage, + EncodingStage, InputValidationStage, LatentPreparationStage, + T5EncodingStage, TimestepPreparationStage) + +# TODO(will): move PRECISION_TO_TYPE to better place + +logger = init_logger(__name__) + + +class WanImageToVideoPipeline(ComposedPipelineBase): + + _required_config_modules = [ + "text_encoder", "tokenizer", "vae", "transformer", "scheduler", \ + "image_encoder", "image_processor" + ] + + def create_pipeline_stages(self, inference_args: InferenceArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage(stage_name="input_validation_stage", + stage=InputValidationStage()) + + self.add_stage(stage_name="prompt_encoding_stage", + stage=T5EncodingStage( + text_encoder=self.get_module("text_encoder"), + tokenizer=self.get_module("tokenizer"), + )) + + self.add_stage(stage_name="image_encoding_stage", + stage=CLIPImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + )) + + self.add_stage(stage_name="conditioning_stage", + stage=ConditioningStage()) + + self.add_stage(stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"))) + + self.add_stage(stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + vae=self.get_module("vae"))) + + self.add_stage(stage_name="image_latent_preparation_stage", + stage=EncodingStage(vae=self.get_module("vae"))) + + self.add_stage(stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"))) + + self.add_stage(stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"))) + + def initialize_pipeline(self, inference_args: InferenceArgs): + """ + Initialize the pipeline. + """ + vae_scale_factor = self.get_module("vae").spatial_compression_ratio + inference_args.vae_scale_factor = vae_scale_factor + + num_channels_latents = self.get_module("transformer").out_channels + inference_args.num_channels_latents = num_channels_latents + + +EntryClass = WanImageToVideoPipeline diff --git a/fastvideo/v1/pipelines/wan/wan_pipeline.py b/fastvideo/v1/pipelines/wan/wan_pipeline.py new file mode 100644 index 000000000..e862588e8 --- /dev/null +++ b/fastvideo/v1/pipelines/wan/wan_pipeline.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Wan video diffusion pipeline implementation. + +This module contains an implementation of the Wan video diffusion pipeline +using the modular pipeline architecture. +""" + +from fastvideo.v1.inference_args import InferenceArgs +from fastvideo.v1.logger import init_logger +from fastvideo.v1.pipelines.composed_pipeline_base import ComposedPipelineBase +from fastvideo.v1.pipelines.stages import (ConditioningStage, DecodingStage, + DenoisingStage, InputValidationStage, + LatentPreparationStage, + T5EncodingStage, + TimestepPreparationStage) + +# TODO(will): move PRECISION_TO_TYPE to better place + +logger = init_logger(__name__) + + +class WanPipeline(ComposedPipelineBase): + + _required_config_modules = [ + "text_encoder", "tokenizer", "vae", "transformer", "scheduler" + ] + + def create_pipeline_stages(self, inference_args: InferenceArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage(stage_name="input_validation_stage", + stage=InputValidationStage()) + + self.add_stage(stage_name="prompt_encoding_stage", + stage=T5EncodingStage( + text_encoder=self.get_module("text_encoder"), + tokenizer=self.get_module("tokenizer"), + )) + + self.add_stage(stage_name="conditioning_stage", + stage=ConditioningStage()) + + self.add_stage(stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"))) + + self.add_stage(stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + vae=self.get_module("vae"))) + + self.add_stage(stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"))) + + self.add_stage(stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"))) + + def initialize_pipeline(self, inference_args: InferenceArgs): + """ + Initialize the pipeline. + """ + vae_scale_factor = self.get_module("vae").spatial_compression_ratio + inference_args.vae_scale_factor = vae_scale_factor + + num_channels_latents = self.get_module("transformer").in_channels + inference_args.num_channels_latents = num_channels_latents + + +EntryClass = WanPipeline diff --git a/fastvideo/v1/tests/conftest.py b/fastvideo/v1/tests/conftest.py index 075afa886..a106e375b 100644 --- a/fastvideo/v1/tests/conftest.py +++ b/fastvideo/v1/tests/conftest.py @@ -1,4 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 +import pytest +import torch.distributed as dist import pytest import torch diff --git a/fastvideo/v1/tests/encoders/test_clip_encoder.py b/fastvideo/v1/tests/encoders/test_clip_encoder.py index 3c1df3296..14fc23bb8 100644 --- a/fastvideo/v1/tests/encoders/test_clip_encoder.py +++ b/fastvideo/v1/tests/encoders/test_clip_encoder.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - # TODO: check if correct import os diff --git a/fastvideo/v1/tests/encoders/test_llama_encoder.py b/fastvideo/v1/tests/encoders/test_llama_encoder.py index 84a0eff4e..0a74c1473 100644 --- a/fastvideo/v1/tests/encoders/test_llama_encoder.py +++ b/fastvideo/v1/tests/encoders/test_llama_encoder.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - import os import numpy as np diff --git a/fastvideo/v1/tests/encoders/test_t5_encoder.py b/fastvideo/v1/tests/encoders/test_t5_encoder.py new file mode 100644 index 000000000..e91e05354 --- /dev/null +++ b/fastvideo/v1/tests/encoders/test_t5_encoder.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 +import os + +import numpy as np +import pytest +import torch +from transformers import AutoConfig, AutoTokenizer, UMT5EncoderModel + +from fastvideo.v1.forward_context import set_forward_context +from fastvideo.v1.logger import init_logger +from fastvideo.v1.models.loader.component_loader import TextEncoderLoader +from fastvideo.v1.utils import maybe_download_model + +logger = init_logger(__name__) + +os.environ["MASTER_ADDR"] = "localhost" +os.environ["MASTER_PORT"] = "29503" + +BASE_MODEL_PATH = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +MODEL_PATH = maybe_download_model(BASE_MODEL_PATH, + local_dir=os.path.join( + 'data', BASE_MODEL_PATH)) +TEXT_ENCODER_PATH = os.path.join(MODEL_PATH, "text_encoder") +TOKENIZER_PATH = os.path.join(MODEL_PATH, "tokenizer") + + +@pytest.mark.usefixtures("distributed_setup") +def test_t5_encoder(): + # Initialize the two model implementations + hf_config = AutoConfig.from_pretrained(TEXT_ENCODER_PATH) + print(hf_config) + + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + precision = torch.float16 + model1 = UMT5EncoderModel.from_pretrained(TEXT_ENCODER_PATH).to( + precision).to(device).eval() + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH) + + loader = TextEncoderLoader() + model2 = loader.load_model(TEXT_ENCODER_PATH, hf_config, device) + + # Convert to float16 and move to device + model2 = model2.to(precision) + model2 = model2.to(device) + model2.eval() + + # Sanity check weights between the two models + logger.info("Comparing model weights for sanity check...") + params1 = dict(model1.named_parameters()) + params2 = dict(model2.named_parameters()) + + # Check number of parameters + logger.info("Model1 has %s parameters", len(params1)) + logger.info("Model2 has %s parameters", len(params2)) + + weight_diffs = [] + # check if embed_tokens are the same + weights = ["encoder.block.{}.layer.0.layer_norm.weight", "encoder.block.{}.layer.0.SelfAttention.relative_attention_bias.weight", \ + "encoder.block.{}.layer.0.SelfAttention.o.weight", "encoder.block.{}.layer.1.DenseReluDense.wi_0.weight", "encoder.block.{}.layer.1.DenseReluDense.wi_1.weight",\ + "encoder.block.{}.layer.1.DenseReluDense.wo.weight", \ + "encoder.block.{}.layer.1.layer_norm.weight", "encoder.final_layer_norm.weight", "shared.weight"] + for idx in range(hf_config.num_hidden_layers): + for w in weights: + name1 = w.format(idx) + name2 = w.format(idx) + p1 = params1[name1] + p2 = params2[name2] + assert p1.dtype == p2.dtype + try: + logger.info("Parameter: %s vs %s", name1, name2) + max_diff = torch.max(torch.abs(p1 - p2)).item() + mean_diff = torch.mean(torch.abs(p1 - p2)).item() + weight_diffs.append((name1, name2, max_diff, mean_diff)) + logger.info(" Max diff: %s, Mean diff: %s", max_diff, + mean_diff) + except Exception as e: + logger.info("Error comparing %s and %s: %s", name1, name2, e) + + # Test with some sample prompts + prompts = [ + "Once upon a time", "The quick brown fox jumps over", + "In a galaxy far, far away" + ] + + logger.info("Testing T5 encoder with sample prompts") + + with torch.no_grad(): + for prompt in prompts: + logger.info("Testing prompt: %s", prompt) + + # Tokenize the prompt + tokens = tokenizer(prompt, + padding="max_length", + max_length=512, + truncation=True, + return_tensors="pt").to(device) + + # Get outputs from HuggingFace implementation + # filter out padding input_ids + # tokens.input_ids = tokens.input_ids[tokens.attention_mask==1] + # tokens.attention_mask = tokens.attention_mask[tokens.attention_mask==1] + outputs1 = model1(input_ids=tokens.input_ids, + attention_mask=tokens.attention_mask, + output_hidden_states=True).last_hidden_state + print("--------------------------------") + logger.info("Testing model2") + + # Get outputs from our implementation + with set_forward_context(current_timestep=0, attn_metadata=None): + outputs2 = model2( + input_ids=tokens.input_ids, + attention_mask=tokens.attention_mask, + ) + + # Compare last hidden states + last_hidden_state1 = outputs1[tokens.attention_mask == 1] + last_hidden_state2 = outputs2[tokens.attention_mask == 1] + + assert last_hidden_state1.shape == last_hidden_state2.shape, \ + f"Hidden state shapes don't match: {last_hidden_state1.shape} vs {last_hidden_state2.shape}" + + max_diff_hidden = torch.max( + torch.abs(last_hidden_state1 - last_hidden_state2)) + mean_diff_hidden = torch.mean( + torch.abs(last_hidden_state1 - last_hidden_state2)) + + logger.info("Maximum difference in last hidden states: %s", + max_diff_hidden.item()) + logger.info("Mean difference in last hidden states: %s", + mean_diff_hidden.item()) + + # Check if outputs are similar (allowing for small numerical differences) + assert mean_diff_hidden < 1e-4, \ + f"Hidden states differ significantly: mean diff = {mean_diff_hidden.item()}" + assert max_diff_hidden < 1e-4, \ + f"Hidden states differ significantly: max diff = {max_diff_hidden.item()}" \ No newline at end of file diff --git a/fastvideo/v1/tests/ssim/README.md b/fastvideo/v1/tests/ssim/README.md index f62ebb836..e32b2908c 100644 --- a/fastvideo/v1/tests/ssim/README.md +++ b/fastvideo/v1/tests/ssim/README.md @@ -1,7 +1,9 @@ The reference videos in the `reference_videos` directory are used as part of an e2e test to ensure consistency in video generation quality across code changes. `test_inference_similarity.py` compares newly generated videos against these references using Structural Similarity Index (SSIM) metrics to detect any regressions in visual quality across code changes. -`reference_videos/FLASH_ATTN/` videos were generated on commit `66107fd5b8469fed25972feb632cd48887dac451`. -`reference_videos/TORCH_SDPA/` videos were generated on commit `4ea008b8a16d7f5678a44b187ebdd7d9d0416ff1`. +`reference_videos/FastHunyuan-diffusers/FLASH_ATTN/` videos were generated on commit `66107fd5b8469fed25972feb632cd48887dac451`. +`reference_videos/FastHunyuan-diffusers/TORCH_SDPA/` videos were generated on commit `4ea008b8a16d7f5678a44b187ebdd7d9d0416ff1`. +`reference_videos/Wan2.1-T2V-1.3B-Diffusers` videos were generated on commit `d085770a70988c7b26632a0c3123c24a57f7ca77`. +`reference_videos/Wan2.1-I2V-14B-480P-Diffusers` videos were generated on commit `d085770a70988c7b26632a0c3123c24a57f7ca77`. ## Generation Details @@ -9,7 +11,7 @@ The reference videos in the `reference_videos` directory are used as part of an ## Generation Parameters -{ +FastHunyuan-diffusers: { "num_gpus": 2, "model_path": "data/FastHunyuan-diffusers", "height": 720, @@ -26,8 +28,51 @@ The reference videos in the `reference_videos` directory are used as part of an "fps": 24 } -### Prompts +Wan2.1-T2V-1.3B-Diffusers: { +"num_gpus": 2, +"model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", +"height": 480, +"width": 832, +"num_frames": 45, +"num_inference_steps": 20, +"guidance_scale": 3, +"embedded_cfg_scale": 6, +"flow_shift": 7.0, +"seed": 1024, +"sp_size": 2, +"tp_size": 2, +"vae_sp": True, +"fps": 24, +"neg_prompt": "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards", +"text-encoder-precision": "fp32" +} -1. Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of a bustling street food market. The scene captures a mix of humor and authenticity. Mid-shot framing, vibrant lighting." +Wan2.1-I2V-14B-480P-Diffusers: { +"num_gpus": 2, +"model_path": "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", +"height": 480, +"width": 832, +"num_frames": 45, +"num_inference_steps": 6, +"guidance_scale": 5.0, +"embedded_cfg_scale": 6, +"flow_shift": 7.0, +"seed": 1024, +"sp_size": 2, +"tp_size": 2, +"vae_sp": True, +"fps": 24, +"neg_prompt": "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards", +"text-encoder-precision": "fp32" +} + +### Text-to-Video Prompts + +1. "Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of a bustling street food market. The scene captures a mix of humor and authenticity. Mid-shot framing, vibrant lighting." 2. "A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscape stretches endlessly beneath, its earthy tones blending into the soft blues of the sky. The scene captures the spirit of exploration and human resilience. High angle, dynamic framing, with soft natural lighting emphasizing the grandeur of nature." + +### Image-to-Video Prompts + +1. "An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space realised in the background. High quality, ultrarealistic detail and breath-taking movie-like camera shot." + Image path: "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" \ No newline at end of file diff --git a/fastvideo/v1/tests/ssim/compute_ssim.py b/fastvideo/v1/tests/ssim/compute_ssim.py index 943adf7f2..4910363e6 100644 --- a/fastvideo/v1/tests/ssim/compute_ssim.py +++ b/fastvideo/v1/tests/ssim/compute_ssim.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - import argparse import os diff --git a/fastvideo/v1/tests/ssim/reference_videos/FLASH_ATTN/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 b/fastvideo/v1/tests/ssim/reference_videos/FastHunyuan-diffusers/FLASH_ATTN/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 similarity index 100% rename from fastvideo/v1/tests/ssim/reference_videos/FLASH_ATTN/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 rename to fastvideo/v1/tests/ssim/reference_videos/FastHunyuan-diffusers/FLASH_ATTN/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 diff --git a/fastvideo/v1/tests/ssim/reference_videos/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 b/fastvideo/v1/tests/ssim/reference_videos/FastHunyuan-diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 similarity index 100% rename from fastvideo/v1/tests/ssim/reference_videos/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 rename to fastvideo/v1/tests/ssim/reference_videos/FastHunyuan-diffusers/FLASH_ATTN/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 diff --git a/fastvideo/v1/tests/ssim/reference_videos/TORCH_SDPA/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 b/fastvideo/v1/tests/ssim/reference_videos/FastHunyuan-diffusers/TORCH_SDPA/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 similarity index 100% rename from fastvideo/v1/tests/ssim/reference_videos/TORCH_SDPA/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 rename to fastvideo/v1/tests/ssim/reference_videos/FastHunyuan-diffusers/TORCH_SDPA/A lone hiker stands atop a towering cliff, silhouetted against the vast horizon. The rugged landscap.mp4 diff --git a/fastvideo/v1/tests/ssim/reference_videos/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 b/fastvideo/v1/tests/ssim/reference_videos/FastHunyuan-diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 similarity index 100% rename from fastvideo/v1/tests/ssim/reference_videos/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 rename to fastvideo/v1/tests/ssim/reference_videos/FastHunyuan-diffusers/TORCH_SDPA/Will Smith casually eats noodles, his relaxed demeanor contrasting with the energetic background of .mp4 diff --git a/fastvideo/v1/tests/ssim/reference_videos/Wan2.1-I2V-14B-480P-Diffusers/FLASH_ATTN/An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space reali.mp4 b/fastvideo/v1/tests/ssim/reference_videos/Wan2.1-I2V-14B-480P-Diffusers/FLASH_ATTN/An astronaut hatching from an egg, on the surface of the moon, the darkness and depth of space reali.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9cdddc1fe73a7243a82ca4d6d11a42ae296ed39c GIT binary patch literal 205123 zcmX_nb9`n$&~I(qeQMiwx5l>Jnon)p?bh1bwr$((c5BIcm zL4JKztV^799j-{VB-1YgR>&@|+*vu<0So}3oudhWg@Xs+$jZaR0$}0dVlm?2_!dYp zegl}~l_Y*JumePXiG3rDO@QA5F*|z?YZFsv04p;y8v`pd3(q&x+}YWlmx;;E&5hC3 z!q~*l+R&EK&e4qNzg`&4oo%eYF?ROO7IwByya1q~ks(lk1>k66D!>LXHZiic16m2N z@G|o<0}O2qtv#Gf1eo2~c$wW1}VDIGdt@>T+I~ohHFfxBD zeHQ>53wIM^{r@bod~4`C8rqtf2(WMhfaZ>NHir7&sw@C!M-yvn3#V_2+nw7O==@Cp z9c%=czg;ji_O!D#5nyGZXJG-D8ag@a+dElV*#C$4Ujq*I`gW$KPA1L*46FcWbH{Iu zZ$lgaYdbqDL-X&J{{LgL0-UTZfZxviKL#_v*71Kj0b1A?I{)X0g{`xRqqX5T=$mh3 z?c!+Yp%1jPu{U)7ZUet_`iR-&Ft*IrT@#c|5mj!@%Xkaz{d7W z`(Ie!4FOg*MrMG+f58YaGje{D_Wv3FpW4t}fRpE&;pA*$FTf74u>T&V?-}tuh~Ksh z9lj^Pf5Qpl4+3IAX%ZF;`t`}{qeSC_%T)g9HCc~$Ze8aK^4lho&&TQ~nR(V{)Ru-6 zS{eD>cDM3H_}`{|Tqd0xb3l^YuvfXttMAZG*1Y)Nxk z`p%8!X4Z}B@)-_Xp1;^&GeO0@m>B6zm@)+b)3X@S2wLE3Gb0%IyUwr2Q5Hj9oUZE7 z5Kw)AY=|6!#E~J_y0rT$Fc)xb(^geuhVPP%Nej)AA~$dE5AsS69G$oRS!Qdp1HSU; z7|HRgsCQE3<}L+f*F)RIx8{>R_9jXx$~5(+z3yGp<{gI>>M!e+REAfgW{qs-T<9uq zq=6k@sa|O_m=zl%v=#3Wb(n?*x=I8aB;z%YPN{4H_*o=%YW@U(kCjLLrp@o5b-V=+fQWvub}7U z0W-m_WmkVG#7dgf6kSuHc~DpX8mr4hFN!N{#Dh(VcyK7=k)_p zFJ5H**qX>)l|YN~yYnY|C10(PHyjJMwE@d|(R-13Q?8MlY$2kyvKPZ6+{@?-UAI7) zEe6~ZH8q95+T|wXtwSU zMpKAr6Le=_NW$?nARn$Oug0G2_Dh4NNDZpId01J_s-RgdWK8oCe&*|EcPYN6M}X@b zB|gDA?nG2`$6xYiU))cT*^qt?+A3Wbc7He7a!=DHcImfBS!BSMQwI&A)f0skb2UxS zk$$y2k1+{e_=Uh;5K~Lr&drknBiDc(TN1}*cds6Bf6Rz#YC{x2^s=-XF(^oyKv@21 z-Al1dz>Aif6ldGAl-f)EMvUJVd6T;Ch%P*AyGpq@ko(IQkmnDtLPCtnXm=K7_wf#& zB9g^%d?M}+{D~=_tTY5;BqS*1BMl!|IuBh|6YR;Y7tIxhp8d)l)J^;up}kNW=HWkd z;4k~xrj^J?){ZUyGKtP=Ptoft6-Ck!gT6+!J+EI193|^oym)3vMJ8e>4zP4$OhZg< z856k^<)n7gN}BqzrN)rn7&TOjFwvGn(8=~Sk&ZNviE~1x>*jPH_DRm{N%rg}?H|KZp>aHnKN^W!b}Tb6=1LcBb>SjciOTA z4aK)KmT+VjygT|NUkANNF8^9vVsW0CR94ILZV; z0eMys<7T||;zGm?UIl6(={a8e9tHqaoEa9LsK6{+=|*MgMS?`^PWe5OW4TZO4~@_kx6Fb0^`M;g!8AFMT5 zFg`!Pxj;(u4{%BEiLCN2pV=W0nO62xn{Qk|Anix((OF=?vU-NX=t01W9c9=rL#JNLn3HPT_bvs^CB+CTL7Uz4w@fVrGxo?%XUrOhGd zK&%E?a_|E1Azv~JOMwB%!_FD4Jw!d_y|nm$>*^@}Bl~F%eViFY@AM^u^|(Yz8}15C z+0M@R55bMgiax(KWzh#bihkFM=)&)(ENOt!&SsSe6Av%9NuD6ligLUT_69OfUSF@; z1;p!LB_RnJz(RxF7|B|l|LzB7>>6A+Z2Ra+Jrj9N|7~R1CJaZsjmI!J;%T?zpos6t zSWqlW5cdKS+DAXdUZ6Bw5`$nIcafHIjBz=-gg`rpYzk81Evf7i`5t$9(+Ph&J1k{| z^7B*5TJVI0eguK_b1zXxT^yR)t!@!w(=J+a^C(Z^OyBwXB~zZ7sIbRI%38AXB2%FF ziOU=f5Z~95Sv?Z#jugW-1fy5CzNi+(yiIRjkOU}kOmkRb=weyVd_Bk>Ic4XJVsSi9 z0cKK=Fl&-&%We|l3wN`A8Xogp0mDU&d^n0U7p!ex<-fP2HAgw?nzO}X_0n&LF0AW! z6yK7}&QLuZF8guIrQiVgn~;Rd??^|NhH6o?S-`3bf9+?S5r)|W*(SoT;kj}X6@ zQ2$_-Y|E4pwEZE5o$%%vXY{j)pQW%&{`X<=%9L$<M_*eG(yR9cIhihP^8F>956qYbmbe>6 zmK=RN4>*qfhnfEBKmqDQ7#^;ern+&hNrH)!PmB=Eyy88zv9`N3x(Q2scc%@JyiXz~ z9YRq$eh;GDI?uAOV{55xJd;S!;wsw_j2ds{Fy{3><;Uc=($Pf4sK;!a;f{ifwqur+0}1NU5o2o%W`|8-KJi=xLIGtbGO~+K z2zLVZ^nQC2{xkanWh+N)DcVk_SztZ5YXbR748|w-M`mMtE}hDRkSWv-*GM^ja!^?x zaB32oGQ`;!(uUJZJf<%awttntwRbFPfYUIf_A2Hw6%|PaErEh?+A;*P(Aq>1@FGW| z&5880ZyoK^evK!|1~!~YB4ab9s-uB+9hvK*h;~#u5Ad6%|GJB2!G1?fomjhtPM~C zGuGniswb_kNY!_a-lEATU%z*u!YtZac5NuHr?=5``e&&?*keq<8|hwdR3Zkxurkbm z+Nuy<4Q$Dy(>RO&UJ2z>vG=XUpHr|DN$G404lPFscl^cnWp3!EH3apo+VZTpHZ$ZC z(#6>D0l#AqlYR|e?04!vp-pLsy!>kyvhnDb>#rA4 z*fwJQYQmK4_w*h1sUZ^OiRWpJ%mtWCgCw9o)=!TLPt-PN8u9;*)MfShK8=j8Dpo+@ z-j2J>a_5Pl{NbePpK2&s)}9~_$)6igr7aj<$fg+^+TnRIm+RQ7?{I7EhuN~Au8YT};RUvdQ=4~X;SjtMMG#c2y~vSd-fKgCk7UqiB{ zvy1WwzpVv_%ZuTn)DTXf1^ZO=0pb_C5}8@fj!i`pIAhs^MB)xPQuW_zx% zG9LPn#P^Q3+Gv7UXlooH{?S+cT^@9gh;n0rUi4`c7fsn!*E`)LkQN;ayKYA?HJqFY2maDSah9HR-i_X2zn+U5I zj1rykmw2VJK2LiB-kCO(_M?qV_-`P_#=usrfN*}B!EYJyD3s1x*Nl>oYt!W?eh{+r5W5}E932seBf|YNNTt=n)sS~=P>gc zsz5T4vV~aW!VNGTkZ2DU8pxRR!!|1e+a>xj3X3WuUN8tIfto#dN?vZ&D-no? zxXz#5ZxxP_b?J4Cg?2q-3fzAt8}mp=>< zw5#D^FC}X!3o5i7+iwCQQDhdE(U(ib&f0UQeFQV-FkG&26t>sU`Hy(2YS?iMDX=psra#Z{kE|WxGmJA_Xm;e zd&~~~n4{52xu=USOHG3ypAUvy_6$E5a2xg>Ssl-8ngaQ>f22$)d96L|W_E`Nqlhy= zEA7Vyxd#B0MSDG6rOGT{rQ4e?LDsr?Ro7GQVoiTS<-f+y~zay(N7v>WQUu3z3A$&J6a=jlB zb!W^hnyFlPyEcWUv*7+A5N)^pQ73T-j{cMq;h*FezhCHm=L$E>8ICnaoCFkPy=3%4 zpqtuoJ)E9Att0*OAu}Ogv}SjV{Epj30LjHpk%(j;!Z!4X>%`NLJW~^B-|Vb) z=ekozigVB#e!OwgA*SycgPb@`Nas@Fe*6eMfCO@~$s?4^&PUcT#+y}6>6u1%bEre2 z@ImVz2n^A0ZNO}KwT)HZDM!~C;-Da^=bOMTD`>L*Q(_$sLc=@m|D#8y)Np-fa%!|| z`1qt zd-A-`1L&M(3i7IGsQoocn61L&B7-+KzQq#-F?MfiC?UZQ;+eX!VYF1*A_8Y((Ezi` zks^{|t9kDms2uTT=O|dvnQ4&Wgsf5=Sj5}_%s`#)zfJei~DjeqO;c^bFvv&L$j(=8*dv|4bHPz;FFd3T6M z#EQ?aFKmS}18(-j=O3NkUHTo^RusIe1-?@yw(3GXLs`mYy>7C$xfQ8{f2!qT&cGyr6irufk(2if__6R4;W#1s7F8IMp_L7Hf>}1mjDD*6yti9^p?yUqo_)w>` zNarttm0cT=0HDQiG_lOvb+%&w*I|(^Ty>BB-G_GYf*|?m*8(B$__XZd_1@Ra6+avM zRp;W4yY12-)jVQ)>31#g|DteFVU-`O0;gczPfa4S;>XvvGS_Pg-HPSdV3hiQGQE4o z77+c(J20SX;y7mi=xja}ptV4}wy!Fkzayrhlr2QMZ_jl!LaZ3mrQn97w~CKjN1}M> ztS*VxYPKv7YRsh{Zs8v~BpXmfu9Cn(^5fM| zj(k<^p1zDm3;k-q^&~wzm0`Xv3nw6-M|VeKB>zr-Zc;!{ z^^|~P@)3pl@7>55$O_7;JMXVQNe!`gvvN;4yAi`VkSIPX8Do63dVO~dg2o|rw631_ zu^s6o=AtuP%+PkXHE7^`LCre9J;-rtyvdy^DP&VAZbmNe!hq=rmr43wI;iY{K~onr zAcYJhsYq*Dk&Gz%Wo^ew&pWY%yO?IM)rf<&NIdlD1tYylUq*#>>Z0JpH2kKu{~J2N z>`|yi5v;%=_{PLjko>ZEugVq#1I7V#2VD) zQTB;AIT=8g;uotpcL7Y)Lp=@%*fvv9BC^c^m*w8(E2O7;s5kn6c7K)DNa<3V$kqh6 z)1c&TllKY6)18$U0h8RM1>z}u-H|~5)64bz^;iPwilh*AxR^2~UA83_x(f%(9NtbO zKrdZ_g(N46s3T&gmnr1T5oLgcZZ=ip6a~{R)jW$kI7SXdHmg&`HE$=IrE%@pM`$S{ z#cOQ>?_6U2^`8QG27a+@JZshq`lCWU-;Q9=1S0p$4Mlfsty*!<2f!7cbH9HcekP6; zYZxk_7!x1$TQ`LD=hzI%p2LyXO)~*kwt6nHXQ~*Vg&6CBHKqeb16J)*7nRH>rqf#| zGQG4^acrGf|LTX5^zl=i6%+}@%Vv&35B+kuGUYWsY%|j~uEg>psi|pxm&etlzSI_k zR?@|3EwZz4tJ-wu-%6~Uf@5Cj*p;*?mM_?PCzZN^putN)KL2TK*?>~pt zO(6@RTegF=1xg0F#biy+aH3UFn-}O~F28*8zb>OVcaS=b_{ZcG?cxQlGXczlIKjZ+ zC=3QN&%*VL8=(rL?uI?|G|+sDmBb0cza63P9(Kyjiia^6W5biUw_P)@I=9S3TiWsF?ZbjHQx^gdYRmeqenET*8XLO^ zSx_(?hciuU*XuURN0^a>2|vpNRjZO9>$1+08Tp^r#tl!Di9Z}}NNuME?r^{}UKkIg zKosV}vRbk$B*F|+GvDxNaRTF5nC1$c)mn;kItQ>~ZD?x03~tS!7=>eJ?6z6Kop0>} z-4YMsiV3s0+01ikCGgrYI-5N>Sqy*wD{MW9KzyK}6dkMRvTg1P@Ef6E91LX(i0iN` zrtS-8bMTC~XmnfKsT_v6eDK5Y9nq6qEx>3#9%Cc6b%mtf?{fA*P{K>SM=$g_IHF#Z z^MgZevr{_;$W7o}a3oEDqKmyR7I>Lfpf|%_kFDK)%}0{AX3OxOpZJB6eSYsw#sf# zHD{g7uClU?RHKZDT zxK}VUDJN3y{W1=ry@h zYj~&P25PB@?r|gP%kSAXU|t$gOW}>IpPMab(td&NGR8!lL>mQ1 zFdse_F3@c6IK&U^vtfZrSK5Ty;ioKG-)#((T-2meZE_GS2M|vq*ZLZbF%-Zf|lDT)JKu+1MXyaL+OtNb7j7T)hC_Lf@_c#j2D z!?EF(^}HRtPA{Bne;S`{IcxGPLyLtw?@m&-O-M)VDKm37igvRc}}c7=(i4qv!aTv*~>V4aK7)t<`%pE z>&0zelB>P3AJ? zAxHQq*c`J%N!m+c{s9i{FUAbWbxbJ<*eOnbD3{0_Le&+UFKd9+|X+9$avFDCkE`=rakb zeq8xl6oV-VpC+bGOm83*a*-c8o+0xjsN}}6bZkAFxdl1+hh~vAXvXn##V%=oXs-(* z9&ll!&LvTPVF0tgLfXx>unj)t?&J(PspZ$^F@;tIiNOSpti~?SX&=|DmRgycI6U~K zqJ&^M6V(K8TRw;s{{*ynZ%q5iFysk?!Q(B@=z8m@`C~7xK%0%Zm@1d>OPPlu$BE+x zGLOo^`F-vpTh}1E(;wNJ=?+;{SI z@C9#};UC)3X$CkUe(WW1fofoR;`N1^nz;hPgXt=QHx*SCUFYZE`F-z8m%sC!!ceo8 zHpFVSuKsb3Mr|B*jYYY7+TJ)@%aZ@q(@n|uqcxFD_YDU=v8Wy3WV7&gCw3plBcoBZ z`Gbjv|p3s&NFk1`p}H%}jv!3Os;qM#xZK#VHT5k=2nI5ho=MSrNhH#RT^JWD>a5|0z~z>B4R(3{*!6cwId%X=BRp-@4?n<{>O~!;FjOt$PhEU z?-kvp`$hIk9ESuW&r*Y%Wc4-@=}vD>xv7?5C{8~yFRfU&IY&|>IZ>`dEH?z7;vUp} zK)wTI3O0%`h8b??#aYhC?9UbWY@RCh+E<73=VP^4DYMCydnK{84N5wm)O~G&Uw7jX zbeJR~FH$iGi*s7f1wapH#2{-dde}d=w%Cn!;DV$&7Os!>np!FE%3@w=Fx!G^=_a$& z%8sN`LPvpfRh*bWpUEQJ%fvc%kJ=J6OxC5w@vZs{ELS4T5NOR-2f;%8 zMHbl{Z5>&MyrSK&53l04JWI!sFxnwY`;~)eb2^0xIzjh!MY0$89d1G)Kevl=|B1?% z`0{_1z6`k;S_#rO^4>UqIcl>_pVS6~MY7mI2L2fr%%On-tBe9;K4IdIN}de^D#Ye* zN-$pdkgSj;p*7cCzYz=W?_$d2XrPCkr%h3TlT(4#3WGg(DAFSau^TD!I za_g#^URu@@%y>0>SEs6dYQyJ+U4McJ?_f>5!2181*-p67hYy*kMJ0V(;SV^?osa(% z!#QETI5hOoMZRQ{#U3?M=xv^vw3iyYgn*mWexq|A5yV*X%9R*(_WM%yi*$J;4YT;Z zYY~gNnz)6h^L1Yw?>ccd)$b*VMr*m|IF}gh8tV!-Qp`*qNTx1r(#}Q=-jlbWpj`H5B_S`36 z*=k=Z>Yrczu~SlZEVRP~ambFaw>4S|bd*n`IKKoKnsS}w;drC_DQq45`0?!7q?Z4V z&5w)eLifG{x1LO(5&}Y?ZsnC|PR>I2w%|S$;!%5b&LjeydnC_8+_%Qa5eTU8f*f-; zi~qrR<6A4zjmcU>!z74U&ntA81;BxQ+vmx}e3fM5k8Auf)7361NNy?%-_&zF=11+{VQ9%cTonokq66f*}U z&5C{#ze|^j5{3cs=Src6Ewr4-hOl^e7rr@?(^u~kE#V|JyBruPC}5*YMERsjdr!z8 z{5R$0yP>=b3Uzt;P?2|}r!@Ly4N%JS(v})s{(zz zKPEqewy8}FAUOuZE>XeEm&}l0yI(ZF^w)=bQBmfE;9~=7x5<&EYYFPi-@gm zCej_-+A2EB@uyC%=JK#*uAouff3r~+&@k9iPrqT5HZ81rp7DMZ(2zjTh1HEh^~Hff zG3``>{Tmt{xzO}41GOj16rs4{pnx}|^J_4rDW9KSl|g!EzMI%Sy&oUI@U)m<2k24% z`g%`jlw1fr0RxFez;4ceOOYN)an~nS$)P7S)l7kc`ttgR5wBa1xVz7D+*D5QNX2SI zbG=`W51^rcq$%zHLe$YjOjp0D0cqD)I(tY48&`~(>MIxapLH|zId%h8Ii%8OG1Vd3+u)Z z+JZzFpVTIuVk}X{Eiq7JJm_9ipr@`Kax%MlwCS5TGujrbHft5P1Q>$0mSkW<&4?a& z^_(%kF2y&4k|`*28~VUZ<-@5+OG0s2U6nR?R|A`R!@2$Q;Vyx2713eolEP$hefKe zE7*SD{@{G%*m+X6VOUqf$~|J$(4P$E&EKujF}QK*Z^rvq)0&kM;VR4$V**Pq;Nlm! zn1~g&l6(T(s-kibtj3NIS3ycoknTzbN0^ zdGg_wF558L@02g!&|>6kxqVQaYAdNy7*1CP82JW5n`^MWfF>quZnnrSe<7itFcDb^ zO*GJViKn>!_;X-lbP<_b_CyO`sF@+Pgm1^5%~RN!Y9(B*X5`l72r|* z(mlPkOZ?u(QVjPx@gh#mBKHxrR3~07L>k`?& z-j8!JpN@AmTe84L#507G1yPH34 z&vC_0$AU{n7UdH=NncPTsW4GaCC@h_tX@iH!++zsou!>6sN|O<_FtFGUh=US6R>e! z1;Y`BRqSzsX0M^tGn$i^8?Y)wc3nk)-bVGYl*=1WG{x!9UX@K$3(Y=$O~a_ij`Xfb zsodY;X@-qism=;*l+SC;!1DlM$~^=OckZ+50j%_TnO^u*sQ8&1q&rXEGD?OB3aeEC zN5ac(F8i*a)B6FD{RPlXMo@233Z3c|jV}{B3b|#L^<22!$3qdq(DX&QHiBXByC6jd zn^j7vaI|6**;%Gt!31u}f09fyek+Px7bk;QPy1ShK{@-4*8^sj5DCjBeB+47Zo4;5Raou|~KiM^!S;CkqJa zS;c)ZfY5J-Fic6`VIk|(r><$q&hJTF44=RFsZ)@nENzVIcHwn?2OkVRbN&xMx!iIy z*(&t{qOUj#dsl=6sL2!uDP+V$#D^2Ed-szsj63g(57)4RBlS_cVlTE#`^*nRX2$g27U&k{QYL#f@AsnH1IaaTdztNNhapv*>7&4DNZ_Q=Py=H; zd#qEtIH_bc2R-8ml{04s@HtCW z@64Kece|CiUPENJHn*u|JIc|+FH}n{=T#Cqd(S{XRO4c-$Pia_F>+|t5BX?rFTVyY zUpv(q5IR~ijTyu2TvjvI))agupgUY>hOl;OCz|6}OYvc@QJQqtqvJwNBDH{#`;+^k zfYIEiqlr$VX>hT7?nSZcfM65Qn46 zYlO9Gd(OdBX#RsEiOI*Zyt}D+{b9(bF+w7i*;#oCq_Z0G*4ry zGR^cqqmA$FNIlAyTG==s=$jjHyG6{ROx0)zFR{ssWyy2>B>=DG{`IeRMkcdBV{*n3 zkr~A={@G>qq45?Nw|g$8zNo@aA`>~3t#H@$^WP=i{+dcR27xUjpRMP{6rd2>SeTdM zrlcA7?4b&O-w1dRGyhmsVhu7!IHidH0GY0{R#~Rs=#?7#ts)~zTEL;Y+oc;@3?hUc zFSe&l4<3Ip!$GHLqEWLc$R5qxoN3F7mW8{W8MN!~(DDerpFU&n>Ji`bEAjUTZzZxQ z<||he4{3VK#R+(n(wXuefosB`^ z@5=B$@RVhn7yysUiQo_iEXltTIGDH7+N1($^+lrz1dfQ2J3(LP-?#G!%!c0drQ7nr zC|hF}+#H0~^&ChY1-k9M>L;?UXuOG+vVIhBy70*FJ4E{-VZEYaEtHX;9(I|nUI<&$ zsrGnd`LGS8G`vOr%|!b1lSyaT1<0w|aavlT z-I8Sg7*3qo@od2?WMy?xMdG|W`5(Sl_qkha4o8`{w7Az2rz;U1n&H=FFq*xK@|t8= zitW6cSX_S?)+AM}>3_$*-AJWnwD9x37txkR@9m5=rnWFKO>1qa^&ZYJCuO%t zAVN7+o6>HlhBu;>ai}7vH?ggZSGCkS844M2>^S%$JaHwSCHVgPpvT@JAm+MR#Mz<* z-RMA;5*skUO8F~bcrVUNk7VFfZ^|mb(%f{d=U$eOG#vTw7Z3|UlZ?gt_JH?8_ zp6g#oM|VG0pj1sMukht~jEKFyFAnq1mm6l_+CGRdaB??~4_JastL~A^DQ*N5^BOqo zTmALfxQ2PGK}kWZ*-g?MGse1x+hnvypgF9}E+DGCBu zQ3^NHbs(MoYoHiUHwU@3?V$J5pXIy$)-5lE%=^>ctx&NuvGY?PVYmJ-R#dxJp-c45 z9RE_6qfb!EklS-JPnp53wt3d-T_`$O3)&h#9V$Yw0AW<{_V*lm25b;^b;=(o*ZG&6 zy4eix>wDvB^aI#7?p4sQ(9%m*Cu8x+xwpJ!l~@ck1?&DBX}|9?Y&GFMVNs3H6C}=f z78+ALd?srZN%7Vp>heGmGnaPKh?~Y?yYl4waU`EW=mJoKMp9A2ReW46YZ`fnfU(l9 z;j1Y)o0P!D{yI3#8~C&$vd_!6(LDTd0klHZ!}gDj)x{t`O^M)XIjErtpY)KWxcw2D zs?{zG)1`{@9~r5X>n>^7+@sSkrBi)Ohg*KAx`4wNa;LM5nY@7XH$TKFSE}!iocAFD zuS`6?p#-W`CySHzKv5670WR5jbOFd>t?@jJ?g^>Em@v@`$jZqCiw-NO%8WdPThawq z7~4aQm*4ACQ((^33PO+5>wFBXzc6g3oAlwUvg-x0T`W93SI&Z^2=e7Q`=!m7qP z%ECnq!mU%PxDt>N=F$YeHZG$7Z||nU*F=Az7G=QT(0lzT%B_jGIQV0Jc0R>%@001xk0WeKn|?cjVZuP=Y~s z?sfwcOxib@=zF3Hn(>28PZCsgR>cJt96$8DZ?|)oKu}$De;_Bgg&W0a3@H~ul5z^7 zd?>>5%RA&g$ih8)^(UV1Z%GnA)Dukjsf|_RO@=VXHNv1bIU~wWXH*_O@McO+)tNlm z(VrELJX^63@N!~zT9|&zqM#`cDAaf)^M3}VMVi3^SO7@{6^B4i6jd*`%e`o?jit9n zn{@ErUVla&V?&LZ2>!AarXtZSay=qYZ4UBs z4pFAMkp6|XJJ!O%${Ixz{9*u*pZ7P_oZ4t>`71nOrEOxuMycwj;{TNbmh#&Q``MXz zovoKPQ|kz13{C2Hm^)^TImyH}r<|D=k+5+V8q;{%VMlJK21uEZM~Xkzlu?vyv$kMWXExaGm1&`3I`?WB0;^$@Q)9tb!)F8pyKN{B71;bfI@5mDM$O4k(P>XJh z6Yd}*&H^1Tj`MHFGR7;KZu%;~eoLbg*`w}tGgrp0f9e=GI66|8(Rd1qAr(S4c1z(_ zwvATX!TAx{pA1u}5}{3!P`EFScWKt9J15gH&#rD1ORgAt`$&n#ilUD8(h)=8S`0l{ zhIKS$o-I4dOpA()%LIY|Y~uX<{<$`h=hxl>jda|#BEk`Ae+0CTnF;%Cc`<9y{0^(3+JNx)+-u|Gp?h+u>FB%jEde_qm(6~r6&%|lRj(A1P_oOhD z{%a<>-~|0)g{cDS#NDJUzo5&n1eqRuH;>ED7r-tv{*j2&3d5vwY(0jNTiVXRFUdRV9%?Y^uSewl!wgJOkE=uF5FjuVRshn{vB zWm{3key#sev)xAE{>nS`nak-=D8JuCjKw}RJO5uG-|M(DZ3WrR|zDd zdGK@mU9wYYR??-$SxBG2A6`u(T$wMEJku#!)r6!`4?5I3Y?k=Gnhi|BK|oL{aMn_<7& zYSnDT9LBIMo|AR`VeQ?Scu{%kH6oVly;EsTCVz1pCv`yArJ0y=jup~*Nw}egt5jWm zUe}G&mTSaVODarH?SyRVw6;-{4RNOCn~YBv;X!wNU)2z1RIabzgU$q($t08}G6u_I zLo66FD@`oT5NgJz5^R`ES6hrIb1%JR5+C*b%a~Lh(x8vyG3dwi#=UhCjqp>-nO;rR zpV%UK$a|nsqh->+=C3M~LOICAm(P4pFRADAt)Af4qtZpU{N4VWb06ske&F)4mmcSE zl>aCQm01Ihp40v6D+tPXZoBC|QQ;m^>hS^TIK%QEe4!e`j|s&;rDzd@k-RO}TM+{+ z@XTAjN0FqB3DdfGC8H5x+_WJ*c5d5^CI}#(vpcOF>Mm{94=GcDCFAci`9{6OZTYDuY~2PLq|L*e6^+=0(egyw@gc8K zxGR`KL?*}@P49DQ$X)X?sGr}g42br8Ky6+DVvW>UPk-&i-;rh<5)qD%o0ldfc3Qt# zx_0h&w0(b$#qF}cc zi#oVk!Y8G|)2^m6h_GI|AkIJ@aQ0TVRtNx85@imLyg zS&1)KL!fng3X4eCzLD92UjRV1O&eLz4*Qc0h8xgq^tHAn9k6lRso>&17Y!hISb_{? zlM}p4mpa&0mx#Q$z+jyF-~i*l=3#U=Pgg1|B_sP!I(*~Sk6&>+uJMFqwd7@8X@tn< zK3eR$n1!<<^wTKWT$sDu$D(0%yz&;Okg$qW%Vqz87dtlq&F_DM>&8~69B*EA-piCx zUuQ{t6=>yoJc(|rT2DhT>B2z*dwrtHmh3(V7&Y1?UA!SyY4)MSILAfE^GOHM`5(Qm zC(|~~+N(rc?P!;Du&if=~$sh^y0rA&6P<&|b<69yfmt1GPtgj_oLpF@e5Huas zvOAS~pTIOL-W)0BH8yY zt8t!(B@$fUONTk~8W2Adph>7#n!BQPmm!ubR632ndGH0eGr?~Z%7SQYO1PthA@?sf zW;4hk2ASB@NFTKl#Y?-8kncg0-_bNV^np7~KYFuyGOP#%GoI;NGq^jsh7ri5!k0GQ zyQ!hY|ATdO6^M^RTE?sr{I+7NwG4FmH2td^G80Y@hkr}YoyAP0i6aIg=FB` zKy7ovJUuP@(v;piN|L&I5NaoqU6^50he=|MSD-c;kYTe#e?!3mbmwc}=@2Sv zBjulz4^n|4RE`vQzz1)bYb_KLHLppps^~ID8h`QyKdJqr^K~BA!Fno7-y> zyv^oe(0aoz0BQDjb|;XRQhNA!9UZA*(`62x0g@b1^+GX9N#844SEKk+M1klaj}r5A zYo03%Kb*u>?7()=3^i;IM?SwUGc~s3miVBfi2=I1MVCn_G3%B;NTVjz0%r?Z-UgAA zd<-9wx4wZz-~qLQY=ho{IJnVFxZzS*figi7kOnE&Wz{yR3+H;MCFa5F>@IRsj zp-)RkpB;dNS(tI9xO|Z3azMr}4JrCk0DQYLU&zqrs8?R7^1FOP+MjXP)Aq*vGFHEZ zVDHo*<%X$Cu#E|l4dD2o1W2Of9q-IJ??o=Zh!2k!UJ#a5^t0A+`GZp888PLJs^H)> z&tgoH-qJu|QP?lD+2!GXYM9BhOz1~=B4~ex0$&E=c{^`DGSM zqmW+PB7+$N+nXaSQ5`sh)E1OjH)?$}1`^%j`uyPxe+IbRs(KKXXHK*|_o9G%#E++^ z8lOXT_L-VrEB%wmk}maHMQO#ZpFWPx{P7Ji6;};VqGkaQ+*!yW!SAONe^A5V5`Ed! z`#)h3beRA?3_R|11g7DjpE<{k`eodjZQsuC{0SLTg9@%m6g$c^M+qz2E`4Z4o7lT_ zT3pT0vW@J2(7{W!NoJjdEOX5&i4twmBA(9b&AYsFXGpS zkrYKfxR-Pfbqo*%7*ljN#eyWf>M#o7--XKx9p1*1MoiOk?Orsu3jAlapsf#pi(|lD zgCt^{S)FCe|8B`LIilZBcZ6O&24lB$BRh>3{=e(nJdgb7l7^v=C|WuoxUBU(8@pu{YPump6d) z$s}W{PgWKiqjX%ViCh6``+Ru*FS(u!hRfu&OX3#Gf_}sm$6g4P{T`?MT#~_Ju6lJWzk@=Z-t?V5-!3I4Cq{1JG0K@Tgry_*+rhi`w{;y9?HgBmMf$U7jS(qAygFIf z(4|An-cZLG3(&0fGwVg= zmNAR*@QszrqG?>sEaferT0}Mrg}={#Mpb{R>&_0B*922Xhr%W@%*FZ2<2qCKJOvU~ zLA#D_LkB$48uc-PoJX!;c3Ao9I%6ca`p2X5{2efcC=q}Uwp2}tUD2Gy#j~lzNE|d? zi7`RLLa@wv+6%11I)k?$F1}Z3FH2$)?Cby&^8dKi>HevCs%eP)a+b*aU*7p3F5vLX zZ#>d#?QvGB)mtzpDeEZT|KSB=SiD*i#{Vt2Krft2R)9~g8xQlOE$NY3oEV8E*#2_W zAq^T1&bEWIgY?(r`5a~ixPSp67l;YX_A>;<2=xhJy_!QAxFxL~xFyFwxIxRf%>*fm zZN2}3FW?4jo;aC^mYWP+H*u3^^PCcP_u>mAYDmPe_w2QczX-CDDDCcxh?1eyVi+@u z{#n`<4d{-OEb^XRkeyJ6lo_g>r_s}EE#*>Udir85mLON$7Z$A?BT7#JqeBS$*?tX zcWhx9L%+K%cQynwH{vU~&IAJvVod|ZX&p2XilAO%U|kog9>0S{gTpmkE}jc}e& zvv(E+*3OA(rpN%*nB-wbBNO??@bt+imgqZ>dr&WNPribgIKAh~avPv<(vfo>x%7PT zaWGk5?fNC1m1jJIEc}&-=7B9QGHUaZ~4%&yQ3++AXc$ z9dYH);@^oLXv~eg^iI|}ieO1;t$BtoecEc*4XPVapBvsewI|tO$el=AQmmU>!icim zrN=-^U%AuB+9g-wOC;00t=4W_H6iI>hm4539#JqnMe)-}n;`w&nqS-u5?(f9V#mT{ zew7DhkxggXJCC7Vt;dusLl@l3J$`|XHSb^=6#d$c)kT1!PeN19lGww6zQ<~4f%RUD zDB4tV?pYc1rG84G>UJq3J^0sU6Zm)^a)CG(m%coN*BV>1U6vR8z@f5fv<616`7^Z` z)+7Nwxl z9Zp94+#0h~JGMn;viJj_3d;WKi@A5;^cAKPJbXNKl2M;jM$3yQjDf#=IG3;R8ZP-1 zNvVxc^IxPA%y21uG72bvE9p5SExcCR$isXKl$;B$F@L!0)LfzzqeLA1opdOnwb$*Q znnqzJpKElzo_mFo)HU_)K;3H3c-01LYNB^~?DYbrkhC{z~Y*@)lth`H71DLYNU+Re5ml5B`?a2Z}2Oxsd% z9U&oB9w{MHiPg?|=LL4mq6v8x~ULYZxZ}C1okg}V- zOv>)plra%D+#~+iy6892PKnwW_n=rPQ!g4&i(!33JH%G*xxCmf`M&ifV(MOR**!Qu zqN!I!(+^ZRpB9>tMKb7s0+Y(!&a7L(1k|?&uV+U-RomAFgD_>KyI06S_0BR0Pp zhF;P83-{S8qdih3ign9W74*aV|*$mmFgB)vkDgvw-C0kF^X%8 zeZ3mphHPK30j3+9(r4JD%KxtxYYKKG0PeE^UCBQ6n=3?A0sd;6W-IK;-N#l|>!SXB z`}-j=d=4>9rAeJ>zFq81m3>!OO=3J=N!(`6U1#Ynxk({P^2${7UIC+wlZwTlf{a8u zVhyh`=Uo;+Fk#xRO69wsKw%yMdT5|Zy}>rp{x3_*yb&H9kc--m(Une}?38p|T$}>A z&Kpt=)EQqYqN8x|n)W;+bfF{RIZ{o-shhyLhNPR*maTl!_D4j4wi*&oNWil4<@x&&y8GgJ8>q(BGYo{blEmR_Lb-yF+S3P-A> z@OxUv;LP z_oFNHDqxSM9!aDk7};@ZR<)_xK8`l#&wN~SX7b>QQG9;dE3tNfHpB{>8(9zdPh5Z( zM+`5Xnoz8jJS%v$haBzs* zO)>p6C6shyM8fhG6PH`HgkDUOm{cak%20H%8BB`B%R<TLdd5 zr%>O;l!?ha5A7EXfe-ZSxmn;Ql;n5WjV={h@_?u_zL; z5e5|%mjE@Sgjid&NTQOZ-uF|VNuX6wLVb#kKtN;2k5ba`R`Z*Ocz|S>fWL#vf{73W z^hqCjUAuae$y+(H?jUgm_!TAiPpVD9m!p~2mAk|6#zxqkXaIt;A0YHmmVwK#pq*NZ z(qkV@gVb=%Y5ikUEFjbfQt9u`f%ZQJfM!cxB@+5kbpDVvz)G}R!oQ(K+bc6tL&@XE zu0H6hdxtYVm3WDP!3}&~hI5@F29|xj@#%#&%_EW;tS}|f+tRs?FokbQ46N>}R4y_L zT*T+mBvcg4;#PkL+Cu+TnQq}ApJb2waTc@)e8`QQV+HeLd9A-!vWx?mH7p~TJ;%OV z2nczKlO8XK8#bdnm|72-&=5}<%PH~{K>G70rT^^|A5^5xChNayXH z_OMKdge*{6L=oWcS=JMc|J>@hN^TN-^;6{z0~9XBtt+=84FNAVlIz9kD;~~$4EGeF z)4OfU7=3PNgiEyatu79g7Tw1xH!>r^1#C=D({Ymn7G)xBK__+Ir?Nv!@7f8OlguHrtlLh+^Xd~YzKXeS2eh_^G?;x_?SU{{SE70)xcjo}22t(EE%pU)xXWA+tarr*6(OUQo7q*1Rw&`}bAu);9r?<) zEDkPx477ZhHno&Ej(?He_41rC@OJkS1Ddk}_K<x)UpZjDgZA5^@d!dmp8~00n+3ea;E1HQbtYmnkJ#M9ZMx0 z(O$lA(;bR}Li<5)mm=?{Pc&!j2F_NUL#*czg6|QrU$BxWGAn%-if%N0Z@M3aln$Tr zVU4}ovCB+$^ippxg*iv0B@gdw*R<#HJbpMM9ERJgSbcc;2DDp9U9}!y;ObR!zIOQQ zU}sZ5A?)*&qEjblJ51L|XQ`PuU~wRr`1wTRI>W&#%9=QGAl?);6F#X6Cjv88rl?Yg zdV`$M{^zKs5Aq(UNcuv+o7KiR$U99+!GC8ct2Ux~!gIPY1>XVGb?4L1#1;Rkk7F`4 zBcoVN^)*i%zHy2u1-Y?;d*kLtn&tcOEBRr+rCH=FWr`M|Dl(;~cC!m=1`Upphf zwYhRh&?Ama*l)r>A)o1K-?)bb;u`mp#j#<_7=j5bqw)_HMB@Ol!6aEq3GJJnq!D>; zN`X{c5Ni)h1pgVNL00oC`z`{oRe&7nB-$LTL@9?Z9btA-tavTjHK;$2>>mm5arDH0 zM6^p9jp3zVRx4&s7%vR5)d{$yn?BXnSR*VV!)L#O!hZRc7H$RuAG-^*ijb^Bk0DCa-f z>r`Qqf$DOlqn+p6OOS()-j1k8IlRvqT~>AHdOAk$z+uZCLfn~4N}@p&Cj1_6_Ou4( zr+K684#tOvs+t&a7CCBJe?u*VZ$ObQ7?;u}_x*;061P4>sFXmS%j)^=qT(M?u&u$3 zD3>7kxFL~*Cd9oLrR3&kX`{9k-*&+KyqU)9hkceF^qA8^%oxhdXU!%1)Q7#h?fatRC^=Ru1{n2l`XmIge-7xX(a~cS zF4&7n!N5tWkj|7i~5i->#g7{U&*7gA_==K9-_uGUpd4 zgZLS(9;joOR__8Z6P>kTxn<$!-=MqHDcJv|tT1Xyazn=GmRFBVnbv8ep!OMxcbu`x zaTRp_dI!x_K&F(3dM$Qm2}n?8?(>5&AG_`k=eR^hZa=Jr`ape5(i)rN_SExJ2=|vr zSiZM_VUJiY$llGAV_ROMm`DqO5XBMLQb{U9(!aRRx)(?8-!c1@dAUU84gxl`8|wXU zZ$ms<{+2O`>Y?9V%FPtNqMKch;F0xCwy9wLpz58do)JAlXJf_A0uw%F&H-ayF;~BQmdeo|;Z74%SA99g zstYzJdJx4V$H`zx0|_LW`^lhH=`GoOBOIUua2^@;k}hW!8{W`I1hDvLb- zx>}-Jtmo!TsvC~Uf$5oA(ILcd`T0JGc_QqAp!6wdW-U>G0=|5v5xz`-xIeb(U?9bL*c>YB^GT=7K!LrxJ2sNdaKZ%e zS(p;=Gt0Ks>M>M`=q^Wdih7ADXb{C&Zw)}h7AX{rl3Yl1<)=x7i(GT{@&(N1fmd?e z-Hr@|^r*2#NwYE2ctDAORqTwUerk8Mf3ppFxvM>6ac$VZYj(*Ih4n5TvVWR#p;%N6 z!L_h2*;}lV>=)LYP1vJ^<^oxz8OtCof-I%^9?i?@jCURf!gg=3^ zR9hb=S*y$)I36@0%^V*_c9jf8?!Tt^#BC<1jizW`KBBzU90K8F_15k}yUD&h%o6&V zM=4@{9Gr0^=1xt}t6X@kYt2D=6NR}SS%}BZcjIV}O*WE-d&^1mm3%{Vd>mPN1jSgW z#w&Z5Kt8-8O9jp$PV#Bw(4*PPRvXQ<4M*p0bRG^LtI@0g_2B&2V-sK>?S?&%Q`HxEXHDjex5^;8x8)29P$VVD2@*TXM7?6D)8tJbAtZQGI&HHgh( z@QFQb)mV9Vg;z&cgj-b&Zq?(wo1(rwf=$k@;gcZ0TjvP}h;ua8`~&x9Su$$TWFt$^ zNKqu1LucPjoFNnRn~R<1=ueDM>rR4u6zMVrH6|1kj1evaarsPeJdP^ER{^15hh z(vPa4KIVubPU-!11uB}T-^I3o{Q?rz!IU8EAw$$3U_@$}#PRsDK6-AXos5!)(Ul|a zSN*%EIdVi(t6R^wb0wm1fM2S>qvQ-69XdWdDOc}8w$|D)iGCVhnYY3KmrKTO^E|5* z>#AB7ls`wCLLqpvP?ne5EKWSex90JHP~lm>As9$9>FX>~ViO0DP%_f}g3*~jy(rm0 znt9UYARg0AeHOapl)FebT>L^}hfR4Bzu|6jfpw`Z2AQ@Vxfb6}#3T$TGUP>JYFK9z zF5vs#@e(I)x)X}jM$X~39n@mf#_4_<^9s2#VtAn^j9u32PNen!+_O$3Rg~y?&X`cq z^E41jQQGtA46j>HW zszI9R?_+GF3luc8)4l03+FaF85E^;8a>G%qS|?=-Q*v#~JREIH^bEZOVpR8KLYR}) zd*xi|&@B-t;qLcSGRx!lFj(p~XqlbWLo$ecVEAVFcCvpZt=qV_DPih>?85*;sc(Jq z=QcL4^d>!`hH7%<(bIZGSE*z1jU5baXwcYFTJ829h1_vo4rh?kMlVK~7FjShz{e0@ zY3(clmN`Svi-{SKc50si7tL!Yxorkh>o#3NCQl_zNGAiqp0=^&q-V8R4B4i8e-)$_ z&1$F)cQO1AC`&iJOKR_xlgomrWncg$6NYSI1XjCuzK7Ou=F5% zU>!}-Q2-U0t8oJImxN-=xKcs!c_|`WzTe|aYj9nDeYgZ*{gVMuy=o;~uhmb+QXe{t&2Kp^ z@NWy)RxW|Fw6Wbj?YYjMH%F*E9`@qYM`KmcS4L6oZtTVQ`j(4u*=jA|ZjSIS!|rr% zP%rFU>Vo3zwZ{FJK@n?IO7Q>CsH&0_JDBurCVtQvBrbd(RiV2*J$7-qYLjEUeMdmO zqW)@%vAf!045;QC94X%E&}*+IYQYl#i~+SIWt|M^XQcV9W%ntYYtNbqm9RMQ&^ELz z8qQJgnRiqi3&BeYGa3CT-so~qbQN$%$s>Aq6izW5nt#XJxtavAPrTuUQDrGIQ;mmj z#W)VfuDqxLts(5xGC2102&)MSyGgzkE=kC!pOCKWs5!$JZ)5q}qMgeAs_gl|SJO*8 zu!GT-x?eP(mD|@)KbM*|UKaTuB3%!O8;>O1LLW|F5Xijy-lj9`<+kD-ys1-3Kkzk| z2-K^1+J(*;n_ZOl9WjHv@P;a$_*12?7}kT^1oizN$dW|BYUc7_qn1%4i^3($x$*Gj zC&2C8OT2HxZCFB^bC#z5(kLeIhExYo%iQv^92V-FZ(?u#pBvLmg2dsiJxs|O-gcrv z)lJgN%G}{LZ-62FfeBjcm#-vKcvQCEJi-YGj!|~+-|NjI2{8m3D~0Izn^&WpEATkb zBQp^L$&0SOXD7i0_v1p@mxTtTih5Q8{d|`MK3Pr2?RMG1U_3K6ca9$qUN^5>o(?hb zN4bFSOjD>JmHp=;!A6kF=&;U@9F$G{oNFiuZsQ*@a4;cpJ4U+2-PqQH{FH8V;jb8} z6arWgFHU!@w0%&Lb295q3uUDy!7CJc%$vtygJTx01$J{7b&&{B{;EE~xlZ6lp(t7UBli5z5 z65${y+@{Y|ToscD>Q065*TcH|8+Z6Ow)gG_|da9 znR8h#ZH4+S(m0Z^ZV;{Cvh_X7EfP~6Bpf}uHFBlR^_(p@Zf1{syJhbajV-O7*K?BIn^0rxK0g8UUSmyRQT!p50N=1Qt2*E&i0DOHcFAf$doo zzZYyANkmBRlU=}gFNy>g!hld6y-A`6`O|5ji*Y2Y*X*y;-Nm~s(Kg)x9AVd5!*Q9s zT#*{xZlP2%R)Q-7Ny~-T%$h_i*@xf08?$NY)*xW&JqfNWR=)MvwUOu!`5F!hnyHV> zNx)9Lq-}cMAeUU&glS?bodN(hEl5KduLm9NOh2Yn1LWR>{NDupFwj&~a{8l>&0}|rEZ}b7-RiQ1Lyfw0zV#ST| z)1lX&RM#VF1#=A2YqW?Igch(0E%C=$kV5O1&I$x1@dLWx3*DLSf-7CLUD(?RLqdp5 z!A|fpseoPgw*M6vfn?tAQO~g!b8hRy6(8>195zt^AVScDHaI6!CEz-2Lr$#3GKfN@)cC+_7wu6kJvXX&|3UdX#|aHbLX7MgEjv?cf}?fRW2(0kl5yj7n?4 z>)s?0;DRU|>xt^sT~oZ1FWQ`;q-?az8Gk#4`_y`t2I)}`Q7#gQG7brIXR28$5%S$- z8E_;Ts9fK#1;m4CH3(XG6%8VLowupsIlm-o%`B`G%A+u~VP zjQzG@b-|-O1IBtwc5gpzTu(QdqB}fo>kTO%T~jLtiJqK>v+aeKc@4`mv#h>p1u{dWBeO_<+s0^nU#!<~y_G~Ddzlu9 zPyt?44*X!e2kCicgk}S}^nHm_GQ%*r8Ew&Q!_v}=C*zrZo_9^FU9K8)kh-eetAejz zaS@kd4cB(Y=m;MNNLc8NWo2HlRpSI6>{QyWUu0=&oCz17_<+rq@qmfY&>i?|t{j|R zKWz|OTCIn2*5^@v#B^f49k<+nRgJB_^%in#p0XxN)5qN}d%$^i*4-K%Mz=BNMr}%HI{?cW|_#d+9RK+J9Adx?` z4a_W^mIscoz!UJZ5hHx+sv6{0Viu~hbQzbubI$X2mA=VT(3Aa5T`j1=GMjM-r^SoT zhZVbE!%KOQe%Q+|yOpi%t4>c2`H%pTjc71@{g#o0m)!Vklphbp*8zg-Jty3vkt0ME zZZuwxKE0dql894oFf;ntDYV9OtYwbvJ&?5jg|2RD_sEV%ZS|Wm&_;rR(JHIi#_spNr%qH3ja({0CzzAi)JnvXww01Jk5rv7_lUi6`#np zV>uC2<*&rW1?Ug%F#e#)f6=8BIPZi`Si^p+d>$=xJ9H1Z`QaYoe&z!J zdYItpJhueMC+hMllSi2K7M| z@vXz!24u3MeZ(IkluyIq&f`ph;=1Er^cW)&uA>U=k|h%LgJa7~mv_gj(O`{U5gS;? z5g0IHXti!aOlBJ%VM;Ka2Dw^=AQ(KRk1YQoJO9z>MrmZj@igc?A0U^V%w`q(+ccfo zr3swIp)-d7Qu?-T^`v^xr4|3#rD)bJk1-KekOk`EB0j#V{>?xO0bF6yz` zKc#)_da4zmf+`)a4e>uJ_0g)<^{i0u`i>(ghenM)v-fB`(cW-49h=WvYum`Q5`*{t z3m+WsMGb}bLLeTyOtKH~HlcXln=k;dh0l2; zPAjLXQ`lRns0ooC@;)X^9CFzGt@Skjpb?d^7IDGm_CUgs=Lyd@t#r48VPPxLOS{XN)hDtyijQHOtmhTi3w-H2>zeD zwOp5rY5{d=PgKer6DAq)Nj5dcX|`}%$`DQJVzF~5P^YE+{h=N;A4}f}_kSgPeio&` zo?S1hIx;5-B7acUCPZhU#XlHCz6^Tl+y_Qvlw=asP39%7`#6oKUC3mDtWZaNg{(b{ zLt&nw*IBLMcu}ih9xh8{vy9qO^Zg>6g29y{@x9oPJNx`=bjm69z0#iuHXZ$Q#u2wX zfAWqAmiulKNS4(J*N6RJjg!YyA%q3wD>0IyRDfK%ME&gZ36u<|B{8W4yYsc5apKz( zXy6#Oziw4>2ji`?il3E`=~YHMv_XdAXP&~;nNGiSkgKrdpQzuxchOsZ-l1AOKxC}2 zv_o7NDgXMIyK39F!U$k$+Pe@7K_*ml3g4s`v6mZJ1i(_D5hQjlfB;x9$A-W!w2T9A z?mDd_k_v|9C_L#aq>8IQ|5}u?Z2_``645FTZ-I(0&^-TkftzmI&`G&dNx1=RJ;rzN zR4$rgJ*2B&zZ^MSOMY#>tcG4~cJ+U(NW|uunIFU?Q8(v>ngK1q^KR}4TD=W-1z4UF z(~E7N)duB*v)rx2G596)ImZRVQ_!71xqU{cjF#hMuQgSZmUsZ}I|1$`(`u}$l%7kY6d zoFMNQM&PmF_U@_t(WVQvV3i1N4SU@#+i{YR5QJ;R_5)OHuPvd!o3{AQb6=1Oe@Ttz zTQEZ!Z9XVpW+q=h8^USdiX6|qQo?=}6w{jbEM+B+`4)t;>P(TIM)Oh_H;RMrTi2We zJ|X^*YPHJabQu=s7s9N*tGh}?d9uh^i?8|7a6}Z;9!6P7QHZb>ojYTvIqIOuco@wX zXLYibnIgIm1V0K6UhqWbWU>@x^9#U=oCupSS_Wn`-=!^pJD&o%DNZ>V4G_hJ5>8Y3 zD0xFZ(bjHzn~Ov)UsEP?izj+kS9MvcY*vr^{Fb%T~~c2;e&k)4k&A zIo!gd9;dtvwwF<`0DQo@c$dx0!%hb6!NX86gVz&QY#e);@dIjGfX~g%iFqAb5BeBi z(0V9LCh+NKGuvoJ5%;v_H^lHKs#ay6^NRT9PFl7ur%k%_uB?5JB)Ukb;|MaZ=GtNlgk_;JK0y#C#$ zeRDtB@_-nUdCJB0*%Qi0`oS((+djwaS~2k=#u7Zw=Yjg}&f*dpS7~k=t3SPTCr)2A zl4l!t!J@&M;Q+`g=D{_u$5;-HM9RLi2W>4yH@i61St5V;sB(el=gbdeJ)PafB~|k- zpxs%G*Ruzs>w&4gA zJW*_uN5pe)B`g5ad;%mF#p!fo1S}ZX{{Jp%l5-&H=pb{yZ0k1}87J8rsL@F^t+kl} zJ^!C23b5g(Lpt&49Jq0#gtxb69p(U+=a^Oi^*q^sbK2vA=UH+wdxjX*W79;!m&nX4 zr!t{~hd7NsjP*^P0t+4z12t0HKWv+yz`+`RM0{mYQN5&B@Jw zV^4!uy(H9dC!ev3Q$hOe$;3Zt+MobiroV*mWB8NOjz%7li45H?=VV!_=jl#q|A032 zDq8c*VLTqZcaR|t6)!|ok@w2 zPo=tRj?i(Xsi>%%m2?LC+Vi`G?n`F3;d9|pBjW`N@7H+b)S+T0>6qVw{1?4&c(Y!gy!XSZhTX6DC-Thv?J; zKQczh4&z}A5B@dL&C*C75C%2F1G$Q}riX$-g;M>bd@b7ntg92 za)^m()qOO7^Oob-Gn!AghPTMR0oHSxY4$4uUMmAim#sD31*jEcM)H)x9^{2hpQ+f2 z-+fQ^Cxzy**dA(T@L7a$Is?aP1;Mv9@EC`7;*!FrTIOEavA90Hp0l!)FRnpv0OvS;+*f z@_zGH%QRIf*&S0FU|8gaxt`7D!x>RX60)XMi9;38BD~LE#J$2>fTRImL$vnJ>}HRD zWO#S;+3PkO1F+xk7#IEt0AW}fW<2sIJvcq^^_z7oBy{l8mBgp~8ba5{HN<+Q($ zl%i$G>_hxY{6}krnc+vE4B{?l-v(kYF4quKBt)FLzYu&3FAc7ORyB)CJ1B@Nt*zfI zjdRpR>k;tJb80RBTdy6JsLKUJPd!ixU}l1B`&kU_ke3G2c1u>BdOi~g7Z!D!TPRrR zftbgCwbLEYm+h~r3V<)rwxx>nd=tM3&dbv&RoRpwEv@1ka~XVJ7IwJYPzJWTfmvpj z10CHFw7Fn9`yp=bvKcR{$QIH07xWBgs|>QT-OH7Qu*V9(-Je6)Nn!DD-G(Lfc?@jn zP`6KP1kR<9OA^iZP<3Ui>2)c8_zM!=AEYHS<|lpKPo}p0O{!HMg-@LF_zuj7mPf0) z#>4sMl;%U51fu8mDka*PM7`=(Lcg!Bm+nc0g{#y{i5F%R;?Ru92^J}#93sQ9t%`E} zIAr%Dy|(Qe6c}+b2iSp3Sc}zdTMa_JLU1{hqVDR@%|)TKnt&-<$9qk)!H!|;z;@IK zsxOGo0SC-vDwxrnk)0Bw1hS}vWWuZxEW66M?)S;L?9zFo(}&dT?*OYZpkGF*UE#Le z0nEE69zWc3a8UlsmwCBQy3uX*_CY0aD5^m7guoN0@eTQ{d7ip-@bL0s9vW2tC!M5W z+4*`pJV2P-k*4UvGDb0d$H-f{xg88W_5Aj!M+SB6jQ=ys5HwXT~_>9pYKhf5PaVW9Hiv%60Jc>uUt-X)mCpzb?Cd z5hlh2U(LyBt=mDySwjae8Eb}Mli!|pf_-O&YUv9Puj~A9ie&@D1 z%F!R_YZs+~^|!x=utE(1AA?OMyb#ME*rr4On#U=_Q&j#BF#T>^+*;fCy;W`B~3!Ia}0v*7^j{vPDj!9r)Rrcz%${EE+Fc4Vpkb zp(dFjP=?UIYDPcq_NSmehFo<}9HPv9T|f>92>3Kv_LPV01R-azMK^n~aw&o>a#s}K zwHeXC^19q)D&LbVRe-Yu4}1fNzoQXWlaU)P`SAvPy6y*Jj5!*dcur7P@zb+aQi7er-fBHR-s0SE2L}If{0qmqF zA6GHXJCD>+h72!bn{V&zXr7z^R<6}-nR@HvJ#pv{qDD#48#w{koNc!03PKB#9@@U% z;Plouhx6>Y6#pMXK5G^a)%@6tk}Dsz{PZYqiAQq3`jenO#TamnKPu_*3<}6_qX8*K z#c$L{-t&IBgBhVfo5;Uw!H{q~BH||NADa*jP?-4U7Kj}nh0LNxL^NRP-3Vh7;?sS& zcL_Jkz|9C4UV6^~*%#tC#J>^Q#4c(at7k)}Q+mpPL(cu4VY!x274fG)pu^3+OHknq zV~Ne1`YeM83X5N_AZu|5NwL)JyVg@_f|{s|@!qX3W^e~bX6vIOT=SBFqNcwP{z0f6 zJt+rbZ^d02v_eI~nZ0wZvJRynMq`_Yk0n`tAK;AOc+}v=Ez%UxA)d#xa-vXE z!SMD7O{i2CbM-)tB zJcQF(kdQD~NSQlQ32-niZk7-KrbDMC<>=J)mpwsWba@lipRw6m17N;5Axo%B@DYUK zV{E7P9;-5d_m7g-;C3~uvplW!D0PfT`pDcQgT1kOfjjaQ_q`Cy+7jo<;hd??4WTj` z%lPc(e-O_QC);TJ5UWGF70Pi9vP_L@DOy#Pb%=9{;EX!oBqd;QrbXL zqlJGF2q_!BzxJE;d%X~wYTtwQqFc_nZja-ZdP8IUMeqxJJKc#Whk6oA90_?l<0U+#<)Q{f{gJ#n%lpL-yVC2H^14p@q}M-;&Y5`I zf6t70Cww=evK0WbD;!#~L>+eEQDx3LoF@_$W_SU>}TJ(3G<{zWo*=nqEr*{Jfu6y%i0jfDuAx& zOW~o>xcB|NJ)`)Fn!GZ?N3m;8TYT!-DKC#4C~SqMSwj?dt)J?Kp%lN~Kiuxrn|6$A zAPTnEybS%ZP>l~Dc`m2>RHkE0LAJ#rEL_2sfZK6sfXN-INU1C`hq*Y3(ggDPA5(M> zJIoEcF;!;$niVBur z*BKwb4Vl2bXBCOcp&-sSty8$vqD^;%!r);-_Jy7+m_A5i*W)*x!kE)@c6nHuv;*PrBQ&oCJyJBjj8lU9-Pbx0<*rdhR>y&jFWZ4IgnNCx2%3C{ojA%7kz=|0mQ0}+*T0lKTXiDM# zkdxR1Ryu3LYGZA6zdL4O`zlWX64|mRqY;AUBsIObn{wfd?xe6YJ5sDXMkH6~4xS4$ zE5}esILZlHYv!M>8aUe+CQAesirMC*OQG-EjK zuRN9HG^wGehEw`3etQG(0Ej(1v+I6$=nB~1q|U~qE13dm{Nf4RkmoO0lnFZ5Uc-&?5kG}99B(hKYg;xxo|rWp1j$NKAjn|@3;d=wN!55}(ewp& zbvoz&l)y%0yQ@Yas@v##B6*s21R9w&Q-|wH8S~)R;)WT(bz)8ihjZW)U7=E}zaL>D zlxZ~yZ@MnCEUo{Wlfb}@{&Safqs&=`k*i=}m|hhf7ar;s{)EZEhD+)Hnt*?TuvpAQ zt;1S`HpMQi+miNCeXz;i;?iV9^75aNmXVea1D-_&cTNrNx-oujZJ~iKVj|YBb>2$l zK~+_h{UVXs4%j;ELnhS98zPBwf5$zvmihY!19!j$K{6-45?|i7H9_AwmJLTPkUOln zc;y(g7CLja*8OH+ z0;z#g^a*Kk92mBxk6a==P>Ef+Zj{1K1vu!XUe&x+ChO!LS~T;Tgw#P49rivK8JNee>1(M12>@?WFDjN{J`>jYoVjs4Fok%aGw~gvXg0 z2d8V%o$Ie3x1z=H@2?I!Hb?G*3&TJT>9Im#9bt4)XfNJzOq%y7LP^i%22?epE8aVj zM{M8Kdc59Y!;LRsR8Qj|>*MB?tl#wyk!m0e8`4AHn{f*+p(t#SHJs3&h8eRG%<3&L ztQvbl2YvGUduTjU$B9lN_k8uRg%eu+6l<2RO}# z_4($4<>ca(mWxozj@8H_-Kq^^)P41zbt%lBV?_!Q=0I*!%_$^y%^;OP|Jd{)#k^Xn zgm18LgG5Q!#16;Zhxg4KUJ;E)-_E?utknJILYIcSR>pa}5G~1OI@l0Yb~%+mZM{I^ znt|Q1r^q*+%!uzr1R>bL_C3M6B~jsh%;h#Pxjj*WA@vJP2nPDeLJ;1o*qOd>!YP|p zX@=sv-jhuVw)1UFJif2Sc3)kmz2;>QcjAKls8o%IEd;K_zY@iklV*kJU@QK5MOsS^ z`#yBDt2G;wdQ-XMpwN4~de9E^+Kj}6rYt$-AdkcG#~O~;6Ugm4B9k}j*EqGTn97)Q zaGxxO*gf5{wv+?^GG`J{Dh;>Z!v?s?PMzwff(U#IZQ0>3dJ8!-&q#aGRWvAJTsG$4 zkf&PsKS%G~Bwu-Z&F{UQzUKD<33TFB=uSG!kxRRq832r%E*ndC`Y5H)K2)nzYDOF-g}YjF)v1 zLt_8vNN-!=&DGH&EzV}<2dIV6IISv@6$I1U-0^G5Z|kip#NqMzo}PlkaXSmZjJa>?HjT63frp9E zU^#;%7T#l1=RvhFyCp@ctU4(y*K91taa9El z#~h?QlTh9jch|@lZA?#PtIAO45h-jdtw9Z5+e3$DeL&N(=#&G8V=2gt?bl3`B%Wr0 zwTKTbA#2C40fiy=OEO}Fs2pO6n{@o|Gj0+ty+E(IeWO$vmSAL!IpSW3GITh~TF2nU zxJzuy-u46pb>~Kxcg^T=I7_nhu7wFDb)S>haAO^r@pOs(+7TER+_=;{vNG#7rvZM` zLz?y~`h;jq%9#yu^!jQPdR?1SEs=>EZt2;+i2CPLmouG{!)Aap^@XK43Y#I{Sv?H! z9j(h%84-BNX(gxwO1v)yzt;6izsA4z_2i>j7bc7pUi)U*y~iov7uFbDu}6nxI0HT2 z&}eGs4zZ65gw8krV{Y5Bl`+o`xtj;ZGgu87+hpC-30qVUFHs<8RGnZTM5F*r67Sd2 zJ)kgF`&RH#7^I5*zJ`z(I$JScyQ{WDhAAd0I+}CtKn#5F;|e1~crm%=1QPhTz8J)4 zEVPU$H(c1LATD{z=1IZ`s}aniZI5kJxr+n>imP^_Dzpjz zC|Tsa{~P3M6kfIQf#O_F(lUSZZm;mI7uhvZ$H?jEF+N-0P8d-{clblE!$n%K)0V*S zbG28&j;oWsV_EXZJfw7pt1J!4FlNjK!Tf1w)7y!W)C_9An;;Cync;(cQ;7#~~M~#v_<06E3hp{TYGc&9!mPw;W{iu3|l!uq2 z%7IDb;rE+OK7eX7B7hYUmO9YQ^LNW`jp4J#pHm1pRs`Uqs{${e5-8nEzacn z`2_Z?kMmj9YjDnL3Na}QtXp$$?Jg+K*8Fqn1uXVi*!S@baZr7m>DdiX>%6u;!Yw+v9P-zffODL53Gtx1G3F6woXLJ8 z@VIA0;)xy_qsN>6*rC~2RQe+|7--|Wudg+bpcAF?A-FBV(+_L9lr0a4SCkGD&-jp^ zu5|JQK>jD-y}bYQJ&`OwT3#RT7)5yz&^_+(r2CDgF8EekBIc_jM2|Fyi~h~fZz^MG z#WyFieBve-)~L=av74KW*tg+<%a=RYLhWy&`}431C9Xhn4m+jYSlMI+c#&oK)@bMs z?L)re(pgx&*d<5Y(3g8{gaTJkW=s5s4SC`Ure3!@l?xnO>8c6N4~-|)kc4BD?X(K5 z_#=#}(EH980*V7m>l2%k33&Cb0QD@zkwbD*cmg*GS+*$F{XJ@Dd ze4BCdsdC)dz##u7H*X05)$$he;*U>PsH7E4pVSx^XjVJ`Ei3H`U8i#T{1=J;345| zv>6ti%LIpL%$;3JZ~<`k@`-oc$k-Is6?z)Ln9SDw1`zP&Grwu|zb*n?Tu0Z;1-?p> zo*>IQSy(9^O^{|Zdm%K-$c;lWjAwDZsgO_vT?Jq>~(B3=DJi5=0qak=Os_zPmbC|B{rUY!?Z|o(m8d|8J(OW z$SDYWL-N|&bFH>{N?!)MaK;VK1bNOM|N7-4m@z^WH-`CG7Qr$ z?e@f!HQLozzw1Cvrol2iFk+R9cSwLu@DB1vDvJcFLl|C5sBc|SyPl@e)~Us8+#GxY zoXD3-Ok*(YnGs#(AhTK=_Bf$?Wv5}VB9cb`BQWDsOR>0l@GMk#78plQ47$Ie3Xr1e z?>AQ7YAc3ai%nc)+I|q;@}Lx_AVEn1JoLHhb|mE&W7Yms?=ozfo14Zpo*k?O)h#}| zqA7oD2oN4WZKmV2I0?-r;PRY6&z%;Orf4asW^&n8tNR2fQcWI?Z2kq3oq0pBQ|1m} zSLuZ%w-hhBkS4TCzCsOBE}pcb)tT%=;wmcSPZd7JQ$SQS8@dzMXtHA-d@e(+anz~8 zk{KLY`O+QHAbh!0H9Qt(rW>LO+!9iauqq%rL>{Q{6O!HXW#-=bZE&jREaWHgb{Nje zR!|Rc;8wH1TDXyesTUXV#Qa;{7|R>H8rXOaS06|oxUxhmEvFQwz+5``^DEg6Eq(Nytj~j{1 z67;(V>bOwI46a;FRn=NF4JQ`Ll9CrIBAfu_grhs8XakFCfOaNnY>dS&)Vch-LUBg< zZ3`Q#C%3qG$9;>%ryl4Y1yu8qlGE)VL!A zv2p*yNeZF`VIE^hEPGgYAls$5@JJR41wy9Ufc|1lkY=rX*mkIk?WQOg6>~knTT)Og zOEY1m=-k+ldOeW;&zy59x*KLn*$1-QKinh&@ok}e%+=N=OJF`rCtbz=BNniXI_QmQ z(5~Mv%BISK5=WIbG03b*xg|HasyYcii7y2T-c*ks;honX9e+O={>#bHFJY;AZ6_j3 zUH`+)$$9!pEjeJV!$hFg$}uPm+5 zOoALv(JS?ZlC{s5$Ag51{({!_=-*|!y!JawIvl4P*(WB^M8|EN%-B}c_7R9yrB9eS z01MBr`^0zcorpn%&Dc7$(`Iyi*5W`pXpp*vUfo=dz1%(syZEH*rCV}r!mOjv!|h@h z&!7)3+&QO}6!vTrF4hPw?@1=5MHvE}wz3}1c?8RLrTE5l0-GrMH?z{b+=62^v&1ZH z(PNRk=j}m~^DiY5FSnBlU8)s=Ze6_m(2WLuD{rjOQrTF)`{7uI`^zUUtwq$4PT zPkwxk*KHAC^eZ^54xm*Bq+YBog4^h(sJK&*43?0@#1+Y!Ir+dYTK0{-o9|jXYY(gi z6uZOHG>NzZ9PX5xZBJ>v>Rya}e<=LsrVAJIs{$dUaXUsD5p5tX7$(~Q?#+tC5WCL@ zJ-@;p1H0be9w@f{fR0^uFz)%Wrq7*gwvp&xZ%Dev+MWzsLtb6^+75=1yWbt79w{FN zU!eJu9xe7NFi;Tup~m@=_LpnXE4+LbjB^1y=s|2z)owfHsQ15i%p(wknBg?nH{B@Q z)bFu;P9M~tv=Rm&Ow51)zLnZta7`3XwXK;LdW!7s#n`I7o#j$9Z}aL z8FSTPrlrWAkr`V)|C{hBnxzWiPJiE?mB$+@*FhzoEd#t!E7if3jea6`TL$H*!?WXD zUVZzbNx2l>F0J7+q)%tjO-NHa=jbWdsrJC4{TG^tG}I|OB27zdVm=qF;jbQD+H1%d zoFD>xOqd=UQUW_BGZ(YfKXHJXmknYi|0w@ytW-@t8!Oc&qLW2|*me+9@w5^05yG-( zYll$@oHZW%cejOw>fi{r~x3UVKJT6kn0npRcO1OaTTb!C8#uldu*W!1m6@Y zWQK6g8s*_o5^3V0s~oKd8RpBsgnxH_vHuq0aA>x33p3Z%{3R$_Oji|Mun@OJG#dI5 zLTdSRbKJWj6x%@h{&_ zv3s$6xJK`TQfGue3avp({+lUG@AFT#?kvEGA|rQ#`cu@dwEVkJ5*bMP4;kI(|58z@ z2zhI0ahXu8&EGz)kPTEEPNX0PwIgTVl@>!~-js<7ip^I<@`E}Bm)_xt7VX*S5cNmZ z3ptAz0P@$u+=Gtr{5}Op|sraTi+J;Y8OBqq_c(0_b zhR>(4JG{NzW_F+l>)HGe&99n4uNvlqJ|OO052-za5BclGg+GQnPiDhJm+;xI?X}r* zP?aTWbOX6m{}2FBxVpOYXv2sL0~dg(_=3@ST*tH&P|%X=+C2Srp+qa9NWeUVKkVOA$X>$nSzbB}>^68oNUswsB8AP%c>b`+|^)jhiXcM&3y zpgQk)@Mes`Qx~2#e$jqZy7j`JJ<4KW{!%sQI))f+ShCe zBgb9T>Fd(bWT{w*HV^OM1nqdWL_)lAtR~&Jm8ER;CS9Ycmy!;6wZedohcj@MHV>o2 z0>67vRbEb1HVa&m!ANlYytV*9+iF5@R~bt8*9;fmV(2e{WH%Tl*<21*I#a2Mp#Y@= zS}l8n3@KmOAlg8D)r-YO1yh#^Pt#)tn(6<9*KiU2-rf}L_^DAte4#DXiM%Cat>ezZzZNLVzDdQ?YoX$kPG`iAc{89kw zSM8H$)00Y5bQ?32cPG22^G!2|myTYi9uhF++|+qU6MIINyBb?kPd&Ken_YM`Om_8KEw1>l z&*2G}mafh;pLrDhQXF0kxy~_{|3NS*fbK$@g(k00@KAws@kAA3Wp7t=6}^qve}zLR z9b{h;Gui-_UfZ>y7-;}R6_(XKJAO4dpep{>3>$--0i%}|o88?(v-VF3PL4IJU9w0$ zwh(}*fk8A$K*Kp}LD45n8;$Zt)pwuAb`>3lgU}E5E8G;Jfq*0$K~>wnUO49Xskb!x zT#mwy8#dG4b@Gi4y0lP|AThEOT-Q8o z%MuAtR9Gw{F(N*SYK9cLZAa{-<1QXB52;*Xm_M+ZG)97(ZsvQXId|A2LX?i(qUZe` zPyE^^*cD81Ue(!+q@rO(4o<3=YaT1Z{!-dGP?n&Ma2Ci^^Z5$ptz^2<2qy8%Ad$)C z;x?^C!y&Ja-wdD_8T-mu<) z*>g_2@8JMFu2s1BD)Y8};#`cl95cPQ7N*GJQYYxp%L-(CG{xTXmF|T_kLhfI=;FMN zp}v(?>(?R{D19fwVFRbt{9KN=#HW7~hcEu^2_e2Xhv_aHXDP@+^?c5L;*o9+bfRSt z`Yim=JpEa7=3?jyqQU6a zTDqy+fdDC?Y@Q|MYZx}Z{aN6U-t4Q}WIVIjgfPN+qX&4;ozky> zL}&pd!4K+7pI^4BN=A{qc64+)As;iua@x2+%1sa41wk(ZObTzyy>FNbR47TKEp;}Q zs!b2Aou=OyMY?j4r}2HF1WamGM*QRtyyy9cfw9atI<-|k)76u#gNHtD``KRfp)Otc z@h-J1Q&pY5{@^j=ox`x|4`Dpl56s@Yn!1rjc|+FB2z<&t!KJ!>A7VDe$YWizF=^S5 zVOzJhFrI`x8mfmm+d>l}pxD#TA$a}~>*3jR)9_9&d1!Iuk5XeFJ)hH1`K;W`$vN#Eu{sl&I0HO+)F9Qw}X zL|WQ}kR2q}!X3P?6mDYto(uK_pt%whb^^l*(|NuXmOq>7F+5M0Y)$Kfyfn$wY=Zv^ zI2wh`uoWfko_xS{6;oi7)(@bFMpO)&um*h^y9N{NB3PCh`JPam%jO5|>LZ)FIWse*#n@-L-EjA0_F(4tAx>eC*Ovcmia=Y66QNrp&IA38|ANldNtAgg#A&SS; zQrJzlyZUNQQ@Sz5zgkFdb~-)8Zz8_P&}aBI{mR=pIYNK5Br5sHy}lt0)WznzrqVKW zXZ*&YW1yt`!CwZRv15Q`p(=k<$U=_);IB_n{$}gJ^3k(#WVVsJ;PVr%Uq!$Ot^-L{ z=s3BT$nTIT&zvO+n=8aOc032#GNOM03z#Fd(o!NOqPqdqNb}kO3?hqe#q&9POaq?C z9$IP?ah0JFTjQzWgv(;IX|>g1HMQVnj;rF~X#!b7>AKs*cJd33R%d>t;^i1iw}bP4 zcbRcoKmJ_rsqcy_J8oU2#%7@%eYp=Isa9=zz z{I-VE_7q+P8s*W1RV=yJ9U}-k!&KTjhF{AVxc@P8VlTlyKDhu5g@!1Gr?o=lcl31! zll(HbLvpre4XEMRggAq)DOGb|P4st0k;FthHVCNT?sD{Dc(x7UF-i#lFP-qwY2*vZ zR8p*k*x0bGqPJb6h6Z>!ZhcUOVc<3AjeI6Q131OoXU=pvytfocBM$*sjjFivydSe;(&vaM`fe^yjjwkjxQ-qaYzE4M4!pQ1K{g3eSg*WXa5a9Zl z*N&!Z&CWrZ(wUjXk^&@jj`_7L-6 z*{!WLQ%R{%0uIj50s#*@i}Pp~&1_f+Of{9XmB;cx*t{geP@=dAHl~a-fqpMOJc(Q? z3(h8=7Zm`P#^{6=l1h9{X#==JrEYw}VFO%NvIKbKs>t?K){{W>(NT}RO*D@?%X354 z_#Bu*_tkp`wZ#0D3qW1=wozvF-_7G3Vrs-Ts1UwIr2pNFxT!P}62W!D*+*F$kSfx8 zJ@r_3^;&9Ip$aOAuL^-EM8%SIKO=XK&6c{=u$8s?h6ZH$4ceZvivt_AB1+HnfdIS$ ziJ-C(H(8! zP~V}=sC4UQ-VNSG^EgG1NT)c${!700&%_GU6!m4C=%S&X58tiMRiX%QN^u4Rb|P<+9ln!u(4!WQB>}SrJ9KDe!21MPfnizEXVH8PU#{kF6ZGg62rf=19uM zt`4x1mC@-a~gL4aRZ)6Mj;Q(oyLURr`)v&d)T;-lQH(3@7tumpKk$Uox8@ zX%6NufTLSqQJ98pDf@&9i zbKJ_Xf>zMvQ%PWw6hahzbInaOeNNb;k&S?SBK4cXt~e(P5})sA&;$2GqQ!7^vKhxb zH;(yZgzL&lC5hGNvO(=#XbG4sAS*?uBm?uT!@Hfjr|%h9(NA&m!bhh_hhcrvv*Zb+ z@iw~Qy|rc<3GCz|+I3!-FY9!}x;(Wm6`{Nsh_oam0{7v7_b`@PvPE9EqHV;yGCRe>TL@t`AoqlpRyD*^l(1O)h5>YNkJ8sBV!fmgtKbm{z{L z6A}DX@ep6jhvlm8+e8#xd&o$VJC3hK7?pUo6$P&@6&zDLZ*ROr)~Z*$vF9(9{eh%^ z8TABiDN%a(7)qPZo7_jE5Oy()fXTg*Gx4NlTprJ{4EA0_o;}2K^|-I6gd21do7=- zOdW`MlsNnqjyl4#2b5c4!3-27S#ob?6MF~VrmURq7<^gHuT*uDoQ0?JE%zc?z{8Vg zv2uexJ~hfYWVQt_NCdmD8aV6)x%~|2MF_j0S+_p;kw2%|k#38DFoFJF_>}$xmr5y+ zk>ERbaLxvx7kLGLj7f4H0*KEt6v?l?QMPK!gw7ndS5UFrcBd^bBk9Xhr`VmBV_i^| zyHnM-xJbEffss5ctYVY|XFv9EwoMJoV^c%3@L41i56#W(YPjl`!guqw^Qjuqptmt* zvG~LEw-Bo_8gQZ21qP4EuNS|K&CI&^#S@fYOk5%1sAhJ(`|E{qS!G*7o5MMSvj2vD zl)+8tX?5B^Az=^c8h+26n>icezr_|s68mgfu?c5!n}lD5DF>e^*&NS0#|SbBF)gH@ zOlV|E$`5vU%~d1fAUhJ+#}dC6Hy4Bs1_$p^&T3?@Phvx~Nzhrfb~aCWTSL!&*lw(% z)k+9JY z%ZSDgUCWzf@kZClld1b?wZO)>=}Q~uBE;C&D$nVtXhP7mmt@XLc+I@X4D$P_S_w4#khE% z7Q4wbd?(ZifHz`Bb1u&<6`aJ6a&MfeHi{6U8r_xaF^S(sF)Zdc+w#(2jHMU8G zsl2dJ7pzdKu`rmWx!P21Z1J|n>-e9`P=I=bMgD`jE# z?(=Q51vlhla%z1tBQ@eGF_~i*ufQ1br_ZR^>lX4j)ATT&tynNEWfY7-tRIk%S=P}* zE|k|SfOyAQ5VYH`$u#qx+c@0(qL!sD1Mh*`oYmZ*zOEw+QVM-UB|5!qSrE5VhMW^? z*cjm+6f>2BgMcC5BUNO=s|?LpCWoLnp^e-l;i3G6w5rk!PJuYAnC_KiU{aF={(sEe zY6yTP(u!OA5SShZ z0r=1%3j%*!rOA&ztG5xk!$6Ew@mk8V-KNH^BercS!+M){6J01z&Y8B zaJqGtK5ED8!~2@XsW!KGAN9Z;FKmH@QLky7M-K5LxL-?*V-_v*^Rkl?VVoAhd?XIL zMh_AtO&CY?4@Zzr0M2NE`KfAnve~-!R9PW#gzVu91|&^3a2d3%pddq*+!R5HKnZrx zIg~TLe5eRnB(#G}!f9ssWvOu0MJ+45;46ysQ?@AG$*r_UwUCkBT&_@Gh7xZKj;Pe4d|R=SBXU+86LjIb(=`x|t}Ct8(jSuvyI;@wWI^HM{I=;MRRsW(ZF;Ph2x*X}@;Mz;goG$!ABFdA9Y`(te`_WQOH zV(Ucaf3^TuWayi3VDs`faE=$u#kXgyKRw>#nopJ@6p9bBFApWk6#~JO=bN70G?Bwr zQ?kZSwUg0;PKa<*jQFtnB;X!&SepgB#94tXyRLxFoLbE3aTf4&xY89E0Y;Ax|HREt zpP_8oLw6w)mV06d9*s45a0k^Aa z0skbS7tw2wFpuiRkv;D4_Fro`J)uH4_y782H=>QOmVfF^Uc3niGqMm(@(=;C%X#|{ zhWw$tr<#4Lbj@rUhV*={9c^m5Kd1iAthIEBw+j`PKji9lEmMYa013p09^MKv4AWeA z)V3zR_nXNRi5yCR5dQ^_hyvE)n4jKV(}I)>$ane|dJa4m2_DshkUR6qP z4&P>jjHhGNP?*3tqBH2(BaeB$5s9AN=zxaF*)|1cdgvs^$MOEp>T=CgFCkCN>Y(~q z>iSH^t4c_kEj~Y+KUkeC7Wf9MK}X4@EZj*qXrDvKY|vYh5)CegygI$*`PIdfG8;kV z(LltXZkOKyfFI5W;Sq@(V~_xfw523}8IP>1I`jyCeO*KAfNC?M<)mmjoZgkgIX1oB znhI1H=Ld$2>6ig$pYWPj$O0o7m}I9j5llGScwKqDH{CWt7;oa~pWxyQE(%egTS9az zidaDWe{o5x5Hlk@!Q@k_;-8q>VuA&vF1;sM^Z3D}+uEu34^=F zmIIR59u=$322x-o4$s;slgSmZ+oB3&KKTUTyzSop8AUoNrWUJ`6x^uS#ohJ4B<6{R z6kJmXs0(tWMU?@IdBch#0vFogT(*i_s9yVFkvXxiq~U8rHp0>7G^^_}+I?oc*aH=H zP!J!4|+uj#{T*eZ&mqxqLh{RHUNj)?Q}^SlJ~XjUwh zYG+3PMW4VjO5p;+NHKtCSRg|69SsU*Ci-j3`7Fx}Ml;jz_R@kFH`dz`IVXvn@JaQQ z_S)2{#Wf{w^E21^6TNbQ(Ktv?t9301Y8}$r=35vEGj`P>GB#Ck*5?#ww> za;{nCJAY!ZH_lk<#%UP+(41h@kXpQm_vIY5Bl^R7g5JoyCfjZXN1>fpf6tPCo1m9y zNgyc-L$U++t3WkIaqg<}RkQf5@QDV|#eF2biT-s6CVbU3C?ZzQmbG$xRSAnnyPNZj&H0vJ|K$(60Q(M|R;eG$ zGHA5LtJO6xsLDs@=ZaKA;hIyED;IVEKhBtC$X#y87`;K#d`jyegOTpKWco!G%)B5T z7RK4O8;HRAU0J_=4^!Oeh@n6kBTu*ps2u-;yVY#N40=}uf>jEgi1W}YskepEKgJA* zzWQ85^Dx9d+ABIQY=ncsTrPsP9ew#|Kt-HE$Abzr&tL*rD8GkCI;<|bJN!xFZMPV?zXF=(Ez$t+HIhzk^?nkrz61wjOn>0zuUkV5EF{4@EDYN8r!m#@JIsGJ4 zwzkjnMTBw2y3BdK5qr(wf2vJW=oMqw7j-y{Me#tMyO6@e6w2u{+9Rk*&3)!7m~)6# z*)VG(tMSc#d={wUO=jJc&@^YSagagxxk14N7P!~pkUA5}kLEofW8DoKPdwf6tHxE+ zz~1ynqxnmRYNIA=??1`!RKAd9n(=Bn7Bmt{RSSIo#?u23%hBVVQ^@vp*_+q>IEU6e z#r6ewL5(|u%W=ZS2LohrJbE)u<|@X;B%9f{R39T~+Ln+=3<$Lm9QG-9bRqFHbhtHC zJ_$37&njP^&Wi%gY)dej;(M3lxaIO`hiQj(@o|l8>~C*^NpvH0Q2)0%r1)f>1&|2E z^PjjTJSJ5zZqVF>d!onyJEm@awE8oeke+*VnbG>mSD2*?X_VwxC3I5EWSH9RM0v!A z%=nd395258#=D{W*g>4vu}aaLWQ8h>%7g<1RKx;Be^M)#f+zH}cf(%>Xih^%iY$9f zR@VUDV$=#;|^E&7mRK$-PZ|AAE#+7`Ez{qig+WLWnM;@56y!->PwdW z>nvJ;1JKph*chC~=wmk@^182dl*7be*V8|kHe|G{r)uGr=RZrjm9d28GQguF6+%52 z)52&P4b~u+0aa(EsXq=6!XVHygeO^~M&y}Mjvq8=>rcdGei<&-nj%cnE>YK{bnYFi zplhD*up>1Kw@bOwx$q@Vn~kJ-o3Ud%MealWYzi^GUEgBO3n0K0ky;`@Vb^e$DoNY* ziFvu(tiP2d%GQG{6ORmwcRkmu!E364tHHZotR~(yNnC+0QHjmPDi)$ToZt;y#ww?X zbC`l$T~ePriGU3=1NxN5D~i|~%c;5$bX=W=A(H*F*?ZB;VA$v7jVM4i$t_P^9vOnJ z-C||5`K%wCPdwZ%T(=^@Pl|iA(baHke9oNk&}(UcN5cq8aj;c#{$Ws# zKhH``dZY}TzeSLU$3+WlJQwj|s(rL(L+Tq;7zMt+J zLdU6G`RmAh=CjEpjowz5wPhGK=DO&U|9nX`IDpeYIk6xeQkLNfGS1)`^=XD7BV2C* z4{M94)sDQ<$U4SJG^#9i`Q>?X$AvNJEF8vG{J5kipB>+&% zunieOlJz;4(b~7YQPyXnU!MxRK#lNeY5wuU=oX;M*T)c+ z0uEbG3Y_+0myASgSf0&B_74xWG!gud@(;^}3+dEFPo-fnu{c|u^`@=xl+aA3Fsnwn z=_rh(-w1%qJ-0ujVmCwldcB%5zE7JzmUOjJL_=+GRt1Gh@^bYX{v_&7PPw4#zO3JH zO=3p0nd#q23|Z0MsBThA&L(QHNM^M()K=RG4IKqbhIMz|rn1?8MG2yszD81v`WtD8;T`MNyBZSYjQ5`ddaICjx*Wo zVcR;vpI({11NIUGFQ&RZV2o(HENtz*4qacdzp@M}Nx20tu40aXg2J!X48o*d)HN9umRIlnB;)BKVBBN^5$$3 zEILHIotemWV=~hl{wvxE6nErtc#+z#(-D1BsqwLtk40PzxIar3NEyPnHTwLOzIGU% zmNk7Z9#!mp#BCGN#Ov*T?=}C$T2qIp==T~-CH1ey-5BBc66W9gAy(SpZyxJRD3PX$ z?&C={v-xY+@8QZs#Tg=%*&&%2Xu{6;9r_+2AXavAwCXGbI`=?dsv0j!4EKs>shXyv?*=BTWmGvK)+C>y-P{u zU(a#WC%jWN3YQ4Nw{pOKCwyJtW8THlt|GBc@x-O5v*uv4V>0tB{OaVj^K8AUbi!H$zB)a@@%FcG%ulBxD_}W1 z6QBaV3q*bk&4XMaCiZm#kbvc`VfknvXO6(HPbffU3v`py_IknlyJq%H^I!p#(i+l= z)x}JcKcxewTRCNCR}*`Q2AKV4sqLhX$>7_OHs5i=FpH`mtosnwswDmQ$^^_6*=TZ7 zyhacI2NW%+^?B}!ga8KVZ0i2Byxa&GFT9{U07&Io3O|l4$*EXXJfR`u%0ep(d+7D> z^csE_EiVf;@ycYL=N(qC8AJHb^B-(#o2QTC>b*P0CWMtM4oV6OI$-LxCE~k15app= zYxu6U|5@r=k~s=Yj0?2U`XRcHH(0+`CewNwfGMnaM8ZcQ6iiYcc(*wa<(00%Cjqx1F|CDu#1ya$4U;?I z&>C`!^w=-u)bpFi7qcXwKT&R|r8a(4x3`=)Sv7NN?VZf}!hbNf=A6u?{FrzU>CTDd z9YT)Pe30`!Mj^6M65r3kPZ$*_b(a;em|d5A|5{M1Ij?N9S-Q|H9N;$$)|zQ3fAKkL zUBr=kZHNl>wm1L5U^dsVcC?gneg@oLCwsMV(TzxM$LX%UW_WrCHNXIt0JqnLx258V zQwlz~Szg3CX-TLSXRlw7Mi%|!5t%n3ibAWdeAR5yXf&&RD(xv0*o})V<=~1JJ5Ph5 zo<&{E${Xj{%`2MR_*}i4wgzUs8o8l*r{7SE3w?n$v|L46AA)wDOa~MovTL&DG*79|)m^5%#-m1pU%?JH0bu=im{(Q`-FG*UV}OYkshOBb2U1`yi_EJd*c^ zFSPaJ@3yM3M-2LX7FDdZxOroY75u%#@80VHa9iRcfni3@oPmr67w4|c_1+L zX27e`nqvDy+{sJxmTZfLN%5h!t9W7-T{3o@BL z9XpeDW(N((_D7O!O5ojJ|BHtbU%jFi_#{CU`|rrP{nd|nCcD+@i2LM~?&r&7_de-{ z*?o8HU-+p4ZN&v0hFg0|D`2v^wfsR^)cz!=Mhe}gpTeb(1m*}+FFgCp2tg>YVUUKh zRXn^uqh!`ChJ^Z5qqmnPet;0#c}MMMZI&|YF%o9ID>#18W0Fa_6&Vw_zHYx4A_7gV z`b_uf0n7_Bcw&e$%&0$H?b64_g@*)~M0eKSM^RL>`?o*avqtuR@GL;)VFI3W0B3OU z!KL4C>6Q^d5iunw&FGFaM zoXe%u2s2J!R<H-iCSHM4qbJ+Ns%ii z{Qh^A@@_sOs8wVKvQ8P5Ot^)VCBTkPhs0jLFdhgJ=JYE;jlI}MfTVL|Tb=&iAXPSk zE@5QActxum@V&8u=jA^0jelehKDZGfMYhIrCYLG$Qx@}Vc4$DM`gqu?1}aFQgkya> zS74_MgCsp)U&ZaiJ284kUeU7|@PtR$Cz$%|+Kn_@>ENZSHb=AXEEiLNVySu(&uKEYTpNEk80v;%`+ZtH1_EJyi@va>3pUTUIK z4#WY8aWz3RW9v?elKasKCIMVrz+ZB}5gBQ)fV1N1*BDeCJS-mA-}DNrsr={P_~8Ft zB%bBc(w$s=btuS~`0%5t^g4!(oy)@sX6WUl!vk&{$OMTpMk140UBpCDH$jf zRE<_RRN6Gk0{z9rRWpvO)89O3n~g5I0@-ty5p}%vFw&VnLt%6mt4$@Fpgpq*P}!nUKvBf z%9}EkoXS9;Q)e*@4!=mJlc~9e(>7`qov1Wb-UsdTuZiczN1o;6S-l#CE{tN8(%=$9 zlEGZ6OS_!FTU<#`OMltV4Rt07UJl-L&)kYJIA;E7n0_5g(lwHsHyG8bnP+A5 z4NuE2zAldlRT1shl4Z%FY6XB|VK2zQEobA&TMTU6kc{6+gvQVss9dr(8QvP9Xw~x8 zL?4^ORPL5@t6>l0U~y!f_f1w$bBXN)aG%xL;819-3a8Do3zlk8##>26rhSnypou4| zHd`h_;(CMv+qI11?tLF_k=E=BIFHy{myUUW)vcgNJ9A86x}c?56hC)Lv9i^AoG9i7 zmK@|JF6Q#rc)@<*5s7S!!i8^_ogjH??@Q-G*mk-LCXDGi|Ig-m_rMxqqo3mTluNU= zmZMzURTy#FZN!NNf<6!#%=<|d5Vl=a$RE}CaVo;lf^egpQF=*`SfTw5-E=tO@uJb2 zii*TD*q4u1*Tky8?`$3(D)=B(J#$NOs}>&Z67=?Xzz~X5A1Nl_5O2rNt-_MFJBvBG z-arVJ+!8cojokI4oFl&^uAg0C1q-!`YfKHNt#q6s0=YbhQF32)FB`-qid!ItaPir=4j0svxzq0 zu9eq*PDC`iXa!fB%0@$*O4?H0F!HXUZ;iM3d-|ZC`rAmaqsA$AQ0fsI80iQ1y>vV# zTIBS~q4o>l5%L6}YO9Y%M7Ita`bW2A?~@jB=K<-S+KU^HeMWn>T&|t6%AFBnU?=g0 zzU5Kn1ZNY0f9m=uq}-9#Gc@krl!U9agYoE3)(Vzt`fX2d?wZ8IHjZ`r$6+WHqvt}; z+@#^%djj(MgmQIiT7Y<=#acxV?2Q(~FpmF{S5eU8{JOwd`t1sFX@I`&i}v>w$$B`e zaAVa+S`U{nvM@NEv&BiP=%%#N@xS_@A%8b>y{n?O`bc~W?Oz`}cW)|ut@@8tOJY`k zFI{$Jc^uxT?_TxtAR}#n-XE3L62BPV~?d2t?qKOLbhQ{ zX^R9nV9*kPQ{8|J9@fx5Y%92gz5mq%ovY>Bt4X#V*0>x(VNm`GP5(HSScT1cA(dKB ztugjp#pl+5*Hb@y1iaOZ8E_-LZA*et_tD?$e@2rupaicv7g2NV^Xo zueS!VUJkqwkYlkITZJScD2m=MX@$RfAQGj^AMpuj`Tx=}(RdYZt|hwM8Hxbsp<`@NM1`b8a2P`Sv$|EWewFOrL89Lycu< zvPj;J|B7$Q3Xp)-k8;h*3O#y^puvE>hriNBt`3?Xkx)e$op7zJ9RmD-w=;V00;%k| zWm>xSHfB)#Wy;K64nE(0y{64M+fAa%lmsec*4^4~ZnO{xKPkl6oAke|be#V$<63&o zvqA_Vk+$4ktRUy0{+g2~Pr3r?xA`7uc^Q!%tBYy_kz6{6)k3cFxy3jVz+7vGZ97Fh zPbnxlggFoOkFFM6?)mXTCQjJM9jwuqmH5yNUI6{SAct@n?>kFO^Qqu;x1gZM3Pi=z zEISdqgdn;!w5oNXDQeH>NYVLM_0X0&y&TVmHmlA-Vd7q!FRe>dS~G_d13xZyVKrTd zC8V&66J5d*AcmhJ?~(+V$g;Zp8wGMyS0UE|Zw2I(l8>94bkIq_DYatBnT)__S=|bh zNMg?ciGfi%8gKXcP@=|cdzFj&{obFj%HM8NWa-8Ot_Zsap8As577-6ExE6Y7M{Eyk z6!F>@axPdQSD9pOu1=vzBK+~aj;&X}XO(LUl>~Z`aeo0lZePcwUpr|J2CCPCk4aEL z<_5rA){KV_a0$1c_Vh<)al#%?_xuSF;;@KjZLQPK|#VH6vsQx1$)IbC_2R z8D11OOqWG?7q|s9#M#KGdv0uX=UtRI3ydQx;R$C-W=7|$ffa4TtELcuX1N$`WTBw* zv>oHv0A?I#X5>{2rzK%+)nk|Mgnz1eU;hs{LoQ_f=P4Ik!KXr8icy(*+y#mfngAYK zrb0BT+29E9%?W`z|BsZi;q)SZ&RK2P>iAVI)!mkj&y(7+lb&t75m8s0*}YY?5#ZKx znTa+VBNtBm6}4muukPJv(SpQzU7393q<>orJu(P804wqd(vn{NKv0o7lN{VqPQacz0^Sl$W@Hx;i;L1hpCI8FN94J zkOa6(-MP{R{NrbRjIkgDQeRtLKRT~%!dbHKCbsV+o|6`f4nx9d-0&S+W03#6^^ z071yC>#Jxiil%7QawVil)#NkJmVHem&7&L<%|O8o6-g*;yR{l4v#Lo5QFz3M_5dRx z?WJ+?`v6rXv2EK5s@;S+sX+CZ)XKtA`f@e}xhRIw^pk2?WHR;=tW3?(YLDghkkLZF zzw@w%;c8`tx&rcoCWIQyHyP$9A6cCe7BiXV4jE#1FJq{kNPQ}Hd=$iH_hwiKR!(D9 zvyX=mOUSe)J5CgDx`rlG8U7H$<%aU7Q@ETS3=f3?yrWDJ`|Q|SgaR;a>nWFO&!Cfu zpzo;+R%BiI229>W07CJh9b1M1PtUyx$@I~#HnwgYjEl)^d~yU^p!WG*08;toumJ>o zhjoI){tH7nOT?&+Pb92c7Wr<~%lzS0RYijR9OW(nY@1H^Tq-;#nQN##b>xR-`7FzJ zk#aeJ&z2`gphH{T8}?T5z;EbkM%aUbg!z<9Nmlb)#izvQT@V(qLY;qlI7GkBhn{Jf z+^}tnA12DFcdoSfCpqsr{h*~B-+3OeS(fC8KI(A$j)FMjkR6k`f)$ME{A0m%8}m+cFRI?_@no^1?3U3`)mMVs?C#6SYE zYz_ol8r&B(v?;ptNv!`q!B7h-ZJ_z_!z6V|xI3a0<(%97ujqpYd(RQY&o>`DA1r5Z z+k7%xE}nw<`7X(Y6IjA*W7}j-ON?CT59|KR7vu?`!cJ-Vuv6^7fiI)NiiB*@ah%+ArrJ3P!myB6{dC(vW9l#^8_z1`1l^P>WdD-)+P zU4fXAC(MbTNh=mujJ9$uKIX;Q9_;oC`ew^?DCipp|5fvId zN6NQuPxk_6HBuH3IY^2+W;MwL3p;tL5ROGwodD}{Hbs~rM}++05r~Bj z?OGFgH8x*wG+BK?y!3?R>(V$YYOHar?id{?lKWz5lv&;ToEGN z6vQ)^DbKc!#HLK?MMs;9R1yXnz$U)oJQOEwPV%wF%cV1_#pc~Vv3o;F0`v6{Mp4sv z-22Pxxq^F#gtj+GW?=Li2~+V7PJ4{_BsK|ayJhm25T)ZU>O;nqDruX-qR1{3r3QL^#9k-@nO16hX^!z{J`Lu0jLc640Tg0^kMsSi zh%z`mY1^#|j;)Ok)j-+{?I_|?NOWSKoP_JtMeE!?C8r(ggPymJ#aA}ab93kt2jLL& zTK;=$uE}U}nm;G9eiqmwV2uL6uXei}aqZyXd4o-Rc}USxsG2e^!KeD6<85IJ=qx9; zb}wG3G^BC#qEZ*SHLUJ(7jq;ByABAhpTRDcS-PA3gSSuzOC0970h~JQ&WJ zd;LS#l>|68nTdI?#F2_Bj-T&9yu(Y-yA!x$m6nGXdU}vb5{Z1wQv-9v;W>pVtx>|# znM?*`9lE0-L@JD$!Kf{qD3suClF|HPe#hC{?pb8os_Hc$Ov>6K>knQbhj0?SLvNR{ z-|-6SXx@Egh2tsHLiDofsZk}Xg-VuIF>Ha8elD52vyedbeGun6)L5o)_jw9{DT z(}PE*)V>#jO2v9CA_y927iy}0CWOfMx~zrGJD;W_O&z5PVtjX5rnG#;DXbvK2q00g zrI6CD;^XAwI8kgL5sbibwv-_ucHdy+`AjmdVrQ?0S>3#Y!g<)$U;R;G?sO2!#Qhdc zsv0SvtBcJIaOwNL?qqX7H}Vmf9(@}RY1gRgvI$5_$YLIshF{?c@@)c|zLDlY_h%2hyk@FN-DK!q}gXhSve8}v53-bT< zW&G#DdxIW(`_dlsUhV;Rr5T1LnX9g7jkZ$GPs<)aKq;5T-*HSX6e#ZCCw!Pv37&5B z{Kwl!d@Ir82kIuNEPfcVoGX_DXxp3DazX7$DnLVP_7$bxIm<8W3AK=GZakV7vhq5% zXvw0YK9)LizpW9%O0O(u@%#zy{ZQEPjMsi|>rXUG}BAP?S8)kAyy5s&7 zz0>I-`XH>=tp;WcUs{dTvva}J<996J{F1lYZ<(1dG19yspmIVstJg%RT?yd^EprK7 z1uGKMdUp=(TFS{0jb1$NJ7Xs*|61PI*_gu0LyW-T^fQRMCxNkqCVj-L5DYY@s2s)l zOt8}jwJ5)4yhBAgK0l1W9!vNA+_>0obX7e687m_bL6(Pw0pD}Q!UD!z7Y2;@8=3D2 z%RW)N@BAen7~2!l$vf(x2M^>&=%>BteAuFNikwtWM!d@_6s(y!Oz8Em{;8%O%{?4M zDw=mhU`QkwMXPWq;tx>uVjX`E5qiu=5kK%N{# zYIh?3hym|*GYZs1xzRyvs(}t?ZiRLkF^8|8iL7{(SC3aIi3v?9iY5ZSe+m>upfTU` z5;CGy+~x-{Y>7kV56K4CqgNHN5m?~3vq)I?V%C=JA0=wFM&ppcwGH_JRXT!F2tVzO zD6)0$K1IA*wp;Yy&>jmA3j-E7itpNV2^1o0%}5GJT$bgN6s~;4-yyF6X2&B#^cMdNdtu=$< zugx!Bg?xlm;;&FK7I(jhQ7N3v3xyvTfRMaQY#F)aIXYG12|bHQg?$e{-;?zn-HteX zVF`~;e!(X1^}!RQBWu^46q^MnH6~w0lrwz}-;oS;yO0F91UX6`484^RZ=qYp0L|vC zM8uJ?8S%b>;9*M)y8~4G2bvYK#SvUj5$|!LLlO71rECCw0?Evi<-afPnU3CQlg{N# z-37`TKb`S_)sxj6X(_{2X$cft1%ukasL$-ZxrS^}l?AmV^HbV}_Js8jP6T^7*y?>u zt_}3&m2lk$p92P5stE91hNYHin@vRi(Ho*;oD!ZF zPGq{4T6yf9H;G8Uk&T4)a4P_#<Zg*cG}r ziP|`y?5ti7WJG#aemKAzCbIrZyppjVG4TmzQzR<>4jnwOQm;81d=Xk8h7=tAQCx(4 zTaG67U4SWt8ah`=QF)xs!u4?WEI>@D=BSvAANVlu z(?75lX`wK0dBuMXC72mVZ3zvoSTo<6-#5V}e}H?smT1J!%%mvJF>boR001vkL7Hbv z;Rr*>a=<|ehk^iu0R$`pC;x7w7qxuA@A)qSWE+c&_5Y~cN$Rz;9dz&63TTt=|Duo+ zfBSe&Fm1(HgvC54EQKoO^H(+?=c@i3oPY%WsP9g`$C(WfZ%=-W5ZQjjV1yldHOYDt z{2I%an>vT>&nxD*XRa(z{t5(Xao2l9>$ym1>_x&G&b^kIBU^vsjBu8vu5ZoDkW{2X(hmQxRvBwKFB)F9+akYIEzEvYR z1a)OWboJsY^*7O@-T$C7kU~7asAZY_0!*Ledkwod1DH?q^3GWDVXKfGQ7zBZa_KK( zgIAD_^J$`}<^v`q{khv#u~^>xe+*^B1`ijwL6A(7OFyzkeCSia3jB5=zKR~ee>Em^ zF$xq1-0m54ffe4_DmUq?4oM?^?ofuMxS##lyJ?3Z3?LC%ToOH#DYaJ^b$Dq7f5;cm z#+2ybbRxK1e{rPP0wBK1ZZ3fYOuqFvVhe!9=qJvJ-PLAF4j1HLP;OkFC~N|hyN;sD z$Hu3YJoKupwy3fN2VBRrP%G!A|72(8tHkRnF9 z_C5i?r~jNZ4FSl%-52uoPPf?AN^z3Y_e#y9d?O9b9xZaLYQuZIoc0X4o@+QtW}K0Mh> zfo5#y0Cm0*{*8pN5cWi={{B5I)ArTSsJ5IS$V)3m@F(=r${qzr;Zo&}=Q~M*N?w3) z(vu1%L64CYIEl+$`kc@b!6pzQ@Ad~;!M=g!oJkx2s%2q%Qz}-0*~tbr_h~S2A z%so-B<%BM(n@eRU(5{PB9nZ%BzKC5sI3q^)jS?_i0h~p}z-fN-&77ni;*#y0cmu_!WX*p zt^-c3-zSdWtIFLSS+-QrR7A2cYl^El@rqbR|FxV?Csd%2ecgv8M@hH>>hH|cBdRZ< z05BO)7x^c%My+X#nz6_nvsl8n1kDQ)Z~4L@b}*@wivp+IJ>Qc!tW9&l@7Nylsk0E& zO$UA>(X2EWSK*Q34G)v_zZt2L8nO6$^qS(pED?&yYI4c-e$$D0*r1Ycge(FGMT9wP zZXFDLk+*K~Z0SnTaYvks7o>U|koZ<`O=#U_or{#r=(dCLd+SNPX-g=FkCY2qST7Px zBrPe7??556GSa)3|G7cWZo40*q;j7WQf&D2m5SM-)L{^6-5co~h@`fH!3}>;SFtS> zziqnIf;pir8x?U^+rN|V#0Ws_quvA_&Zt-~h7Uz~Q)U!eo(c0eS_)^I-Q2?n3ay>9 z&I#T@2+p6w14x%?F$Uw3tG$OrZJ^&32fW66%qI)FRZtS1Z#ceZGNs5wF`fV=9t}(6 zVX$~rmn&M%f26K{e8-U_U#<)Ac`R%?;LsqezI9<)2+p0%Tf}C}N|Rw%6H#u&2g^GM z9jDghOAu$ltS(T?0h8AMaFH9nSIK5lNiD_6S*ZR>QIeeaP4wNNYcY&46ndKwU@p+S zRC%XSc;k;QWerj;#J{M};{^x216szsg~X)Xl0yoa-OP|y2J>LH&~T)E8Q5=^7)i*P zL=py_SXO!60MqMO;IERa7V=n`1e#3GCt_n4*f9vLT_j`foq0T-5#A#PTy+2j z3W`J(zuX~4%d9YJs?zpTe^^NZ5aw38d;jt^o&rUjxtk4gIi34Ir1IC8Hm0rrfOqoJ ztTC$AlsJ#4QFUszgQ?MAw#=<1M*eAeA;B2mM87=PEU6)p`8w#*(XaXmmSJA%Qy${6 zheC_>qg}RrK^&BqzBCU?TQBt+)bd7odJ^V>=6jY5 zYaBiu_U03EOtpunmzZa8NhGHn#j`DLYj_83Oy~vllulW0@47v_GUDD}02FUKjzMMc zH|Zz4y=c=or8cC1TSxBV7vm*WV;D4|#_6}1BaU{te<;2di4ZrS zt|nU0F<0w7l@EwvqdGM$7YPGC{0-WMW}*NeVs>X%k*N~=MD5vK6I_c`qRSziU{xxU z4$6~?M_Clt3>&S;T*P2i5kf|-a`{vwaxJ3|l6l7PY=jTyQSa+jb&vWQokKW1FfYd-nUuHkP!{f-`HPkRYj1|S=lbN;$1ip0&u z^*;}o!0**fz`qf|GI3GlQ&_K?-JoVplLe0jEEnSIM`zxm`Hks^&MLaV}V> zemtOyD+r?1*0=s6JSTS_W zG_t87-D}AvrIkeQizN@^$%0A-@&j=&pF+5PPn2Ng+dS!&EzcmiDyx~GDh}Y3>@Zkj z`m}`-qY}TWje!s=n}^hfbB%Mvyn(gi)0Q)t&9^mRFt3H;VL%vy^Grfx7?Pb2Mjmn2 zDmFgH;D>?w8Sj1wT!i9NYkI)cD3sUMsuw4;M6%$IgwY-Wa)1+V!QdYxu4Pf=&zWL| z?fuz}>e{S*+`OC%#XYc-`Z#eLDIqEFwb0M`OEtzaG;*>L!eNRG8zT1G1=aQC!e&-cz+@gWX1{u2L4iG{LeED52i9}u5>CLAWJ#>r+8kiYqi(Eruq48%pk zCDWhEU9*uh0WM-w!ymqS`*61E0vSfJ3SqpS<4GK!k8K8bgsYx-$9DeA_0w@qn$5o8 z3X;BxZY#z5`f2@2>+r!P#IYr3kmO=<;hfh4pFFN0hkFw~ZUGmNvqYELx-GIh;f$g0 zW)f+yl|}y!(ZK^_G9y#*Kk@7Lb;!IN`p|%)_K_l;tCqz zTz2-=ZEsI(QcZdt~%$7NY&(MLfWUp%T6}x4Q5kHqUN52 z95C;z$qT&9t76z$82AsQ1w?}~#BcTUtecrC))=;3A-t8lh4Or^O=yUHk!r^Lr13%1G-C$H?7mpmE5iAE z*bPUcAff{}aT~nXQ* z?`FN^uZ0?kanwi6@`_h(qMxH8B`RV(hSIuH?@iWqDD|YWR+|!Yf3(bci1r{v)Qe%( z_@A~)X+~=Hg0@nqkKkwjrc1|zc^!EFRm=_1a8_{wlis^!ak-wERG$NppK~{La`D*_ z?~7FUmaDLf!30tS@q0mgPemU++4^9!Cv z;P?ttD7Jbte&V%!xL5kP`$uCe6o8reKp1E??^#dtKvY?Dgi!*&RFAZ2K~Ahw5`s?s zZPp0EeB#n?m6E(*y(|!(f@9aqczNuecMvYU8X|rSn#={IRhnFz9q=2P;12kS%PGhX zYtp?HtSq7TAJ`i4;n>-u?)`$f6{b;D9za-Bfz*C3{?NUO)!g2Q|ALH9pD3&|>Zarz zb_oXW=dWO}KfKxqtURQ{n%o5g6aHJoTll=cTF&}SuF^B1r)Cm{1W&O1RZ;h+BF#4`)##sPQN}+`%KBEf zxOi0M6JwGRdT+Wz=YqsM@J^q(ojx-Qkmr*z@91V*mE2VNiTy z0j~reMw+rwQ8ubZ6THozb20M@T<1Pg+5UCSa>o0oTV`u(lF4(r=yd{6{Sjn&{PghN zlTI1fyNCDtKE>!g#72oV#V$U)gj)Vw$n`-nJ2RYYVfmK+R zwqZgz*bykXRvpkC$Kir;wf_AOyVT@}FJSTv$#gjdE4lk_CvBx6a6m?_pg5x)6U5ZJ zE;G+;-G)b$>%GigD8hqrM86jU^9HKzTMYJA@tc(w@vl*lQ-kGU9UW2FHGLtCK%`Pa z^IMDF*p&EN&_*3;$wU{km|Sf3MLaI5geWAkK931bP?W3vum;ZXRLeeEDp3M_&Wo>rXyEDI*FMCtM zG^{9O@H$q|oIy8Dui`AZ){VBMcw>1Ul#d_?U>NI)9rsqvk^ZVWcb5&^xrfq)_t*UH zZU>mDl8~>P8~spjGradIRgap;M1#Lqvhr{N(x>fUdPb8l_WxF?asR#>R6sN-oY1dk zQ@}%9dvfDH>6HoT`GM+=OB3#M?^{<<^_;qULDNP{X&;kq>d~U;@8t{DS+78FC@nBH zZ|T~!`R0SBr2KuKFQO-JNNde#WGLp5{h-ZSCKZ~QKO&PGA)9$<48*zvRo5KKwuRD` znaAkNgzt%yxsQ9u7{9iNA!dB*Bsu4&w-aQD4kDaG3Dr=@Vv}yi+*dyJlu?A_?A7a* zc3u@1mz;kGxe3iwRehF^<5~DVVdWYhdQjUEOkWMrQV;0N&=>zT-@XaXth3TN3UMTC zH>}G3>KHuLHolEe^&P(5s1jg%TvXnnqNr}&)vsl}+h29ush-zAa+DES+<^fO%H+OL zeg==B2hlS^bw>yw>`x||=>_k+-#T{^>4N;7kfiJ|#Y zC06v~K4%Y35ZElE5Zvz`E7uiQQL%+o|{ZdsQ&OoxTz0QSjAp zm^EHhD*T;pCPA@B^BkN=pSX{Mizk2s_$9%FJ+ zcYtlKa`+%sF$0(zCH2ELGF+)sK|1j)|NYD+PfMeU!{a2!|HZ*w+GeLVsPcMzYtwlQ zh-1`DN4XpChdtK~h^sOWM^@Vwd71=kw$Ut^G|6BPXPJU&OP5_XKAb8#1quQ9{J&M` zBdh^=Q?N;aKW>QoMTEFrvplzde4Dj|+J{=pz@5;SF%}8E5RS1uuZVF)_;KHogL+}cj2XXMBU?jkozCCWurD>{B76M@n@ zOW@tQ!b@8ipgasH7_pRXcI7sjE=0^h!7}byB0uyxs;P+&Q4iE#A4k@RxJ z$k73TRdZ~ZU8vuG_yM#67aW!E3Bpo zktwDLX4NP*v&_#)O0x& zPVY!ie&Uu&elEcPRb(@d%U^=5|7W$+xss(=P@?7>LK64ML-Kon+b2zdbgji|ZUq$L zH*p{U&8g_CqprqKRA$MUi)I_N#({1&@=5r2F455{40y6tI{C;>B%wXQjBh_nRID4T zog+FtLB%Y3WX9&q>%%owus2Qfgi$EePj+Ptn4Ar{0PGayp@fRvkmDsl`QKKA&QpNa z*br#rfGK*V)0~$2xZS$}!-&eRf|cBELwe3?#YS zPulQ z`21v`Z@XgWrMO4l9)>*}QCi?DuUIVj3p{|S8E*+@N47>(rEQID2=}ES6ded6$qOnG zUZqb|u2O65xbyHV%PR_jIXp&GqF~UWDeydUrOWr7@W z?>tZcP^}P0i^aqngJCXm(vm;tNzromFo2bL{!g4<(%W3&a#x;kY@PB9p4LY@xmN$x z_2sbHC0HStY+yK1Ul_N5y46cdixn#P@JhSP6kNYEL6kSlJ5@F#=8*wEyM|B=z#Jzi zXpi~>K=`xweGQqNhw@f@yTn*Pwg!55@`mF(qK#G&;lP_XnZS9aLu3RG9!0zjvR$?9 z54cj`RnHD_a%rvMSeH97NAxbqSiTa-*|9!@@s+D0FU4k7Brgiwp#fR4F1r(rLA8k3 zjmd0DKmS4o{UvAz&zzMq(CMoO?|X1Xd}eYC6p5ds3d3e4G2K7MUBT1p0enlTk+-eH z!|{)IW(|li%!yl_V6ag*Fh9+gt0u54Lc#ZPuYG;2p3@ulT#SMbxMC9FC`TKs z$~S50HjJn9w&iknwM*d40x3#?P@E%ycn}EqSog!mJUXBY;9I{pm7#8M;#Au<2f(Lj z{K&}&ZB=lG>vx<_%QWQ$M7o*K$xGsotDgt3LGAF@4G@WgJx$x&ft23q301AqsZoH? z7(}%{;rdd;(E2ZUo6Y$;G^EX6%;tIk1hl*%KA$nG<3tvqt-En8e)Mh=?^tUOrq&iA zu$1wgUWG(%9h;pYhNV?yXgw-KLb5{8pTi;TdM52;z5Ukua&f$3rVt6dpP?vW0iCD* zMPMqpPPL88U<$K=U9$GI{#&mud5v|@u-jup*f+wFB((f8E%#50R4>eQO%QU|L?mOE z@r>9Mfxgw&GoeKO8)i^Hy42Iydcg8$od?CTN=p7;{f?A)^j)fq2N1$T7>L6SqA&bu zKfChKgOEROF3tSAL*55EmwjKg9*8e>bt3HXN#i$O0!wWmIk#R|;Ejzf`7%LD zgUVtbR-fOXnI@x?#l_2|eoGDT3y8)XCfhp;(kU8QES3D5L=zo~)nKU`t;iLKG|t9 zrx69T!K6)0;0=N&Hwn~>*i^Kq73LSJ6Rv6lIiWkrzm0NGyxQ`lfW5CSlF+-L<*FFF z0g9@08vHU)2p=W#Ka0-&iQm?DOvHZ`$3_WKWUdCQWgid&DoH+?pHbh~-CZ5}@lJM( z4B61smr=MsfaDt%#k}P(Hfdxfv*NE%$WD~-9_PO+**{CCnPIxT9lBuv=;EHmlUUs_ z0vISOZo#@%uPo)F{Q2+?h5A>nHxY!goka;$o1k2qRwiLS4uqgMmXORqm*xVGNBpny zj_@;$c{%rKwaPGXgssVz5l%N7adG%t_6o_1!q};ikji9~(DtyWEp|yd++BhP19Aj^ z&>pii`iIZvuN|LkS!8iWa#VX))iUaUmqRJ5(MK*v?0QJMFk6&+TytE)i{E7jbPen# zmVj5U7`w0+RLjLq8kRmipDSH`flySS#CLLCfY&`_r->i{{PkYA<6*L5n;3mP|AKUr z;>dhd1)2I+Sv0u@hPd^R=J8siR>=8g!(zpWjO?6;4@%B_R2<75+?YLMJCfNxRYgiQ;|tIjBX+Ubg2Fb4z4;$mJ^E z8XgvXQG?>0uYXjOue5YX4PlTy0%rJAuR)jsuhgs!(v#?e86H=5MpzkH>>K$3|HEy& zDkW`|ZxTc*{aza3DrnfNIBAWiC)T;&*cTu!3_3;XUKH@lvMx0&+_Lw%o8`X#sM=nT zEV`>9F;lfTUcB=1qwk`&&+(4VM?h>BGYJO`E;1+0;ND;VDFU|j8q8$#b*-?`|sV7e&o_+qLC`wV2%7sGNfmGCLA^5k@?oI5X8ktORfeaA>#iV%(Z~5*y7|AP=&l zw1d`wsWegzuuEanC6mZvv)eH z&6?ykvbC~m48}R$ z`)bwmItClI*Yfe}uKUfB@qa>1CM&%TnVrrZgvy{qM|4BDto7MVipnA3rnXl@B}Q~f z)_O<1;2;QkPU-yCIk;*oLSL|HtWn}4JOOigDql0RQFACP1(hL;O;{7hcE?U&Sh4xQ zn@s%J*NaKkayu?4^)QG<1_t6hz_}Ypu&vv5xemqfy8c-AkWYJZbo>4Qce%-*TKS|M zeD4(ht-SL(kCfkMq*dhAC=G{v2ZAl`@A0FD@<49?aG%~!Jj96B zUl~V!0uax!+1?mIQL$9E)wROp%Qk$BvIe8;j``yVcu{gXhW#gd#e7e9lQ{|K26xz1 z87c@s64})AOr0q)LHY)@ukM|U$1{B}4o){v1ZNSa>6O5V9~JV#a+1W17}GVk#DOtF z;Cu;mcWi)cdJCb%InbTY_#M<7h z&li_VT}F-F(t3nSN!+ptu>CRRU>R_l65d zAm|}!!qRBI!3d^lC46vLo7~~atZ)LJrzj6>g3;LFP*+>tcH*H9k>`g7kOE*sf5%S# zTW$L03u$Yj)u|t>m(kdEkBFSb*7YJu)Na zqQm<&nM`)#FJQnAA=CU^=VPZ`R@JHYRMJjl93;~x~@>VR)MNjuE@AGwJw$0emRUkZSFqEyWV;A z9*uk-Myzm#KxPpes{F3+RjXkm0C2oJlzwD@ZnT{2An&W1%qY8Dzk|uh$(w=xrtIAA zDAKAH4uW$f$5to4wF{!4zm#kX%06#$+ZQYeP#UlJsqp7VwQin-b10wsqks{0ZK5cc8gi{B zw+7wZo_0wS-bBO`g=KuEl@M9IP1V*iLV48y9`X#Od=zKGk{G4LpiFBrYvLle|1@iZ z(w~~nUZZpaN#`Sb4n=;lzni$1C%JAM7Kk~HZ=y9Xw;B2nHl&g!)8G;fJ2k~c`seSU zRn-{fRO|?Al_DyIIRp+36XhKK05d?$zxF9r-5K08!tL^Sp~MrPL$+qobJ45@ip!?f z{ME;8kqN7%D6BUwL4QNcQaC}fLuJwsHH0ZM^mN&4geth$t^ zYWBm8NH(cZl|D1koAGPp%2-g32Ar#}Ma-hm7Rrzk@UvEtXd5#i(Q_+Z>0{%1&r6## zKI-=_7CTeFcE5XUWx#L9^k7C9%0Q6A%#cI<_DB2Wu9)gc3@%jwkYxVTre4P-Je^A{ zbPN~d%$h3h@c+h$WmBe|NPgn(qxpg863e>BUe+yHpM_~P9Dli5ox|f@Pz=|+3ly49 z>z&MdX>dsMRO97^S?`rUpEYm0igMd$u8({NJ<}>X_xejc6kZ1tr8;cGqcD^3w>rB0cX5g1VnN0!%5RaPFY+wmaG={DWMLHdMdQ@vDZj)7ss&}<|B;Uy? zTPd2qS{V;mrM}Kg@)Af7DMTut)a%$YX~YsGWn>`(v2lFaj|mvLiLuL^T@SVLMXD}A zg7#UYU*vy&2WU}GWbUR^B`B9OjBTG(07XBlO^Gk^Ojf1 zj|~IVkl}pcS=hyn8_wepPJuQ1b6VWe6)$M+JDwO1CYlK957?F2ZBrMi{f&HFus_#y zdQ_sNDooxhV~S}2e_2h~0nq$GD?{~5m8jZR&a&pGO21O#ds0l*VR^lKzns-dOXArW!5?e0+cZYje7>E0lLY2HnJ-;ad$)tK3 z2s1Y&yy4~d3<^cgRsBD5Dt+(0(AuEEkGNL@eO>HEbV=uuj-`xrKs;CQee{i5<2@7w zjEA<-A0g1y_QdQ3wGp$aUmH+LjDPq)|GtZUv@bb+aZOzR4>bGDPX_pVw|W(V*Y|Tq zhX1RIc)SwMioC)*QumqiO9YH)pY!bWC8`JpSGhBJ87g}8B2NKzmC@5rkA)5c9jtvO zHzZHO#4*^sw$01+@l}WYnb$Q5DdU#{Ev$UlgXE-#HnH`vQ=3WQRS+4m_l z?cr~(NZm~Q2^TaMzg{&Z5=_J+eNaiK=cu~e_`X-P7X|+gyMws18g};7@9_Hm4^uB5 zHvdtBQjevpBS9=)N%zkv;doE9Sw|q)T-BD3X28Tj!84kr1v*6Nl6^#I*w~4+lu4A5 zkObbuBseM@nEN-@juQ=%I}yJePd$ueM2AQ#-+it3!?Y zK1vY}%jonZ=!7jjWu^V=s9Ny^Y<%31S-12Vd8BA`vSi2>6Z#H+&-11ty3b7*yFloB zM*&po3%^nbs*%sthxw+?{{$@laXxYE!le}>A-ElJZyz$>D!-T+CcaY9Y zrXDn?#IpGSv~C{pEbpFgIR4?jb{O%5ImmWa_Njq5!(XIX&CO=G3ZXtsn{^ur3XJE^ zRmwe141f!{VhbC*dQ5NZW+yuoTg2DEVvf-$8`Y@|UfAaoJ*w;7Wl!$UZx!_|1wfcZ zN-#1Sdx9A`Wq}Fvh~ah-xRCe-?X4VK&$Oll_BMXP&36 za7-+zA9|?yj4^E)$C0R55%xDPpvw&^r1bI{bXkE@wJG?&2F_@%{(n; z8)Uyfrvr%-KINirsTovl>@r=@kIp&xuZWm{rHkWPOZNKbKm2BBcj$WVVT;FnQ|w4*2Y4ri@BP_;Mq*zd*vZr&$s z4E!#3sz_B+gAu%MbDU5Wz2a0=Gi#+I#TC1;=!dEBWGL4&crf-v}Oz6Xn`>8E&mp!<7-C#wm*mX;#A1%Lu=!aU|^ z#jnGXqHPNP+8tDYdM!}FmbLBXY5unag?0&Ne*XTL6_5Uv>zCO7u{po zH0?fXZb~zjm?7QC!K6heQK(a!RD?#w5DC_97iiSy@f&9XDIpR=SgE|FPiikX2>Z#+ zuK)l9EkT}zMG-vzg7bA#;Z!(*T=vyjhg!bI%iQJfTkN_3Ry?#H6O$f%Df$m?r~=?y zVvtG4SciR9SF}S64CCwKnTTgFSgv+j;`yW;i^ZstKyg$D$L|jU)TF#XyTXFm%l}6B zt;tOE_kieCWs*6fjeEjk`uCgMTC5+8ZukO%vQG#|xYtpUi*Zd-) zJb%$s`x;_}O=B2lmZan41TWO|E5{Jb?SoO<3^lvsVDAU{HD`>@?tS2bjq)$hfG>A7 zg_}Rs92|KOoeo{iNII=-N+G3&8kvWpS=27$MYOJB$s2Y;%LAuaWJVyS8jgav%uY4d zXApFiTho2bHjOfnpr%3^#_j~zez#J|>g0d|-RBPWQSY_JA98IEbVbi*$FYcln(h=K zSI(qW_)JvLfE#ISRLOZ17iQ|;M%c*BrL;ohTq%R9!N44|2utPapmY_;oD9Kp{`Tm+ zEcMhSPPU>~N-tO3-=XDw*COrObF)==NGU^TpNMF+LTE%U`Lt}nXywxljnM{RyX+Pi z_KJ(I$LNPs)mzBFl8PnkPw~PF<$n}3L$Kun!A@Euy^NT8OKdTUiRej-qLgJ-WiH4! z&KS!&8(EwU_=)A@aF~>i`?~b3@ zzXiig5)?J1fp8qnT05GDCxwNZ^%l)d9h!j_qv*7}p>3{X_|>RFUaubNdrk+W0OPWZ zRbK5vQk^Y3C^f+_=t#_RN?@AEHuXP5LQ>{r?3L)u}CMN9t&wY zKjb3d9oGLrt;fz4{{F}CbuVjpkT)bCH&!(aq4cN0I=;aAvV@D4jP0#_J=KMpX}o|x zvBQgJk0J!%^1Oa*d9k>lCxErBBcjQz_ez5eOm#uL_Z>6VN-czEiO{L=rEzc7mm3MH zgA0H8DCHlmR{oyJBc_I*3hidD@xAAhIt;DQtf7@`vVF| zUrzw^9qS&D>ZcW#@1vzm$!Cm<>EywwN{4Xx%L-hb=NvDg!jP!Xk-!#uNkVivy^EoFhkJiOevKBI}hiq_Wjy@@ewPtbHh=)NHW<-r7OIExQ;)B zIzK~r-eowEd?(31WqCu0_jK6G>;zK{ffP1b-^ez+VMhP}0#^Z^q;yaJEyj8$$qjtx zYRTc0@z8$E#e6!}6srDPt-#i>>#Ie?H`>jj7v~IXh?Rxm19dQeo}#p2&Y7X!0q7Bu z#)6zZ`-gEtR-$&OuKAl;QM#vc!Edtdh1lggD+0U4MpT&G>Holv3c zUuXO0?d|u5L_#^9{D1YVyYlK1J|pKh%LZ+ z?3*PYImWe><=Zj!FpcvSWLHlwplOPx=~Xz=84 zRwc3G|1=|JN*F<-ApigazyY47YDE9_J68QL9`EJXxp{Wj1sOG?bTn*YMsv#%Ze-YcG_+_eG&AnY%h+gJ{ctbp$%EqLE40>W^M=XRWS~z6VDTesNndsIZ z*i$^aB|ZOhNgsA8_{MPkl+*p9(UcEB<_&DDSsRs9RA)40IZ5-Q!u1FU=c^KXK5WnQ zYb4e0IyCGSW*j2m8LjbS3e}$QV@b_DTXMqE{k=w>KP_*ka89Yo0}$-iWmO@qpl};R zuR&LK8W;iiLaf*WN!e(Dc4iAMDZbm{Z~f)mRWx6Ukg4-#=LM19EQZeYK7Lu~7)_Qk zeoy~wiBoI{KW*zaYrX)7Yt9m+L&GVF*o)|B-@ zpPn73+VP45qPo#(fy+BAm$`_DXhQ}6Iz#vzp@Xup)Bk=bofWgUINpe?&Pi&A+>Y|^ zte#dgQ;Vj~q+HPqclPgrq+~*kXb5n=M5?$ru-LfSvq0n1rZ9~Xzo;Tw;4NtSRqc+$ zeCU0kcO@6ghmRJ z9AAv)9g*oCyxmZ&Lt$t9md)OK#5qYO}U`r8|nrda;5vss4LnX!jx zj+lWyl<qLewC)Q`9N&%tm zJhK~yoMkMVM>sN;DRZ8}UfB#)u#`DEdx}s}audx5gqEZAtx+Ps5YdWHE$^_T0Hm$E zE{0|(chFx@@;H)l=XP+`y7{9!yd@A)N@$!tFaHwTt0|Pv(#0r|e zu~<_H=WZ==K~lH|TTTt!wogeDoyU)a6b^|Oyl7|{b3q}*b!j7CWJZGhu-J~hXKR;L za)Lz2-?!`hOG*AoL-XI%l(#R9w1TI;`7BbF8+{APS5}8-=c`zup=hKGVO3#0%H~uvj6}t(?Oc6OQ=E+ zS|RWdML6^wDOGLdVwFU{IDY@;NS*&tOLr*K~L0;hE=E5t5!qwIG)xS zg3{>t)GDj94ph*Xaq)3Ha&<=&-F@C1)8nsAEs?=z7S6pAEfO5X0)wAKz~hC>Z(O9* zN+bY@<|r5Y^#o95s-Hl@NVVLuN?*kzWB$#Vk2Zoim%97A=sw*U<_c-%Ck|z=%cQDk zv#ubIwFMXYuvC164kXgD;R$nzR)fwCT;aIoN6*HVL6&~W1E$^IkG|0%p*Z%bkAmx0 zhuD^VLmI2kvO%rC4EYt>tQ-F!4#IB!10LN&O3lekc>j5dZxKpX9s(jGqMQe^Uy@?? zb@kDu{$?7)^wb%ID7sx$S{~$NYBK^?p$d>TvbKuT!2+TpA$yhbZvWk-h+02*f+`HD@UkmOt@^5#=I@r6;bnO$uA9G+xbRpuGxy*J~bVxZjVa`s493wv;^!f|aY&W5*Ed z?VXmi&yNzgE?t92ABd*+kcqodCdC#k-s6W-dIT!cz1;r8Qt7X4Z7h^(szsfFq_khlF}&KhHymt(Gf zy%Em>ac{ST3vI>m#|TxAP#n`7yo!i+MOyAH|mT0i~9VOiSU54EN|7Hvc0ca;l0UPjGY4E z9Cmv4U%R*?J2eZ~YuZ}Tr>0r zm{d}pyaC?(4{)^iQP8WhzzEdTx>mNabs7x3Pc{4I3anA#H10+ku!W|iRj^Jy+K`v)Gx?YkIYk~0=>SqDM*3T4CLe=xrx zno3~fH6~8+S4=bct-HC4UOmLQ7~__C))sj*?fSne<8gVjjf=MaTL31|3v`k$18MBt zBy8`^kOT>1udQ5hqxOUN=04PKF7(W@V8yn-Mn{f+j?GKUf;zM^u$P|kSSjgi15y-W zmPLpDnEGB!GCdZUaHt$o2p1z?2v@xELnK3`L5?WfA=`wb+4K3o5IO8T{o>EFCY(8? zK-(_p5Tj#_uf>M+j0Wpa`S%oAH^Cglfot$z2~>k@x)Npzgy{8Tj<(~j!wzubPTsbt z>QPqKRp|q5@}NqZFfMIxugSQM686wJS{-{>dm&tg_Bd09xpX_xzrv^8`~0jfGw%Vb zoMNg9If<|A+udkLg+H0?`_`CWTr@p1uCw$Z={Zo=BTD_(T<2^iU`vyG-v+vBp_$DH zNCadF5^WL7MVjADOgCbFu7dC%ls1x052bv#T`8g*(_6AE<{2qUo>%78!S{%!`VgMm zSw1h40)eDY?f)U#&_-9x;R973ONFV?jSEQXlKf{q<~%}q*uh|qXw;`Lw94=i|u-VOP{!T&m$QdQ}-LC4A|3jQH3<(J#}`{%Hk53 zMfGl$b42w~{sU6F%~ph7Kup$$Ukni(7W5ul0fK3zkT&`7#M2IFX~%l-`0ERFO2wQN zpmjW3ForC|w|1IF=A)MN?{T>gRc^)-Sd@IN1kt7TWn7slg8}-dLYjO^IQ@M&FK5qD zb$G^|cu}nSR#eQHD?XPGOKT1vb=meKqlfH%8Zu;Hsh=#u_CH>_j+9^H8D*;yFd^6%qdIb3138~@*bZ8o2Lt1+ei&P64KAv| zb}5sxi!1lH)IR)B&JcYUAT(G3`keOv^MVCr`qn{z^E(|pVf=p%zjAQ!l7A2~JIzeU z%X3nDq-;`!s^N2n-MIRwUUl*YvZj!sO#4JXO}9mB0T6$WUJqw%z8Y1)@&5 z^xj80M}FRK=os#Q$L^Wu-w=4OhOOGKxa7wDh1@KdIZ2x#vo0A1DOV1f?8v z4A(((K;zD&rz|@(EY~^k9P{v%YKj>VZx%FF`>Lf|k~xrtfnGM-DbKfyGAs(Kv5!#4 zlPo}X+*&C?vwZ^atl>eMG9v}mZcl|lI*0BN+VoU zvbgA#HkNPdq5FiMdh(Z;iVkSNUGv@+#DQ^(Ud6YvwFXA(4yxB-fhyk*!Iue7q*`V9 z9w2|_ZG#?Ak_Dg>p>h-(A{78n=v_45J8=nL(djQj2=oOxYKPC82%E&KOfVKCp3@1- zcbH-^^`Nq^e$0M%$>C42K)Ly>-{~4O2^dDHlru$rw<0?%+#27=RJ8)wGWG9M^@zwG z_$Naq#)x_4?y7rav!F^^w}846koSe<-|?@5GHQdD4wu#FV9(|bDXs@p5?L2A5my*|EUQbgyZb$e_Ix~$29YOtj<0Q3KUC7is^ZQ|2!F3{QkWbd zQy*rHZA`B^KnGykBO6Z#Z6p2;vv9fo+Fu4~x$7{ReD43|MBo?L)29@i#-t*q`N@`t z+P(F2UaJ#Her|~(J66wl(pgzdL$40LW?AnX9dPH=D!03OM;BT(vggF=^}(`}t7e-p zd1N2%qJ~{LK>6B&3Bfg^R$+eN4snk-#x)>ie!h#LvE6`!f<{u3lwlzpnyt*}=Q|YWi0%RJy^(WW;mvMj zh*rbBmIm&a0@fP_`7;0Dlp5n;>U!TkciZDa=21%qwfcQp!?!_r(PoFXkekrSbtt1n zmLOm8gGU}|$`rAq<6q8k-=jNK1Q3v7u5Lw1>N0*9;vTVioZ-csOGd1X7g$qVf8kx3W#g_28wPnrX%;m0#6(6rs^EB;Z=*4XteS6}_oL+?pKBaQ5eNWcN2v?vaT$@>KMXNh#USBRqB`^wSy zsQAyk=$9(Biv*H+Farr;Vi>blt<-S}X5h-G->Ha0Sdry1$is|AWgYisLJrvRK!qG0 z40@N4m_>Kxb16X_5Gi>w4#QX3wR(m~bX(w6f}7jR*0PFLWq_;4Sm}O56+Ekv?)|V} zrwH?Aahm;>4GKtXkY#+5`;W$LY$wnv)$HM5p~wS&e!M_3;%?MNL3K)B$fNr0ZbT=M zPmcK>uehI@!{?Sw($i$?V5$CysQy#48*Fd@08&cpZ>Hz?dD^Ds*`-Vr?K9d#t=Rwx zJh>v8lw(cWT}6Lj0mpV~=h+*xS!pRp^um>kxHN!pvfj&Lik3)h{$-@J29faFho zykBU^>=>`R-QyODHr95c$DGt|G=Q6mUb);nOBm-lEwRu&2O)K!+7=_hNQib=D1Q-F z+>Y>+E{T<6aKgVHgkFCEKO-{g^&p>IN90w}kOw{b>L@18Z(sypHUDI2 zyb~us?TG*6BoXW4Mlie)>t*GC;cN;IY4913W(^F#t?aClZRtE2)_x@9dIQ=&kn(hL zm1L#>H3^<|U>~hT2o=hC(n_@h*j2qGdy)_b5N0)g3v*-UF7@s&iJ|!(UX^c2lwLP* zXPQH$L4}wergT#A4~M~#stPqTh#|paCER{pmyq1L$V5_S z0R6&Gl#=Z7#-)S2K)A4_4UiqVdJ6MQcy8BkTBJv3oX)pP?M%jA=g#*j=4d>^-cb#E zkpENfy!Gp^PJ-{aod6kEQeCJgt(ysJ%9G@-?60vBvvJKB6x;-9-RcQ0mS!T~>$}P3 zjVu5YH2x0vAHvySbm-V4t{FfxGN&Y*Gt~N+O(M6XszrDEH_)V*U!Tz~DdiwU*Pm&V z!h!p+;v~2Sa6Qiyd>Q&Om4sI<@^Mgg5t!VA&47sYxwrjbDB&{EoU^w%cocq6^Yxpe zhUxXYBokUo_hNRp%yqbg5qL6SBSN~!rD*i(AG2O(>pH=~?TkkO383r;4rgbE^JOc} zVD~;xDg7G64p37!uVwvJ^f8d6J9ZlsPDdb#JD_`L^BYA5M2wmu_ke_82lrvqWl;b) z1g*j)``q4FV?oDEpenpd*oc|gi` zplw5)cfCjrd=ZtdKBfSn5aZ-2CsQ2;S7EiYuRP7{1bT`L*~3{m*3~hIMPp5Z+Jl}5 zfu3G1m+EOVJuMAu9X%bZWur5FB*cg;`iZF@REUSzfpH~djp)e*vA;870;{=i#B<&VpZPQEn7$3%JB9K+i zo&)}pXAoNIqfn`|B;YUiA&nGmV7FC=9`}^v#qxuZcvv3gf^_tCeCc&w2^_G9eUD(P zn6ih{h^%(|(#=rX=)R`WShm@bR2dP3T6x#St-hZt83Onmgh9*Hp>PL?fBc_w9!mCh ze+g&K6D>gBDG@5!iEcaaXd_4sty<9E$=(kUk{ayaCCW|ZG&&scfE=Kn0vii+XM%(i z(6Ga7cY~P}loly74OGQF;WZoO3677~Qee6xq=?^9Q$^!9tdJ9DFg;;9Pf>7-iL{Jg zakcv89fTv}n&86YC@0(m?O+;7KOy%Y*LvWCyefp!X`kW@%ukoxuHuy_3}A|w6HL%> zDobxV`Ri6!#PR?MUzf9RpXPr~|FB|**V#4pQwfPoAcV{wkf9({rTN}(g@_j6}l{-c^1d7xp#X44xtTGtHT zWgNxZ`vxyg1)dDZ|Ga+$e*<~5JjURcRM93Iecx; ziZ<0>{o9+A0Pu=G|4Q1-GPW!eV(DRQqoA;%LIUIYO<5SGG9ROuAzu_9$ z=Zj;Ix-ZD`3qt@5TE~iGY`Vzfkydo+6C2Zm{+HNpEJu=RC(?S}-AqVBd2?B>nqi?eOI{ImI2#naD9Dwx{W{p*eDnEt-dRcTqb-7PRjxLh z-p1af1_J7Q2I>TBX#z%Z1fIA70)|^ARfN>|q6uGtv05GB9G%wkuqzmb$aLDS7F6Jc z%n{U@PiUdz^w4m1wRGBn2ACpMka}W&RzSygrfd|+_OvWuf3bPWsAuy#?|Cv+Uy z#0m%p@&-0E7ORsjjmFf4lA6uKHh9}&VN_y-67p@gES)6p;tT!En{LtdS6Zr(!1slu zmvg#xVG~)80^|Ob8K#`RX^vR+qR1w@#vgE^x%@Plh{E2RpK z4<>yVjl$y0h(0ZOk%V~LtJ&c3y%W929tC-+-%T;x+d)w8hJckquV<4YjTkQBR-B2P z?=mY@GO~ql;M<93&&L21is!3k6b>2xgoPhfNW8Wr)=X+KCA{@?NZ%AtU_tL}NYDV*Hcl+fm*?Q2J zzvtTv76Xtc*Trgauu_+Vqz|K8~Wp_1dX|T!KJ5kos&+T_y#3Do~~iF@#Or6A<(-7N$?y2Ne+}; zgKgNi<;CwL*K!bLnnuz~FBK^D@K&>29l=Aq094Nlt5b+x2TdkbrzB*Lx`^Ttm1RwC=2L6qRB` zM(Qxjs#*n|4C#HVUb71eSlyM5Hu!CL)y^-uccQz_;3|<}XYH(|@SOkeE_NwZ{ppTZ z?NNda{EXoLp<>u=W$P>fS;#F6+YyB>5HAlt)4 zyL$t@{$`LV7>4x1FuPOjw}W+YL2> z+g7)Dws`U`e%Z|aq*=mDObY-le6QH7@q{**zvqN z=5{!vnCy0W@FBP>DnRFY=nrkon3cJ8=aYpwQ~E;I-nGqB;6}J?pi^KN>Dg7?dlr!_ z&5~=6uz{Ss4Vqm}t^J+Mqx^16k3`-W6saOdEw;TEm>FlaYcii#A&1NMXFT-@w>2`N41`r#D+$k}!z z(^vi)^j3moXd(9w<$KGn1vyx{4{rYl+F_wr#tCJM9>d?bhZu5Y|US@lg(b&+A| zp+NGh3bdC!Aa*0fEt=-38-NnO62X$)F#3EEr?+olF%k;dhElcUjR!#gX!TF0ac4fUe#UfCBb| zkXq}mXf2fPk4G2QL$z9bfqKOY&6x{S3mm&rr1Z(A+wBhm8KeS?^NUY+ZOW{2hK#KuJsOK`hk2CSaRlH* zuAgKT?kC5J*t{+#w&W+q7m~IQUj}VJ52$wR2JK$tWm@GU&K>qePQYS0NdtHT zOicVWbSSYRGTvytv|Cv17AUc>FbWz_v@U`8JZU1;2H$BU&JWS79zcy*W%NnIBav>`ll9!f=6)y*u0{roI;YaqG@qNu$EE5pLq|P=}a8 zEO_A*nkO7g?Rv@)Qvrofhu>U$@&BUP=X9YaTflSQFfN20ROwIO@%|7226S8M#wdeV z=vN6&?G#Sv%D@NO@{`#a?22MaEM$fuVJr%&bxraDfUBJRw(pcfWECHQP{kAAZMMPM zIRehgnfPCz%9l1noW=CuIlDJrrj=p|t349R#WU7`u2}jx;Rbiq**XGXZ1^k{DI*YA zIN}rvW|U0iyrhp`cy|yXT-ta-J@kw#?))7uDxk3+X?77yHTk?CC~)j1P~uslnOpF| z-lRAm=NM7Tax($2^kZ7{J&d&Ek{A6Jl*DC;{CHN!fFdErEF4PgllbrG@9_o_Ttf-M z0sf{Dzf8GT_bqqD_7bZiwy@q8u$+bnRAxCIg)G1S%?|1Yh3J&Xair@!xHGUhY`4Rn zA8LeKV8GmOJtC&cR%T91IY+2%z&jWTpXYFQu_|@SMv$ahaWylNSiD`?Z--HlqAgJy z!T<@NY*-DpPZH}j`re`uHKk$^)S{8a3aM1JEf{oL8_D22QE5#9>8F{T$0MkgQa*Fb zXm`V043v}+gVyKu!j2F`+EDYpA6e=!TggjW45ZJSUwv;hqh}cl@)%eIIDLKG^x{dz z_Lgq~oLtu5q-iGW_|0VHbr6t_94k=84spz~B<6&k?b`vrMe|xV3d(l=5y3YQO?M=5 zz&*;9r4Tuuxl(tU`&PJ|L*647$X?2GV8fs}S~2>Y(^NEkgSRWI+}73nhRHWax!h7! z^KIF3g81);Xv=Ox-kqu|ti5mdr}VI#i9qV99`IDC)E4@Kng407PVS3`HD`O3LEXJ8J{ zVchme<1*ul`%_2+6?4(I`JT2CVd8@LGB#t}eK z1gE#Wu*_ZLzhSvp7lcs*((U;$*6d@!ID*jfY|iUSaNRM>?&=55{jS_qv$Z9$wRuLg zH&_#$khM#Krzw62UFx-j6I&QyhK)Sy=%^;sg=vX_KvDxEl`1h{QiurFOZI+~11$`> z2a|fV16j=ls{N&?dOoMPiaaLbx`U~=5@tuuy4>Fe*6ghuKQm=uxM+jnrRg@9pt z(Wkq4YV%PWQPySx@!riew=K>**<)--o)>VHJpEYH^;=4+$!fj6cKkN^L8gnUj#!LC z1PdO*vYqENc8GqTvyg}^%*-j1p>Jkt9^3!fK3lwl_i5OB4(J+3{1us(A*;=-UZ{9t0vws|wNfE|@ z22|m39Tj0)G9}m%4|LLphk161i|OM=s68~7--^yU3zk=eP%#8xebauc8=iJyF6m$M z_i+}RA~_kZ)0=A865QN?_Q_WmtZ_!PJ5Z%J5;CX`>r--z9m9KwQ0$|`|VGtU0E>V?l}@J5n)0BWXArh zoiZx}n7TO3`)tw`0u0z%dl!(aY=J)g$|bUzX_9;C=hF;zU1=f`kouwuW2L8s>g8NN z@Y$I4=nqZBLZ|PC%%1OJ{RQztTyn+n%P!xHqyEY+1}y%pcM!n=S_yg8UfZ5&XLp@k zn3Ut({Em1X@NNg}l$K}G4TJjo;>O&yJo96^HlHt19>g^*2R%B#lpDGo)FThbb=8J4 zS>Weap~oHtp;q=@9h;*amIN^7jqp&X(z!|YSU)^XD>MK?g{kV*&_bwj6bE)(<2A-0 zalyXCA}puii*x!DECtQN;Xa)3)7&R0{xy`+taM!O(Zf1|beYnW{Y*Vs13T*|-w2rJ zyFnfN>9nU$*@{t6`Z$<9A(g@Z^ zYyWvEKxfVRaE!i*z3q&kVJ%4I(Rycp(DgLH5F4~y#mm*(Nf>A6C|Yl_x&NQFWWq(= z(jeGAHt_839v#_%>FnDqFl*o2I!{PkPq<|tU9lS0{A&ef(sv0A4g^EIlN%5M;h>!~ zy9dLNp`S%}!aZLv6p*y`#!CgEZwGiHKJ}`;+1zx994Sr zDDjj?Tn82}GMCWFWXi{4DzfZhr^O=MDLn0tikIW1^p#h11^=amh~r}7XziOs5yC$@ zmsx;x*ayXh?}HGy+769FM?sW;!I$l(m$@oyU-}5_^q`~Su^g-S$4FkwZg3{BfKs&& zk_q*g)0OQg_Gkf)iYE`rcg~a7qU*WSJBf{8K1$q&2FtNlGm@{dh`Nnqt6&FHL9X-6 z?_^w zaa7|rk9c`?`msgR%Kpr_UC?G5M!lc3x-`rq%t_zh>5em=@E!aCB=aBgP9gfk$CRjZDQ#l-?ow4k4xx0P40)6FPpCC-h0@rY+jwUUxl6x zzuP%>&^JFS1!QA^2KW`MmT|{bjq4m&x;3;*L;Ai|T7z}V>eMy9s&}5mV~k;tHD#t4 zYGnb>MpEGz%ltzJm{zdB6y|Cxam4qzQ(71LR<-k!I{mGL zfr6mVtfxHA6goAV{F@;(s4mZEENJ}+hHL06Fqs-fo>Q+?Xm6j*1I44zFClgC>IcmV zY;%+jD#~~-84Y$AhBTfde4Y8Y0vN`HIN9-l&5^pCCbhX;ewOAnzgl4kqCaJuDBd<{ zIEA^xPRMSK6`95lX27eXYdnjl>ak~y*0AyyE&l!+Vo`U_+mOTo)6f?^Yy1~{58ak8 zB~+97{rHFGV#lcAOG?)q_i3Sd0mw(^E9h080Z{e!+P@r{>^QHZ1YUp#=Aeu7lHOqc zD;wk$Gs)lxO=u>ddPVcjkX$bBDAn6#(i`NcF5R5TZ*uNpZ#iJik_;nI zjO0i+YJ!&{U}8IQw{^~uJt=ZH7HW*`yMbflm(ChR4{~Tr?S3Zj(0}g?lSK1het;r-R zdAEF3QG#F;6#L=0ffrnO&BZzV>S!R+?*4Iqnr5jnjXmZp@`k8qE2y?}=1r(oBoz}w ztbTP{fCXsqq3;{4`R49Kp5rBakSsospI--1@WHHhwLzXC{W^?Tyk_z*rK5 zRTov37@IKU$=z#>@ghDZN`w5Y24-)0qif^baJv&H6b8C*L62aLDo>1~vYopdS{!$x zkJDxO+0_%tL6Ht5*DO+#&5aG}vw04KvpnEj)ZTo$TeV^|5#Rzuy>`>`wjV%H5D0Rg zu|YHMN+Dp8_O?+zsKmfweO6mnq0$pHp2cm zTUX9Y6;!TMG@qzTbTW@mal66=ud+qJAs2brSY=?N^-f+RFR`NE3WfF#si&OFqaB+4 ziS9=dp2t{Q$R@FsrYy()h)2FWFXT*owiVHS+WYhR<1_WyuYu6Ddek%^7sYey!F)%@ z8c|kaHzo>K{D~4i>$9~odz?gp_D{HW<5*+}Bnp1!aX^4`YNIqBZ$8p9{G6g=RGFy- zcS;JNf-+eV;bNe=KSdH?l)Iwt{)M}g!kz$-4e9y&RAg>W-ob5&N)F*^&B5|rfN=)Y z(`A+*YAjjP#|iB1meX$20Wh5VRn(Pq>1aK5tgulA`q8C}D0BcHH(n{H&bSD3=Y4rgciuQ2(^^-*3ET+waR+yZy zy+S42$4LU!6?P9=*VbC3K2Vr3wG}*}L8gP)b6nvWW8`MaMaH8qyJOU*M7|R3Q&xmr z+xqIrj^_%!b%av?kuX7z#S3rJ_)jT1i0&wIuD@4KIacayc|&&owc(a%y^lfK>I|Ub zZ)q95)^0#}>mJ~&y3)Vnr`VkrMCC;>P*pQF|7LMt%Ld;(4U{i2!Xfdt4;O=s8T9~S zmuGNSrq}wK#V;Hyp9#Gp-gv}?04+~M578+X06{>$zg~_JYcj!M8B^CdVmK-LN^2W^ z8#Hn`?1l5MdW;Q}W5mwGCaSSLr3&LZMD&sw4Nu>;@yBdVp z1P5bL2M+x0-aFw5(MGagQG{ai_ieO)P@tX|zmT*ApF#D-Wb)77Bpo`C8M%RW zneXpJgyk4&x1klTO~K6e+TX_iQOoO^P}S)+WJoDF$>(p4HhVF81s zwrTrf+o9sT#k*r!bWjnkx_obDHSlRP(#Ps^6!(RYjT!v)SxeO8;a{@JRbLnR0OLI# zc+(S!&Y6~CR^X%Kb5|_)GsNyGjX2`^ga7~rq(Pp^MG-7d)bKFotOj$#!~m<)sZ(bK zr^4<5`53M!s8&jq>(B`hlG~CS^kRr?_}NkL?7?*DUXNAA_@_IGS+VhM_XE!Sfy_i} zGnJa^fiXUN`gLs!L_TUQcV_#v+N7s#GN545I*OsI*QhsnT(eNOf7B}SL#otDmtFL+ zoUzH>On}wKPeEF<)kI`-5G#kvnNMq@%6eMhTVC4!>!7VBi(ds=?kgUplF8ZVYS6~_ zz#lVLulHxO7zJd_q}J&oQ7%sdHY&$qg>r&y6s5x@@hbNVf9y}os#O7DIJ{lxEO8D< znRNi4=Z<rO)G(quD#E_3-Nl+E?(Xb!pxJnF1d-+E_-;6AsHjPfho*)pMv$u5Eg zEFAlID7eg>9JiC`lMo**nzWio)likWlG*p+))iP)0+o!^BrCE%2y`q?b)GT6``o$P z%o+bs0aeGoMlLwWzB-R<_rTIa(4|?Plo82MTMeSA{32zxWq3OJI7xHVr`ECf7vWD6 z05t#mB0DTL(2pmVpG>F=L;02%;z|B7&Cg&7QKNt_`~<5=+)iN>Y#h)spJt1rc{G~e z7ypn~*%1==GCG-aJVzcdSEW4BwuSNy7+JM32&?OKJh%pQgRtb98yN1Fy!8JEjYa?| z6m?8vHa)7^Ic&SXR^7{tqIF?vo<0Lkq*1YY>Via92#-g;z%a@W6y|-CK-;W>Nm5Jc z0~|3_O4ozZN#03Awb6yXke*|8s^%7-)E!wix?}C=R4?$C0>`Mn?kq)%ycKi;bXef*qikuACg-U%ke*^>=6XA{Myi9$t)fni`(` zekcznJp;ze%o%a7EN?%~?oTA&blU9RQ*YiEsCNblsZmB+C%{-pX<7sD4r>&O{CHwV z3H>xN(e#A+_?eM1`Ji!F$BX}o-ZI~s$Wtr8VS`)7cW*E+p&Jo`aBF-#M4qzlZlQ^Q zeP|^Ez|J-%gQ2F*jDz8N0W| z!&|q)WAUi!OMYG*sp~dxqCtu9mzRZm%6JB`0uFrwE7SY`8CB6(sg(Yf)qQUqKH z@V^IHP^kJ=^eV{t<;e88Hx}8R#_O8bgBV@?LhD1*QkqW(xD4`2G9itN?p}-K zaN7j|Qq;cJtK))Ni}&(dRz*nS8Jy=Uh>U(o=qW&-mQnA>_WUk>j60O@%mO=d{P)kM zPTG{DVJb#?5bP?laHlqgb_JK_1*-f{gFEU}S&j*)s4-G*`$pGfjPr~`B-GPlbU8I>I#kENX~cXxJDdvfJ9gzGVGYZ`5;L`xd->=w5_JfSs0SNqjWL+U(%T z#B_-mP#OG-!YoRa>kHT{@u2edXG<;l^s7rc%@(nEb*eHibcmSoa4yfA!`;GW#cw10 z5yQPN6LJvj@3Yl?OTjZmA*$-SI$%U4y7I|KbZ%~W#|yi(tXFZQHsp`t9RGm)4jzb zOU2aHtE78T^wyJlitdTY{jKKWKjdg+_<%7ZepeNz!uOM8G=4>h{%Nm#y+bzMKN*)U zs#y@53t!!L!`kRJISORdRJ<0j>0hsA4b46B&-cp>P0_rK*GfPP9F%RjURM3xZW^7! z+y=aVkcH7ns={hCagi+szA_T$(xwEAIB5NGiqm-)MHLg$`P<72;M!9*V8t(qPp$bb zf>t=CFG=%Gg4>{YB!ECaR>G+W@zeCW{gmyr-h!c?Ct(vyL(eLl2#4Kah64%5S$ml4 zhv6l!O8R1g+fe!~QiIcqF8>QxJ61w_pasq;0n)e30oi@|!JpJ8DO@YzAc#gV$p8QYQvsgnbWi`AjU_oMOI1{0ry1?vfBRJ9smpe1utTO zS_2vsIFY>jFQ-J~=OQL3Yiwbw&RNUrkp%JygNmBXH|{=-$1v^_Un zDa~`pX{Zf%8aZAK!v9Eoyg8%d4Ad7UxoW_N7cLX>hf2xM*xam+4 z$@?P7IZ8wDc0poOUpgb~N-s91%8tbRb`B7(VpXMzbixa}n$sLi8)w+<7dI7BwQbM!EV1nA$5ykQdkao9YjOltrj<(RHi_MTd zX~$`5wm{bya<}UuW|UM)IB@BIB^{OLjnKJM<&8eJbp>4BRrK~|eEHkR%HDGp+S_b( zty)8k(-4%kxnI#|GR1Q=5~{-F{A}oQ64NQK zNuZ)`&q*b;hEFkq;tC9+tfbY;&jCT2UbnLjO+!W;>~GomUkc~t!;@jo1#4Y~`pE({;M8}=o`%L< ztmy;I0blClU04NxY5GaP#O8=*US2e3){c3h7^_vgrCo*1Gb4bV0vdGWsHM3wy(k;I zwv~iVoF$$*YYyCpriBQ(>Hs;vYAyyt0OcAkQ#n#3ht|F)fs2LwC-}hiG>t#vj(D5QfSGEHkTMuGt)a3_)ra&>a|p!irl5q000Jn0iNn=ME@*R6qqK; zJ;`7(2k9t8T3&%J28q-%XcZ`+aesNE>5VFY8aQ?fhZ=0t)S0jU_EtDxUPE09Gq#u^ zJqU&ca&QbrZ!lGnO8D|9U3i~Pcbf&3Q(%WXqC?cK-|>Vr`{&$c z(RCc{f1Wgz2r6k07}8US#Ih*NSutpBvW{{Qlp+d|(o*2^&8U}c7eMY3X|C0YH677r zVE+ajKb`sDL2x8U@U}T%4KG{WTsb@p4Ll~ih>#4mD1Pi#Fc%9O@1&IEV zT{VI`sWV|v8<(mJP)B+M%w1bu*hX+mEAuNo$7FMbuI3OiJzGWogZ6{caW;m2H^ zhrKw!Dc6GKV?9RV{ z8A}cPYxQrfHq?pa^#x(#TN;80WV5`$kIlq2Wo=2cTkZ0{d?|6R1q~;W@2>t23Z3H! z)9Ubv#jz)DCtG-Xq{*_Px4|qHv^#t6oZs^{o>KO*F|fY9)@gA0WGd;KsLPMBD15(X z-2`7uoh7v}9(TB`8kE@JB2?_C;g>#F>MaOkEfw5(kDr(@p4*z{3-+{}vs(-1qSq<_ z4KJ-Kt2So=H*Z>-Ylj0o0syTj)PCJXW_k9#XMg}wnI{WiaApSQ)&ei6&`s@U&JP)X z7xoMSZgnD315zF*S=|V4z*D-Xr*{vmojwbqi{sv$t-6Z`yU6OB626tzzCNCN-cD5} z1V(anH3NmIDDdKibd(=pS52(*iqr07*U~J?dRP$ZA(>1FfyAKb@!z4OoahyDxdC%4 z&+q5>M5eWTO0GLW=N&dGo{#BCn9ycJHfYA~HAr%t_0Du=HhM-Ku#%C5M%apV-4&ED zRTSE`f@*=ds@kwr&`h*Zk#Bjvg??r7P(Y$X_nXtMKql==B4c-z^*6r*d{#Vme9Ty+ zqI+AjLOa=cf&ZMl2OCz{^eUXGU}c!|zpv1<2TE3(nqkzGlSzLCjb}XEgKd}ZhdDGb zV!<2|v^Q*~NXb1X8_nDeh5Pv+OTwfAwDjN?%-hWJZGn34Pu3h}@_ZK6{e=vIFp`!i zO*oS`l&!MZ-leB~#~7e}wNR6p7Is`v-CTJWby);LMn>HYM5;8pU#`Dqhqz~G_W*4w26sHE6teF|MWLFhT<9L>NZgJlvi`^KBxH5;xZ8 zEYY+PNh(-j|BrK>;rKivSXVIT=001qlL7MMLs6lL* zObDOA2>*azF$$rX{OrR^{Fx7$M{Mch!R%^QPb$_6@Byj`?sjPJ%XpR2w2H!)EpinQ1s{6y5YZ8XFsG()lR0g*zDj?=jF?&vW$3Iz*IxQ}4v zb1ym5QVR_(4G>=1ZIO0}$HynP?d*i`Wb>-4Y9v#%k$08n(nt+m*3kRyXFX44vBb;O?_TeOs=q#l4o;e3TTnA-6+yonsc+ zg-VR_@SvvW-XHawGbHW7EQxTA=qD6uLA9nFjy3lej6^oBUs#){q;}|nWtpPhnt1;` zP}f_D?bWFzur1f`oT{V4E2bzS&YwALC<}6ur26y3onvy-p}AHqu3Zj#G^Zs@H{fxc zd%lzS2?fU%?L_P2 z8TPqPD7A7ZX?g3?Nt$yzwi4IJV_X7*l0nfFN0!=znWZvI3)`V|pyOW8Cwa<)@x*0H zaWQHd5TwwL->Qg;c#M!xe_0~6CYW&8eNB51BKIrF8`&R&O=wNHAmVqt@!AcN{R>cz zb!bv0>h>&6H$x7cF@ft9G>>XGEF-x_#Mb9xR&(<|gt;nWFQ^Ikc8LSZGpk3H?i=um zFO_6kJ2MM|53DhAfP9RRJ^eO|1Q-R?58(I2pN!H?i?Bu7Hrym1BvL;#oN@zA{3lC| zl4DD6FE(DIu7Ld<0h&i$t=p(4EzJXiP8vnILl_cEt)^h|ZXdw{HM33NF&JqmFq((j zy!|*@_?Xywxd1-vgLK$p-QNVzUwVl-ZeMB&g!dC@Ot_;w$)g@Ba4-@bZSP`-9e|RqYsA^W_-vNj9LM~lgw-YXbXc;MyJeN4h{&o^Q|dJaN@iTIRXA#O zF`v{9vPT@!6hh6KyRdNfHv_d$m;7l(k>6gi2N9p7 zdQgQnAaO46@RK_V z*EP{rR()6pzlFNXC+2fg73{a1W<|-DDlbl$p128gVLMNwTNZ*2e+61a76tqutI^oa z5*FTUSBU>p`Ta<$Av+`*c!5&`an~`2f^!0RJYyuV{ZzcIPy$NE8Ew96E4;eTekO|p zHSD_B5^be%2yR4Eb9SvTC8Exo^??Er_gUDK-r&*YH?;7Rz)Sbn1yG2JNyFr5NZ&PB z9%H9u*AG+dIMaWM)%WhvAJ zDnR1o7dGsN+^3QUL%8S^T^bQ95xlO~PkacTufmf|`ETHG##N?3YZQzj0ZoxtPy$qx zGDA;mtdMLvV0^2keD+eD*vIRO=VzFQu=xJ?O~4hI3=qhbetSC4tA-+g-B?WyX2ZAU zXsOl$GyojnjHaltcEQR`U7N;sTcRs_shiZrQ&jJ%DhW%yA|cul$+EU6!g79JI7?Z_ z72V%JI(3Lc&`hUoJEkbA)MZ#TZ89SfxCyu zS5Z0Udk7#SrA1pcXGV5vq3>MxQ_dJ-BQ`Qhc`k9M(`N8UqPU2ywwuy1<800~HrRL+mhu97Oh}2$VSTc;8@??R zcU);3tRTjMOYx-7$t9yD0^p^_ex8bzrtw{~^a7O30< zxVCI%7eW!*A;gk%iq)zRSk=ggo{SOY>5XR=B7Hk7aGbQfc5B`lyIUDVB00Z>>E^}# zweqv>Cg&z*Le4flebDD>Mmn|$LaA`qTu|!~BqV>y+>gV)3{RzkHPjY!))na@|aMV0eSDsAq z2u-{{HBSsqi*_>rn5A`L(t3HiMTObUM*9XOOo_zE0iv}hs`5lG%bQ@r*uh+uU@!q> z);|H}d8>G`Ss?Hok2?8B@0*6V`%6?l6iRM89Gvx+Jh^$9iYwIT7Y84p7CEV~RXOq6 ziZ*4$eh6sZ#aYui+nSWIyw9zEIXW>dI*o-hDO1=Io@tnXbs;ZHr;`{;JO$9Q zx}?fb^o-q`?g5T*i%#ae{Q8F<+ebhquQZ!GIXK)HGCaDtA2YU8_%h(eA4--Wc%dLN zAB_w5>b(1)0TAa60BaZEd)ZDaI!fwXGD;}* zzhLgx`}u?b$_=g<8)(lEPPo43wmTnvnYzYmL=Yc68hjhpW}>2_CN@% zcKqh{Ag?rVm$Bhwrpg-`=FNc%+Q7%Ld#3>ixN-Sr zUYkpXQsiD@2zOr<$lQYrR2b|xj7(9`){QP!qE45Vng@Vo-&>POTS|Vm&++cM@jDe> zbJ^ft+f*ue=AY3)se-xOUo<&Ar$5CIrHm)r4M3cLPd)uCl#=eZw6s0BAvAFva(>;l zhiWQDmx!0MVzXWbh7hEa6{p`{!I>`(-2PQcc3xq-oq-=W=GjGfKn@n=PE)nzm!}wS zIe&H}^~O($-^hD|KUy}zY~o95sa0;A{*G3X#KKqjV51RdqbU(2JB94!ig0KZo*H%$ zwp`!lr#6aSFDt|Ls}>STD7rpwdA7eB&uwNb#F7J0MY9vX*SG;7-AFtBi6BT#6 z;2c1^iLc~wSEJ%g#35Tkt%YFD^tf)Ljq+)ZSg3E|Ys&+zrm!!JK3~1%s>#yb0G;0! z$xa+zDGV1!j zjbaN?po%#R$Z5&b^UolmU>v*cy4X1K^=)T%;>hGW3zheXdz7oUZg^^Mymeb3>K@|e zK7z_z@1u#Pds)+L%&lK9En#GG6W`FdNNw`2WeDT@jJn-JMs}ure)4gTl+rA0znd0J z>@n`hB|*vWSV1oT2hxwNa&}i}h@8ON4QityOr8nzcXHG-hmArh@PgGLwZ_w49>O3L z^qaF$W6sjbbQ+j;G<4q;DjUtVM$DN@MlM5>#s(8sImO6;C4yk}_E?WNYd!wY4iehD zE=lpH?j%@{ukn2RYq8#o+SO`_jduK9sv`-!2NlUsOiVTnbgYgY7!>Jv53{hnFdr7V zSSFIFiY!uE^!dE?_u;^Yy>XF|q%aVGLBRH7$_;Pf2Y=UG416vY`%? z(6roLe%Pq{cJ1YgyTxNr3sKYav99rz$4QBJb@`gg%6qjraZxj1Ef*!!7d+p=#vQen zWWYnn(6OGljAR=*PEi?DbHoJ|!;Za`ILTyNcez-t)x1&nO?ypI@in@F=!0 z^2{CVbwFrgxEI6`g1V%j0A356jiS6qU^VL%r;I!r>N|#)Adc0pX7mk-WO||tdi~p)UaN*T0vK@H-SnGSfRL`r!X6}6|1Jv)ylGJ zO9`RH?=glpiM#ZIyKIwYQcO20F52e_&&Ck4DYWPb1CTcAx}is`;TO>-ONQEz(n6yD z@hnbS0W6<4mdgEv9CGE!TIj#-#*AAKOSo1aYNHIw<7 z!DoFi#Iq;yhu93o6(@3hRRR*3|GZUKa&9cao`z@g?whdbbcB>9okWrJH=55>tdnL~ zXVlk=r1AYrngjHm&7;?s@7GrKIhDV>*N_W`$U=mAkx_&^)J2LEaS|(Uye40+nJCK! zc9^as+jqmh(1RXE`Q+D^Nk)E~ooYfRIQEgT>I2&o_1rJLWyL%A?j6BUK6f)+p4=e2 z0-l7_w?0Z1rm@QjKg-^#ynxAQLjsM6| z-|wJ4rPqe4x_HqSZU^i6N3UROL~=y+d!9eei*sGd^A62fF}1Z$dv)vQY>H_ugSW30 zouQWL(iP&}2hscVoW47!Vl5XB)iUAr>Ruj{0wy7gnl8gr*n%-aC4mWZ1F~(Er%ct| zxC>9%zfe-*b8t4wne@H(r23M*Z3f)9yOt~dd z1~^XU`$5atIAX93t~O*fZTa_n60M;WC&$On9=;ADcgY~u_$vkBF8Hu!&18wA^s(qw zDs($pvB=pC`pJ$rE_xbb4s&njSw8$J&la)6gXj;wJT`}*(7Y6F6t?{MGWjHnT^GDI zOgsxb&#+2k&(pHJyEw0sYY!L`N~eVitygW&Ks&ZTn%yJrW^+sD1W^te@rLEq#$oP= zwFrTA7Jr1Z&$&AU+PJZrl~54*^wVN(HI`n?YO1)vt2}nsllrz}#Jvk&@0%h6c(=n0 zM%WRIb3=#pWQ%$ouu)PR_s&^26^|}sJeiuezOh$`lK{f{rBFza-x!r?nVGl1L9{M}+RGne|%D8*_PXWM?A;%$01=yYvr^&hpd zFYPdpZv&Ai;Rp|fmq7AI-2pMno;xZ&8YGH{fFG3LwQ>szpZDAx7+44|cKm8Pxl?1a zO4t08M$w#5RHp^$pZ)nf6aK|kbPZ=qKI~X_xrB?y;
lWSv55MPRf+P(VlQ5y`s zc7Rs0zOf?3u~K#g)fJGvaY0cDjcGFe>x|^@^AZ!9mO*xV6q;(^kjshypG4CsQYn1P zf4cNDuK#_{+iSZPHT^Zjc2QAYTVCA$9T#SCt_&d`U3#181|{1twTG#lW-h`G>iRC4 zP~vYV1HKXs&6@O$?q;Wj-IEJTNd+Qz)VijI*L-Y91ja3~1Xj*fEr0I+ey30&=RC6C9^J{&oe56jdIXO2myD@Bu@BAZQ(gK!u&Q2 z>Cfl7`Xo_q(GlGWa7W>bQd$OhWDsnf2Q2pepxX)j>bt{OuN?H&v<<`M3d(wbh4RUP zk+c(TmttL;XZuSm>>OSGT8kuf!juR$Mi7yJFu7okxdeCL2JQA(Bl->d=f^=P^Qo%py}ZKqf4FwQ!5ovj1%T zgAtDT^}q#%Zv!#jyVj#9b9C9spEDYCxE+&fGIDs^ied9&ggwR4pAh%ffBP>}ORbe( zY7)bS-+aZtx3T=rv!FRq^c($a8b7jNLt|wUm*gXv<4L6eO{PlsK~H!g@4!6FGFwPc zn=4dX1~msG{bYeuOEV*5|+YahMx>-O)`gcAX+ULrLM)cM=7jVEpXHOk!iE*i> zv&446WR8qpA}g>L(@5><(EGIH#kR6RI3xDfFrcIt4zyd@$U?lA+o*>X?E$)UXA57K8O0(pM4?R_s z6w7obxLd%8<80Oy_+ohKjwqBUF%NNixs4_*4Voy%zrJ&oBB#b6tj!OlJdC{P;-4Kv z0o^%0=NLcdU9VS?a_T%}gieB@L5=MB^;AlKFnlmTJ8aWwFY-dh9QE4^6-HcDc=bo@ z5hYVaBW_8|5uoRU&=Sx=k0FfU+3!}plisw52nPZ1Dqtzfq=5Pd`29CB29B9U=f8B$ zxI@B2Ik11zT4Y`wTYaZKtNq`UL4j>$#ltw{VdfZ%7P5@G-Gepfz(0R|8Y-;-jOn?C z5U&KXRP+}+dYlrGN2+Yx&BeGY#I;PEiV^-k1fJ%cRKlP0>o^bOo*eYF#UdFX*4-hE>5s4L~FQc~C=q`qq)zz7+b*PFD%XUio@w93M|GdNN~43Uxww#pwx2iH3SOUTD!o^|X3 z?}y6s;YExE7`BGZXZ57!?O8EaL*niXt-zOVpS#7ayCsomVz-y!K(it90fkn4({ER+ zsJ>zN>Qa{^NL_GspL2TEp}#iY(gKB>Uhie z=O;q^sa1ztHFe6*0q1}#Tq(u=lY*SN^>)8JA$%4zCY{EB*67GmCvY41{g^VzeMV|x z?zL_2muFV~2jNr;(|AVuEvD`VG+2Anj^N##`Uj@vhqYm@&AuZXR=ox-BS3n>QN)M} zn!yD?r~YpBV{hfnip|Y$Xo3C-XrcDd3Q}d56ZKG!(MjNjCG&bn&J^H(pdb9hImxhU z@(YDBnZnPKUiTLzVk08Z^Vpun1Xid$akAz|m3IRhcZcEgGwb7uJ1D3o5m;GGFT@j! zFCuuT_Bhpvr#%XwG;fR8*OpZKIP_3F8k5Ggme-Q}PX_FZXX)eg*=0SYo`u%*IMgEf z5s?1Y&&;g60dd?7Q>%bYrMaJm>uTP0b#K|A#0*?2p*g;G_A5uVkM8~>HYh5;weP&l z=87)5nje)1VgTe*Y%G}Kl^J**?Eoj~debV)UoLK#C(>=GN>=hzT8t-oRb2!|<~h+~ zp9?&~NAyZV)v~3k>K3Bt-n$XX%>ePg!5IlnVu$;!jap#IaV-2A;*~gULka&?CSzD3 zGVf#79;&3L`-r>oghHOvl$sIoTov#c-b91*O?3jCgRA|k`YsgJgi%Y$|5xI!a;pzB z#1dcON>Oe)D+P*3TNkAPqIRyh5!?|Z^38>1;(twLpoDEm50A(?Em+S%O=wCpPvnGp zsA8?4eRxU(00;Edw7bs-V=u%!xcj?kgeMHUN+?X%O(Cd@_8lU|xM(c|bO+3OD)kd# zgkW}-w$Q#&^?AKBW3S}Oiqlj4-!0Aur*a}oyOa(aY@V4b@N~hdL`!G*qIJJBORtb5 zzfZ{Sx8zr4C9gRr6%$$XrQ|l7xVoQw3jKcCfxPIJ*3lE;qNE#tO9?tRG$IVrgpD`V z0&&RIK^P4wG5kI3VLRW<-Z#PwvP8kC9pXTPyOg?5@oEk_2(Onm;Ycw~Hpc)4Rff?1 zxviZ5j2a=Eqr+5q0!FmFgrVI3E5zpS;7fUWM&W^?^ExCG;jAKK?y}k9+9FLKFOTHq zP;FKaE~_@<0}OgQF&h=>B0Ue&?j)XGRTNHFAe1Yd;@nS;GDJc9IaSm6P0UwKFYTA_ zy*}g@?u_R~KmgpKjq9IS9HDLhrEm}Q!1;GYMCEc*KPBK^{8id(Y(-#549C53A>&ir zWEp-;+bji89D*pZ%BzOGSm2|2$K}3W;5Kmde!@87wJHXR9sYYHdakT;p7m550e-=H z!e&ZLI^(_hAJG3Td5MxIfo9jL{{!IL8Vlrk_nj{(Nf+prnSiR%Aq7Bu36`hLG&@Vf zdx5SM?5ikADR5qOxZ9=yT+u2AKw(k_Fjz6i`k zjg%#|G4=`i3Sf8lsQJ0xJd8Jr=E9{6`(1DPiKK>a?@#H@b@3lN*irw56*H zyU9P_cW9gPNkJ{coX=C1j3YLg_oJ%z0h9L5&r*mm@Aw@faw)s0ga<_!P=E_ItWOd z-H?j>)$QZZ;`4_Q^wexHzjYE1BDl90n~fwWmo)+&@os=U{}$%`sTc>}T%y8I;uPKA0trHg6##yHF9t*1cmXn&L@!FnO8=X*PYfZ>lkc{jFrK`ot(HTYj zV{IpcX4OT3c7`)-IMcRJJ$I{w$tQzkAnvUdL{}|&^F*eTx-R?MYULUsj3MhR)vtOm zm!IkgQG8hfh$T7Li~|El@>9`MT6^gAN+-JgP1?acNeGlWQ$SMOAXw^JfAevq1DLh{ zQ7{;qZV@@Mma7F|U0btn6~TxHyQK`}Io(Hl}hE~vnd zcb8O9ELNZ$U}zG!fmzqo&fNj^z1&>We@f|icVu#!fSr5NOS|&<#X0|k5QgsFKX*+a z-NOKpEy6JtFPrpz+vrdHbI07s%`q5WZ(;n*{t`z8_yNWo|+UrzZ`Eq_iGgm(sJu7E%u*V)3w5#6mukSMv?K< z5tHi(>=&Be8g_;MLRGAr9H}_AVJxithBKrfdA~3>l09=Uj%0_EBHQh8Hn^>p^{FYX zTdaLqj!9>ZUQ0elgqRpGP`Az-&X29DapbmwbGN|1f9>%0|0{+kEU2$qLUqiw85(tI zkjdFN?-m#GAt-3IbsyiLnHF81%mnIJV2o%6EkC39<9JZ+iT((hg+i#(uI1hHEd%w$ z$kxKEGw?y55UK-(S@PNcSjMWX22|aAmY4ajfRa&@i5(ZH1Tf$5$oL(*9W2m9<))*| zyMqR~@%V~K|BF`jg2X}YuF#SJhVd%Kfb$I9>TQ0%prcP~i&`hX_q%8dm?4%ibU(BX z87XT%JnJRM`vuL1Rht3>+qgoGp3(}y#EE8xS&XaK_-^vqbYrev^!gFN0FkL+-5$=s z|DspV+gyvoDHsTL%Kzdb7dtUjlBn+BPhkyitD3N>c4uMwbed1bZAoIiL5&;QzQr*H ze`+pz$ll)&Kv_~iy8Y?JC?C7T$~cWR)~VD^W9?Q#@Ov=l?Os-~Ch_FjjBc>Hy)F`` zNti$lmBrM>oBT9kL_?|t@oRPi`<`COO|r;;{M$32B{nzS0w4DC|Ex^mdcI(}7J^%+ zR;q1miG`k>&5%fn)GY!ziL8(dWo=I1b%S7*4h@mk_eR+>TTLgoY#32j&b+P8^~PU; zebo;b@kJj=nsq{)#FS2O4oO=YVyS7NyH^L$3ZamMYHQWiVvAwCpy`_efoN?-%LN(F z6TB*#SLk?epaHiq0M7s-HpsS?sJuzGQACCHJcH#JZ&9fLzV|k?6YE5=z=U8)WGEvP z{A12$B_1U^({0OxHOCFu%u)UYM`mZX!Y%WOk004Q<5V#+N7+TOd>mzl3~&Yl_65wW z`Un-{KE&trB2(i`O1@2kZzeGdYx<;YdVe7JCL`7SUMHM8;;7<#(#iRtm0_fZw0NRG zQ64z8_S3BRe5*`FXVw>i<7aCiK>#emnnGf++sW|p`WOKEH`X@2)G`-|)JpKOELEXy zr7JCshDg2t)eU;Hfv?}vf7Af7bBu^!&J!;&5BHA_2za?70qxsn$gtM_*jb#GT(Wmk zO0DU%@t|q(X9L|6G(l2peAE+UqL8~9HTS7qU~gWA-)^-2&hG&r_CWej8I`_Ir1+(> z01X@6e|jMpfq5QTOKg`+Int8y?)M^DFe&JNYI{Z>_~U;SON`SHMH{ezCMRty7z~-R zt-{Tex#l)Vn}`G|6AH(|#27{;mpNSpx3x!93lWzRSGu-sA8k3yCS20QYkaFe9QDhq z5_hmxzr-@pR*Pk=t?~;N;EfT~Y}0j@q1g2?aVs#>(psJuzQnFfAZdyPvLDwBhk95n zI7DC^^tte){I0@fUuW9Vf<=MhWLmaGe8r?@ib~fH3R3Lhp+isPcKJ$Lh=c;Z1)Za6 z_27JgGj8kk;yEkhh)~6Cf1Q&M%r~O~(4w3`uz`1`o9>_Agw#YeZ<>tCsYx<`e_JWyu_V}Q|_{GCfB~jfy zoPg95$UGsgaS9{o?29f#x_9#`9$7_I11nNA7)s1I6onB-cx@8{r}K%jJP&)6<6Sj? z2>R3E1GH1aZhrt@B>#URmOU;)Xn{oOIK!vA*82^$t*WCmVZ|rxW}`(MOgN<6>8&u% zDrgJ|-*{iVYy92Oc_A_jB%56IRq?5Is0&%~rF8mX z%vrIFF0T@};keA$FBFQTA0@cMsP&p2Dy}j1+n>4l~%W!*$=Ow@7(?8ma@_Pcg!Oa$x@E z-$=M&snOmOcreie{TCAGla58GMSHS6cXuDdi@Oxc8QH*u8zs*0w*1W)`|ibv(4Ov9 z?aVESEM3PUA(@za@vB2Ho^LoDJ+2!3!{qO52x^Z-;3;wl1im!Zde}xBWsJov?hyGg zR#{6OwU=%W|A9OwIcIU|#3!=TW+puI3jDJ9f=(Na3Gcm6Qip0^l0ULX*98t(TMtUY z%A&upv_n-A^%bv8>&=!o+WJt22a%^QG1POFVWyQ%NMKgo|=BjwcPL0XsIbGkq_ZhjWX!96|NPXRXZOIYrBv z*{skO{$+4BdIc*=dbLv&gy%-h@3WnpA~w?5N*H?bSWCodKLq zr>tQGD?y;b8j$wA80M<=AB0RAs9+5MibZVa*2X0O!RQTo6QzaWwOKh z35Nt=s~1`C1v%dLyNOjmZKY1)dKILa2Ym2`o5#t2O|;BlcmLP7G<)Q4EIt3F+K~-v z8%i-nsryvcB!FX5x0K5mz zYrGWK-ebHcrr7J8mKEOv7ZZLB4v^OG6mS%~Bu9A!f#=b45F7366G|*qOZM7;2E0O> zz{g|!VfLTyG#(a26clm@506hzgk!`NUZ1iO_)XseLy=f>(|NiWTNCS|_Ns@MYl