From d5a79a32ee468363bbb3a90d53e413c280afc885 Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Fri, 4 Jul 2025 21:27:09 -0400 Subject: [PATCH 01/13] Add BlockSwap support --- src/core/generation.py | 63 +++++++++++++++++-- src/core/infer.py | 14 +++-- src/core/model_manager.py | 126 +++++++++++++++++++++++++++++++------- 3 files changed, 171 insertions(+), 32 deletions(-) diff --git a/src/core/generation.py b/src/core/generation.py index bc39b52..cdfb176 100644 --- a/src/core/generation.py +++ b/src/core/generation.py @@ -17,6 +17,7 @@ Key Features: """ import os +import gc import torch import time from torchvision.transforms import Compose, Lambda, Normalize @@ -28,6 +29,7 @@ from src.optimization.performance import ( optimized_video_rearrange, optimized_single_video_rearrange, optimized_sample_to_image_format, temporal_latent_blending ) +from src.optimization.blockswap import BlockSwapDebugger from src.common.seed import set_seed try: import comfy.model_management @@ -120,14 +122,19 @@ def generation_step(runner, text_embeds_dict, preserve_vram, cond_latents, tempo conditions = [condition] t = time.time() + # Check if BlockSwap is active + use_blockswap = hasattr(runner, "_blockswap_active") and runner._blockswap_active + # Use adaptive autocast for optimal performance with torch.no_grad(): with torch.autocast("cuda", autocast_dtype, enabled=True): video_tensors = runner.inference( noises=noises, conditions=conditions, - preserve_vram=preserve_vram, # Memory offload optimization + preserve_vram=preserve_vram # Memory offload optimization + and not use_blockswap, # Disable dit_offload if BlockSwap active temporal_overlap=temporal_overlap, + use_blockswap=use_blockswap, **text_embeds_dict, ) @@ -175,7 +182,7 @@ def cut_videos(videos): return result -def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_size=90, preserve_vram=False, temporal_overlap=0, debug=False, progress_callback=None): +def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_size=90, preserve_vram=False, temporal_overlap=0, debug=False, block_swap_config=None, progress_callback=None): """ Main generation loop with context-aware temporal processing @@ -188,6 +195,8 @@ def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_si batch_size (int): Batch size for processing preserve_vram (str/bool): VRAM preservation mode temporal_overlap (int): Frames for temporal continuity + debug (bool): Debug mode + block_swap_config (dict): Optional BlockSwap configuration progress_callback (callable): Optional callback for progress reporting Returns: @@ -202,7 +211,13 @@ def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_si - Real-time progress reporting """ device = "cuda" if torch.cuda.is_available() else "cpu" - + + # Log BlockSwap status + if block_swap_config: + blocks_to_swap = block_swap_config.get("blocks_to_swap", 0) + if blocks_to_swap > 0: + print(f"๐Ÿ”„ Generation starting with BlockSwap: {blocks_to_swap} blocks") + # Adaptive model dtype detection for maximum performance model_dtype = None try: @@ -420,15 +435,53 @@ def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_si #clear_vram_cache() finally: - # Final cleanup of embeddings + # Initializing BlockSwapDebugger logging + enable_debug = block_swap_config.get("enable_debug", False) if block_swap_config else False + debugger = BlockSwapDebugger(enabled=enable_debug) + # Log memory state before cleanup + debugger.log_memory_state( + "Generation complete - Before cleanup", show_tensors=True + ) + debugger.log("๐Ÿงน VRAM cleanup") + + # Move models to CPU text_pos_embeds = text_pos_embeds.to("cpu") text_neg_embeds = text_neg_embeds.to("cpu") runner.dit.to("cpu") runner.vae.to("cpu") + + # Clear runner cache + if hasattr(runner, "cache") and hasattr(runner.cache, "cache"): + # Move cached tensors to CPU before clearing + for value in runner.cache.cache.values(): + if torch.is_tensor(value) and value.is_cuda: + value.data = value.data.cpu() + elif isinstance(value, (list, tuple)): + for item in value: + if torch.is_tensor(item) and item.is_cuda: + item.data = item.data.cpu() + runner.cache.cache.clear() + + # Find and clear any remaining GPU tensors + for obj in gc.get_objects(): + try: + if torch.is_tensor(obj) and obj.is_cuda: + obj.data = torch.empty(0, device=obj.device) + except: + pass + + # Force cleanup + gc.collect() torch.cuda.empty_cache() + torch.cuda.synchronize() + #del text_pos_embeds, text_neg_embeds #clear_vram_cache() - + + # Log final memory state + debugger.log_memory_state( + "After cleanup", show_tensors=True + ) # OPTIMISATION ULTIME : Prรฉ-allocation et copie directe (รฉvite les torch.cat multiples) print(f"๐Ÿ’พ Processing {len(batch_samples)} batch_samples with memory-optimized pre-allocation") diff --git a/src/core/infer.py b/src/core/infer.py index 573025d..3d6d2dd 100644 --- a/src/core/infer.py +++ b/src/core/infer.py @@ -290,6 +290,7 @@ class VideoDiffusionInfer(): cfg_scale: Optional[float] = None, preserve_vram: bool = False, temporal_overlap: int = 0, + use_blockswap: bool = False, ) -> List[Tensor]: assert len(noises) == len(conditions) == len(texts_pos) == len(texts_neg) batch_size = len(noises) @@ -363,10 +364,15 @@ class VideoDiffusionInfer(): self.vae = self.vae.to("cpu") if self.debug: print(f"๐Ÿ”„ VAE to CPU time: {time.time() - t} seconds") - t = time.time() - self.dit = self.dit.to(get_device()) - if self.debug: - print(f"๐Ÿ”„ Dit to GPU time: {time.time() - t} seconds") + # Before sampling, check if BlockSwap is active + if not use_blockswap and not hasattr(self, "_blockswap_active"): + t = time.time() + self.dit = self.dit.to(get_device()) + if self.debug: + print(f"๐Ÿ”„ Dit to GPU time: {time.time() - t} seconds") + else: + # BlockSwap manages device placement + pass t = time.time() diff --git a/src/core/model_manager.py b/src/core/model_manager.py index b866d1b..63df1e8 100644 --- a/src/core/model_manager.py +++ b/src/core/model_manager.py @@ -30,22 +30,26 @@ except ImportError: from src.optimization.memory_manager import get_basic_vram_info, clear_vram_cache from src.optimization.compatibility import FP8CompatibleDiT -from src.optimization.memory_manager import preinitialize_rope_cache +from src.optimization.memory_manager import preinitialize_rope_cache, clear_rope_lru_caches +from src.optimization.blockswap import apply_block_swap_to_dit from src.common.config import load_config, create_object from src.core.infer import VideoDiffusionInfer - # Get script directory for config paths script_directory = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False): +def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False, block_swap_config=None, cached_runner=None): """ Configure and create a VideoDiffusionInfer runner for the specified model Args: model (str): Model filename (e.g., "seedvr2_ema_3b_fp16.safetensors") base_cache_dir (str): Base directory containing model files + preserve_vram (bool): Whether to preserve VRAM + debug (bool): Enable debug logging + block_swap_config (dict): Optional BlockSwap configuration + cached_runner: Optional cached runner to reuse DiT model from Returns: VideoDiffusionInfer: Configured runner instance ready for inference @@ -112,12 +116,45 @@ def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False): # Configure models checkpoint_path = os.path.join(base_cache_dir, f'./{model}') t = time.time() - runner = configure_dit_model_inference(runner, device, checkpoint_path, config, preserve_vram, model_weight, vram_info, debug) - if debug: - print(f"๐Ÿ”„ RUNNER : DIT MODEL INFERENCE TIME: {time.time() - t} seconds") + + # Check if we can reuse cached DiT model + if cached_runner and hasattr(cached_runner, 'dit') and cached_runner.dit is not None: + # Directly reuse the cached DiT model after cleaning + # Transfer the model to the new runner + runner.dit = cached_runner.dit + + # Clear RoPE caches + clear_rope_lru_caches(runner.dit) + + # Get the base model for cleanup + base_model = runner.dit + if isinstance(base_model, FP8CompatibleDiT): + base_model = base_model.dit_model + + # Clear any accumulated buffers (especially RoPE-related) + for name, buffer in list(base_model.named_buffers()): + if 'cache' in name.lower() or 'freq' in name.lower(): + # Re-initialize these buffers to clear accumulated state + if hasattr(buffer, 'zero_'): + buffer.zero_() + + # Ensure model is in eval mode and gradients are disabled + runner.dit.requires_grad_(False).eval() + + # Clear CUDA cache to ensure clean memory state + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + if debug: + print(f"๐Ÿ”„ RUNNER : DiT MODEL DIRECT REUSE TIME: {time.time() - t} seconds") + else: + # Load DiT model normally + runner = configure_dit_model_inference(runner, device, checkpoint_path, config, preserve_vram, model_weight, vram_info, debug, block_swap_config) + if debug: + print(f"๐Ÿ”„ RUNNER : DIT MODEL INFERENCE TIME: {time.time() - t} seconds") t = time.time() checkpoint_path = os.path.join(base_cache_dir, f'./{config.vae.checkpoint}') - runner = configure_vae_model_inference(runner, device, checkpoint_path, config, preserve_vram, model_weight, vram_info, debug) + runner = configure_vae_model_inference(runner, device, checkpoint_path, config, preserve_vram, model_weight, vram_info, debug, block_swap_config) if debug: print(f"๐Ÿ”„ RUNNER : VAE MODEL INFERENCE TIME: {time.time() - t} seconds") @@ -126,11 +163,23 @@ def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False): runner.vae.set_memory_limit(**runner.config.vae.memory_limit) if debug: print(f"๐Ÿ”„ RUNNER : VAE MEMORY LIMIT TIME: {time.time() - t} seconds") - # Pre-initialize RoPE cache for optimal performance - t = time.time() - preinitialize_rope_cache(runner) - if debug: - print(f"๐Ÿ”„ RUNNER : ROPE CACHE PREINITIALIZE TIME: {time.time() - t} seconds") + # Check if BlockSwap is active + blockswap_active = ( + block_swap_config and block_swap_config.get("blocks_to_swap", 0) > 0 + ) + # Pre-initialize RoPE cache for optimal performance if BlockSwap is NOT active + if not blockswap_active: + t = time.time() + preinitialize_rope_cache(runner) + if debug: + print(f"๐Ÿ”„ RUNNER : ROPE CACHE PREINITIALIZE TIME: {time.time() - t} seconds") + else: + if debug: + print(f"๐Ÿ”„ RUNNER : Skipping RoPE pre-init due to BlockSwap") + + # Apply BlockSwap if configured + if blockswap_active: + apply_block_swap_to_dit(runner, block_swap_config) #clear_vram_cache() return runner @@ -196,7 +245,7 @@ def load_quantized_state_dict(checkpoint_path, device="cpu", keep_native_fp8=Tru -def configure_dit_model_inference(runner, device, checkpoint, config, preserve_vram=False, model_weight=None, vram_info=None, debug=False): +def configure_dit_model_inference(runner, device, checkpoint, config, preserve_vram=False, model_weight=None, vram_info=None, debug=False, block_swap_config=None): """ Configure DiT model for inference without distributed decorators @@ -205,20 +254,30 @@ def configure_dit_model_inference(runner, device, checkpoint, config, preserve_v device (str): Target device checkpoint (str): Path to model checkpoint config: Model configuration + block_swap_config (dict): Optional BlockSwap configuration Features: - Automatic format detection and optimal loading - Native FP8 support with universal compatibility wrapper - Gradient checkpointing configuration - Intelligent dtype handling for all model architectures + - BlockSwap support for low VRAM systems """ # Create dit model t = time.time() - loading_device = "cpu" if preserve_vram else device - - with torch.device(device): + # Check if BlockSwap is active + blockswap_active = ( + block_swap_config and block_swap_config.get("blocks_to_swap", 0) > 0 + ) + + loading_device = "cpu" if (preserve_vram or blockswap_active) else device + + if blockswap_active and debug: + print(f"๐Ÿ”„ CONFIG DIT : BlockSwap active - creating model on CPU") + + with torch.device(loading_device): runner.dit = create_object(config.dit.model) # Passer les opรฉrations au modรจle @@ -253,21 +312,41 @@ def configure_dit_model_inference(runner, device, checkpoint, config, preserve_v runner.dit = FP8CompatibleDiT(runner.dit) if debug: print(f"๐Ÿ”„ CONFIG DIT : FP8CompatibleDiT time: {time.time() - t} seconds") - - # Move DiT to CPU to prevent VRAM leaks (especially for 3B model with complex RoPE) - if preserve_vram: + + # Handle model placement and VRAM clearing based on configuration + if preserve_vram and not blockswap_active: + # Only move entire model to CPU if preserve_vram is active WITHOUT BlockSwap if debug: - print(f"๐Ÿ”„ CONFIG DIT : dit to cpu cause preserve_vram: {preserve_vram}") + print(f"๐Ÿ”„ CONFIG DIT : Moving model to CPU for preserve_vram") runner.dit = runner.dit.to("cpu") - if "7b" in model_weight: - clear_vram_cache() + elif blockswap_active: + # BlockSwap will handle its own GPU/CPU placement per block + if debug: + print( + f"๐Ÿ”„ CONFIG DIT : BlockSwap active ({block_swap_config.get('blocks_to_swap', 0)} blocks) - placement handled by BlockSwap" + ) else: + # Normal case: move to GPU if loaded on CPU if state_loading_device == "cpu": + if debug: + print(f"๐Ÿ”„ CONFIG DIT : Moving model from CPU to {device}") runner.dit.to(device) + + # Clear VRAM for both preserve_vram and BlockSwap cases (fixes VRAM leak) + if preserve_vram or blockswap_active: + if debug: + reasons = [] + if preserve_vram: + reasons.append("preserve_vram") + if blockswap_active: + reasons.append(f"BlockSwap") + print(f"๐Ÿ”„ CONFIG DIT : Clearing VRAM cache for: {' + '.join(reasons)}") + clear_vram_cache() + return runner -def configure_vae_model_inference(runner, device, checkpoint_path, config, preserve_vram=False, model_weight=None, vram_info=None, debug=False): +def configure_vae_model_inference(runner, device, checkpoint_path, config, preserve_vram=False, model_weight=None, vram_info=None, debug=False, block_swap_config=None): """ Configure VAE model for inference without distributed decorators @@ -275,6 +354,7 @@ def configure_vae_model_inference(runner, device, checkpoint_path, config, prese runner: VideoDiffusionInfer instance config: Model configuration device (str): Target device + block_swap_config (dict): Optional BlockSwap configuration Features: - Dynamic path resolution for VAE checkpoints From 50dfb771e6efa39d16c84af24d7a46fd2aac93d9 Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Fri, 4 Jul 2025 21:28:47 -0400 Subject: [PATCH 02/13] Add BlockSwap support --- src/optimization/blockswap.py | 611 +++++++++++++++++++++++++++++ src/optimization/compatibility.py | 22 +- src/optimization/memory_manager.py | 12 + 3 files changed, 627 insertions(+), 18 deletions(-) create mode 100644 src/optimization/blockswap.py diff --git a/src/optimization/blockswap.py b/src/optimization/blockswap.py new file mode 100644 index 0000000..e1aa6b5 --- /dev/null +++ b/src/optimization/blockswap.py @@ -0,0 +1,611 @@ +""" +BlockSwap Module for SeedVR2 + +This module implements dynamic block swapping between GPU and CPU memory +to enable running large models on limited VRAM systems. + +Key Features: +- Dynamic transformer block offloading during inference +- Non-blocking GPU transfers for optimal performance +- RoPE computation fallback to CPU on OOM +- Minimal performance overhead with intelligent caching +- I/O component offloading for maximum memory savings +""" + +import time +import types +import torch +import psutil +import gc +import comfy.model_management as mm +from typing import Dict, Any, List, Tuple, Callable + +from src.optimization.memory_manager import get_vram_usage + + +class BlockSwapDebugger: + """ + Debug logger for BlockSwap operations. + + Tracks memory usage, swap timings, and provides detailed logging + for debugging and performance analysis of block swapping operations. + """ + + def __init__(self, enabled: bool = False): + """ + Initialize the debugger. + + Args: + enabled: Whether debug logging is enabled + """ + self.enabled = enabled + self.swap_times: List[Tuple[int, float, str]] = [] + self.vram_history: List[float] = [] + self.device_map: Dict[int, str] = {} + + def log(self, message: str, level: str = "INFO") -> None: + """ + Log a message if debugging is enabled. + + Args: + message: Message to log + level: Log level (INFO, WARN, SWAP, etc.) + """ + if self.enabled: + print(f"[{level}] {message}") + + def log_swap_time(self, block_idx: int, duration: float, direction: str) -> None: + """ + Log block swap timing information. + + Args: + block_idx: Index of the block being swapped + duration: Time taken for the swap in seconds + direction: Direction of swap ("compute" for GPU->CPU->GPU cycle) + """ + self.swap_times.append((block_idx, duration, direction)) + if self.enabled: + self.log( + f"Block {block_idx} swap {direction}: {duration*1000:.1f}ms", "SWAP" + ) + + def log_memory_state(self, stage: str, show_tensors: bool = False) -> None: + """ + Log current memory state for debugging. + + Args: + stage: Description of current stage + show_tensors: Whether to count and display tensor count + """ + # GPU Memory + if torch.cuda.is_available(): + allocated_gb, reserved_gb, peak_gb = get_vram_usage() + vram_info = ( + f"VRAM: {allocated_gb:.2f}/{reserved_gb:.2f}GB (peak: {peak_gb:.2f}GB)" + ) + self.vram_history.append(allocated_gb) + else: + vram_info = "VRAM: CPU mode" + + # RAM Memory + ram_info = "" + if psutil: + try: + process = psutil.Process() + ram_gb = process.memory_info().rss / (1024**3) + ram_info = f" | RAM: {ram_gb:.1f}GB" + except Exception: + # psutil might fail in some environments + pass + + # Tensor count (optional - expensive operation) + tensor_info = "" + if show_tensors: + tensor_count = sum(1 for obj in gc.get_objects() if torch.is_tensor(obj)) + tensor_info = f" | Tensors: {tensor_count}" + + self.log(f"๐Ÿงฎ {stage}: {vram_info}{ram_info}{tensor_info}") + + +class BlockSwapper: + """ + Handles dynamic block swapping between GPU and CPU. + + This class wraps a transformer block's forward method to automatically + move the block to GPU for computation and back to CPU afterwards. + """ + + def __init__( + self, + block_idx: int, + original_forward: Callable, + offload_device: str, + use_non_blocking: bool, + debugger: BlockSwapDebugger, + ): + """ + Initialize the block swapper. + + Args: + block_idx: Index of the block being wrapped + original_forward: Original forward method of the block + offload_device: Device to offload to (typically "cpu") + use_non_blocking: Whether to use non-blocking transfers + debugger: BlockSwapDebugger instance for logging + """ + self.block_idx = block_idx + self.original_forward = original_forward + self.offload_device = offload_device + self.use_non_blocking = use_non_blocking + self.debugger = debugger + self._swap_count = 0 + self._total_swap_time = 0.0 + + def __call__(self, block_self, *args, **kwargs): + """ + Execute the block with automatic GPU/CPU swapping. + + Args: + block_self: The block instance (self from the method binding) + *args: Positional arguments for the forward method + **kwargs: Keyword arguments for the forward method + + Returns: + Output from the block's forward method + """ + self._swap_count += 1 + t_start = time.time() if self.debugger.enabled else None + + # Move block to GPU + block_self.to("cuda", non_blocking=self.use_non_blocking) + if self.use_non_blocking and torch.cuda.is_available(): + torch.cuda.synchronize() + + # Execute forward pass + output = self.original_forward(*args, **kwargs) + + # Log timing if enabled + if self.debugger.enabled and t_start is not None: + self.debugger.log_swap_time( + self.block_idx, time.time() - t_start, "compute" + ) + + # Move block back to CPU + block_self.to(self.offload_device, non_blocking=self.use_non_blocking) + if self.use_non_blocking and torch.cuda.is_available(): + torch.cuda.synchronize() + + # Periodic cache cleanup to prevent memory fragmentation + if self.block_idx % 4 == 0: + torch.cuda.empty_cache() + + if t_start: + self._total_swap_time += time.time() - t_start + + return output + + +class IOSwapper: + """ + Handles dynamic I/O component swapping between GPU and CPU. + + Similar to BlockSwapper but designed for I/O components like + embeddings, patch_embed, etc. that need different handling. + """ + + def __init__( + self, + component_name: str, + original_forward: Callable, + offload_device: str, + use_non_blocking: bool, + debugger: BlockSwapDebugger, + ): + """ + Initialize the I/O component swapper. + + Args: + component_name: Name of the component being wrapped + original_forward: Original forward method + offload_device: Device to offload to (typically "cpu") + use_non_blocking: Whether to use non-blocking transfers + debugger: BlockSwapDebugger instance for logging + """ + self.component_name = component_name + self.original_forward = original_forward + self.offload_device = offload_device + self.use_non_blocking = use_non_blocking + self.debugger = debugger + self._swap_count = 0 + + def __call__(self, component_self, *args, **kwargs): + """ + Execute the component with automatic GPU/CPU swapping. + + Args: + component_self: The component instance + *args: Positional arguments for the forward method + **kwargs: Keyword arguments for the forward method + + Returns: + Output from the component's forward method + """ + self._swap_count += 1 + + # Move to GPU + component_self.to("cuda", non_blocking=self.use_non_blocking) + if self.use_non_blocking: + torch.cuda.synchronize() + + # Forward pass + output = self.original_forward(*args, **kwargs) + + # Move back to CPU + component_self.to(self.offload_device, non_blocking=self.use_non_blocking) + if self.use_non_blocking: + torch.cuda.synchronize() + + # Force VRAM cleanup for I/O components + torch.cuda.empty_cache() + + return output + + +def apply_block_swap_to_dit(runner, block_swap_config: Dict[str, Any]) -> None: + """ + Apply block swap configuration to a DiT model. + + This is the main entry point for configuring block swapping on a model. + It handles block selection, I/O component offloading, RoPE patching, + and device placement. + + Args: + runner: VideoDiffusionInfer instance containing the model + block_swap_config: Configuration dictionary with keys: + - blocks_to_swap: Number of blocks to swap (from the end) + - offload_io_components: Whether to offload I/O components + - use_non_blocking: Whether to use non-blocking transfers + - enable_debug: Whether to enable debug logging + """ + if not block_swap_config: + return + + blocks_to_swap = block_swap_config.get("blocks_to_swap", 0) + offload_io_components = block_swap_config.get("offload_io_components", False) + + # Early exit if nothing to swap + if blocks_to_swap <= 0 and not offload_io_components: + return + + # Initialize debugger + debugger = BlockSwapDebugger(enabled=block_swap_config.get("enable_debug", False)) + debugger.log("Starting BlockSwap configuration...") + + # Determine devices + device = "cuda" if torch.cuda.is_available() else "cpu" + offload_device = str(mm.unet_offload_device()) + use_non_blocking = block_swap_config.get("use_non_blocking", True) + + # Get the actual model (handle FP8CompatibleDiT wrapper) + model = runner.dit + if hasattr(model, "dit_model"): + model = model.dit_model + + # Validate model has blocks + if blocks_to_swap > 0 and not hasattr(model, "blocks"): + debugger.log("Model doesn't have 'blocks' attribute for BlockSwap", "WARN") + return + + # Get total blocks and validate request + total_blocks = len(model.blocks) if hasattr(model, "blocks") else 0 + if blocks_to_swap > 0: + debugger.log(f"Model has {total_blocks} blocks total") + if blocks_to_swap > total_blocks: + debugger.log( + f"WARNING: Requested {blocks_to_swap} blocks but model only has {total_blocks}", + "WARN", + ) + blocks_to_swap = min(blocks_to_swap, total_blocks) + + debugger.log( + f"Configuring: {blocks_to_swap}/{total_blocks} blocks, I/O offload: {offload_io_components}" + ) + debugger.log_memory_state("Before BlockSwap", show_tensors=True) + + # Handle I/O components offloading + io_swap_count = 0 + io_patches: List[Tuple[Any, Callable]] = [] + + for name, module in model.named_children(): + if name != "blocks": # Everything except transformer blocks + if offload_io_components: + # Offload to CPU with dynamic swapping + module.to(offload_device) + original_forward = module.forward + swapper = IOSwapper( + name, + original_forward, + offload_device, + use_non_blocking, + debugger, + ) + module.forward = types.MethodType(swapper, module) + module._io_swapper = swapper + io_patches.append((module, original_forward)) + io_swap_count += 1 + debugger.log(f" {name} โ†’ {offload_device} (with dynamic swapping)") + else: + # Normal behavior: keep on GPU + module.to(device) + debugger.log(f" {name} โ†’ {device}") + + # Store I/O patches for cleanup + if io_swap_count > 0: + runner._io_patches = io_patches + debugger.log(f"โœ… Configured {io_swap_count} I/O components for offloading") + + # Configure block placement + if blocks_to_swap > 0: + blocks_to_keep_on_gpu = total_blocks - blocks_to_swap + + # Move non-swapped blocks to GPU + debugger.log(f"Moving first {blocks_to_keep_on_gpu} blocks to GPU...") + for idx in range(blocks_to_keep_on_gpu): + model.blocks[idx].to(device) + debugger.log(f" Block {idx} โ†’ {device}") + + # Ensure all buffers match device placement + # This prevents device mismatch errors with RoPE and other buffers + for idx in range(blocks_to_keep_on_gpu): + for module in model.blocks[idx].modules(): + if hasattr(module, "_buffers"): + for buffer_name, buffer in module._buffers.items(): + if buffer is not None and buffer.device != torch.device(device): + module._buffers[buffer_name] = buffer.to(device) + + debugger.log(f"โœ… Ensured all buffers match block devices") + + # Patch RoPE modules for robust OOM handling + rope_patches = _patch_rope_for_blockswap(model, debugger) + runner._rope_patches = rope_patches + + # Configure block swapping for the last N blocks + swap_count = 0 + for idx in range(blocks_to_keep_on_gpu, total_blocks): + block = model.blocks[idx] + block._original_forward = block.forward + + # Create and bind swapper + swapper = BlockSwapper( + idx, + block._original_forward, + offload_device, + use_non_blocking, + debugger, + ) + block.forward = types.MethodType(swapper, block) + block._swapper = swapper + swap_count += 1 + + debugger.log(f"โœ… Configured {swap_count} blocks for swapping") + + # Mark BlockSwap as active and store configuration + runner._blockswap_active = True + runner._block_swap_config = { + "blocks_swapped": blocks_to_swap, + "io_components_offloaded": io_swap_count, + "total_blocks": total_blocks, + "use_non_blocking": use_non_blocking, + "offload_device": offload_device, + "main_device": device, + } + + # Protect model from being moved entirely + _protect_model_from_move(runner) + + # Final memory state and cleanup + torch.cuda.empty_cache() + debugger.log_memory_state("After BlockSwap", show_tensors=True) + debugger.log("โœ… BlockSwap configuration complete") + + +def _patch_rope_for_blockswap(model, debugger: BlockSwapDebugger) -> List[Tuple[Any, Callable]]: + """ + Patch RoPE modules to handle OOM gracefully during block swapping. + + RoPE (Rotary Position Embeddings) can cause device mismatches when + blocks are on different devices. This patches the get_axial_freqs + method to handle these cases robustly. + + Args: + model: The model containing RoPE modules + debugger: BlockSwapDebugger instance for logging + + Returns: + List of (module, original_method) tuples for cleanup + """ + rope_patches: List[Tuple[Any, Callable]] = [] + rope_patched = 0 + + # Map RoPE modules to their containing blocks + rope_to_block: Dict[Any, int] = {} + if hasattr(model, "blocks"): + for block_idx, block in enumerate(model.blocks): + for name, module in block.named_modules(): + if "rope" in name.lower() and hasattr(module, "get_axial_freqs"): + rope_to_block[module] = block_idx + + # Patch all RoPE modules + for name, module in model.named_modules(): + if "rope" in name.lower() and hasattr(module, "get_axial_freqs"): + # Store original methods + original_method = module.get_axial_freqs + original_unbound = module.__class__.get_axial_freqs + block_idx = rope_to_block.get(module, -1) + + def make_robust_wrapper( + rope_module, module_name: str, bound_method: Callable, + unbound_method: Callable, parent_block_idx: int + ) -> Callable: + """Create a robust wrapper for RoPE get_axial_freqs method.""" + def robust_method(*args, **kwargs): + try: + # Try the normal bound method first + return bound_method(*args, **kwargs) + except Exception as e: + error_msg = str(e).lower() + # Handle device mismatches and OOM errors + if ( + "device" in error_msg + or "memory" in error_msg + or isinstance(e, (RuntimeError, KeyError)) + ): + debugger.log(f"RoPE issue for {module_name}: {e}") + + # Get current device + current_device = "cuda" + if list(rope_module.parameters()): + current_device = next(rope_module.parameters()).device + + # Try with unbound method (helps with cache context) + try: + return unbound_method(rope_module, *args, **kwargs) + except Exception: + # Final fallback: compute on CPU + debugger.log(f"RoPE fallback to CPU for {module_name}") + rope_module.cpu() + try: + # Clear any LRU cache + if hasattr(bound_method, "cache_clear"): + bound_method.cache_clear() + + result = unbound_method(rope_module, *args, **kwargs) + rope_module.to(current_device) + + # Move result back if it's a tensor + if hasattr(result, "to"): + return result.to(current_device) + return result + except Exception as cpu_error: + # Restore device even on error + rope_module.to(current_device) + raise cpu_error + # Re-raise if not a handled error type + raise + + return robust_method + + # Create and apply patched method + patched_method = make_robust_wrapper( + module, name, original_method, original_unbound, block_idx + ) + module.get_axial_freqs = patched_method + + # Store for cleanup + rope_patches.append((module, original_method)) + rope_patched += 1 + + if rope_patched > 0: + debugger.log( + f"โœ… Patched {rope_patched} RoPE modules with robust device handling" + ) + + return rope_patches + + +def _protect_model_from_move(runner) -> None: + """ + Protect model from being moved entirely to GPU when BlockSwap is active. + + This prevents other code from accidentally moving the entire model to GPU + which would defeat the purpose of block swapping. + + Args: + runner: VideoDiffusionInfer instance containing the model + """ + if hasattr(runner.dit, "dit_model"): + # Handle FP8CompatibleDiT wrapper case + original_dit_to = runner.dit.dit_model.to + runner._original_dit_to = original_dit_to + + def protected_dit_to(device, *args, **kwargs): + """Protected .to() method that blocks GPU moves during BlockSwap.""" + if str(device) != "cpu" and hasattr(runner, "_blockswap_active"): + print("โš ๏ธ Blocked attempt to move blockswapped model to GPU") + return runner.dit.dit_model + return original_dit_to(device, *args, **kwargs) + + runner.dit.dit_model.to = protected_dit_to + else: + # Direct model case + original_dit_to = runner.dit.to + runner._original_dit_to = original_dit_to + + def protected_dit_to(device, *args, **kwargs): + """Protected .to() method that blocks GPU moves during BlockSwap.""" + if str(device) != "cpu" and hasattr(runner, "_blockswap_active"): + print("โš ๏ธ Blocked attempt to move blockswapped model to GPU") + return runner.dit + return original_dit_to(device, *args, **kwargs) + + runner.dit.to = protected_dit_to + + +def cleanup_blockswap(runner) -> None: + """ + Clean up BlockSwap configurations and restore original methods. + + This should be called when BlockSwap is no longer needed to restore + the model to its original state and free up any resources. + + Args: + runner: VideoDiffusionInfer instance to clean up + """ + if not hasattr(runner, "_blockswap_active"): + return + + # Restore original .to() method + if hasattr(runner, "_original_dit_to"): + if hasattr(runner.dit, "dit_model"): + runner.dit.dit_model.to = runner._original_dit_to + else: + runner.dit.to = runner._original_dit_to + delattr(runner, "_original_dit_to") + + # Restore original RoPE methods and clear their caches + if hasattr(runner, "_rope_patches"): + for module, original_method in runner._rope_patches: + if hasattr(original_method, "cache_clear"): + original_method.cache_clear() + module.get_axial_freqs = original_method + runner._rope_patches.clear() + del runner._rope_patches + + # Restore original I/O component methods + if hasattr(runner, "_io_patches"): + for module, original_forward in runner._io_patches: + module.forward = original_forward + if hasattr(module, "_io_swapper"): + delattr(module, "_io_swapper") + runner._io_patches.clear() + del runner._io_patches + + # Clean up BlockSwap markers + if hasattr(runner, "_blockswap_active"): + delattr(runner, "_blockswap_active") + if hasattr(runner, "_block_swap_config"): + delattr(runner, "_block_swap_config") + + # Clean up swapper references from blocks + model = runner.dit + if hasattr(model, "dit_model"): + model = model.dit_model + + if hasattr(model, "blocks"): + for block in model.blocks: + block.to("cpu") + if hasattr(block, "_swapper"): + delattr(block, "_swapper") + if hasattr(block, "_original_forward"): + block.forward = block._original_forward + delattr(block, "_original_forward") \ No newline at end of file diff --git a/src/optimization/compatibility.py b/src/optimization/compatibility.py index 095ba32..9045aae 100644 --- a/src/optimization/compatibility.py +++ b/src/optimization/compatibility.py @@ -19,30 +19,16 @@ class FP8CompatibleDiT(torch.nn.Module): - Flash Attention: Automatic optimization of attention layers """ - def __init__(self, dit_model): + def __init__(self, dit_model, skip_conversion=False): super().__init__() self.dit_model = dit_model self.model_dtype = self._detect_model_dtype() self.is_fp8_model = self.model_dtype in (torch.float8_e4m3fn, torch.float8_e5m2) self.is_fp16_model = self.model_dtype == torch.float16 - # Detect model type - is_nadit_7b = self._is_nadit_model() # NaDiT 7B (dit/nadit) - is_nadit_v2_3b = self._is_nadit_v2_model() # NaDiT v2 3B (dit_v2/nadit) - - if is_nadit_7b: - # ๐ŸŽฏ CRITICAL FIX: ALL NaDiT 7B models (FP8 AND FP16) require BFloat16 conversion - # 7B architecture has dtype compatibility issues regardless of storage format - if self.is_fp8_model: - print("๐ŸŽฏ Detected NaDiT 7B FP8 - Converting all parameters to BFloat16") - self._force_nadit_bfloat16() - else: - print("๐ŸŽฏ Detected NaDiT 7B FP16") - - - elif self.is_fp8_model and is_nadit_v2_3b: - # For NaDiT v2 3B FP8: Convert ALL model to BFloat16 - print("๐ŸŽฏ Detected NaDiT v2 3B FP8 - Converting all parameters to BFloat16") + # Only convert if not already done (e.g., when reusing cached weights) + if not skip_conversion: + # Converting all parameters to BFloat16 to fix dtype compatibility issues self._force_nadit_bfloat16() # ๐Ÿš€ FLASH ATTENTION OPTIMIZATION (Phase 2) diff --git a/src/optimization/memory_manager.py b/src/optimization/memory_manager.py index 4246adb..92e1b39 100644 --- a/src/optimization/memory_manager.py +++ b/src/optimization/memory_manager.py @@ -231,7 +231,19 @@ def fast_model_cleanup(model): if buffer is not None: buffer.data = buffer.data.cpu() + # Deallocate the memory + def deallocate_recursive(m): + for child in m.children(): + deallocate_recursive(child) + for param in m.parameters(): + if param is not None: + param.data = torch.empty(0) + for buffer in m.buffers(): + if buffer is not None: + buffer.data = torch.empty(0) + clear_recursive(model) + deallocate_recursive(model) def fast_ram_cleanup(): From b7412df20de2e026d9c54e914f64c5b8006f792c Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Fri, 4 Jul 2025 21:29:15 -0400 Subject: [PATCH 03/13] Add BlockSwap support --- src/interfaces/comfyui_node.py | 268 +++++++++++++++++++++++++++------ 1 file changed, 223 insertions(+), 45 deletions(-) diff --git a/src/interfaces/comfyui_node.py b/src/interfaces/comfyui_node.py index e377686..c8f952e 100644 --- a/src/interfaces/comfyui_node.py +++ b/src/interfaces/comfyui_node.py @@ -2,9 +2,7 @@ # Clean interface for SeedVR2 VideoUpscaler integration with ComfyUI # Extracted from original seedvr2.py lines 1731-1812 -from datetime import datetime import os -import gc import time import torch from typing import Tuple, Dict, Any @@ -12,7 +10,8 @@ from typing import Tuple, Dict, Any from src.utils.downloads import download_weight, get_base_cache_dir from src.core.model_manager import configure_runner from src.core.generation import generation_loop -from src.optimization.memory_manager import clear_rope_lru_caches, fast_model_cleanup +from src.optimization.memory_manager import clear_rope_lru_caches, fast_model_cleanup, fast_ram_cleanup +from src.optimization.blockswap import apply_block_swap_to_dit, BlockSwapDebugger, cleanup_blockswap # Import ComfyUI progress reporting from server import PromptServer @@ -81,6 +80,12 @@ class SeedVR2: }), "preserve_vram": ("BOOLEAN", {"default": False}), }, + "optional": { + "block_swap_config": ( + "block_swap_config", + {"tooltip": "Optional BlockSwap configuration for low VRAM mode"}, + ), + }, } # Define return types for ComfyUI @@ -90,7 +95,7 @@ class SeedVR2: CATEGORY = "SEEDVR2" def execute(self, images: torch.Tensor, model: str, seed: int, new_resolution: int, - batch_size: int, preserve_vram: bool) -> Tuple[torch.Tensor]: + batch_size: int, preserve_vram: bool, block_swap_config=None) -> Tuple[torch.Tensor]: """Execute SeedVR2 video upscaling with progress reporting""" temporal_overlap = 0 @@ -100,17 +105,42 @@ class SeedVR2: debug = False cfg_scale = 1.0 try: - return self._internal_execute(images, model, seed, new_resolution, cfg_scale, batch_size, preserve_vram, temporal_overlap, debug) + return self._internal_execute(images, model, seed, new_resolution, cfg_scale, batch_size, preserve_vram, temporal_overlap, debug, block_swap_config) except Exception as e: self.cleanup(force_ram_cleanup=True) raise e - def cleanup(self, force_ram_cleanup: bool = True): - """Fast cleanup with minimal logging""" + def cleanup(self, force_ram_cleanup: bool = True, keep_model_cached: bool = False, block_swap_config=None): + """ + Comprehensive cleanup with memory tracking + + Args: + force_ram_cleanup (bool): Whether to perform aggressive RAM cleanup + keep_model_cached (bool): Whether to keep the model in RAM (only applies with BlockSwap) + block_swap_config: Block swap configuration with enable_debug flag + """ + # Determine if we should keep model cached + should_keep_model = False + if self.runner and keep_model_cached: + is_blockswap_active = ( + hasattr(self.runner, "_blockswap_active") + and self.runner._blockswap_active + ) + should_keep_model = is_blockswap_active + + # Initializing BlockSwapDebugger logging + enable_debug = block_swap_config.get("enable_debug", False) if block_swap_config else False + debugger = BlockSwapDebugger(enabled=enable_debug) + cleanup_type = "Partial cleanup - keeping model in RAM" if should_keep_model else "Full cleanup" + debugger.log(f"๐Ÿงน {cleanup_type}") if self.runner: + # Clean up BlockSwap + if hasattr(self.runner, "_blockswap_active"): + cleanup_blockswap(self.runner) + # Clear cache if hasattr(self.runner, 'cache') and hasattr(self.runner.cache, 'cache'): for key, value in list(self.runner.cache.cache.items()): @@ -121,63 +151,114 @@ class SeedVR2: del value self.runner.cache.cache.clear() - # Clear DiT model - if hasattr(self.runner, 'dit') and self.runner.dit is not None: - + # Clear DiT model - only if not keeping cached + if not should_keep_model and hasattr(self.runner, 'dit') and self.runner.dit is not None: clear_rope_lru_caches(self.runner.dit) fast_model_cleanup(self.runner.dit) del self.runner.dit self.runner.dit = None - # Clear VAE model + # Clear VAE model - only clear if not keeping model if hasattr(self.runner, 'vae') and self.runner.vae is not None: - #from src.optimization.memory_manager import fast_model_cleanup fast_model_cleanup(self.runner.vae) del self.runner.vae self.runner.vae = None - - # Clear other components - for component in ['sampler', 'sampling_timesteps', 'schedule', 'config']: - if hasattr(self.runner, component): - setattr(self.runner, component, None) - - del self.runner - self.runner = None + + # Clear other components - only if not keeping model + if not should_keep_model: + for component in ['sampler', 'sampling_timesteps', 'schedule', 'config']: + if hasattr(self.runner, component): + setattr(self.runner, component, None) + + del self.runner + self.runner = None + self.current_model_name = "" - # Clear embeddings + # Clear embeddings - keep on CPU if caching model if self.text_pos_embeds is not None: - if hasattr(self.text_pos_embeds, 'cpu'): - self.text_pos_embeds.cpu() - del self.text_pos_embeds - self.text_pos_embeds = None + if should_keep_model: + if hasattr(self.text_pos_embeds, 'is_cuda') and self.text_pos_embeds.is_cuda: + self.text_pos_embeds = self.text_pos_embeds.cpu() + else: + if hasattr(self.text_pos_embeds, 'cpu'): + self.text_pos_embeds.cpu() + del self.text_pos_embeds + self.text_pos_embeds = None if self.text_neg_embeds is not None: - if hasattr(self.text_neg_embeds, 'cpu'): - self.text_neg_embeds.cpu() - del self.text_neg_embeds - self.text_neg_embeds = None + if should_keep_model: + if hasattr(self.text_neg_embeds, 'is_cuda') and self.text_neg_embeds.is_cuda: + self.text_neg_embeds = self.text_neg_embeds.cpu() + else: + if hasattr(self.text_neg_embeds, 'cpu'): + self.text_neg_embeds.cpu() + del self.text_neg_embeds + self.text_neg_embeds = None - self.current_model_name = "" - - # Fast RAM cleanup - if force_ram_cleanup: - from src.optimization.memory_manager import fast_ram_cleanup + # Fast RAM cleanup - only if not keeping model + if force_ram_cleanup and not should_keep_model: fast_ram_cleanup() + + # Log memory after cleanup + cleanup_type = "partial" if should_keep_model else "full" + debugger.log_memory_state( + f"After {cleanup_type} cleanup", show_tensors=True + ) + + if should_keep_model: + print("โœ… VRAM cleared, model kept in RAM for next inference") - def _internal_execute(self, images, model, seed, new_resolution, cfg_scale, batch_size, preserve_vram, temporal_overlap, debug): + def _internal_execute(self, images, model, seed, new_resolution, cfg_scale, batch_size, preserve_vram, temporal_overlap, debug, block_swap_config): """Internal execution logic with progress tracking""" total_start_time = time.time() - - # Configure runner - if debug: - print("๐Ÿ”„ Configuring inference runner...") + # Check if we should use model caching + use_cache = ( + block_swap_config + and block_swap_config.get("blocks_to_swap", 0) > 0 + and block_swap_config.get("cache_model", False) + ) + + # Check if we need to clear cache due to model change + model_changed = self.current_model_name != model + + if model_changed and self.runner is not None: + print( + f"๐Ÿ”„ Model changed from {self.current_model_name} to {model}, clearing cache..." + ) + self.cleanup( + force_ram_cleanup=True, + keep_model_cached=False, + block_swap_config=block_swap_config, + ) + self.runner = None + runner_start = time.time() - self.runner = configure_runner(model, get_base_cache_dir(), preserve_vram, debug) - if debug: - print(f"๐Ÿ”„ Runner configuration time: {time.time() - runner_start:.2f}s") - + + # Configure runner (always create new runner for clean state) + if use_cache and self.runner is not None and not model_changed: + # Create new runner but reuse cached DiT model + print(f"โ™ป๏ธ Creating new runner with cached DiT model for {model}") + old_runner = self.runner + self.runner = configure_runner( + model, get_base_cache_dir(), preserve_vram, debug, + block_swap_config=block_swap_config, + cached_runner=old_runner + ) + if debug: + print(f"๐Ÿ”„ Runner configuration with cache time: {time.time() - runner_start:.2f}s") + else: + # Create completely new runner + if debug: + print("๐Ÿ”„ Configuring new inference runner...") + self.runner = configure_runner( + model, get_base_cache_dir(), preserve_vram, debug, + block_swap_config=block_swap_config + ) + self.current_model_name = model + if debug: + print(f"๐Ÿ”„ Runner configuration time: {time.time() - runner_start:.2f}s") if debug: print("๐Ÿš€ Starting video upscaling generation...") @@ -187,6 +268,7 @@ class SeedVR2: sample = generation_loop( self.runner, images, cfg_scale, seed, new_resolution, batch_size, preserve_vram, temporal_overlap, debug, + block_swap_config=block_swap_config, progress_callback=self._progress_callback ) @@ -194,7 +276,7 @@ class SeedVR2: print(f"โœ… Video upscaling completed successfully!") # Cleanup print(f"๐Ÿ”„ Total execution time: {time.time() - total_start_time:.2f}s") - self.cleanup(force_ram_cleanup=True) + self.cleanup(force_ram_cleanup=True, keep_model_cached=use_cache, block_swap_config=block_swap_config) return (sample,) def _progress_callback(self, batch_idx, total_batches, current_batch_frames, message=""): @@ -212,19 +294,115 @@ class SeedVR2: def __del__(self): """Destructor""" try: - self.cleanup(force_ram_cleanup=True) + self.cleanup(force_ram_cleanup=True, keep_model_cached=False, block_swap_config=None) except: pass +class SeedVR2BlockSwap: + """Configure block swapping to reduce VRAM usage""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "blocks_to_swap": ( + "INT", + { + "default": 25, + "min": 0, + "max": 36, + "step": 1, + "tooltip": "Number of transformer blocks to swap. 0=disabled, higher=more VRAM savings but slower", + }, + ), + "use_non_blocking": ( + "BOOLEAN", + { + "default": True, + "tooltip": "Use non-blocking GPU transfers. Faster but may use more RAM temporarily", + }, + ), + "offload_io_components": ( + "BOOLEAN", + { + "default": True, + "tooltip": "Offload ALL I/O components (embeddings, patch_embed, etc.) to CPU for maximum VRAM savings", + }, + ), + "cache_model": ( + "BOOLEAN", + { + "default": False, + "tooltip": "Keep model cached in RAM between inferences. Faster subsequent runs but uses more RAM. Only applies when BlockSwap is active", + }, + ), + "enable_debug": ( + "BOOLEAN", + { + "default": False, + "tooltip": "Enable debug logging for BlockSwap operations", + }, + ), + } + } + + RETURN_TYPES = ("block_swap_config",) + FUNCTION = "create_config" + CATEGORY = "SEEDVR2" + DESCRIPTION = """Configure block swapping to reduce VRAM usage. + + Model Info: + - 3B model: 32 transformer blocks + - 7B model: 36 transformer blocks + + Guidelines: + - blocks_to_swap=0: No swapping (fastest, highest VRAM) + - blocks_to_swap=16: Balanced (moderate speed/VRAM) + - blocks_to_swap=36: Maximum saving (slowest, lowest VRAM) + - cache_model=True: Keep model in RAM between runs (faster but uses RAM) + """ + + def create_config( + self, + blocks_to_swap, + use_non_blocking, + offload_io_components, + cache_model, + enable_debug, + ): + if blocks_to_swap > 0 or offload_io_components: + configs = [] + if blocks_to_swap > 0: + configs.append(f"{blocks_to_swap} blocks") + if use_non_blocking: + configs.append("non blocking") + if offload_io_components: + configs.append("I/O components") + if cache_model and blocks_to_swap > 0: + configs.append("model caching") + print(f"๐Ÿ”„ Offloading configured: {', '.join(configs)}") + return ( + { + "blocks_to_swap": blocks_to_swap, + "use_non_blocking": use_non_blocking, + "offload_io_components": offload_io_components, + "cache_model": cache_model, + "enable_debug": enable_debug, + }, + ) + + # ComfyUI Node Mappings NODE_CLASS_MAPPINGS = { "SeedVR2": SeedVR2, + "SeedVR2BlockSwap": SeedVR2BlockSwap, } # Human-readable node display names NODE_DISPLAY_NAME_MAPPINGS = { "SeedVR2": "SeedVR2 Video Upscaler", + "SeedVR2BlockSwap": "SeedVR2 BlockSwap Config", } # Export version and metadata From 4b045456e9720d0e8a04d3d1043ee63a77ddbe40 Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 03:49:28 -0400 Subject: [PATCH 04/13] Blockswap with better memory management --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index a8a0310..fbc3987 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,32 @@ Of course, the output resolution also has an impact, so if your hardware doesn't - `new_resolution`: New desired short edge in px, will keep ratio on other edge - `batch_size`: VERY IMPORTANT!, this model consume a lot of VRAM, All your VRAM, even for the 3B model, so for GPU under 24GB VRAM keep this value Low, good value is "1" without temporal consistency, "5" for temporal consistency, but higher is this value better is the result. - `preserve_vram`: for VRAM < 24GB, If true, It will unload unused models during process, longer but works, otherwise probably OOM with + +4. ๐Ÿงฉ **BlockSwap Configuration (Optional - For Limited VRAM)** + + + + BlockSwap enables running large models on GPUs with limited VRAM by dynamically swapping transformer blocks between GPU and CPU memory during inference. + + **To enable BlockSwap:** + - Add the **SEEDVR2 BlockSwap Config** node to your workflow + - Connect its output to the `block_swap_config` input of the SeedVR2 Video Upscaler node + + **BlockSwap parameters:** + - `blocks_to_swap` (0-32 for 3B model, 0-36 for 7B model): Number of transformer blocks to offload + - 0 = Disabled (fastest, highest VRAM) + - 16 = Balanced (moderate speed/VRAM trade-off) + - 32-36 = Maximum savings (slowest, lowest VRAM) + - `offload_io_components`: Move embeddings/IO layers to CPU (additional VRAM savings, slower) + - `use_non_blocking`: Asynchronous GPU transfers (keep True for better performance) + - `cache_model`: Keep model in RAM between runs (faster for batch processing) + - `enable_debug`: Show detailed memory usage and timing + + **Finding optimal settings:** + - Start with `blocks_to_swap=16`, increase if you get OOM errors, decrease if you have spare VRAM + - Enable debug mode to monitor memory usage + - The first 1-2 blocks might show longer swap times - this is normal + - Combine with `preserve_vram=True` for maximum memory savings ## ๐Ÿ–ฅ๏ธ Run as Standalone From 4756bf082e572ea4b05a1d00b5cb67622a01fab0 Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 03:49:51 -0400 Subject: [PATCH 05/13] Blockswap with better memory management --- docs/BlockSwap.png | Bin 0 -> 99818 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/BlockSwap.png diff --git a/docs/BlockSwap.png b/docs/BlockSwap.png new file mode 100644 index 0000000000000000000000000000000000000000..89672d274f2431f4bc4c3468e5d5b07f6434f997 GIT binary patch literal 99818 zcmce-byQW+*EV`+kOt`%1eERukx-;Wq)Tb(?gj~^1f(00?rxA4DW$s`4qb=%*3sYl zjT`Sf?zn&4dj`YU?7h$0bI$e5XFhZ8u-8g2upf{=fIuMFaxbOdKp;q~5C|eMIx09K zTC6(+{z7(qF83B4e7w+&L&5*aoMbeeRBcV2T;DsGK+J4xtxeb*jT}r&Y#hyPouEi< z;@~7U_(@U@Chwg-*xJy({a|eZ(J(QleNtQJVMutD>*8=;GFan?t^v zrFFeJCFrJO+)Xm39>MNu&kz*2IF5IC>s&-3`k;9??z1lxJXC7xHRnu4yJD+0u#zza2 zn!dfR=QS)P>H)Nu)!6AG_i0TF)V4L+bm#%Yvr7%T^GA1Hi!K{8m_bIHP&c2A^986* zdXqd)`eI#+NNZiIZJNZ1pZp1oKYEx7`v0E+{vSO+M@J`l?)n(LT)PAUk;J3@l$~KI zl|WE5{y-8B;x|cs#tgU6P3vRdWi#rlz!NiSxT(y~JX)?lPg|iv1SKZ$_3)vV#ciu1 zu$iv{pUQwWP8J(>_mA}K_mAez7TkPWZABdSQGQgMxQa?RdO|PQhUat1hM}_yREs)` zb-?;|AAA;3j~k2xoFD!DWK;JTr-uIh^Z_Am*2r^nMaML<|Qkuv4?}j_F z#2tt!qp0Tla$l6gK2mdb?s+~ICWIzpUB8LEm<5-6Memkjn{OSq`s;fg1VB=(0@3k3 zCKlJb6^D&d&2df== z|6qT#qt6)8zo-Uj|Icsar;$_Iik?YZm$xHQ+}w+s{M%h)9}#mgS@GQ&F)Q1O{_9+p zqai_+rKF_ugZScN>+n&8I~bl-hx(KRsux-7 zTj<8@t?KZSFWMx!$~2c%+(kn7EbFxQX=VawpFI@uQR%BJ?tukt4z*3TNUWBq7kSKI zdhB0$%p-`p-bG$)!fJUY^=Z8}{tu9`5>=ON)PEV>A$6Dn^9fYxd_nhw4nsYK=yC4j zsRmCb_2wo-G#r}4f}u1%<3l{Fd#FqEZO5$J;St)=c(1|x=R?Q}NjxYpoP!h6iz%t7 z<%$sMPMUNWQhZA#R?(tf#FN*2EW$1WQkrd99l zhTeHyjV6$O5km}13bVFpTd&w*=eRjSnQJ&znsc~dyF8xdn2&egs2g5NaJ?sRy*pMa z)@~}C-TD1>Xf2~-NKb^zwSSI!HcvLk-;sS!$<=qGw#5~0`bRtblY`ATEc)91E58lB z>W)Mx1-&5n%lM>1UJM)hA|#lm9PQb&#LFRK5|>SdZksu-oOEW0HceNx{IN%mKgVbG zj|J5&_ztWe)k(NfU`Gt@mZ6aaMvE~}#P%xpgtM3}B}Cm3ut;bZh_~H92kbjwcoN>C+bS#>qVH{yl@a3B^p*(_*9|MlptB5)c*Dl%&6B@qNVRYZN+ z=5RY%`Dl-D*s#MH^YHRdR8B#`*>0-+I@YP;{%2eb@6erOi-gU1HiO_Fsvk=XD|xCa zd~LC>TGO$6p)HxXl^}rhO{3GB)FQ>jCqD;qqj_*!y1|-9EJQT;Hm6O2G%dq z&`KVY5(0Y&Gc#*@ad%bu>889Mb=2pu&XjvE!jL)|+3t!5Yx}=CnQXlSs#`yl)Y`8G zP*m)vSThU14f;LzW>2?{u&w@tq~zydsxaYf>vZsO&id5RKF-bhJq+`Cpp@QE%@XzI zi+<10@X)Rm_s{dK={m%Vh!D5c7J+6lWWiKdQZz^0M_hFH@3Y=R{Em6nFK;t^qw?;r ztL3L#(9Tz!jy{|!ZttF*6MwJG2A(txvyS_lry-)n?zY&dxTJH-liNz`VXumHTM1N|l)FP%#HE*8pHnV6 zQbKQGV$w3Q1?#-z*gp1~9Oc0Lsp4+rG9jPJZO(*6PiD=uowr+eCz{F~=au3%zta0^ zh>PnU(e>!^_v9*Uv@0}UM`={po`|RRO9Te$;I5P;EdB`1qMs=_0e>C>2w0vvFv=HH z_eZ)H@iXz&M$H-BdQ-ZhiL!9BnNq*MF&F{^y^#_yohX!!Pys zrLd#n$-C*DE1HYDtz%C&O+o(RdB3Sd&EgyL$1c#>h~f^dM9+O)ueJ?Og>*i$$0m;< zerP^-kaVA$4|~&XXuj95jCU29oc4RtqOc~jFQgfvi}zO-5Wj(7yIA4v#P#_X9?#oUEnjPCg%Po6EP-HM z3;)>>FpU{4#7sEJ+9=Cr$)yvW8W+AecR|7GX_7P>T=J~@Uqb9tKDcj#9bMnw3*TLK z(b#YP7WtBp5h8|m{_di6z9dGbJsmlhF>mF&i0u!7-8K~(-;-N`id~qV+gd`@Qr4n) zP}ilj=-tTbRE;f(DSK*x(zA=}pOd4;7C*#X=K?1e+_7aI%UrI{w*{#eb+_Oyepb{G zK!AvAo*~4yl{b%5{fen1D4wmx;mI44meod8N)^o#zlf#qMa&rub(Qd$ESjXICXQrb z)b$U6`jV=AQE}T|+U#0G7|6HHG#$%gsk-Z*hF4PLLQQQiA?;=N%O--o1)|y&FgC~u=VPW!t3^EvVOO=>2QmNx_h0Tqc=>I>2T$kgiFLvL?b)8KfPS(Hb=mb6w5d)c=8fV%y$ZB!T+QAlqth zt0J+<*d~PO{6xS-^)CxLjs+H_GxNZKJF)`0B(9Lei=uq9f#Fqq%(FM!j5~aHpvCZ* zv0%MdPaLPlxrRMV5b`knX;=RrrY}5C(IA}Hm80kn9{iYG^e$lx__Qx!6QZBjdpXdH z1gS(b%PDPDQe-Duw7HzHPFn<<1wFv8$;Md`&sNut-hzE>sYQ9gk)qD^kN(??V)gqp z09nx1YxWp|*h3HpM|+sVweH2~UTbo5Ublm8>k%3=3W>80ONR)13vCotroSAv(_8G$ zpLfajdK5SPm^B*zvMS+ns+q{Pp&minbODvM|G*_1MLG{h&W?UWK70+U=~fg7C)j?F zXKKzw*WXS%kR*KDb44zX1>#p}%5IP#oTh7}$S5ewpKV)T6WT+j?mAthzsnT54r`R? zecV3zUKCcpqS?U(dDbR=J2&yioYU}^iBY-bB0IP{JQg8vEAvr{GIn;Hj1g%;;ni4k9;+qrMEKtA-0oL{I}U{QF&7HRBl+T5`z|^^ zzkTpl-8U-C02UCv>Wj4*cxtj@xcXX4ZF{ndxUT7WGndo{7KK%>Y19wK>$dU+H~5r-el( z`t6ov{iEaNaE-^_uW25L-#28>z~NY54+3A`bEw-+Mc!eTZF+kaeMtHJh)?mILrwDD zx!m@uz2^`}RR^$XZ#e3o?*75G=dx!Xcy57waq;obpWgsmO_kENkA#)wloKgtyg zQ|Ve9$9??;%h9aCJuzmfj-Q&P#&W3+G=O!$nz2g9)y6qVu_dz{lF~L4PI7mN$oMOD z$ss}Q%K`@#oZEt=F?TyMZ!x(xbZIvp0)HwyIMn`bGk9*{@U%45?(u|A6|vcH#{TKK zeZnpca5kvB)G$k*d$c*tdHg9M z74cw=XVnf6v1vex61ydhZyObDYIZc-AA)l;gLxRso=82eZ8HoN@WYXjDZl2dPs`UOmD>o_H5Ar5FcPxk;beBP4e0P<593|)XS3ig8 z2DnZ2B?woofsglrZlw+jxRWVc^0*Om|;^^Ld z_?9l+bzT~a%5G&yVk>#qv&g$9dSkhIP7($($Ub+}O8L+Eqlr=GEXRd`OSrKv@Lw9Xz< zfdYlP9)^%mc~o)a1~afV+89fM__2x9F6pidu{QXG4NRFs;M;mxS@}+GezVlO36Gk1 z!GiK2rdwWg;&$(<3SPKyIUm2BD_=sD4ti0YtT#{L>$M!kETI~S=6gr`8y*Kqf)6s=OPU*p-dUAZJUoRpOW?Pm_mAtD6$@-`GkO9Zh`j{=@0i_d1~$U;~y` ze8u|#ROuHot7#ew9g35SYi~BdQsE3&Qc{wW6bm>U6W|!!`uLyob!gwLzWzR7n8;E$A03Gwfpz%aPc9-r&d$!hX%q=W_RI_}4e?DGyv}{IGXcV{Tb+0&Ac}XmKe|q9}4jq0Z%1U`+I%xS- zy7m<0Ap6_Y-J>xh<*e%BheZ>mJBM8PWo5lI2AA}ASChUgS7zzGXLb6xkowhl5jZ*B zn`J_aw_D&JyP zj8<6KMx6R=CpwR_TFo?K04qWKd|S_z;R)u_YTeB3{?7#*`Q%Ft0=|D|Y`*^9VtEAs zoOAUjncLDPZYBUqcu9JB7jm?{qijD+b@8s}>-pmm{ySTvqDJw$Jt1ALU?=Bajg@KS3h4J!le~6iK`@!kcQe)VmS9tznbL{ z)R^{gs5$=RaC1fD+hcohkL@74-fmH~vg>THy0AD`6yE^LYC8IDVDLjVN(@Eip^An< zNWc$7K-bCiJxbiS8%Xqh5WcE?t>29FJiPfc+-c664GN{Y1y2^2{aF-<+tyLY;n=G3 z@?FzGh*DAKLZIi->0)|M!u>4)#t}ySd7&jf1;s8u*}Xk9s@|eQ?eb5o?++?@=rd5s zMP>*_a1x%%Q=v{ZU0-jdnKwZvq6vmrP1vzWl=kjbjG!=Fh=A(NZ(sk zDF8vo;U9FXko-PWw#xP+L5HfsxF)|`^Wv_3H_{_O%Pg4T zdcq_nU*vGe%~CiPC!6)U21k2ycEf+ zTA+XqUs#)!wmcJbxr^L1vw_~Wt~WGYu9`$(5h{r9hcwp#I@)|REYSQkB_-iA#-xII z8?7EUmNpXa^I6y4CKykG;*Y7iCzQBjd%NjSBF zR@=gOncG8KlM#A>{2#qu?KY44#Hb_46a^1K96?9A4v!G(cP5>e{%h&<2L#$)hgqe{7!u5-@Q0a*Z7U>ouA}` zaFLY<|M{MIX(a4`Z?rEX2xt*2X{CyGFEf?^-ehmVehsIE)lFML?Bsf3yHl?G3x*W} zh))Q=yEqYrJrg`n3ri^BKGhaO)@*$PVzyh!i7mB&2Np?*uMZgrT=8wh_DJ0O=* z{uS|6o{j9D?Yqdlov=Ks8OU}dQA|^sT)rGSSWKBD%|FsRn&DW#EUgD2bxQ)*y4jj? zT~FaX0;qg*x1s;ERWO5GG{fWq!q0nuzrRRyC8McHTCL3{tr8^LIphJ<;~Fb|iehyZ z@MB?Vi2$i&MJi`nzf=14SAkZE;wHrkjfgmd(MU>5JJ}P;;Ab=cc*jXAC`d}euKP?% zD)73Qb-+jQ=YLKQ3lEnzH_vY?0;eaCO5%Z@kbiVE-rKit>qVIcDh)i$E0jrrW^xu< zUoXhGA}zh9pG(Yim(UTz79xhwD*xTFh#e=pE0zHUw#U7oz-6zk% zXZZX5TKu?$ny-z}E}e?&3JCTemeAi|-(u!etnNMq-5=+?DXdgQ}G3INu>(~ByrFbGPv8|{8pn2M!pNZ9BOr0$NBPq zX;;+{jZ9ibZI)j2z)rkpli&_aYg_dkFtaf>{x%Gw1FN<TJi6zrfEMJMjfOqM#D9Hw!~0Ed==YFth}9qgY5@C z@&x$!Ds@{5lj7jQbOK7sJr~)t1EryVX0F@XKJWpTwQ`b2SRgJO-yd}S6)GA(D&?KH z5`iYhOl~*@blbV-KYaxJ(Z_>A7#F9G8vnadme8Eu$}^NZ2u{w|z`(}-*zwK}h{xTA zlAy!}{j9bkBD>P&<5yVJqNFMO);~sbUfk-X`EmN*d3E*nuAU4_^iEfq&9V&}kqT}M zri26qAs6enkP{IR-Ie0#A$mM|V)nOdpjRq#7b>TuB&(}SSv@o1+Y0=WRa}e}5fP!+ zY5+mU#Oxf&lKPUG8V=c3oXvv?3xDT3T9Dl_r?S z$H$FMi+bIrRdf;(G@ZfNQtImJP5o}NvZw(80k3s*K7+?h*i24NeyysydYnj$j*g3~ zF)Vd|cUxgPNY2a0S73sLibTw=8(Ln@$;;2b(iKLCCxh>OecI9Yh5pNzFaByp>NV2? zbOHh-IWMAQ4Gkat`0)eWcy?o#l-oFz91T*WQD##mimxJH$tr~xWHZhGcOyQ~1UvK= zb`dgaep=|Y*y;^J7pQ2{k6|Rp2s~y(L7?$*FyB;RHk_{RdK-d87SJEdWOQ?R#LUcW zesNIRMk(wBXaxx#Ftdzuy$=)D2ndzI=(^6G0JjcMCJx87sKF z-Ypy7WK>A>1J98v?EGMNqC^#%x_jpFD~Shyf!Gg@bB0OB2}^Z>(_E{|53Sy`Q$ zTEN<{Xv8VQ!oq+XUZ|-N0l!xG-uu+eN5sa)a#>ArGB7e`7ZhNC{V-KysZrzrzLuGt zT{c79J31*T4A|3i&ey?#8=!9_rO?XY`#0m0lSbfKw-(zZ-n@AO((;TO{L3NjV0Y;? zmZuDwu5D}>9uEyQf*FE;vjA)Tp`Slny3&N;Q2l^g!F+^vZ>pkmH5z@r)J0mqFc2}@c2mNG8z-YC8wZ(M<$hs%M!5k ze(OPL`MVCpuI}#TmWu@`4-et%^Zg>~lbsz4tGW6LwKDjQH#Re)2R8*9i!)^~zqGXH zXth@_vgDDlFgZMgpFc;0$C#d8M02w!Sbgo&l$e-+;$l{K>@zS(&z~=}c&g~??u}dS z^m$r=wUcrgB2SbV;HD{syzdOAd;0VNi+Zu0NO1r=o0yP&O2FSFhQa_e&BVk+oZb(- z?-05GxqDbP|-CWv(j8Yc$^@lOH zWDJ8jFF9$1Yz@Nle%<0ZX^xZG#dnr10H5{;b3$D`J%;Um2!bwqnnm2eJ!E8L0HuIS zEgU_m#DXE5_AAoB&Yi~GS@rd#Am%~PkqX#Q27{f((R%qKciDKV{5|{#gWPj_doFg} zdcVHDm*2jBHylb80>46f@ZiCW_kP-AIRXeR6B8Q9tKiE>LD*$8cAsnvb{47?y*4nw zg#b%+?oC(GeR^bJVF7;xAP>?#d4eG)CkOV1bHjO6RTbC!AIRXz9+HscG&fT|dGaJt z(EiaIRn;T#WQEYI(^IEkiCjn!xXccGE-fqj=<#C|a0AUc8!Dw#!O@M~?JZ9QGDMKB zKZ8JcNKEXa#X>^#ncIB(YjW-b)*|wt<*q8|ntLy9Qg1&d6py$kKJ;O^WXa@mtezJq}6GS&2 za2J42FW7$FPMUzqL;JIFjU0nEpO^CTodB#5rkdQ86FCt;3>bKyeU<;62BK7}%JltQ zy}f3Q1rfw^Mc87D8ECp?t=B(IC@;50bGSTCx73~A#0&x?`1b8v{hrkH_un!tEesQ} zV`FcDzd;@=W5Q8ANcQv;;02hgp=*x<@MNBGH4qg$fccZ4^G;l{X5;Dr*WP3qO=VS; zih)5C+@B_MjRZUnYcH*4Yq9-qn>W_h+AV4f&vwT13k%;2Fgt>%dgKJ-q@W;q?VH3I&sRX+p=gJDiLMzT~L+LKzil7nfRcXsYqF75~lU zY7~&}6o^2A9NBj28yK+s%8(EjmzkfB4zj?@SFaG?DUSe%vAesuQYWn3pRPi~#>TcW zh~mc4-R{{3x&RW`O#pw{=l5sn(Gjz=vpKyk?G#e@u_5KQi{fR5u(RC|$v{+)1b*(5 zI=ez2LY@M`uMTNqjkN@I{nfC>qR&z0%WSf=tU968h{fda0;DNaQ>u$x;nA6 zN*9P9J$PG|g{3oz#|-b`!yp7C)asMMCMA7+{URZxI64df_8K&pI5?;f-Fmx@+~Psg z&Y2p^qXkcBft)M23*h6&Cnq|5ejsWy)e?@pVdw8b@LFUS!}lp*<^V)zyl`R1)T$t= z7*`(RRlmAm$rQ45{Xb6+&6^@i9)6$z!x18`En*86HHGtt-v9T4Q+!0i(M0QDEDafw zru1D#WiLh~Vskj77Y1#Ms;=fWHZ_IgH-MqOzCH+KY-}vw$ndNE4gmQPbKZKp<$&3y z&Y$znn@#u2%gc}@`To`w71!J9=90OjyJ z+7m?+1*(s@_;`;!G6E_pY)AslhbFnRFOn$3PaV#p=d)wDv>Zf}aK=Cd`ZssISW zjR418kjVOe{pvEy@J9XI9WXNT8gh=@RMItW&Rb|S`Ze*1>i2g)(u3F37oEg4t_4OkMQv`Av8)uA~ zpOYErg4}A`di8zY?#Zt!5g#{vwoswooS8Zx3NxZSOn|K|Y53aMo zB|GAxX}Ih5xw^`B7arF9DanG&a&|sG;=mM~0CvKuMAX?d0DK3c;bv!MJ`+25b=yLf z41lDvvXb*%J0vJLxDbkBdj+D=Zr=I9!+-pjmXDwR(bK0t^E6}o$F_}I+2AzWVGSK* z8F-U&x3mcd&fP#9xE#zC#21y7+45AN>E#JpKVcb*UQ6Y z_5ONp`;;cK%CrWZ8P-RS5I+;MJ=fLMZEC>>JeN^1IS3HOzj^!)2K7$9YOSp$&dtqz zDJRzfsL+>;j54LUjRA$jgL;wRS3l}~@9+32yhI-_re*CZ7g@uU-2yu@C;rUCGU;8f zT%ZdPz|g>TkHp2Px3{<9nXvl9ANb>XoeTqqOC?{udhqGfC%cs{46u(S_xX51 zxPlbvUPA?fM{oM|i!rkQ7-M{Fj8;f!I#1K~;FkvrD=QX2ChFKGua{6X!>I&yZpm|#|tca@qi0wS@Xi>)d&i~0%~>}^1`;_ zj($FP%uh$xVSNQQZi4;kE5q*$`a@2_X|ug)RN`Kwe%@GafHuQxB3<3N<%5+^Ah%^U zHj;z7+lFb7;BQGH4LE$Q)%5Sj7;)SpUcl4=fq~=pH=u?)eY%ifl+RH3s#9Jb;1v6O z$=B8Gf@X)-7P%glr%%VMYnpvhpTz!hPflBy=ld=?F*f$?JRBVZ<3kONR=moaH`)9% z%)Kw&(+Ua-N^0Re!Tn^Tpz2R=Z!eOaM5OqWC;dfDE3M7Fp@`W3sh+yKyGu;HZIo41 zz9`S9&#Rd7WdOIsoXBC0|nF@3W?g3j93f44b-Ev!PT;z&Sx4 z6`np9 z0{IQFRu7Lx53wikLL3g|*f_7~L%Fw80M;VLGAXN@8QEH{uz)Ql?H@cp21YcQ-)XTD zKYaL*R!nTZsL2EvSU;-Zya(U`J^?`hNX|<;MP=|wfuVG<&y9`3$Y^Nv{QT;QZ*geE z_bL_sOqA%r=_kl_@CO9M5iAUBKM<(TyuDlG6ApoTWHsMN1StOx!0xnD&flNDqQjcZ8FPA_8KomMQoS7DSuFZ8Fd7pz|R5C z|A?OcOLFo9kVdd6_}%8WPUl@0mkWzZpM$vD_>}~wGQc6U%*^OONZx7equ{qXbgL8N z;J^ikr9XUl0!p1PadAP3iKHMc*Bn%`8qL;PC-Pg9+te?kri!@Y0D1*vnCD(zqR*wI zRMgeKm!jyasePpJy}>u?jf8W%*D5OhpwtJBr3Iw~NX$FPp`beN1<3U!Jw0;!PhJj? z%i-n+n0vA48`bI!GhAx-@9XaekyX)~jzi@gg^xo1SF?RM0;<$62?-&9q=C3?N~0|V zKnOfo($nY?4|r8Pn-0LV&j9)!fw~k_Nf;gO{eDWC86Wc&!H_CLF9j;TT|tw6ilG$br$o6#&TL(gYd~ z@$; zfD+oeGP=0}MgxrgOhyI;erMn-P@;lZ1GVxBP{m42QzYCDX+br{Y5I#4C<@QZCRag3 z3fe6}fEYEIwLnAs83oiVM4%NB2&gO_KR@y4=qUV&wY11UvYIQUKiC6GP=0Z7SEghj z$Onj^v%K+(%xbEBkrKLZ`XU%%`HNGtp{bNNc|6652)i*G(5yI2>K zefTm6)W`W59=ZbKzPnT`^Ui>$dYt!_&PfdGLG^9xd5%DS+?nG4Z+Z{7r^C8h@%ThU z;{~_%fW`ml>>QuYW+TSj+1Y7MF8;jrM!$gSp6qBuaLk6=g}>w1<<%_{{v|As|r|(S}%oDJUpiq(*UY~PkzXU zv`Mkm#)3N?sAhu{!@s7>s`fZ|=^;*C;(vVbrhO$upag3W%2f1Dl5N&8z~>FYf&cd} zJj_3jH-SG>XujUvb*tfDZFc^748wmNd(~6pGtC0-t#`^G7tn8f?8*n$J!4!oeN$=h zck_+0f%!aXmD7K}_@_9tKlNy#?&#>aD@6sXN&&m`w$dQ-zZs4>o-K<$vZa01yEZPB0LtY0>dy zMjz8mm^W@x*VYyk^yfWO#_$;EP4l&7DY!88wC2O9K1)3D2Z}rVb1YDD5<~wp?x2-j z$`EMo1C_m(wZy6kCS*VyR+Z0mVp!0NDeJ1Ux;p%x^{k;YXWT-JAFF!&T_&ZcB;L$? zTa!K}&|LhyH?$3$;6v&*t0)Ksu{P3U?F0Dnq7X^@+jp&Y#Ll_BDth%ad9A(_F%6@R zUT7ECmlso4Jn$lH@(ai8*0JMahi2vlHSAgF#bg7LuM9e|S^ZN&Ve05A&;4eJ(zPAQmfP zM>DW6q!5*B(M?-LTM@tHXE9D8iWx-gBVZ_6b?tx9Z-Ky&o2tmhBp;bkLi{qO1pyL+ zUHN9h;-fL2RIL`R)?m37TQsZb%ZUwN{vsJ`-ty$+Jy>nxJJr=au8HrBHo`&v-Foza z0&xt@^jvJKUr|3sz9FY2Pf6x{*JDlUasti7d|=JPC|HW?Kvw`VdK`JeM`o!ojTOStMqc1Ckp70;qs&yD9zE0~U7IdIGtmnWsr%JX{~ zLe1^U1!y(t;{k@GgK_%V%B#z*nq2ebA^TdVg~4(2u&cL~u20Mxkv* zqwA~@&Prm4#TJOTxy$XNMbF}k%eD6Un~ke3D9eRnJo!d7Jm>|<*%&_eQ?HW#&b)~& zF?9|_vLc>h{l1ZoNlaL*c&0z;yR~vJW!$!h(_(6*$;;jbgHB!Mr3ib}HwOKV9y#8O z+}guKd#q0uI794;(K2o2BDW{@_$b{6gyMP6LtJdMed`z2;qP<)u2Aned%y9GVJ_x_ zk3)-J58Ev)-p0BXb$??#s~krxDnN)C{B&82A#N?QUAavvx=YQQ%He<&Wrw9~$NO}O zmAEfPZU2gPnl%~>Pg$$;SVb%dke<4*QDAEG(_Vt12@dKn z>C}!So7Tl|TsQA$cL)Ez_wsFo`g+ZFWb1Uqoa)}m!~9IvyxJpFSn9+v8;86Ehdct^ zTpB%_m|O)jBH!_b_4$bTzJ#UodDE~oj}=^U|7V1lDP$^cw?ghm!T+SldU{CWz3z(_yv-`G1qJe zm*y`ETeN9YRZkJ!@YtuqCoTtprHQ-kSH1?TE!$rq)k;HO-9TMjsNgB*ts_O5I#6Ac z1}20+S_%ynmQ8_8WA1S1Ra7VCxv9{s#k5Q1_{%ymK*`yVOK`Z6&7Fn$I!KVQrKO@< z@SlC4d`*r*8G#&-AGqbFkLQechH73d9IBm%irRZZUNY2*U4j#kuS!894gOZgRVljg zXy!&C&Z?Sjma-MSXLqC(9OhUeb;_eL(V2JoYu!R=Z6!kV1c=C*DDwpyc zC^pNQ*?r*jlMRx6NOdnljxWuI9D%Ndmx*74=j@Tt%V+4wf#^AmfFD3BT+328n?MJN z?5z>bMd&CPeOB~w%iL~cYfp@5F9QA5>PO)C+6I!q-+h^*h1ew8Eu))8f95Zl%Ap&qO=And`>&Lxz+ zmZ+)XaHHxWz+5Cc#d_|B@aAMKG=`{>L5Q55lI3|~%8ajn0<(dJ9YlAM44OZr(7Z(n zonz^Utqo?bCR$XFLVUb1OGnH`Q>V#l5%_#S7T30RwNyjeGqrAW|L~!KM@i5(Q#yw& zC*;St!Ll^Sj?O|Bm_A)sG#ajMI`DQo3kSywEiIv9b=V>jaP3_kEhLZ&{Yk%EXjs_x z2><*b#=0fPuX98A8=q`zH{A!#{_ zExma6YUOhC0M`EV{Q`M0?J?`8QBPXPY2_l{;|+Gw80YFo_U=Vs56xD#n>N*lh&U0s zE(Yq27iIBCv3kwd5iCyY<1b(hgIPS#AU^SEdNL{QH&rp&BuX472*FQ^#S^$gjl0=| z=8IJnNjbM%ZKQmOj`lMdh<9d=1XR!cayi6)XY7gBaX-^t>8mIq_hSZlG1#>i)|U>$ z{1%>yQyl8mk6xU#-J@39fjIj3|cM{_B_?>LhT%M_^M9T$0gpmajR9M)`vgTyzTP<-BxZ%v#> zjXuaCBq+f8l0UaMK#hukZn~H=g?yY2JAk!L;wsyy;YGiNzm>%rUg&3q4ELx<9&4;o z!xY)7(MPeH_BVrQuYIrmCT9l+2W3=LR5adx0rmC%Tm#|w`1oC^Y}6AVY=chK7qJTx z()*K5y>!e^;hjrKbw*34xtDjt^7ale5>RA?Mn;~iT8r(DuuYvaK713yWpJ?fboO)% zZwb3YYWk>Bs=r6>Luyc9LR$49PeJpZ6)*|l~q&*@~fcpE50dfz^kUA&sa z=_T@NBe=TPT3~aes+QqJmKxik!ypfhyh4{QS>SSdR)oW3K8-2qJ*R?7?uq25ZtIsh znt#F)sy(^6!e_zsBdh%R0!roJ=s^1iipcl4mBxXP=?cYyhSBGhLLMy4>uk}F zwIYaUXPH8rx@K1yV6Pu&Ssbt}Gnd)u*eryMThtDZzpLWwtqu*mPA(qo6hw6DlUW=2 zV9mJlRPx#J_{H*+rNo8DEbIabl?HPc9wDGj30~Y3>oKgz1ZvKc9MkGoU1=Yr*}jWk z^NY8bF}!*;;Ni>dQ*p@KsuaVJK|wIHt2Xkb5e3EhW+;W*MkVi0O;WX|$B<#gvv`v- zPe!+$uOms+7%007l0OqIzX%!@PIA?(GgaKrOE{TJfZ+m*^84MMFx)0?{TzWpjF!qB zf7zlCMdH1ahaino1n4r!S~VmGC+?AUy1(V;aEVukBf_YH=Hi^Qdzk-UnS@8cz`1(t3zB@8V^|^RS-WZDGBuOL#C}{ zK^vdndM0RP#n3rLHU~e!z()w)sM^{xhtE^M>uw-u<4_5&o!K*Ff0iGx*%6qtLjrxJ zL(s~3|NEO9Xqfo*8$XwnL;&hz08ngoeXiDl%pe0+=iumA;k>J!V2uN&N#NrUR#vRw z)c`PR5Cx<-Ai(wJ>H>|wwB;PG#kQMP!4;*cT5AfpPy^cGL@et0CPtvUY1D%STE<`q zE3-Bd;U2sw^KTZQmKZ(@0R|~r^yj}yXCCS3;(zVmNUkuz)|&2%a%zb;^J>jLrdEas-+!P?_KY6ws%BrHfI389B(3u>T^69r$ntrH})% zPMwVm@G~kFsb7jsBSwbL^%MAL0z|Xkj&5aT<**Z1VhKb8Iu1?`=-=Ay*Gy)E;urY# zaJwFFv|A>I0ZbD7%8;M|N?a!x4dHS>Him~8d=ACQiMRFUSRVA}e4UdLc=2CoSS)vlWR> zKYm@`^G2Akh~c;5IML;~MuO4djX~BupG4iTK2&x)%3!(uC)4|aqPK)cLk_L8(3`eA zhfnb>_`>ASxzG{)VMN=Bx$}>e^FyU|`z)JoI#SkowM6;VJ$QL^C_^l~TdCv_=^7G5 zUhNtL$^E3vIJE(hc>lNLq^T<@n=f9gJJ>Bm{^t2{*S%8d5X7w!FwiQr=HD2_KXpqZ zYmFc1ZBQzE)m&6i&{3*)-oFIYw-C6<2qaAk0h_n?r1N0TA{dA2Q&*QXALs&LfJWO? zt9MiK<;LbF5~L@bED-z#Oih5XRYa*^tPapCKD^q@kcu~1TVG!W_*TFDJwu(K3i8h#x$#`(qxzR8mrxxHmL1vOnEY0ex1bd3_rj z_Jg*2A1qSNXNo;lm6dozM1SW6BH~{}QmVwdZyI~eRDS^TJ#L$6u4DR=?(XjJX1DXC z9=v&8kdbFypM|)jOeM$TjpWATJwJ~8O3bU>dx&nb5XjGH@Gx$XFESl7J0~wt z*2uPjn0K&+hH}`Fa=6Y*DF+`RA~*uu$Ix=e4JoxG>-!&=wL4k^P@KU)i=fX<{dFRP z9C%IQg`J(Fastr1m%*4oNo}5!aujGKfVKz$0YT=dd2~hw^jJ&t?c1QAKVKzOeE#s^ z17y!PP8je;(6EG$vpETaiAxrsu7Q>qnAFQ^1+V{b6dYwsuZeB0O<2^x5dcibf>#oq zSHP%ui^ToSiTW%z7`Xyp?&b#8@mmw#IY_7k!))+ADMZ zJX$FTGIsUPaugH6z|0B(^n7fHQUppsCFs6mBdb*7HsO60w06w#Ct&$>-h-5jCYd*ejF6~K0Hx*(93CJn_Etpl2fhqZ#f@jHm>a{`aDp!PT5EcTt z3evOJG_Ft3o{hOrwlmKb=Sz4)=)6Nbz3+BD3rp}fwze*;yRP$mUti@<8C<`9h>Sr0 zB2pSC+@L9f51*$En1wgyUpIXdu}lhQ8EBjLf$dZ8Qh?|OBL!YyLMTzxox8N{&I7bQ z;=X>(0t_t#mqu1mag*Xn=++9|fJ0H2MhruO5U1Y;RC;r{6wFn@n-=h)x6V#In`^=0 znNqKbn$&*qK0t@bB9L}Kp9u*StDQbdbfva6y)nA(AvyUc5EOA#a{=Gp$Ih}DG@FS* zCVNl~G`}vV2SAv8>4g`+1*G(jWu)DGck)OF<8No^%7jdxvbXBO2))BxmDc9jHn$X^ zkj*ecgz98y`>Zuaz~TK=6SO`*^6Y3eL(hWN=sh~X&aHtq2Z&N;b#a(3z4jE2%^nllx$rKN`lN7EI3!$QeJ?Z)m_b>k!TmJ#hWgEYN!zx9oR4U18 zDulAPQc?C!GO|hb-b$1Tm5`l;Bzy0o$d)}q2q7zbzvun@e!us9|Htt^j_2rjjPLEf zulu?_^E}VbgMS~3!W0x69)-)oadGCr(vg%GHe3G$!~5nC@TJs^sltl9+&&E-&ShA0 zIND{LnK>x)1i83YL4N+-d-rP4xEJu;EY8+q3c`b?@hoX@g?|UQ%xpX^EQ}^0A)$E= z&8813EWd3ExG$8X0%TxqV{>Y=j=dpe{n%DgjH<@s%o4}TC@XqL$0e7N5${tEt*;z- zbWU@Hc5jZNV(P=fQU~);yW%a0YpN{=KYrA?y_WnjSuHtUao1d9UY*i%G_xPqHhqzV5-~Co~#FO||*rPC|WW=Nm-kDYo|v4SmYW#H^-8 zqR=;#ihl>tu|_Qkp6Czh5TC_Q^ctU7^cI@#9O4Xnj!xgJkdVr6�~K?B2UBP0|l z=p;@~PIoOW(|DdrW}W&5RF=JO(X9kj1mJXSb%&M=zG}W5deW6WKk-vFU?acS0EeUaZzN6f6G~*F<@fnglt!)(_($td8q8^{p z?cp=f=?+v)+^h|~moJkrFfagJ_3HKOYXF(1+{sHoFoXyux!izXoRS? zlifUZb#*2ksq_H>0l=kgGrW>lQX&Fd_8TZ&C63ocMM5DVAplzP)(9LV9hiL;8M*f| zqBLf-fa8@0rlta(J);0PB0oRBDcAhYt5>h!ABgn@a^+^6TX(McGwdR{hUbL;8X2iX zC_2y!rL^)AJeR{|SLdc}nEpI$={ zwQcF@Lw^G#W(Y$czkPe+w~auaW&a5Jp6CzVwXg`lIWjUb`lMA95+&rwY4H8P_8mK3 z)()c&nrYs{i!ZiK9;DFK)djTqN2-?4u6@U^!Rf-it}k@xXqLDfIlrcu2e1l~xJF#{ zL1uI^dj7yUy|;OIf_&lSt5@Z~{6W3#F7QTx*iunbSNZOvwX(DO`txTbNCR2;FaV|M zWga3xLm44e02G6Ykx?E2JfNEI+DA1tG>(^TZ3=CyF2sR7aEyiq{`(91!2$>mxW_;7 zMn*=O{CKw$AjY31uGH~LH?xkiUF74ds;o?CkK1J64SV+NnVl~~-f#wk(Z(u6kK<|LD=9 zN(ov6(Li%HXBtW$y`bLIDSMFgiqqXbmkBnUH)I?p`h4*kK0ZEJJ<<8-*}_Rxyb1jG zK?xKBnu~Y6+Idf(K8@SWw;h!`og;OCG4d3W&#?93GU8{!B~^;b6wbo=lW$^K|Bgsi_Q!i3yC4KWpBT`>nd#4>*GR z_wUavEW~3sHXGJG;yiVX`oFtyBEg%GKqwj|X#dW#zx4d!JpjvsmNE>x@82toYwgOg z9+JRPn6xD91GEw?UTpQcm!~DOInyW#uMviuxF;`QvmOAx!0mv-!2yOh=-%Vfl~|hN zMa+S8my9+4=edc088{C3?mprt-J4n$O3tPB#@#8q zv}G|d*Jrk8G&N&_Z=q2$N5ellT02VT>VG$WRF)()G4Zh`d+0-JAMppVvAR*-v+!Tv zF53V9Kf(X4%74#C8TP;D@4rVJI7#)Nx%=Oz7dH66S3&&r(PJ6Xq5to~Je+yqclZCX zY#WspUjOq!-;K+ze^)zo+bj2q=!-m6epdbT`(Q(Z(BZGt+zxZZTE>2P@;@JbBw^gw z9hqy}qc^M{pVQm<<&9BP!~k!b{~H~>CwGo^@b5cJ{OMi)_ZE%^JDb_MQLpITHCHec z2n-dun9hA-wd@B^-_oPqR9042cY$P6^y@!j=H7U1{;|1bN`@|9$yQAG%BXbGt8VRv5mcD7a;Nvn%4C*PXM% zN!a(M{iPyE;(x_>ZXy7D1j+-f6zt`HTaw}Q|E^i=k50<;s*lft8yr;qp1!I(^ql^= zeY6=*d@y;X4_Re-`F1Z5WnKgY?YImU6yikxP8RGw$^YNS7c<(rLW+69X#4vwtG+8| zYVvKE6FSXToJaD!AW>i>dgQm{XvZI()_Xi7BO~*FyI`uQ>+0)o%F6>$Bmzdmt=kfHJBAB1NkQ&QNF zMV{j03kAUn)DxvR{p~NR4VIUee@#w;rnL{Q8n}KWGst=Pd3kw@=bNP?MO;JRgl;UX zudk!O$q3X5AXBQ{Q-g!R1yb;``*3!(vvB`0}!g6BpXQBXMD76BL> zNKU{mhHIC&3=9rFL|B90L4hI^KJPfAvNIl*K%l$2y2AHYfB*h#ye%2^J%$_&w&1zN z#mfGEMrmnjxEXN8Gz;zgvWy$03=18n<*-b8=SX6I5;J-W)CNmgPHP*^yX0Ddp5r(1>OYl?p(Sr;O0dUeNzH`Npzc_jIJR93ExKw|8@&XYExhze|mU(Qvw_#E3S{oZnjH&$k^%1JxGr*b$MMV)m zv$D3{b{8eLpT*8pURVHF`N}pCpOiXf0Q`z zB%b+_`e(? zO+H2xh8@1>+`?2}zO^^FEjUnrkfDTf=u{#!a&&Y=V%e2v88d$zP|NwLURr!BCEDQp zgetw5hcGgIlv@G@^?LO`1t4C+?*IyXx!VDMjFXeIse^-d-D#qoP0-=*lhV@C)}ZO} z@fsfr>Y1gbs`hrZVK#AD3N%xKWx31kpK@L>$(ux6PDy%z|7oB);;+KrWC(<8JZa;?Zn zIxw&}c7DE3-THU$dd0<^_T1XwBY`0!630=VmgZ)X^z7_b zXLdmiq1qCofE;h{t0{r?U0q?XUL8sb)jrM5t+JZ;_%RvYIo_t$XoB>JOCz!(w=jh@CBF?j_Ag_Om!0J0)psJ`2}mYc|aGROxBDd(F|FzQVbh zEDnQ?d=W?wpL^Rr?{v?_Yr5mpZ7!@$S65+LP-y76;0La|%l^$JM^~hd8PW(jT!0?| zgi@;emKL(U7l|<*;YbIYiySSnk3p^5j}+##fB?;Tib_?MBUmT&JO)Qbh!7D+09DW- zoCjDi`~(Nn^a=0lokch|o!#AffO9VB&u=m&;C75H-0Vm5S>Q*o|L)cAD~kz+C6b_0 z2u&FXoIcpJ$lTs3oJULyu<+H37bLIuRqWrpm$b&W z7<(MY_(A5$2~c*K;L~eEIf8d+LOp`+%Dp&h~X*e;ThYemZY+djtTf5xZz`>Zv1c;IfBt^ z!=t0N|9;=xT%VJNBe@2qXo>4m$iuGAPIWpux{KW0@8N}h7TQyskC3;xcTWnDK)mkX z@L!qWIpOc3<@OamuE1t^4bfJwF+II+3_vVcX%?mS^udIU8;ku(o{O_BP zhbQes?L(udzOK$LclW~+DiZV0DaxtD7d*F^72l2YMY+X4P+HAUzNVP6l-#QPPF&Uf z3z>pW(yewbrtyvv4}C465B>k;PvNZ{L1QPCYhNS}e>V(V+L*=)(cYmgT=-L&^W99f zm4zx#c)z=}yXLt9wcDxuId7zGITXWH-SzUhn@Mf*{HoW;T6wEVRmn=+03zU*xFCyGxmk6M%21dLy z+f+FvpsK0q1yJ5#DzWA40MYFUTxd9-WdM#SLK7;gt1zXTo14du9XkN<>FfF}_Vedo zgsiZ5dT=Z6t!Xrd`AS6c6r4M@Y5S?VLAy&JwFZTMRE6H5A)r}cbG>`Bj*G~J$wDsN z7FSX_goKc&G!Svq)doXOn!dh1bW2V46VLf#nA|x&mn9s z*O!Y|Q@p^DprTeCcazjE12og%-n_00U*Mq!6#wi(%i>om^;+gn4c zTOAq3j}A(FTwU_&JYiTV^jb(LC@4@_nVFyeioXI! zEERW9E=E)qWOxV}jKX`stQD7(;N1=Wt`4%VQPQEpetP@%trtoqrWS5!|6StaOXJ>A zbb`EmgGlrOGn8&K``Ba@KRv;{^V{LL7pxbR+brDwCdehJ+}r7>#;aE6=E`)m-fqYE z`1`-ICk-w47@5~yv@{R+c|-1|%kHk1UFRvr9fqzk1u3eid_}M`)|N~S1N$aC9A%?& zlyBd+$*{b;%~ga=M!IX4o`JzNkQu-Nq!V#bN&ZWG0t;c@ogIrH8267)br#qf2d7yB z1Fw;Lzgk&dPIUCOujr{>X~!}k61?b}W+ zE-qv=oh7c0!Bsk8CuY?>#u-2y%v-8lrM74e;CBw{Iu$jKSD zmnvv{w3BCk;+ff!35T_cqWjK0dw!paOFqHPv+LN~Tkpw)S7Qq9<%ENCrt&d!VAWT` zw|H~9BO@#CDlCchsfW*u88t7T;oe7N5Qs-WmF0F&4u=f0%QM99rfd~VgqT2 z^aii$bpM@={_OJp*WmwQ0hq|;(@!~1j(So&ocR^3^i)Z^Oy-yD1C5k~^mK(iUvbWr zwZp_I9v}7p$$7-V8MQCAPJ53is-%6@z}M<~ zcI^E3PIAq|35B-!_@xm`N*XTGv;b{Ar_`Bi-y0jNnwwwcn)eL;{!NrP;p3NASCtU? zB(H4Y;|2e1(~~(e>&K2hN|0ew_SV+VdpGr3DxUlLuFo9b14ZSRelXusIWO_2wGutg z^t`+@#V!<&kQ4Zm{e0&;$Is9Mh{NtZd*0zhIXOFnd3FPIFmx)N@A?hF-xU+Z28OE( zHC9O5uExF+36Hyl%my5x=1jZx`V<-=`%iKOuihOx)hMBN?^L^HY7(L4uy^l0mBlM5 zFprJ=6pc zPy1x%IXLf!0@!AxdXXL5PDhGs7quydJCZ$q3O7_Cb5yt$xcG?YztRlDaUbzmo^{?2 zjt8cqrv!)h`_lhj9iE`fG9~=<%=aH0Vx!V=@62-)V-VGL%f@=RFP%({`atPrJvpAc zbg!`|t;Ao)Sd4CxUh({qGiT3PkF3*#Y-ButhHk0pyANE728PBz+!Y)h%sa0KdNIt? zv_JkX{W7>cY^_prr1Vm#eCf)gozJM#7Xnk}$J!(lBbAcQ@UR85cBG}>o!uS~BbvHU ztjTT+tv_M0DQAtMyb;*GYu6=b>jV3(=JDJcs_~XqI=pFpb^b@4= zQA+yC`uSP9SjEij;+NQ{tHEoQ({0hpuJW^Mh5cc3Um`VfQg}x^FU6P!CC>h=8zo<3 z{Z!5wS&=0e#vV~8OYj~9v@%R_DE2f+M_g&^&(nKTctUs} znr3-#?NRpAmjqtAKVGt)t+^h4o3|n_i=%>!(zvxrQFyI9E7G68r&HFjEhhQT!6t8N zrcwpNi=ty)HxD17&aj!3N|`K6TZ?*F?rN46aV+9jZ(qc(sW0=>y)m98TO4Xz1sX0M z)s)i5_-yDB8>0255^YU&dme@<8)ur=^Cpz?2Adbe zqMEwJ`iEI{o~*&qc(FKz^PQQt6p4*dsk0O0<=#pf8Xu2n21W?(KdpZ{h0|xJ&nL}n zCDH+6$0+Pr9i~3nacxqSO-4J|_k&JHAyS~ys#JyZ?>##V|0iEEd%E!KgY&K-=jn#i ztgO$p<(J-lP%qLpTWXK&a}2i>UGaS}aMJ#s&eiU0b1EH=`Ll7+G~dW#+%GG>oU&e= z?q$4Xe5?I&>Nz1vq1{an&r&=oIg+dl{uLitF8!~#JkeJVXGKd^FIxQO4~XP1e=;g@ z!*GeduWFNR&Dng_EN4&UHnr3{)uW6a4bK`&zVnHBDC>s##hrVnl%!th!b~Eqb|IGY^l~hsl|0;x!uOo9r=JNm2XIOfE zYssC}rE=%M?$NPWvC>(xhUU1c1bvTl09scc$i#h(6co~cGtkp;qhfM9!DQ}R)6FWQ;8 z_1$AE;Y(jriC{%5neX1;K}%v^Dtf*1?^x4`TEA(UJVHs9Hnw6p*7<5$bS;3LzUZ!* znY57_Q%79dT;G;!;KWv|=ai+943n;>Plj8)jdt`Z=L4-BiXUG`?iQ=~yJQp;9JTYc z?7lY(gBL3IGI}RIK8JGO#Fpl0YnQUy>SUq@AG@(k*q3?c7SrLMPk*j!o?^c?vr#_1 zJ8*AkP4?XVs))4Gn|C`d)6vu2l$9QaAmC=va+zknTWk&Qn%(?Ba$s3`R?fNl3lY}m zhWC>NG8jx5+vZr1#!Dm~%H?}rac=)y&9gKG70qigy6)rd4dJi;&3*NM?thHO^tD`A zi=-Tty?~U2z^*s zuVdzVS#9u1qwHg|yWLo;Pfl6NS=jAqY7hD5$up~>ZK2a*TiS{DWgE-1Y=!pU7xGH+ zXDi#>WcAcpkKko6chW7VpzCz->bvS{%e82f5|}6Kqx2vqkdeWHVLku1q_*dly#ys~ zedyNat6;IT*Ak3wn**c8yc;}fpR~SAPk1C|XCL+y^E7IFi0PVtj{oD0A z7t^Cpq!nVY9bYgDxIs4!h zk)*TR(MykTd`XN8R7vF>-HP%oQC8c`qEs(-%g7Ea+rRnkfjQY5#0f@qVTy@EY>m}l z{4Q}kKRGV(+K%0YbpG!0uK~OrE8f8iB@bN=#vE&mU@T>ByF*-YEs+Dr|(l<@JRQyHAmeO4-ER2dTdM}ANGkWTH>REi78G6OoRq86Fuvq#qQ+vUp zw3pxRuY!;H#$WG7gpr9`A2!c0o!g%7(4x_x!{1nLd6W~rh4+3hiPv!+*e@DUQ>PTF zHDZnkMZ9NESI>_uOZh0yJMH&l-!E%FJg%(VA~9``__MOT{V3}0D9YTFl|6g%JM3c!i{;=hu<82tVQD)i^YVBpDgeyL|1IruSbqo^Bu!{IV4ib%*8t+{cAvR730DyrVxs z_@T(3bnW*wm32$X0@r07G;`<6mZfLcXKNYx1^F}W7L>*$E*p-N2nE8%WIC-3vg{c7 zs99|Da*n!z>-`q`K%e3#;~oc8wpZph=bAaWItO|3>FoK+vne77}J;-`t-d%KSJq5s`cO2OC;-V>m=W68p`A2DbmW;1uH9;LUyWW zy6xW5TGt^dTydztLruEoNKqXgf?Uq7B*>?2Kl2-tW~zrYCg?BBf`tc zd2fDT4DSO%9J^RqSziSO3Aip^Ai6j}@}M#T$-HYP(Dj{p!F+fRz93*g_{lSB4x7cl zw^1(1$jmHnX`ym;U8}Shxq22+!IcxNLeIo_*S|2*Zhi)aNc%y_jMd!Tnq_9f%n79Zf&u^i zd*n|m#Tx5#ocGEAx>`vcHv;z}oyhK#H#d@}TBa%}lfVAc!0R?g!>N4Ni(LJ0lBj-$ zj1G)^c{fdcu$ApeOz37EKBZ#X6|TN9dhh7E@wT8U*$|hZbD}~oUo{^JNzqm_)DpTQ z#F@oupKF$|%cON=T}VLiWk|Hzz>CT3dG2o4F{bg4&*q=oMVT|3GL!IkhxX2dEyXym z&kR$#2`(v%x~!dM*D0z)Jk)tSS9OnrmPVmUVc9=4GYKy>;QhFzCjN*+{Ur9^Qgo=Q zn^<>mJ$;x=B`i~LY2zKioK}~}mtl&AcNngts%mCY>_~0-vpe53Sn)MWXTD9`ygHyC z2JfSEH*{?s94Z?c0)Vuz_?bWZbs>{DWKw{?6KVRTOD~a|5B>iAsGvZA&$8XpCx(RbhGe)@i}9cr9btso;PZ5BzutYC`8Nw z`ok*w_Eb=Hwu?AV*IG*WZPnH8dh_rde+;rZI?^^-N1NNE_a5UR!PRBLF8bhL$+$Qi z;t*LnWGfa?`)YV8vo@6S6Due7Pw`4k#5L`w1}luU1vIWf5HEgMM8-Ao?RIdxGIMRi zw;w^0D|pBwA=P#5r6=Dhu&`WMI?sy6%O(@j_KI_RJD6>Mr=2!1?kj3wVq$934kfGZ z=%Drjy7=8YJ6lnrXfLY_C<4)a?dhR~&LOIiM5T+i=r(2Ke{s>#U(}zKMWRpHNt*ab zdtAAFd9bI8x@RQ!@@>ldqx++}T&>d7j3(B4rSf-v?tNJFXyb^Qq0rR(pWA8<`31-9 zurM-`;Hot`(=XSal%-d{r_W`QjoA5EvtxoL=d0xClP`{G7`}TKrw)IfRy@DQvSjh7 z4#%>_#7am2TUVwF1!^A;^mo}zuI3z{E@FYx{M26@9hz7wb1NowFFB1@X{jh&Xi9e> za}qOuo?UxaU$bISVlMBf7!tSr=0YbzYwaJ}JU+Vr>$ys^V4uvSox%H(rkgWxVU2SX zMJfI%a<^TJG6ziZlr7B6151w7Cic7Sd0oQcuvrik*uU~7J~>2c^2hTJAL?FgyLzKh zhHy9<)2h=BeX8w6G>%kgP?eT?PT+vUWtaPR?<&O6 zvQs_qhI&*(5T`%LhXmEn0N{;cg z2v^iieJ0a-|6NWL_#W(Hz@`XylbM-yY^CnTbN{J&R?CiDdKlpKMg1;Wf1{9kc%kM!UwaT= z5;F*bnT{!+KTId+@#L+=6;gYOW|vgQ#=iB%A(WhZQp*bipZzNtgkRrFcS#|!uFu?+ znXQ0ty71EDL?Iy|ee@vYN2Z?WoP|Iv0JFHUh7X><+N*3Xt&hns8{SO z*5`^T-r7x^m%Kw35S@{E;d!8Oubad;Puh1F7#gnodt(|lUjOc?el+#LDP{5_5M`kp zBc5O6`!~NX#dtQpVb&*c8Y>9>gH(lUwDIEPis_N|4eRQ_pEpyA-BtZ?3}9Z6rCxyX zn)_nR*!rRGX%4HC#a~;ut*#}vm5NYx4Rubv@8nTlMZBY)KQgb&L~#dOif-)UyTu3p zA>*31Sp2V<)xf}LrvLg@@S^Lvh5eT9^GBsU)XRUQ$9f)lbn{IpSti%w@*;IH7sZ>8y+#rB*ljZEZ`>`C^g7V}s*O)q5k{%yJ! zRigaN!o#$c$*TT~JUsDtZshIh@pN0V7R;Ug?n%9M>v+3cS&HZVc=&tC{#TRFNSQ@pKuJlf{>5Ce59t&a9ti+}^rEOB%!(H1$TS(1l z$w=2NaHEHKw{v}AkMFSZ4N?sWIQ>2`=NT0%^h%{W*k>^|2my0qODe-Q5h4k3*}3q_<;~o`xlZD2crFQyg>_fADa9!ZIFFps z*;r!ji{9Yf=CSSk>pgNYLa-TSr+Sa2n^{zqTT*krZ6ukWpZ{JK(OBOZfiX%EK4omJ zf|49^E-#7w5p2r}TYIxG>MN1K3=3P~j#?;1&muzGwYqEFeasT2yzulA-SvO{I>&_X zTHXFEnkiPq`?owVh$CQfI)(06%a6xfb!(K-DIyu!>Az#%c;+9n_erC6OIZ+=Dt`5S z!85hR@>9$<|_|2Ppo*XIa`47=(*>2c> z2R{_F{q{eQPT)K`3Zi&^0NZil!i6W#pPvya8GrAQ(*`|Z;2@V*Mhn>tp-MV)#>&z0 z93E|d9K0d9o-vIcR}Pm zDaCG8o{Mo42UPBMfz4U7c9oK^#^wtEyX>Z7LWeT|6>vnR=sIOW!#TwUm>EJ-7tpax z<3%&^aWLuF3^W?Gf4+pYhQ8l8hDspgjKI zd1bAjb{GuDSU-~y6DcKmJ9C!#2JcuupwT;act38=eTt{w>m}qk&a3s?W7ykjAR4Xk znT@6XxWvRNumW!0xbYU~K9F1kfMESM5@D%Uc&cLPJd#*{T0XVqNMJv-iX7a3l{j$} z&umA;p}X>L7`W#gHG9DJR;G%kP5O!jssiZsA;!$l&kv=`I$F!(2ME1AH&;e?+$%_h zUb4&L-<8XYB9Oi^YKW}HalD;rup5{X2*1C99v34S8QJFY5G8S%-iWyJ0{jT}VGLMr z!Y>wur=Z7cyUmn5f&d9iw;7g#;#jfv|p$HGkyI_a-~Xn8X6ye>f^H$u@Qmu1VO`0D-YNXg6I4LBm(FR7FZ2z zH*5d&l5=1O1fRXppd& z3KtW1_M)E{D;Lx|0W($4HhI}#i!!otw6ibpvmiuBqp6MG8-b#=PIG24Fl_%8I+j8P zeuDdp_AeBK{n3#H*`#qIt*n89qxdr52Pih2Vq)@T^jLndyqsxxlE*W2xc?jpN=Sel zw3!(a5@;78o()LMi`ro{uQP$OhjrA@UCHbBhyXupbE#}A!{o=QDBiZw?mKtxP#r&B z10>Y@%X9=3L^_oH2Z((4&CO|`kxs?*I%N=QhvL8sfB`{3sL%+XE8E&2*yKd38Uo_q z8yZGH{seRRHR2VP#>W6b16Ya|vBTEZ76p<5fI6~dEL2&7+XH<(uC9&MSr@qsu*_zlziAu`(1WmY$Ij3$hTQ z=3N)T7YKmnwx&{dSC8cZf~!JM=0PK=#P-13{Mp;($d=Y z{zMwDjEszWvC}2gra+i}#(FEY34Q?*d=)S_H7Hkouc_Hy<8dbMUfWYxt}tvC1r3cF zkif%WKyd@t*hA+4q#gSEw+`QQo-SE}cYp1h4FE_Kpio)F&amtc0C56)S|6g_=xPFu zt2wdQ--rVWz-N}#K>3~^Yv@tKEK=bBR097?pq2@U?}r;fJm7r)k1oK64{Em61lR@4 za=739XrH)2dAT{#9XT#M0SqP&0Kdj?9r;B?b6?092@3km`dp(jLI((`;@IhaeR2>t zN?>6J{`~n4Awj~iz`}lL5jZB|!kt$9Zx1ZC{1k>2hw^8^BQkK{!XlvpK+7jS@=xL9k) zh+U4#y>!ThlR`ofx)r{&xVNB~7@gsQJ$v{5&T13|r}`m~(y$i(U_${Z2SBbjtXRjG zu-Maa+Sg{HqqJ+TqgR?ZwjQK}O)Gco9^a=}mTFI1eyIVUNJ<3Mf^zC;fN9} z;+=j41YqXzh3K|K|F$1>s%f(S(tQ~MFc#(-o>ai}_ic#%ZDg5-RcRaJ1``08pzdzq~2mYcBxRpRQlEUoB0$9>Am9`Bt&5?0kHD zkkj~r0EvVU#Mg0~-HC8xpkOo_9rj01*?@v_Jx0O=CMG6k>)g6^i=f=VYM(FJ#Qq}m z4n$n$OMqoi8$9d)QEdYV;S>G??kp30Ae8^ZuwYz9b-ysBB4|9}{reeU6g$RTfb-_G zA6J3#u8b70ecvAsGYPs97vdb8E%t?7chuEq5L0F1wIH`MY5S<^vz9WQ!y%7;2?9%E z>_T^*Wuv^<#v_D>%*@0*gE2SL22)s8FV_w^+Xg^fE1<0jo~nNE=-0+Z$^y-XhdU`C zLA~u4zDF?7#KgqVdYBL?A5)CmT?0Vwlfb|)f(j2LlV^ zFeYJDW^CvBEsp*A%U7>Y1z+Ok{}?2bX*8CI14(RJ_tor%@WODCSe$wYS3E*+O7e%49@?#=h*3Mb z9f0pf#CQ`snUL{S2s!w(uI@QPXOm#rn>Ukq4_LN+JOUfL8@t(UWjItdBt<4sE(%j& znoK}(7FSdL8r$qn6(tmDyMaiVT9^kDo##?&N0H1HH67 z9IEra3U?{565#Vx{mvbIb91esx0-VTI;B(vv8LOtUdn2L?*zf(f|^M2+p#xBn8;-x zJ?JJmopbQ)YZaa4lgwNC?R7t&N4wi+9iQ%Vvz*E*E#2I)FVjq_#rNaLryT^waCG$C z&aeA!CaL_x-TrHq=9q9) zU;B9dStc_hqJFZb-QCZgK7Fq(>JRuXCu|YO8*Dp=!ejH`Y};7wXypz9VMh$tfR}ZG z&l-=cMwrHcR-06yGF84xJA1u-*_Jwwt#|Fx+Z>DDa0G<|V`F|%QBi8_JL-8p5^*O; zP;s{Ual*1Bx3!@{tl%I^%W@2mkLC~L`|5d`NdrN&Wa8(46E7Wl@9e#v41Kb)%~e)x zK||0%a>jG)ZEUvRg(OZedy=uGCFSdV@&M}zVMAE>TO=TEGK=K5ekkNP^=n|@3f_Jg zIQ*cdB4a7*?kL-G6YuiR9~UKqT+r;y%v(?>=BN7!qSU*%ID++x&D{}RwfR0XHFnq(dp3wj)4MW-KL!?(uNd|;n zBQ|dum?KUW!gg%OA7!4Nz>a04 zMYrJJ#Ak1dL#a;@>mOoZkOS2it8>#66`**7$_2})GVyEt z$~t|u0jH^nx$`ii<9O@t^ArlT&`%#kxJk(3fN!qB)|eo$hy$xCSSNzx4W1T}zColq zHYh)x)4pVJI6fiaRd_hdun>KN{3E?oIEDYmMBkXh$kXgV@IiV#U zz#vjH8sn**xFO0;1h7cN{r`C&4WXTtLhFQ_q5HU*2lrFZgZkAw_Kz?9$FF2s7~WdaQC8hNe(X4n8viXf#}5t?R_^W_ zOtaoMT9ozf-P16aG)x(|y+gWP(5BqF(k0*Nwu`Dp9h&-e5E3Ob^Y&>JoTU;A??p2d=sG-tI&kmrL+i zaFG=?!K`MQX&)E!aEc%iN!EQ8FY{S&@R73hf6vgtRGu|Y(-}NKK55| zlF5xMFmb)+H%-WKz7b8CX?Y@h2OX9BskklG+jl3FspeZ3B_1R!l^`RkS+amh%*f6* zKzK(?j2T@)dPO)*(T#a!xMl5R-k*(#6!6;FwTnMwSuD3D-=g6%+B?~uliWhL(v9p< z4-w4$uFRcqeXi3bk|1;eo?Ed)XL}m+N7)0Zy7%twA%GT-B%?I=_|Geb+|c>HYQKB~ z5(@SoI}R+wU%U3sBWi-LJ;!^sWGTG7AX$DR;d0jVXA)^1B^NnMEwyG@`RrW2MeL~$ z%Ud~~m2AFt4f{Pe0aDr%iD<{q@E+RO>%&+t--4>>%)kq52&n|xCz>dG{ z_+b{e0Upn%AO4Xaom^dam}n(G1t@lWUfpr5*$1y~7Y&P!Hu#F!A%`f)$w_KP)OVnr zbR#Y8W*?bh!HM;a4M3aD^77`Yr8$o^pGEnJhTG%-vZbcu)|d3Yl51p}5XCc0Ui^)~ zy*0_w*tjafva}lUq5$T3q__;T4oyup)N+*7!OSCC3Td}3aS85q`ed?t`6^N1kcwAg z9V(QH9~}Jl3=`7iT`WbJ#}7R+GZ7t~aGm?a?DF2@daJ=3Dfh>j+@NAj#a-&@?;MqD zwJcr&epjp2(+*taqL@kxPucMfWAkRRr2KsKdu62$MBSsO;*>{zrFu7)EzHj|I5G8f zH<1qEC}?cEF3WRG!2^spkVEU?ace-aWv(}00N)LWXZULp!w214#?z%05rxwm56S`! z%)2sEOsidgnR{n!(^n&>J!Eq7g-ADdaRjt?RY(4}dZ!o7ry8>LLrq0eM$hq|qkeXOn7)kKK zoA~GE^4{++r3Yakz3V;z2^E=!wN35qydP5bKi&ZaYY9;JuAMpvm9APvcMps)ATol4 z^{Bjs&zQ1DsC;EEH4&DNI8%s#_>qF6js5@{bSPRK*ULDzwOl1uX;&QEbmbq?zNVfY zipLcXFvbBDg+XissLg$Y^+5YbIJK}5wZ{FA^}oJNEh(YrC|l>`qCJTLU_hv&zp~gh zeZDC)@;+3d{)~@{J38`_Zr{e>HgmpK0=+Q<5f5Y| z2C&cQ=Dz1+=C#=7ET!<;->l^d#?8+LijA0h^YjQSF%(Dr!cp}LAy=IDlU(d-jfjYl zj6A}r0d^avScwA~zPXrJM#x@4=NbZP#AGZC*WeuZv9wvbGC~{pxhyY_I#6`=Jm|$% z(7?#oWG5vhCDg}EEG?^0{`3`zzIN&h_V{c>|GMu@x25?OdHsQMAE))%s%`W24M?BT zi12=Y&DKR_nKR^xKeBFo|Gs?q?fE@)`^oS`bo-5E#1yg$gw?kCGG@iWARqGU~ebjjgVOi9>G_r5$a`uxsGhrpy?pXyB? zlle7O=OebRO#V+>Fq#HF^<#2!l}Ne3%eTdeiDX1_Ok`M0-U3nd%Z_Dh^rah%cZ_;E zbUf)%VN^#!+G9SB@eAg=L0JmHYHpPS_5$(_?r8VL+dtixdJp(vsH(|lc=DMal8l$A zv)MyYb(-1ku|C8&o2gUZ(GJNg`8k&VC2*DF{i79@~2r^i7(3f zgQ({TghxbJulzG4*xxr~WUgTfySB%Q;rhSOGP+wAQ`M5mQi`Snr#=%a|I1F$dTV2e zm<@>NuQSUyP;xtu&Pu(dhj(JM%O2F3$oKBO2E9chBp4C0%<7scr{MS%`OjRm}Ez5wK-FoBJdLT#4;0H`1`%fwj z6cZn;#SjV^kD%!&Elshx+EGT#>=4CRhOH%c0ja(rS<^QbgK zAYbZS-!M)HqAqe3H^tB_YCz55JkA0*mB7%m-*?#+F?DZ zL$bU@y;B*ae;eu@_jivhQ|BclDIMmC6RPu~MFGd3fSU z!3V@MO0SZAaItu$*n$FQiPx=>ds`eF_U+vp*>qTlSoJq404iVtYVpYRu)ki#TgMk) zKR--bD=f{PL)&eg^NYncqH(nG&shV(sQu!x2f$3X;-@=pl*5o5JN}R=+5<*P4>>_x z_aXw#E9NU;Uf$n3JmKTAgOl z!6$2LYXl{Am-hqaC_ZZ0+Hh<3P-&NNEYp*o@SaHtjKknMB|1|!@Z7OJzt=+2(vm~D zc6DWINm%^!Rgv7Tt-y`%jgzd-*A)-LtCQszCw!ur=0M*BX*fcuf8nx`Z!{s8qm*Q) z(vzE)x0aoo%W;N@DZr(txw-iz-|-z8>d({D(-qClGSn+I9qtPo=|w#0sSW2byPMVJ zCOBlupVt;{?RSlno{~_>I>5E-y5b9|c&I3Q%l!-DIn*oW?PBLP|Mv3FR>SgdE!1H> zSr!&^%7L%<6P1z<96Gc;Nrf1n^YPK*#T8on z?UQ2M7VXD#9@%DGk+27TR`wdU$Xi;jre7Tr$ zi^TI&i7T7v=C7gMT26I`5uuovn_q_!M=Kd@(%SX<0V@n}tQfG8(Wk(cw&1|7{57}o z%Zk6w9aCnYefsC`-(of)sYLE$G1c}`gn`j3Bq!Z&XOCpKVKr2k*79QM$P^FxtDXL) zF)PnT;+CZ@WuNV~78#Mz&+7^Mswd1qx%~kNobP}B_pxRuv0+YP`AB;mis&(V9b2t1 zf+rgfhw@)xy$#I^`&EaAMamfRLsDfr*U z%VUo6mUw7Q7>Z-hzv(%jNgfm9?Az*(i&tQ-6sgxd`0iMT2eCV@Jg(?{6SiPc@q!{3;!w^H3BrZ6>i(fiGda1Z8D$0lWrIGWN6IGI|+^6is15=tmFlCDo zmpO1cqAg7N{8MqTa=+?b%Ia%?Qe$#5g+!cn_#M7ELQ54}il~4U78NBH32>EzAPMyf z>tCO@{aIMO=~;}h?Sgtb2_Pak{^9dgrLzkB5XCMe9VnUb$ zIY%nL+eW`y?Vi4?jFK%We0O~8S#wp}{?J;Vf3(9x-Hr)ER{S#matmHu0#{d$*SbIx zfvC)OzQ+~TJ*`xXHJ|oX4%-NTtKK2=0j36{O^6peJ|6zM3RM`;(0x@{d@i5xf_zz> z5f^U!M{Db&%uG%sDl#?tYY#Qqs~k(`=n)L9Pg;6iP?g`(85Oi@&#gi0$^qNKNi!j=UvAT7+gFE4@AC4pU%z(9Q-2e5A{FXi_v4P{Yk6D&v4l7}`q^i- zMPk7j!@KPy)IFE3T=9HrJ=M}um5M@SKGwxih3|Oa=Xc}g&GR>I^rb#$8h+}gP370} zBErMlOIQVJqZ61>*iAYu#PZye(!rXVJt_KOQpIvx5WT9{BMjNG<_d>sJa^oAH5A6G zj-2m18CZKApk+rrqFqi$a1`PFZDU|GM)#v<}ioXLg9lIrQn1 z^`fOqrw;Ekm^EO0MOF(K6J!G{uDlgr6~h5R8}3VPj1LQwBEayG`9%dMc9XL2C*t~6 z2p;ke;p9HU>j6K0v<0e$A4^{D|L>KdGiT2mP?vn*bC+SE(-+QD?P)}qB%RA6D)Bf~HWDvvP9{lg2Y3(mA=$yF9kGAUijT>{Hs6Mej^PlDW zrJdiu-2C0!-N_Afl{an}cyr9DC$V-#Q*_Uo310A0RMhvTNz>;il=w{|xcPkjYQq-t zhr|3QDKN@I#5o!eplhS~%sNi&P!r}j4?OWwx%q+x#j#m-n74|B+S^s8&o>|5XIY$ecb+nHwD5EX z<|}PeUv|?*f!|?k6}0yuW2Hen#FaN~j41&}jvN`TKm6<{>J@AN=R4=h z>w9(T05zD7QAzBk9ChIzO^RCh+R%B%vr3DB#{%@_3^uU*;p>kdhSM@k{5OqDh`uG; zF`}z_{Tpp}-W$9x2W5?>e838(V|%7&4n8qE)ORQA}BSK3Ydo z5?$Hty*0$cKMtsTn-4n@LvkxZUv2gZIujHS;0;@a;RE?1lm&tvJ9=tFv)nS)sn&|$ zoq6%apvg^OI3ygJ!nG|sc$Y78bEjT;HJG(~us5cbbN!b&?%K7xL$TyJN1uBQP+}Bn z?yD!?_@L4u?Bd0aG>B8NdO{asazn){EtLm{$Bb(4LjfbI33QN|LAhN^!A&u)F$yjF zYsAZf0zb~U|41W#A|Wl{pRznOD|2wjU3j8cls`g6T2l+OKNNA zL!4b2$}UPvq@#{T4DTX3Wtv{YmNGSpKawyV`CwLmk^?yIWmQ#T=O#`vU$xG9KeaCT zQP^g9>((t{0=*V$8bAzb7S#_w`@^SCm0k_+LoY!uEs^jur2#a^Y+b7pIDf^8F7*48 zsuH9aWzuzNx|X5SwcjWs%;fh^8AP)ju;~}R8OnE}Sw$8&VcQH0z%lSizivPvp}=RU z0}`#Vbt?=T-mEqCJB;VIer>uOm$3=j*K3t_^rDfmHgCsz-?x$&(K~@Xw2RL>LbJun z*#XEzD!pU68~ajweO>9FyD6i8u&aAzkM0%^N{*xz-3^^*Z7cI_V@LKaVzd<%6?N&- zrG&~E%SoZua?aCkv!a!ie8O$tNYw~iNc#y}XJ=lvbQz^q{HwOv1!y5OENtz)tszp% zq2{_Dbtnp0ZuctW2hw-36`qJi*v6ihj|ZSZ6_Xp1|KZL*5&uC0I450xAE zMHrnl58eFAEAPXHr?CAU-1tkayyN@rGF(|pp5wxWQancvLX7dHoN{(Mc7$)n5z+&X zh~2J+d`!tc*rcPYRW-QUlC0dTgg+IRnpE!}O>8ifoq)ub zrg%)-d}B&PPO6QL8t09L2>je{Qrsrzj4)2=H$HO6`~TAd@JPfScB(H5TDy`5-MIy- zBS&^2tV3!J*q%NwYRS~WM?;d5^0o|H;CAA~3DG(v=T*Fpn!0Y?kQ{fb*k!869try= zR16>JqD*C!@*P6Wy_wM`0m@>AB^{h)LI6NCN0xnR--Vvy$nwF@-@ZM>ag+^oLHn?l zoQM0(RI)o6m5N9>JBF)mz7W*CgI2#^ld2v^pGqxta6doXTwbr&`=^fsb-$JOlAmPl z_$>z@j3L$u^Pw5|3kIt07QI0tQ^sg}h7Yz1gDGSH5{Jb=KvNXjH%zcVV9U z_WGAiTSvUjY&i>?>(n8Sp4oHJ$o1h58i)1ivS!=>zVT|!Ms^%qqAf%i1VdqI7&CSL z@II1#B)It zW#Z2pX{8&yhLLKoU=?HEj%OoJI%GVUboYF##@d9y z*43*II$XJb_}!Nk$1O(R|2`-8j^b8_&O{liO?Gwyu~2=B?5B$WKx7FV2+^qrJ;&wt zZdTsQrP~^h9Ekrl?99z`_pQ30PTVuj<%IE#q@;n!y3z>2|C~5+fKpw=6^z(fgIrct zR&yrx@33sj8Rt7r6j})gE&6F#_#!^z^LjkB7C-@YENV$*<3j`(CUV$+#WvQ*9VXDt zN?JSuT64YynGF+8U^r03WJQn@$-Ds*ww=r9^q1XV+w(SR2j&2L0x0s+hb$VoMkmn3 zV7A!Az!hK_v|psu^x=Jq4-Gq4(-P15EFs&enH{r^d&{O@i3D?#M?XWa@)f&B=X@ieu?6Hc^ zlNqY;wY=P&GJ}%FNxvXK+qI>+c@UAyHhzzJ@&V+=gMb>rO-wU}d>Rvx^XT~4u;E?k z7e;LP6xHEW{LZLXo7X4D4R-6*Z*+BTQJtGqNp)+(-*3L^jC-xa+w%YMQ}@i41CeLG zQ?;m`n9U)drD{eC_(Y+bG*4PoApLyS&z`-6`{ZAz=_(sOyo*lY)+rY?y(X=GcKK-S zp_=uhv%h`Y!Q)s%S|ajKhlK@KIbHsq3Q}Miz@2hmpNF+f9r}-!d^gEm(_#I6?pww# zqx<@HtKg}n=G{sMHR460zjAZK$<#X}9ID!i3Km`aVpUJ9=C0Zo>uhtM=9c78IpEti zqt!-rSRMOszGH`0jseN+ckd3vl^;oPb(4yVlyYr%3zU&*B)pQhSG`YMlk&X2q97-y zWY_E@vxv1f!#tPk4d1$Lo67ZHc51aG4%mYOsg`%Sq?I?~fC4P>1}2%zoITrPO6a~B zVQV}#TzMsS)bhl&D#r&t-MD3o=af*(cf(K2rX~VH2Ki+xh_I8xwa~mzN9y;m?1vp9 zXo?@^M#jgV+P(EoZG%EYwcMd;QBr26EAG^+s;=(Q&*y-jpKQzzopIy-u?(tFh|jC6os{JA^qX<%&1n2J~|8(~#Cb9ICy}00!=_3O?6(yR70uh&=?q##B ztg@_4rbCBr6W?{MKhl)buD_CE_wQ@yN!R=3t!4!?!h6n>LGQ%uLwRK>J?^jEq|GCd z=MU%%^Mx5icZwiCF#r$>ASW+9~08WEU{aL3`J zZf~kt6Rv$eHumTvv$ro^6l+xmw~o2|YBP|+lu+}1X+_;u^yYJFxwX1bi+4IO1v6?p zX=$l(+jdJGoZ`0hj_VSWDf3^l&L#h(dq(5;ZPVH^Zdi+%b4g{b%=x-`raJhr@=md3 z3iijPGGi^C%1)>gU7c46Q21l=kE*G&W<6rf<%6)xm%F&TyHgTPRgZ4yd-!}non70` zYiUK2Q%{r{?^ezH{ABs7p|)dB-;Um+eB{y9(UlHKr%m@^8wLWJ@S`fJ7psdY9j%qd zJfAQ{ca=k>d;dzjjyzwN%91H%lJcLKZcwyM3VpYF!2%D$z5g`Z`k8HaBP*nK|Fw0- zuzwduE*h?(q2ka6GH`Xp4Hg7+`vcqzaQG4X-?C8&D5AM%r+q(5MX33AH1IIE=6UG8 z)sdDdeCG2}QQi6U!jl#NdxlcL@~9=k@f3?P4&4J-!k+BXK{+1&Y!}O-j@?KhP4N=N zTlTH0vbE0FRl3yJ7Ibj`eoGj4`UFZCqoqsxo2tuE&WY4=?wsMeA1(9TB95#-B&9U1 zG5CAYoaB*lx;ZMx67JBE_{&~o>T$~OZ{4nc*E5(MlRt88`IXX@Ifp_GT@e`svK_zP|DC@&1$K8>fHM z3DJ0tm%6;e<((^64kYP{IB;V2ftD8MPibq%4KOinB0p&DOk5TAPB~%x;Ht{@KIbp3 z$$fw7KWWMGDG@`2gd`^$5Dw%cOq3Zoa$~8v1l_^9mBDzVr8W&Puan7I!1AFDl z&7->b@z3efP4#-Ep)-77fXJ3$g*mA?(4sU8G!b9+t%%4ex)PIo@AmFG#7CY8ML*#G z<>{fL>@U;LXZ%MS8Bh=!`@wvp9f?7m_cM%8M1oonX*9pjqLC*uwt&8S&?%^=o&$0& z8}j(kqjuBh1An{;Nmoeg_^8ix_pHx7HpdvLT3TE=I&S=UtU*C*^hITVmUWhXT{j>(`pd9#+b+hLBXF8``Wl<@4~~)W0^m4o5?TYhrJVa0 zSP&X0Va15|6(Puc=~6FNKsgUd?AX$~T~0=~PMwTMM`9%+iWZyaB^tnxY;sbiyiF~# zdk*Huh)ofifuQ=&h6CQdeT$a7VUxL;ObJQWOu8g_`1qv9@JT%Wnc;7(Dk zlW2`&$v?evlV5B}FYmgyrLA7iVp~R)$|%fm(<&VR?s_yYlbWPn*41s>&<`A2FQtiW z#{|HW_DW+pSqC&n?dnfx56rrBj+Fh{rEq(Ol0}3r)XQ45&~|!f>3M4|NEhFY^3mD0 zC212EOe&lJb>HRke4H?bWkvxoi5HHPf;Wtc>$ZSXJd67ilepp*tSVo&66J z5li}~G<7W4{rT*26>qr?S`*n2RQqgFwGd3A4Hh!_1E4ti_0OjY&GqTRMTWut6AGN9 z(1+|>a^=2TTccY^O;H*b^REBidNu!5bAXwea)Mj;6XJsOclX|w+^oKN%N9U@c^~3a zenn-JYtuvnk9mWULo-&f#5VTFHJ@#tu(nuf9kWR_d35Dq~d%s^~ehc=#lB$Q3I4RXH1{2 z73OBNI^*t4z((}$SxgNxNuNHr7cv={U%z-#9iSHw%zK&IeC|r- zMD7M8;E`{uqEqIWluvEie)@bArZox6*2ldKnCxaKaT=u*GADWcx^+&zx9XhqnH-uL zYOYyPb4S=Ia{~bi48|7ZzIsK2DjH}5gZzqjbI=rxTAdv9HaUg&QRAdPWa!X#`%E;= zvKEE-=e12Lc1T)tV}hDaW}f0O&|T7J=DD)f){5-ifsqmLC@>A*R~ZJ zivHCJf&*|;tZfWdztYfmb;nYBvkQk7y*?9Zo&WX?e9HOr?YYKk>P}@5q`y#^J*#1A^BE&jGP;UV*0`G)^wo^5MkW(Fjp(?j~ z#e~2hzzhY2Lx35A!af}{w0CC_Ef*}fXS15-pLPc$YfIG7EZUL**jTnq-gkzYo}S*p zQ6?ShpQ=*+PUZKgdjN6OgI<0SD=QOs4ik&NUTr?Ds;RCnO)NHx$O)UeM%6QQAKFX5 zFR@K7hD`qGl_jIBqedqGAV=%9VJUS8XpXi*20Il=Ww71w#DF~&^L=iiv!7KE&x4}F z2u04rcm90q^khdj-NUOG-MQ( z4LURsRIaZ>Jp#xw3h))XaVsJRhn*2ZxxBXC?)S%cbYjAj0M=C)Qan!_~`_Rdf&=;%a*?m`}i2_h<<*yLZFcxoU|e%a0;eSwT7SaT{KdR1adO& zey+>T&p!f^LY#b|HUY)P1Be&F3j5sm0K1(Ztlil|K9I}#0nOixojc<)ej#=zRnFj0 zFeM&vag)@BlQ(bN0M<-X3-R^!-T!FafJMd|xn|;+Kt=6C>B+ z_FaR%^7+Pb%FS-p{~j`19pUylBXFIIdf={9-RF&|EQMx+5mvz0GVHB5I?>T%1c*O- z?k%PsVL3VhB8Y|!@Pm8`Bn;dsj)T}U#u~G@xSX|x^ky!MR|@47B`a$qB!N$*JyRug zc9h8e8qvsuT7UfzPy=SCV{<&qSCkGNI@B|~P{qKx4?q;f+6=Z$av6A*!lFNXm~Fzj zI3^XXFYFpT>)A$lXehtl0uvEnwODgW^sSo;Uw`^)ae?p9u9JQ453nEgTe%|5>kkFb zTlAzaCr>P`U>wFca?#*HgWACa0z&=w){w2-$DxhEO*G+kF1wr(4q z1-btbB#B*+;%sDQ4x)IMQE}?mPFC7ugvr=)!De{=3` zbn4B;Q*qxL8e~^&G0WLF4*A0|y(cf1J{WjBC#5bd@~d{9xxgcbS^hUCFmlo5?MjYA zD7wi!r8}M;>e|gC;K(_r;?9(7F4S~6`Nx;5k+CRR6Yvy=xzWvkIw+j&238}7_};v( z;FY%KvZBOjfpc%=Jylx%l~?=ywfB3;>Qw8>Nt1~dP`>j|s1JO$aZX_Kk01V5Op7+s z5~C3j3whNXuA2?%G}-RM-G44VddG1TI3D3@T%` zrz;^L>~jBhG>*wV*1@)e)+`Pq7mvs31?E0TL!(Dba?8j%MrE>+)MUhZgXl*yfs%wWFTd zaYL8tJ%t17_J(zTyjx%Sq{eDIt^H;_(VE_4$J||;;wEPAnmsQ^&0QCnQFnvcpGFT- zy<@X(taWYAorQPj*X~-G+x^|A%=irfMnP2x=l&|wAE0*j2fG%oZORFswOp>sUT+O; z&)bdTkS;J;weiLZiRHIcBGm4$C}e=%djPkLQXvj*w6RfK{KqDih z2O+Dn_BLpbT9wd~f(xv==~c|$165U3Yv!0+Q{Ap^EnO`LlCR{_7w4+iJqEf)Bgw7{ zL`3z~KUY0;9Q*wJo*6XA+7JI7I)^`yA$ZCSvpsS0-zTM9de2vz&(_awpZ@-(o3n$} zioxTG2KcAsZFLB0Z5X#RFs&#vviwQCjlR2)rp=TGP$R`r@dfvg6^mMkME+>QR=cEp znjKAA@dJCR5@pa45#u;t_aIsW{B-mbKCSgP(D@ik1V?j0vr6w&%lyyod_RkVDULPW zU8)fOnl5Z4NwF2$KbQiJ$0_>kZ>3LaGbXq8n%t7@PkIpi3&pFV*d8jR*k2uii{QFs zn%-vZJ&3Mfu=3#EPLyo5FBi@$0(6vleoBMwNUm=Zwuj|@i{>ek{T0Q|80gfJ6H6@= za{haE8YEfn4PW-sx6awk3F>E_Y*rtf%8^MQatj(#;A-&iL}gGtng9Z|RN0&zWj-%_ z(ks`0xYX3LwL)g=vEV>|<(w#^tbG7Gh$Dtd|S(qxjW2yroUu?DuK;1IZSwE zX}C4dc@N$ucu-*aE)vgwN30mt4~I!m_=YZ5lj!ba$GcPTjl13<}y2 zzo@ZR4sISb2bGOG?rS>k6cbiFEcy0$xwV5U@~uaN~~6AN`8dLVXZx8b+pD#hQ#=568IF zVz+Pta4J2qRDZ|xOzD!&_cnlNIw(p#G1G5j zGQSTas7I(&geuWCJ}`0#?^*vpwJqZY(2|ufX~a)~#3l_^wU>fI`TT;T(Q{0D?%yWn zoUW~i!XFeY+p;R9?%AYv?;nSo4YLZI?3Qd2sRb`U^NWPZx_%F!g44H0!)kFh5Zikt zE$y=~0+2=^gs1qLh7{_uqqq>f)CnG~-g|#eS?R=e>%{w1_1;wCkYo@(V%WuD#!BH0 zZyprcsWw)u`lNH}$@=zdRg<-vq1CvdSTvR-%uvwO5oMmbZjXecr z5U;;|$hMca$-(4-rYgr2ABrgWZ6Qzy%l<&`o~b1!3#fX<@f0#3N^P-W3JXM-7R^KX z^vOIzQLNERlLaZ+sGQt069#os;ppFE2K^9Amqa6PLXbp7#9o?iteKI9_G7rOqeKWx zh(Qn7Oxym{la2%nhB3v!0Xi3PAxGOTsgdtSze!A zP*;bhjop+Hv1Qkm8!Li`PVuG9Kwk7{uk0#J&U3%qub}&e<5GX;o8;tQC zarhoo85$sO8ZeqSP;eSeEA!D)%IbYuYw5oS3d{Yno8VR$-va!p$Aw9|5++4A`?Jx(}I*)H7AR6YE_rs&S z+B_YM7rD^GQ5!-d7md^h(wjMNo)@AB2Hkm3Ueupjc>-ikX}Yg*aBJbD>f4?K580%r z>&mhFXD*wd8k04065C(ljzf}@3)ar?h@pf!A06#Cu}7nZMi< z)UI;qcX^%k|7ih|=xw?aE$ARH)%_3}pn#lm|pU&S8`cv3ocOP?QsU3Wd$ z1Sm{0G7P>sOQsZT|BZ-f#SWatMmV^~5yrZshYsz?QDHB5;LEGEIg2+0nJyeAF3ED# zM_S8wjyU{n#nPW&YvW8+Wiy}ND3(%SiSz#bs#;px%|8_#LaudR7(Q5kNAl_WfhXRC z_S$9dDyR~I1#5GbFHhb&?$eUZi+}6DJvkQ{_4Sz-Pn#cfGMg8_B1*UR#0Kf_iMN*A zo_y9!CM>%X$$+XlSS3Zs@-l`XBA(MS>~dXSkQUCTd^c$r^k5aYa8;3wctCdckNygY zv48cT+E(%U)5x$0%c{i1Y0lBF`l^`0A=A4c9I^FzD6K)Cej)~erH0z8EQl#4{p>|VB zpIiInRhZ5dUn9=6%_u12bI~8+{F|b+ro9&*K2IAOan)7(0Ie(15JD?GmFa6|6u+5c zDO~6_1$2te=c~gH`w5#~9HmU;N78y)Hlj=li7^4;Q$tHsMQVR1>35X}9X@_8cs%4; z>5kJ2M#T>de0mV#Sja|%4iT2tUN1k$VjGm1Rvssl<3;Vu0_;VL7e9Twf2-8d-^olh z{B*qKd!`v|l5a?_iax%!{L4kpn?J0o``bV=Kiqa5* zgcKh7_4xQS8=b;_VWY)U>x&47W6w7R>_8LBkH@HykQFBFUbuO-=XRg_W7k#xmfOCM zZt>rn^C!D#KfY6!eb)1}d+y@gqIVa!WNtMKcFXfS+=J1T{oFY4jlt+YY5Fq@d3BlA z_;mhOk}D#C`9D#!2nm}vPoag3d;i5@?8VfCU=;=L-^bQH$SfjbD_qC+`DlPVyx*j* zPiko0s!_MCpHJy__o?-}Hj0=5ri65cUoM6!2%DL*`JbfumfCgLTPikw(Fy0|U)?nR z;)X#BZ1;=QAw5vuH|JvJ@WbuBo}|A1aZ~o)=NE}{Hrj5qwmmk#wn*IK_;|-#fxn|- zw}0n~Pp)d+lc|bcK{0o>jHZ{=zge#J{lmk_{+jU<6=fCr{<&-iI|z|q2d$|@YX6T` zy|Q~fTz*!2W75=MS3IP?O9=s-Of1gjr*qF^;A!^iHK)QXkvjpHv{=e4)eV{|Pwbec}2A6Xz?Wc4pgj3$A`ntDN^o7hdCM*u1dM0o6@C|3ISX zjEZ)&uIH!XfrabmI=!*E*?F<9!qZmr&I(Tp`en#I4wFYlNvc(B2;y#DC}Mb8;*zR| z;^92-i`F&^PT`gG?AsS<GSCT(L|QWJ}}W+{qLqP ze_j^w&(*wJvWn+6Sl0vQxcv7+?vu{+`TW2O^KUN~6GJ3dp}qY2RY=KbqaVJU$xBoa zmJ73i`R!Y-mEi+fw!-;ca;?<0*<8uE&k6DH`aEi7!*isw9-4-a~bmKP9~2gI8LjyO~G zNkm&Y^Ce59c_!2_lmPNa9+ca@xG+fV&kahu9)Gjs*krf%5hY6!wU5r6_WIDCLH&I2 zqG?aWqQjZzxKmsS6)0iRO2eCF{5^4FMpm)r?RKU;K3zsW z7M7T(KqqL}_3>7nfrM&|69)C^;#yFoCnp*mZ-vi*RMSwp8Zs5SCjQBzM?!-uyurdE zBR8$cn_?OR0K{U|ZZHZ+JetbF{A7c#WR$Ntf*F^%bCM&PMK^-uD^G7F;J}iwg z{&vhSJy!B`$+6n}&Y;OXf(>wk$pTafjX-+&h3Yd=v5a4&<7XIzPjn$8|YowoJb3zTOm z)H}K_+Bd1-wp6=b?V1`^Cr{^*AjUqZe~$VwJAOJ$=8&E$Qlv!?dHHS!J1cFtDP824 zzpMZ7!}l<^2~Qf4w&2PufhD6N0ed`MT_y%LArQovZ_fLBiPHxk{SX;(?jGLGlQ*Tf zr#GF|-QFItaA1|S>B!a6rO17$?+Kj{_Ch3YQ(mw3yV}vJK$jrYY*?ElR)cCC`?ich z5Kc&4a6t9=p4Homc6QCSyVY)e@CiV@?i2HG=e_ApN!y_q*s(*g8ft&I2Z75pYFTgT zDKNWZI)6RHc(xn%Zl?K&n!35UwUyhv4(i*hmsb8<)q@C7ipp*89j{$%UhwW+m*1kZ znIe>UN1o@bF~Pra+yFpOl${@7>jyM=1pufC=HQQ#3VZ>uSybI;&J6J1df!S52uz0h z5WbU^l>v{CYu9=bwjX8ye%}MDfyMbfZDE=pL^=Guh9}s5)Kd~`b(i7$7bf11&S&?yZa^uI3haU6d zBR2eaG;gm5$}fQXL9Euom zYoMkk1)nLxATwdhmn*r=ZEVzec=2(gcO>AS5Rwx#N<#)pSvG6eG%_E*1mi} zOw3Z$$ri5P7k_HFy!eV8Kf#oX3*6~Z=$P66tZf*7f12lxmR~_WpJ&{hHn_P*i?nOe zgaY5Fv`XOJSv`gu_Y7mpXm`+G`Y~|tV%{Ke1?B2KR;taK)mfrS-63EFq_uUzaKQoR zFmV$=>%7b&C?hhMl*3%%!75eeokSB&h7}(eN(w;+oXn$DlIL5?0soY>A!0>vMV7Sv zZh zezo@ml?|@Pz6zQwrdd>h2TMy!fv+h4oG`;fhGh5s%VHRod(fgxsWSF4e8h+~dhI#S z7$xop7Xma9r-+!u{N4-%e+)z%h71HF9Ykjb7fud0V$4^dM7%suqECr1SXNN0qD;eA zz;f*Ae;|lY%_|`mYsTM!jC+T_V9lmOg*9#1bW(er`=*$@p?6sgnczsIaf*oUryf>cap zFMDdQuidk;MYg(PvU|CRYWgNJ71@lpRip@ z5NmuBqAuf;0iP$nYdoB+({R7;77hYJOh{2*49*WoN=Uo+v$+nWT=2lWR(>XC8wKdc zPhh-Z!{>l=x$C+xkHQ^fQU!5EEWXDaEBbX2J%p*o_S)j<;$J`m3kKm}_kquIa?)AK`;0?|ClN^`}Euq$vNT1rF zUze#(nUapzrw9Ga)n&2m(bdqM$SQqLY5P?jO8};H5R)tkdm4ogU`>AY-v00`_Mc}j zSkSxQm{nqE4OHO+dZ+ymd&E^SN(qiw0$ePnY+wP=6(ZH1)8;3h1nsJr`IUr_E5vg6 z3570G+cbF56xTxb`^(@xlYMVD55n-?^kA6 z#7qpcJp06q+{O=2N_Xg!JW#`Ah}5iT;Hb`qbnX!`Srj0@oVaP|qc}w7BrtNYD&Cy7 zst_H8F4B`PTsWO!KWk(*Z-PrGy^>Qwk$)r^RQvPoKeT-sT*8;ALOeQO^pIs2PY8! zbl@Q@yzb#q>O)@PbK=oCfc{u*ZylQ=KP>KI%rsM zqVcg(?)2^(|7QAxFw;)6Deo=AmJ~JQxe)3wmcX+Xo zIfdz1+_=(lD>t6(fT^^ip^Lm2c--;PJ1tSvjBnQZXYcxMqT;$A3-mz~n+uH`H&$S|~%QYJz6dFG`b19VVl6*r#~yurE|LDA+1Jl`jHY~a z>8y6{^~iO=fB|A0xOeZdp_cmGW?%qbv}|K|RMe4kgFAOK3j<%bw=<|e^P{DOCP=UN zJqj9Gw}tD&2h>F~txIkqAUHmJa5=X^$YbE8PG#LM-H{ztb<# z80GAv4Z%of)%D@x_N%bI1Y2K9 zM8;o>y&Fu3Pdc{my=e-(Zl>Q+5;{&|MSw3y(Bd(uR`;6oj^Czr|GUtnQ5Tjvr~|v*XAbk?xCTVW<;ZUyL(r1$HMk@ z$eC12Ba8wtO5@Ap7<;z9TH%JD9lbS8$OP;ZsKyV{ReOy%k28^=asjodq-kmFC~@i6 z*3fQolteXe^vW*VQzo^?n}M91~1pGs&$+^ zFle{dYlHQg^1u=Gi*(hG&1#$Qz0n-{akg3q?OGQYt*Gw9E^mMExlYOBO%V#tn!Li+ zeu4M2XKR$O&WiZF z<7AlS?Z(G{=W^o?fUXI!cS`8foOd)^=pY zx$(#2s*l~b3Uzb5lyyBWcR*{BilSmSOS{KSc8fB0Z`^n(t?pp(smp-}Px-89)l_D- zVD+yrODQCq+}f(!Cb~pl^813e+Y*7AS@68S)9y?`*clq?o}99_s7_{5a)kEoyywnN z8<$uB@w6R`iRoR^8E^y;9Oizio7-eS9k5eM(_SD69v@pEiEarLLj?6%t<^c%frJyfooF7d3I5I#{w#R~D-_AOeE(6VvS~98VV^GOi zxqx@O4xlh3Uj6#@D`8hfKiVuUO-(#3{1LOixi~iA$E|RyrmG5Qw1nv!1&A(QvxH*N z<3Jf;fS+AHan-NPTi=qCzU9leI#*qqchDITo|su;?&e6qR^$8_5D(|bcXo~Uo^`pK z?qS#Em6o>L?(WtDIzQ;_5pWE@SQGsj-+l4Q*jVd8E>nD-V`KQ1n3d*!?xM(-s?`!NQPNW*zCYAh^F=w>l~>3}-3rp6A$odB3)@R5ij zY?zi@>FAjn5^-q3qc`-^CSNDsMq?dOYUkkZ>f$2t-Rp0nU2wJI?A5CaFE!@p<~qJ> zNWVDX)>M-f!$FYb3|BiX(-rgQ>neJE{F-Ha_x91x=6xgWieu%j$&c#fbLQ=ckVdJ? z^FMtxQ1Y4N`HeuXH>Y{o!7)+tt@-DZwV%yse!uBz^*NCJ8b=KVGcgg<#bG_ISzYg* z;cefSZS&SYxm9%O*?B|8nEvf)OOS_x)@VOK#Bvrdn!>5q+;|$D#|W%M->p zuHb-jj+nAgLa>+PuPxurF7f@Wj%!e>OM)~i;29E(M)k{?5cC0%&~=t;$>7-b{uNnp)* zN|7}LDK+QZ$nh>OuS2Ez5p#t`(aUR+X<(QK*2Z@Ajg2|W9U^CZsww}ElA0E=Gv>55EA15JQydmyC3uxBF#cy(5tjhrw)+x zXP;Nxdc1KQv4X4pzQ_=%q?TNT?*qC$x(V~Xm#=ATQZf5g)~D)I;q;sX>WdVw-aR#M z@1Y(q42>M;PC%cQQF%dC-EZ^pxegE7^*x@#P|2}^W9J^55dDkHWf|rXLjuD3a9^xj zAdLI-7ca(EEpprt(>=jZZ`t2}!!J%Mo}fV$-QL}DTB~m@Zppk)-3|_ek9xOmRowRJ zNZhxDTP}O~U*#=c{X6NTMed~eoLio~T0^01j~*($eDSCEOe^v&OaEBwgyYJwsfIn4 zax^X#HigC+U8;O{Y3KZ)(VGJNDt1IKqw`L$HuUl1$MnXc`|&MX^bSQPy+dSYN7HA0xPoF%n>PDc zmvnbGH$#0b`C)=G%A5sZ1c;gn#z0py=;+LU0s=Z+Trly=-UQ*J5fyb&>JFGHt(j1> z1MLm+*$PM|TAp1MuTxn839=-##&|C<>8t(aVGWI^Cv(H1=@kv_wi|y@Cki(lNW>C) zjrw;tfwi&^1r3ML5mIp2 lji#{3U58S!E^jb^1{bn(__&E9%6we&hPfc4}ZvFc8 zLKH2$ZRow67hQzu76uXX=O2C2=rj%@fy&@M9L3R7r@WBP8IS5G3{DXB+RtmHZx(L7 zI2yQHs438)3ZZvI&XK^ryGK&p3g^bPcw^vxupf~fP!?Fp413uG!RFHgLv>ClNOm8% zZIbO@cw74ep*zozDYBp|CRs4(DGJdMS8i!fHF=D`ydS`|Sy?kk$rSW^%DX@I5?V4k z8HTg~nVWE`lyr-BjEas9m+ScYbVZrp%2-9!L!*P-g_U^4%T;oL!z9 zvpPe}Qqb>1aj=AETU0oz=4H!tLvxp6HUg6%Fi9kN;DCMqT7pSMH^w!mGNUwZz|%_$ zH$-NXaHtCldvLXBtPtCC>umCFc4y0m5JKR42aJsOvSH}h1=RQg2tu+gM2!@IqJ`&6 zF_jPPrc{-~_qqmtjRKq2RgHd;DJgQi*|Bwg5qIvmKF)o`JU9Y)=C#6p8pG+fteUAtSAi>@xBxaQX%2h^JxJ%6|0sQBij5T#XV>JNVTH0jxsV+V{+#uY~7*p zTeh=P_ShqV1i*qGV&qS+s~t#TeuC0du!&gyF}S`M+^0F?ZZx6PRvd7EVShJ!&xySMvWPB5E=)pUpSNzcu9P;y#dOp_)9clzQ}fd zee$`o@(Ax3v{PUy^sXHlR9dj$#f5O&mZm0E9i4vs%=ij|NKlHi9c^9Ue+uG=!fMNW zT)6%7GbI7Nz>s5BF-E48JNEktp)hAhmOIJ^#^E6eW#8toi)klkN!Ysb*Wnkrbh>+g zHeJYdw%}rcn?8KtV4Y-Y&_C6`kc015Iap1&s7L!~?QAV!#m8WEnn**X341)u7( zk&-HM`~ZeJa3atq3b#DGz;zWj&@!?Qqy!$y$i+pU9aj4R#%V-~uy2=Lm_DqfWnp48 z*oU)q8rfI$FQi++CdS0{K=djcp$eQUMvKC=(VL$M9s65qwvRt`<}Ks4zEWn zIAET^_dc8pad>Hz0dGD)BnI0JyF1PNtnoaKrnlP`;7m-(+ezB6wH0Yb*z-cK%kY!K zb%Bc2BHlM!*{uHftg)_U!a&%(bI(6y+}V$2)GXdGd{W@8xY1h>FL z>i3>JdeoidQzEqTE27Sw7;AoZ*twH)f7fpjWy93f?_oH%MG{KF@7J1k`lkI_XGAX6tf~nXPPfxJ~1k9y=;`%hOvQq%e942<9&ZX z!lQwK6`H-s^62V{@r>V%gM=+Qc(%+l`|>gGJ}69Kk}>XT7T}2KX)oXi4>ED zw~eB5Nv#v#Fz$6{lb+|WTih~x%RLi0njffr0F6!E(>KVS4*&B_5XRCu)c&e=Q-M^$ zLLaY+eWvPZWSO&7DW;jO6nMC|V__C1r<0gtZ#HkJwI4?d{tqUTh7i#H8osC~bi76n zrpitzU-qQ3v{XPG^z(Z88wwWwmUmlqX{gGt_!hhEcCGZCpwAW`Hsyvhp~&6GjBpM8 zF))c>T9|K#mL8i&`)|Z}6POZh9S3J;YuwDt+XUzH96KP)3?|Uf^oKx?5K$8ENO{4N zucV(Q*5Dstoise>@Kpt&hQgrq%eUCZ72kTzOTL+S@Tx+vp0dBmyu|>{`z&IDkLC{L zO~3a_D`HGl380p&crLaw(|XHOU4=hiV4mz0gXVXgves(Kn_F1OGW3f$8qbp#%YO=I z+r)Y5#|9Ba$(rmm`2NuKzSkcW0efhKXEhlPYFZeFEQs7cBdqq}olX2ReLN(MK%2=^ z*@cB-kdor!?U>iT|I=IcREB$HP^}%YiON`vlVRe-q!YbUXl})<{|x!o%hNHI_*}<^ z6ECPFZj?lf>A?+IC)#WKRE~P|Xpvm7?w;0PFOq&s<-{O8vP|ly@?FK01hnOnpyw4b^JI0V>?z)a~7_xChD`;UXKHXmRt z8lsZ_#fW<8(p=X&Qqp@iI>eVh3pYFk3<4?;CNnh{$D`?;yAsc${wc^Tyl3H}8u%t$qNu&P*CF7UG2oq$S=+HCqi!y_+ zI)CZX2~UQKbJ(eP?}zMEUF537V$|NWz!W%>@}>{w4?q0_m%*|m%%2s=EGHBB~uSUR8BmR`1#996LMeV9guL_pCT4ZAkX9j}p0j=8i-B74O8^SSVh& z&;hQ8A;DBMIv6}2c{`?Gkvaez&x1EbMY2;pR*_MZU*3-|+v)^{W^W{%YUfQqGhSK2 zSNpUn-bC#ri1p_=uNI&El$d~xS|aSj=%xlQd|kbruQd3mFtg-Qi0G2NaX)X)EFy5B z!gp~AlmefI*WJXBj0)`!1#&(q=|{4n{`ZHL`VftncL19Y=_%fV`UTJwK^DN&30}9M zq2cYA-de`oS)4|o*QjBHDH!t-o8$7^QO3g30Xy`cVji=7E#C&BISuUCrNuG>j_d*&YQBdafI2)PYG!m;2r+8SsS?FEnL zKmR9RB-d|9nu%I4y#rT5bPrH6=z~xioU0r>car~^@G$_7xfS{UCHtpZssfFoJR24^ zSgt-(J1YbmPabUC-X1f&&p%-?xc*d3hj1QirB_P$Nz_{%?RRM7V%e$z4=mHvPTC z+wg!#f~y7UFT1Pzk)(U!G%ZBJ$8Pc zi4e>L+bU)sX;Zo|wk|e0ag9q(3bQ0JT?dMa*o37OLK%R?jMrM4b1}+DnsDB{7ZnJf z{DI36F~G{aaubJT`;UgLzy*VlSkT-;0*hOVmP0mX@VqKP)4*RbXT zur4|>p+F)5NKqx@gMpDH#5ZR8iy`r{&%$E>tGT9@iWt(DQb8RZ5o!d}FYh{8g?}wf zQDLEtN@IfXSfo4Wia9F{lI@i)k?X?x~?tt7oO$di^|*sKPFvuV&d`i zMa;00fzDsOnps)xkE=6M!dToFe&5)8$An2WJBD{vELdZysoGdu>%M5@_cu{9-hz^+ zdp=VR?6e^({;kbIz0Xcg`pObhrz3CdO--PWWKKPq4*md(Ddvu-wNX5mV2vAQsKFfo zt3nu?0isR*7QT$aF0w2;%zl9$Vj5VxK6JCfFo^^BDJyv1yr+tzRfRnu!LOFxR7C=w z*doo714lq#$r&+0VFhGa%}5S|RJUC67u6)QY#RKHLGc%Jq*vY1SqecXCSp-T_Pk==iDKfzXzETBE63s_W1j?PMuA_F)|c7mn{J)Sg6pG85OX@w$U| zkN;P_OpNmSd;AjDa_&Fbg!luob+II3=Um(|B_hz=wbYbI#7+Mimn8-i5s~hEr)9i; z9|o>>>A%V{*k$xTwmMLDnJ`1X@=6G4A@$o!1a2er14t3gJZovh|2Li|pzwXEcYXnA zaKDuFIqsD2O-j$$&3^mcQ`kg z0?Y@asXRNsgVxzgg)>23SWNail=%CbtX{3cHT=EXxG*iz=0D>^^7wtLc3x?xKlSb{ zH+|^qow*h@Q$6jz?adm+@fA9z{$p0j@BDCYG06fez8!hZoje8~dsM@F7Vdvd2-y-Now!oPUvlZ@X}F zLdn2!l|qL{S%tF64)j5x2G+` zxz%1q&Bd_rVvi!zA=ZeKcMbQ!Yn!x83ltL*lQX0Pn5)D{<}p6Q2dU4IG_h!1hC@g? zhMd{C@h8U8!sI=yOd~>>)L|9ZrQz zX}W((k+<8Gm>3l;E7m^?K9kCcLNvi;YX_}dWJ{8}LU=^Ag8^X?rW?W`S-5(k>C$ie zSr@lzcJ5sD;S=P$<>-xlhNGfYtoX%$tQ%Y27T&7-m|8wk{}1_Bnb?7`4Nd;9R9dC; zy=vFAV;K$KzZVh3Hg4K(=P1N2xeQJpkD4f+oTzt<@6_MIshyY?E^KRc}C@#FDx_C(MHPm0-q}RSbM{ZmTrpd*` zXWlz|6|bTs!(R<43MvK!WBmBSCWZT2c{B5XT%yj>6UUDW>vkk}bf+xZl);WdV3WvP zTjI>I%3)t4vo{zKqoHcgwPRhyJ*gi065=evZcH6PD05cV&_EC!)PM`2IYYO9`l{FR zZ-8s1$y@sg#iwnHfo`6iGrz!zc+wW|Ok2GbBkR z5z#P85<-ItNm`T!>3)7VuiyXrU-xz2zef*`^LL)A<2b&b@qWM7dvG^yKq}{7R3eyo z1$Yh=Bb0!IhYR>u07eoCmm|2A$l!itA-pV16M)_qH0*!@&vJ5vLjw0{3;EM)Npcfe zK_WB;wGIR3;ZVTdHm?i{zcR4@f%qca8P0N4ZQnTG#* zbk69==X$?Sgd_s^gW{7kX|kMtrS1MDLjx;vF>?x?r!?uw;65v!Y2<+RmCAMeBH@5F&KbFYT)k1>|oT!rQ>$0NW!vT7!R8ZfF!ca znt$GH7U3@ltvbugA0WpOT;=$crn+znM=_O9#KNLkm6VjfCe-16T3Xtg^~w^gg+;)V z;)oE~BcqoZ>#Q73>kJ<8{oA+mCb$kJ9UcM<^x3O;YfR92*Kb0f#R}=RdgZuo@kW-O zGg}ukG84=R(xq5&${F2wLnLN9*r{i1Nj&B=+Ohe#t9E?6L8 z)RFLdaD*XIqVcb|^M3s`Qi4cR+;(4gQtjl*gKB=9WMN(7svCZ_9q$)9a410ry~k|S zrU+RXvHeB(JL)&ysE)Pur3XFpsVHcUxKGZ7g_4*@vE7xGX>Uj3vj`GtArYi|W5bI8 zA2q9|{Sdke5o(V9+>k{UYu6Ss#&YxL*|>|IS;w6C;&T4#&UFTb)30VMMHoh)OPE6I zd!}wo4r{?iwJ(vW<=KdVE7VFZ&ip~T5-Dt~IfDsL!+Xg+cyhz~<#5oLux#_wcW{|H zr`MhCS20+md}D!uaGh1DA?cuQ+d|)AM6$o2@uxNgIlO&w(;$4EM0FqD;lp#__FAVnXYdd!B7=Qg5rx0jM z3{rrg+|7rBZ}k5!Edb8Z3*TyM-7_q=-ukuW^Yx{sc4b#ZB*v76M+Q|U&9x>qo5hxg z!7G1e+XS}m4dPKOGo`)-L*P4_ss?BA*k;R=*Bv@2{n5hWm!eMV;;siR77aXB(cv*U zp4HE$b;>%mg<}zcDeTXig<+R3FB#4dnt_7{yN~kL>35F?3+l1Y@Ct4sk@77@YhX5# zrcU9orVRAZ+T(GZ+bAI2_ZRpta{S^cJ--F zCQv5TtOb#ko4qm!>e%CMcrTMB1>J}He}48<#q`=PVq5ML*N0%Mv~ehF5fL6*YF=O( z!~iaX>WLnC7z1&mcu5#4Y#S zMi2gj*f+&ps4p6@N!xBoORR0T$0+r+EF8+6wyvCBR+aB$@lk4j%1Sz;6zP&i!9 zvs$oPzkmN0$E=VGK*<(~DUU=kc?&O*eS$s2PJNd5u6;-a3OTp9i-OL?~-v1 zRjcASfJj3TQ7GoAtzK0AvLDj=Oh?6TUSxh?P+)M(LZ=zVW5x|7jcBqWPs%rVu8R)H z1Plt-@RI>i*XBvKe&m__1Fl8Zy1}6sX#e29#*}cJ(tiVNkpjkz0$adoT z6tKzxuaDEvo(lVpu$7|!uLh+n*_ z+Fj#{`_juV#k3_c1A)&%I5(LOg1lTvdAX>VMeGU)(3l1ia-NA;G~!4haD9?)Bwxo} z`ldU^Cj6=%X)MIjG%*|8lx&ScvW(K^j=r;x^2RN*geIe}$dpY}+O1i^y^tKFd` z!$T(bJQ(lR*!1yxgSoPV^JoTA0A9(_z`)5HbsMTedM^p;ZXGHgGa?DQ-J~zVnT*Gv zbfd=Jq{>0wY17zC_!{nXYXj`RZr$gTTgG>|yXnQcd?>D&eNUdfTFh~g1nYP}M$thc zagY^s2auUwGWhO4CwBMn*+si-onDVQz(?UZ- z^}+~Ot8A#$)Q;JdR%6-63cY(*@e7ar zP9V7Oh52ZcL=k*7S<@0v0?E~mXI?Jn+dk`ER|7GYU3+dp&WweZUz!J5js25ez2Tnn zmW@|6=#ya>rc`VO-Jf6CTJozla`(1IhZ!?c4lI;4RrPxpw@ImK2WgUflGe6FVl+@s zvyIuPt+kV5vrR|8{M7SDN(bZJtxv{ZKk<0MG`DXoT5}Gl?1i_x z{G*c)8M!HM&;n!`B4cOOD34X6!qlzN4IYc%bdPx6?mPct2E~czaQ{HWA z7v@T{?>H$0m?*GZvyVb!uHH6nji1^JdgYWjmSkD5b%w|kE5<0DK+?gDEb<|`bt`_g zb+K;4DuYGjENs3ziBu#}oyN%&Q7Mx{O2ndXc0{j(Z)S{ttemhb4t7KgZKef%+3U@v zLjjXc1qD@9(eB@4?}(`W#xWT7+>#{mtB6?nbTl~}>I_5*ri32GZbPfvgve7WAiym= zFntGeqRxH?eq1FiJQTxr6H_^0A`B^+aO0fg?*jf%Ny#ow;L1XLK{8xlAMcK^s-lK6 zFbJt{XwV&g>c~VNjmqUY>Tj;27#I0yP&i~GZE?`#gXX~F$Q;+?zj58VJw%WSDKcn# z6y~LE(g?1nX8Q&%K@p`63sAby3Zj{M9Dz$# z8pWL{drP>u#oQIdjxBf^FKkLNCIws&M&Jmdex{?R zgT>w(FzrGkx4e*8Qj~!Rf&w6~uq%B&SR;3`>dkF|Y*ZXGXY^yZLB-TDzu8uMle*|Z zA&S-H%><>{-rOiVZ$+;3M|L8zg1r@%A&RtaSPft&3B@ySO zuw07pHIW+#$jiYDHKgS+KNyFqCP{U|nI;&vHjv4J;$@U2QxyB`>iL^%>nzG_lHBis zYzeJ};pJ+w1-sGetJ%q{Dt_9pk9>#ZJGXD2T@2=F#W=>eQ&W{|8LE{_abEhs1nakI2Z% z8@8=dh!O;{29KO9wWmU-^+!ry2S!8WJKDl88(bGdsElIMBXlPYCbbu72W0i>S2ka! zl@*@bc8znAjE;q!60C1GJ}=fw9mZo6&eVeklR$7q5=+cVt=yD9I)H!JFoLv1Aad!_ zmqa5U`_T-+f{@iUH5CI2!fwQr+ClYzLyH0S?UEmz~~qTjrcRl_r#nKswQy^LkOIl^U=#H&5R`P@t9s&g93-q2_9itY2hIB6^V+u+ z6*VSz^hH%tbdBm$Xqm=)KH}#PvAe`F&j`vM6 zA~^t!;tme56xr2_7k5Jx%>-@y30vP~SPO{AXO!`m=TQKeHMb1Kb!p87E&QY#H%837 z+9S;2YrtkJRx3Y`F!8>zj}e|w8Y(vF@sjf@5#c##22ES^7;6oMWoF}#wH6umFJKKx zB5x(1{yqW*{KK}AmV0&He{)fkm#=#0mCSn;QOX$I(LvKLT)2C8x87=B#78aQ!g#bo zObWI~-Pd$CJkNsXV^$HpGJjgxSB9r-Iby2a?H_)Gk#)ZSNM z+_Pu=6&|sLsXFUYD&M;F@apt3Hy6FgAHB@|!w^VFyl~Rd8Zkyreg#Gm#$f@e(VNHz7K(BKL288EU7o{y0=c{tDbKDa#CKIXuV?k&2)BKY1fj&7jTT zjUbr`DmTj*`ZrHWs&F%q3&#dSN`#Y~o`{EycoiCx440pyvJ_AX*qq-Ix1gEyGulf| zC%)=3*3~cgo6*ze{y;8Qa@8>ZA_@hKKmvrSn}5Ex{K}hA1NZ?h!07ia#)=&e--|c{ zvQ#m_GxFXPW)zI$u0YOo;uu>^n>4rV%bLWQ* z{3+6;93~>EVlFGTfBwWK0OkP{o#k}6uC_K~`Rr&zuqR`yUcBeq1L zpT0ecS9_qBMbvUpc-~22$_l|jrTSO@rt-57qgO3rp>Q+EC@OgYrjPol|1dUwWs#Y7qN{ z<@&EgX!~Y9{Xzo^{m8&DA>hosJUuXFr$@crXGfaJ>45 zF6yRs_;Jr-Runex>;Ld+qz@QAbOJkR-8dGtkgoLh?wH!6U2B&7e!xicqO&&*@W7 z=#bKQ-|eA!*Dk@g<+BbByP#N{@_oOia*J<%Qpw6Ar!QU6cKDq3df)clvZNp2|lUxH#?vT26Wd?aZveJ#1xJ(O7d-GGL2Jo7nKJTfe)U%JnR_IZ+#K zd%NOmgW9QZO`aZC8YbUCg(yC%`W<>PPUYe9jwVy0o@BO_JvDIR64@zHQ*2i^$w@cu znVX4|b#G+kXaSBbT}lG!ls9Gx$+mG~D}c`c5GR?EC5iA?vL`zx<1a^UczN5L$T7VT z;9+r*>>;;;=(Ti*USg$H|4$tgC9z-ik*dO4)ul3cL;UnZhkcaqbvd!-pY*|Oc5X=l z|1kRauYAVt#Lo6=M}3;!?`v6o?o(Jq^8NIe@9l1mni7V17dbU^a27?*PQMMKh4h;3 zdhXh_qf$j;=^bEn_uoi#A+AE2Lp{_h>C=|xeARR3f1LB4m%KbP_~5b4f9#Htn;FQjuZtsrU!&@&;7Ak43+OzrK1!6hfS20|<{5qn`*_ z67t{mBV=?tA8~NrG*s?l(y~FDpEMkoTib8CV)?gfArkc0N0_A4Y-;w3Qx2?=YPl?;^(=WyqOwvuF3u z7j@|o{5AG_L*v0Ny)N4|I9zh`eJkhb=frKso5Iu4MM&x@T>h$l8PW0j=1}eP1C$;IWL#IDco^&h%H7FIN33O?PHX6snqq%i)Faljp4OofhsulM&! zTyyZ~?#orA=>VYLzm%12{*>Dv$|79B2q26!!4CMTiN=+~S*VQe(#RzU#jSH+PT_ zGc}dQpAg&w7##l$=z_QrqYgnU$YCjpF9~`eICdH&Gjda8OVE%A2&d|@NmDyO(oiL# z7lOB7OaZ5U79Abrl@J-tf0F#kZpgThyro5Z`qj(0-@M&CF(&2ll2^;SWph=M>qxgMJP{4w zKAv#g_T-_R_n*;=$&qpK^EGjwu4|>6= z$|pE@I>TYOK}&!HjcjqbxvLcysU~p^123Uando3&XuuVB|Jp2aP6*WYuWE?fn7dy9Z!m{H>nhWbV73DO z{6jD>U+T2>%nwQvOTU&DcWFon3VDfggG2v8q5j+z2Ocp%jSJcKNsnoV_bF*QbQW%j z<2paT?M`Yg@X;}yJxis?`O=~>Q|k%_w$P}XR5B?u zNhCjBr>Wp+kdi-Q)`O{Ms8TTIavpOK|z+6p0snHD&&Ds`e^vpT0TJ+ z%GJDpQOA6==DP|KTs9jsR~v*|sM;l}+S*6ppvm$T(^4;7xFC2sj5ySSA_wHtYl8HC z5hsFc{q8id{z z0v&}Z=L-=OscdP#vvJTaVn~H>S^<~DZH->8BF2dc+yUtV0V8+Je%{$>f)ceeON_qw zlOu0YoGv({ToXMIK_8(zhYFR5NObNUh`aei0642XO@6iHH);n@278LU^P@NKrD{|2 zk4@?C8j*m>VxRS0?btTh3Aa4d1EZL>&oz?SbL?0b`c*27#{B&$?x~iRmJ*db`Bu=Y!}qwa zP+f*X@GlwYI&09k($l}GY`Sv5XeeeHf)7RdjYgcP4Y416GAsus)l1X9Vc=A@uk7gN+vbCQ{G1S^ zDl5AB?sGjFFJk}oyTl!S&n8O&(-J{LLW&r&XpOUTA6g}O1%+1ERKImpxwu|Q+xd0z zPHM^~+XqE14W9#<8!GuVaKXh^GnduaI9pg;%uWAQ{Is-_K@LXwC(;m=Qr&0lJUtSy zSU+L!?%SzphaYB@3lttaRCVmwqHK-&v^%3ko{rCrH>RpmHpU%wrA`=@U>MEoz+lX#AE6{AOAYU(`8^1`NGGm-E+-qD^F(f^n3XoadF(L?3c3i&zE<@hTZ zzLWamMRx*zzW7wbZV2HpML>T{0vwe1Oz6h|yCibY3rbCY39}U`y;Xu-yEip>3qB+M zGKj{suxy~;Hkf?2J1OBvp+SIPbwdUd>NDHe_>4)?SaQt%rA&zc< z=gyw}N6#&ec&=3e&A`mWf1PA()zsju&5S^VyclOXz@1iCZ+2$f;D*>VRb3kkk`p9* z__la0Vq?V?7h3~>`5q;K|5I6Y#KgcU1eq0Y8Y%6$rp@pD&MpKa|o$Sr58ZWO7 zxcW>T9OS8lUc*<4*}MeAg;qxAmSd%yh&xy4)P-B3t|DK|#}lHm#$O4}_!mO+7cXAS z_7gu7{=FlUl{*Z&`MlI#Xaw>UPf!{XIx=R$gtR;_LOyT*ZY;y*jQWC;YdUC^Fb9%E z`JMv4Tdy0+v6_w>MjO<{)ogi8%iFP+SGHldlSh<&xBizF;Do7jx3lfz+PdFA>wVs_ z-4frqfj~!q(=zNYr*9u-i*aLAF6sJYk*rXc@1*QFU1PuivqxT?v`#3bC+$he*#hAy zeP?xL^)$v6N;3%0{MKADiH06BD==zo4!T0BvN6Bcbspv)e|f{_0hSk9 zs4X6m<7zj<%6Q|&kOj$8F6BBczIloPEyNOccpG3xZ_F`hS-Kcfh7=-zRS%Y2UcwIV zVe*duT45#V;5>MOgrId`ZTtE8;fSlmRbL@4EdmiT|R zQ8%heeT7MVj+l&J4HAHV;n|xvk8UVPAcwDsJG!pf*m={mh22*>m5(QH7WU*=ug=7< zL6rz|pr6+bE>mGowVLv?7q?#(K1+xE)vj%3aLP;GZ=Su(wqmMMufBcngl_!iJ4JU= z(OdE9!Ohg|b zwo==0VF$4?78k?tNipEC{g>V;Ad!;xM!n;^9-Ph!bLF->k@eZDLtTU?7a~A|&%6*K z9_9P=nQm?U<@mVYRHBmQLyt(S*$*m@bnea)p(sZgT zwuLzKb}2pqh4sT>{rb{Nne#gDooZ#PvSaHVKjMIN`q|~~4$QD5jO19bdcUT@=7Z-iepz^c+lh*RJH5Jo{D7;)9X`<%G2vP*Q(zUj-+eS zt>D$GPd~Pi$1G8;lcD2947Iuax)!SIzj_-X!8_j4w6OBgQZz#FC1(alXLnN4w*UF| zUGLG@91Sm@8rC>xD^c1A*cV^^xEWp})+w_pltfA|n$sr4V1 zz=Zyp%EVZZo4cE}=}lrOvI$yaAvK)r{pB*pHJbA^JZ6d9pQ*mJ}4x3-d1k&bfsMSPaoSK}xFIN(R|C+4QDjLW>!W-;2{0m_e zueqEP$?smfMzbSwH&~~6y z)O!)#tU6}QUeWE4LM-M^B6a{T^d_rg>!Txs6!ITFY$McL!sg-aVms98A9sCKSrvO{ zWMx2Zb1&J*JmztuHWW{sG{ruo8<^~B7z_x`X+yI_17JsrT_5H&!iJw6+K!cjuUaVj z+mFqC`{Y1#=TroEFbd<7T=ojaksU0|MR zVNOrkz3WZWgT3xa2az|UhcyF@(hkWPg>XWoyLH4WpCT?dqWma+IUzT{(ei0)rbSF) z3$f&wIc!phm}i7^vB~bH@j(B=&vd)y`CJNRF>?}a!-6GCKFFsh^bw@|GZ)^TS{bX3rhy127GqtWvypJ+W&J~%LNYwn4_ zSGl=mv0);E0sCXo%Sj~n`JJrRS#B#=hk9l1^H;P_aL`c}k>xK>XLGRSn@RT(Zro+d z;=XRpJTp1&1JVL_G?@`n(YN#l2YvoOz6IUgu~W3k&TbysPrhf*l9%HRFMYpm zNDR53(-{~cQ5lh0HbqP^c5>GKGtFHdLo&O*WOKMkm*&){s;Uae8__k{(kd=Dzhcaj z$B$L1tW*xa`RA9IfR^r{>gy?9V}v2F{-6zB2RhCe)U^<(%<|QZZ1hve@eEHd?ROyJscqc3@`@dwtv5LTI^jD;{k+bG z2Rr;0IR#grIC^vs$sYJ8u6MgH{UIQb_tF+R&YKq)L`H^|1wX6px+**U zjg5M=;qzL(%*AsiY3sc zXA^4#v%>q|UXfS&<0cABMh!4KD6z4pZs)*0ff=dH0Q8uGRm^sU6>z2=;T*Iow?-w* z-f3=OA$7N@d{Z@-UE`S+(Q--kj)gZz?W>D%$TS<#_pafDu^`KGeDB^|O3DQm#F)0e}{_4-;NR&KC5YrBVe4_=iuQgd&2@n@|>kERLBp=~0JY zBsdfjD3ktUI+|#|)2H`wN!Qm;M1@99TzdDhr=EO-iFVby(Dv;W3K?N-58C~EXWktr z-xXbj>j#4IMy%uNA3`S#MZxW2zGX|-_3PK$z6++b#A4**)|WyW5fG^A4C-`VB7$(k zRl}!sHgd-bJ8fDkh7<2K!NDOSMnH4zI8Qss%3i${yHNFtrLr@cCc;9y*W^RT$YpoR zcDa^aYN1${R@^xENZA2pt+($t)qXy8#N@JR9+^HoGOnoyNNNrBT?$luMpzd;5oifY`g4R6BKt!YEM#->`=$I=RWUxzqN4#y7P3)?*S1#!Y1#!^&NOI{cdTbahChk zrWm<2x-4&_cdd`}xp8dk9ExjHI0-yNIVq3;(Hk=`2Cpwz5{kW~bVKqviKG!puBaeT zs0(fi$u52nZW(`}> zlS@N)Go+?lvf&SsX2cJnN=p(h>8z}U;$84P1q<9|TlL!4s(p+R`~*0h;tPl8F-qIZ z-8QN#z`wsss^>|~A>7P3xP#Pa`LHQ|Ia6(HR7g-^JUV>m?QpAy#+SQJdAc&=y~6-G z&pQQ~PP=vmEm2l(W484Ez>ouTj}9EL=#c8di8>3f`FTAnvm6t$$js3!y`-NeqX1{u zm4E)cJUVHfqur<%fAZlXf41Ju(MsMOTHgyHA%oABgt~o0)b=a0`ImtX#ko|MIjeVm z>_NRMHLgwA#}6Nv*471FPIuno9pxT5W92)ATYtH#JY)!-XY^D3)9 zdcTjPwTXC{4)r@3$Id4(1_=CkSxSpNBahAc?e^Pse#o~85jG*krCq+oSfJv0Y0YFF zkyMO#o#Xf1qDg}Ps~vpekr;-7av1a9IR5+T#~vpdfjx2K~w6a0G{sGuPUrJS~#Lt zx(412_p__Z?s&wZ%fxg7EQ;z223VDP;KU;-@NmA^L0315pcPk-)t8mKw?m}C#k3a) zkvF3})FiUjnx3AckUts1eSRo7hqC#T>{o$FpD(v^(JWoY!aI;*x_OFzT~hZM{Dv#m z?Z2LCoGW|BuAuhdRquwa{SDG0tyC2uBmu-F_mY#P(0fyxy^7G{1t&Ka+fCH(k@Ih6 z2$=5hQ`Q-|Qx}RCz!$)uXNuYHdArLWn?=>-s6*D;-9-`KDs zcgYlcmyMpLJ##XlX-H7HE#!Ok+B-kYs8U&|UKD^q9(?{h6v$1$dx((4%vB-)0TAy5 z&Xn@k>DO=WOMdFBqeNQJ^H*Bh4DFxa9z;uwi@9*Y+}Kh>JgfKTt&Sir4$E$Pug>NQ zz>&aI5&t7f5V=p4m7N@%4%V^+M9IUd4P94ey+~H*WtwRwMg0-@i0gG^Ok}-Te+93S zj~|&TQ3S6r(0wOAe`*ItvhoE2e!N||-9$Jrd;^@i{*tE66!N4R3GF3Y1d9p)kIaRJ zuWs;2Tkbb7c1i)L)Rl4PZTB@(H`09q5dk>! zAo(Ke&~d_H_Hcdk?2tvF+vlP!kj8>LNWR7~dzT&1{LbkL*`O{q8r7+UA) zs;6SjR+DF^j_lAlgV(37>~_JNS+gV@zr>?t`_wLhhYt&KY$xbaXkAhzeKak{`d7U& z+C&L#3T}~+yBy+OAh0;xxqyC@zJ^N|DRr=3Z5NgCZuZ0Yoa0}BDZu%07JX}cjWqXy z_&Zp2%-ZL-b)SkWea~q&C-B5h;};oBQ$Y&aM|Q|R!?vOwgG3|fZR4iul$S;8Apn*- zIsAJ8|MYj7OStI3fnsFlyWPz$&my^dqfTYDe$6tB?w%e~NrBHjgU%v-b$+6#!dVQg zS^&6LgRUCob;@quF|kj2Q1NevRl9qQUH=6QJQq4!p7XUVzYh(e2m)g92k-tp38Ryy zcC7!l?c2{2@Ny?mSL&Xq*J#yiZKb26u}P8ya^ossTtu)Z+`O%?^qtNg?}M;K0+y4j5x1Z9Y|qps6s%0Nay%13oO-`wvz!%|RDS)6 zn2tb;PNfk;r+qrb2PSFnlwvDI_z<-LT@vISK@ipTuPt98428-RFK3Qf)Wpv#MkJ?_ z^CVHR-$Hz(!Y-|m9^L9+E~y!s825RXduEBu?UBh}9$AcK7uEduVf$#K5V^hLpmbfX z^5e2esCVu7ewAeZ&l)s!(W2v-Ie98mEi)!+Wk<>N*wnsVyK7E$wtp0eDj+m02tc70 z644yN6_3drNtWE45VJ#)EbsigyG8!fE@L<1J`RHT@Wyd5{taa*qGy`>{3(sH!0l+t z`Iyj~{Je3*JW7N(DxWBvaJQl3_vJvCf=(jgmY$si(D3kKL@Q-fb~iY5UPLqhzsRhg zJ28w3%#ovuHNake@a$PsKLe9CrAMLNRhV73qiPro-iRMA8T_5X`ut;>flfSe80K#d#F)I zDVW~BVHu<{JXkl*jOz)p_^I4m0fH=^3F3@3)RPQJ6f_cvU~~ADe5j~o&52l&+*1jz z&wclMKhPE{h$#{}qkPeA_lb^7}No{%Ty&wX6)+SbB#d7*5gb3FQF(#&4z3 z$vtK)6uxnkX`-MXa_XJ3WtgkP25a+M4(G-KP|e3u!nmEYseg!s7N z-8H?x$IOaTc79+M1!{|H)Qxhs)>c-x^Yn*ZJyR|Ros&B_9l&>W*qQ9Q0c7QHLMsgT zIq4aT#esH&qnEbVZF@(F$omg+{x+L;hJH=VYXqGp32Hys!Tb09Q00Q3?L&*fQxIex zG}=@Xli*x)066wB4fFk~D%EJqyub~T_IIC&1Pv}A#I@pRrE?<*9?IfuOw6hHHw2@m zb~Di2%PQgFn1eWRFf-(jrLDJPM_lHHRQOKBF6mc#l`u4``0#|I04H!s#V7?hT5$v= z)9S*+l;$X93BMaZUDLU~GZ-*OvtZ#U4#h8)h|z)QF~DZT1XRFIQjIQvp%7P)o{`A! z1NfkUBh*fi)x0<2550!T(V-j$nZc(Z(n7dfO&2LW_)BRrTLQsU<=tjQq^vltL?B4M zt<|n01+i#6MMxbVoZt~R7fZ8p+RJ&mV@%=x{iJ+?3EY*N$=cw!0(la1tLe-=8~RH2 z@K-q3aD*3=v6_aoOrXl-`Lo4DRs)0wtT9#`ZWTpfJ-mm4jzB`kSaUJM4$h8KT*yC& zq^Wp!xAJ?Vl3CNd`SUS#uplq6{gJ8*BHcUfdt>xP6h_qg=>gR=d7_sVT{p-z{SE_W z|5`OYU@RnJe)+B{Hzn2J4Qv?>MyM?bG;OP4od0`uI-jgV%<0rL*#-frkB>Ci`I-4W zPyRSMzW6r$pk z)fCZEOg*RiSs}T-R7)2Aw97AxFuPkwD{*>?_JuB;aCDaGCtt6iwjsGU>!VWE_bgaV z*E^8QNg(5qkyb5hs4*n%TaZYgN!_u>{a(6R)O)O)>63uW#eC91rj-UvZJ{Em{^)Bm zHfaOUE_*?sUS^dAf6|4WHLNg@U^*wX+!Yq6_r3cYaGCu}5i-9>9Bo`g3^+JCySb7H zhlpAL6F7bZOPKzgvEYo(5tJE6PA5f8kygO*6fK(wvq)=5`WlUrt(r3gx)?F_{WaqR z4h!+TukQso4PdV_^(+nrkv=x2#eM9I3z)}3k^N~BB1P^)Awr9O+P!oB z-tqe{6o;p^U3w!kxOd-hUQL^UGZuoIueEFn3k&mq_k=o!q#O8bw_a6ey+@N%yj(@L z@tr~!2b5`?$dY?s%i2mnlvpYjQU6`3RP$ajzk%=lZz`qVwx37pWA3zT{_O(jl5l3S z--cx8@+F7$%oi;BbnxY|IUzL7T-;^(q~^7Wd=_IwSa4PTj&~dDZKlb3bbGdTWlaMU zy_aSFE3Cn1P58*A>y8u7A6=pLw9w8q@yrV3t#~sOwViJcx+CyK%sR7MulC%4f2ElD z^643Bhl2`_kUn?q=t?TagEZhfHLcF#t9oD?H+uG+;D7*E!^^cB?#MK)G;TFjA`DbQ ztL8qng?iv&94c5m2A5Tp>bKPx@2fs>;%B*4*~4l){{(weQts{kmJ{T-bvi{27ZzD3 zRhQ?rbHBea^l7R6RNU1na%$cop`m2|j2?C?B&TeQnp&~k%~tnl8oM1x&`!0C_PwpV ztVW}V`;3bsuC5pdqW0nUT@`}kf(HlDG;r;nWsdE}4{=B|k%vg+q}_STgn(-O&i|_! zlK%41AtlE&4-t^5vm2&MmVmsHQm6Pp=Fm^gvAmq$)$6OWNoM$>@P3`! z_}&xzYQ9Mf%1T@Lzv@ZX7#}~$J3JUE63EK;hSN10j5W<;>nGKyRGQh(DJ?QJt9H_D z-ZiSOzNn_F!TPZ^=Cqpn)}Es`eyizSzC9uLt0`ImNNg&(V)fMxLogs2RU2YBXkJW1(PH;kJ%A+6)petw$7??7tSqK zb~7IFiMttd;vc)Xxa&hh@!1h`Rg6yBLV6WdbhWT_31W`ED3&C8ewCPTDUrJ#Oz9&O zXW%lx4g|rlu^yDW;8%YjNJk#exdm;wnn9$o4aV1YD>%I=UVShV2NCVvWoVtRb5JEN zKHR_}W{X~k`KAP)3;7)4e60Y@%qM5Tz`>4b*S~M|H9w&z z@(2{(d=H8+brCOuZi$i*e26OYXhijavv0j(j9j~TWxFrAbGTD!W`pS{ruC_tp$&)9(T05>yJoB_iWMSn>&zIj`Tz#3?^YXml+PHatIkhtT=9jmhiu6dt z0Tcq$;Wuek!JtHmDh8EZ4EN}4GG*?n_2YV<)^hR*4+FtlNHPYWh=}`S_(FQ+8p^U0 zz3;vT89Sb~YI2Is_mJSTanssc7Roy!#Ta$CAnfI~rkf*xSuO=0kC|9dYJb(*xUu-k z(n3Rz2Q8=>#aLk)$@_4(LMl`ewXVo#;S%5p zxiNNvNrYSyLcuMV7KxmXi=aFC)VaGn$o!-<;}RB;2FiPCe>gvhS-*X|9C6`NXq$a4 zfWB|C~^cDnJ)a4Br7zl`kEO{-5WGdy%NW z#~r>3y)8AW2=-U2s{a2KKi&VjN2mGgYKOwslxyPH5)8d~eURxuR)r{a_rOewkbXLh ze_npsW7)CmzRN6G5rf7sJpJH_W5%?=VI*|I93GgkL^3{~0y%v=p6awIzc;_p7y^;1 z%biSl#EpwUa4&vZp(S`wsM^XEFD}lZl-IB2h1r(>NckkP{i(qecg#9J#Mc#Bu|t03 z)fuW4Z#yOYNTNc~RLra-I}h^i#MYUSlOe5ijq(iMsW{F##tANYzf*7W@*Kfq{raUD z{lh=jK{_DlVC~6CogbdZIVuavF!$B6?2UCLcg}gAo7qN--LW^jwC1La-5<%*w}j(A zw@e?9bv*5Jus@xk;pL8V0%0!LkFddR+BaLOQMQ0R6j+Ki8Xxy6tGNqeO^drt-hC0blmMXBg`C#Y?*TUIEaD9VVhdGXoh#5n9h z8)ErL!+z7VyHl=4hVf&XYYduid>H@yu2a$_yLneSaka^jf$u;=NZ?lS4_N;F$kSpw zFCKo%FF2Sy{>eV}7x9C#?cQB4i*j2S!*tMw?q1nVbB5Z6U-yq#8lZ8w!h4;)-ONo! za}?y|?S2}c8`=&eo_2Yzk8IY2A|4;XpevCCc}+IJAOK+ z^>W5m-}sPH`x}igSQg+xLhJIHvum#TYIfZ>#3*XJ=jhLwsS7%!PDB14nL~OpeNF3# z;lSE6TwJus!}1aqgSyIhM1-3C2p|{r$EC$PT;lm0QjtC(sBXvnEr^5n})YxR_+qpFVXw{jpYv%pt)K@>WEwCKWNgnb7CK(q20GQ?&cw zFJd}y%T-Vsj17jJM_(_I&}Ip;7o0=&-yALEUJ*ziosI7ml~n@$Ln zcs8_exz}~$URq;p>a=d{HWN2%fBn>78#^n=oC$}pa8o>*J$cj3EfL*{tgF+z8s}K> zeNSHfJe!$?`w}}0Jv7<7usL060%-Qq_2WjD7+sxr@4nY^j6FzAv);aKcj&w*4k=|m zZSt3utlr(U%{%8@aB=S!+KfmPvs*9)aAF7!@6;Sn>?i3D6TY1+2MRgD*}=T)fbi_8 zMw$n{mRh00@U>W?vTWe~ey{H9MdWwYPqc7&?q+fBW`ylSsp5P0|FLBD(Zxlj$J@Qy z`yAF1w(i@vE#k~i-xkS4Y})xNrQ^{p9Aaqc*bP$!IKwz+X!HR9FhbbQkY!R7z|+P2 zTAm;B1Z=v*@sT1TY&0!Ooe|fGdTB_ILrW~^!e@>zk&hY$+K9U{!-b=apFV(#jQ`Ay z4x^uf{MT>FW1*R@N|M1(s*9PDN*zR;qQnC^%+S=H8_nM#iVj$Jp^c%ohnW%j>*ppt zFKh~EHQs1#)N=Ni`z7CVEVk!mcU3W%aKdxLhi91!F|D}l z8*yv*jW;%8iZAGch~A+!6r+f^-QbbMzjx7iw`Jpb?=Y)>jEsyRk!8W;V{K1rS1>Wx z=!aIQ@BM}%7X8o>BgDo)fh1Jo5=GBhBW)9y|1Pi`S`1BxDv2QBf`dKq)9|ecAL?V4 zKcCV0a?G&}aZ;Y6Ym+}myqr2_+Q>m}oBj3WZkYC2zP$2=!_TWJkhVernx@UT4qwer z<>T?cis3Qd?-I6rY{w|Wq93pG6Lx4_{a$PDvPn2z6m#v#jJW$Wt`C=$kmH~U8=-9w zvS?=qIk_om*<~(O@Asx9ofLy^M2y7oH1J}YGNnN4!~pdc_k=@Wu(yB4r?tB)t_shT zqw_BSG7(XcYyNK6&xqWW15Nvnx7+O4FREc@V|?Sc6bw=I2+L)}H%*ZS4 zqTi%oWqCGeo$~oH>t=E0j_8A$L4EyFX8AH1g5nAAP9zID94TBVYP{*P72c zLyrUsj8r=-C#S?GabBm1ql5e}3~fhWW@&k&Nhr`6S}7X20mn4Aw{+F(skd$39jib7 z?=^4rIurMBm2O$`nyd6h?^7fA0u~o2v66gTr1nGt|67%ec z>J4tSnW9ak6uyQwDw2ot@Oz{Rr07GoO?Ow83l*oPqv_sCx$Hs61_pEg_ z!w=l1t>Q%sFO2A5*!VNZ#c$WXy|`1ZtcTwdxu~$BU?hBs;@rah#hyUt(rBNU)=NoA zq>galB%4(qpYw@YN~GkiUactVa{+=vM*W3T?-R}K;yk4J99A)u!qYZxEI;0>;gqQM zNOTw4Q0Sz^nHHS*h*1%>iD>l|E2r8FSBT$EwzTE*Fgz!1kabRGlq|0K2fk3MesBLC(r@(t3OA<&2p#o$joKwe>P_O8DlW2`UsBv?2P9U&I`io!o`&N~B*W z?w8nWU=8vsf8yNnK0O?;0wpcWJIb=}-K=%V@ZJ9xhkhuiV|C8TK9%+w&?2}TC$^N|bi5pom5PLW$AblO`tInC63XMA+xSXg2N%knRd zbbsO6|3HMB?4{N%TTuBeOId^8=!BMD4FRqTD~hGVcWH0m4cKj2-(t zqK)wWXRYf{sx%H}+AyuqPGES}=3pAJCi(^-W+Ir0ilULI5^Wv zZ#zi1+x2r=;vJ7+MFB{L3V}Y2x=E#Fmlk8ulpH!wv%Zxk` zzAj{4+3CcFYm8>{1_$W^i!yLsYWHffG|9>fv2Hl$Y4xK6ei5L?|GlL*z(Gcimv%Y^- zBSNwW8OFEuCmd$NEY&0K8*LF2F8kQ6s|u*UQ*B!NnYZ@Aui<41pa@jT35 zk4+Mj<@pBWO%e$o0QDD5ANz{04XbMrthp|qT(fdKSSA-F$d9;Map|GdxcriGI%0>K z5Wu*;9uPbIf0F{mFXcrfI~M(s0x<#`gx}QJ`Se$ReevNY2efH0)%+{#5v@7;jA%vv zy_7^aAnfhC|JzNmbIIOjU#N~7ccUOf1m?6EknC=+x_bHYHlut!L0>8Vul~Kb`<<;T zl9s!4L*vZ+;7qhTEr4ff@PMp9ip~<%@NC|nAy%%Y`)pK$e0^p30kN^MgL^Fb^Kl>M z1AFoj@CjK_Liv7l8ECMCRwrutZS63EoB$C)*Tggw_!eL<({a0>^gC7d=bL#@SDrjw z*}s=p7@t$W&Rj`HMSjbGeb00!dZ!s}4_*g8#n;~e9?z7b#S02Z4n=cB_#Y5$grUMe zpEgbU8#1Ksb}DvNd&{F04f8F#c^*FOzGV2?S0Z$y^Es47WQy_|R1^+4X(*Gp+;Z=| zunn~}HOyEXIC5l1a`}Tlb^SZ0Eo#!0obA5-dw3zq1u0sCY7p%YatiS=>d?XnF0`;d z=-G^g-Mc0KJP!>_|L|8q)4no+0GZtVoT!}@0`hI^CaNXMtuNz;%_a88L6f>7!q1wC zQ@S>aHNW2%Jl#Kf=FsjaXq=jysT*h+#X+Z{VzV@-;PDm1GRFyfSuXv2*p~jgBWrJM zJ`%RgKJiM#_vGwqSr7B2B{ln}58y~_8?R*sFf5QJ5ngGuJ>g7Tfg_TlE|9Q^nQ0bdQuh?;O&c?-V2a6rI>dHj@F0i{| z?|jWFm0E!M0F|AqV_$ZEhW>zK5(&v86H>I_t{OGreaQ#AiD!pCE^B(otjD9rkGo#G zcI|A(AB(!DEZG}2v>p6e!-S@oKM;4)-WZLbf;!{KoS%0-1&gQRid0cIGDoehwTj$^ z+06v=y1g~ouOB%pvXy;CLnj|m{eorl9*NY2RwfkoOs={$>I2>-ah)?cOlXIpUYT9& zMnf8$DWj720x(y=6$hC5fLINl+WXZj!?+SEA)z~Bi$bj-SD+KPM@HYT;TOM?t4wfl z5plBjApcR0wq>$;1u7!X+mWKK(v*!j48ndy(xZ&t37(Lz*>YyaBQks=T8KKJ;#Rmf^{?zpS4Q>s)eNIj2TCG9rMpkd4pS>3z&e{*Bj{j0AHXY0UOpd` z?^72p>;f6u#xtcQxcYkfY%atd>98f3-+&wD`!bxC3T#z&l4@z=Q2M2qTyDk1dyd*Dxnoq!0+!exp zOwtMyN81803eyC@1P@js(zbwZf$()XBo#FsB;+=5^r8bAGoh&a3=xy>@_ERW0>K(cnyQNc{Twi|`qI6YxiVqR1!$ae&!8EY5$$u#z1fLe( zjGlkwO?_ZBNZ0{NpN0Vj#DgA-AMdBp6O@HGl3kJrLr+FikmkIHI{-Ik_Ze$Y9M5)o zW3&BPY!Aw&)t%mAyW#C3)kRbw6Qac_t#RgQx#hjucb%^liGG_aa~CW=ifLm#tQ0?-Q`$U76iPS9VT%`ppGHJ2d{bLof~@LX#j| zYg@OzWtOHOMUa9E!3Zl8?e1aTA-7mdiKCNK3kf0|y+103kereW)T+W_+^~|^4D?Py z?+vmkk%*EKnI1z{{LR>l#sI&6@MI85;OJnJ$)p`IYBPGrPccNC-8+yLkn>YJTJt@1 z>%?tex>NEK@FtN67bPE@h&4Z~_u#e?wE+DY(YiW-t2C-Y^ZIY3K>By2N>wC2V3Z45 zZ*Xn(5Z!%a9n00ORW<0!JQn(aFKSOt{CEApzey(=O4JZQT6p}6NT1O!$w92cx}$@m zw0u{-sZSf*8H-J3-a4!|(7#t*H_wzUMwis-*k~3g9FE@D#_4F5@o%?xA137eGm>I zxhK^s=xTyoehqL`gbaqkB|Uj?*$woiA|8qNGx-1M z>`I`iT-)|8I?W=~kw`+jlCsNCh9W~m+8IL9fP@gql%XUtXJ`wdvy(AIM42if964rX zwv#f1geLW0kG}JLXRUAj>tE~NYcK0`c=vvX=edXLzV7Sxe(-dJ-BY2U?(|Ya)tSr} zQ?tp`CGfmJVnNeElK?8XN;(Tfew0mrn34$`{?7Kx03DlRX*S!~ncsE6B_rg~Gn-fDMC-vvE9%v=&>V6`SdFQ8FOWx3emd2Fj(=L>NC z%>`I2z1r;T+2V&2wr$f{mX?-Y+dkKsyOj8Ky{a7<_!iD|jrKhKvU4!z{M)s{%>Fx} z>$9B2)-1VdAlhlIob}^)g}MA<;QnTHeEJg6ODPGvi@_BpaxbJ%j%SLd(8{ix0S9$> zkJ==OjSnq9g&rbyC5*k`2KGAp++sh2NzJH915O`h1+}o2oh`{z!1yV*ODZ{wZbgWw z=DWXmU?&BxSJ$5rnZnDbz#<~5sXqo@$oUW`G9CqM$d&w4YeBq<}AEurki158}gBP4O zwftws8NbQXxatiuWe@GGc16^OY9^R1eeo`!rp_r~0f!^~)Bq{YjID)XJT&8!=scPF z;~Nq=wD}GcmNj%l?a`H~7}7SRP1ej^wBAc2Il5&=W+1h|K=^;OuW*8p1D zREUUR~$kG*O-gwJo&St-Ba@} zwN4~-i+u=J=pS~Q5^DK!AR_AUq{jwUQ_e`m+Qd-Lu#WD+&mIbPX6o}aoO^1oUP#pR zC_X(H?!6~1#$B36igGgNk5s6X5%ne8vUPd9UmdkcqrIxoC305s_mH^L(t3eVn3Of9AtVh-t8LJ`0iCE z{qZseBN9a=sAV8ax2Y>;ry|beHf$9lhBu_yIVchycr36L3Jn z>5sl5vYF>GDe)ax&h%K%im<9Hm{6!+7If=$*4B-5#RH#i@j~~f4;&T3lEHYsyPjpa zxjxm^8bqiJPZsE^p|SBZe2xRwP->DhRtHcEGNhrOLu%EMtBnc4c)H{FPd|~#F4@kf z6ZT@w$Nkoa1&?kJjJuiQZnoOqxVl%{)2lmbP}yVmw1VA5F;;b;Lfa}`8iz_*1W%l6 z)i;T_0J5wLd&a72YL;}`6-+0bdW<0|a7bK;u&W?KqKij#2Tm1mrz(}FF;+_ljT49t z06&%^4-8p^0FVl^Hq$%qN6 z)riWD*k!4@Qk4tN9D~Fl!PgKQn|-`Y5HS-7;jIzf`2JqpVd4rwF7-o?A|ODNiZJ-i zyY7u)L{d~weh_bm`IuFwoeYAlb>nyGZ;IT)g{wpt_Ln{6>5vIXaLbr%+!3NTm^ie3 zms(|t*{h&kijHM%MPm*f{HqhQt8CxwTCv|t;LWqgI#P<$vD|Uj*}g{3>hWsPBx-4Z zqzJ@`J+80dl2VHz6X&wu2-<>jB*Yj_ojf^@Oh8CZNts7k=P^Dn0ZpkO8y^&A2!$5vP*=f*Mwmw$y->#n>r?&zNl@$xFFav;A_j&BiI`UPkDb6vrk@bC1WZo|085 zwB>de%~+GOg)K!KjwicP)~w^>zj#cNDJ^gP1AOt<`iC;N{w2p4itM*AQiFnW)_bvZ zK!$kgzzf%7*jr448y6Sdd6Ca0n5C$yFAR~zkS<9R8>(0QCJw=t?Xt2r8(u20U8dlS zP|L{YUpO`9Ds$30gS{j;`1r8t`dpT@!bHH~qX!rJ+_vFiICXB}ef`kP;}j!0IzP^< zfFTMQ(qIT&j%e z*mvEutCBRu_~BAR7Hf3SyPMeee&s>l!{GFKj0Kap>M zK+pqPy~js%L_QmJ-uS752o4mYB;W@np(t4p-arj*Hg?pA0AWWc@KmE21A`yHth5}x zkuoq~fl`Rt@&`<5-G#x!hRK|&4@396=*+6L(d zjO@}Hl?lubyGEffUaYR!(RKN#w6>1U1`LRXrGgA(IVSD-7k%}{YFAr*`5M)tI&}4q zoyd{Mz*C_`u+|0!>K2D=o&`N(D>z$`@hwh?wtzXk?|NSAypn>^LkP?kRuzEBi8dSl?5Z+vB-f2!dS?;%-;4HvFyX<5h@5gdeway;; zbut-+@pNSN*I7=efJwGM1+~k=v91Z%>q6IAB(J& zNc=;Q2EdS!%|V=~kuFblA#NZX5ZK;Gk4gX8jn><15f1ec*L_0aDKRG?%Up$T{t$t@ab9u z2nIvPRl%EwOA3+T;NrR)j(Wi_jQQ2s)(Cx7Iwyz!dX1HE##A0n*H|!QQR5@Cwn^n5 zz7AoI*4FuWguJ_g>P|N)A~@}xICR`yev>J~;#gC&8K(T9`Uf2b-A8#!bDf{J75^FoM zdS92=b{2Ahq6_U9n5xi*iuJ*wU6#{%p?L()6B&ZTiu1kcnx z)JuSB%WJiWC!FG9dmh`~14OjH{Gs$hK z(Lhfj1_nH7Y+MO0h5%ru-;X72MEOLc<@BBqEdLg3?awnoK|$m0B^bEnsVPv1vKUvz z=9}ild3iiyk!dGvDgv%GD%mQ10#L*PhGS@IDqX0e!_ex#MUAUVp4uF@D7!7=f*^24 zl_-e?J6;wVpU=(9i<)r5Q1#_NL)4IA!oICqwTfuJkqoKq{m2LznKC}O=O-WN^&m@- zaf%SvBLZ?JCGJwzu|#(koNXdl4!{cL3u)zTOd@Le&a3(qzaSKf=GIoEuY6=OCB%Nd zK6mUGbw*PhncxMFun*Q3EE^0*`T5RH`@UV*ZVyCKRghn~f<;LLB#K=nKiJEhZNKzO zxfLCoPsV1<(CSJ$K;U*7Yl+wn3&JIYV0ou05^>+pZz6Rl@?L{^P_KYD zMTw8t1VCK~jEQ10k#s?U{Bi3%da3KcjU(t;pioG%j6OD;zd!0DXi+2Lu?UWc_4+nT znD$Us4F$JVDrM86`GnXD9Z-^4_z3l)SdQICh{Qaq4PJ3C9_=vL9h*{h*q>9^|DxNP zsmrQjg3Ivq=o5H)sOF)42n~S2FD*e-Lois0e3wWd?`2q=?Jnl)nf>6BgL{I4`us}_ zlLRP|A$4&bjCUdWpz&iK3x5GbmGHn63SrCwa{^oS9E7<6_CRfb$aVxW?`d{Ib{Js- z8MZ-k4)jRjwdkty+`VITSm*a+QSU?d5}}8IsfEqQg6c;F#;u$uGr6=M`=RjkeZ+DP zQ9{F4^iH|=ITsOGDc6?!`|Jw%Nekk~k2H|4mcf;g1g<@Y4Ib<{I!kYSq$yTo9u67z zfi-gi{U|sBqQE|_4aY~yDNJRjAK%?DK$1X1gw_D8OHxYeD%4^KWdgYxn&;NRw*W{c z&?gf5-fiArYGVV2&(|yXL$$-?k>di)puZ|!3!$%l0JnkHPwy3PIBzK;W%bh6Ji=j0 zNg&bz`&=^xW&)%&y{%f0p<@WBE1BE0L<2>nWEc>%;SB1#UC(ae_PoPDUP+WMA%%3 zF`#d;2T&nYS|(=7ejdB1{s~HV=#8(!6z|ZNFJGp9*@zh&%aIrpVpu-=7yKF=GcM9j zrjqGT#sKlC7JqpgXWY04gC+2ch=wqz;h}3*o7U5^?jm56a~^pD{bZuFObzrQtrbY} zvWq`z9bHNjegSJH)9zzvRqEK3ILjCz8-GrF$mGam=13`6huFQ*wgG-c6y1rwokaAO^i6L%J3G_ep7ZbP8k@1PFhMWh)EK_( zmyk~AQ9FgP0~s0)BQ(tjo*Vn4qodLFhX<3n>u0+^hdaYz>rfs``#c^KW+3f%HCDO0 zpOc+*ogkFJxi0{>h5Mkv9u&xk!Zmg}xX)MMSYS}e0v+$$NBR0PK8B$+yry@8t;b|+ zBozMj;{hd67CbY`)fzqKqMDfv^q&0SjnZyj#t+<3O(E_MS-*^3CrLc#z^JECbx$-5 zHGjLImWEx}?fr9hw zvY|`TzdXEo-(KuqX3C~i^R=$GI!~voUzk$U;;E3br$*+xO`Zm5+7B;n-(1w^BZMUX z`RIw%yga8C7tqitfQ0*BRt-)*9d!n6NyJto%~ z!l0MX!?a7y;&L*+?JGVRV7TQVJ8+FtR#uC#Z~}Jkfo~=RcS};xPgE) z%nToln^*z*86xmJ?+&yip=Uwy+sFgoi|-cPmdXsX{Xy8K=<_KY&P68WBc#%`QwD=u z$7ZsP&SM`55QOvRk68@f)olJ=R8&NGy_x$Ssf2`vp2GjfU8nC08tp2Q8FC3^5}gQC zBvfI?5NzjpXJllQBW7@z=&wl4-IyCZ#73D?GirJLdd<`q@XH`x1Bz~_doNO?C+4>n zg+&gwb8O#&4n~xiDK;+7r=hrrY9M+mwxoLp1Kb7E;uJCm+|Nz%ay!CEs8!Gu%UCUw_qwg#Oy&+koGw~` z`oVKsbgk)jDHd!1lgdm^{tX;Az8HTI4N?{&O*}*}6D(4AG;|q}h#9@9k&mCU-M*0z zU$D?yX=<9xdUI?3v z9;R{9KlKwTkNFqcF%jK}(eUck+z5mN2Mz!RJ2vTgw$~x@Pt@MHQJiakXv+_0=#QI- z@Enn+{*mndTyYjEVVHy`0^KyBKw;7&3qg-Qz|O%#*Dwfp33}@N81i!e1n^p{Ynnqx zR~l79xPrjZgyv0%eGtex2>~TC2-_a$Uh3Q7v62}<(ldb20y!QYejgfeQFm8wtku># zL%RRue}12Q-QJ5dlXM}Xe+m36KPY-xON(x^M{g%baHAosqa9ZH4kD8tqpCwyqLd zVm52uQ}+}cbErHJk<=CUzb0S@VNeeVdoMGW;0u#+(%ubo{|9hoRPyL%2Hcqg8~1!! zj96;if6w=)@0gqoYY_Y2I%rwL^6)P|2^L=J>5n>1=o9%>Sv{Q>0v#zRF1wk2g#I1FCA8F`E1hdj8sE*yfE!6B116g zF-y_P$${!m;{T!|xgj~wl+mZ2I@1+EodGa{(@ctZ=h{TTJ8dGmhq0Q^02RpDW3o6i zZwsOfebemK7!IOjW0R3Jb31MOf-6c+kTHSoFnTcV8)uS+PRJm(vhCy=kN_gKh+U|b zHsXBoaE^7;C-apK$Wx9LIg^+JQD60{XgI`oX-g#djvE?YY8$@VFzh;g6xPT?Ob zk-v;*+Iw#Kqi1?*st?0oYR&~T4q`k2vRcw1iU5Gah#|RS#}&-n<;UGsTOD?7qH4pc)vI?ot>7A1V8mEo>!>*vUWHbt zeK4B{m=HbG`;+783tBE8y$1p_GSU;XU#Cz%QwK&P&hYX4u~$Wfj0rnY%h=jp>M4iA z5Md!F2$5Jt;E6(?dM$W(AcoSmuYLcCtF^jwa`EOsIHj=54sUEO$Pa_rwsz^#rE06u zB6%%v@6(N}C#<(^Xx~x;sMeZ&93>tIF1j(l{1Y$p-whW0jh^w%r4htRigQC5JL>G8`xzFX9iII& z5Qtbd@L)3#@IYnv5dymlv~9RL%1Dz)h_Kvy*141@?arAOi*f`a9eZlGuF97&q8M+K z4(}0VX-tfy0j#4cN_{clK8Kl$vzcC@B7w{gHu((`F~x2BecjDAFY?K*v#YP;*Nq%%MG4|d(WY|B{-oIJw9lrnSri|)B_bb?4 zY^wC!T{h$+FcbGa_(eB7mipWl9l#&ywJ$5`SC_7Zeb_A4ckXv=YU7o_uM9*7a-`mn zKQPQS-I0Yt7W5>ZJbR`r9fYN8PDP4}AlfEz1GmGJJui8p%x4zW)62~4#espqYTWml zb#EV?#w;{~^a9d_7XwDSkV3DXG46QAElZ)WKhRK7())>3MY}I^jv;|8;Z>=we6P7=g zI>POkZ+StLE>!E-JlNi|D*eTPna<<6WlF8c8&j1yUYyXhfqPu}_-eICk;P9~7ZPg9 zMVRIq5lI4roio&KInM7R{yz12H?Qb)a*KIbY7T}S~CPNcTSbwgN(-ZTn7E;>PBM~X7N zHfpSde}Yxd5^Pre7be*;vnHq6HpS&{e>iunjD+Ixu#OkA9RDsFMi<1duWQyH(pDdl zbIJ*DJ$tycH6*awTr*8={S8;CgZ3V8Xzydx{;v}WsI0Q_5_D5An{*<3#QY2`>oE5@ zp|8oJxpxZkt;;u5=iWNp6r|VR@|NMw+7P#S=)XQwy)+}8ILUurU-k1U=Ra4uWwN#J zjK_}oeL#iAanH-;;Ox&P>+-f(wRDTWxR)lG%B2-7dmrn)c|tAon;z@@$u2&D#2VD( zKgxB#V;PLmP#b?7~Q>uzH z$d&AL{=#qn#(`gMt3p|Q)rECqeLKiDPrP-YQh&nkV?wK*f^%@I`Q4R?Lm~Rbf)_$! z%ew!w;zD&qb9x`vj(s-SWfvKLP~u*Z3M=LuOa@(kS6na(S#g4QSi13k;R7=6U6S2} zp7rl?cPM(}0jHQbeTg}PzYI4Wk@SOI!t0+!ihr0{GuppRnN}sKFX5FNT#bt*FFHe# zfDq(WLlYyGi@c1)fXFFHKIZ)OPn8O^7#ZfR$Q5U!Xo-39zI}hKjo-$}BfVBaV1t{} zDL1#~&I=y8)z*QC=WO!7Pj(fcS5FjoDV$%9(Hgn3Wm?~T){8#oX?@paO{a6EZIMTq zP8IBTblHTOU_si*MY4xO2Y1PMXBQ9)Uq5W?RYmACk9y`+xRulm*jB|72n#Q!`2Xvs aOnbjr=qWOJVqFmhFAdd$Dp|@WeEtXT`-rCi literal 0 HcmV?d00001 From 158a39fa01da5762454f4ecadb0c20ced8b7a37f Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 03:52:06 -0400 Subject: [PATCH 06/13] Blockswap with better memory management --- src/core/generation.py | 52 +++++----------- src/core/model_manager.py | 124 ++++++++++++++++++++++---------------- 2 files changed, 87 insertions(+), 89 deletions(-) diff --git a/src/core/generation.py b/src/core/generation.py index cdfb176..60e9d7f 100644 --- a/src/core/generation.py +++ b/src/core/generation.py @@ -29,7 +29,6 @@ from src.optimization.performance import ( optimized_video_rearrange, optimized_single_video_rearrange, optimized_sample_to_image_format, temporal_latent_blending ) -from src.optimization.blockswap import BlockSwapDebugger from src.common.seed import set_seed try: import comfy.model_management @@ -148,7 +147,7 @@ def generation_step(runner, text_embeds_dict, preserve_vram, cond_latents, tempo cond_latents = cond_latents[0].to("cpu") conditions = conditions[0].to("cpu") condition = condition.to("cpu") - + return samples #, last_latents @@ -311,6 +310,11 @@ def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_si #images = images.to("cpu") #print(f"๐Ÿ”„ Images to CPU time: {time.time() - t} seconds") + # Use existing debugger from runner if available + if hasattr(runner, '_blockswap_debugger') and runner._blockswap_debugger is not None: + debugger = runner._blockswap_debugger + debugger.clear_history() + try: # Main processing loop with context awareness for batch_count, batch_idx in enumerate(range(0, len(images), step)): @@ -429,58 +433,30 @@ def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_si #print(f"๐Ÿ”„ Transformed video to cpu time: {time.time() - tps} seconds") if debug: print(f"๐Ÿ”„ Time batch: {time.time() - tps_loop} seconds") - if preserve_vram: + # Clean VRAM after each batch when preserve_vram is active (but not with blockswap) + if preserve_vram and not (block_swap_config and block_swap_config.get("blocks_to_swap", 0) > 0): torch.cuda.empty_cache() #del transformed_video #clear_vram_cache() + # Log memory state at the end of each batch + if debugger: + debugger.log_memory_state(f"Batch {batch_number} - Memory", show_tensors=True) finally: - # Initializing BlockSwapDebugger logging - enable_debug = block_swap_config.get("enable_debug", False) if block_swap_config else False - debugger = BlockSwapDebugger(enabled=enable_debug) - # Log memory state before cleanup - debugger.log_memory_state( - "Generation complete - Before cleanup", show_tensors=True - ) - debugger.log("๐Ÿงน VRAM cleanup") + debugger.log("๐Ÿงน Generation loop cleanup") - # Move models to CPU + # Final cleanup of embeddings text_pos_embeds = text_pos_embeds.to("cpu") text_neg_embeds = text_neg_embeds.to("cpu") runner.dit.to("cpu") runner.vae.to("cpu") - - # Clear runner cache - if hasattr(runner, "cache") and hasattr(runner.cache, "cache"): - # Move cached tensors to CPU before clearing - for value in runner.cache.cache.values(): - if torch.is_tensor(value) and value.is_cuda: - value.data = value.data.cpu() - elif isinstance(value, (list, tuple)): - for item in value: - if torch.is_tensor(item) and item.is_cuda: - item.data = item.data.cpu() - runner.cache.cache.clear() - - # Find and clear any remaining GPU tensors - for obj in gc.get_objects(): - try: - if torch.is_tensor(obj) and obj.is_cuda: - obj.data = torch.empty(0, device=obj.device) - except: - pass - - # Force cleanup - gc.collect() torch.cuda.empty_cache() - torch.cuda.synchronize() - #del text_pos_embeds, text_neg_embeds #clear_vram_cache() # Log final memory state debugger.log_memory_state( - "After cleanup", show_tensors=True + "Generation loop - After cleanup", show_tensors=True ) # OPTIMISATION ULTIME : Prรฉ-allocation et copie directe (รฉvite les torch.cat multiples) diff --git a/src/core/model_manager.py b/src/core/model_manager.py index 63df1e8..7c973ca 100644 --- a/src/core/model_manager.py +++ b/src/core/model_manager.py @@ -31,9 +31,9 @@ except ImportError: from src.optimization.memory_manager import get_basic_vram_info, clear_vram_cache from src.optimization.compatibility import FP8CompatibleDiT from src.optimization.memory_manager import preinitialize_rope_cache, clear_rope_lru_caches -from src.optimization.blockswap import apply_block_swap_to_dit from src.common.config import load_config, create_object from src.core.infer import VideoDiffusionInfer +from src.optimization.blockswap import apply_block_swap_to_dit # Get script directory for config paths script_directory = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -49,7 +49,7 @@ def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False, bl preserve_vram (bool): Whether to preserve VRAM debug (bool): Enable debug logging block_swap_config (dict): Optional BlockSwap configuration - cached_runner: Optional cached runner to reuse DiT model from + cached_runner: Optional cached runner to reuse entirely (not just DiT) Returns: VideoDiffusionInfer: Configured runner instance ready for inference @@ -60,6 +60,58 @@ def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False, bl - VAE configuration with proper parameter handling - Memory optimization and RoPE cache pre-initialization """ + + # Check if we can fully reuse the cached runner + if cached_runner and block_swap_config and block_swap_config.get("cache_model", False): + # Clear RoPE caches before reuse + if hasattr(cached_runner, 'dit'): + dit_model = cached_runner.dit + if hasattr(dit_model, 'dit_model'): + dit_model = dit_model.dit_model + clear_rope_lru_caches(dit_model) + + print(f"โ™ป๏ธ Reusing cached runner for {model}") + + # Check if blockswap needs to be applied + blockswap_needed = block_swap_config and block_swap_config.get("blocks_to_swap", 0) > 0 + + if blockswap_needed: + # Check if we have cached configuration + has_cached_config = hasattr(cached_runner, "_cached_blockswap_config") + + if has_cached_config: + # Compare configurations + cached_config = cached_runner._cached_blockswap_config + config_matches = ( + cached_config.get("blocks_to_swap") == block_swap_config.get("blocks_to_swap") and + cached_config.get("offload_io_components") == block_swap_config.get("offload_io_components", False) and + cached_config.get("use_non_blocking") == block_swap_config.get("use_non_blocking", True) + ) + + if config_matches: + # Configuration matches - fast re-application + print("โœ… BlockSwap config matches, performing fast re-application") + + # Mark as active before applying + cached_runner._blockswap_active = True + + # Apply BlockSwap (will be fast since model structure is intact) + apply_block_swap_to_dit(cached_runner, block_swap_config) + else: + # Configuration changed - apply new config + print("๐Ÿ”„ BlockSwap configuration changed, applying new config") + apply_block_swap_to_dit(cached_runner, block_swap_config) + else: + # No cached config - apply fresh + print("๐Ÿ”„ Applying BlockSwap to cached runner") + apply_block_swap_to_dit(cached_runner, block_swap_config) + + return cached_runner + else: + # No BlockSwap needed + return cached_runner + + # If we reach here, create a new runner t = time.time() vram_info = get_basic_vram_info() if debug: @@ -80,7 +132,7 @@ def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False, bl print(f"๐Ÿ”„ RUNNER : CONFIG LOAD TIME: {time.time() - t} seconds") # DiT model configuration is now handled directly in the YAML config files # No need for dynamic path resolution here anymore! - + # Load and configure VAE with additional parameters vae_config_path = os.path.join(script_directory, 'src/models/video_vae_v3/s8_c16_t4_inflation_sd3.yaml') t = time.time() @@ -93,11 +145,11 @@ def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False, bl spatial_downsample_factor = vae_config.get('spatial_downsample_factor', 8) temporal_downsample_factor = vae_config.get('temporal_downsample_factor', 4) - vae_config.spatial_downsample_factor = spatial_downsample_factor vae_config.temporal_downsample_factor = temporal_downsample_factor if debug: print(f"๐Ÿ”„ RUNNER : VAE CONFIG SET TIME: {time.time() - t} seconds") + # Merge additional VAE config with main config (preserving __object__ from main config) t = time.time() config.vae.model = OmegaConf.merge(config.vae.model, vae_config) @@ -108,50 +160,21 @@ def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False, bl # Create runner runner = VideoDiffusionInfer(config, debug) OmegaConf.set_readonly(runner.config, False) + # Store model name for cache validation + runner._model_name = model if debug: print(f"๐Ÿ”„ RUNNER : RUNNER VIDEO DIFFUSION INFER TIME: {time.time() - t} seconds") + # Set device device = "cuda" if torch.cuda.is_available() else "cpu" # Configure models checkpoint_path = os.path.join(base_cache_dir, f'./{model}') t = time.time() + runner = configure_dit_model_inference(runner, device, checkpoint_path, config, preserve_vram, model_weight, vram_info, debug, block_swap_config) + if debug: + print(f"๐Ÿ”„ RUNNER : DIT MODEL INFERENCE TIME: {time.time() - t} seconds") - # Check if we can reuse cached DiT model - if cached_runner and hasattr(cached_runner, 'dit') and cached_runner.dit is not None: - # Directly reuse the cached DiT model after cleaning - # Transfer the model to the new runner - runner.dit = cached_runner.dit - - # Clear RoPE caches - clear_rope_lru_caches(runner.dit) - - # Get the base model for cleanup - base_model = runner.dit - if isinstance(base_model, FP8CompatibleDiT): - base_model = base_model.dit_model - - # Clear any accumulated buffers (especially RoPE-related) - for name, buffer in list(base_model.named_buffers()): - if 'cache' in name.lower() or 'freq' in name.lower(): - # Re-initialize these buffers to clear accumulated state - if hasattr(buffer, 'zero_'): - buffer.zero_() - - # Ensure model is in eval mode and gradients are disabled - runner.dit.requires_grad_(False).eval() - - # Clear CUDA cache to ensure clean memory state - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - if debug: - print(f"๐Ÿ”„ RUNNER : DiT MODEL DIRECT REUSE TIME: {time.time() - t} seconds") - else: - # Load DiT model normally - runner = configure_dit_model_inference(runner, device, checkpoint_path, config, preserve_vram, model_weight, vram_info, debug, block_swap_config) - if debug: - print(f"๐Ÿ”„ RUNNER : DIT MODEL INFERENCE TIME: {time.time() - t} seconds") t = time.time() checkpoint_path = os.path.join(base_cache_dir, f'./{config.vae.checkpoint}') runner = configure_vae_model_inference(runner, device, checkpoint_path, config, preserve_vram, model_weight, vram_info, debug, block_swap_config) @@ -163,10 +186,12 @@ def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False, bl runner.vae.set_memory_limit(**runner.config.vae.memory_limit) if debug: print(f"๐Ÿ”„ RUNNER : VAE MEMORY LIMIT TIME: {time.time() - t} seconds") + # Check if BlockSwap is active blockswap_active = ( block_swap_config and block_swap_config.get("blocks_to_swap", 0) > 0 ) + # Pre-initialize RoPE cache for optimal performance if BlockSwap is NOT active if not blockswap_active: t = time.time() @@ -176,7 +201,7 @@ def configure_runner(model, base_cache_dir, preserve_vram=False, debug=False, bl else: if debug: print(f"๐Ÿ”„ RUNNER : Skipping RoPE pre-init due to BlockSwap") - + # Apply BlockSwap if configured if blockswap_active: apply_block_swap_to_dit(runner, block_swap_config) @@ -300,7 +325,7 @@ def configure_dit_model_inference(runner, device, checkpoint, config, preserve_v if 'state' in locals(): del state - + if debug: print(f"๐Ÿ”„ CONFIG DIT : DiT load time: {time.time() - t} seconds") #state.to("cpu") @@ -309,7 +334,9 @@ def configure_dit_model_inference(runner, device, checkpoint, config, preserve_v # Apply universal compatibility wrapper to ALL models # This ensures RoPE compatibility and optimal performance across all architectures t = time.time() - runner.dit = FP8CompatibleDiT(runner.dit) + # Check if already wrapped to avoid double wrapping + if not isinstance(runner.dit, FP8CompatibleDiT): + runner.dit = FP8CompatibleDiT(runner.dit, skip_conversion=False) if debug: print(f"๐Ÿ”„ CONFIG DIT : FP8CompatibleDiT time: {time.time() - t} seconds") @@ -332,15 +359,10 @@ def configure_dit_model_inference(runner, device, checkpoint, config, preserve_v print(f"๐Ÿ”„ CONFIG DIT : Moving model from CPU to {device}") runner.dit.to(device) - # Clear VRAM for both preserve_vram and BlockSwap cases (fixes VRAM leak) - if preserve_vram or blockswap_active: + # Clear VRAM only for preserve_vram (not with BlockSwap which manages its own memory) + if preserve_vram and not blockswap_active: if debug: - reasons = [] - if preserve_vram: - reasons.append("preserve_vram") - if blockswap_active: - reasons.append(f"BlockSwap") - print(f"๐Ÿ”„ CONFIG DIT : Clearing VRAM cache for: {' + '.join(reasons)}") + print(f"? CONFIG DIT : Clearing VRAM cache for preserve_vram") clear_vram_cache() return runner @@ -436,4 +458,4 @@ def configure_vae_model_inference(runner, device, checkpoint_path, config, prese print(f"๐Ÿ”„ CONFIG VAE : VAE SET CAUSAL SLICING TIME: {time.time() - t} seconds") return runner - #runner.vae.to("cpu") + #runner.vae.to("cpu") \ No newline at end of file From 41f5e983ec5de3e1ffa4bba4472eec17f885dfeb Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 03:52:41 -0400 Subject: [PATCH 07/13] Blockswap with better memory management --- src/interfaces/comfyui_node.py | 261 ++++++++++++++++++--------------- 1 file changed, 143 insertions(+), 118 deletions(-) diff --git a/src/interfaces/comfyui_node.py b/src/interfaces/comfyui_node.py index c8f952e..ec9d782 100644 --- a/src/interfaces/comfyui_node.py +++ b/src/interfaces/comfyui_node.py @@ -10,8 +10,14 @@ from typing import Tuple, Dict, Any from src.utils.downloads import download_weight, get_base_cache_dir from src.core.model_manager import configure_runner from src.core.generation import generation_loop -from src.optimization.memory_manager import clear_rope_lru_caches, fast_model_cleanup, fast_ram_cleanup -from src.optimization.blockswap import apply_block_swap_to_dit, BlockSwapDebugger, cleanup_blockswap +from src.optimization.memory_manager import fast_model_cleanup, fast_ram_cleanup +from src.optimization.blockswap import cleanup_blockswap +from src.optimization.memory_manager import ( + clear_rope_lru_caches, + fast_model_cleanup, + fast_ram_cleanup, + clear_all_caches +) # Import ComfyUI progress reporting from server import PromptServer @@ -128,85 +134,102 @@ class SeedVR2: hasattr(self.runner, "_blockswap_active") and self.runner._blockswap_active ) - should_keep_model = is_blockswap_active + # Check if cache_model is enabled in config + cache_model_enabled = block_swap_config and block_swap_config.get("cache_model", False) + should_keep_model = is_blockswap_active and cache_model_enabled - # Initializing BlockSwapDebugger logging - enable_debug = block_swap_config.get("enable_debug", False) if block_swap_config else False - debugger = BlockSwapDebugger(enabled=enable_debug) - cleanup_type = "Partial cleanup - keeping model in RAM" if should_keep_model else "Full cleanup" - debugger.log(f"๐Ÿงน {cleanup_type}") + # Use existing debugger if available + debugger = None + if self.runner and hasattr(self.runner, '_blockswap_debugger'): + debugger = self.runner._blockswap_debugger + debugger.clear_history() - if self.runner: - # Clean up BlockSwap - if hasattr(self.runner, "_blockswap_active"): - cleanup_blockswap(self.runner) + # Perform partial or full cleanup based on model caching + if should_keep_model: + debugger.log("๐Ÿงน Partial cleanup - keeping model in RAM") - # Clear cache - if hasattr(self.runner, 'cache') and hasattr(self.runner.cache, 'cache'): - for key, value in list(self.runner.cache.cache.items()): - if hasattr(value, 'cpu'): - value.cpu() - if hasattr(value, 'detach'): - value.detach() - del value - self.runner.cache.cache.clear() + # Clean BlockSwap with state preservation + if hasattr(self.runner, "_blockswap_active") and self.runner._blockswap_active: + cleanup_blockswap(self.runner, keep_state_for_cache=True) - # Clear DiT model - only if not keeping cached - if not should_keep_model and hasattr(self.runner, 'dit') and self.runner.dit is not None: - clear_rope_lru_caches(self.runner.dit) - fast_model_cleanup(self.runner.dit) - del self.runner.dit - self.runner.dit = None + # Clear all caches + if self.runner: + clear_all_caches(self.runner, debugger) - # Clear VAE model - only clear if not keeping model - if hasattr(self.runner, 'vae') and self.runner.vae is not None: - fast_model_cleanup(self.runner.vae) - del self.runner.vae - self.runner.vae = None + else: + # Full cleanup - existing implementation + debugger.log("๐Ÿงน Full cleanup - clearing everything") - # Clear other components - only if not keeping model - if not should_keep_model: + if self.runner: + # Clean BlockSwap if active + if hasattr(self.runner, "_blockswap_active") and self.runner._blockswap_active: + cleanup_blockswap(self.runner, keep_state_for_cache=False) + + # Clear cache + if hasattr(self.runner, 'cache') and hasattr(self.runner.cache, 'cache'): + for key, value in list(self.runner.cache.cache.items()): + if hasattr(value, 'cpu'): + value.cpu() + if hasattr(value, 'detach'): + value.detach() + del value + self.runner.cache.cache.clear() + + # Clear DiT model + if hasattr(self.runner, 'dit') and self.runner.dit is not None: + # Handle FP8CompatibleDiT wrapper + if hasattr(self.runner.dit, 'dit_model'): + # Clean inner model first + clear_rope_lru_caches(self.runner.dit.dit_model) + fast_model_cleanup(self.runner.dit.dit_model) + # Break reference from wrapper to model + self.runner.dit.dit_model = None + else: + # Direct model cleanup + clear_rope_lru_caches(self.runner.dit) + fast_model_cleanup(self.runner.dit) + + del self.runner.dit + self.runner.dit = None + + # Clear VAE model + if hasattr(self.runner, 'vae') and self.runner.vae is not None: + #from src.optimization.memory_manager import fast_model_cleanup + fast_model_cleanup(self.runner.vae) + del self.runner.vae + self.runner.vae = None + + # Clear other components for component in ['sampler', 'sampling_timesteps', 'schedule', 'config']: if hasattr(self.runner, component): setattr(self.runner, component, None) del self.runner self.runner = None - self.current_model_name = "" - - # Clear embeddings - keep on CPU if caching model + + # Clear embeddings if self.text_pos_embeds is not None: - if should_keep_model: - if hasattr(self.text_pos_embeds, 'is_cuda') and self.text_pos_embeds.is_cuda: - self.text_pos_embeds = self.text_pos_embeds.cpu() - else: - if hasattr(self.text_pos_embeds, 'cpu'): - self.text_pos_embeds.cpu() - del self.text_pos_embeds - self.text_pos_embeds = None + if hasattr(self.text_pos_embeds, 'cpu'): + self.text_pos_embeds.cpu() + del self.text_pos_embeds + self.text_pos_embeds = None if self.text_neg_embeds is not None: - if should_keep_model: - if hasattr(self.text_neg_embeds, 'is_cuda') and self.text_neg_embeds.is_cuda: - self.text_neg_embeds = self.text_neg_embeds.cpu() - else: - if hasattr(self.text_neg_embeds, 'cpu'): - self.text_neg_embeds.cpu() - del self.text_neg_embeds - self.text_neg_embeds = None + if hasattr(self.text_neg_embeds, 'cpu'): + self.text_neg_embeds.cpu() + del self.text_neg_embeds + self.text_neg_embeds = None - # Fast RAM cleanup - only if not keeping model - if force_ram_cleanup and not should_keep_model: + self.current_model_name = "" + + # Fast RAM cleanup + if force_ram_cleanup: fast_ram_cleanup() - # Log memory after cleanup - cleanup_type = "partial" if should_keep_model else "full" - debugger.log_memory_state( - f"After {cleanup_type} cleanup", show_tensors=True - ) - - if should_keep_model: - print("โœ… VRAM cleared, model kept in RAM for next inference") + # BlockSwap debugger memory state + if debugger is not None: + cleanup_stage = "partial" if should_keep_model else "full" + debugger.log_memory_state(f"After {cleanup_stage} cleanup", show_tensors=True) def _internal_execute(self, images, model, seed, new_resolution, cfg_scale, batch_size, preserve_vram, temporal_overlap, debug, block_swap_config): @@ -220,50 +243,39 @@ class SeedVR2: and block_swap_config.get("cache_model", False) ) - # Check if we need to clear cache due to model change - model_changed = self.current_model_name != model + if self.runner is not None: + current_model = getattr(self.runner, '_model_name', None) + model_changed = current_model != model - if model_changed and self.runner is not None: - print( - f"๐Ÿ”„ Model changed from {self.current_model_name} to {model}, clearing cache..." - ) - self.cleanup( - force_ram_cleanup=True, - keep_model_cached=False, - block_swap_config=block_swap_config, - ) - self.runner = None + if model_changed and self.runner is not None: + print( + f"๐Ÿ”„ Model changed from {self.current_model_name} to {model}, clearing cache..." + ) + self.cleanup( + force_ram_cleanup=True, + keep_model_cached=False, + block_swap_config=block_swap_config, + ) + self.runner = None + # Configure runner + if debug: + print("๐Ÿ”„ Configuring inference runner...") runner_start = time.time() - - # Configure runner (always create new runner for clean state) - if use_cache and self.runner is not None and not model_changed: - # Create new runner but reuse cached DiT model - print(f"โ™ป๏ธ Creating new runner with cached DiT model for {model}") - old_runner = self.runner - self.runner = configure_runner( - model, get_base_cache_dir(), preserve_vram, debug, - block_swap_config=block_swap_config, - cached_runner=old_runner - ) - if debug: - print(f"๐Ÿ”„ Runner configuration with cache time: {time.time() - runner_start:.2f}s") - else: - # Create completely new runner - if debug: - print("๐Ÿ”„ Configuring new inference runner...") - self.runner = configure_runner( - model, get_base_cache_dir(), preserve_vram, debug, - block_swap_config=block_swap_config - ) - self.current_model_name = model - if debug: - print(f"๐Ÿ”„ Runner configuration time: {time.time() - runner_start:.2f}s") + self.runner = configure_runner( + model, get_base_cache_dir(), preserve_vram, debug, + block_swap_config=block_swap_config, + cached_runner=self.runner # Pass existing runner if any + ) + + self.current_model_name = model + + if debug: + print(f"๐Ÿ”„ Runner configuration time: {time.time() - runner_start:.2f}s") if debug: print("๐Ÿš€ Starting video upscaling generation...") - # Execute generation with progress callback sample = generation_loop( self.runner, images, cfg_scale, seed, new_resolution, @@ -273,7 +285,7 @@ class SeedVR2: ) - print(f"โœ… Video upscaling completed successfully!") + print(f"โœ… Video upscaling completed successfully!") # Cleanup print(f"๐Ÿ”„ Total execution time: {time.time() - total_start_time:.2f}s") self.cleanup(force_ram_cleanup=True, keep_model_cached=use_cache, block_swap_config=block_swap_config) @@ -309,39 +321,39 @@ class SeedVR2BlockSwap: "blocks_to_swap": ( "INT", { - "default": 25, + "default": 16, "min": 0, "max": 36, "step": 1, - "tooltip": "Number of transformer blocks to swap. 0=disabled, higher=more VRAM savings but slower", + "tooltip": "Number of transformer blocks to swap to CPU. Start with 16 and increase until OOM errors stop. 0=disabled", }, ), "use_non_blocking": ( "BOOLEAN", { "default": True, - "tooltip": "Use non-blocking GPU transfers. Faster but may use more RAM temporarily", + "tooltip": "Use non-blocking GPU transfers for better performance.", }, ), "offload_io_components": ( "BOOLEAN", { - "default": True, - "tooltip": "Offload ALL I/O components (embeddings, patch_embed, etc.) to CPU for maximum VRAM savings", + "default": False, + "tooltip": "Offload embeddings and I/O layers to CPU. Enable if you need additional VRAM savings beyond block swapping", }, ), "cache_model": ( "BOOLEAN", { "default": False, - "tooltip": "Keep model cached in RAM between inferences. Faster subsequent runs but uses more RAM. Only applies when BlockSwap is active", + "tooltip": "Keep model in RAM between runs to avoid model loading time. Useful for batch processing", }, ), "enable_debug": ( "BOOLEAN", { "default": False, - "tooltip": "Enable debug logging for BlockSwap operations", + "tooltip": "Show detailed memory usage and timing information during inference", }, ), } @@ -350,17 +362,30 @@ class SeedVR2BlockSwap: RETURN_TYPES = ("block_swap_config",) FUNCTION = "create_config" CATEGORY = "SEEDVR2" - DESCRIPTION = """Configure block swapping to reduce VRAM usage. + DESCRIPTION = """Configure block swapping to reduce VRAM usage during video upscaling. - Model Info: - - 3B model: 32 transformer blocks - - 7B model: 36 transformer blocks +BlockSwap dynamically moves transformer blocks between GPU and CPU/RAM during inference, enabling large models to run on limited VRAM systems with minimal performance impact. - Guidelines: - - blocks_to_swap=0: No swapping (fastest, highest VRAM) - - blocks_to_swap=16: Balanced (moderate speed/VRAM) - - blocks_to_swap=36: Maximum saving (slowest, lowest VRAM) - - cache_model=True: Keep model in RAM between runs (faster but uses RAM) +Configuration Guidelines: + - blocks_to_swap=0: Disabled (fastest, highest VRAM usage) + - blocks_to_swap=16: Balanced mode (moderate speed/VRAM trade-off) + - blocks_to_swap=32-36: Maximum savings (slowest, lowest VRAM) + +Advanced Options: + - use_non_blocking: Enables asynchronous GPU transfers for better performance + - offload_io_components: Moves embeddings and I/O layers to CPU for additional VRAM savings (slower) + - cache_model: Keeps model in RAM between runs (avoids model loading time on subsequent generations) + - enable_debug: Shows detailed memory usage and timing information + +Performance Tips: + - Start with blocks_to_swap=16 and increase until you no longer get OOM errors or decrease if you have spare VRAM + - Enable offload_io_components if you still need additional VRAM savings + - Note: Even if inference succeeds, you may still OOM during VAE decoding - combine BlockSwap with VAE tiling if needed (feature in development) + - Enable cache_model for batch processing to skip model reloading between runs + - Keep non_blocking=True for better performance (default) + - Combine with smaller batch_size for maximum VRAM savings + +The actual memory savings depend on your specific model architecture and will be shown in the debug output when enabled. """ def create_config( @@ -381,7 +406,7 @@ class SeedVR2BlockSwap: configs.append("I/O components") if cache_model and blocks_to_swap > 0: configs.append("model caching") - print(f"๐Ÿ”„ Offloading configured: {', '.join(configs)}") + print(f"๐Ÿ”„ BlockSwap configured: {', '.join(configs)}") return ( { "blocks_to_swap": blocks_to_swap, From c88c9e61c17f1fed912b3f0d74cf9816f351b3a7 Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 03:53:24 -0400 Subject: [PATCH 08/13] Blockswap with better memory management --- src/optimization/blockswap.py | 1010 ++++++++++++++++------------ src/optimization/memory_manager.py | 157 ++++- 2 files changed, 710 insertions(+), 457 deletions(-) diff --git a/src/optimization/blockswap.py b/src/optimization/blockswap.py index e1aa6b5..e7c6f02 100644 --- a/src/optimization/blockswap.py +++ b/src/optimization/blockswap.py @@ -15,14 +15,33 @@ Key Features: import time import types import torch +import weakref import psutil import gc import comfy.model_management as mm -from typing import Dict, Any, List, Tuple, Callable - +from typing import Dict, Any, List, Tuple, Optional, Union from src.optimization.memory_manager import get_vram_usage + +def get_module_memory_mb(module: torch.nn.Module) -> float: + """ + Calculate memory usage of a module in MB. + + Args: + module: PyTorch module to measure + + Returns: + Memory usage in megabytes + """ + total_bytes = sum( + param.nelement() * param.element_size() + for param in module.parameters() + if param.data is not None + ) + return total_bytes / (1024 * 1024) + + class BlockSwapDebugger: """ Debug logger for BlockSwap operations. @@ -41,48 +60,47 @@ class BlockSwapDebugger: self.enabled = enabled self.swap_times: List[Tuple[int, float, str]] = [] self.vram_history: List[float] = [] - self.device_map: Dict[int, str] = {} def log(self, message: str, level: str = "INFO") -> None: - """ - Log a message if debugging is enabled. - - Args: - message: Message to log - level: Log level (INFO, WARN, SWAP, etc.) - """ + """Log a message if debugging is enabled.""" if self.enabled: print(f"[{level}] {message}") - def log_swap_time(self, block_idx: int, duration: float, direction: str) -> None: + def log_swap_time(self, component_id, duration: float, component_type: str = "block", direction: str = "compute") -> None: """ - Log block swap timing information. + Log swap timing information for any component (blocks or I/O). Args: - block_idx: Index of the block being swapped - duration: Time taken for the swap in seconds - direction: Direction of swap ("compute" for GPU->CPU->GPU cycle) + component_id: Block index (int) or I/O component name (str) + duration: Time taken for the swap operation + component_type: Type of component ("block" or "io") + direction: Direction of swap ("compute" or "offload") """ - self.swap_times.append((block_idx, duration, direction)) + # Store timing data with component info + self.swap_times.append({ + 'component_id': component_id, + 'component_type': component_type, + 'duration': duration, + 'direction': direction + }) + if self.enabled: - self.log( - f"Block {block_idx} swap {direction}: {duration*1000:.1f}ms", "SWAP" - ) + # Format message based on component type + if component_type == "block": + message = f"Block {component_id} swap {direction}: {duration*1000:.1f}ms" + elif component_type == "io": + message = f"I/O {component_id} swap {direction}: {duration*1000:.1f}ms" + else: + message = f"{component_type} {component_id} swap {direction}: {duration*1000:.1f}ms" + + self.log(message, "SWAP") def log_memory_state(self, stage: str, show_tensors: bool = False) -> None: - """ - Log current memory state for debugging. - - Args: - stage: Description of current stage - show_tensors: Whether to count and display tensor count - """ + """Log current memory state for debugging.""" # GPU Memory if torch.cuda.is_available(): allocated_gb, reserved_gb, peak_gb = get_vram_usage() - vram_info = ( - f"VRAM: {allocated_gb:.2f}/{reserved_gb:.2f}GB (peak: {peak_gb:.2f}GB)" - ) + vram_info = f"VRAM: {allocated_gb:.2f}/{reserved_gb:.2f}GB (peak: {peak_gb:.2f}GB)" self.vram_history.append(allocated_gb) else: vram_info = "VRAM: CPU mode" @@ -95,7 +113,6 @@ class BlockSwapDebugger: ram_gb = process.memory_info().rss / (1024**3) ram_info = f" | RAM: {ram_gb:.1f}GB" except Exception: - # psutil might fail in some environments pass # Tensor count (optional - expensive operation) @@ -106,163 +123,24 @@ class BlockSwapDebugger: self.log(f"๐Ÿงฎ {stage}: {vram_info}{ram_info}{tensor_info}") + def clear_history(self) -> None: + """Clear accumulated history.""" + self.swap_times.clear() + self.vram_history.clear() -class BlockSwapper: - """ - Handles dynamic block swapping between GPU and CPU. - - This class wraps a transformer block's forward method to automatically - move the block to GPU for computation and back to CPU afterwards. - """ - - def __init__( - self, - block_idx: int, - original_forward: Callable, - offload_device: str, - use_non_blocking: bool, - debugger: BlockSwapDebugger, - ): - """ - Initialize the block swapper. - - Args: - block_idx: Index of the block being wrapped - original_forward: Original forward method of the block - offload_device: Device to offload to (typically "cpu") - use_non_blocking: Whether to use non-blocking transfers - debugger: BlockSwapDebugger instance for logging - """ - self.block_idx = block_idx - self.original_forward = original_forward - self.offload_device = offload_device - self.use_non_blocking = use_non_blocking - self.debugger = debugger - self._swap_count = 0 - self._total_swap_time = 0.0 - - def __call__(self, block_self, *args, **kwargs): - """ - Execute the block with automatic GPU/CPU swapping. - - Args: - block_self: The block instance (self from the method binding) - *args: Positional arguments for the forward method - **kwargs: Keyword arguments for the forward method - - Returns: - Output from the block's forward method - """ - self._swap_count += 1 - t_start = time.time() if self.debugger.enabled else None - - # Move block to GPU - block_self.to("cuda", non_blocking=self.use_non_blocking) - if self.use_non_blocking and torch.cuda.is_available(): - torch.cuda.synchronize() - - # Execute forward pass - output = self.original_forward(*args, **kwargs) - - # Log timing if enabled - if self.debugger.enabled and t_start is not None: - self.debugger.log_swap_time( - self.block_idx, time.time() - t_start, "compute" - ) - - # Move block back to CPU - block_self.to(self.offload_device, non_blocking=self.use_non_blocking) - if self.use_non_blocking and torch.cuda.is_available(): - torch.cuda.synchronize() - - # Periodic cache cleanup to prevent memory fragmentation - if self.block_idx % 4 == 0: - torch.cuda.empty_cache() - - if t_start: - self._total_swap_time += time.time() - t_start - - return output - - -class IOSwapper: - """ - Handles dynamic I/O component swapping between GPU and CPU. - - Similar to BlockSwapper but designed for I/O components like - embeddings, patch_embed, etc. that need different handling. - """ - - def __init__( - self, - component_name: str, - original_forward: Callable, - offload_device: str, - use_non_blocking: bool, - debugger: BlockSwapDebugger, - ): - """ - Initialize the I/O component swapper. - - Args: - component_name: Name of the component being wrapped - original_forward: Original forward method - offload_device: Device to offload to (typically "cpu") - use_non_blocking: Whether to use non-blocking transfers - debugger: BlockSwapDebugger instance for logging - """ - self.component_name = component_name - self.original_forward = original_forward - self.offload_device = offload_device - self.use_non_blocking = use_non_blocking - self.debugger = debugger - self._swap_count = 0 - - def __call__(self, component_self, *args, **kwargs): - """ - Execute the component with automatic GPU/CPU swapping. - - Args: - component_self: The component instance - *args: Positional arguments for the forward method - **kwargs: Keyword arguments for the forward method - - Returns: - Output from the component's forward method - """ - self._swap_count += 1 - - # Move to GPU - component_self.to("cuda", non_blocking=self.use_non_blocking) - if self.use_non_blocking: - torch.cuda.synchronize() - - # Forward pass - output = self.original_forward(*args, **kwargs) - - # Move back to CPU - component_self.to(self.offload_device, non_blocking=self.use_non_blocking) - if self.use_non_blocking: - torch.cuda.synchronize() - - # Force VRAM cleanup for I/O components - torch.cuda.empty_cache() - - return output def apply_block_swap_to_dit(runner, block_swap_config: Dict[str, Any]) -> None: """ - Apply block swap configuration to a DiT model. + Apply block swapping configuration to a DIT model with OOM protection. This is the main entry point for configuring block swapping on a model. - It handles block selection, I/O component offloading, RoPE patching, - and device placement. + It handles block selection, I/O component offloading, and device placement. Args: runner: VideoDiffusionInfer instance containing the model block_swap_config: Configuration dictionary with keys: - - blocks_to_swap: Number of blocks to swap (from the end) + - blocks_to_swap: Number of blocks to swap (from the start) - offload_io_components: Whether to offload I/O components - use_non_blocking: Whether to use non-blocking transfers - enable_debug: Whether to enable debug logging @@ -271,287 +149,421 @@ def apply_block_swap_to_dit(runner, block_swap_config: Dict[str, Any]) -> None: return blocks_to_swap = block_swap_config.get("blocks_to_swap", 0) - offload_io_components = block_swap_config.get("offload_io_components", False) - - # Early exit if nothing to swap - if blocks_to_swap <= 0 and not offload_io_components: + if blocks_to_swap <= 0: return - # Initialize debugger - debugger = BlockSwapDebugger(enabled=block_swap_config.get("enable_debug", False)) - debugger.log("Starting BlockSwap configuration...") + # Always use fresh debugger for clean state + enable_debug = block_swap_config.get("enable_debug", False) - # Determine devices - device = "cuda" if torch.cuda.is_available() else "cpu" - offload_device = str(mm.unet_offload_device()) - use_non_blocking = block_swap_config.get("use_non_blocking", True) + # Clean up old debugger if exists + if hasattr(runner, '_blockswap_debugger'): + old_debugger = runner._blockswap_debugger + if old_debugger: + old_debugger.clear_history() + delattr(runner, '_blockswap_debugger') + + # Create new debugger + debugger = BlockSwapDebugger(enabled=enable_debug) + runner._blockswap_debugger = debugger # Get the actual model (handle FP8CompatibleDiT wrapper) model = runner.dit if hasattr(model, "dit_model"): model = model.dit_model - # Validate model has blocks - if blocks_to_swap > 0 and not hasattr(model, "blocks"): + # Determine devices + device = "cuda" if torch.cuda.is_available() else "cpu" + offload_device = str(mm.unet_offload_device()) + use_non_blocking = block_swap_config.get("use_non_blocking", True) + + # Validate model structure + if not hasattr(model, "blocks"): debugger.log("Model doesn't have 'blocks' attribute for BlockSwap", "WARN") return - # Get total blocks and validate request - total_blocks = len(model.blocks) if hasattr(model, "blocks") else 0 - if blocks_to_swap > 0: - debugger.log(f"Model has {total_blocks} blocks total") - if blocks_to_swap > total_blocks: - debugger.log( - f"WARNING: Requested {blocks_to_swap} blocks but model only has {total_blocks}", - "WARN", - ) - blocks_to_swap = min(blocks_to_swap, total_blocks) + total_blocks = len(model.blocks) + debugger.log(f"Model has {total_blocks} blocks total") + blocks_to_swap = min(blocks_to_swap, total_blocks) - debugger.log( - f"Configuring: {blocks_to_swap}/{total_blocks} blocks, I/O offload: {offload_io_components}" - ) + # Configure model with blockswap attributes + model.blocks_to_swap = blocks_to_swap - 1 # Convert to 0-indexed + model.main_device = device + model.offload_device = offload_device + model.use_non_blocking = use_non_blocking + + debugger.log(f"Configuring: {blocks_to_swap}/{total_blocks} blocks for swapping") + debugger.log_memory_state("Before BlockSwap", show_tensors=True) - # Handle I/O components offloading - io_swap_count = 0 - io_patches: List[Tuple[Any, Callable]] = [] + # Configure I/O components + offload_io_components = block_swap_config.get("offload_io_components", False) + io_components_offloaded = _configure_io_components(model, device, offload_device, use_non_blocking, + offload_io_components, debugger) + + # Configure block placement and memory tracking + memory_stats = _configure_blocks(model, device, offload_device, use_non_blocking, debugger) + memory_stats['io_components'] = io_components_offloaded + + # Log memory summary + _log_memory_summary(memory_stats, offload_device, device, offload_io_components, + use_non_blocking, debugger) - for name, module in model.named_children(): - if name != "blocks": # Everything except transformer blocks - if offload_io_components: - # Offload to CPU with dynamic swapping - module.to(offload_device) - original_forward = module.forward - swapper = IOSwapper( - name, - original_forward, - offload_device, - use_non_blocking, - debugger, - ) - module.forward = types.MethodType(swapper, module) - module._io_swapper = swapper - io_patches.append((module, original_forward)) - io_swap_count += 1 - debugger.log(f" {name} โ†’ {offload_device} (with dynamic swapping)") - else: - # Normal behavior: keep on GPU - module.to(device) - debugger.log(f" {name} โ†’ {device}") + # Wrap block forward methods for dynamic swapping + for b, block in enumerate(model.blocks): + if b <= model.blocks_to_swap: + _wrap_block_forward(block, b, model, debugger) - # Store I/O patches for cleanup - if io_swap_count > 0: - runner._io_patches = io_patches - debugger.log(f"โœ… Configured {io_swap_count} I/O components for offloading") + # Patch RoPE modules for robust error handling + _patch_rope_for_blockswap(model, debugger) - # Configure block placement - if blocks_to_swap > 0: - blocks_to_keep_on_gpu = total_blocks - blocks_to_swap - - # Move non-swapped blocks to GPU - debugger.log(f"Moving first {blocks_to_keep_on_gpu} blocks to GPU...") - for idx in range(blocks_to_keep_on_gpu): - model.blocks[idx].to(device) - debugger.log(f" Block {idx} โ†’ {device}") - - # Ensure all buffers match device placement - # This prevents device mismatch errors with RoPE and other buffers - for idx in range(blocks_to_keep_on_gpu): - for module in model.blocks[idx].modules(): - if hasattr(module, "_buffers"): - for buffer_name, buffer in module._buffers.items(): - if buffer is not None and buffer.device != torch.device(device): - module._buffers[buffer_name] = buffer.to(device) - - debugger.log(f"โœ… Ensured all buffers match block devices") - - # Patch RoPE modules for robust OOM handling - rope_patches = _patch_rope_for_blockswap(model, debugger) - runner._rope_patches = rope_patches - - # Configure block swapping for the last N blocks - swap_count = 0 - for idx in range(blocks_to_keep_on_gpu, total_blocks): - block = model.blocks[idx] - block._original_forward = block.forward - - # Create and bind swapper - swapper = BlockSwapper( - idx, - block._original_forward, - offload_device, - use_non_blocking, - debugger, - ) - block.forward = types.MethodType(swapper, block) - block._swapper = swapper - swap_count += 1 - - debugger.log(f"โœ… Configured {swap_count} blocks for swapping") - - # Mark BlockSwap as active and store configuration + # Mark BlockSwap as active runner._blockswap_active = True + + # Store configuration for debugging and cleanup runner._block_swap_config = { "blocks_swapped": blocks_to_swap, - "io_components_offloaded": io_swap_count, + "offload_io_components": offload_io_components, "total_blocks": total_blocks, "use_non_blocking": use_non_blocking, "offload_device": offload_device, "main_device": device, + "enable_debug": block_swap_config.get("enable_debug", False), + "offload_memory": memory_stats['offload_memory'], + "main_memory": memory_stats['main_memory'] } # Protect model from being moved entirely - _protect_model_from_move(runner) + _protect_model_from_move(model, runner, debugger) - # Final memory state and cleanup - torch.cuda.empty_cache() debugger.log_memory_state("After BlockSwap", show_tensors=True) debugger.log("โœ… BlockSwap configuration complete") -def _patch_rope_for_blockswap(model, debugger: BlockSwapDebugger) -> List[Tuple[Any, Callable]]: +def _configure_io_components(model, device: str, offload_device: str, + use_non_blocking: bool, offload_io_components: bool, + debugger: BlockSwapDebugger) -> List[str]: + """Configure I/O component placement and wrapping.""" + io_components_offloaded = [] + + # Process non-block parameters + for name, param in model.named_parameters(): + if "block" not in name: + target_device = offload_device if offload_io_components else device + param.data = param.data.to(target_device, non_blocking=use_non_blocking) + status = "(offloaded)" if offload_io_components else "" + debugger.log(f" {name} โ†’ {target_device} {status}") + + # Handle I/O modules with dynamic swapping + for name, module in model.named_children(): + if name != "blocks": + if offload_io_components: + module.to(offload_device) + _wrap_io_forward(module, name, model, debugger) + io_components_offloaded.append(name) + debugger.log(f" {name} โ†’ {offload_device} (with dynamic swapping)") + else: + module.to(device) + debugger.log(f" {name} โ†’ {device}") + + return io_components_offloaded + + +def _configure_blocks(model, device: str, offload_device: str, + use_non_blocking: bool, debugger: BlockSwapDebugger) -> Dict[str, float]: + """Configure block placement and calculate memory statistics.""" + total_offload_memory = 0.0 + total_main_memory = 0.0 + + # Move blocks based on swap configuration + for b, block in enumerate(model.blocks): + block_memory = get_module_memory_mb(block) + + if b > model.blocks_to_swap: + block.to(device) + total_main_memory += block_memory + else: + block.to(offload_device, non_blocking=use_non_blocking) + total_offload_memory += block_memory + + # Ensure all buffers match their containing module's device + for b, block in enumerate(model.blocks): + target_device = device if b > model.blocks_to_swap else offload_device + for name, buffer in block.named_buffers(): + if buffer.device != torch.device(target_device): + buffer.data = buffer.data.to(target_device) + + # Clean up memory + mm.soft_empty_cache() + gc.collect() + + return { + "offload_memory": total_offload_memory, + "main_memory": total_main_memory, + "io_components": [] # Will be populated by caller + } + + +def _log_memory_summary(memory_stats: Dict[str, float], offload_device: str, + device: str, offload_io_components: bool, + use_non_blocking: bool, debugger: BlockSwapDebugger) -> None: + """Log memory usage summary.""" + debugger.log("----------------------") + debugger.log("Block swap memory summary:") + debugger.log(f"Transformer blocks on {offload_device}: {memory_stats['offload_memory']:.2f}MB") + debugger.log(f"Transformer blocks on {device}: {memory_stats['main_memory']:.2f}MB") + total_memory = memory_stats['offload_memory'] + memory_stats['main_memory'] + debugger.log(f"Total memory used by transformer blocks: {total_memory:.2f}MB") + + if offload_io_components and memory_stats.get('io_components'): + debugger.log(f"I/O components offloaded: {', '.join(memory_stats['io_components'])}") + + debugger.log(f"Non-blocking memory transfer: {use_non_blocking}") + debugger.log("----------------------") + + + +def _wrap_block_forward(block: torch.nn.Module, block_idx: int, model: torch.nn.Module, debugger: BlockSwapDebugger) -> None: + """Wrap individual block forward to handle device movement using weak references to prevent leaks.""" + + if hasattr(block, '_original_forward'): + return # Already wrapped + + # Store original forward method + original_forward = block.forward + + # Create weak references + model_ref = weakref.ref(model) + debugger_ref = weakref.ref(debugger) + + # Store block_idx on the block itself to avoid closure issues + block._block_idx = block_idx + + def wrapped_forward(self, *args, **kwargs): + # Retrieve weak references + model = model_ref() + debugger = debugger_ref() + + if not model: + # Model has been garbage collected, fall back to original + return original_forward(*args, **kwargs) + + # Check if block swap is active for this block + if hasattr(model, 'blocks_to_swap') and self._block_idx <= model.blocks_to_swap: + t_start = time.time() if debugger and debugger.enabled else None + + # Only move to GPU if necessary + current_device = next(self.parameters()).device + target_device = torch.device(model.main_device) + + if current_device != target_device: + self.to(model.main_device, non_blocking=model.use_non_blocking) + + # Synchronize if needed + if hasattr(model, 'use_non_blocking') and not model.use_non_blocking: + torch.cuda.synchronize() + + # Execute forward pass with OOM protection + output = original_forward(*args, **kwargs) + + # Move back to offload device + self.to(model.offload_device, non_blocking=model.use_non_blocking) + + # Log timing if debugger is available + if debugger and t_start is not None: + debugger.log_swap_time( + component_id=self._block_idx, + duration=time.time() - t_start, + component_type="block", + direction="compute" + ) + + # Only clear cache under memory pressure + if torch.cuda.memory_allocated() > torch.cuda.get_device_properties(0).total_memory * 0.9: + mm.soft_empty_cache() + else: + output = original_forward(*args, **kwargs) + + return output + + # Bind the wrapped function as a method to the block + block.forward = types.MethodType(wrapped_forward, block) + + # Store reference to original forward for cleanup + block._original_forward = original_forward + + +def _wrap_io_forward(module: torch.nn.Module, module_name: str, model: torch.nn.Module, debugger: BlockSwapDebugger) -> None: + """Wrap I/O component forward using weak references to prevent memory leaks.""" + + if hasattr(module, '_is_io_wrapped') and module._is_io_wrapped: + return # Already wrapped + + # Store original forward method + original_forward = module.forward + + # Create weak references + model_ref = weakref.ref(model) + debugger_ref = weakref.ref(debugger) if debugger else lambda: None + + # Store module name on the module itself + module._module_name = module_name + module._original_forward = original_forward + + def wrapped_io_forward(self, *args, **kwargs): + # Retrieve weak references + model = model_ref() + debugger = debugger_ref() + + if not model: + # Model has been garbage collected, fall back to original + return self._original_forward(*args, **kwargs) + + t_start = time.time() if debugger and debugger.enabled else None + + # Check current device to avoid unnecessary moves + current_device = next(self.parameters()).device + target_device = torch.device(model.main_device) + + # Move to GPU for computation if needed + if current_device != target_device: + self.to(model.main_device) + + # Synchronize if not using non-blocking transfers + if hasattr(model, 'use_non_blocking') and not model.use_non_blocking: + torch.cuda.synchronize() + + # Execute forward pass + output = self._original_forward(*args, **kwargs) + + # Move back to offload device + self.to(model.offload_device, non_blocking=model.use_non_blocking) + + # Log timing if debugger is available + if debugger and t_start is not None: + debugger.log_swap_time( + component_id=self._module_name, + duration=time.time() - t_start, + component_type="io", + direction="compute" + ) + + # Only clear cache under memory pressure + if torch.cuda.memory_allocated() > torch.cuda.get_device_properties(0).total_memory * 0.9: + mm.soft_empty_cache() + + return output + + # Bind as a method + module.forward = types.MethodType(wrapped_io_forward, module) + module._is_io_wrapped = True + + # Store module reference for restoration + if not hasattr(model, '_io_swappers'): + model._io_swappers = [] + model._io_swappers.append((module, module_name)) + + +def _patch_rope_for_blockswap(model, debugger: BlockSwapDebugger) -> None: """ - Patch RoPE modules to handle OOM gracefully during block swapping. + Patch RoPE modules to handle device mismatches gracefully. RoPE (Rotary Position Embeddings) can cause device mismatches when blocks are on different devices. This patches the get_axial_freqs method to handle these cases robustly. - - Args: - model: The model containing RoPE modules - debugger: BlockSwapDebugger instance for logging - - Returns: - List of (module, original_method) tuples for cleanup """ - rope_patches: List[Tuple[Any, Callable]] = [] - rope_patched = 0 - - # Map RoPE modules to their containing blocks - rope_to_block: Dict[Any, int] = {} - if hasattr(model, "blocks"): - for block_idx, block in enumerate(model.blocks): - for name, module in block.named_modules(): - if "rope" in name.lower() and hasattr(module, "get_axial_freqs"): - rope_to_block[module] = block_idx - - # Patch all RoPE modules + rope_patches = [] + for name, module in model.named_modules(): if "rope" in name.lower() and hasattr(module, "get_axial_freqs"): - # Store original methods original_method = module.get_axial_freqs - original_unbound = module.__class__.get_axial_freqs - block_idx = rope_to_block.get(module, -1) - - def make_robust_wrapper( - rope_module, module_name: str, bound_method: Callable, - unbound_method: Callable, parent_block_idx: int - ) -> Callable: - """Create a robust wrapper for RoPE get_axial_freqs method.""" - def robust_method(*args, **kwargs): - try: - # Try the normal bound method first - return bound_method(*args, **kwargs) - except Exception as e: - error_msg = str(e).lower() - # Handle device mismatches and OOM errors - if ( - "device" in error_msg - or "memory" in error_msg - or isinstance(e, (RuntimeError, KeyError)) - ): - debugger.log(f"RoPE issue for {module_name}: {e}") - - # Get current device - current_device = "cuda" - if list(rope_module.parameters()): - current_device = next(rope_module.parameters()).device - - # Try with unbound method (helps with cache context) + + def robust_rope_wrapper(self, *args, **kwargs): + try: + return original_method(*args, **kwargs) + except (RuntimeError, KeyError) as e: + error_msg = str(e).lower() + if "device" in error_msg or "memory" in error_msg or "allocation" in error_msg: + debugger.log(f"RoPE issue for {name}: {e}") + + # Get current device from parameters + current_device = "cuda" + if list(self.parameters()): + current_device = next(self.parameters()).device + + # Try with cleared cache first + if hasattr(original_method, 'cache_clear'): + original_method.cache_clear() try: - return unbound_method(rope_module, *args, **kwargs) - except Exception: - # Final fallback: compute on CPU - debugger.log(f"RoPE fallback to CPU for {module_name}") - rope_module.cpu() - try: - # Clear any LRU cache - if hasattr(bound_method, "cache_clear"): - bound_method.cache_clear() - - result = unbound_method(rope_module, *args, **kwargs) - rope_module.to(current_device) - - # Move result back if it's a tensor - if hasattr(result, "to"): - return result.to(current_device) - return result - except Exception as cpu_error: - # Restore device even on error - rope_module.to(current_device) - raise cpu_error - # Re-raise if not a handled error type + return original_method(*args, **kwargs) + except: + pass + + # Fallback to CPU computation + debugger.log(f"RoPE fallback to CPU for {name}") + self.cpu() + + try: + result = original_method(*args, **kwargs) + + # Move module back to original device + self.to(current_device) + + # Move result to appropriate device if it's a tensor + if hasattr(result, 'to'): + if len(args) > 0 and hasattr(args[0], 'device'): + return result.to(args[0].device) + return result.to(current_device) + return result + except Exception as cpu_error: + # Always restore device even on error + self.to(current_device) + raise cpu_error + else: raise - - return robust_method - - # Create and apply patched method - patched_method = make_robust_wrapper( - module, name, original_method, original_unbound, block_idx - ) - module.get_axial_freqs = patched_method - - # Store for cleanup + + module.get_axial_freqs = types.MethodType(robust_rope_wrapper, module) rope_patches.append((module, original_method)) - rope_patched += 1 - - if rope_patched > 0: - debugger.log( - f"โœ… Patched {rope_patched} RoPE modules with robust device handling" - ) - - return rope_patches + + if rope_patches: + model._rope_patches = rope_patches + debugger.log(f"โœ… Patched {len(rope_patches)} RoPE modules with robust device handling") -def _protect_model_from_move(runner) -> None: +def _protect_model_from_move(model, runner, debugger: BlockSwapDebugger) -> None: """ Protect model from being moved entirely to GPU when BlockSwap is active. This prevents other code from accidentally moving the entire model to GPU which would defeat the purpose of block swapping. - - Args: - runner: VideoDiffusionInfer instance containing the model """ - if hasattr(runner.dit, "dit_model"): - # Handle FP8CompatibleDiT wrapper case - original_dit_to = runner.dit.dit_model.to - runner._original_dit_to = original_dit_to - - def protected_dit_to(device, *args, **kwargs): - """Protected .to() method that blocks GPU moves during BlockSwap.""" - if str(device) != "cpu" and hasattr(runner, "_blockswap_active"): - print("โš ๏ธ Blocked attempt to move blockswapped model to GPU") - return runner.dit.dit_model - return original_dit_to(device, *args, **kwargs) - - runner.dit.dit_model.to = protected_dit_to - else: - # Direct model case - original_dit_to = runner.dit.to - runner._original_dit_to = original_dit_to - - def protected_dit_to(device, *args, **kwargs): - """Protected .to() method that blocks GPU moves during BlockSwap.""" - if str(device) != "cpu" and hasattr(runner, "_blockswap_active"): - print("โš ๏ธ Blocked attempt to move blockswapped model to GPU") - return runner.dit - return original_dit_to(device, *args, **kwargs) - - runner.dit.to = protected_dit_to + if not hasattr(model, '_original_to'): + # Store runner reference as weak reference to avoid circular refs + model._blockswap_runner_ref = weakref.ref(runner) + model._original_to = model.to + + # Define the protected method without closures + def protected_model_to(self, device, *args, **kwargs): + # Check blockswap status using weak reference + if str(device) != "cpu": + runner_ref = getattr(self, '_blockswap_runner_ref', None) + if runner_ref: + runner_obj = runner_ref() + if runner_obj and hasattr(runner_obj, "_blockswap_active") and runner_obj._blockswap_active: + print("[INFO] โš ๏ธ Blocked attempt to move blockswapped model to GPU") + return self + + # Use original method stored as attribute + if hasattr(self, '_original_to'): + return self._original_to(device, *args, **kwargs) + else: + # This shouldn't happen, but fallback to super().to() + return super(type(self), self).to(device, *args, **kwargs) + + # Bind as a method to the model instance + model.to = types.MethodType(protected_model_to, model) -def cleanup_blockswap(runner) -> None: +def cleanup_blockswap(runner, keep_state_for_cache: bool = False) -> None: """ Clean up BlockSwap configurations and restore original methods. @@ -560,52 +572,164 @@ def cleanup_blockswap(runner) -> None: Args: runner: VideoDiffusionInfer instance to clean up + keep_state_for_cache: If True, stores configuration for fast re-application """ - if not hasattr(runner, "_blockswap_active"): + # Early return if BlockSwap not active + if not hasattr(runner, "_blockswap_active") or not runner._blockswap_active: + print("[INFO] โš ๏ธ BlockSwap not active, skipping cleanup") return - # Restore original .to() method - if hasattr(runner, "_original_dit_to"): - if hasattr(runner.dit, "dit_model"): - runner.dit.dit_model.to = runner._original_dit_to - else: - runner.dit.to = runner._original_dit_to - delattr(runner, "_original_dit_to") + # Use existing debugger if available + debugger = getattr(runner, '_blockswap_debugger', None) + if debugger is None: + # Create new debugger only if none exists + debugger = BlockSwapDebugger(enabled=runner._block_swap_config.get("enable_debug", False)) + runner._blockswap_debugger = debugger + else: + debugger.clear_history() + + debugger.log("๐Ÿงน Starting BlockSwap cleanup") - # Restore original RoPE methods and clear their caches - if hasattr(runner, "_rope_patches"): - for module, original_method in runner._rope_patches: - if hasattr(original_method, "cache_clear"): - original_method.cache_clear() - module.get_axial_freqs = original_method - runner._rope_patches.clear() - del runner._rope_patches - - # Restore original I/O component methods - if hasattr(runner, "_io_patches"): - for module, original_forward in runner._io_patches: - module.forward = original_forward - if hasattr(module, "_io_swapper"): - delattr(module, "_io_swapper") - runner._io_patches.clear() - del runner._io_patches - - # Clean up BlockSwap markers - if hasattr(runner, "_blockswap_active"): - delattr(runner, "_blockswap_active") - if hasattr(runner, "_block_swap_config"): - delattr(runner, "_block_swap_config") - - # Clean up swapper references from blocks + # Get the actual model (handle FP8CompatibleDiT wrapper) model = runner.dit if hasattr(model, "dit_model"): model = model.dit_model - - if hasattr(model, "blocks"): - for block in model.blocks: - block.to("cpu") - if hasattr(block, "_swapper"): - delattr(block, "_swapper") - if hasattr(block, "_original_forward"): + + # Store configuration BEFORE cleanup if caching + cached_config = None + if keep_state_for_cache and hasattr(runner, "_block_swap_config"): + cached_config = { + "blocks_to_swap": runner._block_swap_config.get("blocks_swapped"), + "offload_io_components": runner._block_swap_config.get("offload_io_components"), + "use_non_blocking": runner._block_swap_config.get("use_non_blocking"), + "offload_device": runner._block_swap_config.get("offload_device"), + "main_device": runner._block_swap_config.get("main_device"), + "enable_debug": runner._block_swap_config.get("enable_debug", False), + } + runner._cached_blockswap_config = cached_config + debugger.log("๐Ÿ“ฆ Storing configuration for fast re-application") + + # Restore block forward methods + if hasattr(model, 'blocks'): + restored_count = 0 + for idx, block in enumerate(model.blocks): + if hasattr(block, '_original_forward'): block.forward = block._original_forward - delattr(block, "_original_forward") \ No newline at end of file + delattr(block, '_original_forward') + restored_count += 1 + + # Clean up ALL wrapper attributes + attrs_to_clean = ['_block_idx', '_model_ref', '_debugger_ref', '_blockswap_wrapped'] + for attr in attrs_to_clean: + if hasattr(block, attr): + delattr(block, attr) + + # Clear gradients to free memory + block.zero_grad(set_to_none=True) + + # Move block to CPU and ensure all buffers follow + if not keep_state_for_cache: + block.to("cpu") + # Force memory deallocation for all parameters and buffers + for param in block.parameters(): + if param.data.numel() > 0: + param.data.set_() + for buffer in block.buffers(): + if buffer.data.numel() > 0: + buffer.data.set_() + + if restored_count > 0: + debugger.log(f"โœ… Restored original forward for {restored_count} blocks") + + # Restore RoPE methods and clear LRU caches + if hasattr(model, '_rope_patches'): + for module, original_method in model._rope_patches: + # Clear the LRU cache before restoring + if hasattr(module.get_axial_freqs, 'cache_clear'): + module.get_axial_freqs.cache_clear() + module.get_axial_freqs = original_method + debugger.log(f"โœ… Restored {len(model._rope_patches)} RoPE modules") + delattr(model, '_rope_patches') + else: + # Fallback: Clear RoPE caches without restoration + cleared_count = 0 + for module in model.modules(): + if hasattr(module, 'get_axial_freqs') and hasattr(module.get_axial_freqs, 'cache_clear'): + module.get_axial_freqs.cache_clear() + cleared_count += 1 + if cleared_count > 0: + debugger.log(f"โœ… Cleared {cleared_count} RoPE LRU caches") + + # Restore I/O component forward methods + if hasattr(model, '_io_swappers'): + for module, module_name in model._io_swappers: + if hasattr(module, '_is_io_wrapped') and hasattr(module, '_original_forward'): + module.forward = module._original_forward + # Clean up wrapper attributes + attrs_to_clean = ['_original_forward', '_model_ref', '_debugger_ref', + '_module_name', '_is_io_wrapped'] + for attr in attrs_to_clean: + if hasattr(module, attr): + delattr(module, attr) + debugger.log(f"โœ… Restored {len(model._io_swappers)} I/O component wrappers") + delattr(model, '_io_swappers') + + # Restore original .to() method + if hasattr(model, '_original_to'): + model.to = model._original_to + delattr(model, '_original_to') + debugger.log("โœ… Restored original .to() method") + + # Clean up weak reference on model + if hasattr(model, '_blockswap_runner_ref'): + delattr(model, '_blockswap_runner_ref') + + # Clean up BlockSwap attributes from model + attrs_to_remove = ["blocks_to_swap", "main_device", "offload_device", "use_non_blocking"] + for attr in attrs_to_remove: + if hasattr(model, attr): + delattr(model, attr) + + # Mark model as not configured + if hasattr(model, '_blockswap_configured'): + delattr(model, '_blockswap_configured') + + # Move model to CPU to free VRAM (safe now that wrappers are removed) + if not keep_state_for_cache: + model.to("cpu") + debugger.log("๐Ÿ“ฆ Moved model to CPU") + + # Clean up runner attributes + runner._blockswap_active = False + + # Remove all config attributes if not caching + if not cached_config: + if hasattr(runner, "_cached_blockswap_config"): + delattr(runner, "_cached_blockswap_config") + if hasattr(runner, "_block_swap_config"): + delattr(runner, "_block_swap_config") + + # Clear debugger reference (only if not caching) + if not keep_state_for_cache and hasattr(runner, '_blockswap_debugger'): + delattr(runner, '_blockswap_debugger') + + # Clear local debugger reference + debugger = None + + # Force garbage collection (multiple passes for thorough cleanup) + gc.collect(2) # Full collection including oldest generation + gc.collect() + gc.collect() + + # Final memory cleanup + mm.soft_empty_cache() + + + + + + + + + + diff --git a/src/optimization/memory_manager.py b/src/optimization/memory_manager.py index 92e1b39..3c59565 100644 --- a/src/optimization/memory_manager.py +++ b/src/optimization/memory_manager.py @@ -12,7 +12,7 @@ import time from typing import Tuple, Optional from src.common.cache import Cache from src.models.dit_v2.rope import RotaryEmbeddingBase - +from comfy import model_management as mm def get_basic_vram_info(): """๐Ÿ” Mรฉthode basique avec PyTorch natif""" @@ -231,19 +231,7 @@ def fast_model_cleanup(model): if buffer is not None: buffer.data = buffer.data.cpu() - # Deallocate the memory - def deallocate_recursive(m): - for child in m.children(): - deallocate_recursive(child) - for param in m.parameters(): - if param is not None: - param.data = torch.empty(0) - for buffer in m.buffers(): - if buffer is not None: - buffer.data = torch.empty(0) - clear_recursive(model) - deallocate_recursive(model) def fast_ram_cleanup(): @@ -260,4 +248,145 @@ def fast_ram_cleanup(): try: torch._C._clear_cache() except: - pass \ No newline at end of file + pass + + +def clear_all_caches(runner, debugger=None) -> int: + """ + Aggressively clear all caches from runner and model. + Optimized to only process what's necessary. + + Args: + runner: The runner instance to clear caches from + debugger: Optional BlockSwapDebugger instance for logging + """ + if not runner: + return 0 + + # Try to get debugger from runner if not provided + if debugger is None and hasattr(runner, '_blockswap_debugger'): + debugger = runner._blockswap_debugger + + # Helper function for logging + def log_message(message, level="INFO"): + if debugger and debugger.enabled: + debugger.log(message, level) + else: + print(f" {message}") + + cleaned_items = 0 + + # Early exit if no caches to clear + has_cache = hasattr(runner, 'cache') and hasattr(runner.cache, 'cache') + if not has_cache and not hasattr(runner, 'dit'): + return 0 + + # Clear main runner cache efficiently + if has_cache and runner.cache.cache: + cache_entries = len(runner.cache.cache) + + # Process all cache items to properly free memory + for key, value in list(runner.cache.cache.items()): + if torch.is_tensor(value): + # Force deallocation of tensor storage + if value.is_cuda: + value.data = value.data.cpu() + value.grad = None + if value.numel() > 0: + value.data.set_() # Release underlying storage + elif isinstance(value, (list, tuple)): + for item in value: + if torch.is_tensor(item): + if item.is_cuda: + item.data = item.data.cpu() + item.grad = None + if item.numel() > 0: + item.data.set_() + + # Clear the cache after processing + runner.cache.cache.clear() + cleaned_items += cache_entries + log_message(f"โœ… Cleared {cache_entries} cache entries") + + # Clear any accumulated state in blocks + if hasattr(runner, 'dit'): + model = runner.dit + if hasattr(model, 'dit_model'): + model = model.dit_model + + # Clear RoPE LRU caches + rope_caches_cleared = clear_rope_lru_caches(model) + cleaned_items += rope_caches_cleared + if rope_caches_cleared > 0: + log_message(f"โœ… Cleared {rope_caches_cleared} RoPE LRU caches") + + # Clear block attributes if needed + if hasattr(model, 'blocks'): + block_attrs_cleared = 0 + + # Define PyTorch's essential attributes that must NOT be deleted + essential_attrs = { + '_modules', '_parameters', '_buffers', + '_forward_hooks', '_forward_pre_hooks', + '_backward_hooks', '_backward_pre_hooks', + '_state_dict_hooks', '_state_dict_pre_hooks', + '_load_state_dict_pre_hooks', '_load_state_dict_post_hooks', + '_non_persistent_buffers_set', '_version', + '_is_full_backward_hook', 'training', + '_original_forward', # BlockSwap attribute + '_is_io_wrapped', # BlockSwap attribute + '_block_idx', # BlockSwap attribute + } + + for idx, block in enumerate(model.blocks): + # Get all attributes that look like caches + attrs_to_remove = [] + for attr_name in list(block.__dict__.keys()): + # Only remove cache-like attributes, not essential PyTorch attributes + if (attr_name not in essential_attrs and + ('cache' in attr_name or + 'temp' in attr_name or + (attr_name.startswith('_') and + not attr_name.startswith('__') and + attr_name not in essential_attrs))): + attrs_to_remove.append(attr_name) + + # Remove the identified attributes + for attr_name in attrs_to_remove: + try: + delattr(block, attr_name) + block_attrs_cleared += 1 + except AttributeError: + pass # Already deleted or doesn't exist + + if block_attrs_cleared > 0: + log_message(f"โœ… Cleared {block_attrs_cleared} temporary attributes from blocks") + + # Clear any temporary attributes that might accumulate + temp_attrs = ['_temp_cache', '_block_cache', '_swap_cache', '_generation_cache', + '_rope_cache', '_intermediate_cache', '_backward_cache'] + + # Check both runner and model for these attributes + for obj in [runner, getattr(runner, 'dit', None)]: + if obj is None: + continue + + # Handle wrapped models + if hasattr(obj, 'dit_model'): + obj = obj.dit_model + + for attr in temp_attrs: + if hasattr(obj, attr): + delattr(obj, attr) + cleaned_items += 1 + log_message(f"โœ… Cleared {attr} from {type(obj).__name__}") + + # Force garbage collection + gc.collect(2) # Collect all generations + + # Clear CUDA cache + if torch.cuda.is_available(): + torch.cuda.empty_cache() + mm.soft_empty_cache() + + return cleaned_items \ No newline at end of file From 0551e474afe8adffb80df33746889b5b464cc223 Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 09:13:25 -0400 Subject: [PATCH 09/13] FIx debugger log error when not using BlockSwap --- src/interfaces/comfyui_node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/interfaces/comfyui_node.py b/src/interfaces/comfyui_node.py index ec9d782..092b036 100644 --- a/src/interfaces/comfyui_node.py +++ b/src/interfaces/comfyui_node.py @@ -158,7 +158,8 @@ class SeedVR2: else: # Full cleanup - existing implementation - debugger.log("๐Ÿงน Full cleanup - clearing everything") + if debugger: + debugger.log("๐Ÿงน Full cleanup - clearing everything") if self.runner: # Clean BlockSwap if active From a0bf32cb97cb60ed470c1ddd54a69a3e94ae2911 Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 09:29:56 -0400 Subject: [PATCH 10/13] Reverted bf16 conversion logic - don't force it --- src/optimization/compatibility.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/optimization/compatibility.py b/src/optimization/compatibility.py index 9045aae..14699c4 100644 --- a/src/optimization/compatibility.py +++ b/src/optimization/compatibility.py @@ -28,8 +28,24 @@ class FP8CompatibleDiT(torch.nn.Module): # Only convert if not already done (e.g., when reusing cached weights) if not skip_conversion: - # Converting all parameters to BFloat16 to fix dtype compatibility issues - self._force_nadit_bfloat16() + # Detect model type + is_nadit_7b = self._is_nadit_model() # NaDiT 7B (dit/nadit) + is_nadit_v2_3b = self._is_nadit_v2_model() # NaDiT v2 3B (dit_v2/nadit) + + if is_nadit_7b: + # ๐ŸŽฏ CRITICAL FIX: ALL NaDiT 7B models (FP8 AND FP16) require BFloat16 conversion + # 7B architecture has dtype compatibility issues regardless of storage format + if self.is_fp8_model: + print("๐ŸŽฏ Detected NaDiT 7B FP8 - Converting all parameters to BFloat16") + self._force_nadit_bfloat16() + else: + print("๐ŸŽฏ Detected NaDiT 7B FP16") + + + elif self.is_fp8_model and is_nadit_v2_3b: + # For NaDiT v2 3B FP8: Convert ALL model to BFloat16 + print("๐ŸŽฏ Detected NaDiT v2 3B FP8 - Converting all parameters to BFloat16") + self._force_nadit_bfloat16() # ๐Ÿš€ FLASH ATTENTION OPTIMIZATION (Phase 2) self._apply_flash_attention_optimization() From 83f7a5d865eb9fc1e1de859a70f54f198a223009 Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 09:39:17 -0400 Subject: [PATCH 11/13] FIx debugger log error when not using BlockSwap --- src/core/generation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/generation.py b/src/core/generation.py index 60e9d7f..dee0c4d 100644 --- a/src/core/generation.py +++ b/src/core/generation.py @@ -443,7 +443,8 @@ def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_si debugger.log_memory_state(f"Batch {batch_number} - Memory", show_tensors=True) finally: - debugger.log("๐Ÿงน Generation loop cleanup") + if debugger: + debugger.log("๐Ÿงน Generation loop cleanup") # Final cleanup of embeddings text_pos_embeds = text_pos_embeds.to("cpu") @@ -455,9 +456,8 @@ def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_si #clear_vram_cache() # Log final memory state - debugger.log_memory_state( - "Generation loop - After cleanup", show_tensors=True - ) + if debugger: + debugger.log_memory_state("Generation loop - After cleanup", show_tensors=True) # OPTIMISATION ULTIME : Prรฉ-allocation et copie directe (รฉvite les torch.cat multiples) print(f"๐Ÿ’พ Processing {len(batch_samples)} batch_samples with memory-optimized pre-allocation") From 1ce39c51c4131e66e7cc24fd9733617a7753135a Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 09:58:04 -0400 Subject: [PATCH 12/13] Simplified/reverted logic closer to original --- src/core/model_manager.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/core/model_manager.py b/src/core/model_manager.py index 7c973ca..a2181f1 100644 --- a/src/core/model_manager.py +++ b/src/core/model_manager.py @@ -340,30 +340,20 @@ def configure_dit_model_inference(runner, device, checkpoint, config, preserve_v if debug: print(f"๐Ÿ”„ CONFIG DIT : FP8CompatibleDiT time: {time.time() - t} seconds") - # Handle model placement and VRAM clearing based on configuration + # Move DiT to CPU to prevent VRAM leaks (especially for 3B model with complex RoPE) if preserve_vram and not blockswap_active: - # Only move entire model to CPU if preserve_vram is active WITHOUT BlockSwap if debug: - print(f"๐Ÿ”„ CONFIG DIT : Moving model to CPU for preserve_vram") + print(f"๐Ÿ”„ CONFIG DIT : dit to cpu cause preserve_vram: {preserve_vram}") runner.dit = runner.dit.to("cpu") - elif blockswap_active: - # BlockSwap will handle its own GPU/CPU placement per block - if debug: - print( - f"๐Ÿ”„ CONFIG DIT : BlockSwap active ({block_swap_config.get('blocks_to_swap', 0)} blocks) - placement handled by BlockSwap" - ) + if "7b" in model_weight: + clear_vram_cache() else: - # Normal case: move to GPU if loaded on CPU - if state_loading_device == "cpu": - if debug: - print(f"๐Ÿ”„ CONFIG DIT : Moving model from CPU to {device}") + if state_loading_device == "cpu" and not blockswap_active: runner.dit.to(device) - # Clear VRAM only for preserve_vram (not with BlockSwap which manages its own memory) - if preserve_vram and not blockswap_active: - if debug: - print(f"? CONFIG DIT : Clearing VRAM cache for preserve_vram") - clear_vram_cache() + # Log BlockSwap status if active + if blockswap_active and debug: + print(f"๐Ÿ”„ CONFIG DIT : BlockSwap active ({block_swap_config.get('blocks_to_swap', 0)} blocks) - placement handled by BlockSwap") return runner From 3e1e51d847df18cdd7e724dbb4240e2ce010f749 Mon Sep 17 00:00:00 2001 From: Adrien Toupet Date: Wed, 9 Jul 2025 15:31:37 -0400 Subject: [PATCH 13/13] Fix debugger error for non blockswap user --- src/core/generation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/generation.py b/src/core/generation.py index dee0c4d..cddc857 100644 --- a/src/core/generation.py +++ b/src/core/generation.py @@ -311,6 +311,7 @@ def generation_loop(runner, images, cfg_scale=1.0, seed=666, res_w=720, batch_si #print(f"๐Ÿ”„ Images to CPU time: {time.time() - t} seconds") # Use existing debugger from runner if available + debugger = None if hasattr(runner, '_blockswap_debugger') and runner._blockswap_debugger is not None: debugger = runner._blockswap_debugger debugger.clear_history()