changes to LoRAs

- MemoryEfficientSafeOpen now supports low_memory=True which disables mmap
- Uses direct file reads with readinto() to avoid OS page caching
- _deserialize_tensor avoids bytearray copy when input is already bytearray
- merge_loras_to_model now uses low_memory=True for base model

This should reduce peak RAM from 2x model size to 1x + overhead

__init__.py

@@ -1,6 +1,7 @@
 from typing_extensions import override
 from comfy_api.latest import ComfyExtension, io
 
+from .nodes.lora_merger import LoRAMultiMerge, LoRAMultiMergeDARE
 from .nodes.metakeys import (
     ModelMetaKeys, TextEncoderMetaKeys, LoRAMetaKeys,
     CheckpointMetaKeys, EmbeddingMetaKeys
@@ -26,9 +27,7 @@ from .nodes.lora_extract_svd import (
 from .nodes.lora_resize import (
     LoRAResizeFixed, LoRAResizeRatio,
     LoRAResizeFrobenius, LoRAResizeCumulative,
-    LoRAResizeViaBaseFixed, LoRAResizeViaBaseRatio,
-    LoRAResizeViaBaseFrobenius, LoRAResizeViaBaseCumulative,
-    LoRAMultiMerge
+    LoRAMergeToModel
 )
 
 
@@ -57,11 +56,10 @@ class ModelUtilsExtension(ComfyExtension):
             # LoRA Resize
             LoRAResizeFixed, LoRAResizeRatio,
             LoRAResizeFrobenius, LoRAResizeCumulative,
-            # LoRA Fast Resize (via base model)
-            LoRAResizeViaBaseFixed, LoRAResizeViaBaseRatio,
-            LoRAResizeViaBaseFrobenius, LoRAResizeViaBaseCumulative,
             # LoRA Multi-Merge
-            LoRAMultiMerge,
+            LoRAMultiMerge, LoRAMultiMergeDARE,
+            # LoRA Merge To Model
+            LoRAMergeToModel,
         ]
 
 

docs/merger_2_model_modes.md

@@ -23,12 +23,47 @@
 ---
 
 ## Power-Up (DARE)
-> Adds the unique capabilities of Model B to Model A using the Drop and Rescale (DARE) technique, which often preserves the knowledge of the base model better than simple additions.
+> Adds the unique capabilities of Model B to Model A using the Drop and Rescale (DARE) technique. This implementation handles shape mismatches between models by padding and uses a randomized dropout mask.
 
 **Models Used:** A, B
 **Parameters:**
-- **Alpha:** The dropout rate. This is the proportion of unique weights from Model B that are *dropped* before merging. A higher value means less of B is merged.
+- **Alpha:** The dropout rate ($p$). This is the proportion of delta parameters from Model B that are randomly set to zero.
 - **Beta:** A final multiplier for the rescaled difference before it's added to Model A.
+- **Rescaling Logic:** Remaining weights are automatically rescaled by $1/(1-p)$ as per the DARE paper to approximate the original embeddings.
+
+---
+
+## Enhanced Man Interp
+> Sophisticated interpolation between values from A and B depending on their difference relative to other values, with manual threshold control.
+
+**Models Used:** A, B
+**Parameters:**
+- **Alpha:** Interpolation strength.
+- **Beta:** Lower mean threshold for filtering differences.
+- **Gamma:** Upper mean threshold for filtering differences.
+- **Delta:** Smoothness factor (mix between randomized mask and powered differences).
+
+---
+
+## Enhanced Auto Interp
+> Automated version of the enhanced interpolation mode that dynamically calculates thresholds based on mean differences.
+
+**Models Used:** A, B
+**Parameters:**
+- **Alpha:** Interpolation strength.
+- **Beta:** Threshold adjustment factor.
+- **Gamma:** Smoothness factor.
+
+---
+
+## Weight-Sum Cutoff
+> A linear interpolation mode that only applies the merge to weights whose differences fall within a specific threshold range.
+
+**Models Used:** A, B
+**Parameters:**
+- **Alpha:** Interpolation weight (multiplier for the difference).
+- **Beta:** Upper threshold for the difference cutoff.
+- **Gamma:** Lower threshold for the difference cutoff.
 
 ---
 
@@ -84,4 +119,4 @@ Layers matching any pattern will be **removed entirely** from the output.
 | `text_model` | All text encoder layers |
 | `\.norm` | All normalization layers |
 | `attn\.(q\|k\|v)` | Query, key, value attention weights |
-| `block\.[0-9]\.` | Blocks 0-9 |
+| `block\.[0-9]\.` | Blocks 0-9 |

docs/merger_3_model_modes.md

@@ -22,19 +22,19 @@
 ---
 
 ## Extract-Features
-> A powerful mode that identifies features present in both `(B - A)` and `(C - A)` and adds them to A. Allows for fine-grained control over combining aspects based on their similarity.
+> A powerful mode that identifies features present in both `(B - A)` and `(C - A)` and adds them to A. It uses per-vector cosine similarity to decide how much of each feature to keep, allowing for fine-grained control over combining aspects.
 
 **Models Used:** A, B, C
 **Parameters:**
 - **Alpha:** Weights the merge between Model B (`0.0`) and Model C (`1.0`).
 - **Beta:** Controls the focus on similarity (`0.0`) versus dissimilarity (`1.0`).
-- **Gamma:** A bias exponent for similarity. Higher values increase the bias.
+- **Gamma:** A bias exponent for similarity calculation.
 - **Delta:** A final multiplier for the extracted features before they are added to Model A.
 
 ---
 
 ## Add-Dissimilarities
-> Identifies features that are dissimilar between Model B and Model C and adds them to Model A. Useful for combining unique aspects of two different models.
+> Identifies features that are dissimilar between Model B and Model C (relative to A) and adds them to Model A. Useful for combining unique aspects of two different models.
 
 **Models Used:** A, B, C
 **Parameters:**
@@ -80,4 +80,4 @@ Layers matching any pattern will be **removed entirely** from the output.
 **Pattern format:**
 - Whitespace-separated regex patterns
 - Patterns use **substring matching** (not full match)
-- Example: `text_model lora` matches any key containing "text_model" OR "lora"
+- Example: `text_model lora` matches any key containing "text_model" OR "lora"

nodes/lora_extract_svd.py

@@ -423,7 +423,6 @@ def extract_lora_from_files(
     mode: str,
     linear_param: float,
     conv_param: float,
-    prefix: str,
     device: str,
     save_dtype: str,
     linear_max_rank: int = None,
@@ -444,7 +443,6 @@ def extract_lora_from_files(
         mode: Rank selection mode
         linear_param: Mode parameter for linear layers
         conv_param: Mode parameter for conv layers
-        prefix: LoRA key prefix (e.g. "lora_unet_")
         device: Computation device
         save_dtype: Output dtype
         linear_max_rank: Max rank for linear layers
@@ -484,8 +482,7 @@ def extract_lora_from_files(
         stats = {"extracted": 0, "full": 0, "skipped": 0, "chunked": 0}
 
         for key in tqdm(weight_keys, desc="Extracting LoRA", unit="layers"):
-            lora_key = key[:-7]
-            lora_name = prefix + lora_key.replace(".", "_")
+            lora_name = _format_lora_key(key)
 
             if _matches_any_pattern(key, skip_patterns):
                 stats["skipped"] += 1
@@ -567,7 +564,7 @@ def extract_lora_from_files(
                         if lora_up is not None:
                             output_sd[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
                             output_sd[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
-                            output_sd[f"{lora_name}.alpha"] = torch.tensor([rank]).to(save_torch_dtype)
+                            output_sd[f"{lora_name}.alpha"] = torch.tensor(rank).to(save_torch_dtype)
                             stats["chunked"] += 1
                             del weight_diff
                             pbar.update(1)
@@ -587,7 +584,7 @@ def extract_lora_from_files(
                 lora_down, lora_up, _ = result
                 output_sd[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
                 output_sd[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
-                output_sd[f"{lora_name}.alpha"] = torch.tensor([lora_down.shape[0]]).to(save_torch_dtype)
+                output_sd[f"{lora_name}.alpha"] = torch.tensor(lora_down.shape[0]).to(save_torch_dtype)
                 stats["extracted"] += 1
 
             del weight_diff
@@ -632,6 +629,84 @@ def _get_model_inputs():
     ]
 
 
+def _reconstruct_dots(key: str) -> str:
+    """Reconstruct dot structure from underscored key using heuristics."""
+    # Block indices: input_blocks_1 -> input_blocks.1
+    blocks = [
+        "input_blocks", "output_blocks", "middle_block",
+        "transformer_blocks", "single_transformer_blocks",
+        "double_blocks", "single_blocks",
+        "down_blocks", "up_blocks", "mid_block",
+        "attentions", "resnets", "upsamplers", "downsamplers"
+    ]
+    for b in blocks:
+        key = re.sub(f"{b}_(\\d+)", f"{b}.\\1", key)
+
+    # Sequential numbers: .1_1 -> .1.1
+    key = re.sub(r"(\.\d+)_(\d+)", r"\1.\2", key)
+
+    # Separators that should be preceded by dot
+    separators = [
+        "transformer_blocks", "single_transformer_blocks",
+        "attn1", "attn2", "attn",
+        "img_attn", "txt_attn",
+        "to_q", "to_k", "to_v", "to_out",
+        "q_proj", "k_proj", "v_proj", "out_proj",
+        "qkv", "proj", "ff", "net",
+        "norm1", "norm2", "norm3", "norm",
+        "time_emb_proj"
+    ]
+    for s in separators:
+        key = key.replace(f"_{s}", f".{s}")
+
+    # Special case: to_out_0 -> to_out.0
+    key = re.sub(r"to_out_(\d+)", r"to_out.\1", key)
+
+    return key
+
+
+def _format_lora_key(key: str) -> str:
+    """
+    Format the key for LoRA saving.
+    Standardizes to 'diffusion_model.' or 'transformer.' prefix.
+    """
+    if key.endswith(".weight"):
+        key = key[:-7]
+
+    # Handle ComfyUI Checkpoint wrapper
+    if key.startswith("model.diffusion_model."):
+        # Check if it's a Diffusers-format transformer inside a checkpoint
+        inner_key = key[22:]  # len("model.diffusion_model.")
+        if inner_key.startswith("transformer_blocks") or inner_key.startswith("single_transformer_blocks"):
+            return f"transformer.{inner_key}"
+        # Standard checkpoint or Original Flux
+        return f"diffusion_model.{inner_key}"
+
+    # Handle direct Diffusers keys
+    if key.startswith("transformer_blocks") or key.startswith("single_transformer_blocks"):
+        return f"transformer.{key}"
+
+    # Handle legacy lora_unet_ prefix (for resizing without base)
+    if key.startswith("lora_unet_"):
+        core = key[10:]
+        dotted = _reconstruct_dots(core)
+        # Check if reconstructed key implies transformer format
+        if dotted.startswith("transformer_blocks") or dotted.startswith("single_transformer_blocks"):
+            return f"transformer.{dotted}"
+        return f"diffusion_model.{dotted}"
+
+    # Handle already correct prefixes
+    if key.startswith("diffusion_model.") or key.startswith("transformer."):
+        return key
+    
+    # Handle known Unet blocks (Diffusers format UNet)
+    if any(key.startswith(p) for p in ["down_blocks", "up_blocks", "mid_block", "conv_in", "conv_out", "time_embedding", "class_embedding"]):
+        return f"diffusion_model.{key}"
+
+    # Default fallback to diffusion_model
+    return f"diffusion_model.{key}"
+
+
 def _get_common_inputs():
     return [
         io.Boolean.Input("chunk_large_layers", default=False,
@@ -683,7 +758,7 @@ class LoRAExtractFixed(io.ComfyNode):
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "fixed", linear_dim, conv_dim,
-            "lora_unet_", device, save_dtype, linear_dim, conv_dim,
+            device, save_dtype, linear_dim, conv_dim,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers, svd_niter
         )
 
@@ -706,9 +781,9 @@ class LoRAExtractRatio(io.ComfyNode):
                               tooltip="Ratio threshold for linear layers (higher = more SVs kept)"),
                 io.Float.Input("conv_ratio", default=2.0, min=1.0, max=100.0, step=0.1,
                               tooltip="Ratio threshold for conv layers (higher = more SVs kept)"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("linear_max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("conv_max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
@@ -726,7 +801,7 @@ class LoRAExtractRatio(io.ComfyNode):
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "ratio", linear_ratio, conv_ratio,
-            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
+            device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
         )
 
@@ -749,9 +824,9 @@ class LoRAExtractQuantile(io.ComfyNode):
                               tooltip="Target cumulative % for linear layers"),
                 io.Float.Input("conv_quantile", default=0.9, min=0.0, max=1.0, step=0.01,
                               tooltip="Target cumulative % for conv layers"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("linear_max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("conv_max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
@@ -769,7 +844,7 @@ class LoRAExtractQuantile(io.ComfyNode):
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "quantile", linear_quantile, conv_quantile,
-            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
+            device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
         )
 
@@ -790,9 +865,9 @@ class LoRAExtractKnee(io.ComfyNode):
                 *_get_model_inputs(),
                 io.Combo.Input("knee_method", options=["sv_knee", "sv_cumulative_knee"],
                               default="sv_knee", tooltip="Knee detection method"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("linear_max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("conv_max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
@@ -810,7 +885,7 @@ class LoRAExtractKnee(io.ComfyNode):
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, knee_method, 0, 0,
-            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
+            device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
         )
 
@@ -833,9 +908,9 @@ class LoRAExtractFrobenius(io.ComfyNode):
                               tooltip="Target Frobenius norm fraction for linear"),
                 io.Float.Input("conv_target", default=0.9, min=0.0, max=1.0, step=0.01,
                               tooltip="Target Frobenius norm fraction for conv"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("linear_max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("conv_max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
@@ -853,7 +928,7 @@ class LoRAExtractFrobenius(io.ComfyNode):
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "sv_fro", linear_target, conv_target,
-            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
+            device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
         )
 

nodes/lora_merger.py

@@ -0,0 +1,549 @@
+"""
+LoRA Multi-Merge - Merge multiple LoRAs into a single LoRA file.
+Resolves different naming conventions and ranks via concatenation.
+"""
+import os
+import torch
+import folder_paths
+import comfy.utils
+from tqdm import tqdm
+from comfy_api.latest import io
+from safetensors.torch import save_file
+from typing import List, Dict, Tuple, Optional
+from .merger_utils import MemoryEfficientSafeOpen, transfer_to_gpu_pinned
+from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation
+from .lora_resize import (
+    detect_lora_format, 
+    extract_lora_pairs, 
+    detect_lora_rank,
+    _compile_patterns,
+    _matches_any_pattern,
+    _format_lora_key
+)
+
+def merge_multi_loras(
+    lora_paths: List[str],
+    lora_weights: List[float],
+    merge_mode: str,
+    device: str,
+    save_dtype: torch.dtype,
+    output_filename: str,
+    verbose: bool = True,
+) -> str:
+    """
+    Merge multiple LoRAs into a single LoRA file.
+    
+    Modes:
+    - concatenate: Mathematically sound merge by stacking ranks.
+                   New rank = sum(input ranks). Alpha set to new rank.
+    - weighted_sum: Weighted sum of A and B weights (Kohya style).
+                    If ranks differ, smaller ones are zero-padded to match the largest rank.
+                    New rank = max(input ranks).
+    """
+    # Estimate memory
+    total_size_gb = sum(estimate_model_size(p) for p in lora_paths)
+    if verbose:
+        print(f"[LoRA Multi-Merge] Preparing memory for {total_size_gb:.2f}GB operation...")
+        print(f"[LoRA Multi-Merge] Mode: {merge_mode}")
+    
+    prepare_for_large_operation(total_size_gb * 2.5, torch.device(device))
+
+    handlers = [MemoryEfficientSafeOpen(p) for p in lora_paths]
+    
+    try:
+        # 1. Analyze all LoRAs
+        layer_map = {}
+        lora_infos = []
+        
+        for i, handler in enumerate(handlers):
+            keys = handler.keys()
+            fmt = detect_lora_format(keys)
+            pairs = extract_lora_pairs(keys, fmt)
+            rank, alpha = detect_lora_rank(handler, pairs)
+            
+            info = {
+                "name": os.path.basename(lora_paths[i]),
+                "format": fmt,
+                "pairs": pairs,
+                "rank": rank,
+                "alpha": alpha,
+                "weight": lora_weights[i],
+                "scale": alpha / rank if rank > 0 else 1.0
+            }
+            lora_infos.append(info)
+            
+            if verbose:
+                print(f"[LoRA Multi-Merge] LoRA {i+1} ({info['name']}): {len(pairs)} layers, dim={rank}, format={fmt['format']}")
+            
+            for block_name in pairs.keys():
+                core = _format_lora_key(block_name)
+                if core not in layer_map:
+                    layer_map[core] = []
+                layer_map[core].append(i)
+
+        if verbose:
+            print(f"[LoRA Multi-Merge] Total unique layers after resolving naming: {len(layer_map)}")
+
+        output_sd = {}
+        pbar = comfy.utils.ProgressBar(len(layer_map))
+
+        # 2. Merge each core layer
+        with torch.no_grad():
+            for core, indices in tqdm(layer_map.items(), desc="Merging layers", unit="layers"):
+                downs = []
+                ups = []
+                is_conv = False
+                max_rank = 0
+                
+                # First pass: load and determine max rank
+                for idx in indices:
+                    info = lora_infos[idx]
+                    handler = handlers[idx]
+                    orig_block = next(b for b in info["pairs"].keys() if _format_lora_key(b) == core)
+                    block_keys = info["pairs"][orig_block]
+                    
+                    if "down" not in block_keys or "up" not in block_keys:
+                        continue
+                        
+                    t_down = handler.get_tensor(block_keys["down"])
+                    t_up = handler.get_tensor(block_keys["up"])
+                    
+                    if device == 'cuda':
+                        t_down = transfer_to_gpu_pinned(t_down, device, torch.float32)
+                        t_up = transfer_to_gpu_pinned(t_up, device, torch.float32)
+                    else:
+                        t_down = t_down.to(device=device, dtype=torch.float32)
+                        t_up = t_up.to(device=device, dtype=torch.float32)
+                    
+                    # Store with original info for padding/weighting
+                    is_conv = len(t_down.shape) == 4
+                    current_rank = t_down.shape[0]
+                    max_rank = max(max_rank, current_rank)
+                    
+                    # Apply scale immediately for concatenate mode, or keep for later
+                    if merge_mode == "concatenate":
+                        effective_weight = info["weight"] * info["scale"]
+                        t_up = t_up * effective_weight
+                    
+                    downs.append((t_down, info))
+                    ups.append((t_up, info))
+
+                if not downs:
+                    pbar.update(1)
+                    continue
+
+                if merge_mode == "concatenate":
+                    # Stack ranks
+                    merged_down = torch.cat([d[0] for d in downs], dim=0)
+                    merged_up = torch.cat([u[0] for u in ups], dim=1)
+                    new_rank = merged_down.shape[0]
+                    new_alpha = float(new_rank)
+                else:
+                    # weighted_sum (Kohya style)
+                    # Result = sum( weight_i * Padded(B_i) ), sum( weight_i * Padded(A_i) )
+                    # Note: We apply scale to weights here to normalize different alphas
+                    merged_down = torch.zeros_like(downs[0][0])
+                    # Need to handle different ranks via padding
+                    target_down_shape = list(downs[0][0].shape)
+                    target_down_shape[0] = max_rank
+                    target_up_shape = list(ups[0][0].shape)
+                    target_up_shape[1] = max_rank
+                    
+                    merged_down = torch.zeros(target_down_shape, device=device, dtype=torch.float32)
+                    merged_up = torch.zeros(target_up_shape, device=device, dtype=torch.float32)
+                    
+                    for (t_d, info_d), (t_u, info_u) in zip(downs, ups):
+                        r = t_d.shape[0]
+                        w = info_d["weight"]
+                        # We also apply sqrt(scale) to both to distribute the scale factor?
+                        # Kohya just uses weights. But to be safe with different alphas,
+                        # we apply scale to the final delta.
+                        # For direct weight merge, we'll just use weights.
+                        
+                        if r < max_rank:
+                            # Pad down: [r, in...] -> [max_rank, in...]
+                            pad_d = [0] * (len(t_d.shape) * 2)
+                            pad_d[-1] = max_rank - r # last dim in pad is first dim in tensor (reversed)
+                            # Wait, F.pad uses reverse order of dims.
+                            # For [r, in], padding is (0,0, 0, max_rank-r)
+                            padding_d = [0, 0] * (len(t_d.shape) - 1) + [0, max_rank - r]
+                            t_d = torch.nn.functional.pad(t_d, tuple(padding_d))
+                            
+                            # Pad up: [out, r...] -> [out, max_rank...]
+                            # For [out, r], padding is (0, max_rank-r, 0, 0)
+                            padding_u = [0, max_rank - r] + [0, 0] * (len(t_u.shape) - 1)
+                            t_u = torch.nn.functional.pad(t_u, tuple(padding_u))
+                        
+                        merged_down += w * t_d
+                        merged_up += w * t_u
+                    
+                    new_rank = max_rank
+                    # Alpha is usually max of alphas or same as rank
+                    new_alpha = float(max_rank)
+
+                # Store
+                output_sd[f"{core}.lora_down.weight"] = merged_down.to(save_dtype).cpu().contiguous()
+                output_sd[f"{core}.lora_up.weight"] = merged_up.to(save_dtype).cpu().contiguous()
+                output_sd[f"{core}.alpha"] = torch.tensor(new_alpha, dtype=save_dtype)
+                
+                # Cleanup
+                del merged_down, merged_up
+                for d, _ in downs: del d
+                for u, _ in ups: del u
+                
+                pbar.update(1)
+
+        # 3. Final Summary
+        if verbose:
+            matched_stats = {i: 0 for i in range(len(lora_paths))}
+            for indices in layer_map.values():
+                for idx in indices:
+                    matched_stats[idx] += 1
+            
+            print(f"[LoRA Multi-Merge] --- Merge Summary ---")
+            for i, info in enumerate(lora_infos):
+                print(f"[LoRA Multi-Merge] LoRA {i+1}: {matched_stats[i]}/{len(info['pairs'])} layers used in merge")
+            print(f"[LoRA Multi-Merge] Output state dict has {len(output_sd)} tensors")
+
+        # 4. Metadata
+        metadata = {
+            "ss_training_comment": f"Merged {len(lora_paths)} LoRAs via concatenation",
+            "ss_network_module": "networks.lora",
+        }
+        
+        # 4. Save
+        output_dir = folder_paths.get_folder_paths("loras")[0]
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors")
+        
+        save_file(output_sd, output_path, metadata)
+        if verbose:
+            print(f"[LoRA Multi-Merge] Saved merged LoRA to {output_path}")
+            
+        return output_path
+
+    finally:
+        for h in handlers:
+            h.__exit__(None, None, None)
+        cleanup_after_operation()
+
+
+class LoRAMultiMerge(io.ComfyNode):
+    """Merge multiple LoRAs into a single LoRA file."""
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="LoRAMultiMerge",
+            display_name="LoRA Multi-Merge",
+            category="ModelUtils/LoRA/Merge",
+            description="Merge 1-8 LoRAs into a single LoRA file. This resolves different ranks and naming conventions.",
+            inputs=[
+                io.Combo.Input("lora_count", options=[str(i) for i in range(1, 9)], default="2",
+                              tooltip="Number of LoRAs to merge"),
+                # LoRA 1
+                io.Combo.Input("lora_1", options=folder_paths.get_filename_list("loras"), tooltip="First LoRA"),
+                io.Float.Input("weight_1", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 2
+                io.Combo.Input("lora_2", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_2", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 3
+                io.Combo.Input("lora_3", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_3", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 4
+                io.Combo.Input("lora_4", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_4", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 5
+                io.Combo.Input("lora_5", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_5", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 6
+                io.Combo.Input("lora_6", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_6", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 7
+                io.Combo.Input("lora_7", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_7", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 8
+                io.Combo.Input("lora_8", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_8", default=1.0, min=-10.0, max=10.0, step=0.01),
+                
+                io.Combo.Input("merge_mode", options=["concatenate", "weighted_sum"], default="concatenate",
+                              tooltip="concatenate: safe, increases rank. weighted_sum: fixed rank (to max input rank), mathematically lossy but standard in Kohya."),
+                io.String.Input("output_filename", default="merged_lora"),
+                io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
+                io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
+            ],
+            outputs=[io.String.Output(display_name="output_path")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, lora_count,
+                lora_1, weight_1, lora_2, weight_2, lora_3, weight_3, lora_4, weight_4,
+                lora_5, weight_5, lora_6, weight_6, lora_7, weight_7, lora_8, weight_8,
+                merge_mode, output_filename, save_dtype, device) -> io.NodeOutput:
+        
+        count = int(lora_count)
+        names = [lora_1, lora_2, lora_3, lora_4, lora_5, lora_6, lora_7, lora_8]
+        weights = [weight_1, weight_2, weight_3, weight_4, weight_5, weight_6, weight_7, weight_8]
+        
+        valid_paths = []
+        valid_weights = []
+        
+        for i in range(count):
+            if names[i] and names[i] != "None":
+                path = folder_paths.get_full_path_or_raise("loras", names[i])
+                valid_paths.append(path)
+                valid_weights.append(weights[i])
+                
+        if not valid_paths:
+            raise ValueError("No LoRAs selected for merging.")
+            
+        dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
+        
+        path = merge_multi_loras(
+            lora_paths=valid_paths,
+            lora_weights=valid_weights,
+            merge_mode=merge_mode,
+            device=device,
+            save_dtype=dtype,
+            output_filename=output_filename
+        )
+        
+        return io.NodeOutput(path)
+
+
+class LoRAMultiMergeDARE(io.ComfyNode):
+    """Merge multiple LoRAs into a single LoRA file using DARE-Ties method."""
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="LoRAMultiMergeDARE",
+            display_name="LoRA Multi-Merge (DARE-Ties)",
+            category="ModelUtils/LoRA/Merge",
+            description="Merge 1-8 LoRAs into a single LoRA file using DARE-Ties. Drops small values and resolves sign conflicts.",
+            inputs=[
+                io.Combo.Input("lora_count", options=[str(i) for i in range(1, 9)], default="2"),
+                # LoRA 1
+                io.Combo.Input("lora_1", options=folder_paths.get_filename_list("loras"), tooltip="First LoRA"),
+                io.Float.Input("weight_1", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # ...
+                io.Combo.Input("lora_2", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_2", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_3", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_3", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_4", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_4", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_5", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_5", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_6", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_6", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_7", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_7", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_8", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_8", default=1.0, min=-10.0, max=10.0, step=0.01),
+
+                io.Float.Input("drop_rate", default=0.1, min=0.0, max=1.0, step=0.01, tooltip="DARE drop rate"),
+                io.Float.Input("trim_quantile", default=0.2, min=0.0, max=1.0, step=0.01, tooltip="TIES trim quantile (drops smallest values)"),
+                io.Int.Input("seed", default=42, min=0, max=0xffffffffffffffff),
+                io.String.Input("output_filename", default="merged_lora_dare"),
+                io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
+                io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
+            ],
+            outputs=[io.String.Output(display_name="output_path")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, lora_count,
+                lora_1, weight_1, lora_2, weight_2, lora_3, weight_3, lora_4, weight_4,
+                lora_5, weight_5, lora_6, weight_6, lora_7, weight_7, lora_8, weight_8,
+                drop_rate, trim_quantile, seed, output_filename, save_dtype, device) -> io.NodeOutput:
+        
+        count = int(lora_count)
+        names = [lora_1, lora_2, lora_3, lora_4, lora_5, lora_6, lora_7, lora_8]
+        weights = [weight_1, weight_2, weight_3, weight_4, weight_5, weight_6, weight_7, weight_8]
+        
+        valid_paths = []
+        valid_weights = []
+        
+        for i in range(count):
+            if names[i] and names[i] != "None":
+                path = folder_paths.get_full_path_or_raise("loras", names[i])
+                valid_paths.append(path)
+                valid_weights.append(weights[i])
+                
+        if not valid_paths:
+            raise ValueError("No LoRAs selected for merging.")
+            
+        dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
+        
+        path = merge_multi_loras_dare(
+            lora_paths=valid_paths,
+            lora_weights=valid_weights,
+            drop_rate=drop_rate,
+            trim_quantile=trim_quantile,
+            seed=seed,
+            device=device,
+            save_dtype=dtype,
+            output_filename=output_filename
+        )
+        
+        return io.NodeOutput(path)
+
+def merge_multi_loras_dare(
+    lora_paths: List[str],
+    lora_weights: List[float],
+    drop_rate: float,
+    trim_quantile: float,
+    seed: int,
+    device: str,
+    save_dtype: torch.dtype,
+    output_filename: str,
+    verbose: bool = True,
+) -> str:
+    """
+    Merge multiple LoRAs using DARE-Ties method applied to weights.
+    Different ranks are handled via zero-padding to the max rank.
+    """
+    total_size_gb = sum(estimate_model_size(p) for p in lora_paths)
+    if verbose:
+        print(f"[LoRA Multi-Merge DARE] Preparing memory for {total_size_gb:.2f}GB operation...")
+        print(f"[LoRA Multi-Merge DARE] Drop rate: {drop_rate}, Trim quantile: {trim_quantile}")
+
+    prepare_for_large_operation(total_size_gb * 2.5, torch.device(device))
+    handlers = [MemoryEfficientSafeOpen(p) for p in lora_paths]
+    
+    rng = torch.Generator(device=device).manual_seed(seed)
+
+    try:
+        # 1. Analyze
+        layer_map = {}
+        lora_infos = []
+        for i, handler in enumerate(handlers):
+            keys = handler.keys()
+            fmt = detect_lora_format(keys)
+            pairs = extract_lora_pairs(keys, fmt)
+            rank, alpha = detect_lora_rank(handler, pairs)
+            info = {
+                "name": os.path.basename(lora_paths[i]),
+                "pairs": pairs,
+                "rank": rank,
+                "alpha": alpha,
+                "weight": lora_weights[i]
+            }
+            lora_infos.append(info)
+            
+            if verbose:
+                print(f"[LoRA Multi-Merge DARE] LoRA {i+1} ({info['name']}): {len(pairs)} layers, dim={rank}")
+
+            for block_name in pairs.keys():
+                core = _format_lora_key(block_name)
+                if core not in layer_map: layer_map[core] = []
+                layer_map[core].append(i)
+
+        if verbose:
+            print(f"[LoRA Multi-Merge DARE] Total unique layers after resolving naming: {len(layer_map)}")
+
+        output_sd = {}
+        pbar = comfy.utils.ProgressBar(len(layer_map))
+
+        # 2. Merge
+        with torch.no_grad():
+            for core, indices in tqdm(layer_map.items(), desc="Merging layers (DARE)", unit="layers"):
+                downs = []
+                ups = []
+                max_rank = 0
+                
+                for idx in indices:
+                    info = lora_infos[idx]
+                    orig_block = next(b for b in info["pairs"].keys() if _format_lora_key(b) == core)
+                    block_keys = info["pairs"][orig_block]
+                    if "down" not in block_keys or "up" not in block_keys: continue
+                    
+                    t_d = handlers[idx].get_tensor(block_keys["down"]).to(device=device, dtype=torch.float32)
+                    t_u = handlers[idx].get_tensor(block_keys["up"]).to(device=device, dtype=torch.float32)
+                    
+                    max_rank = max(max_rank, t_d.shape[0])
+                    downs.append((t_d, info["weight"]))
+                    ups.append((t_u, info["weight"]))
+
+                if not downs:
+                    pbar.update(1)
+                    continue
+
+                def process_ties_dare(tensors, weights, dim_to_pad):
+                    # Pad all to max_rank
+                    padded = []
+                    for t, w in tensors:
+                        if t.shape[dim_to_pad] < max_rank:
+                            padding = [0] * (len(t.shape) * 2)
+                            # dim_to_pad 0 (down) -> last pair in pad
+                            # dim_to_pad 1 (up) -> second to last pair in pad
+                            rev_dim = len(t.shape) - 1 - dim_to_pad
+                            padding[rev_dim*2 + 1] = max_rank - t.shape[dim_to_pad]
+                            t = torch.nn.functional.pad(t, tuple(padding))
+                        padded.append(t * w)
+                    
+                    # DARE
+                    if drop_rate > 0:
+                        for i in range(len(padded)):
+                            mask = (torch.rand(padded[i].shape, generator=rng, device=device) > drop_rate).float()
+                            padded[i] = (padded[i] * mask) / (1 - drop_rate)
+                    
+                    # TIES
+                    # 1. Trim
+                    if trim_quantile > 0:
+                        for i in range(len(padded)):
+                            flat = padded[i].abs().flatten()
+                            k = int(len(flat) * trim_quantile)
+                            if k > 0:
+                                threshold = torch.kthvalue(flat, k).values
+                                padded[i] = torch.where(padded[i].abs() < threshold, torch.zeros_like(padded[i]), padded[i])
+                    
+                    # 2. Elect & Merge
+                    stacked = torch.stack(padded) # [N, ...]
+                    signs = torch.sign(stacked)
+                    sum_signs = signs.sum(dim=0)
+                    dominant_sign = torch.sign(sum_signs)
+                    
+                    # Filter those matching dominant sign
+                    mask = (signs == dominant_sign) & (dominant_sign != 0)
+                    filtered = torch.where(mask, stacked, torch.zeros_like(stacked))
+                    
+                    # Average matching signs
+                    count = mask.sum(dim=0)
+                    result = filtered.sum(dim=0) / torch.clamp(count, min=1.0)
+                    return result
+
+                merged_down = process_ties_dare(downs, [1.0]*len(downs), 0)
+                merged_up = process_ties_dare(ups, [1.0]*len(ups), 1)
+
+                output_sd[f"{core}.lora_down.weight"] = merged_down.to(save_dtype).cpu().contiguous()
+                output_sd[f"{core}.lora_up.weight"] = merged_up.to(save_dtype).cpu().contiguous()
+                output_sd[f"{core}.alpha"] = torch.tensor(float(max_rank), dtype=save_dtype)
+                
+                pbar.update(1)
+
+        # Final Summary
+        if verbose:
+            matched_stats = {i: 0 for i in range(len(lora_paths))}
+            for indices in layer_map.values():
+                for idx in indices:
+                    matched_stats[idx] += 1
+            
+            print(f"[LoRA Multi-Merge DARE] --- Merge Summary ---")
+            for i, info in enumerate(lora_infos):
+                print(f"[LoRA Multi-Merge DARE] LoRA {i+1}: {matched_stats[i]}/{len(info['pairs'])} layers used in merge")
+            print(f"[LoRA Multi-Merge DARE] Output state dict has {len(output_sd)} tensors")
+
+        # Save
+        output_dir = folder_paths.get_folder_paths("loras")[0]
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors")
+        save_file(output_sd, output_path, {"ss_training_comment": "Merged via DARE-Ties"})
+        return output_path
+
+    finally:
+        for h in handlers: h.__exit__(None, None, None)
+        cleanup_after_operation()

nodes/merger.py

@@ -93,6 +93,7 @@ class MergerLogic:
         mismatch_mode_str = recipe_params.get('mismatch_mode', 'skip')
         mismatch_mode = MissingTensorBehavior(mismatch_mode_str)
         recipe_params['mismatch_mode'] = mismatch_mode
+        alignment_mode = recipe_params.get('alignment_mode', 'pad/crop')
 
         # Compile filter patterns
         exclude_patterns = _compile_patterns(recipe_params.get('exclude_patterns', ''))
@@ -168,6 +169,15 @@ class MergerLogic:
                 for r_key, r_tensor in result.items():
                     merged_state_dict[r_key] = r_tensor.to(save_torch_dtype).cpu().clone()
             else:
+                # Ensure compatibility with Model A's architecture.
+                # If alignment_mode is 'pad/crop', we crop results that were padded.
+                # If alignment_mode is 'interpolate', resizing happened during operators.
+                if alignment_mode == 'pad/crop':
+                    target_shape = recipe_params['_tensor_a_shape']
+                    if result.shape != target_shape:
+                        slices = tuple(slice(0, min(res_s, tgt_s)) for res_s, tgt_s in zip(result.shape, target_shape))
+                        result = result[slices]
+
                 merged_state_dict[key] = result.to(save_torch_dtype).cpu().clone()
 
             # Clean up tensor_a reference to allow GC
@@ -219,9 +229,11 @@ class CheckpointTwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("checkpoints")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_checkpoint"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
@@ -237,8 +249,8 @@ class CheckpointTwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
@@ -246,8 +258,8 @@ class CheckpointTwoMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
@@ -272,9 +284,11 @@ class ModelTwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("diffusion_models")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_model"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
@@ -290,8 +304,8 @@ class ModelTwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
@@ -299,8 +313,8 @@ class ModelTwoMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
@@ -325,9 +339,11 @@ class TextEncoderTwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("text_encoders")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_textencoder"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
@@ -343,8 +359,8 @@ class TextEncoderTwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
@@ -352,8 +368,8 @@ class TextEncoderTwoMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
@@ -378,9 +394,11 @@ class LoRATwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("loras")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_lora"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
@@ -396,8 +414,8 @@ class LoRATwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
@@ -405,8 +423,8 @@ class LoRATwoMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
@@ -431,9 +449,11 @@ class EmbeddingTwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("embeddings")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_embedding"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
@@ -449,8 +469,8 @@ class EmbeddingTwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
@@ -458,8 +478,8 @@ class EmbeddingTwoMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
@@ -487,6 +507,7 @@ class CheckpointThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("checkpoints")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
@@ -506,7 +527,7 @@ class CheckpointThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
@@ -515,7 +536,7 @@ class CheckpointThreeMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
@@ -542,6 +563,7 @@ class ModelThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("diffusion_models")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
@@ -561,7 +583,7 @@ class ModelThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
@@ -570,7 +592,7 @@ class ModelThreeMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
@@ -597,6 +619,7 @@ class TextEncoderThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("text_encoders")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
@@ -616,7 +639,7 @@ class TextEncoderThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
@@ -625,7 +648,7 @@ class TextEncoderThreeMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
@@ -652,6 +675,7 @@ class LoRAThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("loras")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
@@ -671,7 +695,7 @@ class LoRAThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
@@ -680,7 +704,7 @@ class LoRAThreeMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
@@ -707,6 +731,7 @@ class EmbeddingThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("embeddings")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
                 io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
                 io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
@@ -726,7 +751,7 @@ class EmbeddingThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
@@ -735,7 +760,7 @@ class EmbeddingThreeMerger(io.ComfyNode):
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,

nodes/merger_ops.py

@@ -15,6 +15,92 @@ class MissingTensorError(Exception):
     """Raised when a tensor is missing and behavior is ERROR."""
     pass
 
+# ################# #
+# UTILITIES
+# ################# #
+def resize_by_interpolation(tensor1, tensor2):
+    """Resizes tensor2 to match tensor1's shape using interpolation."""
+    if tensor1 is None or tensor2 is None:
+        return tensor1, tensor2
+    if tensor1.shape == tensor2.shape:
+        return tensor1, tensor2
+
+    dims1 = len(tensor1.shape)
+    dims2 = len(tensor2.shape)
+
+    if dims1 != dims2 or dims1 > 4 or dims1 == 0:
+        return tensor1, tensor2
+
+    orig_dtype = tensor2.dtype
+    t2_f = tensor2.float()
+
+    # F.interpolate expects [B, C, ...]
+    # We treat the tensor as [1, 1, ...dims]
+    view_shape = [1, 1] + list(tensor2.shape)
+    t2_f = t2_f.view(view_shape)
+
+    # Select mode based on dimensions
+    if dims1 == 4:
+        # [Out, In, H, W] -> [1, 1, Out, In, H*W] (5D)
+        view_shape = [1, 1, tensor2.shape[0], tensor2.shape[1], tensor2.shape[2] * tensor2.shape[3]]
+        target_size = [tensor1.shape[0], tensor1.shape[1], tensor1.shape[2] * tensor1.shape[3]]
+        t2_f = t2_f.view(view_shape)
+        t2_res = F.interpolate(t2_f, size=target_size, mode='trilinear', align_corners=False)
+        return tensor1, t2_res.view(tensor1.shape).to(orig_dtype)
+
+    if dims1 == 1:
+        mode = 'linear'
+    elif dims1 == 2:
+        mode = 'bilinear'
+    else:
+        mode = 'trilinear'
+
+    t2_res = F.interpolate(t2_f, size=list(tensor1.shape), mode=mode, align_corners=False)
+
+    return tensor1, t2_res.view(tensor1.shape).to(orig_dtype)
+
+
+def resize_tensors(tensor1, tensor2, mode='pad/crop'):
+    """Resizes tensors to match shapes using either padding or interpolation."""
+    if tensor1 is None or tensor2 is None:
+        return tensor1, tensor2
+    if tensor1.shape == tensor2.shape:
+        return tensor1, tensor2
+
+    if mode == 'interpolate':
+        return resize_by_interpolation(tensor1, tensor2)
+
+    dims1 = len(tensor1.shape)
+    dims2 = len(tensor2.shape)
+
+    if dims1 != dims2 or dims1 > 4 or dims1 == 0:
+        return tensor1, tensor2
+
+    padding1 = []
+    padding2 = []
+
+    # F.pad expects: (last_dim_left, last_dim_right, prev_dim_top, prev_dim_bottom, ...)
+    for i in range(dims1 - 1, -1, -1):
+        s1 = tensor1.shape[i]
+        s2 = tensor2.shape[i]
+        if s1 < s2:
+            padding1.extend([0, s2 - s1])
+            padding2.extend([0, 0])
+        elif s2 < s1:
+            padding1.extend([0, 0])
+            padding2.extend([0, s1 - s2])
+        else:
+            padding1.extend([0, 0])
+            padding2.extend([0, 0])
+
+    if any(p > 0 for p in padding1):
+        tensor1 = F.pad(tensor1, tuple(padding1))
+    if any(p > 0 for p in padding2):
+        tensor2 = F.pad(tensor2, tuple(padding2))
+
+    return tensor1, tensor2
+
+
 # ################# #
 # MERGE OPERATORS
 # ################# #
@@ -23,6 +109,7 @@ class Operation:
         self.key = key
         self.sources = tuple(sources)
         self.merge_func = self.recurse
+        self.alignment_mode = 'pad/crop'
         self.alpha = None
         self.beta = None
         self.gamma = None
@@ -96,6 +183,7 @@ class Add(Operation):
             return b
         if b is None:
             return a
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
         return a + b
 
 
@@ -103,15 +191,33 @@ class Sub(Operation):
     def oper(self, a, b) -> torch.Tensor:
         if a is None or b is None:
             return None  # Cannot compute difference with missing operand
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
         return a - b
 
 class SVD(Operation):
-    def __init__(self, key, alpha, beta, gamma, *sources):
+    def __init__(self, key, alpha, beta, gamma, *sources, target_shape=None):
         super().__init__(key, *sources)
         self.alpha, self.beta, self.gamma = int(alpha), int(beta), gamma
+        self.target_shape = target_shape
+
     def oper(self, a, b) -> torch.Tensor:
         if a is None or b is None:
             return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+
+        # For SVD extraction, we must ensure the diff matches the original Model A shape
+        # so the resulting LoRA weights are compatible with the target architecture.
+        if self.target_shape and a.shape != self.target_shape:
+            if self.alignment_mode == 'interpolate':
+                # Re-align both to match target_shape via interpolation
+                dummy = torch.empty(self.target_shape, device=a.device)
+                _, a = resize_by_interpolation(dummy, a)
+                _, b = resize_by_interpolation(dummy, b)
+            else:
+                slices = tuple(slice(0, min(res_s, tgt_s)) for res_s, tgt_s in zip(a.shape, self.target_shape))
+                a = a[slices]
+                b = b[slices]
+
         diff, weights, conv2d = a - b, {}, len(a.size()) == 4
         kernel_size = None if not conv2d else a.size()[2:4]
         conv2d_3x3 = conv2d and kernel_size != (1, 1)
@@ -141,6 +247,10 @@ class TrainDiff(Operation):
     def oper(self, a, b, c) -> torch.Tensor:
         if a is None or b is None or c is None:
             return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+        a, c = resize_tensors(a, c, mode=self.alignment_mode)
+        b, c = resize_tensors(b, c, mode=self.alignment_mode)
+
         if torch.allclose(b.float(), c.float()): return torch.zeros_like(a)
         diff_bc = b.float() - c.float()
         dist_bc, dist_ab = torch.abs(diff_bc), torch.abs(b.float() - a.float())
@@ -153,11 +263,17 @@ class ExtractOp(Operation):
     def oper(self, base: torch.Tensor, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
         if a is None or b is None:
             return None
+        if base is not None:
+            base, a = resize_tensors(base, a, mode=self.alignment_mode)
+            base, b = resize_tensors(base, b, mode=self.alignment_mode)
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+
+        dtype = base.dtype if base is not None else a.dtype
         base_f = base.float() if base is not None else 0
         a_f, b_f = a.float() - base_f, b.float() - base_f
-        c = torch.cosine_similarity(a_f.flatten(), b_f.flatten(), 0).clamp(-1, 1)
+        c = torch.cosine_similarity(a_f, b_f, -1).clamp(-1, 1).unsqueeze(-1)
         d = ((c + 1) / 2)**self.gamma
-        return (torch.lerp(a_f, b_f, self.alpha) * torch.lerp(d, 1 - d, self.beta)).to(a.dtype)
+        return (torch.lerp(a_f, b_f, self.alpha) * torch.lerp(d, 1 - d, self.beta)).to(dtype)
 
 
 class Similarities(ExtractOp):
@@ -172,10 +288,14 @@ class PowerUpOp(Operation):
     def oper(self, a, b):
         if a is None or b is None:
             return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
         delta = b - a
+        keep_prob = 1 - self.alpha
+        if keep_prob <= 0:
+            return torch.zeros_like(delta)
         rng = torch.Generator(device=delta.device).manual_seed(self.seed)
-        m = torch.bernoulli(torch.full_like(delta, 1 - self.alpha), generator=rng)
-        return (m * delta) / (1 - self.alpha) if (1 - self.alpha) > 0 else torch.zeros_like(delta)
+        m = (torch.empty_like(delta).uniform_(0, 1, generator=rng) < keep_prob).to(delta.dtype)
+        return (m * delta) / keep_prob
 
 
 class InterpolateDifference(Operation):
@@ -185,6 +305,7 @@ class InterpolateDifference(Operation):
     def oper(self, a, b):
         if a is None or b is None:
             return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
         alpha, delta = max(self.alpha, 0.001), torch.abs(a - b)
         max_delta = torch.max(delta)
         if max_delta == 0: return a
@@ -194,6 +315,65 @@ class InterpolateDifference(Operation):
         mask = torch.lerp(torch.bernoulli(torch.clamp(diff, 0, 1), generator=rng), diff, self.gamma)
         return a * (1 - mask) + b * mask
 
+class ManualEnhancedInterpolateDifferenceOp(Operation):
+    def __init__(self, key, alpha, beta, gamma, delta, seed, *sources):
+        super().__init__(key, *sources)
+        self.alpha, self.beta, self.gamma, self.delta, self.seed = alpha, beta, gamma, delta, seed
+
+    def oper(self, a, b):
+        if a is None or b is None: return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+        delta = torch.abs(a - b)
+        diff = torch.nan_to_num((torch.max(delta) - delta) / torch.max(delta))
+        mean_diff = torch.mean(diff, 0, keepdim=True)
+        mask = torch.logical_and(self.beta < mean_diff, mean_diff < self.gamma)
+        powered_diff = torch.nan_to_num(diff ** (1 / max(self.alpha, 0.001) - 1))
+        masked_diff = powered_diff * mask.float()
+        rng = torch.Generator(device=a.device).manual_seed(self.seed)
+        random_mask = torch.bernoulli(torch.clamp(masked_diff, 0, 1), generator=rng)
+        interpolated_mask = torch.lerp(random_mask, masked_diff, self.delta)
+        return a * (1 - interpolated_mask) + b * interpolated_mask
+
+
+class AutoEnhancedInterpolateDifferenceOp(Operation):
+    def __init__(self, key, alpha, beta, gamma, seed, *sources):
+        super().__init__(key, *sources)
+        self.alpha, self.beta, self.gamma, self.seed = alpha, beta, gamma, seed
+
+    def oper(self, a, b):
+        if a is None or b is None: return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+        delta = torch.abs(a - b)
+        max_delta = torch.max(delta)
+        diff = torch.nan_to_num((max_delta - delta) / max_delta)
+        mean_diff = torch.mean(diff)
+        lower_threshold = mean_diff * (1 - self.beta)
+        upper_threshold = mean_diff * (1 + self.beta)
+        mask = torch.logical_and(lower_threshold < diff, diff < upper_threshold)
+        powered_diff = torch.nan_to_num(diff ** (1 / max(self.alpha, 0.001) - 1))
+        masked_diff = powered_diff * mask.float()
+        rng = torch.Generator(device=a.device).manual_seed(self.seed)
+        random_mask = torch.bernoulli(torch.clamp(masked_diff, 0, 1), generator=rng)
+        interpolated_mask = torch.lerp(random_mask, masked_diff, self.gamma)
+        return a * (1 - interpolated_mask) + b * interpolated_mask
+
+
+class WeightSumCutoffOp(Operation):
+    def __init__(self, key, alpha, beta, gamma, *sources):
+        super().__init__(key, *sources)
+        self.alpha, self.beta, self.gamma = alpha, beta, gamma
+
+    def oper(self, a, b):
+        if a is None or b is None: return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+        delta = torch.abs(a - b)
+        diff = torch.nan_to_num((torch.max(delta) - delta) / torch.max(delta))
+        mean = torch.mean(diff, 0, True)
+        mask = torch.logical_and(mean < self.beta, self.gamma < mean)
+        mul = self.alpha * mask
+        return a * (1 - mul) + b * mul
+
+
 # ################# #
 # CALCULATION MODES
 # ################# #
@@ -236,6 +416,7 @@ class WeightSum(CalcMode):
     param_docs = {'alpha': 'Interpolation weight. 0.0 is 100% Model A, 1.0 is 100% Model B.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         # Use pre-loaded tensor_a if available
         if '_tensor_a' in kwargs:
@@ -251,7 +432,9 @@ class WeightSum(CalcMode):
         if kwargs['alpha'] <= 0: return a
         c = Multiply(key, 1 - kwargs['alpha'], a)
         d = Multiply(key, kwargs['alpha'], b)
-        return Add(key, c, d)
+        res = Add(key, c, d)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class AddDifference(CalcMode):
@@ -261,6 +444,7 @@ class AddDifference(CalcMode):
     param_docs = {'alpha': 'Multiplier for the (B - C) difference.', 'beta': 'Smoothing (0=Off, 1=On).'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -272,9 +456,12 @@ class AddDifference(CalcMode):
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         diff = Sub(key, b, c)
+        diff.alignment_mode = alignment_mode
         if kwargs['beta'] == 1: diff = Smooth(key, diff)
         diffm = Multiply(key, kwargs['alpha'], diff)
-        return Add(key, a, diffm)
+        res = Add(key, a, diffm)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class TrainDifference(CalcMode):
@@ -284,6 +471,7 @@ class TrainDifference(CalcMode):
     param_docs = {'alpha': 'Multiplier for the calculated difference.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -295,8 +483,11 @@ class TrainDifference(CalcMode):
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         diff = TrainDiff(key, a, b, c)
+        diff.alignment_mode = alignment_mode
         diffm = Multiply(key, kwargs['alpha'], diff)
-        return Add(key, a, diffm)
+        res = Add(key, a, diffm)
+        res.alignment_mode = alignment_mode
+        return res
 class InterpDifference(CalcMode):
     name = 'Comparative-Interpolation'
     description = 'Interpolates A and B based on relative tensor value differences.'
@@ -304,6 +495,7 @@ class InterpDifference(CalcMode):
     param_docs = {'alpha': 'Curve strength.', 'beta': 'Style (0=Similarity, 1=Difference).', 'gamma': 'Mix (0=Random, 1=Linear).'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -317,7 +509,9 @@ class InterpDifference(CalcMode):
 
         loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
-        return InterpolateDifference(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['seed'], a, b)
+        res = InterpolateDifference(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['seed'], a, b)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class Extract(CalcMode):
@@ -327,6 +521,7 @@ class Extract(CalcMode):
     param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Similarity vs dissimilarity.', 'gamma': 'Similarity bias.', 'delta': 'Multiplier for final features.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -338,8 +533,11 @@ class Extract(CalcMode):
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         extracted = ExtractOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'] * 15, a, b, c)
+        extracted.alignment_mode = alignment_mode
         multiplied = Multiply(key, kwargs['delta'], extracted)
-        return Add(key, a, multiplied)
+        res = Add(key, a, multiplied)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class AddDisimilarity(CalcMode):
@@ -349,6 +547,7 @@ class AddDisimilarity(CalcMode):
     param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Multiplier for final features.', 'gamma': 'Similarity bias.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -360,8 +559,11 @@ class AddDisimilarity(CalcMode):
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         extracted = Similarities(key, kwargs['alpha'], 1, kwargs['gamma'] * 15, b, c)
+        extracted.alignment_mode = alignment_mode
         multiplied = Multiply(key, kwargs['beta'], extracted)
-        return Add(key, a, multiplied)
+        res = Add(key, a, multiplied)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class PowerUp(CalcMode):
@@ -371,6 +573,7 @@ class PowerUp(CalcMode):
     param_docs = {'alpha': 'Dropout rate (proportion of weights from B to *drop*).', 'beta': 'Multiplier for the final difference.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -381,8 +584,11 @@ class PowerUp(CalcMode):
         loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
         deltahat = PowerUpOp(key, kwargs['alpha'], kwargs['seed'], a, b)
-        res = Multiply(key, kwargs['beta'], deltahat)
-        return Add(key, a, res)
+        deltahat.alignment_mode = alignment_mode # Important!
+        multiplied = Multiply(key, kwargs['beta'], deltahat)
+        res = Add(key, a, multiplied)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class SVDMode(CalcMode):
@@ -392,6 +598,7 @@ class SVDMode(CalcMode):
     param_docs = {'alpha': 'Rank for standard layers.', 'beta': 'Rank for 3x3 conv layers.', 'gamma': 'Clamp quantile for weights.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -401,8 +608,82 @@ class SVDMode(CalcMode):
 
         loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
-        return SVD(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], a, b)
+        res = SVD(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], a, b, target_shape=kwargs.get('_tensor_a_shape'))
+        res.alignment_mode = alignment_mode
+        return res
 
 
-TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), SVDMode()]
+class ManEnhInterpDifference(CalcMode):
+    name = 'Enhanced Man Interp'
+    description = 'Enhanced interpolation between each pair of values from A and B depending on their difference relative to other values.'
+    models_used = ['A', 'B']
+    param_docs = {'alpha': 'Interpolation strength.', 'beta': 'Lower mean threshold.', 'gamma': 'Upper mean threshold.', 'delta': 'Smoothness factor.'}
+    def create_recipe(self, key, **kwargs):
+        mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
+
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+
+        if key.startswith('cond_stage_model.transformer.text_model.embeddings'):
+            return a
+
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
+        b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
+        res = ManualEnhancedInterpolateDifferenceOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['delta'], kwargs['seed'], a, b)
+        res.alignment_mode = alignment_mode
+        return res
+
+
+class AutoEnhInterpDifference(CalcMode):
+    name = 'Enhanced Auto Interp'
+    description = 'Interpolates between each pair of values from A and B depending on their difference relative to other values.'
+    models_used = ['A', 'B']
+    param_docs = {'alpha': 'Interpolation strength.', 'beta': 'Threshold adjustment factor.', 'gamma': 'Smoothness factor.'}
+    def create_recipe(self, key, **kwargs):
+        mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
+
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+
+        if key.startswith('cond_stage_model.transformer.text_model.embeddings'):
+            return a
+
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
+        b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
+        res = AutoEnhancedInterpolateDifferenceOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['seed'], a, b)
+        res.alignment_mode = alignment_mode
+        return res
+
+
+class WeightSumCutoffMode(CalcMode):
+    name = 'Weight-Sum Cutoff'
+    description = 'Weight-sum with cutoff based on value differences.'
+    models_used = ['A', 'B']
+    param_docs = {'alpha': 'Interpolation weight.', 'beta': 'Upper threshold.', 'gamma': 'Lower threshold.'}
+    def create_recipe(self, key, **kwargs):
+        mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
+
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
+        b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
+        res = WeightSumCutoffOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], a, b)
+        res.alignment_mode = alignment_mode
+        return res
+
+
+TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), SVDMode(), ManEnhInterpDifference(), AutoEnhInterpDifference(), WeightSumCutoffMode()]
 THREE_MODEL_MODES = [AddDifference(), TrainDifference(), Extract(), AddDisimilarity()]

nodes/merger_utils.py

@@ -93,6 +93,7 @@ class MemoryEfficientSafeOpen:
 
     Features:
     - mmap mode: Zero-copy tensor access via memory-mapped file
+    - low_memory mode: Direct file reads to minimize OS page cache usage
     - Parallel loading: Multi-threaded tensor reads for 2-4x speedup
     - Sorted batch reads: Keys sorted by file offset for sequential I/O
     - Auto-optimized workers: Adjusts parallelism based on device capabilities
@@ -102,12 +103,15 @@ class MemoryEfficientSafeOpen:
         filename: Path to safetensors file
         device: Target device (default 'cpu')
         mmap_mode: Use memory-mapped file for zero-copy (default True)
+        low_memory: Use direct file reads to minimize memory (overrides mmap_mode)
     """
 
-    def __init__(self, filename: str, device: str = 'cpu', mmap_mode: bool = True):
+    def __init__(self, filename: str, device: str = 'cpu', mmap_mode: bool = True, low_memory: bool = False):
         self.filename = filename
         self.device = device
-        self.mmap_mode = mmap_mode
+        # low_memory mode forces mmap off to avoid OS page caching
+        self.low_memory = low_memory
+        self.mmap_mode = mmap_mode and not low_memory
         self.header, self.header_size = self._read_header()
         self.file = open(filename, "rb")
         self.mmap_obj = None
@@ -123,6 +127,7 @@ class MemoryEfficientSafeOpen:
             self.mmap_obj.close()
         self.file.close()
 
+
     def keys(self) -> List[str]:
         """Return all tensor keys (excluding metadata)."""
         return [k for k in self.header.keys() if k != "__metadata__"]
@@ -156,13 +161,17 @@ class MemoryEfficientSafeOpen:
             else:
                 tensor_bytes = None
         else:
+            # Non-mmap mode: use pre-allocated bytearray with readinto for minimal copies
             tensor_bytes = None
             if offset_start != offset_end:
                 self.file.seek(self.header_size + 8 + offset_start)
-                tensor_bytes = self.file.read(offset_end - offset_start)
+                # Pre-allocate writable buffer and read directly into it
+                tensor_bytes = bytearray(offset_end - offset_start)
+                self.file.readinto(tensor_bytes)
 
         return self._deserialize_tensor(tensor_bytes, metadata)
 
+
     def get_tensor_to_gpu(
         self,
         key: str,
@@ -276,13 +285,19 @@ class MemoryEfficientSafeOpen:
         if tensor_bytes is None:
             byte_tensor = torch.empty(0, dtype=torch.uint8)
         else:
-            byte_tensor = torch.frombuffer(bytearray(tensor_bytes), dtype=torch.uint8)
+            # Avoid extra copy if already a bytearray (low_memory mode)
+            if isinstance(tensor_bytes, bytearray):
+                byte_tensor = torch.frombuffer(tensor_bytes, dtype=torch.uint8)
+            else:
+                # mmap memoryview needs copy to create writable tensor
+                byte_tensor = torch.frombuffer(bytearray(tensor_bytes), dtype=torch.uint8)
 
         if dtype_str in ["F8_E5M2", "F8_E4M3"]:
             return self._convert_float8(byte_tensor, dtype_str, shape)
 
         return byte_tensor.view(dtype).reshape(shape)
 
+
     @staticmethod
     def _get_torch_dtype(dtype_str):
         dtype_map = {

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "ComfyUI-ModelUtils"
 description = "[WIP]Custom nodes for handling, inspecting, modifying and creating various model files."
-version = "0.2.3"
+version = "0.2.7"
 license = { file = "LICENSE" }
 
 [project.urls]