Add ability to adjust paramaters on a ler layer basis using regex/substring for layer colon separated with parameter=float for specific parameters

__init__.py

@@ -1,7 +1,7 @@
 from typing_extensions import override
 from comfy_api.latest import ComfyExtension, io
 
-from .nodes.lora_merger import LoRAMultiMerge, LoRAMultiMergeDARE, LoRAMultiMergeDAREEnhanced
+from .nodes.lora_merger import LoRAMultiMerge, LoRAMultiMergeDARE
 from .nodes.metakeys import (
     ModelMetaKeys, TextEncoderMetaKeys, LoRAMetaKeys,
     CheckpointMetaKeys, EmbeddingMetaKeys
@@ -24,14 +24,6 @@ from .nodes.lora_extract_svd import (
     LoRAExtractFixed, LoRAExtractRatio, LoRAExtractQuantile,
     LoRAExtractKnee, LoRAExtractFrobenius
 )
-from .nodes.dora_extract_wd import (
-    DoRAExtractFixed, DoRAExtractRatio, DoRAExtractQuantile,
-    DoRAExtractKnee, DoRAExtractFrobenius
-)
-from .nodes.dora_learned_wd import (
-    DoRALearnedExtractFixed, DoRALearnedExtractRatio, DoRALearnedExtractQuantile,
-    DoRALearnedExtractKnee, DoRALearnedExtractFrobenius
-)
 from .nodes.lora_resize import (
     LoRAResizeFixed, LoRAResizeRatio,
     LoRAResizeFrobenius, LoRAResizeCumulative,
@@ -47,8 +39,6 @@ from .nodes.model_info_nodes import (
 )
 
 
-from .nodes.lora_rename import AnimaLoraRename
-
 class ModelUtilsExtension(ComfyExtension):
     @override
     async def get_node_list(self) -> list[type[io.ComfyNode]]:
@@ -71,21 +61,13 @@ class ModelUtilsExtension(ComfyExtension):
             # LoRA Extraction
             LoRAExtractFixed, LoRAExtractRatio, LoRAExtractQuantile,
             LoRAExtractKnee, LoRAExtractFrobenius,
-            # DoRA Extraction
-            DoRAExtractFixed, DoRAExtractRatio, DoRAExtractQuantile,
-            DoRAExtractKnee, DoRAExtractFrobenius,
-            # Learned DoRA Extraction
-            DoRALearnedExtractFixed, DoRALearnedExtractRatio, DoRALearnedExtractQuantile,
-            DoRALearnedExtractKnee, DoRALearnedExtractFrobenius,
             # LoRA Resize
             LoRAResizeFixed, LoRAResizeRatio,
             LoRAResizeFrobenius, LoRAResizeCumulative,
             # LoRA Multi-Merge
-            LoRAMultiMerge, LoRAMultiMergeDARE, LoRAMultiMergeDAREEnhanced,
+            LoRAMultiMerge, LoRAMultiMergeDARE,
             # LoRA Merge To Model
             LoRAMergeToModel,
-            # LoRA Utilities
-            AnimaLoraRename,
             # Downloaders
             CheckpointInfoMetaDownloader, DiffusionModelInfoMetaDownloader, LoRAInfoMetaDownloader, EmbeddingInfoMetaDownloader,
             VAEInfoMetaDownloader, ControlNetInfoMetaDownloader, ManualPathInfoMetaDownloader,

docs/merger_2_model_modes.md

@@ -121,15 +121,6 @@ When merging models with different layer structures (e.g., fine-tuned models wit
 
 ---
 
-## Dtype Preservation & Override
-
-By default, the merger preserves the highest precision `dtype` for each key found across all active source models (Model A and Model B) and the requested `save_dtype`.
-
-- **Preserve Higher Precision**: If a key is `float32` in any of the source models, but you requested `save_dtype="bf16"` or `"fp16"`, that key will be saved as `float32` anyway. This prevents degrading critical high-precision layers (such as normalization layers or layers with restored outliers).
-- **Explicit Override**: If you prefer to force the entire model to be saved strictly as the selected `save_dtype` (e.g. for complete downcasting to `bf16`), toggle `override_dtype` to `True`.
-
----
-
 ## Layer Filtering (Regex Patterns)
 
 Use regex patterns to control which layers are merged:

docs/merger_3_model_modes.md

@@ -67,15 +67,6 @@ When merging models with different layer structures, the `mismatch_mode` paramet
 
 ---
 
-## Dtype Preservation & Override
-
-By default, the merger preserves the highest precision `dtype` for each key found across all active source models (Model A, B, and C) and the requested `save_dtype`.
-
-- **Preserve Higher Precision**: If a key is `float32` in any of the active source models, but you requested `save_dtype="bf16"` or `"fp16"`, that key will be saved as `float32` anyway. This prevents degrading critical high-precision layers (such as normalization layers or layers with restored outliers).
-- **Explicit Override**: If you prefer to force the entire model to be saved strictly as the selected `save_dtype` (e.g. for complete downcasting to `bf16`), toggle `override_dtype` to `True`.
-
----
-
 ## Layer Filtering (Regex Patterns)
 
 Use regex patterns to control which layers are merged:

nodes/dora_extract_wd.py

@@ -1,981 +0,0 @@
-"""
-DoRA Extraction via SVD and Weight Decomposition.
-
-Native implementation of Weight-Decomposed Low-Rank Adapter extraction
-with multiple rank selection modes. Extracted models use the DoRA structure.
-"""
-import fnmatch
-import os
-import re
-import torch
-import torch.linalg as linalg
-import folder_paths
-import comfy.utils
-from tqdm import tqdm
-from comfy_api.latest import io
-from .device_utils import (
-    estimate_model_size, prepare_for_large_operation,
-    cleanup_after_operation, get_device_capabilities
-)
-
-from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned, IncrementalSafetensorsWriter
-
-
-
-
-# =============================================================================
-# Rank Index Functions - Determine LoRA rank from singular values
-# =============================================================================
-
-def _index_sv_fixed(S: torch.Tensor, dim: int) -> int:
-    """Fixed rank mode - use specified dimension."""
-    return max(1, min(dim, len(S) - 1))
-
-
-def _index_sv_threshold(S: torch.Tensor, threshold: float) -> int:
-    """Threshold mode - keep singular values above absolute threshold."""
-    rank = int(torch.sum(S > threshold).item())
-    return max(1, min(rank, len(S) - 1))
-
-
-def _index_sv_ratio(S: torch.Tensor, ratio: float) -> int:
-    """Ratio mode - keep singular values > max(S) / ratio (kohya convention)."""
-    if ratio <= 0:
-        return len(S) - 1
-    min_sv = S[0] / ratio
-    rank = int(torch.sum(S > min_sv).item())
-    return max(1, min(rank, len(S) - 1))
-
-
-def _index_sv_cumulative(S: torch.Tensor, target: float, max_rank: int = None) -> int:
-    """Cumulative mode - keep enough SVs to reach target % of total.
-
-    Calculates relative to max_rank if provided, otherwise relative to full.
-    """
-    if max_rank is not None and max_rank < len(S):
-        total = torch.sum(S[:max_rank])
-    else:
-        total = torch.sum(S)
-
-    if total < 1e-8:
-        return 1
-    cumsum = torch.cumsum(S, dim=0) / total
-    rank = int(torch.searchsorted(cumsum, target).item()) + 1
-    return max(1, min(rank, len(S) - 1))
-
-
-def _index_sv_fro(S: torch.Tensor, target: float, max_rank: int = None) -> int:
-    """Frobenius norm mode - preserve target fraction of Frobenius norm.
-
-    Calculates relative to max_rank if provided, otherwise relative to full.
-    This means "retain target% of what's achievable within max_rank".
-    """
-    if max_rank is not None and max_rank < len(S):
-        # Calculate relative to what's achievable within max_rank
-        S_capped = S[:max_rank]
-        S_sq = S_capped.pow(2)
-        total_sq = torch.sum(S_sq)
-    else:
-        S_sq = S.pow(2)
-        total_sq = torch.sum(S_sq)
-
-    if total_sq < 1e-8:
-        return 1
-
-    # Cumsum of all S (not capped) to find where we reach target
-    cumsum = torch.cumsum(S.pow(2), dim=0) / total_sq
-    rank = int(torch.searchsorted(cumsum, target ** 2).item()) + 1
-    return max(1, min(rank, len(S) - 1))
-
-
-def _index_sv_knee(S: torch.Tensor, min_sv: float = 1e-8) -> int:
-    """Knee detection - find elbow point on singular value curve."""
-    n = len(S)
-    if n < 3:
-        return 1
-    s_max, s_min = S[0].item(), S[-1].item()
-    if s_max - s_min < min_sv:
-        return 1
-    # Normalize to [0, 1]
-    s_norm = (S - s_min) / (s_max - s_min)
-    x_norm = torch.linspace(0, 1, n, device=S.device, dtype=S.dtype)
-    # Distance from diagonal
-    distances = (x_norm + s_norm - 1).abs()
-    rank = torch.argmax(distances).item() + 1
-    return max(1, min(rank, n - 1))
-
-
-def _index_sv_cumulative_knee(S: torch.Tensor, min_sv: float = 1e-8) -> int:
-    """Knee detection on cumulative singular value curve."""
-    n = len(S)
-    if n < 3:
-        return 1
-    total = torch.sum(S)
-    if total < min_sv:
-        return 1
-    cumsum = torch.cumsum(S, dim=0) / total
-    y_min, y_max = cumsum[0].item(), cumsum[-1].item()
-    if y_max - y_min < min_sv:
-        return 1
-    y_norm = (cumsum - y_min) / (y_max - y_min)
-    x_norm = torch.linspace(0, 1, n, device=S.device, dtype=S.dtype)
-    distances = (y_norm - x_norm).abs()
-    rank = torch.argmax(distances).item() + 1
-    return max(1, min(rank, n - 1))
-
-
-def _index_sv_rel_decrease(S: torch.Tensor, tau: float = 0.1) -> int:
-    """Relative decrease mode - stop when consecutive SV ratio drops below tau."""
-    if len(S) < 2:
-        return 1
-    ratios = S[1:] / (S[:-1] + 1e-8)
-    for k in range(len(ratios)):
-        if ratios[k] < tau:
-            return k + 1
-    return len(S)
-
-
-def _compute_rank(S: torch.Tensor, mode: str, mode_param: float,
-                  max_rank: int = None) -> int:
-    """Compute rank based on mode and parameters."""
-    if mode == "fixed":
-        rank = _index_sv_fixed(S, int(mode_param))
-    elif mode == "threshold":
-        rank = _index_sv_threshold(S, mode_param)
-    elif mode == "ratio":
-        rank = _index_sv_ratio(S, mode_param)
-    elif mode == "quantile" or mode == "sv_cumulative":
-        rank = _index_sv_cumulative(S, mode_param, max_rank)
-    elif mode == "sv_fro":
-        rank = _index_sv_fro(S, mode_param, max_rank)
-    elif mode == "sv_knee":
-        rank = _index_sv_knee(S)
-    elif mode == "sv_cumulative_knee":
-        rank = _index_sv_cumulative_knee(S)
-    elif mode == "sv_rel_decrease":
-        rank = _index_sv_rel_decrease(S, mode_param)
-    else:
-        rank = _index_sv_fixed(S, int(mode_param) if mode_param else 64)
-
-    if max_rank is not None:
-        rank = min(rank, max_rank)
-    return rank
-
-
-# =============================================================================
-# SVD Extraction Functions
-# =============================================================================
-
-def _svd_extract_linear_lowrank(
-    weight: torch.Tensor,
-    rank: int,
-    device: str,
-    clamp_quantile: float = 0.99,
-    niter: int = 2,
-) -> tuple:
-    """
-    Low-rank SVD decomposition using svd_lowrank for fixed rank extraction.
-
-    Much faster than full SVD when rank << min(m, n) because it only computes
-    the top-k singular values using randomized algorithms.
-
-    Args:
-        niter: Power iterations for SVD accuracy (higher = more accurate but slower)
-
-    Returns:
-        (lora_down, lora_up, diff), "low rank" or (weight, "full")
-    """
-    weight = weight.to(device=device, dtype=torch.float32)
-    out_dim, in_dim = weight.shape
-
-    # Clamp rank to valid range
-    max_possible_rank = min(out_dim, in_dim)
-    rank = max(1, min(rank, max_possible_rank))
-
-    # Check if decomposition is worthwhile
-    if rank >= max_possible_rank:
-        return weight, "full"
-
-    # Use svd_lowrank for faster low-rank approximation
-    try:
-        U, S, V = torch.svd_lowrank(weight, q=rank, niter=niter)
-        # U: [out_dim, rank], S: [rank], V: [in_dim, rank]
-        Vh = V.T  # [rank, in_dim]
-    except Exception as e:
-        raise RuntimeError(f"svd_lowrank failed: {e}")
-
-    # Clamp outliers
-    if clamp_quantile < 1.0 and len(S) > 0:
-        try:
-            max_val = torch.quantile(S, clamp_quantile)
-            S = S.clamp(max=max_val)
-        except RuntimeError:
-            pass
-
-    # Construct LoRA matrices
-    lora_up = U @ torch.diag(S)  # [out_dim, rank]
-    lora_down = Vh               # [rank, in_dim]
-
-    # Compute reconstruction diff
-    diff = weight - (lora_up @ lora_down)
-
-    return (lora_down, lora_up, diff), "low rank"
-
-
-def _svd_extract_linear(
-    weight: torch.Tensor,
-    mode: str,
-    mode_param: float,
-    device: str,
-    max_rank: int = None,
-    clamp_quantile: float = 0.99,
-    niter: int = 2,
-) -> tuple:
-    """
-    SVD decomposition for linear layers.
-
-    For fixed rank mode, uses svd_lowrank which is much faster.
-    For adaptive modes, uses full SVD to analyze all singular values.
-
-    Returns:
-        (lora_down, lora_up, rank), "low rank" or (weight, "full")
-    """
-    weight = weight.to(device=device, dtype=torch.float32)
-    out_dim, in_dim = weight.shape
-
-    # For fixed rank, use the faster svd_lowrank
-    if mode == "fixed" and mode_param > 0:
-        target_rank = int(mode_param)
-        if max_rank is not None:
-            target_rank = min(target_rank, max_rank)
-        return _svd_extract_linear_lowrank(weight, target_rank, device, clamp_quantile, niter)
-
-    # For adaptive modes, use full SVD to analyze singular values
-    try:
-        U, S, Vh = linalg.svd(weight, full_matrices=False)
-    except Exception as e:
-        raise RuntimeError(f"SVD failed: {e}")
-
-    # Compute rank based on mode
-    rank = _compute_rank(S, mode, mode_param, max_rank)
-
-    # Check if decomposition is worthwhile
-    if rank >= min(out_dim, in_dim):
-        return weight, "full"
-
-    # Truncate to rank
-    U = U[:, :rank]
-    S = S[:rank]
-    Vh = Vh[:rank, :]
-
-    # Clamp outliers
-    if clamp_quantile < 1.0 and len(S) > 0:
-        try:
-            max_val = torch.quantile(S, clamp_quantile)
-            S = S.clamp(max=max_val)
-        except RuntimeError:
-            pass  # Skip clamping if quantile fails
-
-    # Construct LoRA matrices
-    # lora_up = U @ diag(S), lora_down = Vh
-    lora_up = U @ torch.diag(S)  # [out_dim, rank]
-    lora_down = Vh               # [rank, in_dim]
-
-    # Compute reconstruction diff for optional sparse bias
-    diff = weight - (lora_up @ lora_down)
-
-    return (lora_down, lora_up, diff), "low rank"
-
-
-def _svd_extract_conv(
-    weight: torch.Tensor,
-    mode: str,
-    mode_param: float,
-    device: str,
-    max_rank: int = None,
-    clamp_quantile: float = 0.99,
-) -> tuple:
-    """
-    SVD decomposition for conv2d layers.
-
-    Returns:
-        (lora_down, lora_up, rank), "low rank" or (weight, "full")
-    """
-    out_ch, in_ch, kh, kw = weight.shape
-    is_1x1 = (kh == 1 and kw == 1)
-
-    # Flatten for SVD
-    if is_1x1:
-        mat = weight.view(out_ch, in_ch)  # [out_ch, in_ch]
-    else:
-        mat = weight.reshape(out_ch, -1)  # [out_ch, in_ch*k*k]
-
-    result, mode_str = _svd_extract_linear(mat, mode, mode_param, device, max_rank, clamp_quantile)
-
-    if mode_str == "full":
-        return weight, "full"
-
-    lora_down, lora_up, diff = result
-    rank = lora_down.shape[0]
-
-    # Reshape for conv
-    if is_1x1:
-        lora_down = lora_down.unsqueeze(-1).unsqueeze(-1)  # [rank, in_ch, 1, 1]
-        lora_up = lora_up.unsqueeze(-1).unsqueeze(-1)      # [out_ch, rank, 1, 1]
-    else:
-        lora_down = lora_down.reshape(rank, in_ch, kh, kw)
-        lora_up = lora_up.reshape(out_ch, rank, 1, 1)
-
-    return (lora_down, lora_up, diff), "low rank"
-
-
-# =============================================================================
-# Chunked Extraction for Large Layers
-# =============================================================================
-
-def _detect_fused_layer(key: str, shape: tuple) -> int:
-    """Detect fused QKV/MLP layers and return number of chunks."""
-    if len(shape) != 2:
-        return 1
-
-    out_dim, in_dim = shape
-    key_lower = key.lower()
-
-    # QKV layers (3 chunks)
-    if "qkv" in key_lower and out_dim % 3 == 0 and out_dim // 3 >= in_dim // 4:
-        return 3
-
-    # KV layers (2 chunks)
-    if "kv" in key_lower and "qkv" not in key_lower and out_dim % 2 == 0:
-        return 2
-
-    # Fused MLP (2 chunks)
-    if ("linear1" in key_lower or "gate_up" in key_lower) and out_dim % 2 == 0 and out_dim >= in_dim * 2:
-        return 2
-
-    return 1
-
-
-def _extract_chunked_layer(
-    weight_diff: torch.Tensor,
-    num_chunks: int,
-    mode: str,
-    mode_param: float,
-    device: str,
-    max_rank: int = None,
-) -> tuple:
-    """Extract LoRA from fused layer by chunking."""
-    out_dim, in_dim = weight_diff.shape
-    chunk_size = out_dim // num_chunks
-
-    all_lora_up = []
-    all_lora_down = []
-
-    for i in range(num_chunks):
-        chunk = weight_diff[i * chunk_size:(i + 1) * chunk_size, :]
-
-        try:
-            result, mode_str = _svd_extract_linear(chunk, mode, mode_param, device, max_rank)
-            if mode_str == "full":
-                return None, None, 0
-            lora_down, lora_up, _ = result
-            all_lora_down.append(lora_down)
-            all_lora_up.append(lora_up)
-        except Exception:
-            return None, None, 0
-
-    # Combine chunks
-    # Note: Using the first chunk's down matrix as the basis for all chunks
-    # is a better approximation than a simple mean if the chunks share the same input space (like QKV).
-    # Ideally, we would use a more sophisticated joint SVD, but this is a reasonable fallback.
-    combined_lora_up = torch.cat(all_lora_up, dim=0)
-    combined_lora_down = all_lora_down[0]
-    rank = combined_lora_down.shape[0]
-
-    return combined_lora_up, combined_lora_down, rank
-
-
-# =============================================================================
-# Pattern Matching
-# =============================================================================
-
-def _compile_patterns(pattern_string: str, glob_mode: bool = False) -> list:
-    """Compile patterns from whitespace-separated string.
-
-    In regex mode (default) returns compiled re.Pattern objects.
-    In glob mode returns plain strings; fnmatch handles matching.
-    """
-    if not pattern_string or not pattern_string.strip():
-        return []
-    patterns = []
-    for p in pattern_string.split():
-        p = p.strip()
-        if not p:
-            continue
-        if glob_mode:
-            patterns.append(p)
-        else:
-            try:
-                patterns.append(re.compile(p))
-            except re.error:
-                pass
-    return patterns
-
-
-def _matches_any_pattern(key: str, patterns: list, glob_mode: bool = False) -> bool:
-    """Check if key matches any pattern.
-
-    Glob mode: each pattern is matched as a substring glob using fnmatch
-    (dots are literal, * matches any sequence of characters).
-    Regex mode: each pattern is a compiled re.Pattern, matched via search().
-    """
-    if glob_mode:
-        return any(fnmatch.fnmatch(key, f"*{p}*") for p in patterns)
-    return any(p.search(key) for p in patterns)
-
-
-# =============================================================================
-# Main Extraction Function
-# =============================================================================
-
-def extract_dora_from_files(
-    model_a_path: str,
-    model_b_path: str,
-    mode: str,
-    linear_param: float,
-    conv_param: float,
-    device: str,
-    save_dtype: str,
-    output_path: str,
-    linear_max_rank: int = None,
-    conv_max_rank: int = None,
-    clamp_quantile: float = 0.99,
-    min_diff: float = 0.0,
-    skip_patterns_str: str = "",
-    mismatch_mode: str = "skip",
-    chunk_large_layers: bool = True,
-    svd_niter: int = 2,
-    lazy_load: bool = True,
-    force_clear_cache: bool = True,
-    glob_skip_patterns: bool = False,
-) -> None:
-    """
-    Extract LoRA from difference between two models, writing incrementally to disk.
-
-    Args:
-        model_a_path: Finetuned model path
-        model_b_path: Base model path
-        mode: Rank selection mode
-        linear_param: Mode parameter for linear layers
-        conv_param: Mode parameter for conv layers
-        device: Computation device
-        save_dtype: Output dtype
-        output_path: Full path to output safetensors file
-        linear_max_rank: Max rank for linear layers
-        conv_max_rank: Max rank for conv layers
-        clamp_quantile: Quantile for weight clamping
-        min_diff: Minimum difference threshold
-        skip_patterns_str: Patterns for layers to skip
-        mismatch_mode: How to handle mismatches
-        chunk_large_layers: Enable chunked extraction
-        glob_skip_patterns: When True, treat skip_patterns as glob (* wildcard).
-                            When False (default), treat as Python regex.
-    """
-    save_torch_dtype = {
-        "fp32": torch.float32,
-        "fp16": torch.float16,
-        "bf16": torch.bfloat16,
-    }.get(save_dtype, torch.float16)
-
-    skip_patterns = _compile_patterns(skip_patterns_str, glob_mode=glob_skip_patterns)
-
-    # Prepare memory before heavy operation
-    total_size_gb = estimate_model_size(model_a_path) + estimate_model_size(model_b_path)
-    print(f"[DoRA Extract] Preparing memory for {total_size_gb:.2f}GB operation...")
-    prepare_for_large_operation(total_size_gb * 1.5, torch.device(device))  # 1.5x for SVD headroom
-
-    handler_a = MemoryEfficientSafeOpen(model_a_path, low_memory=lazy_load)
-    handler_b = MemoryEfficientSafeOpen(model_b_path, low_memory=lazy_load)
-
-    try:
-        keys_a = set(handler_a.keys())
-        keys_b = set(handler_b.keys())
-        weight_keys = [k for k in keys_a if k.endswith(".weight")]
-        pbar = comfy.utils.ProgressBar(len(weight_keys))
-        stats = {"extracted": 0, "full": 0, "skipped": 0, "chunked": 0}
-
-        def _process_layer(key):
-            lora_name = _format_lora_key(key)
-
-            if _matches_any_pattern(key, skip_patterns, glob_mode=glob_skip_patterns):
-                return "skipped", None
-
-            # Load tensors with pinned memory for CUDA
-            use_pinned = device == 'cuda'
-
-            if key not in keys_b:
-                if mismatch_mode == "skip":
-                    return "skipped", None
-                if mismatch_mode == "error":
-                    raise ValueError(f"Key {key} not found in model B")
-
-                cpu_a = handler_a.get_tensor(key)
-                if use_pinned:
-                    weight_diff = transfer_to_gpu_pinned(cpu_a, device, torch.float32)
-                else:
-                    weight_diff = cpu_a.to(device=device, dtype=torch.float32)
-                del cpu_a
-
-                # Without base, DoRA fallback just treats tensor_a as diff
-                dora_scale = None
-            else:
-                cpu_a = handler_a.get_tensor(key)
-                cpu_b = handler_b.get_tensor(key)
-
-                if use_pinned:
-                    tensor_a = transfer_to_gpu_pinned(cpu_a, device, torch.float32)
-                    tensor_b = transfer_to_gpu_pinned(cpu_b, device, torch.float32)
-                else:
-                    tensor_a = cpu_a.to(device=device, dtype=torch.float32)
-                    tensor_b = cpu_b.to(device=device, dtype=torch.float32)
-                del cpu_a, cpu_b
-
-                if tensor_a.shape != tensor_b.shape:
-                    if mismatch_mode == "skip":
-                        del tensor_a, tensor_b
-                        return "skipped", None
-                    if mismatch_mode == "error":
-                        raise ValueError(f"Shape mismatch for {key}: {tensor_a.shape} vs {tensor_b.shape}")
-
-                    weight_diff = tensor_a
-                    dora_scale = None
-                    del tensor_b
-                else:
-                    # Weight Decomposition calculation
-                    is_conv_dim = tensor_a.ndim == 4
-                    
-                    if is_conv_dim:
-                        out_ch = tensor_a.shape[0]
-                        flat_a = tensor_a.reshape(out_ch, -1)
-                        flat_b = tensor_b.reshape(out_ch, -1)
-                        norm_a = torch.linalg.norm(flat_a, dim=1, keepdim=True)
-                        norm_b = torch.linalg.norm(flat_b, dim=1, keepdim=True)
-                    else:
-                        norm_a = torch.linalg.norm(tensor_a, dim=1, keepdim=True)
-                        norm_b = torch.linalg.norm(tensor_b, dim=1, keepdim=True)
-                    
-                    # Compute directional diff: W_dir = W_ft * (norm_base / norm_ft)
-                    # Use a small epsilon to avoid division by zero
-                    eps = 1e-8
-                    norm_ratio = norm_b / (norm_a + eps)
-                    if is_conv_dim:
-                        norm_ratio = norm_ratio.view(-1, 1, 1, 1)
-                        dora_scale = norm_a.view(-1, 1, 1, 1)
-                    else:
-                        dora_scale = norm_a
-                        
-                    w_dir = tensor_a * norm_ratio
-                    weight_diff = w_dir - tensor_b
-                    del tensor_a, tensor_b, w_dir
-
-            # Skip small differences
-            if min_diff > 0 and weight_diff.abs().max() < min_diff:
-                del weight_diff
-                return "skipped", None
-
-            # Skip 1D tensors
-            if weight_diff.ndim < 2:
-                del weight_diff
-                return "skipped", None
-
-            is_conv = weight_diff.ndim == 4
-            layer_results = {}
-
-            try:
-                if is_conv:
-                    result, mode_str = _svd_extract_conv(
-                        weight_diff, mode, conv_param, device, conv_max_rank, clamp_quantile
-                    )
-                else:
-                    result, mode_str = _svd_extract_linear(
-                        weight_diff, mode, linear_param, device, linear_max_rank, clamp_quantile, svd_niter
-                    )
-            except Exception as e:
-                # Try chunked extraction for large tensors
-                if chunk_large_layers and not is_conv:
-                    num_chunks = _detect_fused_layer(key, weight_diff.shape)
-                    if num_chunks > 1:
-                        print(f"[DoRA Extract] Chunked: {key} ({num_chunks} chunks)")
-                        lora_up, lora_down, rank = _extract_chunked_layer(
-                            weight_diff, num_chunks, mode, linear_param, device, linear_max_rank
-                        )
-                        if lora_up is not None:
-                            layer_results[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
-                            layer_results[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
-                            layer_results[f"{lora_name}.alpha"] = torch.tensor(rank, dtype=save_torch_dtype)
-                            if dora_scale is not None:
-                                layer_results[f"{lora_name}.dora_scale"] = dora_scale.to(save_torch_dtype).cpu().contiguous()
-                            del weight_diff
-                            return "chunked", layer_results
-
-                print(f"[DoRA Extract] Failed: {key}: {e}")
-                del weight_diff
-                return "skipped", None
-
-            # Store result
-            if mode_str == "full":
-                layer_results[f"{lora_name}.diff"] = weight_diff.to(save_torch_dtype).cpu().contiguous()
-                status = "full"
-            else:
-                lora_down, lora_up, rank_or_diff = result
-                # rank is the size of lora_down's first dimension
-                rank = lora_down.shape[0]
-                layer_results[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
-                layer_results[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
-                layer_results[f"{lora_name}.alpha"] = torch.tensor(rank, dtype=save_torch_dtype)
-                if dora_scale is not None:
-                    layer_results[f"{lora_name}.dora_scale"] = dora_scale.to(save_torch_dtype).cpu().contiguous()
-                status = "extracted"
-
-            del weight_diff
-            return status, layer_results
-
-        writer = IncrementalSafetensorsWriter(output_path)
-        writer.__enter__()
-        try:
-            for key in tqdm(weight_keys, desc="Extracting DoRA", unit="layers"):
-                status, layer_sd = _process_layer(key)
-                stats[status] += 1
-                if layer_sd:
-                    writer.write_dict(layer_sd)
-
-                if force_clear_cache:
-                    import gc
-                    gc.collect()
-                    if torch.cuda.is_available():
-                        torch.cuda.empty_cache()
-
-                pbar.update(1)
-        finally:
-            writer.__exit__(None, None, None)
-
-        print(f"[DoRA Extract] Done: {stats['extracted']} extracted, {stats['chunked']} chunked, "
-              f"{stats['full']} full, {stats['skipped']} skipped")
-
-    finally:
-        handler_a.__exit__(None, None, None)
-        handler_b.__exit__(None, None, None)
-        cleanup_after_operation()
-
-
-def _build_lora_output_path(output_filename: str) -> str:
-    """Build output path for LoRA file."""
-    output_dir = folder_paths.get_folder_paths("loras")[0]
-    os.makedirs(output_dir, exist_ok=True)
-    return os.path.join(output_dir, f"{output_filename.strip()}.safetensors")
-
-
-# =============================================================================
-# Node Definitions
-# =============================================================================
-
-def _get_model_inputs():
-    return [
-        io.Combo.Input("model_a", options=folder_paths.get_filename_list("diffusion_models"),
-                      tooltip="Finetuned model (A - B = LoRA)"),
-        io.Combo.Input("model_b", options=folder_paths.get_filename_list("diffusion_models"),
-                      tooltip="Base model (A - B = LoRA)"),
-    ]
-
-
-def _format_lora_key(key: str) -> str:
-    """
-    Format the key for LoRA saving.
-    Always standardizes to 'diffusion_model.' prefix for maximum compatibility.
-
-    ComfyUI maps 'diffusion_model.<layer>' generically for all model types, making
-    it universally compatible. The 'transformer.' prefix only works for a subset of
-    model types via model-specific mapping code and is intentionally not used here.
-    """
-    if key.endswith(".weight"):
-        key = key[:-7]
-
-    # Handle ComfyUI Checkpoint wrapper (ldm/sgm format: model.diffusion_model.*)
-    if key.startswith("model.diffusion_model."):
-        inner_key = key[22:]  # strip len("model.diffusion_model.")
-        return f"diffusion_model.{inner_key}"
-
-    # Handle net.* wrapper
-    if key.startswith("net."):
-        inner_key = key[4:]  # strip len("net.")
-        return f"diffusion_model.{inner_key}"
-
-    # Handle direct Diffusers keys (transformer_blocks.* / single_transformer_blocks.* without prefix)
-    if key.startswith("transformer_blocks") or key.startswith("single_transformer_blocks"):
-        return f"diffusion_model.{key}"
-
-    # Handle legacy lora_unet_ prefix (for resizing without base)
-    if key.startswith("lora_unet_"):
-        return key
-
-    # Handle already-prefixed keys
-    if key.startswith("diffusion_model."):
-        return key
-
-    # Absorb now-invalid transformer. prefix into diffusion_model. for compatibility
-    if key.startswith("transformer."):
-        return f"diffusion_model.{key[12:]}"
-
-    # Handle known Diffusers UNet blocks
-    if any(key.startswith(p) for p in ["down_blocks", "up_blocks", "mid_block", "conv_in", "conv_out", "time_embedding", "class_embedding"]):
-        return f"diffusion_model.{key}"
-
-    # Default fallback
-    return f"diffusion_model.{key}"
-
-
-def _get_common_inputs():
-    return [
-        io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
-        io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-        io.Boolean.Input("chunk_large_layers", default=False,
-                        tooltip="Split large fused layers (QKV, MLP) into chunks"),
-        io.Float.Input("clamp_quantile", default=0.99, min=0.5, max=1.0, step=0.01,
-                      tooltip="Clamp outlier singular values"),
-        io.Float.Input("min_diff", default=0.0, min=0.0, max=1.0, step=0.001,
-                      tooltip="Skip layers with max difference below this"),
-        io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-        io.String.Input("output_filename", default="extracted_lora"),
-        io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
-        io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
-        io.String.Input("skip_patterns", default="", multiline=True,
-                       tooltip="Patterns for layers to skip (regex or glob depending on glob_skip_patterns)"),
-        io.Boolean.Input("glob_skip_patterns", default=False,
-                        tooltip="When True, skip_patterns use glob syntax (* = any sequence, ? = any char, dots are literal). "
-                                "When False (default), patterns are Python regex matched as substrings."),
-    ]
-
-
-class DoRAExtractFixed(io.ComfyNode):
-    """Extract DoRA with fixed rank."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRAExtractFixed",
-            display_name="DoRA Extract (Fixed Rank)",
-            category="ModelUtils/DoRA",
-            description="Extract DoRA with specified fixed rank for each layer type.",
-            inputs=[
-                *_get_model_inputs(),
-                io.Int.Input("linear_dim", default=64, min=1, max=16384,
-                            tooltip="Rank for linear/attention layers"),
-                io.Int.Input("conv_dim", default=32, min=1, max=16384,
-                            tooltip="Rank for conv layers"),
-                io.Int.Input("svd_niter", default=2, min=0, max=10,
-                            tooltip="SVD power iterations (higher = more accurate but slower)"),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, linear_dim, conv_dim, svd_niter, chunk_large_layers,
-                clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_from_files(
-            model_a_path, model_b_path, "fixed", linear_dim, conv_dim,
-            device, save_dtype, output_path, linear_dim, conv_dim,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers, svd_niter,
-            lazy_load, force_clear_cache, glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)
-
-
-class DoRAExtractRatio(io.ComfyNode):
-    """Extract DoRA with singular value ratio threshold."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRAExtractRatio",
-            display_name="DoRA Extract (Ratio)",
-            category="ModelUtils/DoRA",
-            description="Keep singular values > max(S) / ratio.",
-            inputs=[
-                *_get_model_inputs(),
-                io.Float.Input("linear_ratio", default=2.0, min=1.0, max=100.0, step=0.1,
-                              tooltip="Ratio threshold for linear layers (higher = more SVs kept)"),
-                io.Float.Input("conv_ratio", default=2.0, min=1.0, max=100.0, step=0.1,
-                              tooltip="Ratio threshold for conv layers (higher = more SVs kept)"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=16384,
-                            tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=16384,
-                            tooltip="Maximum rank for conv layers"),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, linear_ratio, conv_ratio, linear_max_rank, conv_max_rank,
-                chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_from_files(
-            model_a_path, model_b_path, "ratio", linear_ratio, conv_ratio,
-            device, save_dtype, output_path, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache,
-            glob_skip_patterns=glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)
-
-
-class DoRAExtractQuantile(io.ComfyNode):
-    """Extract DoRA with cumulative singular value quantile."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRAExtractQuantile",
-            display_name="DoRA Extract (Quantile)",
-            category="ModelUtils/DoRA",
-            description="Keep enough singular values to reach target percentage.",
-            inputs=[
-                *_get_model_inputs(),
-                io.Float.Input("linear_quantile", default=0.9, min=0.0, max=1.0, step=0.01,
-                              tooltip="Target cumulative % for linear layers"),
-                io.Float.Input("conv_quantile", default=0.9, min=0.0, max=1.0, step=0.01,
-                              tooltip="Target cumulative % for conv layers"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=16384,
-                            tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=16384,
-                            tooltip="Maximum rank for conv layers"),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, linear_quantile, conv_quantile, linear_max_rank, conv_max_rank,
-                chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_from_files(
-            model_a_path, model_b_path, "quantile", linear_quantile, conv_quantile,
-            device, save_dtype, output_path, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache,
-            glob_skip_patterns=glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)
-
-
-class DoRAExtractKnee(io.ComfyNode):
-    """Extract DoRA with automatic knee detection."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRAExtractKnee",
-            display_name="DoRA Extract (Knee Detection)",
-            category="ModelUtils/DoRA",
-            description="Automatically find optimal rank using knee detection on singular value curve.",
-            inputs=[
-                *_get_model_inputs(),
-                io.Combo.Input("knee_method", options=["sv_knee", "sv_cumulative_knee"],
-                              default="sv_knee", tooltip="Knee detection method"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=16384,
-                            tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=16384,
-                            tooltip="Maximum rank for conv layers"),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, knee_method, linear_max_rank, conv_max_rank,
-                chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_from_files(
-            model_a_path, model_b_path, knee_method, 0, 0,
-            device, save_dtype, output_path, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache,
-            glob_skip_patterns=glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)
-
-
-class DoRAExtractFrobenius(io.ComfyNode):
-    """Extract DoRA preserving target fraction of Frobenius norm."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRAExtractFrobenius",
-            display_name="DoRA Extract (Frobenius)",
-            category="ModelUtils/DoRA",
-            description="Preserve target fraction of Frobenius norm.",
-            inputs=[
-                *_get_model_inputs(),
-                io.Float.Input("linear_target", default=0.9, min=0.0, max=1.0, step=0.01,
-                              tooltip="Target Frobenius norm fraction for linear"),
-                io.Float.Input("conv_target", default=0.9, min=0.0, max=1.0, step=0.01,
-                              tooltip="Target Frobenius norm fraction for conv"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=16384,
-                            tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=16384,
-                            tooltip="Maximum rank for conv layers"),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, linear_target, conv_target, linear_max_rank, conv_max_rank,
-                chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_from_files(
-            model_a_path, model_b_path, "sv_fro", linear_target, conv_target,
-            device, save_dtype, output_path, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache,
-            glob_skip_patterns=glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)

nodes/dora_learned_wd.py

@@ -1,790 +0,0 @@
-"""
-Learned DoRA Extraction via Optimization.
-
-Extracts Weight-Decomposed Low-Rank Adapters using gradient descent optimization
-to minimize the reconstruction MSE loss, matching the true non-linear DoRA formulation.
-Uses analytical SVD purely as the initialization seed.
-"""
-import fnmatch
-import os
-import re
-import math
-import torch
-import torch.linalg as linalg
-import folder_paths
-import comfy.utils
-from tqdm import tqdm
-from comfy_api.latest import io
-from .device_utils import (
-    estimate_model_size, prepare_for_large_operation,
-    cleanup_after_operation
-)
-
-from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned, IncrementalSafetensorsWriter
-
-# Import the existing SV ranking and chunking logic so we don't have to duplicate it
-from .dora_extract_wd import (
-    _compute_rank, _svd_extract_linear_lowrank, _svd_extract_linear,
-    _svd_extract_conv, _detect_fused_layer, _compile_patterns,
-    _matches_any_pattern, _build_lora_output_path, _get_model_inputs,
-    _format_lora_key, _get_common_inputs
-)
-
-
-def _adaptive_lr_update_cosine(curr_lr: float, improved: bool, worse_loss_counter: int, iteration: int, tensor_shape: tuple, min_lr: float, early_stop_stall: int, lr_factor: float = 0.95, lr_cooldown: int = 1) -> tuple:
-    M, N = tensor_shape
-    shape_ratio = abs(M - N) / max(M, N) if max(M, N) > 0 else 0
-
-    t = min(worse_loss_counter / max(early_stop_stall, 1), 1.0)
-    u_factor = (1 + math.cos(2 * math.pi * t)) / 2
-
-    if improved:
-        base_boost = 1.25
-        distance = base_boost - 1.0
-        scaled_distance = distance * (1.0 + 0.5 * shape_ratio)
-        boost_mult = 1.0 + scaled_distance * (1.0 - u_factor)
-        return min(curr_lr * boost_mult, 100.0), True
-    else:
-        cooldown = max(lr_cooldown, 1)
-        if iteration % cooldown != 0:
-            return curr_lr, False
-
-        min_decay = lr_factor - 0.03 * shape_ratio
-        max_decay = 0.995
-        decay_mult = min_decay + (max_decay - min_decay) * u_factor
-        return max(curr_lr * decay_mult, min_lr), True
-
-
-def _compute_shape_aware_plateau_params(M: int, N: int, lr_patience: int, lr_factor: float, lr_cooldown: int, lr_shape_influence: float = 1.0) -> tuple:
-    if lr_shape_influence > 0:
-        aspect_ratio = max(M, N) / min(M, N) if min(M, N) > 0 else 1.0
-        ar_factor = math.sqrt(aspect_ratio)
-        blend = lr_shape_influence
-
-        effective_patience = lr_patience
-        raw_factor = lr_factor
-        aggressive_factor = raw_factor**ar_factor
-        effective_factor = raw_factor + (aggressive_factor - raw_factor) * blend
-        effective_cooldown = lr_cooldown
-    else:
-        effective_patience = lr_patience
-        effective_factor = lr_factor
-        effective_cooldown = lr_cooldown
-
-    return effective_patience, effective_factor, effective_cooldown
-
-
-def _optimize_dora(
-    tensor_base: torch.Tensor,
-    tensor_ft: torch.Tensor,
-    init_down: torch.Tensor,
-    init_up: torch.Tensor,
-    optimize_iters: int,
-    learning_rate: float,
-    optimizer_type: str = "prodigy",
-    lr_schedule: str = "plateau",
-    lr_patience: int = 2,
-    lr_factor: float = 0.9,
-    lr_cooldown: int = 2,
-    early_stop_loss: float = 1e-6,
-    early_stop_stall: int = 2000,
-    early_stop_lr: float = 9.01e-9,
-    layer_name: str = "Layer",
-) -> tuple:
-    """
-    Run gradient descent to optimize lora_down and lora_up against the actual DoRA formula.
-
-    Args:
-        tensor_base: Base model weight
-        tensor_ft: Finetuned model weight
-        init_down: SVD-initialized lora_down matrix
-        init_up: SVD-initialized lora_up matrix
-        optimize_iters: Number of iterations for AdamW
-        learning_rate: Learning rate for AdamW
-        optimizer_type: Optimizer choice (adamw, radam, prodigy)
-        lr_schedule: LR Schedule choice (exponential, plateau, adaptive)
-        early_stop_loss: Target loss to stop optimizing
-        early_stop_stall: Number of non-improving iterations before stopping
-
-    Returns:
-        (optimized_down, optimized_up)
-    """
-    if optimize_iters <= 0:
-        return init_down, init_up
-
-    is_conv = tensor_base.ndim == 4
-    if is_conv:
-        out_ch, in_ch, kh, kw = tensor_base.shape
-        flat_base = tensor_base.reshape(out_ch, -1)
-        flat_ft = tensor_ft.reshape(out_ch, -1)
-
-        # Flatten init matrices for matmul: (out, rank) @ (rank, in*k*k)
-        rank = init_down.shape[0]
-        flat_up = init_up.reshape(out_ch, rank)
-        flat_down = init_down.reshape(rank, -1)
-    else:
-        flat_base = tensor_base
-        flat_ft = tensor_ft
-        flat_up = init_up
-        flat_down = init_down
-
-    # Magnitude of the finetuned model (fixed DoRA scale)
-    m = torch.linalg.norm(flat_ft, dim=1, keepdim=True).detach()
-    eps = 1e-8
-
-    # Set up learnable parameters
-    with torch.inference_mode(False), torch.enable_grad():
-        # Clone inference tensors to normal tensors for autograd
-        flat_base_opt = flat_base.clone()
-        flat_ft_opt = flat_ft.clone()
-        m_opt = m.clone()
-
-        # Free original tensors to save memory
-        del flat_base, flat_ft, m
-
-        B = torch.nn.Parameter(flat_up.clone())
-        A = torch.nn.Parameter(flat_down.clone())
-
-        opt_type = optimizer_type.lower()
-        if opt_type == "radam":
-            optimizer = torch.optim.RAdam([B, A], lr=learning_rate)
-        elif opt_type == "prodigy":
-            try:
-                from prodigyplus.prodigy_plus_schedulefree import ProdigyPlusScheduleFree
-                optimizer = ProdigyPlusScheduleFree([B, A], lr=learning_rate, use_schedulefree=False)
-            except ImportError:
-                print("ProdigyPlusScheduleFree optimizer not found. Falling back to AdamW. To install: pip install prodigyplus")
-                optimizer = torch.optim.AdamW([B, A], lr=learning_rate)
-        else:
-            optimizer = torch.optim.AdamW([B, A], lr=learning_rate)
-
-        # Learning rate scheduling and early stopping state
-        best_loss = float('inf')
-        worse_loss_counter = 0
-        plateau_counter = 0
-        cooldown_counter = 0
-        current_lr = learning_rate
-        schedule_type = lr_schedule.lower()
-
-        M_dim, N_dim = flat_base_opt.shape[0], flat_base_opt.shape[1]
-        eff_patience, eff_factor, eff_cooldown = _compute_shape_aware_plateau_params(M_dim, N_dim, lr_patience, lr_factor, lr_cooldown)
-
-        # Optimization loop
-        opt_pbar = tqdm(range(optimize_iters), desc=f"  └─ Optimizing {layer_name}", leave=False, dynamic_ncols=True, mininterval=0.5)
-        for i in opt_pbar:
-            optimizer.zero_grad()
-
-            # 1. Reconstruct directional weight: W_dir = W_base + B @ A
-            w_approx = flat_base_opt + (B @ A)
-
-            # 2. Normalize directional weight
-            approx_norm = torch.linalg.norm(w_approx, dim=1, keepdim=True) + eps
-
-            # 3. Scale by target magnitude
-            w_reconstructed = m_opt * (w_approx / approx_norm)
-
-            # 4. Compute MSE loss against actual finetuned weight
-            loss = torch.nn.functional.mse_loss(w_reconstructed, flat_ft_opt)
-            loss_val = loss.item()
-
-            if i == 0:
-                initial_loss = loss_val
-                best_loss = loss_val
-
-            loss.backward()
-            optimizer.step()
-
-            # --- Early Stopping & Schedulers ---
-            improved = False
-            if loss_val < best_loss * 0.9999: # 0.01% improvement threshold
-                best_loss = loss_val
-                worse_loss_counter = 0
-                plateau_counter = 0
-                improved = True
-            else:
-                worse_loss_counter += 1
-                plateau_counter += 1
-
-            if loss_val <= early_stop_loss:
-                break
-
-            if worse_loss_counter > early_stop_stall:
-                break
-
-            if current_lr <= early_stop_lr:
-                break
-
-            # LR Schedulers
-            if schedule_type == "adaptive":
-                current_lr, updated = _adaptive_lr_update_cosine(
-                    current_lr, improved, worse_loss_counter, i, (M_dim, N_dim), early_stop_lr, early_stop_stall, lr_factor, lr_cooldown
-                )
-                if updated:
-                    for param_group in optimizer.param_groups:
-                        param_group['lr'] = current_lr
-            elif schedule_type == "plateau":
-                if cooldown_counter > 0:
-                    cooldown_counter -= 1
-                elif plateau_counter >= max(eff_patience, 1): # Decay if patience threshold reached
-                    current_lr = max(current_lr * eff_factor, early_stop_lr)
-                    for param_group in optimizer.param_groups:
-                        param_group['lr'] = current_lr
-                    plateau_counter = 0 # reset counter after decay
-                    cooldown_counter = eff_cooldown
-            elif schedule_type == "exponential":
-                current_lr = max(current_lr * 0.99, early_stop_lr)
-                for param_group in optimizer.param_groups:
-                    param_group['lr'] = current_lr
-
-            if i % 10 == 0 or i == optimize_iters - 1:
-                if schedule_type == "plateau":
-                    opt_pbar.set_postfix({"loss": f"{loss_val:.4e}", "lr": f"{current_lr:.2e}", "plat": f"{plateau_counter}/{eff_patience}"}, refresh=False)
-                else:
-                    opt_pbar.set_postfix({"loss": f"{loss_val:.4e}", "lr": f"{current_lr:.2e}", "stall": f"{worse_loss_counter}"}, refresh=False)
-
-        final_loss = loss.item()
-        opt_pbar.close()
-        print(f"  └─ {layer_name} | Loss: {initial_loss:.6f} -> {final_loss:.6f} (stopped at iter {i+1})")
-
-        final_up = B.detach()
-        final_down = A.detach()
-
-    # Reshape back to original dimensions
-    if is_conv:
-        final_up = final_up.reshape(out_ch, rank, 1, 1)
-        final_down = final_down.reshape(rank, in_ch, kh, kw)
-
-    return final_down, final_up
-
-
-def _extract_chunked_learned_layer(
-    tensor_base: torch.Tensor,
-    tensor_ft: torch.Tensor,
-    num_chunks: int,
-    mode: str,
-    mode_param: float,
-    device: str,
-    max_rank: int = None,
-    optimize_iters: int = 500,
-    learning_rate: float = 1.0,
-    optimizer_type: str = "prodigy",
-    lr_schedule: str = "plateau",
-    lr_patience: int = 2,
-    lr_factor: float = 0.9,
-    lr_cooldown: int = 2,
-    early_stop_loss: float = 1e-6,
-    early_stop_stall: int = 2000,
-    early_stop_lr: float = 9.01e-9,
-    layer_name: str = "Layer",
-) -> tuple:
-    """Extract Learned DoRA from fused layer by chunking."""
-    out_dim, in_dim = tensor_base.shape
-    chunk_size = out_dim // num_chunks
-
-    all_lora_up = []
-    all_lora_down = []
-
-    for i in range(num_chunks):
-        chunk_base = tensor_base[i * chunk_size:(i + 1) * chunk_size, :]
-        chunk_ft = tensor_ft[i * chunk_size:(i + 1) * chunk_size, :]
-
-        norm_base = torch.linalg.norm(chunk_base, dim=1, keepdim=True)
-        norm_ft = torch.linalg.norm(chunk_ft, dim=1, keepdim=True)
-        eps = 1e-8
-
-        w_dir = chunk_ft * (norm_base / (norm_ft + eps))
-        weight_diff = w_dir - chunk_base
-
-        try:
-            result, mode_str = _svd_extract_linear(weight_diff, mode, mode_param, device, max_rank)
-            if mode_str == "full":
-                return None, None, 0
-            lora_down, lora_up, _ = result
-
-            # Apply learned optimization
-            lora_down, lora_up = _optimize_dora(
-                chunk_base, chunk_ft, lora_down, lora_up, optimize_iters, learning_rate,
-                optimizer_type, lr_schedule, lr_patience, lr_factor, lr_cooldown,
-                early_stop_loss, early_stop_stall, early_stop_lr,
-                layer_name=f"{layer_name} (Chunk {i+1}/{num_chunks})"
-            )
-
-            all_lora_down.append(lora_down)
-            all_lora_up.append(lora_up)
-        except Exception:
-            return None, None, 0
-
-    combined_lora_up = torch.cat(all_lora_up, dim=0)
-    combined_lora_down = all_lora_down[0]
-    rank = combined_lora_down.shape[0]
-
-    return combined_lora_up, combined_lora_down, rank
-
-
-def extract_dora_learned_from_files(
-    model_a_path: str,
-    model_b_path: str,
-    mode: str,
-    linear_param: float,
-    conv_param: float,
-    device: str,
-    save_dtype: str,
-    output_path: str,
-    optimize_iters: int = 500,
-    learning_rate: float = 1.0,
-    optimizer_type: str = "prodigy",
-    lr_schedule: str = "plateau",
-    lr_patience: int = 2,
-    lr_factor: float = 0.9,
-    lr_cooldown: int = 2,
-    early_stop_loss: float = 1e-6,
-    early_stop_stall: int = 2000,
-    early_stop_lr: float = 9.01e-9,
-    linear_max_rank: int = None,
-    conv_max_rank: int = None,
-    clamp_quantile: float = 0.99,
-    min_diff: float = 0.0,
-    skip_patterns_str: str = "",
-    mismatch_mode: str = "skip",
-    chunk_large_layers: bool = True,
-    svd_niter: int = 2,
-    lazy_load: bool = True,
-    force_clear_cache: bool = True,
-    glob_skip_patterns: bool = False,
-) -> None:
-    """
-    Extract Learned DoRA from difference between two models.
-    """
-    save_torch_dtype = {
-        "fp32": torch.float32,
-        "fp16": torch.float16,
-        "bf16": torch.bfloat16,
-    }.get(save_dtype, torch.float16)
-
-    skip_patterns = _compile_patterns(skip_patterns_str, glob_mode=glob_skip_patterns)
-
-    total_size_gb = estimate_model_size(model_a_path) + estimate_model_size(model_b_path)
-    print(f"[Learned DoRA Extract] Preparing memory for {total_size_gb:.2f}GB operation...")
-    prepare_for_large_operation(total_size_gb * 1.5, torch.device(device))
-
-    handler_a = MemoryEfficientSafeOpen(model_a_path, low_memory=lazy_load)
-    handler_b = MemoryEfficientSafeOpen(model_b_path, low_memory=lazy_load)
-
-    try:
-        keys_a = set(handler_a.keys())
-        keys_b = set(handler_b.keys())
-        weight_keys = [k for k in keys_a if k.endswith(".weight")]
-        pbar = comfy.utils.ProgressBar(len(weight_keys))
-        stats = {"extracted": 0, "full": 0, "skipped": 0, "chunked": 0}
-
-        def _process_layer(key):
-            lora_name = _format_lora_key(key)
-
-            if _matches_any_pattern(key, skip_patterns, glob_mode=glob_skip_patterns):
-                return "skipped", None
-
-            use_pinned = device == 'cuda'
-
-            if key not in keys_b:
-                if mismatch_mode == "skip":
-                    return "skipped", None
-                if mismatch_mode == "error":
-                    raise ValueError(f"Key {key} not found in model B")
-
-                # Cannot extract true DoRA without base. Fallback to diff.
-                cpu_a = handler_a.get_tensor(key)
-                tensor_a = transfer_to_gpu_pinned(cpu_a, device, torch.float32) if use_pinned else cpu_a.to(device=device, dtype=torch.float32)
-                del cpu_a
-                weight_diff = tensor_a
-                tensor_base = torch.zeros_like(tensor_a)
-                tensor_ft = tensor_a
-                dora_scale = None
-            else:
-                cpu_a = handler_a.get_tensor(key)
-                cpu_b = handler_b.get_tensor(key)
-
-                if use_pinned:
-                    tensor_a = transfer_to_gpu_pinned(cpu_a, device, torch.float32)
-                    tensor_b = transfer_to_gpu_pinned(cpu_b, device, torch.float32)
-                else:
-                    tensor_a = cpu_a.to(device=device, dtype=torch.float32)
-                    tensor_b = cpu_b.to(device=device, dtype=torch.float32)
-                del cpu_a, cpu_b
-
-                if tensor_a.shape != tensor_b.shape:
-                    if mismatch_mode == "skip":
-                        del tensor_a, tensor_b
-                        return "skipped", None
-                    if mismatch_mode == "error":
-                        raise ValueError(f"Shape mismatch for {key}: {tensor_a.shape} vs {tensor_b.shape}")
-
-                    weight_diff = tensor_a
-                    tensor_base = torch.zeros_like(tensor_a)
-                    tensor_ft = tensor_a
-                    dora_scale = None
-                    del tensor_b
-                else:
-                    tensor_base = tensor_b
-                    tensor_ft = tensor_a
-
-                    is_conv_dim = tensor_a.ndim == 4
-                    if is_conv_dim:
-                        out_ch = tensor_a.shape[0]
-                        flat_a = tensor_a.reshape(out_ch, -1)
-                        flat_b = tensor_b.reshape(out_ch, -1)
-                        norm_a = torch.linalg.norm(flat_a, dim=1, keepdim=True)
-                        norm_b = torch.linalg.norm(flat_b, dim=1, keepdim=True)
-                    else:
-                        norm_a = torch.linalg.norm(tensor_a, dim=1, keepdim=True)
-                        norm_b = torch.linalg.norm(tensor_b, dim=1, keepdim=True)
-
-                    eps = 1e-8
-                    norm_ratio = norm_b / (norm_a + eps)
-                    if is_conv_dim:
-                        norm_ratio = norm_ratio.view(-1, 1, 1, 1)
-                        dora_scale = norm_a.view(-1, 1, 1, 1)
-                    else:
-                        dora_scale = norm_a
-
-                    w_dir = tensor_a * norm_ratio
-                    weight_diff = w_dir - tensor_b
-                    del w_dir
-
-            if min_diff > 0 and weight_diff.abs().max() < min_diff:
-                del weight_diff, tensor_base, tensor_ft
-                return "skipped", None
-
-            if weight_diff.ndim < 2:
-                del weight_diff, tensor_base, tensor_ft
-                return "skipped", None
-
-            is_conv = weight_diff.ndim == 4
-            layer_results = {}
-
-            try:
-                # 1. Run Analytical SVD (Initialization)
-                if is_conv:
-                    result, mode_str = _svd_extract_conv(
-                        weight_diff, mode, conv_param, device, conv_max_rank, clamp_quantile
-                    )
-                else:
-                    result, mode_str = _svd_extract_linear(
-                        weight_diff, mode, linear_param, device, linear_max_rank, clamp_quantile, svd_niter
-                    )
-
-                # 2. Run Optimization (Learned Rounding)
-                if mode_str != "full" and dora_scale is not None:
-                    lora_down, lora_up, _ = result
-                    lora_down, lora_up = _optimize_dora(
-                        tensor_base, tensor_ft, lora_down, lora_up, optimize_iters, learning_rate,
-                        optimizer_type, lr_schedule, lr_patience, lr_factor, lr_cooldown,
-                        early_stop_loss, early_stop_stall, early_stop_lr,
-                        layer_name=lora_name
-                    )
-                    result = (lora_down, lora_up, _)
-
-            except Exception as e:
-                if chunk_large_layers and not is_conv:
-                    num_chunks = _detect_fused_layer(key, weight_diff.shape)
-                    if num_chunks > 1:
-                        print(f"[Learned DoRA] Chunked: {key} ({num_chunks} chunks)")
-                        lora_up, lora_down, rank = _extract_chunked_learned_layer(
-                            tensor_base, tensor_ft, num_chunks, mode, linear_param, device, linear_max_rank, optimize_iters, learning_rate,
-                            optimizer_type, lr_schedule, lr_patience, lr_factor, lr_cooldown,
-                            early_stop_loss, early_stop_stall, early_stop_lr,
-                            layer_name=lora_name
-                        )
-                        if lora_up is not None:
-                            layer_results[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
-                            layer_results[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
-                            layer_results[f"{lora_name}.alpha"] = torch.tensor(rank, dtype=save_torch_dtype)
-                            if dora_scale is not None:
-                                layer_results[f"{lora_name}.dora_scale"] = dora_scale.to(save_torch_dtype).cpu().contiguous()
-                            del weight_diff, tensor_base, tensor_ft
-                            return "chunked", layer_results
-
-                print(f"[Learned DoRA] Failed: {key}: {e}")
-                del weight_diff, tensor_base, tensor_ft
-                return "skipped", None
-
-            if mode_str == "full":
-                layer_results[f"{lora_name}.diff"] = weight_diff.to(save_torch_dtype).cpu().contiguous()
-                status = "full"
-            else:
-                lora_down, lora_up, _ = result
-                rank = lora_down.shape[0]
-                layer_results[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
-                layer_results[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
-                layer_results[f"{lora_name}.alpha"] = torch.tensor(rank, dtype=save_torch_dtype)
-                if dora_scale is not None:
-                    layer_results[f"{lora_name}.dora_scale"] = dora_scale.to(save_torch_dtype).cpu().contiguous()
-                status = "extracted"
-
-            del weight_diff, tensor_base, tensor_ft
-            return status, layer_results
-
-        writer = IncrementalSafetensorsWriter(output_path)
-        writer.__enter__()
-        try:
-            for key in tqdm(weight_keys, desc="Extracting Learned DoRA", unit="layers"):
-                status, layer_sd = _process_layer(key)
-                stats[status] += 1
-                if layer_sd:
-                    writer.write_dict(layer_sd)
-
-                if force_clear_cache:
-                    import gc
-                    gc.collect()
-                    if torch.cuda.is_available():
-                        torch.cuda.empty_cache()
-
-                pbar.update(1)
-        finally:
-            writer.__exit__(None, None, None)
-
-        print(f"[Learned DoRA] Done: {stats['extracted']} extracted, {stats['chunked']} chunked, "
-              f"{stats['full']} full, {stats['skipped']} skipped")
-
-    finally:
-        handler_a.__exit__(None, None, None)
-        handler_b.__exit__(None, None, None)
-        cleanup_after_operation()
-
-
-# =============================================================================
-# Node Definitions
-# =============================================================================
-
-def _get_learned_inputs():
-    return [
-        io.Int.Input("optimize_iters", default=500, min=0, max=10000,
-                    tooltip="Number of gradient descent iterations (0 = skip optimization)"),
-        io.Float.Input("learning_rate", default=1.0, min=0.0001, max=100.0, step=0.001,
-                      tooltip="Base learning rate. AdamW prefers ~0.01, Prodigy prefers ~1.0"),
-        io.Combo.Input("optimizer", options=["prodigy", "adamw", "radam"], default="prodigy",
-                      tooltip="Optimization algorithm"),
-        io.Combo.Input("lr_schedule", options=["plateau", "adaptive", "exponential", "constant"], default="plateau",
-                      tooltip="Learning rate scheduling strategy"),
-        io.Int.Input("lr_patience", default=2, min=0, max=1000, tooltip="Steps to wait before decaying LR (Plateau)"),
-        io.Float.Input("lr_factor", default=0.9, min=0.01, max=1.0, step=0.01, tooltip="Factor to decay LR by (Plateau)"),
-        io.Int.Input("lr_cooldown", default=2, min=0, max=1000, tooltip="Steps to wait after decay before checking again (Plateau)"),
-        io.Float.Input("early_stop_loss", default=1e-6, min=0.0, max=1.0, step=1e-9,
-                      tooltip="Stop early if MSE loss drops below this value"),
-        io.Int.Input("early_stop_stall", default=2000, min=0, max=10000,
-                      tooltip="Stop early if loss doesn't improve for this many iterations"),
-        io.Float.Input("early_stop_lr", default=9.01e-9, min=0.0, max=1.0, step=1e-10,
-                      tooltip="Stop early if learning rate drops below this value"),
-    ]
-
-
-class DoRALearnedExtractFixed(io.ComfyNode):
-    """Extract Learned DoRA with fixed rank."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRALearnedExtractFixed",
-            display_name="DoRA Learned Extract (Fixed Rank) (WIP)",
-            category="ModelUtils/DoRA",
-            description="Extract DoRA via MSE Loss minimization with fixed rank.",
-            is_experimental=True,
-            inputs=[
-                *_get_model_inputs(),
-                io.Int.Input("linear_dim", default=64, min=1, max=16384, tooltip="Rank for linear/attention layers"),
-                io.Int.Input("conv_dim", default=32, min=1, max=16384, tooltip="Rank for conv layers"),
-                *_get_learned_inputs(),
-                io.Int.Input("svd_niter", default=2, min=0, max=10, tooltip="SVD iterations for initialization"),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, linear_dim, conv_dim, optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, svd_niter, chunk_large_layers,
-                clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_learned_from_files(
-            model_a_path, model_b_path, "fixed", linear_dim, conv_dim,
-            device, save_dtype, output_path, optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, linear_dim, conv_dim,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers, svd_niter,
-            lazy_load, force_clear_cache, glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)
-
-
-class DoRALearnedExtractRatio(io.ComfyNode):
-    """Extract Learned DoRA with singular value ratio threshold."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRALearnedExtractRatio",
-            display_name="DoRA Learned Extract (Ratio) (WIP)",
-            category="ModelUtils/DoRA",
-            description="Learned extraction keeping SVs > max(S) / ratio.",
-            is_experimental=True,
-            inputs=[
-                *_get_model_inputs(),
-                io.Float.Input("linear_ratio", default=2.0, min=1.0, max=100.0, step=0.1),
-                io.Float.Input("conv_ratio", default=2.0, min=1.0, max=100.0, step=0.1),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=16384),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=16384),
-                *_get_learned_inputs(),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, linear_ratio, conv_ratio, linear_max_rank, conv_max_rank,
-                optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, chunk_large_layers, clamp_quantile, min_diff, mismatch_mode,
-                output_filename, save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_learned_from_files(
-            model_a_path, model_b_path, "ratio", linear_ratio, conv_ratio,
-            device, save_dtype, output_path, optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache, glob_skip_patterns=glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)
-
-
-class DoRALearnedExtractQuantile(io.ComfyNode):
-    """Extract Learned DoRA with cumulative singular value quantile."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRALearnedExtractQuantile",
-            display_name="DoRA Learned Extract (Quantile) (WIP)",
-            category="ModelUtils/DoRA",
-            description="Learned extraction keeping enough SVs to reach target percentage.",
-            is_experimental=True,
-            inputs=[
-                *_get_model_inputs(),
-                io.Float.Input("linear_quantile", default=0.9, min=0.0, max=1.0, step=0.01),
-                io.Float.Input("conv_quantile", default=0.9, min=0.0, max=1.0, step=0.01),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=16384),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=16384),
-                *_get_learned_inputs(),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, linear_quantile, conv_quantile, linear_max_rank, conv_max_rank,
-                optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, chunk_large_layers, clamp_quantile, min_diff, mismatch_mode,
-                output_filename, save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_learned_from_files(
-            model_a_path, model_b_path, "quantile", linear_quantile, conv_quantile,
-            device, save_dtype, output_path, optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache, glob_skip_patterns=glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)
-
-
-class DoRALearnedExtractKnee(io.ComfyNode):
-    """Extract Learned DoRA with automatic knee detection."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRALearnedExtractKnee",
-            display_name="DoRA Learned Extract (Knee Detection) (WIP)",
-            category="ModelUtils/DoRA",
-            description="Learned extraction automatically finding optimal rank using knee detection.",
-            is_experimental=True,
-            inputs=[
-                *_get_model_inputs(),
-                io.Combo.Input("knee_method", options=["sv_knee", "sv_cumulative_knee"], default="sv_knee"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=16384),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=16384),
-                *_get_learned_inputs(),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, knee_method, linear_max_rank, conv_max_rank,
-                optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, chunk_large_layers, clamp_quantile, min_diff, mismatch_mode,
-                output_filename, save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_learned_from_files(
-            model_a_path, model_b_path, knee_method, 0, 0,
-            device, save_dtype, output_path, optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache, glob_skip_patterns=glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)
-
-
-class DoRALearnedExtractFrobenius(io.ComfyNode):
-    """Extract Learned DoRA preserving target fraction of Frobenius norm."""
-
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="DoRALearnedExtractFrobenius",
-            display_name="DoRA Learned Extract (Frobenius) (WIP)",
-            category="ModelUtils/DoRA",
-            description="Learned extraction preserving target fraction of Frobenius norm.",
-            is_experimental=True,
-            inputs=[
-                *_get_model_inputs(),
-                io.Float.Input("linear_target", default=0.9, min=0.0, max=1.0, step=0.01),
-                io.Float.Input("conv_target", default=0.9, min=0.0, max=1.0, step=0.01),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=16384),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=16384),
-                *_get_learned_inputs(),
-                *_get_common_inputs(),
-            ],
-            outputs=[io.String.Output(display_name="output_path")],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, model_a, model_b, linear_target, conv_target, linear_max_rank, conv_max_rank,
-                optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, chunk_large_layers, clamp_quantile, min_diff, mismatch_mode,
-                output_filename, save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
-
-        model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
-        model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
-
-        extract_dora_learned_from_files(
-            model_a_path, model_b_path, "sv_fro", linear_target, conv_target,
-            device, save_dtype, output_path, optimize_iters, learning_rate, optimizer, lr_schedule, lr_patience, lr_factor, lr_cooldown, early_stop_loss, early_stop_stall, early_stop_lr, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache, glob_skip_patterns=glob_skip_patterns
-        )
-
-        return io.NodeOutput(output_path)

nodes/lora_extract_svd.py

@@ -4,7 +4,6 @@ LoRA Extraction via SVD decomposition.
 Native implementation of Low-Rank Adapter extraction with multiple rank selection modes.
 No external dependencies required.
 """
-import fnmatch
 import os
 import re
 import torch
@@ -13,12 +12,13 @@ import folder_paths
 import comfy.utils
 from tqdm import tqdm
 from comfy_api.latest import io
+from safetensors.torch import save_file
 from .device_utils import (
     estimate_model_size, prepare_for_large_operation,
     cleanup_after_operation, get_device_capabilities
 )
 
-from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned, IncrementalSafetensorsWriter
+from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned
 
 
 
@@ -399,22 +399,14 @@ def _extract_chunked_layer(
 # Pattern Matching
 # =============================================================================
 
-def _compile_patterns(pattern_string: str, glob_mode: bool = False) -> list:
-    """Compile patterns from whitespace-separated string.
-
-    In regex mode (default) returns compiled re.Pattern objects.
-    In glob mode returns plain strings; fnmatch handles matching.
-    """
+def _compile_patterns(pattern_string: str) -> list:
+    """Compile regex patterns from whitespace-separated string."""
     if not pattern_string or not pattern_string.strip():
         return []
     patterns = []
     for p in pattern_string.split():
         p = p.strip()
-        if not p:
-            continue
-        if glob_mode:
-            patterns.append(p)
-        else:
+        if p:
             try:
                 patterns.append(re.compile(p))
             except re.error:
@@ -422,15 +414,8 @@ def _compile_patterns(pattern_string: str, glob_mode: bool = False) -> list:
     return patterns
 
 
-def _matches_any_pattern(key: str, patterns: list, glob_mode: bool = False) -> bool:
-    """Check if key matches any pattern.
-
-    Glob mode: each pattern is matched as a substring glob using fnmatch
-    (dots are literal, * matches any sequence of characters).
-    Regex mode: each pattern is a compiled re.Pattern, matched via search().
-    """
-    if glob_mode:
-        return any(fnmatch.fnmatch(key, f"*{p}*") for p in patterns)
+def _matches_any_pattern(key: str, patterns: list) -> bool:
+    """Check if key matches any pattern."""
     return any(p.search(key) for p in patterns)
 
 
@@ -446,7 +431,6 @@ def extract_lora_from_files(
     conv_param: float,
     device: str,
     save_dtype: str,
-    output_path: str,
     linear_max_rank: int = None,
     conv_max_rank: int = None,
     clamp_quantile: float = 0.99,
@@ -457,10 +441,9 @@ def extract_lora_from_files(
     svd_niter: int = 2,
     lazy_load: bool = True,
     force_clear_cache: bool = True,
-    glob_skip_patterns: bool = False,
-) -> None:
+) -> dict[str, torch.Tensor]:
     """
-    Extract LoRA from difference between two models, writing incrementally to disk.
+    Extract LoRA from difference between two models.
 
     Args:
         model_a_path: Finetuned model path
@@ -470,7 +453,6 @@ def extract_lora_from_files(
         conv_param: Mode parameter for conv layers
         device: Computation device
         save_dtype: Output dtype
-        output_path: Full path to output safetensors file
         linear_max_rank: Max rank for linear layers
         conv_max_rank: Max rank for conv layers
         clamp_quantile: Quantile for weight clamping
@@ -478,16 +460,16 @@ def extract_lora_from_files(
         skip_patterns_str: Patterns for layers to skip
         mismatch_mode: How to handle mismatches
         chunk_large_layers: Enable chunked extraction
-        glob_skip_patterns: When True, treat skip_patterns as glob (* wildcard).
-                            When False (default), treat as Python regex.
     """
+    output_sd = {}
+
     save_torch_dtype = {
         "fp32": torch.float32,
         "fp16": torch.float16,
         "bf16": torch.bfloat16,
     }.get(save_dtype, torch.float16)
 
-    skip_patterns = _compile_patterns(skip_patterns_str, glob_mode=glob_skip_patterns)
+    skip_patterns = _compile_patterns(skip_patterns_str)
 
     # Prepare memory before heavy operation
     total_size_gb = estimate_model_size(model_a_path) + estimate_model_size(model_b_path)
@@ -507,7 +489,7 @@ def extract_lora_from_files(
         def _process_layer(key):
             lora_name = _format_lora_key(key)
 
-            if _matches_any_pattern(key, skip_patterns, glob_mode=glob_skip_patterns):
+            if _matches_any_pattern(key, skip_patterns):
                 return "skipped", None
 
             # Load tensors with pinned memory for CUDA
@@ -588,8 +570,9 @@ def extract_lora_from_files(
                             weight_diff, num_chunks, mode, linear_param, device, linear_max_rank
                         )
                         if lora_up is not None:
-                            layer_results[f"{lora_name}.lora_B.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
-                            layer_results[f"{lora_name}.lora_A.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
+                            layer_results[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
+                            layer_results[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
+                            layer_results[f"{lora_name}.alpha"] = torch.tensor(rank).to(save_torch_dtype)
                             del weight_diff
                             return "chunked", layer_results
 
@@ -603,31 +586,27 @@ def extract_lora_from_files(
                 status = "full"
             else:
                 lora_down, lora_up, _ = result
-                layer_results[f"{lora_name}.lora_A.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
-                layer_results[f"{lora_name}.lora_B.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
+                layer_results[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
+                layer_results[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
+                layer_results[f"{lora_name}.alpha"] = torch.tensor(lora_down.shape[0]).to(save_torch_dtype)
                 status = "extracted"
 
             del weight_diff
             return status, layer_results
 
-        writer = IncrementalSafetensorsWriter(output_path)
-        writer.__enter__()
-        try:
-            for key in tqdm(weight_keys, desc="Extracting LoRA", unit="layers"):
-                status, layer_sd = _process_layer(key)
-                stats[status] += 1
-                if layer_sd:
-                    writer.write_dict(layer_sd)
+        for key in tqdm(weight_keys, desc="Extracting LoRA", unit="layers"):
+            status, layer_sd = _process_layer(key)
+            stats[status] += 1
+            if layer_sd:
+                output_sd.update(layer_sd)
 
-                if force_clear_cache:
-                    import gc
-                    gc.collect()
-                    if torch.cuda.is_available():
-                        torch.cuda.empty_cache()
+            if force_clear_cache:
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
 
-                pbar.update(1)
-        finally:
-            writer.__exit__(None, None, None)
+            pbar.update(1)
 
         print(f"[LoRA Extract] Done: {stats['extracted']} extracted, {stats['chunked']} chunked, "
               f"{stats['full']} full, {stats['skipped']} skipped")
@@ -637,12 +616,22 @@ def extract_lora_from_files(
         handler_b.__exit__(None, None, None)
         cleanup_after_operation()
 
+    return output_sd
+
+
+def _save_lora(output_sd: dict, output_filename: str) -> str:
+    """Save LoRA to file."""
+    if not output_sd:
+        raise ValueError("No LoRA weights extracted")
 
-def _build_lora_output_path(output_filename: str) -> str:
-    """Build output path for LoRA file."""
     output_dir = folder_paths.get_folder_paths("loras")[0]
     os.makedirs(output_dir, exist_ok=True)
-    return os.path.join(output_dir, f"{output_filename.strip()}.safetensors")
+    output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors")
+
+    save_file(output_sd, output_path)
+    print(f"[LoRA Extract] Saved to {output_path}")
+
+    return output_path
 
 
 # =============================================================================
@@ -658,49 +647,85 @@ def _get_model_inputs():
     ]
 
 
+def _reconstruct_dots(key: str) -> str:
+    """Reconstruct dot structure from underscored key using heuristics."""
+    # Block indices: input_blocks_1 -> input_blocks.1
+    blocks = [
+        "input_blocks", "output_blocks", "middle_block",
+        "transformer_blocks", "single_transformer_blocks",
+        "double_blocks", "single_blocks",
+        "down_blocks", "up_blocks", "mid_block",
+        "attentions", "resnets", "upsamplers", "downsamplers"
+    ]
+    for b in blocks:
+        key = re.sub(f"{b}_(\\d+)", f"{b}.\\1", key)
+
+    # Sequential numbers: .1_1 -> .1.1
+    key = re.sub(r"(\.\d+)_(\d+)", r"\1.\2", key)
+
+    # Separators that should be preceded by dot
+    separators = [
+        "transformer_blocks", "single_transformer_blocks",
+        "attn1", "attn2", "attn",
+        "img_attn", "txt_attn",
+        "to_q", "to_k", "to_v", "to_out",
+        "q_proj", "k_proj", "v_proj", "out_proj",
+        "qkv", "proj", "ff", "net",
+        "norm1", "norm2", "norm3", "norm",
+        "time_emb_proj"
+    ]
+    for s in separators:
+        key = key.replace(f"_{s}", f".{s}")
+
+    # Special case: to_out_0 -> to_out.0
+    key = re.sub(r"to_out_(\d+)", r"to_out.\1", key)
+
+    return key
+
+
 def _format_lora_key(key: str) -> str:
     """
     Format the key for LoRA saving.
-    Always standardizes to 'diffusion_model.' prefix for maximum compatibility.
-
-    ComfyUI maps 'diffusion_model.<layer>' generically for all model types, making
-    it universally compatible. The 'transformer.' prefix only works for a subset of
-    model types via model-specific mapping code and is intentionally not used here.
+    Standardizes to 'diffusion_model.' or 'transformer.' prefix.
     """
     if key.endswith(".weight"):
         key = key[:-7]
 
-    # Handle ComfyUI Checkpoint wrapper (ldm/sgm format: model.diffusion_model.*)
-    if key.startswith("model.diffusion_model."):
-        inner_key = key[22:]  # strip len("model.diffusion_model.")
+    # Handle ComfyUI Checkpoint wrapper
+    if key.startswith("model.diffusion_model.") or key.startswith("net."):
+        # Check if it's a Diffusers-format transformer inside a checkpoint
+        if key.startswith("model.diffusion_model."):
+            inner_key = key[22:]  # len("model.diffusion_model.")
+        else:
+            inner_key = key[4:]   # len("net.")
+
+        if inner_key.startswith("transformer_blocks") or inner_key.startswith("single_transformer_blocks"):
+            return f"transformer.{inner_key}"
+        # Standard checkpoint or Original Flux
         return f"diffusion_model.{inner_key}"
 
-    # Handle net.* wrapper
-    if key.startswith("net."):
-        inner_key = key[4:]  # strip len("net.")
-        return f"diffusion_model.{inner_key}"
-
-    # Handle direct Diffusers keys (transformer_blocks.* / single_transformer_blocks.* without prefix)
+    # Handle direct Diffusers keys
     if key.startswith("transformer_blocks") or key.startswith("single_transformer_blocks"):
-        return f"diffusion_model.{key}"
+        return f"transformer.{key}"
 
     # Handle legacy lora_unet_ prefix (for resizing without base)
     if key.startswith("lora_unet_"):
+        core = key[10:]
+        dotted = _reconstruct_dots(core)
+        # Check if reconstructed key implies transformer format
+        if dotted.startswith("transformer_blocks") or dotted.startswith("single_transformer_blocks"):
+            return f"transformer.{dotted}"
+        return f"diffusion_model.{dotted}"
+
+    # Handle already correct prefixes
+    if key.startswith("diffusion_model.") or key.startswith("transformer."):
         return key
 
-    # Handle already-prefixed keys
-    if key.startswith("diffusion_model."):
-        return key
-
-    # Absorb now-invalid transformer. prefix into diffusion_model. for compatibility
-    if key.startswith("transformer."):
-        return f"diffusion_model.{key[12:]}"
-
-    # Handle known Diffusers UNet blocks
+    # Handle known Unet blocks (Diffusers format UNet)
     if any(key.startswith(p) for p in ["down_blocks", "up_blocks", "mid_block", "conv_in", "conv_out", "time_embedding", "class_embedding"]):
         return f"diffusion_model.{key}"
 
-    # Default fallback
+    # Default fallback to diffusion_model
     return f"diffusion_model.{key}"
 
 
@@ -719,10 +744,7 @@ def _get_common_inputs():
         io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
         io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
         io.String.Input("skip_patterns", default="", multiline=True,
-                       tooltip="Patterns for layers to skip (regex or glob depending on glob_skip_patterns)"),
-        io.Boolean.Input("glob_skip_patterns", default=False,
-                        tooltip="When True, skip_patterns use glob syntax (* = any sequence, ? = any char, dots are literal). "
-                                "When False (default), patterns are Python regex matched as substrings."),
+                       tooltip="Regex patterns for layers to skip"),
     ]
 
 
@@ -753,20 +775,19 @@ class LoRAExtractFixed(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, linear_dim, conv_dim, svd_niter, chunk_large_layers,
                 clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
 
-        extract_lora_from_files(
+        output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "fixed", linear_dim, conv_dim,
-            device, save_dtype, output_path, linear_dim, conv_dim,
+            device, save_dtype, linear_dim, conv_dim,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers, svd_niter,
-            lazy_load, force_clear_cache, glob_skip_patterns
+            lazy_load, force_clear_cache
         )
 
-        return io.NodeOutput(output_path)
+        return io.NodeOutput(_save_lora(output_sd, output_filename))
 
 
 class LoRAExtractRatio(io.ComfyNode):
@@ -798,21 +819,19 @@ class LoRAExtractRatio(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, linear_ratio, conv_ratio, linear_max_rank, conv_max_rank,
                 chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
 
-        extract_lora_from_files(
+        output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "ratio", linear_ratio, conv_ratio,
-            device, save_dtype, output_path, linear_max_rank, conv_max_rank,
+            device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache,
-            glob_skip_patterns=glob_skip_patterns
+            lazy_load=lazy_load, force_clear_cache=force_clear_cache
         )
 
-        return io.NodeOutput(output_path)
+        return io.NodeOutput(_save_lora(output_sd, output_filename))
 
 
 class LoRAExtractQuantile(io.ComfyNode):
@@ -844,21 +863,19 @@ class LoRAExtractQuantile(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, linear_quantile, conv_quantile, linear_max_rank, conv_max_rank,
                 chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
 
-        extract_lora_from_files(
+        output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "quantile", linear_quantile, conv_quantile,
-            device, save_dtype, output_path, linear_max_rank, conv_max_rank,
+            device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache,
-            glob_skip_patterns=glob_skip_patterns
+            lazy_load=lazy_load, force_clear_cache=force_clear_cache
         )
 
-        return io.NodeOutput(output_path)
+        return io.NodeOutput(_save_lora(output_sd, output_filename))
 
 
 class LoRAExtractKnee(io.ComfyNode):
@@ -888,21 +905,19 @@ class LoRAExtractKnee(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, knee_method, linear_max_rank, conv_max_rank,
                 chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
 
-        extract_lora_from_files(
+        output_sd = extract_lora_from_files(
             model_a_path, model_b_path, knee_method, 0, 0,
-            device, save_dtype, output_path, linear_max_rank, conv_max_rank,
+            device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache,
-            glob_skip_patterns=glob_skip_patterns
+            lazy_load=lazy_load, force_clear_cache=force_clear_cache
         )
 
-        return io.NodeOutput(output_path)
+        return io.NodeOutput(_save_lora(output_sd, output_filename))
 
 
 class LoRAExtractFrobenius(io.ComfyNode):
@@ -934,18 +949,16 @@ class LoRAExtractFrobenius(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, linear_target, conv_target, linear_max_rank, conv_max_rank,
                 chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns, glob_skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
-        output_path = _build_lora_output_path(output_filename)
 
-        extract_lora_from_files(
+        output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "sv_fro", linear_target, conv_target,
-            device, save_dtype, output_path, linear_max_rank, conv_max_rank,
+            device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache,
-            glob_skip_patterns=glob_skip_patterns
+            lazy_load=lazy_load, force_clear_cache=force_clear_cache
         )
 
-        return io.NodeOutput(output_path)
+        return io.NodeOutput(_save_lora(output_sd, output_filename))

nodes/lora_rename.py

@@ -1,119 +0,0 @@
-import os
-import folder_paths
-import comfy.utils
-from tqdm import tqdm
-from comfy_api.latest import io
-from safetensors import safe_open
-from .utils import convert_pt_to_safetensors
-from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation
-
-from unifiedefficientloader import MemoryEfficientSafeOpen, IncrementalSafetensorsWriter
-
-REVERSE_RENAME_DICT = {
-    "transformer_blocks.": "blocks.",
-    "norm1.linear_1": "adaln_modulation_self_attn.1",
-    "norm1.linear_2": "adaln_modulation_self_attn.2",
-    "norm2.linear_1": "adaln_modulation_cross_attn.1",
-    "norm2.linear_2": "adaln_modulation_cross_attn.2",
-    "norm3.linear_1": "adaln_modulation_mlp.1",
-    "norm3.linear_2": "adaln_modulation_mlp.2",
-    "attn1.to_q": "self_attn.q_proj",
-    "attn1.to_k": "self_attn.k_proj",
-    "attn1.to_v": "self_attn.v_proj",
-    "attn1.to_out.0": "self_attn.output_proj",
-    "attn2.to_q": "cross_attn.q_proj",
-    "attn2.to_k": "cross_attn.k_proj",
-    "attn2.to_v": "cross_attn.v_proj",
-    "attn2.to_out.0": "cross_attn.output_proj",
-    "ff.net.0.proj": "mlp.layer1",
-    "ff.net.2": "mlp.layer2",
-    "norm_out.linear_1": "final_layer.adaln_modulation.1",
-    "norm_out.linear_2": "final_layer.adaln_modulation.2",
-    "proj_out": "final_layer.linear",
-    "time_embed.t_embedder": "t_embedder.1",
-    "time_embed.norm": "t_embedding_norm",
-    "patch_embed.proj": "x_embedder.proj.1",
-}
-
-def _convert_diffusers_to_non_diffusers_anima_lora(lora_name: str, output_filename: str) -> str:
-    """
-    Converts a Diffusers Anima LoRA back to its non-diffusers format.
-    """
-    model_type = "loras"
-    model_path = folder_paths.get_full_path_or_raise(model_type, lora_name)
-
-    if model_path.endswith(('.pt', '.pth', '.bin', '.ckpt')):
-        temp_safe_path = model_path + ".safetensors"
-        if not os.path.exists(temp_safe_path):
-            conversion_successful, error_message = convert_pt_to_safetensors(model_path, temp_safe_path)
-            if not conversion_successful:
-                raise Exception(f"Conversion failed: {error_message}")
-        model_path_to_load = temp_safe_path
-    else:
-        model_path_to_load = model_path
-
-    # Prepare memory before operation
-    model_size_gb = estimate_model_size(model_path_to_load)
-    prepare_for_large_operation(model_size_gb * 1.2)
-
-    # Get metadata from safe_open (header only, no tensor loading)
-    with safe_open(model_path_to_load, framework="pt", device="cpu") as f:
-        metadata = f.metadata() or {}
-
-    model_dir = folder_paths.get_folder_paths(model_type)[-1]
-    output_path = os.path.join(model_dir, f"{output_filename.strip()}.safetensors")
-
-    # Stream tensors, rename on the fly, write immediately
-    writer = IncrementalSafetensorsWriter(output_path, metadata=metadata)
-    writer.__enter__()
-    try:
-        with MemoryEfficientSafeOpen(model_path_to_load) as handler:
-            original_keys = handler.keys()
-            pbar = comfy.utils.ProgressBar(len(original_keys))
-            for key in tqdm(original_keys, desc="Renaming Anima LoRA keys", unit="keys"):
-                new_key = key
-
-                if key.startswith("text_conditioner."):
-                    new_key = f"diffusion_model.llm_adapter.{key.removeprefix('text_conditioner.')}"
-                elif key.startswith("transformer."):
-                    new_key = key.removeprefix("transformer.")
-                    for old_part, new_part in REVERSE_RENAME_DICT.items():
-                        new_key = new_key.replace(old_part, new_part)
-                    new_key = f"diffusion_model.{new_key}"
-
-                writer.write(new_key, handler.get_tensor(key).contiguous())
-                pbar.update(1)
-    finally:
-        writer.__exit__(None, None, None)
-
-    # Cleanup after operation
-    cleanup_after_operation()
-
-    return output_path
-
-
-class AnimaLoraRename(io.ComfyNode):
-    @classmethod
-    def define_schema(cls):
-        return io.Schema(
-            node_id="AnimaLoraRename",
-            display_name="Anima LoRA Rename (Diffusers to Non-Diffusers)",
-            category="ModelUtils/LoRA",
-            description="Loads a Diffusers Anima LoRA, reverses its key names to the non-diffusers format, and saves it as a new safetensors file.",
-            inputs=[
-                io.Combo.Input(
-                    "lora_name",
-                    options=folder_paths.get_filename_list("loras"),
-                ),
-                io.String.Input("output_filename", default="anima_lora_non_diffusers"),
-            ],
-            outputs=[
-                io.String.Output(display_name="output_path"),
-            ],
-            is_output_node=True,
-        )
-
-    @classmethod
-    def execute(cls, lora_name: str, output_filename: str) -> io.NodeOutput:
-        path = _convert_diffusers_to_non_diffusers_anima_lora(lora_name, output_filename)
-        return io.NodeOutput(path)

nodes/lora_resize.py

@@ -12,10 +12,11 @@ import folder_paths
 import comfy.utils
 from tqdm import tqdm
 from comfy_api.latest import io
+from safetensors.torch import save_file
+from typing import Optional, Dict, Tuple, List
 from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation
 
-from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned, IncrementalSafetensorsWriter
-from typing import Optional, Dict, Tuple, List
+from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned
 
 
 
@@ -377,8 +378,6 @@ def resize_lora_file(
     output_filename: str,
     verbose: bool = True,
     svd_niter: int = 2,
-    lazy_load: bool = True,
-    force_clear_cache: bool = False,
 ) -> str:
     """
     Resize a LoRA file to a new rank.
@@ -393,8 +392,6 @@ def resize_lora_file(
         output_filename: Output filename (without extension)
         verbose: Print progress info
         svd_niter: Power iterations for SVD accuracy
-        lazy_load: Low memory mode: load tensors from disk on demand
-        force_clear_cache: Clear CUDA cache after each layer
 
     Returns:
         Path to saved resized LoRA
@@ -403,7 +400,7 @@ def resize_lora_file(
     lora_size_gb = estimate_model_size(lora_path)
     prepare_for_large_operation(lora_size_gb * 2, torch.device(device))
 
-    handler = MemoryEfficientSafeOpen(lora_path, low_memory=lazy_load)
+    handler = MemoryEfficientSafeOpen(lora_path)
 
     try:
         metadata = handler.metadata().copy()
@@ -422,7 +419,66 @@ def resize_lora_file(
             print(f"[LoRA Resize] Original dim={network_dim}, alpha={network_alpha:.1f}, scale={scale:.3f}")
             print(f"[LoRA Resize] Resizing with method={method_str}, max_rank={new_rank}")
 
-        # Build metadata and output path before loop (all values known at this point)
+        output_sd = {}
+        fro_list = []
+        pbar = comfy.utils.ProgressBar(len(pairs))
+
+        with torch.no_grad():
+            for block_name, block_keys in tqdm(pairs.items(), desc="Resizing layers", unit="layers"):
+                if "down" not in block_keys or "up" not in block_keys:
+                    pbar.update(1)
+                    continue
+
+                lora_down = handler.get_tensor(block_keys["down"])
+                lora_up = handler.get_tensor(block_keys["up"])
+
+                is_conv = len(lora_down.shape) == 4
+
+                # Transfer to GPU with pinned memory if available
+                if device == 'cuda':
+                    lora_down = transfer_to_gpu_pinned(lora_down, device, torch.float32)
+                    lora_up = transfer_to_gpu_pinned(lora_up, device, torch.float32)
+
+                # Merge and re-extract
+                if is_conv:
+                    weight = _merge_conv(lora_down, lora_up, device)
+                    result = _extract_conv(weight, new_rank, dynamic_method, dynamic_param, scale, svd_niter)
+                else:
+                    weight = _merge_linear(lora_down, lora_up, device)
+                    result = _extract_linear(weight, new_rank, dynamic_method, dynamic_param, scale, svd_niter)
+
+                del weight, lora_down, lora_up
+
+                fro_list.append(result['fro_retained'])
+
+                # Store using same format suffixes as input
+                down_suffix = format_info["down_suffix"]
+                up_suffix = format_info["up_suffix"]
+                alpha_suffix = format_info["alpha_suffix"]
+
+                # Standardize prefix using heuristics if possible
+                new_block_name = _format_lora_key(block_name)
+
+                output_sd[f"{new_block_name}{down_suffix}"] = result["lora_down"].to(save_dtype)
+                output_sd[f"{new_block_name}{up_suffix}"] = result["lora_up"].to(save_dtype)
+                if alpha_suffix:
+                    output_sd[f"{new_block_name}{alpha_suffix}"] = torch.tensor(result["new_alpha"]).to(save_dtype)
+
+                del result
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
+                pbar.update(1)
+
+        if verbose and fro_list:
+            import numpy as np
+            avg_fro = np.mean(fro_list)
+            std_fro = np.std(fro_list)
+            print(f"[LoRA Resize] Average Frobenius retention: {avg_fro:.1%} ± {std_fro:.3f}")
+
+        # Update metadata
         if dynamic_method:
             metadata["ss_training_comment"] = f"Dynamic resize with {dynamic_method}: {dynamic_param} from dim {network_dim}"
             metadata["ss_network_dim"] = "Dynamic"
@@ -432,79 +488,14 @@ def resize_lora_file(
             metadata["ss_network_dim"] = str(new_rank)
             metadata["ss_network_alpha"] = str(scale * new_rank)
 
+        # Save
         output_dir = folder_paths.get_folder_paths("loras")[0]
         os.makedirs(output_dir, exist_ok=True)
         output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors")
 
-        fro_list = []
-        pbar = comfy.utils.ProgressBar(len(pairs))
-
-        writer = IncrementalSafetensorsWriter(output_path, metadata=metadata)
-        writer.__enter__()
-        try:
-            with torch.no_grad():
-                for block_name, block_keys in tqdm(pairs.items(), desc="Resizing layers", unit="layers"):
-                    if "down" not in block_keys or "up" not in block_keys:
-                        pbar.update(1)
-                        continue
-
-                    lora_down = handler.get_tensor(block_keys["down"])
-                    lora_up = handler.get_tensor(block_keys["up"])
-
-                    is_conv = len(lora_down.shape) == 4
-
-                    # Transfer to GPU with pinned memory if available
-                    if device == 'cuda':
-                        lora_down = transfer_to_gpu_pinned(lora_down, device, torch.float32)
-                        lora_up = transfer_to_gpu_pinned(lora_up, device, torch.float32)
-
-                    # Merge and re-extract
-                    if is_conv:
-                        weight = _merge_conv(lora_down, lora_up, device)
-                        result = _extract_conv(weight, new_rank, dynamic_method, dynamic_param, scale, svd_niter)
-                    else:
-                        weight = _merge_linear(lora_down, lora_up, device)
-                        result = _extract_linear(weight, new_rank, dynamic_method, dynamic_param, scale, svd_niter)
-
-                    del weight, lora_down, lora_up
-
-                    fro_list.append(result['fro_retained'])
-
-                    # Store using same format suffixes as input
-                    down_suffix = format_info["down_suffix"]
-                    up_suffix = format_info["up_suffix"]
-                    alpha_suffix = format_info["alpha_suffix"]
-
-                    # Preserve original key format from input LoRA verbatim
-                    new_block_name = block_name
-
-                    block_sd = {
-                        f"{new_block_name}{down_suffix}": result["lora_down"].to(save_dtype),
-                        f"{new_block_name}{up_suffix}": result["lora_up"].to(save_dtype),
-                    }
-                    if alpha_suffix:
-                        block_sd[f"{new_block_name}{alpha_suffix}"] = torch.tensor(result["new_alpha"]).to(save_dtype)
-
-                    writer.write_dict(block_sd)
-
-                    del result
-                    if force_clear_cache:
-                        import gc
-                        gc.collect()
-                        if torch.cuda.is_available():
-                            torch.cuda.empty_cache()
-
-                    pbar.update(1)
-        finally:
-            writer.__exit__(None, None, None)
-
-        if verbose and fro_list:
-            import numpy as np
-            avg_fro = np.mean(fro_list)
-            std_fro = np.std(fro_list)
-            print(f"[LoRA Resize] Average Frobenius retention: {avg_fro:.1%} ± {std_fro:.3f}")
-
+        save_file(output_sd, output_path, metadata)
         print(f"[LoRA Resize] Saved to {output_path}")
+
         return output_path
 
     finally:
@@ -536,22 +527,17 @@ class LoRAResizeFixed(io.ComfyNode):
                 io.String.Input("output_filename", default="resized_lora"),
                 io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
                 io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
-                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
-                io.Boolean.Input("force_clear_cache", default=False, tooltip="Clear CUDA cache after each layer (slower but saves VRAM)"),
             ],
             outputs=[io.String.Output(display_name="output_path")],
             is_output_node=True,
         )
 
     @classmethod
-    def execute(cls, lora_name, new_rank, svd_niter, output_filename, save_dtype, device, lazy_load, force_clear_cache) -> io.NodeOutput:
+    def execute(cls, lora_name, new_rank, svd_niter, output_filename, save_dtype, device) -> io.NodeOutput:
         lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
         dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
 
-        path = resize_lora_file(
-            lora_path, new_rank, None, None, device, dtype, output_filename,
-            svd_niter=svd_niter, lazy_load=lazy_load, force_clear_cache=force_clear_cache
-        )
+        path = resize_lora_file(lora_path, new_rank, None, None, device, dtype, output_filename, svd_niter=svd_niter)
         return io.NodeOutput(path)
 
 
@@ -575,22 +561,17 @@ class LoRAResizeRatio(io.ComfyNode):
                 io.String.Input("output_filename", default="resized_lora_ratio"),
                 io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
                 io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
-                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
-                io.Boolean.Input("force_clear_cache", default=False, tooltip="Clear CUDA cache after each layer (slower but saves VRAM)"),
             ],
             outputs=[io.String.Output(display_name="output_path")],
             is_output_node=True,
         )
 
     @classmethod
-    def execute(cls, lora_name, max_rank, ratio, output_filename, save_dtype, device, lazy_load, force_clear_cache) -> io.NodeOutput:
+    def execute(cls, lora_name, max_rank, ratio, output_filename, save_dtype, device) -> io.NodeOutput:
         lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
         dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
 
-        path = resize_lora_file(
-            lora_path, max_rank, "sv_ratio", ratio, device, dtype, output_filename,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache
-        )
+        path = resize_lora_file(lora_path, max_rank, "sv_ratio", ratio, device, dtype, output_filename)
         return io.NodeOutput(path)
 
 
@@ -614,22 +595,17 @@ class LoRAResizeFrobenius(io.ComfyNode):
                 io.String.Input("output_filename", default="resized_lora_fro"),
                 io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
                 io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
-                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
-                io.Boolean.Input("force_clear_cache", default=False, tooltip="Clear CUDA cache after each layer (slower but saves VRAM)"),
             ],
             outputs=[io.String.Output(display_name="output_path")],
             is_output_node=True,
         )
 
     @classmethod
-    def execute(cls, lora_name, max_rank, target, output_filename, save_dtype, device, lazy_load, force_clear_cache) -> io.NodeOutput:
+    def execute(cls, lora_name, max_rank, target, output_filename, save_dtype, device) -> io.NodeOutput:
         lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
         dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
 
-        path = resize_lora_file(
-            lora_path, max_rank, "sv_fro", target, device, dtype, output_filename,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache
-        )
+        path = resize_lora_file(lora_path, max_rank, "sv_fro", target, device, dtype, output_filename)
         return io.NodeOutput(path)
 
 
@@ -653,22 +629,17 @@ class LoRAResizeCumulative(io.ComfyNode):
                 io.String.Input("output_filename", default="resized_lora_cumulative"),
                 io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
                 io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
-                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
-                io.Boolean.Input("force_clear_cache", default=False, tooltip="Clear CUDA cache after each layer (slower but saves VRAM)"),
             ],
             outputs=[io.String.Output(display_name="output_path")],
             is_output_node=True,
         )
 
     @classmethod
-    def execute(cls, lora_name, max_rank, target, output_filename, save_dtype, device, lazy_load, force_clear_cache) -> io.NodeOutput:
+    def execute(cls, lora_name, max_rank, target, output_filename, save_dtype, device) -> io.NodeOutput:
         lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
         dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
 
-        path = resize_lora_file(
-            lora_path, max_rank, "sv_cumulative", target, device, dtype, output_filename,
-            lazy_load=lazy_load, force_clear_cache=force_clear_cache
-        )
+        path = resize_lora_file(lora_path, max_rank, "sv_cumulative", target, device, dtype, output_filename)
         return io.NodeOutput(path)
 
 
@@ -687,8 +658,6 @@ def merge_loras_to_model(
     output_filename: str,
     skip_patterns_str: str = "",
     verbose: bool = True,
-    lazy_load: bool = True,
-    force_clear_cache: bool = False,
 ) -> str:
     """
     Merge multiple LoRAs into a base model and save the result directly.
@@ -706,8 +675,6 @@ def merge_loras_to_model(
         output_filename: Output filename (without extension)
         skip_patterns_str: Regex patterns for layers to skip
         verbose: Print progress info
-        lazy_load: Low memory mode: load tensors from disk on demand
-        force_clear_cache: Clear CUDA cache after each layer
 
     Returns:
         Path to saved merged model
@@ -723,8 +690,8 @@ def merge_loras_to_model(
     prepare_for_large_operation(total_size_gb * 1.5, torch.device(device))
 
     # Open all files - use low_memory for base model to avoid OS page caching
-    base_handler = MemoryEfficientSafeOpen(base_model_path, low_memory=lazy_load)
-    lora_handlers = [MemoryEfficientSafeOpen(lp, low_memory=lazy_load) for lp in lora_paths]
+    base_handler = MemoryEfficientSafeOpen(base_model_path, low_memory=True)
+    lora_handlers = [MemoryEfficientSafeOpen(lp) for lp in lora_paths]
 
 
     try:
@@ -748,7 +715,7 @@ def merge_loras_to_model(
                 print(f"[LoRA Merge To Model] LoRA {i+1}: {format_info['format']}, {len(pairs)} layers, dim={network_dim}")
 
         # Common prefixes
-        BASE_PREFIXES = ["model.diffusion_model.", "diffusion_model.", "transformer.", "model.", "net."]
+        BASE_PREFIXES = ["model.diffusion_model.", "diffusion_model.", "transformer.", "model."]
         LORA_PREFIXES = [
             "lora_unet_", "lora_transformer_", "lora_te1_", "lora_te2_", "lora_te_",
             "lycoris_", "diffusion_model.", "transformer.", "unet."
@@ -794,11 +761,7 @@ def merge_loras_to_model(
         base_metadata = base_handler.metadata().copy() if base_handler.metadata() else {}
         base_metadata["merge_comment"] = f"Merged {len(lora_paths)} LoRAs with weights: {lora_weights}"
 
-        # Build output path before loop
-        base_dir = os.path.dirname(base_model_path)
-        os.makedirs(base_dir, exist_ok=True)
-        output_path = os.path.join(base_dir, f"{output_filename.strip()}.safetensors")
-
+        output_sd = {}
         stats = {"merged": 0, "copied": 0, "skipped": 0}
         base_keys = list(base_handler.keys())
         pbar = comfy.utils.ProgressBar(len(base_keys))
@@ -806,117 +769,118 @@ def merge_loras_to_model(
         if verbose:
             print(f"[LoRA Merge To Model] Processing {len(base_keys)} base model keys...")
 
-        writer = IncrementalSafetensorsWriter(output_path, metadata=base_metadata)
-        writer.__enter__()
-        try:
-            with torch.no_grad():
-                for base_key in tqdm(base_keys, desc="Merging to model", unit="keys"):
-                    # Check skip patterns
-                    if _matches_any_pattern(base_key, skip_patterns):
-                        stats["skipped"] += 1
-                        pbar.update(1)
-                        continue
+        with torch.no_grad():
+            for base_key in tqdm(base_keys, desc="Merging to model", unit="keys"):
+                # Check skip patterns
+                if _matches_any_pattern(base_key, skip_patterns):
+                    stats["skipped"] += 1
+                    pbar.update(1)
+                    continue
 
-                    # Load base weight
-                    cpu_base = base_handler.get_tensor(base_key)
+                # Load base weight
+                cpu_base = base_handler.get_tensor(base_key)
 
-                    # Only process weight tensors for LoRA merging
-                    if base_key.endswith(".weight"):
-                        core = extract_core_layer_base(base_key)
-                        core_underscored = core.replace(".", "_")
+                # Only process weight tensors for LoRA merging
+                if base_key.endswith(".weight"):
+                    core = extract_core_layer_base(base_key)
+                    core_underscored = core.replace(".", "_")
 
-                        # Check if any LoRA contributes to this layer
-                        if core_underscored in lora_lookup:
-                            # Transfer to GPU for computation
-                            if device == 'cuda':
-                                base_weight = transfer_to_gpu_pinned(cpu_base, device, torch.float32)
-                            else:
-                                base_weight = cpu_base.to(device=device, dtype=torch.float32)
-                            del cpu_base
-
-                            # Accumulate deltas from all contributing LoRAs
-                            for info, block_keys in lora_lookup[core_underscored]:
-                                is_full_diff = info["format_info"].get("is_full_diff", False)
-
-                                if is_full_diff:
-                                    # Full diff format
-                                    if "diff" not in block_keys:
-                                        continue
-                                    cpu_diff = info["handler"].get_tensor(block_keys["diff"])
-                                    if device == 'cuda':
-                                        delta = transfer_to_gpu_pinned(cpu_diff, device, torch.float32)
-                                    else:
-                                        delta = cpu_diff.to(device=device, dtype=torch.float32)
-                                    del cpu_diff
-                                    effective_scale = info["weight"]
-                                else:
-                                    # Standard LoRA format
-                                    if "down" not in block_keys or "up" not in block_keys:
-                                        continue
-
-                                    cpu_down = info["handler"].get_tensor(block_keys["down"])
-                                    cpu_up = info["handler"].get_tensor(block_keys["up"])
-                                    if device == 'cuda':
-                                        lora_down = transfer_to_gpu_pinned(cpu_down, device, torch.float32)
-                                        lora_up = transfer_to_gpu_pinned(cpu_up, device, torch.float32)
-                                    else:
-                                        lora_down = cpu_down.to(device=device, dtype=torch.float32)
-                                        lora_up = cpu_up.to(device=device, dtype=torch.float32)
-                                    del cpu_down, cpu_up
-
-                                    # Get alpha
-                                    if "alpha" in block_keys:
-                                        alpha_tensor = info["handler"].get_tensor(block_keys["alpha"])
-                                        layer_alpha = float(alpha_tensor.item())
-                                    else:
-                                        layer_alpha = float(info["network_dim"])
-                                    layer_scale = layer_alpha / info["network_dim"] if info["network_dim"] > 0 else 1.0
-                                    effective_scale = layer_scale * info["weight"]
-
-                                    # Compute delta
-                                    is_conv = len(lora_down.shape) == 4
-                                    if is_conv:
-                                        in_rank, in_size, kernel_size, k_ = lora_down.shape
-                                        out_size, out_rank, _, _ = lora_up.shape
-                                        delta = lora_up.reshape(out_size, -1) @ lora_down.reshape(in_rank, -1)
-                                        delta = delta.reshape(out_size, in_size, kernel_size, kernel_size)
-                                    else:
-                                        delta = lora_up @ lora_down
-                                    del lora_down, lora_up
-
-                                # Apply delta to base weight
-                                base_weight = base_weight + effective_scale * delta
-                                del delta
-
-                            # Write merged weight
-                            writer.write(base_key, base_weight.to(save_dtype).cpu().contiguous())
-                            del base_weight
-                            stats["merged"] += 1
+                    # Check if any LoRA contributes to this layer
+                    if core_underscored in lora_lookup:
+                        # Transfer to GPU for computation
+                        if device == 'cuda':
+                            base_weight = transfer_to_gpu_pinned(cpu_base, device, torch.float32)
                         else:
-                            # No LoRA contribution, write as-is
-                            writer.write(base_key, cpu_base.to(save_dtype).contiguous())
-                            del cpu_base
-                            stats["copied"] += 1
+                            base_weight = cpu_base.to(device=device, dtype=torch.float32)
+                        del cpu_base
+
+                        # Accumulate deltas from all contributing LoRAs
+                        for info, block_keys in lora_lookup[core_underscored]:
+                            is_full_diff = info["format_info"].get("is_full_diff", False)
+
+                            if is_full_diff:
+                                # Full diff format
+                                if "diff" not in block_keys:
+                                    continue
+                                cpu_diff = info["handler"].get_tensor(block_keys["diff"])
+                                if device == 'cuda':
+                                    delta = transfer_to_gpu_pinned(cpu_diff, device, torch.float32)
+                                else:
+                                    delta = cpu_diff.to(device=device, dtype=torch.float32)
+                                del cpu_diff
+                                effective_scale = info["weight"]
+                            else:
+                                # Standard LoRA format
+                                if "down" not in block_keys or "up" not in block_keys:
+                                    continue
+
+                                cpu_down = info["handler"].get_tensor(block_keys["down"])
+                                cpu_up = info["handler"].get_tensor(block_keys["up"])
+                                if device == 'cuda':
+                                    lora_down = transfer_to_gpu_pinned(cpu_down, device, torch.float32)
+                                    lora_up = transfer_to_gpu_pinned(cpu_up, device, torch.float32)
+                                else:
+                                    lora_down = cpu_down.to(device=device, dtype=torch.float32)
+                                    lora_up = cpu_up.to(device=device, dtype=torch.float32)
+                                del cpu_down, cpu_up
+
+                                # Get alpha
+                                if "alpha" in block_keys:
+                                    alpha_tensor = info["handler"].get_tensor(block_keys["alpha"])
+                                    layer_alpha = float(alpha_tensor.item())
+                                else:
+                                    layer_alpha = float(info["network_dim"])
+                                layer_scale = layer_alpha / info["network_dim"] if info["network_dim"] > 0 else 1.0
+                                effective_scale = layer_scale * info["weight"]
+
+                                # Compute delta
+                                is_conv = len(lora_down.shape) == 4
+                                if is_conv:
+                                    in_rank, in_size, kernel_size, k_ = lora_down.shape
+                                    out_size, out_rank, _, _ = lora_up.shape
+                                    delta = lora_up.reshape(out_size, -1) @ lora_down.reshape(in_rank, -1)
+                                    delta = delta.reshape(out_size, in_size, kernel_size, kernel_size)
+                                else:
+                                    delta = lora_up @ lora_down
+                                del lora_down, lora_up
+
+                            # Apply delta to base weight
+                            base_weight = base_weight + effective_scale * delta
+                            del delta
+
+                        # Store merged weight
+                        output_sd[base_key] = base_weight.to(save_dtype).cpu().contiguous()
+                        del base_weight
+                        stats["merged"] += 1
                     else:
-                        # Non-weight tensor (bias, norm, etc.), write as-is
-                        writer.write(base_key, cpu_base.to(save_dtype).contiguous())
+                        # No LoRA contribution, copy as-is
+                        output_sd[base_key] = cpu_base.to(save_dtype).contiguous()
                         del cpu_base
                         stats["copied"] += 1
+                else:
+                    # Non-weight tensor (bias, norm, etc.), copy as-is
+                    output_sd[base_key] = cpu_base.to(save_dtype).contiguous()
+                    del cpu_base
+                    stats["copied"] += 1
 
-                    if force_clear_cache:
-                        import gc
-                        gc.collect()
-                        if torch.cuda.is_available():
-                            torch.cuda.empty_cache()
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
 
-                    pbar.update(1)
-        finally:
-            writer.__exit__(None, None, None)
+                pbar.update(1)
 
         if verbose:
             print(f"[LoRA Merge To Model] Done: {stats['merged']} merged, {stats['copied']} copied, {stats['skipped']} skipped")
 
+        # Save to base model directory
+        base_dir = os.path.dirname(base_model_path)
+        os.makedirs(base_dir, exist_ok=True)
+        output_path = os.path.join(base_dir, f"{output_filename.strip()}.safetensors")
+
+        save_file(output_sd, output_path, base_metadata)
         print(f"[LoRA Merge To Model] Saved to {output_path}")
+
         return output_path
 
     finally:
@@ -987,8 +951,6 @@ class LoRAMergeToModel(io.ComfyNode):
                 io.String.Input("output_filename", default="merged_model"),
                 io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
                 io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
-                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
-                io.Boolean.Input("force_clear_cache", default=False, tooltip="Clear CUDA cache after each layer (slower but saves VRAM)"),
             ],
             outputs=[io.String.Output(display_name="output_path")],
             is_output_node=True,
@@ -998,7 +960,7 @@ class LoRAMergeToModel(io.ComfyNode):
     def execute(cls, base_model, lora_count,
                 lora_1, weight_1, lora_2, weight_2, lora_3, weight_3, lora_4, weight_4,
                 lora_5, weight_5, lora_6, weight_6, lora_7, weight_7, lora_8, weight_8,
-                skip_patterns, output_filename, save_dtype, device, lazy_load, force_clear_cache) -> io.NodeOutput:
+                skip_patterns, output_filename, save_dtype, device) -> io.NodeOutput:
 
         # Build LoRA list based on count
         count = int(lora_count)
@@ -1023,7 +985,5 @@ class LoRAMergeToModel(io.ComfyNode):
             save_dtype=dtype,
             output_filename=output_filename,
             skip_patterns_str=skip_patterns,
-            lazy_load=lazy_load,
-            force_clear_cache=force_clear_cache,
         )
         return io.NodeOutput(path)

nodes/merger.py

@@ -1,4 +1,3 @@
-import fnmatch
 import os
 import re
 import torch
@@ -6,7 +5,9 @@ import folder_paths
 import comfy.utils
 from tqdm import tqdm
 from comfy_api.latest import io
+from safetensors.torch import save_file
 from .merger_ops import TWO_MODEL_MODES, THREE_MODEL_MODES, MissingTensorBehavior, MissingTensorError
+from .merger_utils import parse_custom_weights, get_custom_parameters
 from .device_utils import (
     estimate_model_size, prepare_for_large_operation, cleanup_after_operation
 )
@@ -25,38 +26,26 @@ def load_documentation_from_file(filename):
         return f"# Documentation File Not Found\n\nPlease ensure `{filename}` exists in the `docs` directory of the custom node."
 
 
-def _compile_patterns(pattern_string, glob_mode=False):
-    """Compiles whitespace-separated patterns into a list.
+def _compile_patterns(pattern_string):
+    """Compiles whitespace-separated regex patterns into a list of compiled patterns.
 
-    In regex mode (default): returns compiled re.Pattern objects.
-      Raises ValueError if any pattern is invalid regex.
-    In glob mode: returns plain strings; fnmatch handles matching.
+    Returns empty list if pattern_string is empty or whitespace-only.
+    Raises ValueError if any pattern is invalid regex.
     """
     if not pattern_string or not pattern_string.strip():
         return []
 
     patterns = []
     for pattern in pattern_string.split():
-        if not pattern:
-            continue
-        if glob_mode:
-            patterns.append(pattern)
-        else:
-            try:
-                patterns.append(re.compile(pattern))
-            except re.error as e:
-                raise ValueError(f"Invalid regex pattern '{pattern}': {e}")
+        try:
+            patterns.append(re.compile(pattern))
+        except re.error as e:
+            raise ValueError(f"Invalid regex pattern '{pattern}': {e}")
     return patterns
 
 
-def _matches_any_pattern(key, patterns, glob_mode=False):
-    """Returns True if key matches any pattern.
-
-    Glob mode: fnmatch substring match — dots are literal, * matches any sequence.
-    Regex mode: compiled re.Pattern substring search.
-    """
-    if glob_mode:
-        return any(fnmatch.fnmatch(key, f"*{p}*") for p in patterns)
+def _matches_any_pattern(key, patterns):
+    """Returns True if key matches any of the compiled regex patterns (substring search)."""
     for pattern in patterns:
         if pattern.search(key):
             return True
@@ -112,9 +101,11 @@ class MergerLogic:
         alignment_mode = recipe_params.get('alignment_mode', 'pad/crop')
 
         # Compile filter patterns
-        glob_mode = recipe_params.get('glob_patterns', False)
-        exclude_patterns = _compile_patterns(recipe_params.get('exclude_patterns', ''), glob_mode=glob_mode)
-        discard_patterns = _compile_patterns(recipe_params.get('discard_patterns', ''), glob_mode=glob_mode)
+        exclude_patterns = _compile_patterns(recipe_params.get('exclude_patterns', ''))
+        discard_patterns = _compile_patterns(recipe_params.get('discard_patterns', ''))
+
+        # Parse custom per-layer weights
+        parsed_custom_weights = parse_custom_weights(recipe_params.get('custom_weights', ''))
 
         # Pre-compute key differences for logging
         primary_keys = set(all_keys)
@@ -132,25 +123,6 @@ class MergerLogic:
         pbar = comfy.utils.ProgressBar(len(all_keys))
         save_dtype = recipe_params.pop('save_dtype')
         save_torch_dtype = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16}.get(save_dtype)
-        override_dtype = recipe_params.pop('override_dtype', False)
-
-        def determine_target_dtype(original_dtypes, target_dtype, override):
-            if override:
-                return target_dtype
-            priority_map = {
-                torch.float32: 3,
-                torch.bfloat16: 2,
-                torch.float16: 2,
-            }
-            highest_dtype = target_dtype
-            highest_priority = priority_map.get(highest_dtype, 1)
-            for dtype in original_dtypes:
-                p = priority_map.get(dtype, 1)
-                if p > highest_priority:
-                    highest_priority = p
-                    highest_dtype = dtype
-            return highest_dtype
-
         recipe_params.update({"handlers": handlers})
         process_device = recipe_params.get('device', 'cpu')
         process_dtype = recipe_params.get('dtype', torch.float32)
@@ -194,21 +166,12 @@ class MergerLogic:
         with torch.no_grad():
             for key in tqdm(all_keys, desc="Merging layers", unit="layers"):
                 # Check discard patterns first - skip entirely
-                if _matches_any_pattern(key, discard_patterns, glob_mode=glob_mode):
+                if _matches_any_pattern(key, discard_patterns):
                     discarded_keys += 1
                     continue
 
                 # Pre-load Model A's tensor with pinned memory for CUDA
                 cpu_tensor = primary_handler.get_tensor(key)
-
-                # Determine original dtypes across all active models for this key
-                original_dtypes = [primary_handler.get_dtype(key)]
-                for name, handler in handlers.items():
-                    if name != primary_model_name and key in handler.keys():
-                        original_dtypes.append(handler.get_dtype(key))
-
-                target_dtype = determine_target_dtype(original_dtypes, save_torch_dtype, override_dtype)
-
                 if process_device == 'cuda':
                     tensor_a = transfer_to_gpu_pinned(cpu_tensor, process_device, process_dtype)
                 else:
@@ -216,8 +179,8 @@ class MergerLogic:
                 del cpu_tensor
 
                 # Check exclude patterns - use Model A only, no merge
-                if _matches_any_pattern(key, exclude_patterns, glob_mode=glob_mode):
-                    t = tensor_a.detach().to(target_dtype).cpu()
+                if _matches_any_pattern(key, exclude_patterns):
+                    t = tensor_a.detach().to(save_torch_dtype).cpu()
 
                     batch_buffer[key] = t
                     batch_bytes += t.numel() * t.element_size()
@@ -234,7 +197,9 @@ class MergerLogic:
                 recipe_params['_tensor_a_dtype'] = tensor_a.dtype
 
                 try:
-                    recipe = calc_mode_class.create_recipe(key=key, **recipe_params)
+                    # Apply per-layer parameter overrides (regex/substring matching)
+                    layer_params = get_custom_parameters(key, parsed_custom_weights, recipe_params)
+                    recipe = calc_mode_class.create_recipe(key=key, **layer_params)
                     result = recipe.merge()
                 except MissingTensorError as e:
                     if mismatch_mode == MissingTensorBehavior.ERROR:
@@ -252,7 +217,7 @@ class MergerLogic:
 
                 if isinstance(result, dict):
                     for r_key, r_tensor in result.items():
-                        t = r_tensor.detach().to(target_dtype).cpu()
+                        t = r_tensor.detach().to(save_torch_dtype).cpu()
 
                         batch_buffer[r_key] = t
                         batch_bytes += t.numel() * t.element_size()
@@ -266,7 +231,7 @@ class MergerLogic:
                             slices = tuple(slice(0, min(res_s, tgt_s)) for res_s, tgt_s in zip(result.shape, target_shape))
                             result = result[slices]
 
-                    t = result.detach().to(target_dtype).cpu()
+                    t = result.detach().to(save_torch_dtype).cpu()
 
                     batch_buffer[key] = t
                     batch_bytes += t.numel() * t.element_size()
@@ -337,21 +302,16 @@ class CheckpointTwoMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_checkpoint"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -362,9 +322,10 @@ class CheckpointTwoMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -372,11 +333,11 @@ class CheckpointTwoMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
@@ -403,21 +364,16 @@ class ModelTwoMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_model"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -428,9 +384,10 @@ class ModelTwoMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -438,11 +395,11 @@ class ModelTwoMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
@@ -469,21 +426,16 @@ class TextEncoderTwoMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_textencoder"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -494,9 +446,10 @@ class TextEncoderTwoMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -504,11 +457,11 @@ class TextEncoderTwoMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
@@ -535,21 +488,16 @@ class LoRATwoMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_lora"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -560,9 +508,10 @@ class LoRATwoMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -570,11 +519,11 @@ class LoRATwoMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
@@ -601,21 +550,16 @@ class EmbeddingTwoMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_embedding"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -626,9 +570,10 @@ class EmbeddingTwoMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -636,11 +581,11 @@ class EmbeddingTwoMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
@@ -670,21 +615,16 @@ class CheckpointThreeMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.5, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_checkpoint"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -695,9 +635,10 @@ class CheckpointThreeMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -705,11 +646,11 @@ class CheckpointThreeMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}
@@ -737,21 +678,16 @@ class ModelThreeMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.5, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_model"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -762,9 +698,10 @@ class ModelThreeMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -772,11 +709,11 @@ class ModelThreeMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}
@@ -804,21 +741,16 @@ class TextEncoderThreeMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_textencoder"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -829,9 +761,10 @@ class TextEncoderThreeMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -839,11 +772,11 @@ class TextEncoderThreeMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}
@@ -871,21 +804,16 @@ class LoRAThreeMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.5, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_lora"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -896,9 +824,10 @@ class LoRAThreeMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -906,11 +835,11 @@ class LoRAThreeMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}
@@ -938,21 +867,16 @@ class EmbeddingThreeMerger(io.ComfyNode):
                 io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Float.Input("gamma", default=0.5, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("delta", default=2.0, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("epsilon", default=0.01, min=-10.0, max=10.0, step=0.01),
-                io.Float.Input("zeta", default=0.0, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_embedding"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
-                io.Boolean.Input("glob_patterns", default=False,
-                                 tooltip="When True, exclude/discard patterns use glob syntax (* = any sequence, dots are literal). "
-                                         "When False (default), patterns are Python regex matched as substrings."),
+                io.String.Input("custom_weights", default="", multiline=True, tooltip="Per-layer weights/params. Syntax: 'regex:weight' or 'regex:param=val,param2=val'"),
                 io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
                 io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
-                io.Boolean.Input("override_dtype", default=False, tooltip="Force the entire model to be saved as the selected save_dtype. If False (default), higher precision dtypes are preserved."),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -963,9 +887,10 @@ class EmbeddingThreeMerger(io.ComfyNode):
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
                 calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
-                gamma: float, delta: float, epsilon: float, zeta: float, seed: int, output_filename: str, save_dtype: str,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
                 process_device: str, exclude_patterns: str, discard_patterns: str,
-                glob_patterns: bool, lazy_load: bool, force_clear_cache: bool, override_dtype: bool) -> io.NodeOutput:
+                custom_weights: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
@@ -973,11 +898,11 @@ class EmbeddingThreeMerger(io.ComfyNode):
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
             "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "epsilon": epsilon, "zeta": zeta, "seed": seed,
-            "output_filename": output_filename, "save_dtype": save_dtype, "override_dtype": override_dtype,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
+            "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
-            "glob_patterns": glob_patterns,
+            "custom_weights": custom_weights,
             "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}

nodes/merger_ops.py

@@ -793,143 +793,5 @@ class DARETIESMode(CalcMode):
         return op
 
 
-class EnhancedPowerUpOp(Operation):
-    def __init__(self, key, alpha, beta, gamma, delta, seed, *sources):
-        super().__init__(key, *sources)
-        self.mask_power = alpha       # Mask power
-        self.min_keep_prob = beta     # Min keep prob
-        self.mask_smooth = gamma      # Smoothness (0 or 1)
-        self.delta = delta            # Final multiplier
-        self.seed = seed
-
-    def oper(self, a, b):
-        if a is None or b is None: return None
-        a, b = resize_tensors(a, b, mode=self.alignment_mode)
-        delta_val = (b - a).float()
-        abs_delta = torch.abs(delta_val)
-        max_delta = torch.max(abs_delta)
-        if max_delta == 0: return torch.zeros_like(delta_val)
-
-        prob = torch.clamp((abs_delta / max_delta) ** max(self.mask_power, 0.001), min=self.min_keep_prob, max=1.0)
-
-        rng = torch.Generator(device=a.device).manual_seed(self.seed)
-        random_mask = torch.bernoulli(prob, generator=rng)
-        interpolated_mask = torch.lerp(random_mask, prob, self.mask_smooth)
-
-        rescaled_delta = delta_val * interpolated_mask
-        return rescaled_delta
-
-
-class EnhancedDARETIESOp(Operation):
-    def __init__(self, key, alpha, beta, gamma, delta, epsilon, zeta, seed, *sources):
-        super().__init__(key, *sources)
-        self.alpha = alpha       # Unused directly for drop scale
-        self.beta = beta         # TIES trim quantile
-        self.gamma = gamma       # TIES lambda scale
-        self.mask_power = delta
-        self.min_keep_prob = epsilon
-        self.mask_smooth = zeta
-        self.seed = seed
-
-    def oper(self, a, b):
-        if a is None or b is None: return None
-        a, b = resize_tensors(a, b, mode=self.alignment_mode)
-        delta_val = (b - a).float()
-        abs_delta = torch.abs(delta_val)
-        max_delta = torch.max(abs_delta)
-
-        if max_delta > 0:
-            prob = torch.clamp((abs_delta / max_delta) ** max(self.mask_power, 0.001), min=self.min_keep_prob, max=1.0)
-
-            rng = torch.Generator(device=a.device).manual_seed(self.seed)
-            random_mask = torch.bernoulli(prob, generator=rng)
-            interpolated_mask = torch.lerp(random_mask, prob, self.mask_smooth)
-
-            delta_val = delta_val * interpolated_mask
-
-        if self.beta > 0:
-            flat_abs = delta_val.abs().flatten()
-            k = max(1, int(flat_abs.numel() * self.beta))
-            threshold = torch.kthvalue(flat_abs, k).values
-            delta_val = torch.where(delta_val.abs() < threshold, torch.zeros_like(delta_val), delta_val)
-
-        dominant_sign = torch.sign(delta_val)
-        delta_val = torch.where(torch.sign(delta_val) == dominant_sign, delta_val, torch.zeros_like(delta_val))
-
-        return (a + self.gamma * delta_val).to(a.dtype)
-
-
-class EnhancedPowerUp(CalcMode):
-    name = 'Power-Up Enhanced (DARE)'
-    description = 'Adds capabilities of B to A using Enhanced Drop and Rescale based on relative tensor differences.'
-    models_used = ['A', 'B']
-    param_docs = {
-        'alpha': 'Mask power (Curve. 2.0 = quadratic).',
-        'beta': 'Minimum keep probability (Floor to prevent explosion).',
-        'gamma': 'Mask smoothness factor (0.0 = pure dropout, 1.0 = soft scale).',
-        'delta': 'Multiplier for the final difference.',
-    }
-    def create_recipe(self, key, **kwargs):
-        mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
-
-        if '_tensor_a' in kwargs:
-            a = PreloadedTensor(key, kwargs['_tensor_a'])
-        else:
-            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
-
-        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
-        b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
-
-        deltahat = EnhancedPowerUpOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['delta'], kwargs['seed'], a, b)
-        deltahat.alignment_mode = alignment_mode
-        multiplied = Multiply(key, kwargs['delta'], deltahat)
-        res = Add(key, a, multiplied)
-        res.alignment_mode = alignment_mode
-        return res
-
-
-class EnhancedDARETIESMode(CalcMode):
-    name = 'Power-Up Enhanced (DARE+TIES)'
-    description = 'Adds capabilities of B to A using Enhanced DARE mask followed by TIES.'
-    models_used = ['A', 'B']
-    param_docs = {
-        'alpha': 'Unused (Drop scale handled via mask interpolation).',
-        'beta': 'TIES trim quantile (fraction of smallest-magnitude delta zeroed).',
-        'gamma': 'TIES Lambda scale.',
-        'delta': 'Mask power (Curve. 2.0 = quadratic).',
-        'epsilon': 'Minimum keep probability (Floor to prevent explosion).',
-        'zeta': 'Mask smoothness factor (0.0 = pure dropout, 1.0 = soft scale).',
-    }
-
-    def create_recipe(self, key, **kwargs):
-        mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
-
-        if '_tensor_a' in kwargs:
-            a = PreloadedTensor(key, kwargs['_tensor_a'])
-        else:
-            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
-
-        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
-        b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
-
-        op = EnhancedDARETIESOp(
-            key,
-            kwargs.get('alpha', 0.5),
-            kwargs.get('beta', 0.2),
-            kwargs.get('gamma', 1.0),
-            kwargs.get('delta', 0.5),
-            kwargs.get('epsilon', 0.5),
-            kwargs.get('zeta', 0.5),
-            kwargs['seed'],
-            a, b,
-        )
-        op.alignment_mode = alignment_mode
-        return op
-
-
-TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), DARETIESMode(), SVDMode(), ManEnhInterpDifference(), AutoEnhInterpDifference(), WeightSumCutoffMode(), EnhancedPowerUp(), EnhancedDARETIESMode()]
+TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), DARETIESMode(), SVDMode(), ManEnhInterpDifference(), AutoEnhInterpDifference(), WeightSumCutoffMode()]
 THREE_MODEL_MODES = [AddDifference(), TrainDifference(), Extract(), AddDisimilarity()]

nodes/merger_utils.py

@@ -1,7 +1,8 @@
 """
 Utilities for model merging operations.
 """
-from typing import Dict, List
+from typing import Dict, List, Tuple, Any
+import re
 
 def get_union_keys(*handlers) -> List[str]:
     """Returns the union of all tensor keys across multiple model handlers."""
@@ -21,3 +22,84 @@ def get_intersection_keys(*handlers) -> List[str]:
     for handler in handlers[1:]:
         all_keys &= set(handler.keys())
     return sorted(all_keys)
+
+
+def parse_custom_weights(custom_weights_str: str) -> List[Tuple[re.Pattern, Dict[str, float]]]:
+    """Parses multiline string of custom weights.
+    Format: pattern:weight OR pattern:param=value,param2=value
+    Returns list of (compiled_regex, parameter_dict).
+    """
+    if not custom_weights_str or not custom_weights_str.strip():
+        return []
+
+    results = []
+    lines = custom_weights_str.strip().split('\n')
+    for line in lines:
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+
+        if ':' not in line:
+            continue
+
+        pattern_str, weight_str = line.rsplit(':', 1)
+        pattern_str = pattern_str.strip()
+        weight_str = weight_str.strip()
+
+        try:
+            pattern = re.compile(pattern_str)
+        except re.error:
+            # Fallback to literal substring if regex invalid
+            pattern = re.compile(re.escape(pattern_str))
+
+        params = {}
+        if '=' in weight_str:
+            # param=value,param2=value
+            parts = weight_str.split(',')
+            for part in parts:
+                if '=' in part:
+                    k, v = part.split('=', 1)
+                    k = k.strip()
+                    # Map common aliases
+                    if k in ['a', 'alpha']: k = 'alpha'
+                    elif k in ['b', 'beta']: k = 'beta'
+                    elif k in ['g', 'gamma']: k = 'gamma'
+                    elif k in ['d', 'delta']: k = 'delta'
+                    elif k in ['w', 'weight']: k = 'weight'
+                    elif re.match(r'^w[1-8]$', k): pass # Keep w1-w8 as is
+
+                    try:
+                        params[k] = float(v.strip())
+                    except ValueError:
+                        pass
+        else:
+            # single float or comma-separated list of floats (for multi-LoRA)
+            if ',' in weight_str:
+                parts = weight_str.split(',')
+                for i, p in enumerate(parts):
+                    try:
+                        params[f'w{i+1}'] = float(p.strip())
+                    except ValueError:
+                        pass
+            else:
+                try:
+                    val = float(weight_str)
+                    params['alpha'] = val # default for merger
+                    params['weight'] = val # default for lora_merger
+                except ValueError:
+                    pass
+
+        if params:
+            results.append((pattern, params))
+
+    return results
+
+
+def get_custom_parameters(key: str, parsed_weights: List[Tuple[re.Pattern, Dict[str, float]]], base_params: Dict[str, Any]) -> Dict[str, Any]:
+    """Returns updated parameters if key matches any pattern."""
+    new_params = base_params.copy()
+    for pattern, custom_dict in parsed_weights:
+        if pattern.search(key):
+            new_params.update(custom_dict)
+            break # First match wins
+    return new_params

nodes/prunekeys.py

@@ -4,11 +4,12 @@ import folder_paths
 import comfy.utils
 from tqdm import tqdm
 from comfy_api.latest import io
+from safetensors.torch import save_file
 from safetensors import safe_open
 from .utils import convert_pt_to_safetensors
 from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation
 
-from unifiedefficientloader import MemoryEfficientSafeOpen, IncrementalSafetensorsWriter
+from unifiedefficientloader import MemoryEfficientSafeOpen
 
 
 def _prune_keys(model_name: str, model_type: str, keys_to_prune_str: str,
@@ -39,33 +40,32 @@ def _prune_keys(model_name: str, model_type: str, keys_to_prune_str: str,
     if not patterns:
         raise ValueError("No keys/patterns provided to prune.")
 
+    # Stream tensors and filter on the fly
+    pruned_tensors = {}
+    with MemoryEfficientSafeOpen(model_path_to_load) as handler:
+        all_keys = handler.keys()
+        pbar = comfy.utils.ProgressBar(len(all_keys))
+        for key in tqdm(all_keys, desc="Pruning keys", unit="keys"):
+            is_match = False
+            if use_regex:
+                if any(re.search(pattern, key) for pattern in patterns):
+                    is_match = True
+            else:
+                if any(pattern in key for pattern in patterns):
+                    is_match = True
+
+            if not is_match:
+                pruned_tensors[key] = handler.get_tensor(key)
+            pbar.update(1)
+
     # Use [-1] for diffusion_models to get the actual diffusion_models folder, not legacy unet
     model_dir = folder_paths.get_folder_paths(model_type)[-1]
     output_path = os.path.join(model_dir, f"{output_filename.strip()}.safetensors")
 
-    # Stream tensors, filter on the fly, write immediately
-    writer = IncrementalSafetensorsWriter(output_path, metadata=metadata)
-    writer.__enter__()
-    try:
-        with MemoryEfficientSafeOpen(model_path_to_load) as handler:
-            all_keys = handler.keys()
-            pbar = comfy.utils.ProgressBar(len(all_keys))
-            for key in tqdm(all_keys, desc="Pruning keys", unit="keys"):
-                is_match = False
-                if use_regex:
-                    if any(re.search(pattern, key) for pattern in patterns):
-                        is_match = True
-                else:
-                    if any(pattern in key for pattern in patterns):
-                        is_match = True
-
-                if not is_match:
-                    writer.write(key, handler.get_tensor(key).contiguous())
-                pbar.update(1)
-    finally:
-        writer.__exit__(None, None, None)
+    save_file(pruned_tensors, output_path, metadata)
 
     # Cleanup after operation
+    del pruned_tensors
     cleanup_after_operation()
 
     return output_path

nodes/renamekeys.py

@@ -3,11 +3,12 @@ import folder_paths
 import comfy.utils
 from tqdm import tqdm
 from comfy_api.latest import io
+from safetensors.torch import save_file
 from safetensors import safe_open
 from .utils import convert_pt_to_safetensors
 from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation
 
-from unifiedefficientloader import MemoryEfficientSafeOpen, IncrementalSafetensorsWriter
+from unifiedefficientloader import MemoryEfficientSafeOpen
 
 
 
@@ -42,25 +43,24 @@ def _rename_keys(model_name: str, model_type: str, old_keys_str: str,
 
     key_map = dict(zip(old_keys, new_keys))
 
+    # Stream tensors and rename on the fly
+    renamed_tensors = {}
+    with MemoryEfficientSafeOpen(model_path_to_load) as handler:
+        original_keys = handler.keys()
+        pbar = comfy.utils.ProgressBar(len(original_keys))
+        for key in tqdm(original_keys, desc="Renaming keys", unit="keys"):
+            new_key = key_map.get(key, key)
+            renamed_tensors[new_key] = handler.get_tensor(key)
+            pbar.update(1)
+
     # Use [-1] for diffusion_models to get the actual diffusion_models folder, not legacy unet
     model_dir = folder_paths.get_folder_paths(model_type)[-1]
     output_path = os.path.join(model_dir, f"{output_filename.strip()}.safetensors")
 
-    # Stream tensors, rename on the fly, write immediately
-    writer = IncrementalSafetensorsWriter(output_path, metadata=metadata)
-    writer.__enter__()
-    try:
-        with MemoryEfficientSafeOpen(model_path_to_load) as handler:
-            original_keys = handler.keys()
-            pbar = comfy.utils.ProgressBar(len(original_keys))
-            for key in tqdm(original_keys, desc="Renaming keys", unit="keys"):
-                new_key = key_map.get(key, key)
-                writer.write(new_key, handler.get_tensor(key).contiguous())
-                pbar.update(1)
-    finally:
-        writer.__exit__(None, None, None)
+    save_file(renamed_tensors, output_path, metadata)
 
     # Cleanup after operation
+    del renamed_tensors
     cleanup_after_operation()
 
     return output_path

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "ComfyUI-ModelUtils"
 description = "[WIP]Custom nodes for handling, inspecting, modifying and creating various model files."
-version = "0.4.4"
+version = "0.3.5"
 license = { file = "LICENSE" }
 
 classifiers = [

requirements.txt

@@ -1,5 +1,5 @@
 requests
 Pillow
 mutagen
-unifiedefficientloader>=0.5.2
+unifiedefficientloader==0.4.4
 av