Add node to extract text encoder LoRA

Feature/preview meta wf loader

__init__.py

@@ -1,6 +1,7 @@
 from typing_extensions import override
 from comfy_api.latest import ComfyExtension, io
 
+from .nodes.lora_merger import LoRAMultiMerge, LoRAMultiMergeDARE
 from .nodes.metakeys import (
     ModelMetaKeys, TextEncoderMetaKeys, LoRAMetaKeys,
     CheckpointMetaKeys, EmbeddingMetaKeys
@@ -21,11 +22,20 @@ from .nodes.merger import (
 )
 from .nodes.lora_extract_svd import (
     LoRAExtractFixed, LoRAExtractRatio, LoRAExtractQuantile,
-    LoRAExtractKnee, LoRAExtractFrobenius
+    LoRAExtractKnee, LoRAExtractFrobenius, LoRATextEncoderExtractFixed
 )
 from .nodes.lora_resize import (
     LoRAResizeFixed, LoRAResizeRatio,
-    LoRAResizeFrobenius, LoRAResizeCumulative
+    LoRAResizeFrobenius, LoRAResizeCumulative,
+    LoRAMergeToModel
+)
+from .nodes.downloader_nodes import (
+    CheckpointInfoMetaDownloader, DiffusionModelInfoMetaDownloader, LoRAInfoMetaDownloader, EmbeddingInfoMetaDownloader,
+    VAEInfoMetaDownloader, ControlNetInfoMetaDownloader, ManualPathInfoMetaDownloader
+)
+from .nodes.model_info_nodes import (
+    CheckpointInfoLoader, LoRAInfoLoader, EmbeddingInfoLoader,
+    VAEInfoLoader, ControlNetInfoLoader, DiffusionModelInfoLoader
 )
 
 
@@ -50,10 +60,20 @@ class ModelUtilsExtension(ComfyExtension):
             CheckpointThreeMerger, EmbeddingThreeMerger,
             # LoRA Extraction
             LoRAExtractFixed, LoRAExtractRatio, LoRAExtractQuantile,
-            LoRAExtractKnee, LoRAExtractFrobenius,
+            LoRAExtractKnee, LoRAExtractFrobenius, LoRATextEncoderExtractFixed,
             # LoRA Resize
             LoRAResizeFixed, LoRAResizeRatio,
             LoRAResizeFrobenius, LoRAResizeCumulative,
+            # LoRA Multi-Merge
+            LoRAMultiMerge, LoRAMultiMergeDARE,
+            # LoRA Merge To Model
+            LoRAMergeToModel,
+            # Downloaders
+            CheckpointInfoMetaDownloader, DiffusionModelInfoMetaDownloader, LoRAInfoMetaDownloader, EmbeddingInfoMetaDownloader,
+            VAEInfoMetaDownloader, ControlNetInfoMetaDownloader, ManualPathInfoMetaDownloader,
+            # Info Loaders
+            CheckpointInfoLoader, LoRAInfoLoader, EmbeddingInfoLoader,
+            VAEInfoLoader, ControlNetInfoLoader, DiffusionModelInfoLoader,
         ]
 
 

docs/DEVELOPMENT.md

@@ -294,3 +294,28 @@ ComfyUI-ModelUtils/
 | Metadata reading | **100x+** (header only) |
 | Parallel I/O | **2-4x** (ThreadPoolExecutor) |
 | Memory management | Auto-unload models before heavy ops |
+| **LoRA Resize Fast** | **~91x** (merge-to-base extraction) |
+
+---
+
+## Phase 13: Fast LoRA Resize (2026-01-11)
+
+### Summary
+Added fast LoRA resize via base model extraction path, achieving ~91x speedup.
+
+### Changes
+- Added `resize_lora_via_base()` function using extraction code path
+- Added 4 new fast resize nodes requiring base model input
+
+### Files Modified
+
+| File | Changes |
+|------|---------|
+| `nodes/lora_resize.py` | Added `resize_lora_via_base()`, 4 new node classes |
+| `__init__.py` | Registered new nodes |
+
+### New Nodes
+- `LoRAResizeViaBaseFixed` - Fast fixed rank resize
+- `LoRAResizeViaBaseRatio` - Fast SV ratio resize  
+- `LoRAResizeViaBaseFrobenius` - Fast Frobenius resize
+- `LoRAResizeViaBaseCumulative` - Fast cumulative resize

docs/merger_2_model_modes.md

@@ -23,12 +23,71 @@
 ---
 
 ## Power-Up (DARE)
-> Adds the unique capabilities of Model B to Model A using the Drop and Rescale (DARE) technique, which often preserves the knowledge of the base model better than simple additions.
+> Adds the unique capabilities of Model B to Model A using the Drop and Rescale (DARE) technique. This implementation handles shape mismatches between models by padding and uses a randomized dropout mask.
 
 **Models Used:** A, B
 **Parameters:**
-- **Alpha:** The dropout rate. This is the proportion of unique weights from Model B that are *dropped* before merging. A higher value means less of B is merged.
+- **Alpha:** The dropout rate ($p$). This is the proportion of delta parameters from Model B that are randomly set to zero.
 - **Beta:** A final multiplier for the rescaled difference before it's added to Model A.
+- **Rescaling Logic:** Remaining weights are automatically rescaled by $1/(1-p)$ as per the DARE paper to approximate the original embeddings.
+
+---
+
+## Power-Up (DARE+TIES)
+> Combines DARE (Drop and Rescale) with TIES (Trim, Elect Sign) — the current best-practice algorithm for merging fine-tuned models, particularly LoRA-tuned ones. DARE sparsifies the task vector via random dropout and rescaling; TIES then removes residual low-magnitude noise and enforces sign consistency. The result is a cleaner, more targeted capability transfer from B to A with less interference.
+
+**Models Used:** A, B
+**Parameters:**
+- **Alpha:** DARE drop rate. Fraction of delta parameters randomly zeroed. Higher values produce a sparser, more targeted delta. Typical range: `0.5`–`0.9`.
+- **Beta:** TIES trim quantile. Fraction of the smallest-magnitude delta parameters zeroed after DARE, removing residual noise. `0.0` disables trimming. Typical range: `0.1`–`0.3`.
+- **Gamma:** Lambda scale. Final multiplier applied to the merged delta before adding to Model A. Equivalent to the task arithmetic scaling coefficient. Typical range: `0.5`–`1.5`.
+- **Seed:** Random seed for the DARE dropout mask. Controls reproducibility.
+
+**Algorithm:**
+1. Compute task vector `δ = B − A`
+2. **DARE:** apply random binary mask with keep probability `1 − alpha`; rescale survivors by `1/(1−alpha)`
+3. **TIES trim:** zero out parameters below the `beta`-quantile magnitude threshold
+4. **TIES elect:** determine dominant sign per position; zero out disagreeing parameters
+5. Return `A + gamma × δ_filtered`
+
+**Guidance:**
+- Start with `alpha=0.9, beta=0.2, gamma=1.0` for LoRA-derived models
+- Lower `alpha` (e.g. `0.5`) for full fine-tunes where more delta mass should be retained
+- Increase `gamma` beyond `1.0` to amplify B's influence; decrease below `1.0` to soften it
+
+---
+
+## Enhanced Man Interp
+> Sophisticated interpolation between values from A and B depending on their difference relative to other values, with manual threshold control.
+
+**Models Used:** A, B
+**Parameters:**
+- **Alpha:** Interpolation strength.
+- **Beta:** Lower mean threshold for filtering differences.
+- **Gamma:** Upper mean threshold for filtering differences.
+- **Delta:** Smoothness factor (mix between randomized mask and powered differences).
+
+---
+
+## Enhanced Auto Interp
+> Automated version of the enhanced interpolation mode that dynamically calculates thresholds based on mean differences.
+
+**Models Used:** A, B
+**Parameters:**
+- **Alpha:** Interpolation strength.
+- **Beta:** Threshold adjustment factor.
+- **Gamma:** Smoothness factor.
+
+---
+
+## Weight-Sum Cutoff
+> A linear interpolation mode that only applies the merge to weights whose differences fall within a specific threshold range.
+
+**Models Used:** A, B
+**Parameters:**
+- **Alpha:** Interpolation weight (multiplier for the difference).
+- **Beta:** Upper threshold for the difference cutoff.
+- **Gamma:** Lower threshold for the difference cutoff.
 
 ---
 
@@ -84,4 +143,4 @@ Layers matching any pattern will be **removed entirely** from the output.
 | `text_model` | All text encoder layers |
 | `\.norm` | All normalization layers |
 | `attn\.(q\|k\|v)` | Query, key, value attention weights |
-| `block\.[0-9]\.` | Blocks 0-9 |
+| `block\.[0-9]\.` | Blocks 0-9 |

docs/merger_3_model_modes.md

@@ -22,19 +22,19 @@
 ---
 
 ## Extract-Features
-> A powerful mode that identifies features present in both `(B - A)` and `(C - A)` and adds them to A. Allows for fine-grained control over combining aspects based on their similarity.
+> A powerful mode that identifies features present in both `(B - A)` and `(C - A)` and adds them to A. It uses per-vector cosine similarity to decide how much of each feature to keep, allowing for fine-grained control over combining aspects.
 
 **Models Used:** A, B, C
 **Parameters:**
 - **Alpha:** Weights the merge between Model B (`0.0`) and Model C (`1.0`).
 - **Beta:** Controls the focus on similarity (`0.0`) versus dissimilarity (`1.0`).
-- **Gamma:** A bias exponent for similarity. Higher values increase the bias.
+- **Gamma:** A bias exponent for similarity calculation.
 - **Delta:** A final multiplier for the extracted features before they are added to Model A.
 
 ---
 
 ## Add-Dissimilarities
-> Identifies features that are dissimilar between Model B and Model C and adds them to Model A. Useful for combining unique aspects of two different models.
+> Identifies features that are dissimilar between Model B and Model C (relative to A) and adds them to Model A. Useful for combining unique aspects of two different models.
 
 **Models Used:** A, B, C
 **Parameters:**
@@ -80,4 +80,4 @@ Layers matching any pattern will be **removed entirely** from the output.
 **Pattern format:**
 - Whitespace-separated regex patterns
 - Patterns use **substring matching** (not full match)
-- Example: `text_model lora` matches any key containing "text_model" OR "lora"
+- Example: `text_model lora` matches any key containing "text_model" OR "lora"

nodes/downloader_nodes.py

@@ -0,0 +1,341 @@
+import os
+import folder_paths
+from comfy_api.latest import io
+from .downloader_utils import scan_and_process
+
+def get_subdirectories(folder_name: str) -> list[str]:
+    """Helper to get a list of subdirectories for a given model folder name."""
+    paths = folder_paths.get_folder_paths(folder_name)
+    if not paths:
+        return ["/"]
+
+    subdirs = set()
+    for base_path in paths:
+        if not os.path.exists(base_path):
+            continue
+        for root, dirs, _ in os.walk(base_path):
+            # Ignore hidden directories directly during walk
+            dirs[:] = [d for d in dirs if not d.startswith('.')]
+
+            rel_path = os.path.relpath(root, base_path)
+            if rel_path == ".":
+                subdirs.add("/")
+            else:
+                # Store with forward slashes for consistency
+                subdirs.add("/" + rel_path.replace("\\", "/"))
+
+    return sorted(list(subdirs))
+
+class BaseInfoMetaDownloaderNode(io.ComfyNode):
+    CATEGORY = "ModelUtils/Downloader"
+    OUTPUT_NODE = True
+
+    @classmethod
+    def get_common_inputs(cls):
+        return {
+            "recursive": io.Boolean.Input("recursive", default=True, tooltip="Whether to scan subdirectories recursively."),
+            "nsfw_level": io.Combo.Input("nsfw_level", options=["None", "Soft", "Mature", "X", "XXX", "All"], default="All", tooltip="Filter models based on NSFW level in metadata."),
+            "max_examples": io.Int.Input("max_examples", default=0, min=0, max=100, tooltip="Maximum number of examples to process."),
+            "threads": io.Int.Input("threads", default=4, min=1, max=32, tooltip="Number of threads to use for processing."),
+            "api_key": io.String.Input("api_key", default="", tooltip="API key for accessing the model repository.")
+        }
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name=cls.__name__.replace("Downloader", " Downloader"),
+            category=cls.CATEGORY,
+            inputs=list(cls.get_common_inputs().values()),
+            outputs=[io.String.Output(display_name="status")],
+            is_output_node=True,
+            description=("Scans specified directories for models, extracts metadata, generates previews, and optionally compresses files.")
+        )
+
+    @classmethod
+    def execute(cls, recursive, nsfw_level, max_examples, threads, api_key, **kwargs):
+        scan_dirs = cls.get_scan_dirs(**kwargs)
+        if not scan_dirs:
+            return io.NodeOutput("No directories to scan.")
+
+        # Call the scan_and_process function
+        stats = scan_and_process(
+            scan_dirs=scan_dirs,
+            recursive=recursive,
+            nsfw_level=nsfw_level,
+            max_examples=max_examples,
+            api_key=api_key if api_key else None,
+            threads=threads
+        )
+
+        mb_saved = stats.get("space_saved", 0) / (1024 * 1024)
+
+        output_msg = (
+            f"Scan Complete for {len(scan_dirs)} directory/directories.\n"
+            f"Total Models Processed: {stats.get('total_processed', 0)}\n"
+            f"New Models Hashed: {stats.get('new_hashes', 0)}\n"
+            f"Images Processed: {stats.get('images_processed', 0)}\n"
+            f"Workflows Extracted: {stats.get('workflows_extracted', 0)}\n"
+            f"Space Saved via Compression: {mb_saved:.2f} MB"
+        )
+
+        return io.NodeOutput(output_msg)
+
+    @classmethod
+    def get_scan_dirs(cls, **kwargs) -> list[str]:
+        return []
+
+class CheckpointInfoMetaDownloader(BaseInfoMetaDownloaderNode):
+    @classmethod
+    def define_schema(cls):
+        inputs = list(cls.get_common_inputs().values())
+        inputs.insert(0, io.Combo.Input("subdirectory", options=get_subdirectories("checkpoints"), default="/"))
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="Checkpoint Downloader",
+            category=cls.CATEGORY,
+            inputs=inputs,
+            outputs=[io.String.Output(display_name="status")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, subdirectory, recursive, nsfw_level, max_examples, threads, api_key, **kwargs):
+        return super().execute(
+            recursive=recursive, nsfw_level=nsfw_level, max_examples=max_examples,
+            threads=threads, api_key=api_key, subdirectory=subdirectory, **kwargs
+        )
+
+    @classmethod
+    def get_scan_dirs(cls, **kwargs):
+        base_paths = folder_paths.get_folder_paths("checkpoints")
+        if not base_paths:
+            return []
+        sub = kwargs.get("subdirectory", "/")
+        if sub == "/":
+            return base_paths
+
+        target_paths = []
+        for bp in base_paths:
+            target_path = os.path.join(bp, sub.lstrip("/"))
+            if os.path.exists(target_path):
+                target_paths.append(target_path)
+        return target_paths
+
+class DiffusionModelInfoMetaDownloader(BaseInfoMetaDownloaderNode):
+    @classmethod
+    def define_schema(cls):
+        inputs = list(cls.get_common_inputs().values())
+        inputs.insert(0, io.Combo.Input("subdirectory", options=get_subdirectories("diffusion_models"), default="/"))
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="Diffusion Model Downloader",
+            category=cls.CATEGORY,
+            inputs=inputs,
+            outputs=[io.String.Output(display_name="status")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, subdirectory, recursive, nsfw_level, max_examples, threads, api_key, **kwargs):
+        return super().execute(
+            recursive=recursive, nsfw_level=nsfw_level, max_examples=max_examples,
+            threads=threads, api_key=api_key, subdirectory=subdirectory, **kwargs
+        )
+
+    @classmethod
+    def get_scan_dirs(cls, **kwargs):
+        base_paths = folder_paths.get_folder_paths("diffusion_models")
+        if not base_paths:
+            return []
+        sub = kwargs.get("subdirectory", "/")
+        if sub == "/":
+            return base_paths
+
+        target_paths = []
+        for bp in base_paths:
+            target_path = os.path.join(bp, sub.lstrip("/"))
+            if os.path.exists(target_path):
+                target_paths.append(target_path)
+        return target_paths
+
+class LoRAInfoMetaDownloader(BaseInfoMetaDownloaderNode):
+    @classmethod
+    def define_schema(cls):
+        inputs = list(cls.get_common_inputs().values())
+        inputs.insert(0, io.Combo.Input("subdirectory", options=get_subdirectories("loras"), default="/"))
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="LoRA Downloader",
+            category=cls.CATEGORY,
+            inputs=inputs,
+            outputs=[io.String.Output(display_name="status")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, subdirectory, recursive, nsfw_level, max_examples, threads, api_key, **kwargs):
+        return super().execute(
+            recursive=recursive, nsfw_level=nsfw_level, max_examples=max_examples,
+            threads=threads, api_key=api_key, subdirectory=subdirectory, **kwargs
+        )
+
+    @classmethod
+    def get_scan_dirs(cls, **kwargs):
+        base_paths = folder_paths.get_folder_paths("loras")
+        if not base_paths:
+            return []
+        sub = kwargs.get("subdirectory", "/")
+        if sub == "/":
+            return base_paths
+
+        target_paths = []
+        for bp in base_paths:
+            target_path = os.path.join(bp, sub.lstrip("/"))
+            if os.path.exists(target_path):
+                target_paths.append(target_path)
+        return target_paths
+
+class EmbeddingInfoMetaDownloader(BaseInfoMetaDownloaderNode):
+    @classmethod
+    def define_schema(cls):
+        inputs = list(cls.get_common_inputs().values())
+        inputs.insert(0, io.Combo.Input("subdirectory", options=get_subdirectories("embeddings"), default="/"))
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="Embedding Downloader",
+            category=cls.CATEGORY,
+            inputs=inputs,
+            outputs=[io.String.Output(display_name="status")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, subdirectory, recursive, nsfw_level, max_examples, threads, api_key, **kwargs):
+        return super().execute(
+            recursive=recursive, nsfw_level=nsfw_level, max_examples=max_examples,
+            threads=threads, api_key=api_key, subdirectory=subdirectory, **kwargs
+        )
+
+    @classmethod
+    def get_scan_dirs(cls, **kwargs):
+        base_paths = folder_paths.get_folder_paths("embeddings")
+        if not base_paths:
+            return []
+        sub = kwargs.get("subdirectory", "/")
+        if sub == "/":
+            return base_paths
+
+        target_paths = []
+        for bp in base_paths:
+            target_path = os.path.join(bp, sub.lstrip("/"))
+            if os.path.exists(target_path):
+                target_paths.append(target_path)
+        return target_paths
+
+class VAEInfoMetaDownloader(BaseInfoMetaDownloaderNode):
+    @classmethod
+    def define_schema(cls):
+        inputs = list(cls.get_common_inputs().values())
+        inputs.insert(0, io.Combo.Input("subdirectory", options=get_subdirectories("vae"), default="/"))
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="VAE Downloader",
+            category=cls.CATEGORY,
+            inputs=inputs,
+            outputs=[io.String.Output(display_name="status")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, subdirectory, recursive, nsfw_level, max_examples, threads, api_key, **kwargs):
+        return super().execute(
+            recursive=recursive, nsfw_level=nsfw_level, max_examples=max_examples,
+            threads=threads, api_key=api_key, subdirectory=subdirectory, **kwargs
+        )
+
+    @classmethod
+    def get_scan_dirs(cls, **kwargs):
+        base_paths = folder_paths.get_folder_paths("vae")
+        if not base_paths:
+            return []
+        sub = kwargs.get("subdirectory", "/")
+        if sub == "/":
+            return base_paths
+
+        target_paths = []
+        for bp in base_paths:
+            target_path = os.path.join(bp, sub.lstrip("/"))
+            if os.path.exists(target_path):
+                target_paths.append(target_path)
+        return target_paths
+
+class ControlNetInfoMetaDownloader(BaseInfoMetaDownloaderNode):
+    @classmethod
+    def define_schema(cls):
+        inputs = list(cls.get_common_inputs().values())
+        inputs.insert(0, io.Combo.Input("subdirectory", options=get_subdirectories("controlnet"), default="/"))
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="ControlNet Downloader",
+            category=cls.CATEGORY,
+            inputs=inputs,
+            outputs=[io.String.Output(display_name="status")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, subdirectory, recursive, nsfw_level, max_examples, threads, api_key, **kwargs):
+        return super().execute(
+            recursive=recursive, nsfw_level=nsfw_level, max_examples=max_examples,
+            threads=threads, api_key=api_key, subdirectory=subdirectory, **kwargs
+        )
+
+    @classmethod
+    def get_scan_dirs(cls, **kwargs):
+        base_paths = folder_paths.get_folder_paths("controlnet")
+        if not base_paths:
+            return []
+        sub = kwargs.get("subdirectory", "/")
+        if sub == "/":
+            return base_paths
+
+        target_paths = []
+        for bp in base_paths:
+            target_path = os.path.join(bp, sub.lstrip("/"))
+            if os.path.exists(target_path):
+                target_paths.append(target_path)
+        return target_paths
+
+class ManualPathInfoMetaDownloader(BaseInfoMetaDownloaderNode):
+    @classmethod
+    def define_schema(cls):
+        inputs = list(cls.get_common_inputs().values())
+        inputs.append(io.String.Input("scan_dir", default=""))
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="Manual Path Downloader",
+            category=cls.CATEGORY,
+            inputs=inputs,
+            outputs=[io.String.Output(display_name="status")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, recursive, nsfw_level, max_examples, threads, api_key, scan_dir, **kwargs):
+        return super().execute(
+            recursive=recursive,
+            nsfw_level=nsfw_level,
+            max_examples=max_examples,
+            threads=threads,
+            api_key=api_key,
+            scan_dir=scan_dir,
+            **kwargs
+        )
+
+    @classmethod
+    def get_scan_dirs(cls, **kwargs):
+        scan_dir = kwargs.get("scan_dir", "")
+        if scan_dir.strip():
+            return [scan_dir.strip()]
+        return []

nodes/downloader_utils.py

@@ -0,0 +1,695 @@
+import os
+import sys
+import hashlib
+import json
+import sqlite3
+import requests
+import glob
+from pathlib import Path
+import threading
+import concurrent.futures
+import codecs
+import shutil
+
+try:
+    from PIL import Image
+    HAS_PIL = True
+except ImportError:
+    HAS_PIL = False
+    print("Warning: Pillow not found. Image workflow extraction disabled.")
+
+try:
+    from mutagen.mp4 import MP4
+    HAS_MUTAGEN = True
+except ImportError:
+    HAS_MUTAGEN = False
+    print("Warning: mutagen not found. MP4 tag extraction will rely entirely on av fallback.")
+
+try:
+    import av
+    HAS_AV = True
+except ImportError:
+    HAS_AV = False
+    print("Warning: av (PyAV) not found. MP4 metadata extraction fallback disabled.")
+
+# Add the reference module to the Python path
+# Assuming downloader_utils.py is inside nodes/, the root is one level up
+root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(os.path.join(root_dir, 'unifiedefficientloader-reference'))
+from unifiedefficientloader import UnifiedSafetensorsLoader
+
+# --- Database Path Logic ---
+try:
+    import folder_paths
+    base_user_dir = folder_paths.get_public_user_directory("default")
+    if not base_user_dir:
+        base_user_dir = root_dir
+except (ImportError, AttributeError):
+    base_user_dir = root_dir
+
+PLUGIN_DATA_DIR = os.path.join(base_user_dir, "ComfyUI-ModelUtils")
+os.makedirs(PLUGIN_DATA_DIR, exist_ok=True)
+DEFAULT_DB_PATH = os.path.join(PLUGIN_DATA_DIR, "cache.db")
+
+
+# --- Workflow Extraction Logic ---
+
+def unescape_and_parse_nested_json(json_string):
+    if not isinstance(json_string, str): return None
+    stripped = json_string.strip()
+    if not (stripped.startswith('{') or stripped.startswith('[')): return None
+    try: return json.loads(stripped)
+    except json.JSONDecodeError: pass
+    try:
+        first_parse = json.loads(stripped)
+        if isinstance(first_parse, str): return json.loads(first_parse)
+    except json.JSONDecodeError: pass
+    try:
+        decoded = codecs.decode(stripped, "unicode_escape")
+        return json.loads(decoded)
+    except (json.JSONDecodeError, UnicodeDecodeError, ValueError): pass
+    try:
+        unescaped = stripped.replace('\\\\"', '\\"').replace('\\"', '"').replace("\\\\", "\\")
+        unescaped = unescaped.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t")
+        return json.loads(unescaped)
+    except json.JSONDecodeError: pass
+    return None
+
+def get_workflow_blob(metadata):
+    for key in ['workflow', 'prompt']:
+        val = metadata.get(key)
+        if val:
+            parsed = unescape_and_parse_nested_json(val)
+            if parsed: return parsed
+    for val in metadata.values():
+        parsed = unescape_and_parse_nested_json(val)
+        if isinstance(parsed, dict) and (any(k.isdigit() for k in parsed.keys()) or 'nodes' in parsed):
+            return parsed
+    return None
+
+def process_image_workflow_and_compress(filepath_str):
+    if not HAS_PIL: return {"images_processed": 0, "workflows_extracted": 0, "space_saved": 0}
+    p = Path(filepath_str)
+    if p.suffix.lower() not in ['.png', '.jpg', '.jpeg']: return {"images_processed": 0, "workflows_extracted": 0, "space_saved": 0}
+
+    workflow = None
+    actual_format = None
+    stats = {"images_processed": 1, "workflows_extracted": 0, "space_saved": 0}
+    try:
+        with Image.open(p) as img:
+            actual_format = img.format
+            # CivitAI sometimes serves PNGs with .jpeg extension in URL
+            if actual_format == 'PNG':
+                workflow = get_workflow_blob(img.info)
+                if workflow:
+                    # Save workflow
+                    json_path = p.with_suffix('.json')
+                    if not json_path.exists():
+                        with open(json_path, 'w', encoding='utf-8') as f:
+                            json.dump(workflow, f, indent=2, ensure_ascii=False)
+                        print(f"Extracted workflow to {json_path}")
+                        stats["workflows_extracted"] = 1
+
+                    # Compress image to WebP
+                    webp_path = p.with_suffix('.webp')
+                    if not webp_path.exists():
+                        orig_size = os.path.getsize(p)
+                        img.save(webp_path, 'WEBP', quality=85, method=4)
+                        new_size = os.path.getsize(webp_path)
+                        stats["space_saved"] = max(0, orig_size - new_size)
+                        print(f"Compressed {p.name} (was actually {actual_format}) to {webp_path.name}")
+
+        # Delete original if we created a webp replacement
+        if workflow and p.with_suffix('.webp').exists() and actual_format == 'PNG':
+            os.remove(p)
+            print(f"Deleted original uncompressed image: {p.name}")
+    except Exception as e:
+        print(f"Failed to process image {p.name}: {e}")
+    return stats
+
+# --- Video Workflow Logic ---
+def get_tag_value(mp4, keys):
+    for key in keys:
+        if key in mp4.tags: return mp4.tags[key][0] if mp4.tags[key] else None
+    for key in keys:
+        search_suffix = key.split(':')[-1].lower()
+        for tag_key, val in mp4.tags.items():
+            if tag_key.lower().endswith(f":{search_suffix}"): return val[0] if val else None
+    return None
+
+def extract_metadata_av(mp4_path):
+    if not HAS_AV: return None
+    try:
+        with av.open(str(mp4_path)) as container:
+            return {k.lower(): v for k, v in container.metadata.items()}
+    except Exception as e:
+        print(f"av extraction error for {mp4_path}: {e}")
+        return None
+
+def process_video_workflow(filepath_str):
+    p = Path(filepath_str)
+    if p.suffix.lower() != '.mp4': return {"images_processed": 0, "workflows_extracted": 0, "space_saved": 0}
+
+    json_path = p.with_suffix('.json')
+    if json_path.exists(): return {"images_processed": 0, "workflows_extracted": 0, "space_saved": 0}
+
+    workflow_data = None
+    mp4 = None
+    stats = {"images_processed": 0, "workflows_extracted": 0, "space_saved": 0}
+    if HAS_MUTAGEN:
+        try: mp4 = MP4(p)
+        except: pass
+
+    if mp4:
+        for tag_keys in [['----:QuickTime:Workflow', 'workflow'], ['----:QuickTime:Prompt', 'prompt'], ['©cmt']]:
+            val = get_tag_value(mp4, tag_keys)
+            if val:
+                if isinstance(val, bytes): val = val.decode('utf-8', errors='ignore')
+                try:
+                    parsed = json.loads(val)
+                    if isinstance(parsed, dict) and ('prompt' in parsed or any(k.isdigit() for k in parsed.keys()) or 'nodes' in parsed):
+                        if 'prompt' in parsed and isinstance(parsed['prompt'], str):
+                            try: workflow_data = json.loads(parsed['prompt'])
+                            except: workflow_data = parsed
+                        else:
+                            workflow_data = parsed
+                        break
+                except json.JSONDecodeError:
+                    # Try more aggressive parsing if it's a nested string
+                    parsed = unescape_and_parse_nested_json(val)
+                    if parsed:
+                        workflow_data = parsed
+                        break
+
+    if not workflow_data:
+        av_data = extract_metadata_av(p)
+        if av_data:
+            for key in ['workflow', 'prompt', 'comment', 'description']:
+                if key in av_data:
+                    val = av_data[key]
+                    try:
+                        parsed = json.loads(val)
+                        if isinstance(parsed, dict):
+                            if 'prompt' in parsed and isinstance(parsed['prompt'], str):
+                                try: workflow_data = json.loads(parsed['prompt'])
+                                except: workflow_data = parsed
+                            else:
+                                workflow_data = parsed
+                            break
+                    except json.JSONDecodeError:
+                        parsed = unescape_and_parse_nested_json(val)
+                        if parsed:
+                            workflow_data = parsed
+                            break
+
+    if workflow_data:
+        try:
+            with open(json_path, 'w', encoding='utf-8') as f:
+                json.dump(workflow_data, f, indent=2, ensure_ascii=False)
+            print(f"Extracted workflow to {json_path.name}")
+            stats["workflows_extracted"] = 1
+        except Exception as e:
+            print(f"Error saving workflow for {p.name}: {e}")
+    return stats
+
+# --- Constants for Civitai NSFW levels ---
+NSFW_LEVELS = {
+    "None": 1,
+    "Soft": 2,
+    "Mature": 4,
+    "X": 8,
+    "XXX": 16,
+    "All": 32  # Anything
+}
+
+class HashCache:
+    def __init__(self, db_path=None):
+        self.db_path = db_path if db_path else DEFAULT_DB_PATH
+        self.is_sqlite = self.db_path.endswith('.db') or self.db_path.endswith('.sqlite')
+        self.lock = threading.Lock()
+        if self.is_sqlite:
+            self._init_sqlite()
+        else:
+            self.cache = self._load_json()
+
+    def _init_sqlite(self):
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                CREATE TABLE IF NOT EXISTS file_hashes (
+                    filepath TEXT PRIMARY KEY,
+                    mtime REAL,
+                    size INTEGER,
+                    hash TEXT
+                )
+            ''')
+            conn.commit()
+
+    def _load_json(self):
+        if os.path.exists(self.db_path):
+            with open(self.db_path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        return {}
+
+    def _save_json(self):
+        with open(self.db_path, "w", encoding="utf-8") as f:
+            json.dump(self.cache, f, indent=4)
+
+    def _get_sqlite_hash(self, filepath, mtime, size):
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT mtime, size, hash FROM file_hashes WHERE filepath=?", (filepath,))
+            row = cursor.fetchone()
+            if row:
+                db_mtime, db_size, db_hash = row
+                if db_mtime == mtime and db_size == size:
+                    return db_hash
+        return None
+
+    def _set_sqlite_hash(self, filepath, mtime, size, file_hash):
+        with sqlite3.connect(self.db_path, timeout=10.0) as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                INSERT OR REPLACE INTO file_hashes (filepath, mtime, size, hash)
+                VALUES (?, ?, ?, ?)
+            ''', (filepath, mtime, size, file_hash))
+            conn.commit()
+
+    def get_hash(self, filepath):
+        if not os.path.exists(filepath):
+            return None
+        mtime = os.path.getmtime(filepath)
+        size = os.path.getsize(filepath)
+        key = filepath
+
+        with self.lock:
+            if self.is_sqlite:
+                cached_hash = self._get_sqlite_hash(key, mtime, size)
+                if cached_hash: return cached_hash
+            else:
+                if key in self.cache:
+                    entry = self.cache[key]
+                    if entry["mtime"] == mtime and entry["size"] == size:
+                        return entry["hash"]
+
+        # Calculate new hash (outside the lock to allow parallel hashing)
+        print(f"Calculating SHA256 for: {filepath}")
+        sha256_hash = self.calculate_sha256(filepath)
+
+        with self.lock:
+            if self.is_sqlite:
+                self._set_sqlite_hash(key, mtime, size, sha256_hash)
+            else:
+                self.cache[key] = {
+                    "mtime": mtime,
+                    "size": size,
+                    "hash": sha256_hash
+                }
+                self._save_json()
+
+        return sha256_hash
+
+    @staticmethod
+    def calculate_sha256(filename):
+        sha256_hash = hashlib.sha256()
+        with open(filename, "rb") as f:
+            # Read and update hash string value in blocks of 4K
+            for byte_block in iter(lambda: f.read(4096), b""):
+                sha256_hash.update(byte_block)
+        return sha256_hash.hexdigest()
+
+class CivitaiAPI:
+    def __init__(self, api_key=None):
+        self.api_key = api_key
+        self.base_url = "https://civitai.com/api/v1"
+
+    def get_headers(self):
+        headers = {}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    def get_model_info_by_hash(self, file_hash):
+        url = f"{self.base_url}/model-versions/by-hash/{file_hash}"
+        response = requests.get(url, headers=self.get_headers())
+        if response.status_code == 200:
+            return response.json()
+        elif response.status_code == 404:
+            return None
+        else:
+            response.raise_for_status()
+
+class Downloader:
+    @staticmethod
+    def download_file(url, target_path, headers=None):
+        if headers is None:
+            headers = {}
+
+        temp_path = target_path + ".downloading"
+
+        downloaded_size = 0
+        if os.path.exists(temp_path):
+            downloaded_size = os.path.getsize(temp_path)
+
+        if downloaded_size > 0:
+            headers["Range"] = f"bytes={downloaded_size}-"
+
+        response = requests.get(url, headers=headers, stream=True)
+
+        if response.status_code == 416: # Range not satisfiable, file probably changed or completed
+            os.remove(temp_path)
+            downloaded_size = 0
+            if "Range" in headers:
+                del headers["Range"]
+            response = requests.get(url, headers=headers, stream=True)
+
+        if response.status_code not in [200, 206]:
+            print(f"Failed to download from {url}, status code: {response.status_code}")
+            return False
+
+        mode = "ab" if response.status_code == 206 else "wb"
+
+        total_size = int(response.headers.get('content-length', 0)) + downloaded_size
+
+        print(f"Downloading: {target_path} ({downloaded_size}/{total_size} bytes)")
+
+        with open(temp_path, mode) as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    f.write(chunk)
+
+        os.rename(temp_path, target_path)
+        return True
+
+def get_potential_preview_files(base_path):
+    exts = ["png", "jpg", "jpeg", "webp", "mp4"]
+    files = []
+    for ext in exts:
+        files.append(f"{base_path}.preview.{ext}")
+        files.append(f"{base_path}.{ext}")
+    return files
+
+def preview_exists(base_path):
+    previews = get_potential_preview_files(base_path)
+    for p in previews:
+        if os.path.exists(p):
+            return True
+    return False
+
+def next_example_image_path(base_path, ext):
+    i = 0
+    while len(glob.glob(f"{base_path}.example.{i}.*")) > 0:
+        i += 1
+    return f"{base_path}.example.{i}.{ext}"
+
+def filter_images_by_nsfw(images, threshold_level):
+    valid_images = []
+    for img in images:
+        rating = img.get("nsfwLevel", 1)
+        if rating <= threshold_level:
+            valid_images.append(img)
+    return valid_images
+
+def extract_safetensors_metadata(filepath):
+    try:
+        # Use low_memory=True to only read the header and metadata without loading tensors
+        loader = UnifiedSafetensorsLoader(filepath, low_memory=True)
+        metadata = loader.metadata()
+        loader.close()
+        return metadata if metadata else None
+    except Exception as e:
+        print(f"Failed to read safetensors metadata from {filepath} using unifiedefficientloader: {e}")
+        return None
+
+def process_file(filepath, cache, api, nsfw_threshold, max_examples):
+    filepath_str = str(filepath)
+    base_path = os.path.splitext(filepath_str)[0]
+    is_safetensors = filepath_str.lower().endswith(".safetensors")
+
+    stats = {"new_hashes": 0, "images_processed": 0, "workflows_extracted": 0, "space_saved": 0}
+
+    # Check if all targets already exist
+    info_path = f"{base_path}.civitai.info"
+    st_metadata_path = f"{base_path}.metadata.json"
+
+    has_info = os.path.exists(info_path)
+    has_preview = preview_exists(base_path)
+    has_st_metadata = not is_safetensors or os.path.exists(st_metadata_path)
+
+    # Process existing images/videos for workflow extraction & compression BEFORE we possibly skip the file
+    for p in get_potential_preview_files(base_path):
+        if os.path.exists(p):
+            ext_lower = p.lower()
+            if ext_lower.endswith(('.png', '.jpg', '.jpeg')):
+                res = process_image_workflow_and_compress(p)
+                for k in stats: stats[k] += res.get(k, 0)
+            elif ext_lower.endswith('.mp4'):
+                res = process_video_workflow(p)
+                for k in stats: stats[k] += res.get(k, 0)
+
+    for ex in glob.glob(f"{base_path}.example.*"):
+        ext_lower = ex.lower()
+        if ext_lower.endswith(('.png', '.jpg', '.jpeg')):
+            res = process_image_workflow_and_compress(ex)
+            for k in stats: stats[k] += res.get(k, 0)
+        elif ext_lower.endswith('.mp4'):
+            res = process_video_workflow(ex)
+            for k in stats: stats[k] += res.get(k, 0)
+
+    # If everything exists and we don't want examples or already have enough examples, we might skip API logic
+    if has_info and has_preview and has_st_metadata and max_examples == 0:
+        print(f"Skipping fully populated model: {filepath_str}")
+        return stats
+
+    print(f"Processing model: {filepath_str}")
+
+    # Safetensors built-in metadata extraction
+    if is_safetensors and not has_st_metadata:
+        metadata = extract_safetensors_metadata(filepath_str)
+        if metadata:
+            with open(st_metadata_path, "w", encoding="utf-8") as f:
+                json.dump(metadata, f, indent=4)
+            print(f"Saved safetensors metadata to {st_metadata_path}")
+
+    mtime = os.path.getmtime(filepath_str)
+    size = os.path.getsize(filepath_str)
+
+    # We want to know if it's a NEW hash or cached
+    is_cached = False
+    with cache.lock:
+        if cache.is_sqlite:
+            cached_hash = cache._get_sqlite_hash(filepath_str, mtime, size)
+            if cached_hash: is_cached = True
+        else:
+            if filepath_str in cache.cache:
+                entry = cache.cache[filepath_str]
+                if entry["mtime"] == mtime and entry["size"] == size:
+                    is_cached = True
+
+    file_hash = cache.get_hash(filepath_str)
+    if not file_hash:
+        return stats
+
+    if not is_cached:
+        stats["new_hashes"] += 1
+
+    try:
+        model_info = api.get_model_info_by_hash(file_hash)
+    except Exception as e:
+        print(f"API Error for {filepath_str}: {e}")
+        return stats
+
+    if not model_info:
+        print(f"Model not found on Civitai: {filepath_str}")
+        return stats
+
+    # 1. Save Info
+    if not has_info:
+        with open(info_path, "w", encoding="utf-8") as f:
+            json.dump(model_info, f, indent=4)
+        print(f"Saved metadata to {info_path}")
+
+    images = model_info.get("images", [])
+    valid_images = filter_images_by_nsfw(images, nsfw_threshold)
+
+    if not valid_images:
+        print(f"No valid images found matching NSFW criteria for {filepath_str}")
+        return stats
+
+    # 2. Save Preview
+    if not has_preview and len(valid_images) > 0:
+        preview_img = valid_images[0]
+        url = preview_img.get("url")
+        ext = "jpeg" # Default fallback
+        if ".png" in url.lower(): ext = "png"
+        elif ".mp4" in url.lower(): ext = "mp4"
+        elif ".jpg" in url.lower() or ".jpeg" in url.lower(): ext = "jpeg"
+
+        preview_path = f"{base_path}.preview.{ext}"
+        if Downloader.download_file(url, preview_path):
+            if ext in ("png", "jpg", "jpeg"):
+                res = process_image_workflow_and_compress(preview_path)
+                for k in stats: stats[k] += res.get(k, 0)
+            elif ext == "mp4":
+                res = process_video_workflow(preview_path)
+                for k in stats: stats[k] += res.get(k, 0)
+
+    # 3. Save Examples
+    if max_examples > 0:
+        existing_examples_count = len(glob.glob(f"{base_path}.example.*"))
+        # We divide by 2 roughly if there are json files, let's just count unique example indices
+        existing_indices = set()
+        for f in glob.glob(f"{base_path}.example.*"):
+            parts = f.split('.example.')
+            if len(parts) > 1:
+                idx = parts[1].split('.')[0]
+                if idx.isdigit(): existing_indices.add(idx)
+
+        needed_examples = max_examples - len(existing_indices)
+
+        if needed_examples > 0:
+            example_images = valid_images[1:max_examples+1]
+            # Skip images we probably already downloaded by skipping the first N
+            example_images_to_dl = example_images[len(existing_indices):]
+
+            for img in example_images_to_dl:
+                url = img.get("url")
+                ext = "jpeg"
+                if ".png" in url.lower(): ext = "png"
+                elif ".mp4" in url.lower(): ext = "mp4"
+                elif ".jpg" in url.lower() or ".jpeg" in url.lower(): ext = "jpeg"
+
+                example_path = next_example_image_path(base_path, ext)
+                if Downloader.download_file(url, example_path):
+                    if ext in ("png", "jpg", "jpeg"):
+                        res = process_image_workflow_and_compress(example_path)
+                        for k in stats: stats[k] += res.get(k, 0)
+                    elif ext == "mp4":
+                        res = process_video_workflow(example_path)
+                        for k in stats: stats[k] += res.get(k, 0)
+    return stats
+
+import numpy as np
+from PIL import Image, ImageOps
+import torch
+
+def load_image_tensor(path):
+    if not os.path.exists(path):
+        return None
+    try:
+        img = Image.open(path)
+        img = ImageOps.exif_transpose(img)
+        image = img.convert("RGB")
+        image = np.array(image).astype(np.float32) / 255.0
+        image = torch.from_numpy(image)[None,]
+        return image
+    except Exception as e:
+        print(f"Error loading image {path}: {e}")
+        return None
+
+def get_model_workflows(base_path):
+    workflows = []
+    # Main workflow
+    main_json = f"{base_path}.json"
+    if os.path.exists(main_json):
+        workflows.append(main_json)
+
+    # Example workflows
+    # They are named like base_path.example.N.json
+    # or base_path.preview.json (if extracted from preview)
+    preview_json = f"{base_path}.preview.json"
+    if os.path.exists(preview_json) and preview_json not in workflows:
+        workflows.append(preview_json)
+
+    example_jsons = glob.glob(f"{base_path}.example.*.json")
+    # Sort them by index
+    example_jsons.sort()
+    for ej in example_jsons:
+        if ej not in workflows:
+            workflows.append(ej)
+
+    return workflows
+
+def get_model_metadata_file(base_path):
+    # Prefer civitai.info if available
+    civitai_info = f"{base_path}.civitai.info"
+    if os.path.exists(civitai_info):
+        return civitai_info
+
+    # Fallback to metadata.json (safetensors metadata)
+    metadata_json = f"{base_path}.metadata.json"
+    if os.path.exists(metadata_json):
+        return metadata_json
+
+    return None
+
+def scan_and_process(scan_dirs, recursive=True, db_path=None, nsfw_level="All", max_examples=0, api_key=None, threads=4):
+    """
+    Scans directories for models and fetches their metadata/previews.
+    Adapted for direct Python API calls rather than argparse CLI.
+    """
+    db_path = db_path if db_path else DEFAULT_DB_PATH
+    cache = HashCache(db_path)
+    api = CivitaiAPI(api_key)
+
+    model_exts = [".safetensors", ".ckpt", ".pt"]
+
+    files_to_process = []
+    if isinstance(scan_dirs, str):
+        scan_dirs = [scan_dirs]
+
+    for scan_dir in scan_dirs:
+        path = Path(scan_dir)
+        if recursive:
+            for ext in model_exts:
+                # rglob includes all directories, so we must filter out hidden ones
+                for f in path.rglob(f"*{ext}"):
+                    if not any(part.startswith('.') for part in f.parts if part != '.' and part != '..'):
+                        files_to_process.append(f)
+        else:
+            for ext in model_exts:
+                for f in path.glob(f"*{ext}"):
+                    if not any(part.startswith('.') for part in f.parts if part != '.' and part != '..'):
+                        files_to_process.append(f)
+
+    nsfw_threshold = NSFW_LEVELS.get(nsfw_level, NSFW_LEVELS["All"])
+
+    print(f"Found {len(files_to_process)} models to process.")
+
+    global_stats = {
+        "total_processed": len(files_to_process),
+        "new_hashes": 0,
+        "images_processed": 0,
+        "workflows_extracted": 0,
+        "space_saved": 0
+    }
+
+    # Import and use ComfyUI's ProgressBar
+    try:
+        import comfy.utils
+        pbar = comfy.utils.ProgressBar(len(files_to_process))
+    except ImportError:
+        pbar = None
+
+    if threads > 1:
+        print(f"Starting {threads} threads...")
+        with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
+            futures = [executor.submit(process_file, fp, cache, api, nsfw_threshold, max_examples) for fp in files_to_process]
+            for future in concurrent.futures.as_completed(futures):
+                try:
+                    res = future.result()
+                    if res:
+                        for k in global_stats: global_stats[k] += res.get(k, 0)
+                except Exception as e:
+                    print(f"Error in thread: {e}")
+                if pbar:
+                    pbar.update(1)
+    else:
+        for filepath in files_to_process:
+            res = process_file(filepath, cache, api, nsfw_threshold, max_examples)
+            if res:
+                for k in global_stats: global_stats[k] += res.get(k, 0)
+            if pbar:
+                pbar.update(1)
+
+    return global_stats

nodes/lora_extract_svd.py

@@ -13,12 +13,15 @@ import comfy.utils
 from tqdm import tqdm
 from comfy_api.latest import io
 from safetensors.torch import save_file
-from .merger_utils import MemoryEfficientSafeOpen, transfer_to_gpu_pinned
 from .device_utils import (
     estimate_model_size, prepare_for_large_operation,
     cleanup_after_operation, get_device_capabilities
 )
 
+from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned
+
+
+
 
 # =============================================================================
 # Rank Index Functions - Determine LoRA rank from singular values
@@ -44,9 +47,16 @@ def _index_sv_ratio(S: torch.Tensor, ratio: float) -> int:
     return max(1, min(rank, len(S) - 1))
 
 
-def _index_sv_cumulative(S: torch.Tensor, target: float) -> int:
-    """Cumulative mode - keep enough SVs to reach target % of total."""
-    total = torch.sum(S)
+def _index_sv_cumulative(S: torch.Tensor, target: float, max_rank: int = None) -> int:
+    """Cumulative mode - keep enough SVs to reach target % of total.
+
+    Calculates relative to max_rank if provided, otherwise relative to full.
+    """
+    if max_rank is not None and max_rank < len(S):
+        total = torch.sum(S[:max_rank])
+    else:
+        total = torch.sum(S)
+
     if total < 1e-8:
         return 1
     cumsum = torch.cumsum(S, dim=0) / total
@@ -54,13 +64,26 @@ def _index_sv_cumulative(S: torch.Tensor, target: float) -> int:
     return max(1, min(rank, len(S) - 1))
 
 
-def _index_sv_fro(S: torch.Tensor, target: float) -> int:
-    """Frobenius norm mode - preserve target fraction of Frobenius norm."""
-    S_sq = S.pow(2)
-    total_sq = torch.sum(S_sq)
+def _index_sv_fro(S: torch.Tensor, target: float, max_rank: int = None) -> int:
+    """Frobenius norm mode - preserve target fraction of Frobenius norm.
+
+    Calculates relative to max_rank if provided, otherwise relative to full.
+    This means "retain target% of what's achievable within max_rank".
+    """
+    if max_rank is not None and max_rank < len(S):
+        # Calculate relative to what's achievable within max_rank
+        S_capped = S[:max_rank]
+        S_sq = S_capped.pow(2)
+        total_sq = torch.sum(S_sq)
+    else:
+        S_sq = S.pow(2)
+        total_sq = torch.sum(S_sq)
+
     if total_sq < 1e-8:
         return 1
-    cumsum = torch.cumsum(S_sq, dim=0) / total_sq
+
+    # Cumsum of all S (not capped) to find where we reach target
+    cumsum = torch.cumsum(S.pow(2), dim=0) / total_sq
     rank = int(torch.searchsorted(cumsum, target ** 2).item()) + 1
     return max(1, min(rank, len(S) - 1))
 
@@ -122,9 +145,9 @@ def _compute_rank(S: torch.Tensor, mode: str, mode_param: float,
     elif mode == "ratio":
         rank = _index_sv_ratio(S, mode_param)
     elif mode == "quantile" or mode == "sv_cumulative":
-        rank = _index_sv_cumulative(S, mode_param)
+        rank = _index_sv_cumulative(S, mode_param, max_rank)
     elif mode == "sv_fro":
-        rank = _index_sv_fro(S, mode_param)
+        rank = _index_sv_fro(S, mode_param, max_rank)
     elif mode == "sv_knee":
         rank = _index_sv_knee(S)
     elif mode == "sv_cumulative_knee":
@@ -152,27 +175,27 @@ def _svd_extract_linear_lowrank(
 ) -> tuple:
     """
     Low-rank SVD decomposition using svd_lowrank for fixed rank extraction.
-    
+
     Much faster than full SVD when rank << min(m, n) because it only computes
     the top-k singular values using randomized algorithms.
-    
+
     Args:
         niter: Power iterations for SVD accuracy (higher = more accurate but slower)
-    
+
     Returns:
         (lora_down, lora_up, diff), "low rank" or (weight, "full")
     """
     weight = weight.to(device=device, dtype=torch.float32)
     out_dim, in_dim = weight.shape
-    
+
     # Clamp rank to valid range
     max_possible_rank = min(out_dim, in_dim)
-    rank = max(1, min(rank, max_possible_rank - 1))
-    
+    rank = max(1, min(rank, max_possible_rank))
+
     # Check if decomposition is worthwhile
-    if rank >= max_possible_rank // 2:
+    if rank >= max_possible_rank:
         return weight, "full"
-    
+
     # Use svd_lowrank for faster low-rank approximation
     try:
         U, S, V = torch.svd_lowrank(weight, q=rank, niter=niter)
@@ -180,7 +203,7 @@ def _svd_extract_linear_lowrank(
         Vh = V.T  # [rank, in_dim]
     except Exception as e:
         raise RuntimeError(f"svd_lowrank failed: {e}")
-    
+
     # Clamp outliers
     if clamp_quantile < 1.0 and len(S) > 0:
         try:
@@ -188,14 +211,14 @@ def _svd_extract_linear_lowrank(
             S = S.clamp(max=max_val)
         except RuntimeError:
             pass
-    
+
     # Construct LoRA matrices
     lora_up = U @ torch.diag(S)  # [out_dim, rank]
     lora_down = Vh               # [rank, in_dim]
-    
+
     # Compute reconstruction diff
     diff = weight - (lora_up @ lora_down)
-    
+
     return (lora_down, lora_up, diff), "low rank"
 
 
@@ -219,7 +242,7 @@ def _svd_extract_linear(
     """
     weight = weight.to(device=device, dtype=torch.float32)
     out_dim, in_dim = weight.shape
-    
+
     # For fixed rank, use the faster svd_lowrank
     if mode == "fixed" and mode_param > 0:
         target_rank = int(mode_param)
@@ -237,7 +260,7 @@ def _svd_extract_linear(
     rank = _compute_rank(S, mode, mode_param, max_rank)
 
     # Check if decomposition is worthwhile
-    if rank >= min(out_dim, in_dim) // 2:
+    if rank >= min(out_dim, in_dim):
         return weight, "full"
 
     # Truncate to rank
@@ -283,7 +306,7 @@ def _svd_extract_conv(
 
     # Flatten for SVD
     if is_1x1:
-        mat = weight.squeeze()  # [out_ch, in_ch]
+        mat = weight.view(out_ch, in_ch)  # [out_ch, in_ch]
     else:
         mat = weight.reshape(out_ch, -1)  # [out_ch, in_ch*k*k]
 
@@ -362,8 +385,11 @@ def _extract_chunked_layer(
             return None, None, 0
 
     # Combine chunks
+    # Note: Using the first chunk's down matrix as the basis for all chunks
+    # is a better approximation than a simple mean if the chunks share the same input space (like QKV).
+    # Ideally, we would use a more sophisticated joint SVD, but this is a reasonable fallback.
     combined_lora_up = torch.cat(all_lora_up, dim=0)
-    combined_lora_down = torch.stack(all_lora_down, dim=0).mean(dim=0)
+    combined_lora_down = all_lora_down[0]
     rank = combined_lora_down.shape[0]
 
     return combined_lora_up, combined_lora_down, rank
@@ -403,7 +429,6 @@ def extract_lora_from_files(
     mode: str,
     linear_param: float,
     conv_param: float,
-    prefix: str,
     device: str,
     save_dtype: str,
     linear_max_rank: int = None,
@@ -414,6 +439,9 @@ def extract_lora_from_files(
     mismatch_mode: str = "skip",
     chunk_large_layers: bool = True,
     svd_niter: int = 2,
+    lazy_load: bool = True,
+    force_clear_cache: bool = True,
+    is_text_encoder: bool = False,
 ) -> dict[str, torch.Tensor]:
     """
     Extract LoRA from difference between two models.
@@ -424,7 +452,6 @@ def extract_lora_from_files(
         mode: Rank selection mode
         linear_param: Mode parameter for linear layers
         conv_param: Mode parameter for conv layers
-        prefix: LoRA key prefix (e.g. "lora_unet_")
         device: Computation device
         save_dtype: Output dtype
         linear_max_rank: Max rank for linear layers
@@ -434,6 +461,7 @@ def extract_lora_from_files(
         skip_patterns_str: Patterns for layers to skip
         mismatch_mode: How to handle mismatches
         chunk_large_layers: Enable chunked extraction
+        is_text_encoder: Whether extracting from text encoder models
     """
     output_sd = {}
 
@@ -450,46 +478,46 @@ def extract_lora_from_files(
     print(f"[LoRA Extract] Preparing memory for {total_size_gb:.2f}GB operation...")
     prepare_for_large_operation(total_size_gb * 1.5, torch.device(device))  # 1.5x for SVD headroom
 
-    handler_a = MemoryEfficientSafeOpen(model_a_path)
-    handler_b = MemoryEfficientSafeOpen(model_b_path)
+    handler_a = MemoryEfficientSafeOpen(model_a_path, low_memory=lazy_load)
+    handler_b = MemoryEfficientSafeOpen(model_b_path, low_memory=lazy_load)
 
     try:
         keys_a = set(handler_a.keys())
         keys_b = set(handler_b.keys())
-
         weight_keys = [k for k in keys_a if k.endswith(".weight")]
-
         pbar = comfy.utils.ProgressBar(len(weight_keys))
-
         stats = {"extracted": 0, "full": 0, "skipped": 0, "chunked": 0}
 
-        for key in tqdm(weight_keys, desc="Extracting LoRA", unit="layers"):
-            lora_key = key[:-7]
-            lora_name = prefix + lora_key.replace(".", "_")
+        def _process_layer(key):
+            lora_name = _format_lora_key(key, is_text_encoder=is_text_encoder)
 
             if _matches_any_pattern(key, skip_patterns):
-                stats["skipped"] += 1
-                pbar.update(1)
-                continue
+                return "skipped", None
 
             # Load tensors with pinned memory for CUDA
             use_pinned = device == 'cuda'
-            
+
             if key not in keys_b:
                 if mismatch_mode == "skip":
-                    stats["skipped"] += 1
-                    pbar.update(1)
-                    continue
+                    return "skipped", None
+                if mismatch_mode == "error":
+                    raise ValueError(f"Key {key} not found in model B")
+
                 cpu_a = handler_a.get_tensor(key)
                 if use_pinned:
                     weight_diff = transfer_to_gpu_pinned(cpu_a, device, torch.float32)
                 else:
                     weight_diff = cpu_a.to(device=device, dtype=torch.float32)
                 del cpu_a
+
+                if mismatch_mode == "zeros":
+                    # For zeros mode, we treat the missing base as a zeroed tensor of same shape
+                    # which is already captured by weight_diff = tensor_a
+                    pass
             else:
                 cpu_a = handler_a.get_tensor(key)
                 cpu_b = handler_b.get_tensor(key)
-                
+
                 if use_pinned:
                     tensor_a = transfer_to_gpu_pinned(cpu_a, device, torch.float32)
                     tensor_b = transfer_to_gpu_pinned(cpu_b, device, torch.float32)
@@ -500,10 +528,12 @@ def extract_lora_from_files(
 
                 if tensor_a.shape != tensor_b.shape:
                     if mismatch_mode == "skip":
-                        stats["skipped"] += 1
-                        pbar.update(1)
                         del tensor_a, tensor_b
-                        continue
+                        return "skipped", None
+                    if mismatch_mode == "error":
+                        raise ValueError(f"Shape mismatch for {key}: {tensor_a.shape} vs {tensor_b.shape}")
+
+                    # For zeros/fallback, use tensor_a as the difference
                     weight_diff = tensor_a
                     del tensor_b
                 else:
@@ -512,19 +542,16 @@ def extract_lora_from_files(
 
             # Skip small differences
             if min_diff > 0 and weight_diff.abs().max() < min_diff:
-                stats["skipped"] += 1
                 del weight_diff
-                pbar.update(1)
-                continue
+                return "skipped", None
 
             # Skip 1D tensors
             if weight_diff.ndim < 2:
-                stats["skipped"] += 1
                 del weight_diff
-                pbar.update(1)
-                continue
+                return "skipped", None
 
             is_conv = weight_diff.ndim == 4
+            layer_results = {}
 
             try:
                 if is_conv:
@@ -545,32 +572,42 @@ def extract_lora_from_files(
                             weight_diff, num_chunks, mode, linear_param, device, linear_max_rank
                         )
                         if lora_up is not None:
-                            output_sd[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
-                            output_sd[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
-                            output_sd[f"{lora_name}.alpha"] = torch.tensor([rank]).to(save_torch_dtype)
-                            stats["chunked"] += 1
+                            layer_results[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
+                            layer_results[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
+                            layer_results[f"{lora_name}.alpha"] = torch.tensor(rank).to(save_torch_dtype)
                             del weight_diff
-                            pbar.update(1)
-                            continue
+                            return "chunked", layer_results
 
                 print(f"[LoRA Extract] Failed: {key}: {e}")
-                stats["skipped"] += 1
                 del weight_diff
-                pbar.update(1)
-                continue
+                return "skipped", None
 
             # Store result
             if mode_str == "full":
-                output_sd[f"{lora_name}.diff"] = weight_diff.to(save_torch_dtype).cpu().contiguous()
-                stats["full"] += 1
+                layer_results[f"{lora_name}.diff"] = weight_diff.to(save_torch_dtype).cpu().contiguous()
+                status = "full"
             else:
                 lora_down, lora_up, _ = result
-                output_sd[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
-                output_sd[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
-                output_sd[f"{lora_name}.alpha"] = torch.tensor([lora_down.shape[0]]).to(save_torch_dtype)
-                stats["extracted"] += 1
+                layer_results[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous()
+                layer_results[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous()
+                layer_results[f"{lora_name}.alpha"] = torch.tensor(lora_down.shape[0]).to(save_torch_dtype)
+                status = "extracted"
 
             del weight_diff
+            return status, layer_results
+
+        for key in tqdm(weight_keys, desc="Extracting LoRA", unit="layers"):
+            status, layer_sd = _process_layer(key)
+            stats[status] += 1
+            if layer_sd:
+                output_sd.update(layer_sd)
+
+            if force_clear_cache:
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
             pbar.update(1)
 
         print(f"[LoRA Extract] Done: {stats['extracted']} extracted, {stats['chunked']} chunked, "
@@ -612,8 +649,105 @@ def _get_model_inputs():
     ]
 
 
+def _get_clip_model_inputs():
+    return [
+        io.Combo.Input("model_a", options=folder_paths.get_filename_list("text_encoders"),
+                      tooltip="Finetuned CLIP/Text Encoder (A - B = LoRA)"),
+        io.Combo.Input("model_b", options=folder_paths.get_filename_list("text_encoders"),
+                      tooltip="Base CLIP/Text Encoder (A - B = LoRA)"),
+    ]
+
+
+def _reconstruct_dots(key: str) -> str:
+    """Reconstruct dot structure from underscored key using heuristics."""
+    # Block indices: input_blocks_1 -> input_blocks.1
+    blocks = [
+        "input_blocks", "output_blocks", "middle_block",
+        "transformer_blocks", "single_transformer_blocks",
+        "double_blocks", "single_blocks",
+        "down_blocks", "up_blocks", "mid_block",
+        "attentions", "resnets", "upsamplers", "downsamplers"
+    ]
+    for b in blocks:
+        key = re.sub(f"{b}_(\\d+)", f"{b}.\\1", key)
+
+    # Sequential numbers: .1_1 -> .1.1
+    key = re.sub(r"(\.\d+)_(\d+)", r"\1.\2", key)
+
+    # Separators that should be preceded by dot
+    separators = [
+        "transformer_blocks", "single_transformer_blocks",
+        "attn1", "attn2", "attn",
+        "img_attn", "txt_attn",
+        "to_q", "to_k", "to_v", "to_out",
+        "q_proj", "k_proj", "v_proj", "out_proj",
+        "qkv", "proj", "ff", "net",
+        "norm1", "norm2", "norm3", "norm",
+        "time_emb_proj"
+    ]
+    for s in separators:
+        key = key.replace(f"_{s}", f".{s}")
+
+    # Special case: to_out_0 -> to_out.0
+    key = re.sub(r"to_out_(\d+)", r"to_out.\1", key)
+
+    return key
+
+
+def _format_lora_key(key: str, is_text_encoder: bool = False) -> str:
+    """
+    Format the key for LoRA saving.
+    Standardizes to 'diffusion_model.' or 'transformer.' prefix.
+    If is_text_encoder is True, use 'text_encoders.transformer.'
+    """
+    if key.endswith(".weight"):
+        key = key[:-7]
+
+    if is_text_encoder:
+        return f"text_encoders.transformer.{key}"
+
+    # Handle ComfyUI Checkpoint wrapper
+    if key.startswith("model.diffusion_model.") or key.startswith("net."):
+        # Check if it's a Diffusers-format transformer inside a checkpoint
+        if key.startswith("model.diffusion_model."):
+            inner_key = key[22:]  # len("model.diffusion_model.")
+        else:
+            inner_key = key[4:]   # len("net.")
+
+        if inner_key.startswith("transformer_blocks") or inner_key.startswith("single_transformer_blocks"):
+            return f"transformer.{inner_key}"
+        # Standard checkpoint or Original Flux
+        return f"diffusion_model.{inner_key}"
+
+    # Handle direct Diffusers keys
+    if key.startswith("transformer_blocks") or key.startswith("single_transformer_blocks"):
+        return f"transformer.{key}"
+
+    # Handle legacy lora_unet_ prefix (for resizing without base)
+    if key.startswith("lora_unet_"):
+        core = key[10:]
+        dotted = _reconstruct_dots(core)
+        # Check if reconstructed key implies transformer format
+        if dotted.startswith("transformer_blocks") or dotted.startswith("single_transformer_blocks"):
+            return f"transformer.{dotted}"
+        return f"diffusion_model.{dotted}"
+
+    # Handle already correct prefixes
+    if key.startswith("diffusion_model.") or key.startswith("transformer."):
+        return key
+
+    # Handle known Unet blocks (Diffusers format UNet)
+    if any(key.startswith(p) for p in ["down_blocks", "up_blocks", "mid_block", "conv_in", "conv_out", "time_embedding", "class_embedding"]):
+        return f"diffusion_model.{key}"
+
+    # Default fallback to diffusion_model
+    return f"diffusion_model.{key}"
+
+
 def _get_common_inputs():
     return [
+        io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+        io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
         io.Boolean.Input("chunk_large_layers", default=False,
                         tooltip="Split large fused layers (QKV, MLP) into chunks"),
         io.Float.Input("clamp_quantile", default=0.99, min=0.5, max=1.0, step=0.01,
@@ -641,9 +775,9 @@ class LoRAExtractFixed(io.ComfyNode):
             description="Extract LoRA with specified fixed rank for each layer type.",
             inputs=[
                 *_get_model_inputs(),
-                io.Int.Input("linear_dim", default=64, min=1, max=4096,
+                io.Int.Input("linear_dim", default=64, min=1, max=16384,
                             tooltip="Rank for linear/attention layers"),
-                io.Int.Input("conv_dim", default=32, min=1, max=4096,
+                io.Int.Input("conv_dim", default=32, min=1, max=16384,
                             tooltip="Rank for conv layers"),
                 io.Int.Input("svd_niter", default=2, min=0, max=10,
                             tooltip="SVD power iterations (higher = more accurate but slower)"),
@@ -656,15 +790,16 @@ class LoRAExtractFixed(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, linear_dim, conv_dim, svd_niter, chunk_large_layers,
                 clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "fixed", linear_dim, conv_dim,
-            "lora_unet_", device, save_dtype, linear_dim, conv_dim,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers, svd_niter
+            device, save_dtype, linear_dim, conv_dim,
+            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers, svd_niter,
+            lazy_load, force_clear_cache
         )
 
         return io.NodeOutput(_save_lora(output_sd, output_filename))
@@ -686,9 +821,9 @@ class LoRAExtractRatio(io.ComfyNode):
                               tooltip="Ratio threshold for linear layers (higher = more SVs kept)"),
                 io.Float.Input("conv_ratio", default=2.0, min=1.0, max=100.0, step=0.1,
                               tooltip="Ratio threshold for conv layers (higher = more SVs kept)"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("linear_max_rank", default=128, min=1, max=16384,
                             tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("conv_max_rank", default=128, min=1, max=16384,
                             tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
@@ -699,15 +834,16 @@ class LoRAExtractRatio(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, linear_ratio, conv_ratio, linear_max_rank, conv_max_rank,
                 chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "ratio", linear_ratio, conv_ratio,
-            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
+            device, save_dtype, linear_max_rank, conv_max_rank,
+            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
+            lazy_load=lazy_load, force_clear_cache=force_clear_cache
         )
 
         return io.NodeOutput(_save_lora(output_sd, output_filename))
@@ -729,9 +865,9 @@ class LoRAExtractQuantile(io.ComfyNode):
                               tooltip="Target cumulative % for linear layers"),
                 io.Float.Input("conv_quantile", default=0.9, min=0.0, max=1.0, step=0.01,
                               tooltip="Target cumulative % for conv layers"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("linear_max_rank", default=128, min=1, max=16384,
                             tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("conv_max_rank", default=128, min=1, max=16384,
                             tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
@@ -742,15 +878,16 @@ class LoRAExtractQuantile(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, linear_quantile, conv_quantile, linear_max_rank, conv_max_rank,
                 chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "quantile", linear_quantile, conv_quantile,
-            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
+            device, save_dtype, linear_max_rank, conv_max_rank,
+            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
+            lazy_load=lazy_load, force_clear_cache=force_clear_cache
         )
 
         return io.NodeOutput(_save_lora(output_sd, output_filename))
@@ -770,9 +907,9 @@ class LoRAExtractKnee(io.ComfyNode):
                 *_get_model_inputs(),
                 io.Combo.Input("knee_method", options=["sv_knee", "sv_cumulative_knee"],
                               default="sv_knee", tooltip="Knee detection method"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("linear_max_rank", default=128, min=1, max=16384,
                             tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("conv_max_rank", default=128, min=1, max=16384,
                             tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
@@ -783,15 +920,16 @@ class LoRAExtractKnee(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, knee_method, linear_max_rank, conv_max_rank,
                 chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, knee_method, 0, 0,
-            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
+            device, save_dtype, linear_max_rank, conv_max_rank,
+            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
+            lazy_load=lazy_load, force_clear_cache=force_clear_cache
         )
 
         return io.NodeOutput(_save_lora(output_sd, output_filename))
@@ -813,9 +951,9 @@ class LoRAExtractFrobenius(io.ComfyNode):
                               tooltip="Target Frobenius norm fraction for linear"),
                 io.Float.Input("conv_target", default=0.9, min=0.0, max=1.0, step=0.01,
                               tooltip="Target Frobenius norm fraction for conv"),
-                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("linear_max_rank", default=128, min=1, max=16384,
                             tooltip="Maximum rank for linear layers"),
-                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                io.Int.Input("conv_max_rank", default=128, min=1, max=16384,
                             tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
@@ -826,15 +964,56 @@ class LoRAExtractFrobenius(io.ComfyNode):
     @classmethod
     def execute(cls, model_a, model_b, linear_target, conv_target, linear_max_rank, conv_max_rank,
                 chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
-                save_dtype, device, skip_patterns) -> io.NodeOutput:
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
         model_b_path = folder_paths.get_full_path_or_raise("diffusion_models", model_b)
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "sv_fro", linear_target, conv_target,
-            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
+            device, save_dtype, linear_max_rank, conv_max_rank,
+            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers,
+            lazy_load=lazy_load, force_clear_cache=force_clear_cache
+        )
+
+        return io.NodeOutput(_save_lora(output_sd, output_filename))
+
+
+class LoRATextEncoderExtractFixed(io.ComfyNode):
+    """Extract LoRA from Text Encoder with fixed rank."""
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="LoRATextEncoderExtractFixed",
+            display_name="LoRA Extract (Text Encoder, Fixed Rank)",
+            category="ModelUtils/LoRA",
+            description="Extract LoRA from Text Encoder/CLIP models with specified fixed rank.",
+            inputs=[
+                *_get_clip_model_inputs(),
+                io.Int.Input("linear_dim", default=32, min=1, max=16384,
+                            tooltip="Rank for linear layers"),
+                io.Int.Input("svd_niter", default=2, min=0, max=10,
+                            tooltip="SVD power iterations"),
+                *_get_common_inputs(),
+            ],
+            outputs=[io.String.Output(display_name="output_path")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, model_a, model_b, linear_dim, svd_niter, chunk_large_layers,
+                clamp_quantile, min_diff, mismatch_mode, output_filename,
+                save_dtype, device, skip_patterns, lazy_load, force_clear_cache) -> io.NodeOutput:
+
+        model_a_path = folder_paths.get_full_path_or_raise("text_encoders", model_a)
+        model_b_path = folder_paths.get_full_path_or_raise("text_encoders", model_b)
+
+        output_sd = extract_lora_from_files(
+            model_a_path, model_b_path, "fixed", linear_dim, 0,
+            device, save_dtype, linear_dim, 0,
+            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers, svd_niter,
+            lazy_load, force_clear_cache, is_text_encoder=True
         )
 
         return io.NodeOutput(_save_lora(output_sd, output_filename))

nodes/lora_merger.py

@@ -0,0 +1,568 @@
+"""
+LoRA Multi-Merge - Merge multiple LoRAs into a single LoRA file.
+Resolves different naming conventions and ranks via concatenation.
+"""
+import os
+import torch
+import folder_paths
+import comfy.utils
+from tqdm import tqdm
+from comfy_api.latest import io
+from safetensors.torch import save_file
+from typing import List, Dict, Tuple, Optional
+from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation
+from .lora_resize import (
+    detect_lora_format,
+    extract_lora_pairs,
+    detect_lora_rank,
+    _compile_patterns,
+    _matches_any_pattern,
+    _format_lora_key
+)
+
+from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned
+
+
+
+def merge_multi_loras(
+    lora_paths: List[str],
+    lora_weights: List[float],
+    merge_mode: str,
+    device: str,
+    save_dtype: torch.dtype,
+    output_filename: str,
+    verbose: bool = True,
+) -> str:
+    """
+    Merge multiple LoRAs into a single LoRA file.
+
+    Modes:
+    - concatenate: Mathematically sound merge by stacking ranks.
+                   New rank = sum(input ranks). Alpha set to new rank.
+    - weighted_sum: Weighted sum of A and B weights (Kohya style).
+                    If ranks differ, smaller ones are zero-padded to match the largest rank.
+                    New rank = max(input ranks).
+    """
+    # Estimate memory
+    total_size_gb = sum(estimate_model_size(p) for p in lora_paths)
+    if verbose:
+        print(f"[LoRA Multi-Merge] Preparing memory for {total_size_gb:.2f}GB operation...")
+        print(f"[LoRA Multi-Merge] Mode: {merge_mode}")
+
+    prepare_for_large_operation(total_size_gb * 2.5, torch.device(device))
+
+    handlers = [MemoryEfficientSafeOpen(p, low_memory=True) for p in lora_paths]
+
+    try:
+        # 1. Analyze all LoRAs
+        layer_map = {}
+        lora_infos = []
+
+        for i, handler in enumerate(handlers):
+            keys = handler.keys()
+            fmt = detect_lora_format(keys)
+            pairs = extract_lora_pairs(keys, fmt)
+            rank, alpha = detect_lora_rank(handler, pairs)
+
+            info = {
+                "name": os.path.basename(lora_paths[i]),
+                "format": fmt,
+                "pairs": pairs,
+                "rank": rank,
+                "alpha": alpha,
+                "weight": lora_weights[i],
+                "scale": alpha / rank if rank > 0 else 1.0
+            }
+            lora_infos.append(info)
+
+            if verbose:
+                print(f"[LoRA Multi-Merge] LoRA {i+1} ({info['name']}): {len(pairs)} layers, dim={rank}, format={fmt['format']}")
+
+            for block_name in pairs.keys():
+                core = _format_lora_key(block_name)
+                if core not in layer_map:
+                    layer_map[core] = []
+                layer_map[core].append(i)
+
+        if verbose:
+            print(f"[LoRA Multi-Merge] Total unique layers after resolving naming: {len(layer_map)}")
+
+        output_sd = {}
+        pbar = comfy.utils.ProgressBar(len(layer_map))
+
+        # 2. Merge each core layer
+        with torch.no_grad():
+            for core, indices in tqdm(layer_map.items(), desc="Merging layers", unit="layers"):
+                downs = []
+                ups = []
+                is_conv = False
+                max_rank = 0
+
+                # First pass: load and determine max rank
+                for idx in indices:
+                    info = lora_infos[idx]
+                    handler = handlers[idx]
+                    orig_block = next(b for b in info["pairs"].keys() if _format_lora_key(b) == core)
+                    block_keys = info["pairs"][orig_block]
+
+                    if "down" not in block_keys or "up" not in block_keys:
+                        continue
+
+                    t_down = handler.get_tensor(block_keys["down"])
+                    t_up = handler.get_tensor(block_keys["up"])
+
+                    if device == 'cuda':
+                        t_down = transfer_to_gpu_pinned(t_down, device, torch.float32)
+                        t_up = transfer_to_gpu_pinned(t_up, device, torch.float32)
+                    else:
+                        t_down = t_down.to(device=device, dtype=torch.float32)
+                        t_up = t_up.to(device=device, dtype=torch.float32)
+
+                    # Store with original info for padding/weighting
+                    is_conv = len(t_down.shape) == 4
+                    current_rank = t_down.shape[0]
+                    max_rank = max(max_rank, current_rank)
+
+                    # Apply scale immediately for concatenate mode, or keep for later
+                    if merge_mode == "concatenate":
+                        effective_weight = info["weight"] * info["scale"]
+                        t_up = t_up * effective_weight
+
+                    downs.append((t_down, info))
+                    ups.append((t_up, info))
+
+                if not downs:
+                    pbar.update(1)
+                    continue
+
+                if merge_mode == "concatenate":
+                    # Stack ranks
+                    merged_down = torch.cat([d[0] for d in downs], dim=0)
+                    merged_up = torch.cat([u[0] for u in ups], dim=1)
+                    new_rank = merged_down.shape[0]
+                    new_alpha = float(new_rank)
+                else:
+                    # weighted_sum (Kohya style)
+                    # Result = sum( weight_i * Padded(B_i) ), sum( weight_i * Padded(A_i) )
+                    # Note: We apply scale to weights here to normalize different alphas
+                    merged_down = torch.zeros_like(downs[0][0])
+                    # Need to handle different ranks via padding
+                    target_down_shape = list(downs[0][0].shape)
+                    target_down_shape[0] = max_rank
+                    target_up_shape = list(ups[0][0].shape)
+                    target_up_shape[1] = max_rank
+
+                    merged_down = torch.zeros(target_down_shape, device=device, dtype=torch.float32)
+                    merged_up = torch.zeros(target_up_shape, device=device, dtype=torch.float32)
+
+                    for (t_d, info_d), (t_u, info_u) in zip(downs, ups):
+                        r = t_d.shape[0]
+                        w = info_d["weight"]
+                        # We also apply sqrt(scale) to both to distribute the scale factor?
+                        # Kohya just uses weights. But to be safe with different alphas,
+                        # we apply scale to the final delta.
+                        # For direct weight merge, we'll just use weights.
+
+                        if r < max_rank:
+                            # Pad down: [r, in...] -> [max_rank, in...]
+                            pad_d = [0] * (len(t_d.shape) * 2)
+                            pad_d[-1] = max_rank - r # last dim in pad is first dim in tensor (reversed)
+                            # Wait, F.pad uses reverse order of dims.
+                            # For [r, in], padding is (0,0, 0, max_rank-r)
+                            padding_d = [0, 0] * (len(t_d.shape) - 1) + [0, max_rank - r]
+                            t_d = torch.nn.functional.pad(t_d, tuple(padding_d))
+
+                            # Pad up: [out, r...] -> [out, max_rank...]
+                            # For [out, r], padding is (0, max_rank-r, 0, 0)
+                            padding_u = [0, max_rank - r] + [0, 0] * (len(t_u.shape) - 1)
+                            t_u = torch.nn.functional.pad(t_u, tuple(padding_u))
+
+                        merged_down += w * t_d
+                        merged_up += w * t_u
+
+                    new_rank = max_rank
+                    # Alpha is usually max of alphas or same as rank
+                    new_alpha = float(max_rank)
+
+                # Store
+                output_sd[f"{core}.lora_down.weight"] = merged_down.to(save_dtype).cpu().contiguous()
+                output_sd[f"{core}.lora_up.weight"] = merged_up.to(save_dtype).cpu().contiguous()
+                output_sd[f"{core}.alpha"] = torch.tensor(new_alpha, dtype=save_dtype)
+
+                # Cleanup
+                del merged_down, merged_up
+                for d, _ in downs: del d
+                for u, _ in ups: del u
+                downs.clear()
+                ups.clear()
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
+                pbar.update(1)
+
+        # 3. Final Summary
+        if verbose:
+            matched_stats = {i: 0 for i in range(len(lora_paths))}
+            for indices in layer_map.values():
+                for idx in indices:
+                    matched_stats[idx] += 1
+
+            print(f"[LoRA Multi-Merge] --- Merge Summary ---")
+            for i, info in enumerate(lora_infos):
+                print(f"[LoRA Multi-Merge] LoRA {i+1}: {matched_stats[i]}/{len(info['pairs'])} layers used in merge")
+            print(f"[LoRA Multi-Merge] Output state dict has {len(output_sd)} tensors")
+
+        # 4. Metadata
+        metadata = {
+            "ss_training_comment": f"Merged {len(lora_paths)} LoRAs via concatenation",
+            "ss_network_module": "networks.lora",
+        }
+
+        # 4. Save
+        output_dir = folder_paths.get_folder_paths("loras")[0]
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors")
+
+        save_file(output_sd, output_path, metadata)
+        if verbose:
+            print(f"[LoRA Multi-Merge] Saved merged LoRA to {output_path}")
+
+        return output_path
+
+    finally:
+        for h in handlers:
+            h.__exit__(None, None, None)
+        cleanup_after_operation()
+
+
+class LoRAMultiMerge(io.ComfyNode):
+    """Merge multiple LoRAs into a single LoRA file."""
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="LoRAMultiMerge",
+            display_name="LoRA Multi-Merge",
+            category="ModelUtils/LoRA/Merge",
+            description="Merge 1-8 LoRAs into a single LoRA file. This resolves different ranks and naming conventions.",
+            inputs=[
+                io.Combo.Input("lora_count", options=[str(i) for i in range(1, 9)], default="2",
+                              tooltip="Number of LoRAs to merge"),
+                # LoRA 1
+                io.Combo.Input("lora_1", options=folder_paths.get_filename_list("loras"), tooltip="First LoRA"),
+                io.Float.Input("weight_1", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 2
+                io.Combo.Input("lora_2", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_2", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 3
+                io.Combo.Input("lora_3", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_3", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 4
+                io.Combo.Input("lora_4", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_4", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 5
+                io.Combo.Input("lora_5", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_5", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 6
+                io.Combo.Input("lora_6", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_6", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 7
+                io.Combo.Input("lora_7", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_7", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # LoRA 8
+                io.Combo.Input("lora_8", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_8", default=1.0, min=-10.0, max=10.0, step=0.01),
+
+                io.Combo.Input("merge_mode", options=["concatenate", "weighted_sum"], default="concatenate",
+                              tooltip="concatenate: safe, increases rank. weighted_sum: fixed rank (to max input rank), mathematically lossy but standard in Kohya."),
+                io.String.Input("output_filename", default="merged_lora"),
+                io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
+                io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
+            ],
+            outputs=[io.String.Output(display_name="output_path")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, lora_count,
+                lora_1, weight_1, lora_2, weight_2, lora_3, weight_3, lora_4, weight_4,
+                lora_5, weight_5, lora_6, weight_6, lora_7, weight_7, lora_8, weight_8,
+                merge_mode, output_filename, save_dtype, device) -> io.NodeOutput:
+
+        count = int(lora_count)
+        names = [lora_1, lora_2, lora_3, lora_4, lora_5, lora_6, lora_7, lora_8]
+        weights = [weight_1, weight_2, weight_3, weight_4, weight_5, weight_6, weight_7, weight_8]
+
+        valid_paths = []
+        valid_weights = []
+
+        for i in range(count):
+            if names[i] and names[i] != "None":
+                path = folder_paths.get_full_path_or_raise("loras", names[i])
+                valid_paths.append(path)
+                valid_weights.append(weights[i])
+
+        if not valid_paths:
+            raise ValueError("No LoRAs selected for merging.")
+
+        dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
+
+        path = merge_multi_loras(
+            lora_paths=valid_paths,
+            lora_weights=valid_weights,
+            merge_mode=merge_mode,
+            device=device,
+            save_dtype=dtype,
+            output_filename=output_filename
+        )
+
+        return io.NodeOutput(path)
+
+
+class LoRAMultiMergeDARE(io.ComfyNode):
+    """Merge multiple LoRAs into a single LoRA file using DARE-Ties method."""
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="LoRAMultiMergeDARE",
+            display_name="LoRA Multi-Merge (DARE-Ties)",
+            category="ModelUtils/LoRA/Merge",
+            description="Merge 1-8 LoRAs into a single LoRA file using DARE-Ties. Drops small values and resolves sign conflicts.",
+            inputs=[
+                io.Combo.Input("lora_count", options=[str(i) for i in range(1, 9)], default="2"),
+                # LoRA 1
+                io.Combo.Input("lora_1", options=folder_paths.get_filename_list("loras"), tooltip="First LoRA"),
+                io.Float.Input("weight_1", default=1.0, min=-10.0, max=10.0, step=0.01),
+                # ...
+                io.Combo.Input("lora_2", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_2", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_3", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_3", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_4", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_4", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_5", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_5", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_6", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_6", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_7", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_7", default=1.0, min=-10.0, max=10.0, step=0.01),
+                io.Combo.Input("lora_8", options=["None"] + folder_paths.get_filename_list("loras"), default="None"),
+                io.Float.Input("weight_8", default=1.0, min=-10.0, max=10.0, step=0.01),
+
+                io.Float.Input("drop_rate", default=0.1, min=0.0, max=1.0, step=0.01, tooltip="DARE drop rate"),
+                io.Float.Input("trim_quantile", default=0.2, min=0.0, max=1.0, step=0.01, tooltip="TIES trim quantile (drops smallest values)"),
+                io.Int.Input("seed", default=42, min=0, max=0xffffffffffffffff),
+                io.String.Input("output_filename", default="merged_lora_dare"),
+                io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
+                io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
+            ],
+            outputs=[io.String.Output(display_name="output_path")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, lora_count,
+                lora_1, weight_1, lora_2, weight_2, lora_3, weight_3, lora_4, weight_4,
+                lora_5, weight_5, lora_6, weight_6, lora_7, weight_7, lora_8, weight_8,
+                drop_rate, trim_quantile, seed, output_filename, save_dtype, device) -> io.NodeOutput:
+
+        count = int(lora_count)
+        names = [lora_1, lora_2, lora_3, lora_4, lora_5, lora_6, lora_7, lora_8]
+        weights = [weight_1, weight_2, weight_3, weight_4, weight_5, weight_6, weight_7, weight_8]
+
+        valid_paths = []
+        valid_weights = []
+
+        for i in range(count):
+            if names[i] and names[i] != "None":
+                path = folder_paths.get_full_path_or_raise("loras", names[i])
+                valid_paths.append(path)
+                valid_weights.append(weights[i])
+
+        if not valid_paths:
+            raise ValueError("No LoRAs selected for merging.")
+
+        dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
+
+        path = merge_multi_loras_dare(
+            lora_paths=valid_paths,
+            lora_weights=valid_weights,
+            drop_rate=drop_rate,
+            trim_quantile=trim_quantile,
+            seed=seed,
+            device=device,
+            save_dtype=dtype,
+            output_filename=output_filename
+        )
+
+        return io.NodeOutput(path)
+
+def merge_multi_loras_dare(
+    lora_paths: List[str],
+    lora_weights: List[float],
+    drop_rate: float,
+    trim_quantile: float,
+    seed: int,
+    device: str,
+    save_dtype: torch.dtype,
+    output_filename: str,
+    verbose: bool = True,
+) -> str:
+    """
+    Merge multiple LoRAs using DARE-Ties method applied to weights.
+    Different ranks are handled via zero-padding to the max rank.
+    """
+    total_size_gb = sum(estimate_model_size(p) for p in lora_paths)
+    if verbose:
+        print(f"[LoRA Multi-Merge DARE] Preparing memory for {total_size_gb:.2f}GB operation...")
+        print(f"[LoRA Multi-Merge DARE] Drop rate: {drop_rate}, Trim quantile: {trim_quantile}")
+
+    prepare_for_large_operation(total_size_gb * 2.5, torch.device(device))
+    handlers = [MemoryEfficientSafeOpen(p, low_memory=True) for p in lora_paths]
+
+    rng = torch.Generator(device=device).manual_seed(seed)
+
+    try:
+        # 1. Analyze
+        layer_map = {}
+        lora_infos = []
+        for i, handler in enumerate(handlers):
+            keys = handler.keys()
+            fmt = detect_lora_format(keys)
+            pairs = extract_lora_pairs(keys, fmt)
+            rank, alpha = detect_lora_rank(handler, pairs)
+            info = {
+                "name": os.path.basename(lora_paths[i]),
+                "pairs": pairs,
+                "rank": rank,
+                "alpha": alpha,
+                "weight": lora_weights[i]
+            }
+            lora_infos.append(info)
+
+            if verbose:
+                print(f"[LoRA Multi-Merge DARE] LoRA {i+1} ({info['name']}): {len(pairs)} layers, dim={rank}")
+
+            for block_name in pairs.keys():
+                core = _format_lora_key(block_name)
+                if core not in layer_map: layer_map[core] = []
+                layer_map[core].append(i)
+
+        if verbose:
+            print(f"[LoRA Multi-Merge DARE] Total unique layers after resolving naming: {len(layer_map)}")
+
+        output_sd = {}
+        pbar = comfy.utils.ProgressBar(len(layer_map))
+
+        # 2. Merge
+        with torch.no_grad():
+            for core, indices in tqdm(layer_map.items(), desc="Merging layers (DARE)", unit="layers"):
+                downs = []
+                ups = []
+                max_rank = 0
+
+                for idx in indices:
+                    info = lora_infos[idx]
+                    orig_block = next(b for b in info["pairs"].keys() if _format_lora_key(b) == core)
+                    block_keys = info["pairs"][orig_block]
+                    if "down" not in block_keys or "up" not in block_keys: continue
+
+                    t_d = handlers[idx].get_tensor(block_keys["down"]).to(device=device, dtype=torch.float32)
+                    t_u = handlers[idx].get_tensor(block_keys["up"]).to(device=device, dtype=torch.float32)
+
+                    max_rank = max(max_rank, t_d.shape[0])
+                    downs.append((t_d, info["weight"]))
+                    ups.append((t_u, info["weight"]))
+
+                if not downs:
+                    pbar.update(1)
+                    continue
+
+                def process_ties_dare(tensors, weights, dim_to_pad):
+                    # Pad all to max_rank
+                    padded = []
+                    for t, w in tensors:
+                        if t.shape[dim_to_pad] < max_rank:
+                            padding = [0] * (len(t.shape) * 2)
+                            # dim_to_pad 0 (down) -> last pair in pad
+                            # dim_to_pad 1 (up) -> second to last pair in pad
+                            rev_dim = len(t.shape) - 1 - dim_to_pad
+                            padding[rev_dim*2 + 1] = max_rank - t.shape[dim_to_pad]
+                            t = torch.nn.functional.pad(t, tuple(padding))
+                        padded.append(t * w)
+
+                    # DARE
+                    if drop_rate > 0:
+                        for i in range(len(padded)):
+                            mask = (torch.rand(padded[i].shape, generator=rng, device=device) > drop_rate).float()
+                            padded[i] = (padded[i] * mask) / (1 - drop_rate)
+
+                    # TIES
+                    # 1. Trim
+                    if trim_quantile > 0:
+                        for i in range(len(padded)):
+                            flat = padded[i].abs().flatten()
+                            k = int(len(flat) * trim_quantile)
+                            if k > 0:
+                                threshold = torch.kthvalue(flat, k).values
+                                padded[i] = torch.where(padded[i].abs() < threshold, torch.zeros_like(padded[i]), padded[i])
+
+                    # 2. Elect & Merge
+                    stacked = torch.stack(padded) # [N, ...]
+                    signs = torch.sign(stacked)
+                    sum_signs = signs.sum(dim=0)
+                    dominant_sign = torch.sign(sum_signs)
+
+                    # Filter those matching dominant sign
+                    mask = (signs == dominant_sign) & (dominant_sign != 0)
+                    filtered = torch.where(mask, stacked, torch.zeros_like(stacked))
+
+                    # Average matching signs
+                    count = mask.sum(dim=0)
+                    result = filtered.sum(dim=0) / torch.clamp(count, min=1.0)
+                    return result
+
+                merged_down = process_ties_dare(downs, [1.0]*len(downs), 0)
+                merged_up = process_ties_dare(ups, [1.0]*len(ups), 1)
+
+                output_sd[f"{core}.lora_down.weight"] = merged_down.to(save_dtype).cpu().contiguous()
+                output_sd[f"{core}.lora_up.weight"] = merged_up.to(save_dtype).cpu().contiguous()
+                output_sd[f"{core}.alpha"] = torch.tensor(float(max_rank), dtype=save_dtype)
+
+                del merged_down, merged_up
+                for d, _ in downs: del d
+                for u, _ in ups: del u
+                downs.clear()
+                ups.clear()
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
+                pbar.update(1)
+
+        # Final Summary
+        if verbose:
+            matched_stats = {i: 0 for i in range(len(lora_paths))}
+            for indices in layer_map.values():
+                for idx in indices:
+                    matched_stats[idx] += 1
+
+            print(f"[LoRA Multi-Merge DARE] --- Merge Summary ---")
+            for i, info in enumerate(lora_infos):
+                print(f"[LoRA Multi-Merge DARE] LoRA {i+1}: {matched_stats[i]}/{len(info['pairs'])} layers used in merge")
+            print(f"[LoRA Multi-Merge DARE] Output state dict has {len(output_sd)} tensors")
+
+        # Save
+        output_dir = folder_paths.get_folder_paths("loras")[0]
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors")
+        save_file(output_sd, output_path, {"ss_training_comment": "Merged via DARE-Ties"})
+        return output_path
+
+    finally:
+        for h in handlers: h.__exit__(None, None, None)
+        cleanup_after_operation()

nodes/lora_resize.py

@@ -14,14 +14,22 @@ from tqdm import tqdm
 from comfy_api.latest import io
 from safetensors.torch import save_file
 from typing import Optional, Dict, Tuple, List
-from .merger_utils import MemoryEfficientSafeOpen, transfer_to_gpu_pinned
 from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation
 
+from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned
+
+
+
 # Reuse rank functions from extraction module
 from .lora_extract_svd import (
     _index_sv_ratio,
     _index_sv_cumulative,
     _index_sv_fro,
+    _svd_extract_linear,
+    _svd_extract_conv,
+    _compile_patterns,
+    _matches_any_pattern,
+    _format_lora_key,
 )
 
 
@@ -35,14 +43,16 @@ MIN_SV = 1e-6
 def detect_lora_format(keys: List[str]) -> Dict:
     """
     Detect LoRA format and key patterns from various sources.
-    
+
     Supports:
     - Kohya/A1111: lora_unet_*, lora_te_*
     - Diffusers: *.lora_A.weight, *.lora_B.weight
     - PEFT: base_model.model.*.lora_A/B
     - ComfyUI native: *.lora_down.weight, *.lora_up.weight
+    - HuggingFace: *.lora.down.weight, *.lora.up.weight (dots instead of underscores)
+    - Full diff: *.diff, *.diff_b (full weight differences, not low-rank)
     - Flux/SDXL: Various transformer key patterns
-    
+
     Returns:
         Dict with format info and key patterns
     """
@@ -53,117 +63,153 @@ def detect_lora_format(keys: List[str]) -> Dict:
         "alpha_suffix": ".alpha",
         "has_alpha": False,
         "key_count": 0,
+        "is_full_diff": False,  # New flag for full diff format
     }
-    
+
     # Check for alpha keys
     alpha_keys = [k for k in keys if '.alpha' in k or k.endswith('alpha')]
     format_info["has_alpha"] = len(alpha_keys) > 0
-    
+
     # Detect format by key patterns
-    sample_keys = keys[:20]  # Check first 20 keys
-    
+    sample_keys = keys[:50]  # Check more keys for full_diff detection
+
+    # Full diff format: *.diff (NOT .lora_down, just straight weight diff)
+    if any(k.endswith('.diff') for k in sample_keys):
+        format_info["format"] = "full_diff"
+        format_info["down_suffix"] = ".diff"  # Used for pairing, but no up
+        format_info["up_suffix"] = None  # No up weight in full diff
+        format_info["alpha_suffix"] = None
+        format_info["is_full_diff"] = True
+
     # Kohya/A1111 format: lora_unet_down_blocks_0_*.lora_down.weight
-    if any('lora_unet_' in k or 'lora_te' in k for k in sample_keys):
+    elif any('lora_unet_' in k or 'lora_te' in k for k in sample_keys):
         format_info["format"] = "kohya"
         format_info["down_suffix"] = ".lora_down.weight"
         format_info["up_suffix"] = ".lora_up.weight"
-    
+
     # Diffusers format: transformer.*.lora_A.weight
     elif any('.lora_A.weight' in k for k in sample_keys):
         format_info["format"] = "diffusers"
         format_info["down_suffix"] = ".lora_A.weight"
         format_info["up_suffix"] = ".lora_B.weight"
         format_info["alpha_suffix"] = ".alpha"
-    
+
     # PEFT format: base_model.model.*.lora_A.default.weight
     elif any('base_model.model.' in k and 'lora_' in k for k in sample_keys):
         format_info["format"] = "peft"
         # Handle nested structure
         format_info["down_suffix"] = ".lora_A.default.weight"
         format_info["up_suffix"] = ".lora_B.default.weight"
-    
+
+    # HuggingFace format: transformer.*.lora.down.weight (dots instead of underscores)
+    elif any('.lora.down.weight' in k for k in sample_keys):
+        format_info["format"] = "huggingface"
+        format_info["down_suffix"] = ".lora.down.weight"
+        format_info["up_suffix"] = ".lora.up.weight"
+        format_info["alpha_suffix"] = ".alpha"
+
     # ComfyUI native / standard
     elif any('.lora_down.weight' in k for k in sample_keys):
         format_info["format"] = "comfy"
         format_info["down_suffix"] = ".lora_down.weight"
         format_info["up_suffix"] = ".lora_up.weight"
-    
+
     # Count LoRA layers
     down_keys = [k for k in keys if format_info["down_suffix"] in k]
     format_info["key_count"] = len(down_keys)
-    
+
     return format_info
 
 
+
 def extract_lora_pairs(keys: List[str], format_info: Dict) -> Dict[str, Dict[str, str]]:
     """
     Group LoRA keys into down/up/alpha pairs.
-    
+
+    For full_diff format, groups .diff and .diff_b keys.
+
     Returns:
         Dict[block_name, {"down": key, "up": key, "alpha": key}]
+        For full_diff: {"diff": key, "diff_b": key}
     """
     down_suffix = format_info["down_suffix"]
     up_suffix = format_info["up_suffix"]
     alpha_suffix = format_info["alpha_suffix"]
-    
+    is_full_diff = format_info.get("is_full_diff", False)
+
     pairs = {}
-    
-    for key in keys:
-        if down_suffix in key:
-            block_name = key.replace(down_suffix, "")
-            if block_name not in pairs:
-                pairs[block_name] = {}
-            pairs[block_name]["down"] = key
-        elif up_suffix in key:
-            block_name = key.replace(up_suffix, "")
-            if block_name not in pairs:
-                pairs[block_name] = {}
-            pairs[block_name]["up"] = key
-        elif alpha_suffix in key or key.endswith('.alpha'):
-            # Handle various alpha key formats
-            block_name = key.replace(alpha_suffix, "").replace(".alpha", "")
-            if block_name not in pairs:
-                pairs[block_name] = {}
-            pairs[block_name]["alpha"] = key
-    
+
+    if is_full_diff:
+        # Full diff format: group .diff and .diff_b
+        for key in keys:
+            if key.endswith('.diff'):
+                block_name = key[:-5]  # Remove .diff
+                if block_name not in pairs:
+                    pairs[block_name] = {}
+                pairs[block_name]["diff"] = key
+            elif key.endswith('.diff_b'):
+                block_name = key[:-7]  # Remove .diff_b
+                if block_name not in pairs:
+                    pairs[block_name] = {}
+                pairs[block_name]["diff_b"] = key
+    else:
+        # Standard LoRA format
+        for key in keys:
+            if down_suffix and down_suffix in key:
+                block_name = key.replace(down_suffix, "")
+                if block_name not in pairs:
+                    pairs[block_name] = {}
+                pairs[block_name]["down"] = key
+            elif up_suffix and up_suffix in key:
+                block_name = key.replace(up_suffix, "")
+                if block_name not in pairs:
+                    pairs[block_name] = {}
+                pairs[block_name]["up"] = key
+            elif alpha_suffix and (alpha_suffix in key or key.endswith('.alpha')):
+                # Handle various alpha key formats
+                block_name = key.replace(alpha_suffix, "").replace(".alpha", "")
+                if block_name not in pairs:
+                    pairs[block_name] = {}
+                pairs[block_name]["alpha"] = key
+
     return pairs
 
 
 def detect_lora_rank(handler: MemoryEfficientSafeOpen, pairs: Dict) -> Tuple[int, float]:
     """
     Detect the rank and alpha of an existing LoRA.
-    
+
     Returns:
         (network_dim, network_alpha)
     """
     network_dim = None
     network_alpha = None
-    
+
     for block_name, block_keys in pairs.items():
         if "down" not in block_keys:
             continue
-        
+
         # Get dim from down weight shape
         if network_dim is None:
             down_key = block_keys["down"]
-            shape = handler.header[down_key]["shape"]
+            shape = handler.get_shape(down_key)
             # Linear: [rank, in_features] or Conv: [rank, in_ch, k, k]
             network_dim = shape[0]
-        
+
         # Get alpha if present
         if network_alpha is None and "alpha" in block_keys:
             alpha_tensor = handler.get_tensor(block_keys["alpha"])
             network_alpha = float(alpha_tensor.item())
-        
+
         if network_dim is not None and network_alpha is not None:
             break
-    
+
     # Default alpha to dim if not found
     if network_alpha is None:
         network_alpha = float(network_dim) if network_dim else 1.0
     if network_dim is None:
         network_dim = 1
-    
+
     return network_dim, network_alpha
 
 
@@ -183,10 +229,10 @@ def _merge_conv(lora_down: torch.Tensor, lora_up: torch.Tensor, device: str) ->
     in_rank, in_size, kernel_size, k_ = lora_down.shape
     out_size, out_rank, _, _ = lora_up.shape
     assert in_rank == out_rank, f"rank mismatch: {in_rank} vs {out_rank}"
-    
+
     lora_down = lora_down.to(device=device, dtype=torch.float32)
     lora_up = lora_up.to(device=device, dtype=torch.float32)
-    
+
     merged = lora_up.reshape(out_size, -1) @ lora_down.reshape(in_rank, -1)
     return merged.reshape(out_size, in_size, kernel_size, kernel_size)
 
@@ -205,7 +251,7 @@ def _extract_linear(
 ) -> Dict:
     """Extract LoRA from merged linear weight."""
     out_size, in_size = weight.shape
-    
+
     if dynamic_method is None:
         # Fixed rank - use svd_lowrank for 10x speedup
         rank = min(max_rank, min(out_size, in_size) - 1)
@@ -221,10 +267,10 @@ def _extract_linear(
         U = U[:, :new_rank]
         S = S[:new_rank]
         Vh = Vh[:new_rank, :]
-    
+
     lora_up = U @ torch.diag(S)
     lora_down = Vh
-    
+
     return {
         "lora_down": lora_down.cpu().contiguous(),
         "lora_up": lora_up.cpu().contiguous(),
@@ -245,7 +291,7 @@ def _extract_conv(
     """Extract LoRA from merged conv weight."""
     out_ch, in_ch, kh, kw = weight.shape
     mat = weight.reshape(out_ch, -1)
-    
+
     if dynamic_method is None:
         # Fixed rank - use svd_lowrank for 10x speedup
         rank = min(max_rank, min(mat.shape) - 1)
@@ -261,10 +307,10 @@ def _extract_conv(
         U = U[:, :new_rank]
         S = S[:new_rank]
         Vh = Vh[:new_rank, :]
-    
+
     lora_up = (U @ torch.diag(S)).reshape(out_ch, new_rank, 1, 1)
     lora_down = Vh.reshape(new_rank, in_ch, kh, kw)
-    
+
     return {
         "lora_down": lora_down.cpu().contiguous(),
         "lora_up": lora_up.cpu().contiguous(),
@@ -282,40 +328,39 @@ def _compute_resize(
     scale: float
 ) -> Tuple[int, float, Dict]:
     """Compute new rank and alpha based on resize method."""
-    
+
     if dynamic_method == "sv_ratio" and dynamic_param is not None:
-        # Note: _index_sv_ratio uses S[0]*ratio, kohya uses S[0]/ratio
-        # Use kohya convention here for consistency with their tool
+        # Use kohya convention: S[0]/ratio
         min_sv = S[0] / dynamic_param
         new_rank = max(1, int(torch.sum(S > min_sv).item()))
     elif dynamic_method == "sv_cumulative" and dynamic_param is not None:
-        new_rank = _index_sv_cumulative(S, dynamic_param)
+        new_rank = _index_sv_cumulative(S, dynamic_param, max_rank)
     elif dynamic_method == "sv_fro" and dynamic_param is not None:
-        new_rank = _index_sv_fro(S, dynamic_param)
+        new_rank = _index_sv_fro(S, dynamic_param, max_rank)
     else:
         new_rank = max_rank
-    
+
     # Clamp rank
     if S[0] < MIN_SV:
         new_rank = 1
     else:
         new_rank = max(1, min(new_rank, max_rank, len(S) - 1))
-    
+
     new_alpha = float(scale * new_rank)
-    
+
     # Compute retention stats
     s_sum = float(torch.sum(torch.abs(S)))
     s_rank = float(torch.sum(torch.abs(S[:new_rank]))) if new_rank <= len(S) else s_sum
-    
+
     S_sq = S.pow(2)
     s_fro = float(torch.sqrt(torch.sum(S_sq)))
     s_red_fro = float(torch.sqrt(torch.sum(S_sq[:new_rank]))) if new_rank <= len(S) else s_fro
-    
+
     stats = {
         "sum_retained": s_rank / s_sum if s_sum > 0 else 1.0,
         "fro_retained": s_red_fro / s_fro if s_fro > 0 else 1.0,
     }
-    
+
     return new_rank, new_alpha, stats
 
 
@@ -336,7 +381,7 @@ def resize_lora_file(
 ) -> str:
     """
     Resize a LoRA file to a new rank.
-    
+
     Args:
         lora_path: Path to input LoRA
         new_rank: Target rank (max rank for dynamic methods)
@@ -347,53 +392,53 @@ def resize_lora_file(
         output_filename: Output filename (without extension)
         verbose: Print progress info
         svd_niter: Power iterations for SVD accuracy
-    
+
     Returns:
         Path to saved resized LoRA
     """
     # Prepare memory
     lora_size_gb = estimate_model_size(lora_path)
     prepare_for_large_operation(lora_size_gb * 2, torch.device(device))
-    
+
     handler = MemoryEfficientSafeOpen(lora_path)
-    
+
     try:
         metadata = handler.metadata().copy()
         all_keys = handler.keys()
-        
+
         # Detect format and extract pairs
         format_info = detect_lora_format(all_keys)
         pairs = extract_lora_pairs(all_keys, format_info)
         network_dim, network_alpha = detect_lora_rank(handler, pairs)
-        
+
         scale = network_alpha / network_dim if network_dim > 0 else 1.0
-        
+
         if verbose:
             method_str = f"{dynamic_method}: {dynamic_param}" if dynamic_method else "fixed"
             print(f"[LoRA Resize] Format: {format_info['format']}, layers: {format_info['key_count']}")
             print(f"[LoRA Resize] Original dim={network_dim}, alpha={network_alpha:.1f}, scale={scale:.3f}")
             print(f"[LoRA Resize] Resizing with method={method_str}, max_rank={new_rank}")
-        
+
         output_sd = {}
         fro_list = []
         pbar = comfy.utils.ProgressBar(len(pairs))
-        
+
         with torch.no_grad():
             for block_name, block_keys in tqdm(pairs.items(), desc="Resizing layers", unit="layers"):
                 if "down" not in block_keys or "up" not in block_keys:
                     pbar.update(1)
                     continue
-                
+
                 lora_down = handler.get_tensor(block_keys["down"])
                 lora_up = handler.get_tensor(block_keys["up"])
-                
+
                 is_conv = len(lora_down.shape) == 4
-                
+
                 # Transfer to GPU with pinned memory if available
                 if device == 'cuda':
                     lora_down = transfer_to_gpu_pinned(lora_down, device, torch.float32)
                     lora_up = transfer_to_gpu_pinned(lora_up, device, torch.float32)
-                
+
                 # Merge and re-extract
                 if is_conv:
                     weight = _merge_conv(lora_down, lora_up, device)
@@ -401,28 +446,38 @@ def resize_lora_file(
                 else:
                     weight = _merge_linear(lora_down, lora_up, device)
                     result = _extract_linear(weight, new_rank, dynamic_method, dynamic_param, scale, svd_niter)
-                
+
                 del weight, lora_down, lora_up
-                
+
                 fro_list.append(result['fro_retained'])
-                
+
                 # Store using same format suffixes as input
                 down_suffix = format_info["down_suffix"]
                 up_suffix = format_info["up_suffix"]
                 alpha_suffix = format_info["alpha_suffix"]
-                
-                output_sd[f"{block_name}{down_suffix}"] = result["lora_down"].to(save_dtype)
-                output_sd[f"{block_name}{up_suffix}"] = result["lora_up"].to(save_dtype)
-                output_sd[f"{block_name}{alpha_suffix}"] = torch.tensor([result["new_alpha"]]).to(save_dtype)
-                
+
+                # Standardize prefix using heuristics if possible
+                new_block_name = _format_lora_key(block_name)
+
+                output_sd[f"{new_block_name}{down_suffix}"] = result["lora_down"].to(save_dtype)
+                output_sd[f"{new_block_name}{up_suffix}"] = result["lora_up"].to(save_dtype)
+                if alpha_suffix:
+                    output_sd[f"{new_block_name}{alpha_suffix}"] = torch.tensor(result["new_alpha"]).to(save_dtype)
+
+                del result
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
                 pbar.update(1)
-        
+
         if verbose and fro_list:
             import numpy as np
             avg_fro = np.mean(fro_list)
             std_fro = np.std(fro_list)
             print(f"[LoRA Resize] Average Frobenius retention: {avg_fro:.1%} ± {std_fro:.3f}")
-        
+
         # Update metadata
         if dynamic_method:
             metadata["ss_training_comment"] = f"Dynamic resize with {dynamic_method}: {dynamic_param} from dim {network_dim}"
@@ -432,17 +487,17 @@ def resize_lora_file(
             metadata["ss_training_comment"] = f"Resized from dim {network_dim} to {new_rank}"
             metadata["ss_network_dim"] = str(new_rank)
             metadata["ss_network_alpha"] = str(scale * new_rank)
-        
+
         # Save
         output_dir = folder_paths.get_folder_paths("loras")[0]
         os.makedirs(output_dir, exist_ok=True)
         output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors")
-        
+
         save_file(output_sd, output_path, metadata)
         print(f"[LoRA Resize] Saved to {output_path}")
-        
+
         return output_path
-        
+
     finally:
         handler.__exit__(None, None, None)
         cleanup_after_operation()
@@ -454,7 +509,7 @@ def resize_lora_file(
 
 class LoRAResizeFixed(io.ComfyNode):
     """Resize LoRA to a fixed rank."""
-    
+
     @classmethod
     def define_schema(cls):
         return io.Schema(
@@ -465,7 +520,7 @@ class LoRAResizeFixed(io.ComfyNode):
             inputs=[
                 io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"),
                               tooltip="LoRA to resize"),
-                io.Int.Input("new_rank", default=64, min=1, max=1024,
+                io.Int.Input("new_rank", default=64, min=1, max=3072,
                             tooltip="Target rank"),
                 io.Int.Input("svd_niter", default=2, min=0, max=10,
                             tooltip="SVD power iterations (higher = more accurate but slower)"),
@@ -476,19 +531,19 @@ class LoRAResizeFixed(io.ComfyNode):
             outputs=[io.String.Output(display_name="output_path")],
             is_output_node=True,
         )
-    
+
     @classmethod
     def execute(cls, lora_name, new_rank, svd_niter, output_filename, save_dtype, device) -> io.NodeOutput:
         lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
         dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
-        
+
         path = resize_lora_file(lora_path, new_rank, None, None, device, dtype, output_filename, svd_niter=svd_niter)
         return io.NodeOutput(path)
 
 
 class LoRAResizeRatio(io.ComfyNode):
     """Resize LoRA keeping SVs above ratio threshold."""
-    
+
     @classmethod
     def define_schema(cls):
         return io.Schema(
@@ -499,7 +554,7 @@ class LoRAResizeRatio(io.ComfyNode):
             inputs=[
                 io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"),
                               tooltip="LoRA to resize"),
-                io.Int.Input("max_rank", default=128, min=1, max=1024,
+                io.Int.Input("max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum allowed rank"),
                 io.Float.Input("ratio", default=2.0, min=1.0, max=100.0, step=0.1,
                               tooltip="Keep SVs where S[i] > S[0]/ratio"),
@@ -510,19 +565,19 @@ class LoRAResizeRatio(io.ComfyNode):
             outputs=[io.String.Output(display_name="output_path")],
             is_output_node=True,
         )
-    
+
     @classmethod
     def execute(cls, lora_name, max_rank, ratio, output_filename, save_dtype, device) -> io.NodeOutput:
         lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
         dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
-        
+
         path = resize_lora_file(lora_path, max_rank, "sv_ratio", ratio, device, dtype, output_filename)
         return io.NodeOutput(path)
 
 
 class LoRAResizeFrobenius(io.ComfyNode):
     """Resize LoRA to preserve Frobenius norm target."""
-    
+
     @classmethod
     def define_schema(cls):
         return io.Schema(
@@ -533,7 +588,7 @@ class LoRAResizeFrobenius(io.ComfyNode):
             inputs=[
                 io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"),
                               tooltip="LoRA to resize"),
-                io.Int.Input("max_rank", default=128, min=1, max=1024,
+                io.Int.Input("max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum allowed rank"),
                 io.Float.Input("target", default=0.9, min=0.1, max=1.0, step=0.01,
                               tooltip="Target Frobenius norm retention (0.9 = 90%)"),
@@ -544,19 +599,19 @@ class LoRAResizeFrobenius(io.ComfyNode):
             outputs=[io.String.Output(display_name="output_path")],
             is_output_node=True,
         )
-    
+
     @classmethod
     def execute(cls, lora_name, max_rank, target, output_filename, save_dtype, device) -> io.NodeOutput:
         lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
         dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
-        
+
         path = resize_lora_file(lora_path, max_rank, "sv_fro", target, device, dtype, output_filename)
         return io.NodeOutput(path)
 
 
 class LoRAResizeCumulative(io.ComfyNode):
     """Resize LoRA to preserve cumulative SV target."""
-    
+
     @classmethod
     def define_schema(cls):
         return io.Schema(
@@ -567,7 +622,7 @@ class LoRAResizeCumulative(io.ComfyNode):
             inputs=[
                 io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"),
                               tooltip="LoRA to resize"),
-                io.Int.Input("max_rank", default=128, min=1, max=1024,
+                io.Int.Input("max_rank", default=128, min=1, max=3072,
                             tooltip="Maximum allowed rank"),
                 io.Float.Input("target", default=0.9, min=0.1, max=1.0, step=0.01,
                               tooltip="Target cumulative SV retention (0.9 = 90%)"),
@@ -578,11 +633,357 @@ class LoRAResizeCumulative(io.ComfyNode):
             outputs=[io.String.Output(display_name="output_path")],
             is_output_node=True,
         )
-    
+
     @classmethod
     def execute(cls, lora_name, max_rank, target, output_filename, save_dtype, device) -> io.NodeOutput:
         lora_path = folder_paths.get_full_path_or_raise("loras", lora_name)
         dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
-        
+
         path = resize_lora_file(lora_path, max_rank, "sv_cumulative", target, device, dtype, output_filename)
         return io.NodeOutput(path)
+
+
+
+
+# =============================================================================
+# LoRA Merge To Model (Save merged model, skip extraction)
+# =============================================================================
+
+def merge_loras_to_model(
+    lora_paths: List[str],
+    lora_weights: List[float],
+    base_model_path: str,
+    device: str,
+    save_dtype: torch.dtype,
+    output_filename: str,
+    skip_patterns_str: str = "",
+    verbose: bool = True,
+) -> str:
+    """
+    Merge multiple LoRAs into a base model and save the result directly.
+
+    Unlike merge_multi_loras_via_base, this function:
+    - Does NOT extract the result back to a LoRA
+    - Saves the merged full model to the base model's directory
+
+    Args:
+        lora_paths: List of paths to LoRA files
+        lora_weights: List of weight strengths (0.0-2.0) for each LoRA
+        base_model_path: Path to base model
+        device: Processing device
+        save_dtype: Output dtype
+        output_filename: Output filename (without extension)
+        skip_patterns_str: Regex patterns for layers to skip
+        verbose: Print progress info
+
+    Returns:
+        Path to saved merged model
+    """
+    # Estimate memory and prepare
+    total_size_gb = estimate_model_size(base_model_path)
+    for lp in lora_paths:
+        total_size_gb += estimate_model_size(lp)
+
+    if verbose:
+        print(f"[LoRA Merge To Model] Preparing memory for {total_size_gb:.2f}GB operation...")
+        print(f"[LoRA Merge To Model] Merging {len(lora_paths)} LoRAs with weights: {lora_weights}")
+    prepare_for_large_operation(total_size_gb * 1.5, torch.device(device))
+
+    # Open all files - use low_memory for base model to avoid OS page caching
+    base_handler = MemoryEfficientSafeOpen(base_model_path, low_memory=True)
+    lora_handlers = [MemoryEfficientSafeOpen(lp) for lp in lora_paths]
+
+
+    try:
+        # Detect format and extract pairs for each LoRA
+        lora_infos = []
+
+        for i, handler in enumerate(lora_handlers):
+            keys = handler.keys()
+            format_info = detect_lora_format(keys)
+            pairs = extract_lora_pairs(keys, format_info)
+            network_dim, network_alpha = detect_lora_rank(handler, pairs)
+            lora_infos.append({
+                "handler": handler,
+                "format_info": format_info,
+                "pairs": pairs,
+                "network_dim": network_dim,
+                "network_alpha": network_alpha,
+                "weight": lora_weights[i],
+            })
+            if verbose:
+                print(f"[LoRA Merge To Model] LoRA {i+1}: {format_info['format']}, {len(pairs)} layers, dim={network_dim}")
+
+        # Common prefixes
+        BASE_PREFIXES = ["model.diffusion_model.", "diffusion_model.", "transformer.", "model."]
+        LORA_PREFIXES = [
+            "lora_unet_", "lora_transformer_", "lora_te1_", "lora_te2_", "lora_te_",
+            "lycoris_", "diffusion_model.", "transformer.", "unet."
+        ]
+
+        def extract_core_layer_base(key: str) -> str:
+            result = key
+            if result.endswith(".weight"):
+                result = result[:-7]
+            elif result.endswith(".bias"):
+                result = result[:-5]
+            for prefix in BASE_PREFIXES:
+                if result.startswith(prefix):
+                    result = result[len(prefix):]
+                    break
+            return result
+
+        def extract_core_layer_lora(block_name: str) -> str:
+            result = block_name
+            for prefix in LORA_PREFIXES:
+                if result.startswith(prefix):
+                    result = result[len(prefix):]
+                    break
+            # Strip trailing .lora suffix (HuggingFace format leaves this after suffix removal)
+            if result.endswith(".lora"):
+                result = result[:-5]
+            return result.replace(".", "_")
+
+
+        # Build LoRA lookup: core layer name (underscored) -> list of (info, block_keys)
+        lora_lookup = {}
+        for info in lora_infos:
+            for block_name, block_keys in info["pairs"].items():
+                core = extract_core_layer_lora(block_name)
+                if core not in lora_lookup:
+                    lora_lookup[core] = []
+                lora_lookup[core].append((info, block_keys))
+
+        # Compile skip patterns
+        skip_patterns = _compile_patterns(skip_patterns_str)
+
+        # Preserve metadata from base model
+        base_metadata = base_handler.metadata().copy() if base_handler.metadata() else {}
+        base_metadata["merge_comment"] = f"Merged {len(lora_paths)} LoRAs with weights: {lora_weights}"
+
+        output_sd = {}
+        stats = {"merged": 0, "copied": 0, "skipped": 0}
+        base_keys = list(base_handler.keys())
+        pbar = comfy.utils.ProgressBar(len(base_keys))
+
+        if verbose:
+            print(f"[LoRA Merge To Model] Processing {len(base_keys)} base model keys...")
+
+        with torch.no_grad():
+            for base_key in tqdm(base_keys, desc="Merging to model", unit="keys"):
+                # Check skip patterns
+                if _matches_any_pattern(base_key, skip_patterns):
+                    stats["skipped"] += 1
+                    pbar.update(1)
+                    continue
+
+                # Load base weight
+                cpu_base = base_handler.get_tensor(base_key)
+
+                # Only process weight tensors for LoRA merging
+                if base_key.endswith(".weight"):
+                    core = extract_core_layer_base(base_key)
+                    core_underscored = core.replace(".", "_")
+
+                    # Check if any LoRA contributes to this layer
+                    if core_underscored in lora_lookup:
+                        # Transfer to GPU for computation
+                        if device == 'cuda':
+                            base_weight = transfer_to_gpu_pinned(cpu_base, device, torch.float32)
+                        else:
+                            base_weight = cpu_base.to(device=device, dtype=torch.float32)
+                        del cpu_base
+
+                        # Accumulate deltas from all contributing LoRAs
+                        for info, block_keys in lora_lookup[core_underscored]:
+                            is_full_diff = info["format_info"].get("is_full_diff", False)
+
+                            if is_full_diff:
+                                # Full diff format
+                                if "diff" not in block_keys:
+                                    continue
+                                cpu_diff = info["handler"].get_tensor(block_keys["diff"])
+                                if device == 'cuda':
+                                    delta = transfer_to_gpu_pinned(cpu_diff, device, torch.float32)
+                                else:
+                                    delta = cpu_diff.to(device=device, dtype=torch.float32)
+                                del cpu_diff
+                                effective_scale = info["weight"]
+                            else:
+                                # Standard LoRA format
+                                if "down" not in block_keys or "up" not in block_keys:
+                                    continue
+
+                                cpu_down = info["handler"].get_tensor(block_keys["down"])
+                                cpu_up = info["handler"].get_tensor(block_keys["up"])
+                                if device == 'cuda':
+                                    lora_down = transfer_to_gpu_pinned(cpu_down, device, torch.float32)
+                                    lora_up = transfer_to_gpu_pinned(cpu_up, device, torch.float32)
+                                else:
+                                    lora_down = cpu_down.to(device=device, dtype=torch.float32)
+                                    lora_up = cpu_up.to(device=device, dtype=torch.float32)
+                                del cpu_down, cpu_up
+
+                                # Get alpha
+                                if "alpha" in block_keys:
+                                    alpha_tensor = info["handler"].get_tensor(block_keys["alpha"])
+                                    layer_alpha = float(alpha_tensor.item())
+                                else:
+                                    layer_alpha = float(info["network_dim"])
+                                layer_scale = layer_alpha / info["network_dim"] if info["network_dim"] > 0 else 1.0
+                                effective_scale = layer_scale * info["weight"]
+
+                                # Compute delta
+                                is_conv = len(lora_down.shape) == 4
+                                if is_conv:
+                                    in_rank, in_size, kernel_size, k_ = lora_down.shape
+                                    out_size, out_rank, _, _ = lora_up.shape
+                                    delta = lora_up.reshape(out_size, -1) @ lora_down.reshape(in_rank, -1)
+                                    delta = delta.reshape(out_size, in_size, kernel_size, kernel_size)
+                                else:
+                                    delta = lora_up @ lora_down
+                                del lora_down, lora_up
+
+                            # Apply delta to base weight
+                            base_weight = base_weight + effective_scale * delta
+                            del delta
+
+                        # Store merged weight
+                        output_sd[base_key] = base_weight.to(save_dtype).cpu().contiguous()
+                        del base_weight
+                        stats["merged"] += 1
+                    else:
+                        # No LoRA contribution, copy as-is
+                        output_sd[base_key] = cpu_base.to(save_dtype).contiguous()
+                        del cpu_base
+                        stats["copied"] += 1
+                else:
+                    # Non-weight tensor (bias, norm, etc.), copy as-is
+                    output_sd[base_key] = cpu_base.to(save_dtype).contiguous()
+                    del cpu_base
+                    stats["copied"] += 1
+
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
+                pbar.update(1)
+
+        if verbose:
+            print(f"[LoRA Merge To Model] Done: {stats['merged']} merged, {stats['copied']} copied, {stats['skipped']} skipped")
+
+        # Save to base model directory
+        base_dir = os.path.dirname(base_model_path)
+        os.makedirs(base_dir, exist_ok=True)
+        output_path = os.path.join(base_dir, f"{output_filename.strip()}.safetensors")
+
+        save_file(output_sd, output_path, base_metadata)
+        print(f"[LoRA Merge To Model] Saved to {output_path}")
+
+        return output_path
+
+    finally:
+        base_handler.__exit__(None, None, None)
+        for handler in lora_handlers:
+            handler.__exit__(None, None, None)
+        cleanup_after_operation()
+
+
+class LoRAMergeToModel(io.ComfyNode):
+    """Merge multiple LoRAs into base model and save as full model."""
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="LoRAMergeToModel",
+            display_name="LoRA Merge To Model",
+            category="ModelUtils/LoRA/Merge",
+            description="Merge 1-8 LoRAs into a base model and save the result. Saves to base model directory.",
+            inputs=[
+                io.Combo.Input("base_model", options=folder_paths.get_filename_list("diffusion_models"),
+                              tooltip="Base model the LoRAs were trained on"),
+                io.Combo.Input("lora_count", options=["1", "2", "3", "4", "5", "6", "7", "8"], default="2",
+                              tooltip="Number of LoRAs to merge"),
+                # LoRA 1
+                io.Combo.Input("lora_1", options=folder_paths.get_filename_list("loras"),
+                              tooltip="First LoRA"),
+                io.Float.Input("weight_1", default=1.0, min=-10.0, max=10.0, step=0.01,
+                              tooltip="Weight strength for LoRA 1"),
+                # LoRA 2
+                io.Combo.Input("lora_2", options=["None"] + folder_paths.get_filename_list("loras"),
+                              default="None", tooltip="Second LoRA"),
+                io.Float.Input("weight_2", default=1.0, min=-10.0, max=10.0, step=0.01,
+                              tooltip="Weight strength for LoRA 2"),
+                # LoRA 3
+                io.Combo.Input("lora_3", options=["None"] + folder_paths.get_filename_list("loras"),
+                              default="None", tooltip="Third LoRA"),
+                io.Float.Input("weight_3", default=1.0, min=-10.0, max=10.0, step=0.01,
+                              tooltip="Weight strength for LoRA 3"),
+                # LoRA 4
+                io.Combo.Input("lora_4", options=["None"] + folder_paths.get_filename_list("loras"),
+                              default="None", tooltip="Fourth LoRA"),
+                io.Float.Input("weight_4", default=1.0, min=-10.0, max=10.0, step=0.01,
+                              tooltip="Weight strength for LoRA 4"),
+                # LoRA 5
+                io.Combo.Input("lora_5", options=["None"] + folder_paths.get_filename_list("loras"),
+                              default="None", tooltip="Fifth LoRA"),
+                io.Float.Input("weight_5", default=1.0, min=-10.0, max=10.0, step=0.01,
+                              tooltip="Weight strength for LoRA 5"),
+                # LoRA 6
+                io.Combo.Input("lora_6", options=["None"] + folder_paths.get_filename_list("loras"),
+                              default="None", tooltip="Sixth LoRA"),
+                io.Float.Input("weight_6", default=1.0, min=-10.0, max=10.0, step=0.01,
+                              tooltip="Weight strength for LoRA 6"),
+                # LoRA 7
+                io.Combo.Input("lora_7", options=["None"] + folder_paths.get_filename_list("loras"),
+                              default="None", tooltip="Seventh LoRA"),
+                io.Float.Input("weight_7", default=1.0, min=-10.0, max=10.0, step=0.01,
+                              tooltip="Weight strength for LoRA 7"),
+                # LoRA 8
+                io.Combo.Input("lora_8", options=["None"] + folder_paths.get_filename_list("loras"),
+                              default="None", tooltip="Eighth LoRA"),
+                io.Float.Input("weight_8", default=1.0, min=-10.0, max=10.0, step=0.01,
+                              tooltip="Weight strength for LoRA 8"),
+                # Settings
+                io.String.Input("skip_patterns", default="", multiline=True,
+                               tooltip="Regex patterns for layers to skip"),
+                io.String.Input("output_filename", default="merged_model"),
+                io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"),
+                io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"),
+            ],
+            outputs=[io.String.Output(display_name="output_path")],
+            is_output_node=True,
+        )
+
+    @classmethod
+    def execute(cls, base_model, lora_count,
+                lora_1, weight_1, lora_2, weight_2, lora_3, weight_3, lora_4, weight_4,
+                lora_5, weight_5, lora_6, weight_6, lora_7, weight_7, lora_8, weight_8,
+                skip_patterns, output_filename, save_dtype, device) -> io.NodeOutput:
+
+        # Build LoRA list based on count
+        count = int(lora_count)
+        lora_names = [lora_1, lora_2, lora_3, lora_4, lora_5, lora_6, lora_7, lora_8][:count]
+        lora_weights = [weight_1, weight_2, weight_3, weight_4, weight_5, weight_6, weight_7, weight_8][:count]
+
+        # Filter out "None" entries
+        valid_loras = [(name, weight) for name, weight in zip(lora_names, lora_weights) if name != "None"]
+        if not valid_loras:
+            raise ValueError("At least one LoRA must be selected")
+
+        lora_names, lora_weights = zip(*valid_loras)
+        lora_paths = [folder_paths.get_full_path_or_raise("loras", name) for name in lora_names]
+        base_path = folder_paths.get_full_path_or_raise("diffusion_models", base_model)
+        dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype]
+
+        path = merge_loras_to_model(
+            lora_paths=list(lora_paths),
+            lora_weights=list(lora_weights),
+            base_model_path=base_path,
+            device=device,
+            save_dtype=dtype,
+            output_filename=output_filename,
+            skip_patterns_str=skip_patterns,
+        )
+        return io.NodeOutput(path)

nodes/merger.py

@@ -7,11 +7,12 @@ from tqdm import tqdm
 from comfy_api.latest import io
 from safetensors.torch import save_file
 from .merger_ops import TWO_MODEL_MODES, THREE_MODEL_MODES, MissingTensorBehavior, MissingTensorError
-from .merger_utils import MemoryEfficientSafeOpen, transfer_to_gpu_pinned
 from .device_utils import (
     estimate_model_size, prepare_for_large_operation, cleanup_after_operation
 )
 
+from unifiedefficientloader import MemoryEfficientSafeOpen, transfer_to_gpu_pinned, IncrementalSafetensorsWriter
+
 
 def load_documentation_from_file(filename):
     """Loads documentation from a markdown file in the ../docs/ directory."""
@@ -76,6 +77,9 @@ class MergerLogic:
             process_device = recipe_params.get('device', 'cpu')
             print(f"[Merger] Preparing memory for {total_size_gb:.2f}GB merge operation...")
             prepare_for_large_operation(total_size_gb * 1.2, torch.device(process_device))
+            flush_threshold_bytes = int(total_size_gb * 1024**3 / 16)
+        else:
+            flush_threshold_bytes = 256 * 1024 * 1024  # 256MB fallback (min 8GB RAM assumed)
 
         handlers = {}
         for name in model_names.values():
@@ -83,16 +87,17 @@ class MergerLogic:
                 path = folder_paths.get_full_path(model_type, name)
                 if not path:
                     raise FileNotFoundError(f"Model '{name}' not found.")
-                handlers[name] = MemoryEfficientSafeOpen(path)
+                handlers[name] = MemoryEfficientSafeOpen(path, low_memory=True)
 
         primary_handler = handlers[primary_model_name]
         all_keys = primary_handler.keys()
-        metadata = primary_handler.header.get("__metadata__", {})
+        metadata = primary_handler.metadata()
 
         # Convert mismatch_mode string to enum
         mismatch_mode_str = recipe_params.get('mismatch_mode', 'skip')
         mismatch_mode = MissingTensorBehavior(mismatch_mode_str)
         recipe_params['mismatch_mode'] = mismatch_mode
+        alignment_mode = recipe_params.get('alignment_mode', 'pad/crop')
 
         # Compile filter patterns
         exclude_patterns = _compile_patterns(recipe_params.get('exclude_patterns', ''))
@@ -123,56 +128,125 @@ class MergerLogic:
         discarded_keys = 0
         error_keys = []
 
-        for key in tqdm(all_keys, desc="Merging layers", unit="layers"):
-            # Check discard patterns first - skip entirely
-            if _matches_any_pattern(key, discard_patterns):
-                discarded_keys += 1
-                pbar.update(1)
-                continue
+        output_folder = "loras" if calc_mode == "SVD LoRA Extraction" else model_type
+        # Use [-1] for diffusion_models to get the actual diffusion_models folder, not legacy unet
+        output_dir = folder_paths.get_folder_paths(output_folder)[-1]
+        os.makedirs(output_dir, exist_ok=True)
+        output_filename = recipe_params.get("output_filename")
+        output_path = os.path.join(output_dir, f"{output_filename}.safetensors")
 
-            # Pre-load Model A's tensor with pinned memory for CUDA
-            cpu_tensor = primary_handler.get_tensor(key)
-            if process_device == 'cuda':
-                tensor_a = transfer_to_gpu_pinned(cpu_tensor, process_device, process_dtype)
-            else:
-                tensor_a = cpu_tensor.to(device=process_device, dtype=process_dtype)
-            del cpu_tensor
+        writer = None
 
-            # Check exclude patterns - use Model A only, no merge
-            if _matches_any_pattern(key, exclude_patterns):
-                merged_state_dict[key] = tensor_a.to(save_torch_dtype).cpu().clone()
-                excluded_keys += 1
-                pbar.update(1)
-                continue
+        writer = IncrementalSafetensorsWriter(output_path, metadata=metadata)
+        writer.__enter__()
 
-            # Pass tensor_a metadata to recipes for zeros mode and fallback
-            recipe_params['_tensor_a'] = tensor_a
-            recipe_params['_tensor_a_shape'] = tensor_a.shape
-            recipe_params['_tensor_a_dtype'] = tensor_a.dtype
+        batch_buffer = {}
+        batch_bytes = 0
+        batch_key_count = 0
 
-            try:
-                recipe = calc_mode_class.create_recipe(key=key, **recipe_params)
-                result = recipe.merge()
-            except MissingTensorError as e:
-                if mismatch_mode == MissingTensorBehavior.ERROR:
-                    raise ValueError(f"Layer mismatch error (mismatch_mode='error'): {e}")
-                result = None
-                error_keys.append(key)
+        def _flush_batch():
+            nonlocal batch_buffer, batch_bytes, batch_key_count
+            if not batch_buffer:
+                return
+            writer.write_dict(batch_buffer)
+            pbar.update(batch_key_count)
+            batch_buffer = {}
+            batch_bytes = 0
+            batch_key_count = 0
+            if recipe_params.get('force_clear_cache', True):
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
 
-            # Handle None result (mismatch occurred with skip mode)
-            if result is None:
-                result = tensor_a
-                skipped_keys += 1
+        with torch.no_grad():
+            for key in tqdm(all_keys, desc="Merging layers", unit="layers"):
+                # Check discard patterns first - skip entirely
+                if _matches_any_pattern(key, discard_patterns):
+                    discarded_keys += 1
+                    continue
 
-            if isinstance(result, dict):
-                for r_key, r_tensor in result.items():
-                    merged_state_dict[r_key] = r_tensor.to(save_torch_dtype).cpu().clone()
-            else:
-                merged_state_dict[key] = result.to(save_torch_dtype).cpu().clone()
+                # Pre-load Model A's tensor with pinned memory for CUDA
+                cpu_tensor = primary_handler.get_tensor(key)
+                if process_device == 'cuda':
+                    tensor_a = transfer_to_gpu_pinned(cpu_tensor, process_device, process_dtype)
+                else:
+                    tensor_a = cpu_tensor.to(device=process_device, dtype=process_dtype)
+                del cpu_tensor
 
-            # Clean up tensor_a reference to allow GC
-            del recipe_params['_tensor_a']
-            pbar.update(1)
+                # Check exclude patterns - use Model A only, no merge
+                if _matches_any_pattern(key, exclude_patterns):
+                    t = tensor_a.detach().to(save_torch_dtype).cpu()
+            
+                    batch_buffer[key] = t
+                    batch_bytes += t.numel() * t.element_size()
+                    batch_key_count += 1
+                    excluded_keys += 1
+                    del tensor_a
+                    if (batch_bytes >= flush_threshold_bytes or batch_key_count >= 32):
+                        _flush_batch()
+                    continue
+
+                # Pass tensor_a metadata to recipes for zeros mode and fallback
+                recipe_params['_tensor_a'] = tensor_a
+                recipe_params['_tensor_a_shape'] = tensor_a.shape
+                recipe_params['_tensor_a_dtype'] = tensor_a.dtype
+
+                try:
+                    recipe = calc_mode_class.create_recipe(key=key, **recipe_params)
+                    result = recipe.merge()
+                except MissingTensorError as e:
+                    if mismatch_mode == MissingTensorBehavior.ERROR:
+                
+                        _flush_batch()
+                        writer.__exit__(None, None, None)
+                        raise ValueError(f"Layer mismatch error (mismatch_mode='error'): {e}")
+                    result = None
+                    error_keys.append(key)
+
+                # Handle None result (mismatch occurred with skip mode)
+                if result is None:
+                    result = tensor_a
+                    skipped_keys += 1
+
+                if isinstance(result, dict):
+                    for r_key, r_tensor in result.items():
+                        t = r_tensor.detach().to(save_torch_dtype).cpu()
+                
+                        batch_buffer[r_key] = t
+                        batch_bytes += t.numel() * t.element_size()
+                else:
+                    # Ensure compatibility with Model A's architecture.
+                    # If alignment_mode is 'pad/crop', we crop results that were padded.
+                    # If alignment_mode is 'interpolate', resizing happened during operators.
+                    if alignment_mode == 'pad/crop':
+                        target_shape = recipe_params['_tensor_a_shape']
+                        if result.shape != target_shape:
+                            slices = tuple(slice(0, min(res_s, tgt_s)) for res_s, tgt_s in zip(result.shape, target_shape))
+                            result = result[slices]
+
+                    t = result.detach().to(save_torch_dtype).cpu()
+            
+                    batch_buffer[key] = t
+                    batch_bytes += t.numel() * t.element_size()
+
+        
+                batch_key_count += 1
+
+                # Clean up references to allow GC immediately.
+                # Local loop variables must be explicitly deleted to prevent PyTorch from keeping tensors in VRAM.
+                recipe.clean()
+                del recipe_params['_tensor_a']
+                del tensor_a
+                del recipe
+                del result
+
+                if (batch_bytes >= flush_threshold_bytes or batch_key_count >= 32):
+                    _flush_batch()
+
+
+        _flush_batch()
+        writer.__exit__(None, None, None)
 
         # Log summary
         if excluded_keys > 0:
@@ -193,10 +267,9 @@ class MergerLogic:
         os.makedirs(output_dir, exist_ok=True)
         output_filename = recipe_params.get("output_filename")
         output_path = os.path.join(output_dir, f"{output_filename}.safetensors")
-        save_file(merged_state_dict, output_path, metadata=metadata)
-
+        if os.path.exists(output_path):
+            print(f"[Merger] Output saved to {output_path}")
         # Cleanup after heavy operation
-        del merged_state_dict
         cleanup_after_operation()
 
         return f"{output_filename}.safetensors"
@@ -219,15 +292,19 @@ class CheckpointTwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("checkpoints")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_checkpoint"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -237,20 +314,22 @@ class CheckpointTwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
         filename = MergerLogic.execute_merge(model_names, calc_mode, TWO_MODEL_MODES, recipe_params, cls.MODEL_TYPE)
@@ -272,15 +351,19 @@ class ModelTwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("diffusion_models")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_model"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -290,20 +373,22 @@ class ModelTwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
         filename = MergerLogic.execute_merge(model_names, calc_mode, TWO_MODEL_MODES, recipe_params, cls.MODEL_TYPE)
@@ -325,15 +410,19 @@ class TextEncoderTwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("text_encoders")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_textencoder"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -343,20 +432,22 @@ class TextEncoderTwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
         filename = MergerLogic.execute_merge(model_names, calc_mode, TWO_MODEL_MODES, recipe_params, cls.MODEL_TYPE)
@@ -378,15 +469,19 @@ class LoRATwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("loras")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_lora"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -396,20 +491,22 @@ class LoRATwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
         filename = MergerLogic.execute_merge(model_names, calc_mode, TWO_MODEL_MODES, recipe_params, cls.MODEL_TYPE)
@@ -431,15 +528,19 @@ class EmbeddingTwoMerger(io.ComfyNode):
                 io.Combo.Input("model_b", options=["None"] + folder_paths.get_filename_list("embeddings")),
                 io.Combo.Input("calc_mode", options=[m.name for m in TWO_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.99, min=0.0, max=1.0, step=0.001),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_2_embedding"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -449,20 +550,22 @@ class EmbeddingTwoMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
-                gamma: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
+                gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_2_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
-            "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
+            "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b}
         filename = MergerLogic.execute_merge(model_names, calc_mode, TWO_MODEL_MODES, recipe_params, cls.MODEL_TYPE)
@@ -487,16 +590,19 @@ class CheckpointThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("checkpoints")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_checkpoint"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -506,20 +612,22 @@ class CheckpointThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}
         filename = MergerLogic.execute_merge(model_names, calc_mode, THREE_MODEL_MODES, recipe_params, cls.MODEL_TYPE)
@@ -542,16 +650,19 @@ class ModelThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("diffusion_models")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_model"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -561,20 +672,22 @@ class ModelThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}
         filename = MergerLogic.execute_merge(model_names, calc_mode, THREE_MODEL_MODES, recipe_params, cls.MODEL_TYPE)
@@ -597,16 +710,19 @@ class TextEncoderThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("text_encoders")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.99, min=-10.0, max=10.0, step=0.001),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_textencoder"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -616,20 +732,22 @@ class TextEncoderThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}
         filename = MergerLogic.execute_merge(model_names, calc_mode, THREE_MODEL_MODES, recipe_params, cls.MODEL_TYPE)
@@ -652,16 +770,19 @@ class LoRAThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("loras")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_lora"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -671,20 +792,22 @@ class LoRAThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}
         filename = MergerLogic.execute_merge(model_names, calc_mode, THREE_MODEL_MODES, recipe_params, cls.MODEL_TYPE)
@@ -707,16 +830,19 @@ class EmbeddingThreeMerger(io.ComfyNode):
                 io.Combo.Input("model_c", options=["None"] + folder_paths.get_filename_list("embeddings")),
                 io.Combo.Input("calc_mode", options=[m.name for m in THREE_MODEL_MODES]),
                 io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"),
-                io.Float.Input("alpha", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("beta", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("gamma", default=0.5, min=-2.0, max=3.0, step=0.01),
-                io.Float.Input("delta", default=0.5, min=-2.0, max=3.0, step=0.01),
+                io.Combo.Input("alignment_mode", options=["pad/crop", "interpolate"], default="pad/crop"),
+                io.Float.Input("alpha", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("beta", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("gamma", default=0.5, min=-10.0, max=10.0, step=0.01),
+                io.Float.Input("delta", default=0.5, min=-10.0, max=10.0, step=0.01),
                 io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
                 io.String.Input("output_filename", default="merged_3_embedding"),
                 io.Combo.Input("save_dtype", options=["fp32", "fp16", "bf16"]),
                 io.Combo.Input("process_device", options=["cuda", "cpu"]),
                 io.String.Input("exclude_patterns", default="", multiline=True),
                 io.String.Input("discard_patterns", default="", multiline=True),
+                io.Boolean.Input("lazy_load", default=True, tooltip="Low memory mode: load tensors from disk on demand"),
+                io.Boolean.Input("force_clear_cache", default=True, tooltip="Clear CUDA cache after each layer"),
             ],
             outputs=[
                 io.String.Output(display_name="output_filename"),
@@ -726,20 +852,22 @@ class EmbeddingThreeMerger(io.ComfyNode):
 
     @classmethod
     def execute(cls, execution_mode: str, model_a: str, model_b: str, model_c: str,
-                calc_mode: str, mismatch_mode: str, alpha: float, beta: float,
+                calc_mode: str, mismatch_mode: str, alignment_mode: str, alpha: float, beta: float,
                 gamma: float, delta: float, seed: int, output_filename: str, save_dtype: str,
-                process_device: str, exclude_patterns: str, discard_patterns: str) -> io.NodeOutput:
+                process_device: str, exclude_patterns: str, discard_patterns: str,
+                lazy_load: bool, force_clear_cache: bool) -> io.NodeOutput:
         doc = load_documentation_from_file('merger_3_model_modes.md')
         if execution_mode == "DOCUMENTATION ONLY":
             return io.NodeOutput("Documentation mode active. No merge performed.", doc)
 
         recipe_params = {
             "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode,
-            "mismatch_mode": mismatch_mode,
+            "mismatch_mode": mismatch_mode, "alignment_mode": alignment_mode,
             "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed,
             "output_filename": output_filename, "save_dtype": save_dtype,
             "device": process_device, "dtype": torch.float32,
             "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns,
+            "lazy_load": lazy_load, "force_clear_cache": force_clear_cache,
         }
         model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c}
         filename = MergerLogic.execute_merge(model_names, calc_mode, THREE_MODEL_MODES, recipe_params, cls.MODEL_TYPE)

nodes/merger_ops.py

@@ -4,6 +4,8 @@ import torch.nn.functional as F
 from enum import Enum
 
 
+from unifiedefficientloader import transfer_to_gpu_pinned
+
 class MissingTensorBehavior(Enum):
     """Controls behavior when a tensor key is missing from a model."""
     ERROR = "error"      # Raise exception (strict mode)
@@ -15,6 +17,92 @@ class MissingTensorError(Exception):
     """Raised when a tensor is missing and behavior is ERROR."""
     pass
 
+# ################# #
+# UTILITIES
+# ################# #
+def resize_by_interpolation(tensor1, tensor2):
+    """Resizes tensor2 to match tensor1's shape using interpolation."""
+    if tensor1 is None or tensor2 is None:
+        return tensor1, tensor2
+    if tensor1.shape == tensor2.shape:
+        return tensor1, tensor2
+
+    dims1 = len(tensor1.shape)
+    dims2 = len(tensor2.shape)
+
+    if dims1 != dims2 or dims1 > 4 or dims1 == 0:
+        return tensor1, tensor2
+
+    orig_dtype = tensor2.dtype
+    t2_f = tensor2.float()
+
+    # F.interpolate expects [B, C, ...]
+    # We treat the tensor as [1, 1, ...dims]
+    view_shape = [1, 1] + list(tensor2.shape)
+    t2_f = t2_f.view(view_shape)
+
+    # Select mode based on dimensions
+    if dims1 == 4:
+        # [Out, In, H, W] -> [1, 1, Out, In, H*W] (5D)
+        view_shape = [1, 1, tensor2.shape[0], tensor2.shape[1], tensor2.shape[2] * tensor2.shape[3]]
+        target_size = [tensor1.shape[0], tensor1.shape[1], tensor1.shape[2] * tensor1.shape[3]]
+        t2_f = t2_f.view(view_shape)
+        t2_res = F.interpolate(t2_f, size=target_size, mode='trilinear', align_corners=False)
+        return tensor1, t2_res.view(tensor1.shape).to(orig_dtype)
+
+    if dims1 == 1:
+        mode = 'linear'
+    elif dims1 == 2:
+        mode = 'bilinear'
+    else:
+        mode = 'trilinear'
+
+    t2_res = F.interpolate(t2_f, size=list(tensor1.shape), mode=mode, align_corners=False)
+
+    return tensor1, t2_res.view(tensor1.shape).to(orig_dtype)
+
+
+def resize_tensors(tensor1, tensor2, mode='pad/crop'):
+    """Resizes tensors to match shapes using either padding or interpolation."""
+    if tensor1 is None or tensor2 is None:
+        return tensor1, tensor2
+    if tensor1.shape == tensor2.shape:
+        return tensor1, tensor2
+
+    if mode == 'interpolate':
+        return resize_by_interpolation(tensor1, tensor2)
+
+    dims1 = len(tensor1.shape)
+    dims2 = len(tensor2.shape)
+
+    if dims1 != dims2 or dims1 > 4 or dims1 == 0:
+        return tensor1, tensor2
+
+    padding1 = []
+    padding2 = []
+
+    # F.pad expects: (last_dim_left, last_dim_right, prev_dim_top, prev_dim_bottom, ...)
+    for i in range(dims1 - 1, -1, -1):
+        s1 = tensor1.shape[i]
+        s2 = tensor2.shape[i]
+        if s1 < s2:
+            padding1.extend([0, s2 - s1])
+            padding2.extend([0, 0])
+        elif s2 < s1:
+            padding1.extend([0, 0])
+            padding2.extend([0, s1 - s2])
+        else:
+            padding1.extend([0, 0])
+            padding2.extend([0, 0])
+
+    if any(p > 0 for p in padding1):
+        tensor1 = F.pad(tensor1, tuple(padding1))
+    if any(p > 0 for p in padding2):
+        tensor2 = F.pad(tensor2, tuple(padding2))
+
+    return tensor1, tensor2
+
+
 # ################# #
 # MERGE OPERATORS
 # ################# #
@@ -23,6 +111,7 @@ class Operation:
         self.key = key
         self.sources = tuple(sources)
         self.merge_func = self.recurse
+        self.alignment_mode = 'pad/crop'
         self.alpha = None
         self.beta = None
         self.gamma = None
@@ -40,12 +129,21 @@ class Operation:
         raise NotImplementedError
 
     def recurse(self, operation):
-        source_tensors = [source_oper.merge() for source_oper in operation.sources]
-        return operation.oper(*source_tensors)
+        self._source_tensors = [source_oper.merge() for source_oper in operation.sources]
+        return operation.oper(*self._source_tensors)
 
     def merge(self):
         return self.merge_func(self)
 
+    def clean(self):
+        if hasattr(self, '_source_tensors'):
+            for t in self._source_tensors:
+                del t
+            del self._source_tensors
+        for source in self.sources:
+            if hasattr(source, 'clean'):
+                source.clean()
+
 class LoadTensor(Operation):
     def __init__(self, key, model_name, handlers, device, dtype,
                  on_missing=MissingTensorBehavior.ERROR, fallback_shape=None, fallback_dtype=None):
@@ -78,7 +176,17 @@ class LoadTensor(Operation):
                 dtype = self.fallback_dtype if self.fallback_dtype else self.dtype
                 return torch.zeros(self.fallback_shape, device=self.device, dtype=dtype)
 
-        return handler.get_tensor(self.key).to(device=self.device, dtype=self.dtype)
+        cpu_tensor = handler.get_tensor(self.key)
+        if self.device == 'cuda':
+            self._tensor = transfer_to_gpu_pinned(cpu_tensor, self.device, self.dtype)
+        else:
+            self._tensor = cpu_tensor.to(device=self.device, dtype=self.dtype)
+        del cpu_tensor
+        return self._tensor
+
+    def clean(self):
+        if hasattr(self, '_tensor'):
+            del self._tensor
 
 class Multiply(Operation):
     def __init__(self, key, alpha, *sources):
@@ -96,6 +204,7 @@ class Add(Operation):
             return b
         if b is None:
             return a
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
         return a + b
 
 
@@ -103,15 +212,33 @@ class Sub(Operation):
     def oper(self, a, b) -> torch.Tensor:
         if a is None or b is None:
             return None  # Cannot compute difference with missing operand
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
         return a - b
 
 class SVD(Operation):
-    def __init__(self, key, alpha, beta, gamma, *sources):
+    def __init__(self, key, alpha, beta, gamma, *sources, target_shape=None):
         super().__init__(key, *sources)
         self.alpha, self.beta, self.gamma = int(alpha), int(beta), gamma
+        self.target_shape = target_shape
+
     def oper(self, a, b) -> torch.Tensor:
         if a is None or b is None:
             return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+
+        # For SVD extraction, we must ensure the diff matches the original Model A shape
+        # so the resulting LoRA weights are compatible with the target architecture.
+        if self.target_shape and a.shape != self.target_shape:
+            if self.alignment_mode == 'interpolate':
+                # Re-align both to match target_shape via interpolation
+                dummy = torch.empty(self.target_shape, device=a.device)
+                _, a = resize_by_interpolation(dummy, a)
+                _, b = resize_by_interpolation(dummy, b)
+            else:
+                slices = tuple(slice(0, min(res_s, tgt_s)) for res_s, tgt_s in zip(a.shape, self.target_shape))
+                a = a[slices]
+                b = b[slices]
+
         diff, weights, conv2d = a - b, {}, len(a.size()) == 4
         kernel_size = None if not conv2d else a.size()[2:4]
         conv2d_3x3 = conv2d and kernel_size != (1, 1)
@@ -141,6 +268,10 @@ class TrainDiff(Operation):
     def oper(self, a, b, c) -> torch.Tensor:
         if a is None or b is None or c is None:
             return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+        a, c = resize_tensors(a, c, mode=self.alignment_mode)
+        b, c = resize_tensors(b, c, mode=self.alignment_mode)
+
         if torch.allclose(b.float(), c.float()): return torch.zeros_like(a)
         diff_bc = b.float() - c.float()
         dist_bc, dist_ab = torch.abs(diff_bc), torch.abs(b.float() - a.float())
@@ -153,11 +284,17 @@ class ExtractOp(Operation):
     def oper(self, base: torch.Tensor, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
         if a is None or b is None:
             return None
+        if base is not None:
+            base, a = resize_tensors(base, a, mode=self.alignment_mode)
+            base, b = resize_tensors(base, b, mode=self.alignment_mode)
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+
+        dtype = base.dtype if base is not None else a.dtype
         base_f = base.float() if base is not None else 0
         a_f, b_f = a.float() - base_f, b.float() - base_f
-        c = torch.cosine_similarity(a_f.flatten(), b_f.flatten(), 0).clamp(-1, 1)
+        c = torch.cosine_similarity(a_f, b_f, -1).clamp(-1, 1).unsqueeze(-1)
         d = ((c + 1) / 2)**self.gamma
-        return (torch.lerp(a_f, b_f, self.alpha) * torch.lerp(d, 1 - d, self.beta)).to(a.dtype)
+        return (torch.lerp(a_f, b_f, self.alpha) * torch.lerp(d, 1 - d, self.beta)).to(dtype)
 
 
 class Similarities(ExtractOp):
@@ -172,10 +309,61 @@ class PowerUpOp(Operation):
     def oper(self, a, b):
         if a is None or b is None:
             return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
         delta = b - a
+        keep_prob = 1 - self.alpha
+        if keep_prob <= 0:
+            return torch.zeros_like(delta)
         rng = torch.Generator(device=delta.device).manual_seed(self.seed)
-        m = torch.bernoulli(torch.full_like(delta, 1 - self.alpha), generator=rng)
-        return (m * delta) / (1 - self.alpha) if (1 - self.alpha) > 0 else torch.zeros_like(delta)
+        m = (torch.empty_like(delta).uniform_(0, 1, generator=rng) < keep_prob).to(delta.dtype)
+        return (m * delta) / keep_prob
+
+
+class DARETIESOp(Operation):
+    """DARE + TIES combined operation on a single task vector (delta = b - a).
+
+    Step 1 — DARE: randomly drop `drop_rate` fraction of delta parameters,
+                   rescale survivors by 1/(1-drop_rate).
+    Step 2 — TIES trim: zero out parameters below the `trim_quantile` magnitude threshold.
+    Step 3 — TIES elect: determine dominant sign per parameter position.
+    Step 4 — TIES merge: zero out parameters that disagree with the dominant sign.
+
+    For a two-model merge the TIES elect/merge step degenerates to a no-op (single vector,
+    sign is always self-consistent) but the trim step still reduces interference from
+    near-zero noisy deltas. The dominant sign filter is included for correctness and
+    forward compatibility with n-model extensions.
+    """
+    def __init__(self, key, alpha, beta, gamma, seed, *sources):
+        super().__init__(key, *sources)
+        # alpha = drop_rate (DARE), beta = trim_quantile (TIES), gamma = lambda scale
+        self.alpha, self.beta, self.gamma, self.seed = alpha, beta, gamma, seed
+
+    def oper(self, a, b):
+        if a is None or b is None:
+            return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+        delta = (b - a).float()
+
+        # --- DARE: drop and rescale ---
+        keep_prob = 1.0 - self.alpha
+        if keep_prob <= 0:
+            return torch.zeros_like(a)
+        rng = torch.Generator(device=delta.device).manual_seed(self.seed)
+        dare_mask = (torch.empty_like(delta).uniform_(0, 1, generator=rng) < keep_prob).float()
+        delta = (delta * dare_mask) / keep_prob
+
+        # --- TIES trim: zero parameters below magnitude threshold ---
+        if self.beta > 0:
+            flat_abs = delta.abs().flatten()
+            k = max(1, int(flat_abs.numel() * self.beta))
+            threshold = torch.kthvalue(flat_abs, k).values
+            delta = torch.where(delta.abs() < threshold, torch.zeros_like(delta), delta)
+
+        # --- TIES elect: dominant sign (trivially self-consistent for single vector) ---
+        dominant_sign = torch.sign(delta)
+        delta = torch.where(torch.sign(delta) == dominant_sign, delta, torch.zeros_like(delta))
+
+        return (a + self.gamma * delta).to(a.dtype)
 
 
 class InterpolateDifference(Operation):
@@ -185,6 +373,7 @@ class InterpolateDifference(Operation):
     def oper(self, a, b):
         if a is None or b is None:
             return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
         alpha, delta = max(self.alpha, 0.001), torch.abs(a - b)
         max_delta = torch.max(delta)
         if max_delta == 0: return a
@@ -194,6 +383,65 @@ class InterpolateDifference(Operation):
         mask = torch.lerp(torch.bernoulli(torch.clamp(diff, 0, 1), generator=rng), diff, self.gamma)
         return a * (1 - mask) + b * mask
 
+class ManualEnhancedInterpolateDifferenceOp(Operation):
+    def __init__(self, key, alpha, beta, gamma, delta, seed, *sources):
+        super().__init__(key, *sources)
+        self.alpha, self.beta, self.gamma, self.delta, self.seed = alpha, beta, gamma, delta, seed
+
+    def oper(self, a, b):
+        if a is None or b is None: return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+        delta = torch.abs(a - b)
+        diff = torch.nan_to_num((torch.max(delta) - delta) / torch.max(delta))
+        mean_diff = torch.mean(diff, 0, keepdim=True)
+        mask = torch.logical_and(self.beta < mean_diff, mean_diff < self.gamma)
+        powered_diff = torch.nan_to_num(diff ** (1 / max(self.alpha, 0.001) - 1))
+        masked_diff = powered_diff * mask.float()
+        rng = torch.Generator(device=a.device).manual_seed(self.seed)
+        random_mask = torch.bernoulli(torch.clamp(masked_diff, 0, 1), generator=rng)
+        interpolated_mask = torch.lerp(random_mask, masked_diff, self.delta)
+        return a * (1 - interpolated_mask) + b * interpolated_mask
+
+
+class AutoEnhancedInterpolateDifferenceOp(Operation):
+    def __init__(self, key, alpha, beta, gamma, seed, *sources):
+        super().__init__(key, *sources)
+        self.alpha, self.beta, self.gamma, self.seed = alpha, beta, gamma, seed
+
+    def oper(self, a, b):
+        if a is None or b is None: return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+        delta = torch.abs(a - b)
+        max_delta = torch.max(delta)
+        diff = torch.nan_to_num((max_delta - delta) / max_delta)
+        mean_diff = torch.mean(diff)
+        lower_threshold = mean_diff * (1 - self.beta)
+        upper_threshold = mean_diff * (1 + self.beta)
+        mask = torch.logical_and(lower_threshold < diff, diff < upper_threshold)
+        powered_diff = torch.nan_to_num(diff ** (1 / max(self.alpha, 0.001) - 1))
+        masked_diff = powered_diff * mask.float()
+        rng = torch.Generator(device=a.device).manual_seed(self.seed)
+        random_mask = torch.bernoulli(torch.clamp(masked_diff, 0, 1), generator=rng)
+        interpolated_mask = torch.lerp(random_mask, masked_diff, self.gamma)
+        return a * (1 - interpolated_mask) + b * interpolated_mask
+
+
+class WeightSumCutoffOp(Operation):
+    def __init__(self, key, alpha, beta, gamma, *sources):
+        super().__init__(key, *sources)
+        self.alpha, self.beta, self.gamma = alpha, beta, gamma
+
+    def oper(self, a, b):
+        if a is None or b is None: return None
+        a, b = resize_tensors(a, b, mode=self.alignment_mode)
+        delta = torch.abs(a - b)
+        diff = torch.nan_to_num((torch.max(delta) - delta) / torch.max(delta))
+        mean = torch.mean(diff, 0, True)
+        mask = torch.logical_and(mean < self.beta, self.gamma < mean)
+        mul = self.alpha * mask
+        return a * (1 - mul) + b * mul
+
+
 # ################# #
 # CALCULATION MODES
 # ################# #
@@ -229,6 +477,10 @@ class PreloadedTensor(Operation):
     def merge(self) -> torch.Tensor:
         return self.tensor
 
+    def clean(self):
+        if hasattr(self, 'tensor'):
+            del self.tensor
+
 class WeightSum(CalcMode):
     name = 'Weight-Sum'
     description = 'A * (1 - α) + B * α. Simple linear interpolation.'
@@ -236,6 +488,7 @@ class WeightSum(CalcMode):
     param_docs = {'alpha': 'Interpolation weight. 0.0 is 100% Model A, 1.0 is 100% Model B.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         # Use pre-loaded tensor_a if available
         if '_tensor_a' in kwargs:
@@ -251,7 +504,9 @@ class WeightSum(CalcMode):
         if kwargs['alpha'] <= 0: return a
         c = Multiply(key, 1 - kwargs['alpha'], a)
         d = Multiply(key, kwargs['alpha'], b)
-        return Add(key, c, d)
+        res = Add(key, c, d)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class AddDifference(CalcMode):
@@ -261,6 +516,7 @@ class AddDifference(CalcMode):
     param_docs = {'alpha': 'Multiplier for the (B - C) difference.', 'beta': 'Smoothing (0=Off, 1=On).'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -272,9 +528,12 @@ class AddDifference(CalcMode):
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         diff = Sub(key, b, c)
+        diff.alignment_mode = alignment_mode
         if kwargs['beta'] == 1: diff = Smooth(key, diff)
         diffm = Multiply(key, kwargs['alpha'], diff)
-        return Add(key, a, diffm)
+        res = Add(key, a, diffm)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class TrainDifference(CalcMode):
@@ -284,6 +543,7 @@ class TrainDifference(CalcMode):
     param_docs = {'alpha': 'Multiplier for the calculated difference.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -295,8 +555,11 @@ class TrainDifference(CalcMode):
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         diff = TrainDiff(key, a, b, c)
+        diff.alignment_mode = alignment_mode
         diffm = Multiply(key, kwargs['alpha'], diff)
-        return Add(key, a, diffm)
+        res = Add(key, a, diffm)
+        res.alignment_mode = alignment_mode
+        return res
 class InterpDifference(CalcMode):
     name = 'Comparative-Interpolation'
     description = 'Interpolates A and B based on relative tensor value differences.'
@@ -304,6 +567,7 @@ class InterpDifference(CalcMode):
     param_docs = {'alpha': 'Curve strength.', 'beta': 'Style (0=Similarity, 1=Difference).', 'gamma': 'Mix (0=Random, 1=Linear).'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -317,7 +581,9 @@ class InterpDifference(CalcMode):
 
         loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
-        return InterpolateDifference(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['seed'], a, b)
+        res = InterpolateDifference(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['seed'], a, b)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class Extract(CalcMode):
@@ -327,6 +593,7 @@ class Extract(CalcMode):
     param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Similarity vs dissimilarity.', 'gamma': 'Similarity bias.', 'delta': 'Multiplier for final features.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -338,8 +605,11 @@ class Extract(CalcMode):
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         extracted = ExtractOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'] * 15, a, b, c)
+        extracted.alignment_mode = alignment_mode
         multiplied = Multiply(key, kwargs['delta'], extracted)
-        return Add(key, a, multiplied)
+        res = Add(key, a, multiplied)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class AddDisimilarity(CalcMode):
@@ -349,6 +619,7 @@ class AddDisimilarity(CalcMode):
     param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Multiplier for final features.', 'gamma': 'Similarity bias.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -360,8 +631,11 @@ class AddDisimilarity(CalcMode):
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         extracted = Similarities(key, kwargs['alpha'], 1, kwargs['gamma'] * 15, b, c)
+        extracted.alignment_mode = alignment_mode
         multiplied = Multiply(key, kwargs['beta'], extracted)
-        return Add(key, a, multiplied)
+        res = Add(key, a, multiplied)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class PowerUp(CalcMode):
@@ -371,6 +645,7 @@ class PowerUp(CalcMode):
     param_docs = {'alpha': 'Dropout rate (proportion of weights from B to *drop*).', 'beta': 'Multiplier for the final difference.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -381,8 +656,11 @@ class PowerUp(CalcMode):
         loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
         deltahat = PowerUpOp(key, kwargs['alpha'], kwargs['seed'], a, b)
-        res = Multiply(key, kwargs['beta'], deltahat)
-        return Add(key, a, res)
+        deltahat.alignment_mode = alignment_mode # Important!
+        multiplied = Multiply(key, kwargs['beta'], deltahat)
+        res = Add(key, a, multiplied)
+        res.alignment_mode = alignment_mode
+        return res
 
 
 class SVDMode(CalcMode):
@@ -392,6 +670,7 @@ class SVDMode(CalcMode):
     param_docs = {'alpha': 'Rank for standard layers.', 'beta': 'Rank for 3x3 conv layers.', 'gamma': 'Clamp quantile for weights.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
 
         if '_tensor_a' in kwargs:
             a = PreloadedTensor(key, kwargs['_tensor_a'])
@@ -401,8 +680,118 @@ class SVDMode(CalcMode):
 
         loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
-        return SVD(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], a, b)
+        res = SVD(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], a, b, target_shape=kwargs.get('_tensor_a_shape'))
+        res.alignment_mode = alignment_mode
+        return res
 
 
-TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), SVDMode()]
+class ManEnhInterpDifference(CalcMode):
+    name = 'Enhanced Man Interp'
+    description = 'Enhanced interpolation between each pair of values from A and B depending on their difference relative to other values.'
+    models_used = ['A', 'B']
+    param_docs = {'alpha': 'Interpolation strength.', 'beta': 'Lower mean threshold.', 'gamma': 'Upper mean threshold.', 'delta': 'Smoothness factor.'}
+    def create_recipe(self, key, **kwargs):
+        mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
+
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+
+        if key.startswith('cond_stage_model.transformer.text_model.embeddings'):
+            return a
+
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
+        b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
+        res = ManualEnhancedInterpolateDifferenceOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['delta'], kwargs['seed'], a, b)
+        res.alignment_mode = alignment_mode
+        return res
+
+
+class AutoEnhInterpDifference(CalcMode):
+    name = 'Enhanced Auto Interp'
+    description = 'Interpolates between each pair of values from A and B depending on their difference relative to other values.'
+    models_used = ['A', 'B']
+    param_docs = {'alpha': 'Interpolation strength.', 'beta': 'Threshold adjustment factor.', 'gamma': 'Smoothness factor.'}
+    def create_recipe(self, key, **kwargs):
+        mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
+
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+
+        if key.startswith('cond_stage_model.transformer.text_model.embeddings'):
+            return a
+
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
+        b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
+        res = AutoEnhancedInterpolateDifferenceOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['seed'], a, b)
+        res.alignment_mode = alignment_mode
+        return res
+
+
+class WeightSumCutoffMode(CalcMode):
+    name = 'Weight-Sum Cutoff'
+    description = 'Weight-sum with cutoff based on value differences.'
+    models_used = ['A', 'B']
+    param_docs = {'alpha': 'Interpolation weight.', 'beta': 'Upper threshold.', 'gamma': 'Lower threshold.'}
+    def create_recipe(self, key, **kwargs):
+        mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
+
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
+        b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
+        res = WeightSumCutoffOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], a, b)
+        res.alignment_mode = alignment_mode
+        return res
+
+
+class DARETIESMode(CalcMode):
+    name = 'Power-Up (DARE+TIES)'
+    description = 'Adds capabilities of B to A using DARE (Drop and Rescale) followed by TIES (Trim, Elect Sign). Best practice for LoRA-tuned model merging.'
+    models_used = ['A', 'B']
+    param_docs = {
+        'alpha': 'DARE drop rate — fraction of delta parameters randomly zeroed. Higher = more aggressive sparsification.',
+        'beta':  'TIES trim quantile — fraction of smallest-magnitude delta parameters zeroed after DARE. Removes residual noise.',
+        'gamma': 'Lambda — scale multiplier applied to the final merged delta before adding to Model A.',
+        'seed':  'Random seed for the DARE dropout mask.',
+    }
+
+    def create_recipe(self, key, **kwargs):
+        mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
+        alignment_mode = kwargs.get('alignment_mode', 'pad/crop')
+
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
+        b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
+
+        op = DARETIESOp(
+            key,
+            kwargs['alpha'],   # drop_rate
+            kwargs['beta'],    # trim_quantile
+            kwargs['gamma'],   # lambda scale
+            kwargs['seed'],
+            a, b,
+        )
+        op.alignment_mode = alignment_mode
+        return op
+
+
+TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), DARETIESMode(), SVDMode(), ManEnhInterpDifference(), AutoEnhInterpDifference(), WeightSumCutoffMode()]
 THREE_MODEL_MODES = [AddDifference(), TrainDifference(), Extract(), AddDisimilarity()]

nodes/merger_utils.py

@@ -1,358 +1,7 @@
 """
-Memory-efficient safetensors utilities with parallel I/O and pinned memory support.
-
-Based on benchmarking, parallel reading with ThreadPoolExecutor provides 2-4x speedup
-for large models compared to sequential reads.
-
-Worker count is automatically optimized based on device capabilities when workers=None.
-Pinned memory enables faster CPU→GPU transfers (2-3x improvement).
+Utilities for model merging operations.
 """
-import os
-import mmap
-import json
-import struct
-import torch
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Dict, List, Optional, Any, Tuple
-
-# Import ComfyUI's memory pinning functions if available
-try:
-    from comfy.model_management import pin_memory, unpin_memory, is_device_cuda
-    PINNING_AVAILABLE = True
-except ImportError:
-    PINNING_AVAILABLE = False
-    def pin_memory(tensor): return False
-    def unpin_memory(tensor): return False
-    def is_device_cuda(device): return str(device).startswith('cuda')
-
-
-class PinnedTensor:
-    """Context manager for automatic pinned memory management.
-    
-    Usage:
-        with PinnedTensor(cpu_tensor) as pinned:
-            gpu_tensor = pinned.to('cuda')
-        # Memory automatically unpinned after transfer
-    """
-    
-    def __init__(self, tensor: torch.Tensor, auto_pin: bool = True):
-        self.tensor = tensor
-        self.is_pinned = False
-        self.auto_pin = auto_pin and PINNING_AVAILABLE
-        
-    def __enter__(self) -> torch.Tensor:
-        if self.auto_pin and self.tensor.device.type == 'cpu':
-            self.is_pinned = pin_memory(self.tensor)
-        return self.tensor
-    
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self.is_pinned:
-            unpin_memory(self.tensor)
-            self.is_pinned = False
-
-
-def transfer_to_gpu_pinned(
-    tensor: torch.Tensor,
-    device: str = 'cuda',
-    dtype: Optional[torch.dtype] = None
-) -> torch.Tensor:
-    """Transfer tensor to GPU using pinned memory for faster transfer.
-    
-    Args:
-        tensor: CPU tensor to transfer
-        device: Target GPU device
-        dtype: Optional dtype conversion
-        
-    Returns:
-        Tensor on GPU device
-    """
-    if not PINNING_AVAILABLE or not is_device_cuda(torch.device(device)):
-        # Fall back to regular transfer
-        if dtype:
-            return tensor.to(device=device, dtype=dtype)
-        return tensor.to(device=device)
-    
-    # Pin memory for faster transfer
-    was_pinned = pin_memory(tensor)
-    try:
-        if dtype:
-            result = tensor.to(device=device, dtype=dtype, non_blocking=True)
-        else:
-            result = tensor.to(device=device, non_blocking=True)
-        # Sync to ensure transfer is complete before unpinning
-        if was_pinned:
-            torch.cuda.current_stream().synchronize()
-        return result
-    finally:
-        if was_pinned:
-            unpin_memory(tensor)
-
-
-class MemoryEfficientSafeOpen:
-    """Memory-efficient safetensors file reader with mmap, parallel I/O, and pinned memory support.
-
-    Features:
-    - mmap mode: Zero-copy tensor access via memory-mapped file
-    - Parallel loading: Multi-threaded tensor reads for 2-4x speedup
-    - Sorted batch reads: Keys sorted by file offset for sequential I/O
-    - Auto-optimized workers: Adjusts parallelism based on device capabilities
-    - Pinned memory: Faster CPU→GPU transfers when transferring to CUDA
-
-    Args:
-        filename: Path to safetensors file
-        device: Target device (default 'cpu')
-        mmap_mode: Use memory-mapped file for zero-copy (default True)
-    """
-
-    def __init__(self, filename: str, device: str = 'cpu', mmap_mode: bool = True):
-        self.filename = filename
-        self.device = device
-        self.mmap_mode = mmap_mode
-        self.header, self.header_size = self._read_header()
-        self.file = open(filename, "rb")
-        self.mmap_obj = None
-
-        if self.mmap_mode:
-            self.mmap_obj = mmap.mmap(self.file.fileno(), 0, access=mmap.ACCESS_READ)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self.mmap_obj:
-            self.mmap_obj.close()
-        self.file.close()
-
-    def keys(self) -> List[str]:
-        """Return all tensor keys (excluding metadata)."""
-        return [k for k in self.header.keys() if k != "__metadata__"]
-
-    def metadata(self) -> Dict[str, str]:
-        """Return file metadata."""
-        return self.header.get("__metadata__", {})
-
-    def keys_sorted_by_offset(self) -> List[str]:
-        """Return keys sorted by file offset for optimal sequential I/O."""
-        keys_with_offsets = []
-        for key in self.keys():
-            offset = self.header[key]["data_offsets"][0]
-            keys_with_offsets.append((key, offset))
-        keys_with_offsets.sort(key=lambda x: x[1])
-        return [k for k, _ in keys_with_offsets]
-
-    def get_tensor(self, key: str) -> torch.Tensor:
-        """Load a single tensor by key."""
-        if key not in self.header:
-            raise KeyError(f"Tensor '{key}' not found in the file")
-
-        metadata = self.header[key]
-        offset_start, offset_end = metadata["data_offsets"]
-
-        if self.mmap_mode and self.mmap_obj:
-            if offset_start != offset_end:
-                start = self.header_size + 8 + offset_start
-                end = self.header_size + 8 + offset_end
-                tensor_bytes = memoryview(self.mmap_obj)[start:end]
-            else:
-                tensor_bytes = None
-        else:
-            tensor_bytes = None
-            if offset_start != offset_end:
-                self.file.seek(self.header_size + 8 + offset_start)
-                tensor_bytes = self.file.read(offset_end - offset_start)
-
-        return self._deserialize_tensor(tensor_bytes, metadata)
-    
-    def get_tensor_to_gpu(
-        self,
-        key: str,
-        device: str = 'cuda',
-        dtype: Optional[torch.dtype] = None
-    ) -> torch.Tensor:
-        """Load a tensor and transfer to GPU using pinned memory for faster transfer.
-        
-        Args:
-            key: Tensor key to load
-            device: Target GPU device (default 'cuda')
-            dtype: Optional dtype conversion
-            
-        Returns:
-            Tensor on GPU device
-        """
-        cpu_tensor = self.get_tensor(key)
-        return transfer_to_gpu_pinned(cpu_tensor, device, dtype)
-
-    def get_tensors_parallel(
-        self,
-        keys: Optional[List[str]] = None,
-        workers: Optional[int] = None,
-        sort_by_offset: bool = True
-    ) -> Dict[str, torch.Tensor]:
-        """Load multiple tensors in parallel using ThreadPoolExecutor.
-
-        Args:
-            keys: List of keys to load (default: all keys)
-            workers: Number of worker threads (default: auto-calculated based on device)
-            sort_by_offset: Sort keys by file offset before loading (default: True)
-
-        Returns:
-            Dict mapping key -> tensor
-        """
-        if keys is None:
-            keys = self.keys()
-
-        # Auto-calculate workers based on device capabilities
-        if workers is None:
-            try:
-                from .device_utils import get_optimal_workers, estimate_model_size
-                model_size = estimate_model_size(self.filename)
-                workers = get_optimal_workers(model_size)
-            except ImportError:
-                workers = min(4, os.cpu_count() or 4)
-
-        if sort_by_offset:
-            # Sort by offset for better I/O patterns
-            keys_with_offsets = [(k, self.header[k]["data_offsets"][0]) for k in keys]
-            keys_with_offsets.sort(key=lambda x: x[1])
-            keys = [k for k, _ in keys_with_offsets]
-
-        result = {}
-
-        with ThreadPoolExecutor(max_workers=workers) as executor:
-            futures = {
-                executor.submit(
-                    _read_tensor_from_file,
-                    self.filename,
-                    key,
-                    self.header[key],
-                    self.header_size
-                ): key
-                for key in keys
-            }
-
-            for future in as_completed(futures):
-                key = futures[future]
-                try:
-                    tensor = future.result()
-                    if tensor is not None:
-                        result[key] = tensor
-                except Exception as e:
-                    print(f"[MemoryEfficientSafeOpen] Failed to load {key}: {e}")
-
-        return result
-
-    def get_tensor_as_dict(self, key: str) -> Dict[str, Any]:
-        """Load a tensor and decode as JSON dict (for config tensors)."""
-        if key not in self.header:
-            raise KeyError(f"Tensor '{key}' not found")
-
-        metadata = self.header[key]
-        offset_start, offset_end = metadata["data_offsets"]
-
-        if offset_start == offset_end:
-            return {}
-
-        if self.mmap_mode and self.mmap_obj:
-            start = self.header_size + 8 + offset_start
-            end = self.header_size + 8 + offset_end
-            tensor_bytes = bytes(self.mmap_obj[start:end])
-        else:
-            self.file.seek(self.header_size + 8 + offset_start)
-            tensor_bytes = self.file.read(offset_end - offset_start)
-
-        return json.loads(tensor_bytes.decode("utf-8"))
-
-    def _read_header(self):
-        with open(self.filename, "rb") as f:
-            header_size = struct.unpack("<Q", f.read(8))[0]
-            header_json = f.read(header_size).decode("utf-8")
-        return json.loads(header_json), header_size
-
-    def _deserialize_tensor(self, tensor_bytes, metadata):
-        dtype_str = metadata["dtype"]
-        shape = metadata["shape"]
-        dtype = self._get_torch_dtype(dtype_str)
-
-        if tensor_bytes is None:
-            byte_tensor = torch.empty(0, dtype=torch.uint8)
-        else:
-            byte_tensor = torch.frombuffer(bytearray(tensor_bytes), dtype=torch.uint8)
-
-        if dtype_str in ["F8_E5M2", "F8_E4M3"]:
-            return self._convert_float8(byte_tensor, dtype_str, shape)
-
-        return byte_tensor.view(dtype).reshape(shape)
-
-    @staticmethod
-    def _get_torch_dtype(dtype_str):
-        dtype_map = {
-            "F64": torch.float64, "F32": torch.float32, "F16": torch.float16, "BF16": torch.bfloat16,
-            "I64": torch.int64, "I32": torch.int32, "I16": torch.int16, "I8": torch.int8,
-            "U8": torch.uint8, "BOOL": torch.bool,
-        }
-        if hasattr(torch, "float8_e5m2"):
-            dtype_map["F8_E5M2"] = torch.float8_e5m2
-        if hasattr(torch, "float8_e4m3fn"):
-            dtype_map["F8_E4M3"] = torch.float8_e4m3fn
-
-        dtype = dtype_map.get(dtype_str)
-        if dtype is None:
-            raise ValueError(f"Unsupported dtype: {dtype_str}")
-        return dtype
-
-    @staticmethod
-    def _convert_float8(byte_tensor, dtype_str, shape):
-        if dtype_str == "F8_E5M2" and hasattr(torch, "float8_e5m2"):
-            return byte_tensor.view(torch.float8_e5m2).reshape(shape)
-        elif dtype_str == "F8_E4M3" and hasattr(torch, "float8_e4m3fn"):
-            return byte_tensor.view(torch.float8_e4m3fn).reshape(shape)
-        else:
-            raise ValueError(f"Unsupported float8 type: {dtype_str}. Your PyTorch version may be too old.")
-
-
-def _read_tensor_from_file(
-    filepath: str,
-    key: str,
-    metadata: Dict,
-    header_size: int
-) -> Optional[torch.Tensor]:
-    """Helper function to read a single tensor from file (for parallel execution).
-
-    Each worker opens its own file handle for thread safety.
-    """
-    offset_start, offset_end = metadata["data_offsets"]
-    dtype_str = metadata["dtype"]
-    shape = metadata["shape"]
-
-    if offset_start == offset_end:
-        return None
-
-    dtype_map = {
-        "F64": torch.float64, "F32": torch.float32, "F16": torch.float16, "BF16": torch.bfloat16,
-        "I64": torch.int64, "I32": torch.int32, "I16": torch.int16, "I8": torch.int8,
-        "U8": torch.uint8, "BOOL": torch.bool,
-    }
-    if hasattr(torch, "float8_e5m2"):
-        dtype_map["F8_E5M2"] = torch.float8_e5m2
-    if hasattr(torch, "float8_e4m3fn"):
-        dtype_map["F8_E4M3"] = torch.float8_e4m3fn
-
-    dtype = dtype_map.get(dtype_str, torch.float32)
-
-    with open(filepath, "rb") as f:
-        f.seek(header_size + 8 + offset_start)
-        tensor_bytes = f.read(offset_end - offset_start)
-
-    byte_tensor = torch.frombuffer(bytearray(tensor_bytes), dtype=torch.uint8)
-
-    if dtype_str in ["F8_E5M2", "F8_E4M3"]:
-        if dtype_str == "F8_E5M2" and hasattr(torch, "float8_e5m2"):
-            return byte_tensor.view(torch.float8_e5m2).reshape(shape)
-        elif dtype_str == "F8_E4M3" and hasattr(torch, "float8_e4m3fn"):
-            return byte_tensor.view(torch.float8_e4m3fn).reshape(shape)
-
-    return byte_tensor.view(dtype).reshape(shape)
-
+from typing import Dict, List
 
 def get_union_keys(*handlers) -> List[str]:
     """Returns the union of all tensor keys across multiple model handlers."""
@@ -371,4 +20,4 @@ def get_intersection_keys(*handlers) -> List[str]:
     all_keys = set(handlers[0].keys())
     for handler in handlers[1:]:
         all_keys &= set(handler.keys())
-    return sorted(all_keys)
+    return sorted(all_keys)

nodes/model_info_nodes.py

@@ -0,0 +1,184 @@
+import os
+import folder_paths
+from comfy_api.latest import io
+from .downloader_utils import get_potential_preview_files, load_image_tensor, get_model_workflows, get_model_metadata_file
+
+class BaseModelInfoLoader(io.ComfyNode):
+    CATEGORY = "ModelUtils/Info"
+
+    @classmethod
+    def get_info(cls, category, name, workflow_index):
+        full_path = folder_paths.get_full_path(category, name)
+        if not full_path or not os.path.exists(full_path):
+            return (None, "{}", "{}")
+
+        base_path = os.path.splitext(full_path)[0]
+
+        # 1. Image Preview
+        image_tensor = None
+        previews = get_potential_preview_files(base_path)
+        for p in previews:
+            if os.path.exists(p) and not p.lower().endswith('.mp4'):
+                image_tensor = load_image_tensor(p)
+                if image_tensor is not None:
+                    break
+
+        # 2. Workflow
+        workflow_json = "{}"
+        workflows = get_model_workflows(base_path)
+        if workflows and 0 <= workflow_index < len(workflows):
+            try:
+                with open(workflows[workflow_index], 'r', encoding='utf-8') as f:
+                    workflow_json = f.read()
+            except Exception as e:
+                print(f"Error reading workflow {workflows[workflow_index]}: {e}")
+
+        # 3. Metadata
+        metadata_json = "{}"
+        metadata_file = get_model_metadata_file(base_path)
+        if metadata_file:
+            try:
+                with open(metadata_file, 'r', encoding='utf-8') as f:
+                    metadata_json = f.read()
+            except Exception as e:
+                print(f"Error reading metadata {metadata_file}: {e}")
+
+        return (image_tensor, workflow_json, metadata_json)
+
+class CheckpointInfoLoader(BaseModelInfoLoader):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="Checkpoint Info Loader",
+            category=cls.CATEGORY,
+            inputs=[
+                io.Combo.Input("checkpoint", options=folder_paths.get_filename_list("checkpoints")),
+                io.Int.Input("workflow_index", default=0, min=0, max=100)
+            ],
+            outputs=[
+                io.Image.Output(display_name="preview"),
+                io.String.Output(display_name="workflow_json"),
+                io.String.Output(display_name="metadata_json")
+            ]
+        )
+
+    @classmethod
+    def execute(cls, checkpoint, workflow_index):
+        res = cls.get_info("checkpoints", checkpoint, workflow_index)
+        return io.NodeOutput(res[0], res[1], res[2])
+
+class LoRAInfoLoader(BaseModelInfoLoader):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="LoRA Info Loader",
+            category=cls.CATEGORY,
+            inputs=[
+                io.Combo.Input("lora", options=folder_paths.get_filename_list("loras")),
+                io.Int.Input("workflow_index", default=0, min=0, max=100)
+            ],
+            outputs=[
+                io.Image.Output(display_name="preview"),
+                io.String.Output(display_name="workflow_json"),
+                io.String.Output(display_name="metadata_json")
+            ]
+        )
+
+    @classmethod
+    def execute(cls, lora, workflow_index):
+        res = cls.get_info("loras", lora, workflow_index)
+        return io.NodeOutput(res[0], res[1], res[2])
+
+class EmbeddingInfoLoader(BaseModelInfoLoader):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="Embedding Info Loader",
+            category=cls.CATEGORY,
+            inputs=[
+                io.Combo.Input("embedding", options=folder_paths.get_filename_list("embeddings")),
+                io.Int.Input("workflow_index", default=0, min=0, max=100)
+            ],
+            outputs=[
+                io.Image.Output(display_name="preview"),
+                io.String.Output(display_name="workflow_json"),
+                io.String.Output(display_name="metadata_json")
+            ]
+        )
+
+    @classmethod
+    def execute(cls, embedding, workflow_index):
+        res = cls.get_info("embeddings", embedding, workflow_index)
+        return io.NodeOutput(res[0], res[1], res[2])
+
+class VAEInfoLoader(BaseModelInfoLoader):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="VAE Info Loader",
+            category=cls.CATEGORY,
+            inputs=[
+                io.Combo.Input("vae", options=folder_paths.get_filename_list("vae")),
+                io.Int.Input("workflow_index", default=0, min=0, max=100)
+            ],
+            outputs=[
+                io.Image.Output(display_name="preview"),
+                io.String.Output(display_name="workflow_json"),
+                io.String.Output(display_name="metadata_json")
+            ]
+        )
+
+    @classmethod
+    def execute(cls, vae, workflow_index):
+        res = cls.get_info("vae", vae, workflow_index)
+        return io.NodeOutput(res[0], res[1], res[2])
+
+class ControlNetInfoLoader(BaseModelInfoLoader):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="ControlNet Info Loader",
+            category=cls.CATEGORY,
+            inputs=[
+                io.Combo.Input("controlnet", options=folder_paths.get_filename_list("controlnet")),
+                io.Int.Input("workflow_index", default=0, min=0, max=100)
+            ],
+            outputs=[
+                io.Image.Output(display_name="preview"),
+                io.String.Output(display_name="workflow_json"),
+                io.String.Output(display_name="metadata_json")
+            ]
+        )
+
+    @classmethod
+    def execute(cls, controlnet, workflow_index):
+        res = cls.get_info("controlnet", controlnet, workflow_index)
+        return io.NodeOutput(res[0], res[1], res[2])
+
+class DiffusionModelInfoLoader(BaseModelInfoLoader):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id=cls.__name__,
+            display_name="Diffusion Model Info Loader",
+            category=cls.CATEGORY,
+            inputs=[
+                io.Combo.Input("diffusion_model", options=folder_paths.get_filename_list("diffusion_models")),
+                io.Int.Input("workflow_index", default=0, min=0, max=100)
+            ],
+            outputs=[
+                io.Image.Output(display_name="preview"),
+                io.String.Output(display_name="workflow_json"),
+                io.String.Output(display_name="metadata_json")
+            ]
+        )
+
+    @classmethod
+    def execute(cls, diffusion_model, workflow_index):
+        res = cls.get_info("diffusion_models", diffusion_model, workflow_index)
+        return io.NodeOutput(res[0], res[1], res[2])

nodes/prunekeys.py

@@ -7,9 +7,10 @@ from comfy_api.latest import io
 from safetensors.torch import save_file
 from safetensors import safe_open
 from .utils import convert_pt_to_safetensors
-from .merger_utils import MemoryEfficientSafeOpen
 from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation
 
+from unifiedefficientloader import MemoryEfficientSafeOpen
+
 
 def _prune_keys(model_name: str, model_type: str, keys_to_prune_str: str,
                 use_regex: bool, output_filename: str) -> str:

nodes/renamekeys.py

@@ -6,9 +6,11 @@ from comfy_api.latest import io
 from safetensors.torch import save_file
 from safetensors import safe_open
 from .utils import convert_pt_to_safetensors
-from .merger_utils import MemoryEfficientSafeOpen
 from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation
 
+from unifiedefficientloader import MemoryEfficientSafeOpen
+
+
 
 def _rename_keys(model_name: str, model_type: str, old_keys_str: str,
                  new_keys_str: str, output_filename: str) -> str:

pyproject.toml

@@ -1,9 +1,24 @@
 [project]
 name = "ComfyUI-ModelUtils"
 description = "[WIP]Custom nodes for handling, inspecting, modifying and creating various model files."
-version = "0.2.0"
+version = "0.3.5"
 license = { file = "LICENSE" }
 
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "unifiedefficientloader==0.4.4"
+]
+
 [project.urls]
 Repository = "https://github.com/silveroxides/ComfyUI-ModelUtils"
 #  Used by Comfy Registry https://comfyregistry.org

requirements.txt

@@ -1 +1,5 @@
-# No external dependencies required
+requests
+Pillow
+mutagen
+unifiedefficientloader==0.4.4
+av