add new feature to resize LoRA by first adding it to base model

Feature/lora resize

__init__.py

@@ -23,6 +23,13 @@ from .nodes.lora_extract_svd import (
     LoRAExtractFixed, LoRAExtractRatio, LoRAExtractQuantile,
     LoRAExtractKnee, LoRAExtractFrobenius
 )
+from .nodes.lora_resize import (
+    LoRAResizeFixed, LoRAResizeRatio,
+    LoRAResizeFrobenius, LoRAResizeCumulative,
+    LoRAResizeViaBaseFixed, LoRAResizeViaBaseRatio,
+    LoRAResizeViaBaseFrobenius, LoRAResizeViaBaseCumulative,
+    LoRAMultiMerge
+)
 
 
 class ModelUtilsExtension(ComfyExtension):
@@ -47,6 +54,14 @@ class ModelUtilsExtension(ComfyExtension):
             # LoRA Extraction
             LoRAExtractFixed, LoRAExtractRatio, LoRAExtractQuantile,
             LoRAExtractKnee, LoRAExtractFrobenius,
+            # LoRA Resize
+            LoRAResizeFixed, LoRAResizeRatio,
+            LoRAResizeFrobenius, LoRAResizeCumulative,
+            # LoRA Fast Resize (via base model)
+            LoRAResizeViaBaseFixed, LoRAResizeViaBaseRatio,
+            LoRAResizeViaBaseFrobenius, LoRAResizeViaBaseCumulative,
+            # LoRA Multi-Merge
+            LoRAMultiMerge,
         ]
 
 

docs/DEVELOPMENT.md

@@ -0,0 +1,321 @@
+# ComfyUI-ModelUtils - Complete Project History
+
+## Version Timeline
+
+| Version | Commits | Key Features |
+|---------|---------|--------------|
+| 0.1.1 | `70d6c13`-`d67cbca` | Initial nodes (MetaKeys) |
+| 0.1.2 | `6b662a4` | Icon assets |
+| 0.1.4 | `cbe3502` | Key renaming/pruning nodes |
+| 0.1.5 | `78b3198` | Model merging nodes |
+| 0.1.6 | `31989be` | Embedding nodes, namespace fixes |
+| 0.1.7 | `b6fae8d` | Mismatch handling, layer filtering, V3 migration |
+| **0.2.0** | `3df073f` | LoRA extraction, memory optimization, pinned memory |
+
+---
+
+## Phase 1: Initial Release (v0.1.1)
+
+**Commit:** `70d6c13` - "Add first nodes, toml and publish"
+
+### Files Created
+- `nodes/metakeys.py` - Get model metadata and layer keys
+- `nodes/utils.py` - Shared utilities
+- `__init__.py` - Node registration
+- `pyproject.toml` - Package configuration
+
+### Nodes
+- `ModelMetaKeys` - Get diffusion model metadata
+- `CheckpointMetaKeys` - Get checkpoint metadata
+- `TextEncoderMetaKeys` - Get text encoder metadata
+- `LoRAMetaKeys` - Get LoRA metadata
+- `EmbeddingMetaKeys` - Get embedding metadata
+
+---
+
+## Phase 2: Key Manipulation (v0.1.4)
+
+**Commit:** `4f08fe6` - "add key renaming and key pruning nodes"
+
+### Files Created
+- `nodes/renamekeys.py` - Rename tensor keys
+- `nodes/prunekeys.py` - Remove tensor keys by pattern
+
+### Nodes (5 variants each for model types)
+- `ModelRenameKeys`, `CheckpointRenameKeys`, `TextEncoderRenameKeys`, `LoRARenameKeys`, `EmbeddingRenameKeys`
+- `ModelPruneKeys`, `CheckpointPruneKeys`, `TextEncoderPruneKeys`, `LoRAPruneKeys`, `EmbeddingPruneKeys`
+
+---
+
+## Phase 3: Model Merging (v0.1.5)
+
+**Commit:** `7959370` - "Add model merging nodes"
+
+### Files Created
+- `nodes/merger.py` - Main merging logic
+- `nodes/merger_ops.py` - Merge operation modes
+- `nodes/merger_utils.py` - `MemoryEfficientSafeOpen` class
+- `docs/merger_2_model_modes.md` - Documentation
+- `docs/merger_3_model_modes.md` - Documentation
+
+### Nodes
+- `CheckpointTwoMerger`, `DiffusionModelTwoMerger`, `TextEncoderTwoMerger`, `LoRATwoMerger`
+- `CheckpointThreeMerger`, `DiffusionModelThreeMerger`, `TextEncoderThreeMerger`, `LoRAThreeMerger`
+
+### Merge Modes (TWO_MODEL_MODES)
+- Weighted Sum, Add Difference, Smooth Add Difference
+- Train Difference, Sum Twice, Triple Sum
+- Euclidean Add Difference, Distribution Crossover, Ties
+- Slerp, Lerp, SVD Supermerger, SVD LoRA Extraction
+
+### Merge Modes (THREE_MODEL_MODES)
+- Add Difference (A + (B - C) × α)
+
+### MemoryEfficientSafeOpen (initial)
+```python
+class MemoryEfficientSafeOpen:
+    def __init__(self, filename, device='cpu', mmap_mode=True)
+    def keys() -> List[str]
+    def get_tensor(key) -> Tensor
+    # mmap for zero-copy reads
+```
+
+---
+
+## Phase 4: Embedding Nodes (v0.1.6)
+
+**Commit:** `0d035ef` - "Add embedding nodes"
+
+### Additions
+- Added embedding variants to all node types
+- Namespace conflict fixes (`68f22fa`, `5ea63ab`)
+- Refactored names for universality (`232dc76`)
+
+---
+
+## Phase 5: Mismatch Handling (v0.1.7)
+
+**Commits:** `19bc047` → `de27186`
+
+### Features Added
+- `mismatch_mode` parameter: skip/zeros/error
+- Layer filtering with regex patterns
+- `exclude_patterns` - Keep model A only (no merge)
+- `discard_patterns` - Remove from output entirely
+
+### Files Modified
+- `merger.py` - Added mismatch handling logic
+- `merger_ops.py` - Added `MissingTensorBehavior` enum, `MissingTensorError`
+
+---
+
+## Phase 6: V3 API Migration (v0.1.7)
+
+**Commit:** `a3116cb` - "migrate to v3 schema and add LoRA extraction node"
+
+### Changes
+- Migrated all nodes from legacy API to `comfy_api.latest.io`
+- `io.Schema`, `io.ComfyNode`, `io.NodeOutput`
+- Added initial LoRA extraction node (diff-based)
+
+---
+
+## Phase 7: LyCORIS Extraction (pre-0.2.0)
+
+**Commits:** `92d3c55` → `18679cc`
+
+### Files Created
+- `nodes/lycoris_extract.py` - LyCORIS-based extraction
+- `nodes/lora_extract.py` - Simple diff extraction
+
+### Initial Implementation
+- Used `lycoris-lora` external dependency
+- `lycoris.utils.extract_linear`, `extract_conv`
+- Added `requirements.txt` with `lycoris-lora`
+
+### Nodes Split (`18679cc`)
+- `LyCORISExtractFixed`, `LyCORISExtractThreshold`
+- `LyCORISExtractRatio`, `LyCORISExtractQuantile`
+
+---
+
+## Phase 8: Native SVD Port (0.2.0)
+
+**Commits:** `6d7b2d1` → `eef7377`
+
+### Removed
+- `lycoris-lora` dependency
+- `lycoris_extract.py`
+- `lora_extract.py`
+
+### NEW: [lora_extract_svd.py](file:///f:/ComfyUI/custom_nodes/ComfyUI-ModelUtils/nodes/lora_extract_svd.py)
+
+**8 Rank Selection Functions:**
+```python
+_index_sv_fixed(S, dim)           # Fixed rank
+_index_sv_threshold(S, threshold) # Absolute threshold
+_index_sv_ratio(S, ratio)         # Relative to max
+_index_sv_cumulative(S, target)   # Cumulative sum %
+_index_sv_fro(S, target)          # Frobenius norm
+_index_sv_knee(S)                 # Knee detection
+_index_sv_cumulative_knee(S)      # Cumsum knee
+_index_sv_rel_decrease(S, tau)    # Ratio drop
+```
+
+**SVD Functions:**
+```python
+_svd_extract_linear_lowrank(weight, rank, device, clamp_quantile)
+_svd_extract_linear(weight, mode, mode_param, device, max_rank, clamp_quantile)
+_svd_extract_conv(weight, mode, mode_param, device, max_rank, clamp_quantile)
+_detect_fused_layer(key, shape)  # QKV/MLP detection
+_extract_chunked_layer(...)      # Chunked extraction
+```
+
+**Nodes:**
+- `LoRAExtractFixed` - Uses `svd_lowrank` (10x faster)
+- `LoRAExtractRatio` - Relative threshold
+- `LoRAExtractQuantile` - Cumulative target
+- `LoRAExtractKnee` - Auto knee detection
+- `LoRAExtractFrobenius` - Norm preservation
+
+---
+
+## Phase 9: Memory Management (0.2.0)
+
+**Commit:** `6d7b2d1` - "refactor nodes and implement memory management"
+
+### NEW: [device_utils.py](file:///f:/ComfyUI/custom_nodes/ComfyUI-ModelUtils/nodes/device_utils.py)
+
+Integrates with `comfy.model_management`:
+
+```python
+@dataclass
+class DeviceCapabilities:
+    device_type, total_vram_gb, free_vram_gb, total_ram_gb, free_ram_gb
+    cpu_cores, recommended_workers, max_tensor_size_gb
+    supports_pinned_memory, supports_async_streams
+
+get_device_capabilities(device) -> DeviceCapabilities
+estimate_model_size(path) -> float  # GB
+can_fit_in_memory(paths, device, safety_factor) -> (bool, str)
+prepare_for_large_operation(gb, device) -> bool  # Calls free_memory()
+cleanup_after_operation()  # gc.collect() + soft_empty_cache()
+get_optimal_workers(size_gb) -> int
+```
+
+### Updates to Existing Nodes
+- `metakeys.py` - Header-only reading (no tensor loading)
+- `renamekeys.py` - `MemoryEfficientSafeOpen` + memory mgmt
+- `prunekeys.py` - `MemoryEfficientSafeOpen` + memory mgmt
+- `merger.py` - Memory prep/cleanup
+
+---
+
+## Phase 10: Pinned Memory (0.2.0)
+
+**Commit:** `b770786` - "Add pinned memory for faster merging and extraction"
+
+### [merger_utils.py](file:///f:/ComfyUI/custom_nodes/ComfyUI-ModelUtils/nodes/merger_utils.py) Enhancements
+
+```python
+# Context manager
+class PinnedTensor:
+    def __enter__(self) -> Tensor  # Pins memory
+    def __exit__(...)              # Unpins
+
+# Transfer helper (2-3x faster)
+def transfer_to_gpu_pinned(tensor, device='cuda', dtype=None) -> Tensor
+
+# MemoryEfficientSafeOpen additions
+def get_tensor_to_gpu(key, device, dtype)  # Pinned transfer
+def keys_sorted_by_offset() -> List[str]   # Sequential I/O
+def get_tensors_parallel(keys, workers, sort_by_offset) -> Dict  # Parallel
+def get_tensor_as_dict(key) -> Dict        # JSON config parsing
+```
+
+---
+
+## Phase 11: SVD Optimization (0.2.0)
+
+**Commit:** `65c1ae3` - "add svd_lowrank for fixed rank extraction"
+
+### Changes
+- Added `_svd_extract_linear_lowrank()` using `torch.svd_lowrank`
+- Routes fixed mode to lowrank for **10x speedup**
+- Changed `chunk_large_layers` default to `False`
+
+---
+
+## Phase 12: Bug Fixes (0.2.0)
+
+**Commit:** `eef7377` - "Added .contiguous() to saves"
+
+### Fix
+Non-contiguous tensor error from SVD slices:
+```python
+tensor.to(dtype).cpu().contiguous()  # Added .contiguous()
+```
+
+---
+
+## Current File Structure
+
+```
+ComfyUI-ModelUtils/
+├── __init__.py
+├── pyproject.toml
+├── requirements.txt (empty - no external deps)
+├── README.md
+├── docs/
+│   ├── merger_2_model_modes.md
+│   └── merger_3_model_modes.md
+├── nodes/
+│   ├── utils.py
+│   ├── metakeys.py
+│   ├── renamekeys.py
+│   ├── prunekeys.py
+│   ├── merger.py
+│   ├── merger_ops.py
+│   ├── merger_utils.py
+│   ├── device_utils.py
+│   └── lora_extract_svd.py
+└── assets/
+    └── icon.png
+```
+
+---
+
+## Performance Summary
+
+| Feature | Improvement |
+|---------|-------------|
+| Fixed-rank SVD | **10x** (svd_lowrank) |
+| CPU→GPU transfer | **2-3x** (pinned memory) |
+| Metadata reading | **100x+** (header only) |
+| Parallel I/O | **2-4x** (ThreadPoolExecutor) |
+| Memory management | Auto-unload models before heavy ops |
+| **LoRA Resize Fast** | **~91x** (merge-to-base extraction) |
+
+---
+
+## Phase 13: Fast LoRA Resize (2026-01-11)
+
+### Summary
+Added fast LoRA resize via base model extraction path, achieving ~91x speedup.
+
+### Changes
+- Added `resize_lora_via_base()` function using extraction code path
+- Added 4 new fast resize nodes requiring base model input
+
+### Files Modified
+
+| File | Changes |
+|------|---------|
+| `nodes/lora_resize.py` | Added `resize_lora_via_base()`, 4 new node classes |
+| `__init__.py` | Registered new nodes |
+
+### New Nodes
+- `LoRAResizeViaBaseFixed` - Fast fixed rank resize
+- `LoRAResizeViaBaseRatio` - Fast SV ratio resize  
+- `LoRAResizeViaBaseFrobenius` - Fast Frobenius resize
+- `LoRAResizeViaBaseCumulative` - Fast cumulative resize

nodes/lora_extract_svd.py

@@ -36,17 +36,24 @@ def _index_sv_threshold(S: torch.Tensor, threshold: float) -> int:
 
 
 def _index_sv_ratio(S: torch.Tensor, ratio: float) -> int:
-    """Ratio mode - keep singular values > max(S) * ratio."""
+    """Ratio mode - keep singular values > max(S) / ratio (kohya convention)."""
     if ratio <= 0:
         return len(S) - 1
-    min_sv = S[0] * ratio
+    min_sv = S[0] / ratio
     rank = int(torch.sum(S > min_sv).item())
     return max(1, min(rank, len(S) - 1))
 
 
-def _index_sv_cumulative(S: torch.Tensor, target: float) -> int:
+def _index_sv_cumulative(S: torch.Tensor, target: float, max_rank: int = None) -> int:
-    """Cumulative mode - keep enough SVs to reach target % of total."""
+    """Cumulative mode - keep enough SVs to reach target % of total.
-    total = torch.sum(S)
+    
+    Calculates relative to max_rank if provided, otherwise relative to full.
+    """
+    if max_rank is not None and max_rank < len(S):
+        total = torch.sum(S[:max_rank])
+    else:
+        total = torch.sum(S)
+    
     if total < 1e-8:
         return 1
     cumsum = torch.cumsum(S, dim=0) / total
@@ -54,13 +61,26 @@ def _index_sv_cumulative(S: torch.Tensor, target: float) -> int:
     return max(1, min(rank, len(S) - 1))
 
 
-def _index_sv_fro(S: torch.Tensor, target: float) -> int:
+def _index_sv_fro(S: torch.Tensor, target: float, max_rank: int = None) -> int:
-    """Frobenius norm mode - preserve target fraction of Frobenius norm."""
+    """Frobenius norm mode - preserve target fraction of Frobenius norm.
-    S_sq = S.pow(2)
+    
-    total_sq = torch.sum(S_sq)
+    Calculates relative to max_rank if provided, otherwise relative to full.
+    This means "retain target% of what's achievable within max_rank".
+    """
+    if max_rank is not None and max_rank < len(S):
+        # Calculate relative to what's achievable within max_rank
+        S_capped = S[:max_rank]
+        S_sq = S_capped.pow(2)
+        total_sq = torch.sum(S_sq)
+    else:
+        S_sq = S.pow(2)
+        total_sq = torch.sum(S_sq)
+    
     if total_sq < 1e-8:
         return 1
-    cumsum = torch.cumsum(S_sq, dim=0) / total_sq
+    
+    # Cumsum of all S (not capped) to find where we reach target
+    cumsum = torch.cumsum(S.pow(2), dim=0) / total_sq
     rank = int(torch.searchsorted(cumsum, target ** 2).item()) + 1
     return max(1, min(rank, len(S) - 1))
 
@@ -122,9 +142,9 @@ def _compute_rank(S: torch.Tensor, mode: str, mode_param: float,
     elif mode == "ratio":
         rank = _index_sv_ratio(S, mode_param)
     elif mode == "quantile" or mode == "sv_cumulative":
-        rank = _index_sv_cumulative(S, mode_param)
+        rank = _index_sv_cumulative(S, mode_param, max_rank)
     elif mode == "sv_fro":
-        rank = _index_sv_fro(S, mode_param)
+        rank = _index_sv_fro(S, mode_param, max_rank)
     elif mode == "sv_knee":
         rank = _index_sv_knee(S)
     elif mode == "sv_cumulative_knee":
@@ -148,6 +168,7 @@ def _svd_extract_linear_lowrank(
     rank: int,
     device: str,
     clamp_quantile: float = 0.99,
+    niter: int = 2,
 ) -> tuple:
     """
     Low-rank SVD decomposition using svd_lowrank for fixed rank extraction.
@@ -155,6 +176,9 @@ def _svd_extract_linear_lowrank(
     Much faster than full SVD when rank << min(m, n) because it only computes
     the top-k singular values using randomized algorithms.
     
+    Args:
+        niter: Power iterations for SVD accuracy (higher = more accurate but slower)
+    
     Returns:
         (lora_down, lora_up, diff), "low rank" or (weight, "full")
     """
@@ -170,9 +194,8 @@ def _svd_extract_linear_lowrank(
         return weight, "full"
     
     # Use svd_lowrank for faster low-rank approximation
-    # niter controls accuracy (higher = more accurate but slower)
     try:
-        U, S, V = torch.svd_lowrank(weight, q=rank, niter=2)
+        U, S, V = torch.svd_lowrank(weight, q=rank, niter=niter)
         # U: [out_dim, rank], S: [rank], V: [in_dim, rank]
         Vh = V.T  # [rank, in_dim]
     except Exception as e:
@@ -203,6 +226,7 @@ def _svd_extract_linear(
     device: str,
     max_rank: int = None,
     clamp_quantile: float = 0.99,
+    niter: int = 2,
 ) -> tuple:
     """
     SVD decomposition for linear layers.
@@ -221,7 +245,7 @@ def _svd_extract_linear(
         target_rank = int(mode_param)
         if max_rank is not None:
             target_rank = min(target_rank, max_rank)
-        return _svd_extract_linear_lowrank(weight, target_rank, device, clamp_quantile)
+        return _svd_extract_linear_lowrank(weight, target_rank, device, clamp_quantile, niter)
 
     # For adaptive modes, use full SVD to analyze singular values
     try:
@@ -409,6 +433,7 @@ def extract_lora_from_files(
     skip_patterns_str: str = "",
     mismatch_mode: str = "skip",
     chunk_large_layers: bool = True,
+    svd_niter: int = 2,
 ) -> dict[str, torch.Tensor]:
     """
     Extract LoRA from difference between two models.
@@ -528,7 +553,7 @@ def extract_lora_from_files(
                     )
                 else:
                     result, mode_str = _svd_extract_linear(
-                        weight_diff, mode, linear_param, device, linear_max_rank, clamp_quantile
+                        weight_diff, mode, linear_param, device, linear_max_rank, clamp_quantile, svd_niter
                     )
             except Exception as e:
                 # Try chunked extraction for large tensors
@@ -640,6 +665,8 @@ class LoRAExtractFixed(io.ComfyNode):
                             tooltip="Rank for linear/attention layers"),
                 io.Int.Input("conv_dim", default=32, min=1, max=4096,
                             tooltip="Rank for conv layers"),
+                io.Int.Input("svd_niter", default=2, min=0, max=10,
+                            tooltip="SVD power iterations (higher = more accurate but slower)"),
                 *_get_common_inputs(),
             ],
             outputs=[io.String.Output(display_name="output_path")],
@@ -647,7 +674,7 @@ class LoRAExtractFixed(io.ComfyNode):
         )
 
     @classmethod
-    def execute(cls, model_a, model_b, linear_dim, conv_dim, chunk_large_layers,
+    def execute(cls, model_a, model_b, linear_dim, conv_dim, svd_niter, chunk_large_layers,
                 clamp_quantile, min_diff, mismatch_mode, output_filename,
                 save_dtype, device, skip_patterns) -> io.NodeOutput:
 
@@ -657,7 +684,7 @@ class LoRAExtractFixed(io.ComfyNode):
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "fixed", linear_dim, conv_dim,
             "lora_unet_", device, save_dtype, linear_dim, conv_dim,
-            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
+            clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers, svd_niter
         )
 
         return io.NodeOutput(_save_lora(output_sd, output_filename))
@@ -672,13 +699,17 @@ class LoRAExtractRatio(io.ComfyNode):
             node_id="LoRAExtractRatio",
             display_name="LoRA Extract (Ratio)",
             category="ModelUtils/LoRA",
-            description="Keep singular values > max(S) × ratio.",
+            description="Keep singular values > max(S) / ratio.",
             inputs=[
                 *_get_model_inputs(),
-                io.Float.Input("linear_ratio", default=0.5, min=0.0, max=1.0, step=0.01,
+                io.Float.Input("linear_ratio", default=2.0, min=1.0, max=100.0, step=0.1,
-                              tooltip="Ratio threshold for linear layers"),
+                              tooltip="Ratio threshold for linear layers (higher = more SVs kept)"),
-                io.Float.Input("conv_ratio", default=0.5, min=0.0, max=1.0, step=0.01,
+                io.Float.Input("conv_ratio", default=2.0, min=1.0, max=100.0, step=0.1,
-                              tooltip="Ratio threshold for conv layers"),
+                              tooltip="Ratio threshold for conv layers (higher = more SVs kept)"),
+                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                            tooltip="Maximum rank for linear layers"),
+                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                            tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
             outputs=[io.String.Output(display_name="output_path")],
@@ -686,8 +717,8 @@ class LoRAExtractRatio(io.ComfyNode):
         )
 
     @classmethod
-    def execute(cls, model_a, model_b, linear_ratio, conv_ratio, chunk_large_layers,
+    def execute(cls, model_a, model_b, linear_ratio, conv_ratio, linear_max_rank, conv_max_rank,
-                clamp_quantile, min_diff, mismatch_mode, output_filename,
+                chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
                 save_dtype, device, skip_patterns) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
@@ -695,7 +726,7 @@ class LoRAExtractRatio(io.ComfyNode):
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "ratio", linear_ratio, conv_ratio,
-            "lora_unet_", device, save_dtype, None, None,
+            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
         )
 
@@ -718,6 +749,10 @@ class LoRAExtractQuantile(io.ComfyNode):
                               tooltip="Target cumulative % for linear layers"),
                 io.Float.Input("conv_quantile", default=0.9, min=0.0, max=1.0, step=0.01,
                               tooltip="Target cumulative % for conv layers"),
+                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                            tooltip="Maximum rank for linear layers"),
+                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                            tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
             outputs=[io.String.Output(display_name="output_path")],
@@ -725,8 +760,8 @@ class LoRAExtractQuantile(io.ComfyNode):
         )
 
     @classmethod
-    def execute(cls, model_a, model_b, linear_quantile, conv_quantile, chunk_large_layers,
+    def execute(cls, model_a, model_b, linear_quantile, conv_quantile, linear_max_rank, conv_max_rank,
-                clamp_quantile, min_diff, mismatch_mode, output_filename,
+                chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
                 save_dtype, device, skip_patterns) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
@@ -734,7 +769,7 @@ class LoRAExtractQuantile(io.ComfyNode):
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "quantile", linear_quantile, conv_quantile,
-            "lora_unet_", device, save_dtype, None, None,
+            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
         )
 
@@ -755,6 +790,10 @@ class LoRAExtractKnee(io.ComfyNode):
                 *_get_model_inputs(),
                 io.Combo.Input("knee_method", options=["sv_knee", "sv_cumulative_knee"],
                               default="sv_knee", tooltip="Knee detection method"),
+                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                            tooltip="Maximum rank for linear layers"),
+                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                            tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
             outputs=[io.String.Output(display_name="output_path")],
@@ -762,8 +801,8 @@ class LoRAExtractKnee(io.ComfyNode):
         )
 
     @classmethod
-    def execute(cls, model_a, model_b, knee_method, chunk_large_layers,
+    def execute(cls, model_a, model_b, knee_method, linear_max_rank, conv_max_rank,
-                clamp_quantile, min_diff, mismatch_mode, output_filename,
+                chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
                 save_dtype, device, skip_patterns) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
@@ -771,7 +810,7 @@ class LoRAExtractKnee(io.ComfyNode):
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, knee_method, 0, 0,
-            "lora_unet_", device, save_dtype, None, None,
+            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
         )
 
@@ -794,6 +833,10 @@ class LoRAExtractFrobenius(io.ComfyNode):
                               tooltip="Target Frobenius norm fraction for linear"),
                 io.Float.Input("conv_target", default=0.9, min=0.0, max=1.0, step=0.01,
                               tooltip="Target Frobenius norm fraction for conv"),
+                io.Int.Input("linear_max_rank", default=128, min=1, max=1024,
+                            tooltip="Maximum rank for linear layers"),
+                io.Int.Input("conv_max_rank", default=128, min=1, max=1024,
+                            tooltip="Maximum rank for conv layers"),
                 *_get_common_inputs(),
             ],
             outputs=[io.String.Output(display_name="output_path")],
@@ -801,8 +844,8 @@ class LoRAExtractFrobenius(io.ComfyNode):
         )
 
     @classmethod
-    def execute(cls, model_a, model_b, linear_target, conv_target, chunk_large_layers,
+    def execute(cls, model_a, model_b, linear_target, conv_target, linear_max_rank, conv_max_rank,
-                clamp_quantile, min_diff, mismatch_mode, output_filename,
+                chunk_large_layers, clamp_quantile, min_diff, mismatch_mode, output_filename,
                 save_dtype, device, skip_patterns) -> io.NodeOutput:
 
         model_a_path = folder_paths.get_full_path_or_raise("diffusion_models", model_a)
@@ -810,7 +853,7 @@ class LoRAExtractFrobenius(io.ComfyNode):
 
         output_sd = extract_lora_from_files(
             model_a_path, model_b_path, "sv_fro", linear_target, conv_target,
-            "lora_unet_", device, save_dtype, None, None,
+            "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank,
             clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers
         )
 

nodes/merger_utils.py

@@ -28,23 +28,23 @@ except ImportError:
 
 class PinnedTensor:
     """Context manager for automatic pinned memory management.
-    
+
     Usage:
         with PinnedTensor(cpu_tensor) as pinned:
             gpu_tensor = pinned.to('cuda')
         # Memory automatically unpinned after transfer
     """
-    
+
     def __init__(self, tensor: torch.Tensor, auto_pin: bool = True):
         self.tensor = tensor
         self.is_pinned = False
         self.auto_pin = auto_pin and PINNING_AVAILABLE
-        
+
     def __enter__(self) -> torch.Tensor:
         if self.auto_pin and self.tensor.device.type == 'cpu':
             self.is_pinned = pin_memory(self.tensor)
         return self.tensor
-    
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.is_pinned:
             unpin_memory(self.tensor)
@@ -57,12 +57,12 @@ def transfer_to_gpu_pinned(
     dtype: Optional[torch.dtype] = None
 ) -> torch.Tensor:
     """Transfer tensor to GPU using pinned memory for faster transfer.
-    
+
     Args:
         tensor: CPU tensor to transfer
         device: Target GPU device
         dtype: Optional dtype conversion
-        
+
     Returns:
         Tensor on GPU device
     """
@@ -71,7 +71,7 @@ def transfer_to_gpu_pinned(
         if dtype:
             return tensor.to(device=device, dtype=dtype)
         return tensor.to(device=device)
-    
+
     # Pin memory for faster transfer
     was_pinned = pin_memory(tensor)
     try:
@@ -162,7 +162,7 @@ class MemoryEfficientSafeOpen:
                 tensor_bytes = self.file.read(offset_end - offset_start)
 
         return self._deserialize_tensor(tensor_bytes, metadata)
-    
+
     def get_tensor_to_gpu(
         self,
         key: str,
@@ -170,12 +170,12 @@ class MemoryEfficientSafeOpen:
         dtype: Optional[torch.dtype] = None
     ) -> torch.Tensor:
         """Load a tensor and transfer to GPU using pinned memory for faster transfer.
-        
+
         Args:
             key: Tensor key to load
             device: Target GPU device (default 'cuda')
             dtype: Optional dtype conversion
-            
+
         Returns:
             Tensor on GPU device
         """

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "ComfyUI-ModelUtils"
 description = "[WIP]Custom nodes for handling, inspecting, modifying and creating various model files."
-version = "0.2.0"
+version = "0.2.3"
 license = { file = "LICENSE" }
 
 [project.urls]