diff --git a/__init__.py b/__init__.py index 7996542..e29d660 100644 --- a/__init__.py +++ b/__init__.py @@ -1,6 +1,7 @@ from typing_extensions import override from comfy_api.latest import ComfyExtension, io +from .nodes.lora_merger import LoRAMultiMerge, LoRAMultiMergeDARE from .nodes.metakeys import ( ModelMetaKeys, TextEncoderMetaKeys, LoRAMetaKeys, CheckpointMetaKeys, EmbeddingMetaKeys @@ -26,9 +27,7 @@ from .nodes.lora_extract_svd import ( from .nodes.lora_resize import ( LoRAResizeFixed, LoRAResizeRatio, LoRAResizeFrobenius, LoRAResizeCumulative, - LoRAResizeViaBaseFixed, LoRAResizeViaBaseRatio, - LoRAResizeViaBaseFrobenius, LoRAResizeViaBaseCumulative, - LoRAMultiMerge, LoRAMergeToModel + LoRAMergeToModel ) @@ -57,11 +56,8 @@ class ModelUtilsExtension(ComfyExtension): # LoRA Resize LoRAResizeFixed, LoRAResizeRatio, LoRAResizeFrobenius, LoRAResizeCumulative, - # LoRA Fast Resize (via base model) - LoRAResizeViaBaseFixed, LoRAResizeViaBaseRatio, - LoRAResizeViaBaseFrobenius, LoRAResizeViaBaseCumulative, # LoRA Multi-Merge - LoRAMultiMerge, + LoRAMultiMerge, LoRAMultiMergeDARE, # LoRA Merge To Model LoRAMergeToModel, ] diff --git a/nodes/lora_extract_svd.py b/nodes/lora_extract_svd.py index 72562eb..2e83efd 100644 --- a/nodes/lora_extract_svd.py +++ b/nodes/lora_extract_svd.py @@ -423,7 +423,6 @@ def extract_lora_from_files( mode: str, linear_param: float, conv_param: float, - prefix: str, device: str, save_dtype: str, linear_max_rank: int = None, @@ -444,7 +443,6 @@ def extract_lora_from_files( mode: Rank selection mode linear_param: Mode parameter for linear layers conv_param: Mode parameter for conv layers - prefix: LoRA key prefix (e.g. "lora_unet_") device: Computation device save_dtype: Output dtype linear_max_rank: Max rank for linear layers @@ -484,8 +482,7 @@ def extract_lora_from_files( stats = {"extracted": 0, "full": 0, "skipped": 0, "chunked": 0} for key in tqdm(weight_keys, desc="Extracting LoRA", unit="layers"): - lora_key = key[:-7] - lora_name = prefix + lora_key.replace(".", "_") + lora_name = _format_lora_key(key) if _matches_any_pattern(key, skip_patterns): stats["skipped"] += 1 @@ -567,7 +564,7 @@ def extract_lora_from_files( if lora_up is not None: output_sd[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous() output_sd[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous() - output_sd[f"{lora_name}.alpha"] = torch.tensor([rank]).to(save_torch_dtype) + output_sd[f"{lora_name}.alpha"] = torch.tensor(rank).to(save_torch_dtype) stats["chunked"] += 1 del weight_diff pbar.update(1) @@ -587,7 +584,7 @@ def extract_lora_from_files( lora_down, lora_up, _ = result output_sd[f"{lora_name}.lora_down.weight"] = lora_down.to(save_torch_dtype).cpu().contiguous() output_sd[f"{lora_name}.lora_up.weight"] = lora_up.to(save_torch_dtype).cpu().contiguous() - output_sd[f"{lora_name}.alpha"] = torch.tensor([lora_down.shape[0]]).to(save_torch_dtype) + output_sd[f"{lora_name}.alpha"] = torch.tensor(lora_down.shape[0]).to(save_torch_dtype) stats["extracted"] += 1 del weight_diff @@ -632,6 +629,84 @@ def _get_model_inputs(): ] +def _reconstruct_dots(key: str) -> str: + """Reconstruct dot structure from underscored key using heuristics.""" + # Block indices: input_blocks_1 -> input_blocks.1 + blocks = [ + "input_blocks", "output_blocks", "middle_block", + "transformer_blocks", "single_transformer_blocks", + "double_blocks", "single_blocks", + "down_blocks", "up_blocks", "mid_block", + "attentions", "resnets", "upsamplers", "downsamplers" + ] + for b in blocks: + key = re.sub(f"{b}_(\\d+)", f"{b}.\\1", key) + + # Sequential numbers: .1_1 -> .1.1 + key = re.sub(r"(\.\d+)_(\d+)", r"\1.\2", key) + + # Separators that should be preceded by dot + separators = [ + "transformer_blocks", "single_transformer_blocks", + "attn1", "attn2", "attn", + "img_attn", "txt_attn", + "to_q", "to_k", "to_v", "to_out", + "q_proj", "k_proj", "v_proj", "out_proj", + "qkv", "proj", "ff", "net", + "norm1", "norm2", "norm3", "norm", + "time_emb_proj" + ] + for s in separators: + key = key.replace(f"_{s}", f".{s}") + + # Special case: to_out_0 -> to_out.0 + key = re.sub(r"to_out_(\d+)", r"to_out.\1", key) + + return key + + +def _format_lora_key(key: str) -> str: + """ + Format the key for LoRA saving. + Standardizes to 'diffusion_model.' or 'transformer.' prefix. + """ + if key.endswith(".weight"): + key = key[:-7] + + # Handle ComfyUI Checkpoint wrapper + if key.startswith("model.diffusion_model."): + # Check if it's a Diffusers-format transformer inside a checkpoint + inner_key = key[22:] # len("model.diffusion_model.") + if inner_key.startswith("transformer_blocks") or inner_key.startswith("single_transformer_blocks"): + return f"transformer.{inner_key}" + # Standard checkpoint or Original Flux + return f"diffusion_model.{inner_key}" + + # Handle direct Diffusers keys + if key.startswith("transformer_blocks") or key.startswith("single_transformer_blocks"): + return f"transformer.{key}" + + # Handle legacy lora_unet_ prefix (for resizing without base) + if key.startswith("lora_unet_"): + core = key[10:] + dotted = _reconstruct_dots(core) + # Check if reconstructed key implies transformer format + if dotted.startswith("transformer_blocks") or dotted.startswith("single_transformer_blocks"): + return f"transformer.{dotted}" + return f"diffusion_model.{dotted}" + + # Handle already correct prefixes + if key.startswith("diffusion_model.") or key.startswith("transformer."): + return key + + # Handle known Unet blocks (Diffusers format UNet) + if any(key.startswith(p) for p in ["down_blocks", "up_blocks", "mid_block", "conv_in", "conv_out", "time_embedding", "class_embedding"]): + return f"diffusion_model.{key}" + + # Default fallback to diffusion_model + return f"diffusion_model.{key}" + + def _get_common_inputs(): return [ io.Boolean.Input("chunk_large_layers", default=False, @@ -683,7 +758,7 @@ class LoRAExtractFixed(io.ComfyNode): output_sd = extract_lora_from_files( model_a_path, model_b_path, "fixed", linear_dim, conv_dim, - "lora_unet_", device, save_dtype, linear_dim, conv_dim, + device, save_dtype, linear_dim, conv_dim, clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers, svd_niter ) @@ -706,9 +781,9 @@ class LoRAExtractRatio(io.ComfyNode): tooltip="Ratio threshold for linear layers (higher = more SVs kept)"), io.Float.Input("conv_ratio", default=2.0, min=1.0, max=100.0, step=0.1, tooltip="Ratio threshold for conv layers (higher = more SVs kept)"), - io.Int.Input("linear_max_rank", default=128, min=1, max=1024, + io.Int.Input("linear_max_rank", default=128, min=1, max=3072, tooltip="Maximum rank for linear layers"), - io.Int.Input("conv_max_rank", default=128, min=1, max=1024, + io.Int.Input("conv_max_rank", default=128, min=1, max=3072, tooltip="Maximum rank for conv layers"), *_get_common_inputs(), ], @@ -726,7 +801,7 @@ class LoRAExtractRatio(io.ComfyNode): output_sd = extract_lora_from_files( model_a_path, model_b_path, "ratio", linear_ratio, conv_ratio, - "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank, + device, save_dtype, linear_max_rank, conv_max_rank, clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers ) @@ -749,9 +824,9 @@ class LoRAExtractQuantile(io.ComfyNode): tooltip="Target cumulative % for linear layers"), io.Float.Input("conv_quantile", default=0.9, min=0.0, max=1.0, step=0.01, tooltip="Target cumulative % for conv layers"), - io.Int.Input("linear_max_rank", default=128, min=1, max=1024, + io.Int.Input("linear_max_rank", default=128, min=1, max=3072, tooltip="Maximum rank for linear layers"), - io.Int.Input("conv_max_rank", default=128, min=1, max=1024, + io.Int.Input("conv_max_rank", default=128, min=1, max=3072, tooltip="Maximum rank for conv layers"), *_get_common_inputs(), ], @@ -769,7 +844,7 @@ class LoRAExtractQuantile(io.ComfyNode): output_sd = extract_lora_from_files( model_a_path, model_b_path, "quantile", linear_quantile, conv_quantile, - "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank, + device, save_dtype, linear_max_rank, conv_max_rank, clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers ) @@ -790,9 +865,9 @@ class LoRAExtractKnee(io.ComfyNode): *_get_model_inputs(), io.Combo.Input("knee_method", options=["sv_knee", "sv_cumulative_knee"], default="sv_knee", tooltip="Knee detection method"), - io.Int.Input("linear_max_rank", default=128, min=1, max=1024, + io.Int.Input("linear_max_rank", default=128, min=1, max=3072, tooltip="Maximum rank for linear layers"), - io.Int.Input("conv_max_rank", default=128, min=1, max=1024, + io.Int.Input("conv_max_rank", default=128, min=1, max=3072, tooltip="Maximum rank for conv layers"), *_get_common_inputs(), ], @@ -810,7 +885,7 @@ class LoRAExtractKnee(io.ComfyNode): output_sd = extract_lora_from_files( model_a_path, model_b_path, knee_method, 0, 0, - "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank, + device, save_dtype, linear_max_rank, conv_max_rank, clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers ) @@ -833,9 +908,9 @@ class LoRAExtractFrobenius(io.ComfyNode): tooltip="Target Frobenius norm fraction for linear"), io.Float.Input("conv_target", default=0.9, min=0.0, max=1.0, step=0.01, tooltip="Target Frobenius norm fraction for conv"), - io.Int.Input("linear_max_rank", default=128, min=1, max=1024, + io.Int.Input("linear_max_rank", default=128, min=1, max=3072, tooltip="Maximum rank for linear layers"), - io.Int.Input("conv_max_rank", default=128, min=1, max=1024, + io.Int.Input("conv_max_rank", default=128, min=1, max=3072, tooltip="Maximum rank for conv layers"), *_get_common_inputs(), ], @@ -853,7 +928,7 @@ class LoRAExtractFrobenius(io.ComfyNode): output_sd = extract_lora_from_files( model_a_path, model_b_path, "sv_fro", linear_target, conv_target, - "lora_unet_", device, save_dtype, linear_max_rank, conv_max_rank, + device, save_dtype, linear_max_rank, conv_max_rank, clamp_quantile, min_diff, skip_patterns, mismatch_mode, chunk_large_layers ) diff --git a/nodes/lora_merger.py b/nodes/lora_merger.py new file mode 100644 index 0000000..c439012 --- /dev/null +++ b/nodes/lora_merger.py @@ -0,0 +1,549 @@ +""" +LoRA Multi-Merge - Merge multiple LoRAs into a single LoRA file. +Resolves different naming conventions and ranks via concatenation. +""" +import os +import torch +import folder_paths +import comfy.utils +from tqdm import tqdm +from comfy_api.latest import io +from safetensors.torch import save_file +from typing import List, Dict, Tuple, Optional +from .merger_utils import MemoryEfficientSafeOpen, transfer_to_gpu_pinned +from .device_utils import estimate_model_size, prepare_for_large_operation, cleanup_after_operation +from .lora_resize import ( + detect_lora_format, + extract_lora_pairs, + detect_lora_rank, + _compile_patterns, + _matches_any_pattern, + _format_lora_key +) + +def merge_multi_loras( + lora_paths: List[str], + lora_weights: List[float], + merge_mode: str, + device: str, + save_dtype: torch.dtype, + output_filename: str, + verbose: bool = True, +) -> str: + """ + Merge multiple LoRAs into a single LoRA file. + + Modes: + - concatenate: Mathematically sound merge by stacking ranks. + New rank = sum(input ranks). Alpha set to new rank. + - weighted_sum: Weighted sum of A and B weights (Kohya style). + If ranks differ, smaller ones are zero-padded to match the largest rank. + New rank = max(input ranks). + """ + # Estimate memory + total_size_gb = sum(estimate_model_size(p) for p in lora_paths) + if verbose: + print(f"[LoRA Multi-Merge] Preparing memory for {total_size_gb:.2f}GB operation...") + print(f"[LoRA Multi-Merge] Mode: {merge_mode}") + + prepare_for_large_operation(total_size_gb * 2.5, torch.device(device)) + + handlers = [MemoryEfficientSafeOpen(p) for p in lora_paths] + + try: + # 1. Analyze all LoRAs + layer_map = {} + lora_infos = [] + + for i, handler in enumerate(handlers): + keys = handler.keys() + fmt = detect_lora_format(keys) + pairs = extract_lora_pairs(keys, fmt) + rank, alpha = detect_lora_rank(handler, pairs) + + info = { + "name": os.path.basename(lora_paths[i]), + "format": fmt, + "pairs": pairs, + "rank": rank, + "alpha": alpha, + "weight": lora_weights[i], + "scale": alpha / rank if rank > 0 else 1.0 + } + lora_infos.append(info) + + if verbose: + print(f"[LoRA Multi-Merge] LoRA {i+1} ({info['name']}): {len(pairs)} layers, dim={rank}, format={fmt['format']}") + + for block_name in pairs.keys(): + core = _format_lora_key(block_name) + if core not in layer_map: + layer_map[core] = [] + layer_map[core].append(i) + + if verbose: + print(f"[LoRA Multi-Merge] Total unique layers after resolving naming: {len(layer_map)}") + + output_sd = {} + pbar = comfy.utils.ProgressBar(len(layer_map)) + + # 2. Merge each core layer + with torch.no_grad(): + for core, indices in tqdm(layer_map.items(), desc="Merging layers", unit="layers"): + downs = [] + ups = [] + is_conv = False + max_rank = 0 + + # First pass: load and determine max rank + for idx in indices: + info = lora_infos[idx] + handler = handlers[idx] + orig_block = next(b for b in info["pairs"].keys() if _format_lora_key(b) == core) + block_keys = info["pairs"][orig_block] + + if "down" not in block_keys or "up" not in block_keys: + continue + + t_down = handler.get_tensor(block_keys["down"]) + t_up = handler.get_tensor(block_keys["up"]) + + if device == 'cuda': + t_down = transfer_to_gpu_pinned(t_down, device, torch.float32) + t_up = transfer_to_gpu_pinned(t_up, device, torch.float32) + else: + t_down = t_down.to(device=device, dtype=torch.float32) + t_up = t_up.to(device=device, dtype=torch.float32) + + # Store with original info for padding/weighting + is_conv = len(t_down.shape) == 4 + current_rank = t_down.shape[0] + max_rank = max(max_rank, current_rank) + + # Apply scale immediately for concatenate mode, or keep for later + if merge_mode == "concatenate": + effective_weight = info["weight"] * info["scale"] + t_up = t_up * effective_weight + + downs.append((t_down, info)) + ups.append((t_up, info)) + + if not downs: + pbar.update(1) + continue + + if merge_mode == "concatenate": + # Stack ranks + merged_down = torch.cat([d[0] for d in downs], dim=0) + merged_up = torch.cat([u[0] for u in ups], dim=1) + new_rank = merged_down.shape[0] + new_alpha = float(new_rank) + else: + # weighted_sum (Kohya style) + # Result = sum( weight_i * Padded(B_i) ), sum( weight_i * Padded(A_i) ) + # Note: We apply scale to weights here to normalize different alphas + merged_down = torch.zeros_like(downs[0][0]) + # Need to handle different ranks via padding + target_down_shape = list(downs[0][0].shape) + target_down_shape[0] = max_rank + target_up_shape = list(ups[0][0].shape) + target_up_shape[1] = max_rank + + merged_down = torch.zeros(target_down_shape, device=device, dtype=torch.float32) + merged_up = torch.zeros(target_up_shape, device=device, dtype=torch.float32) + + for (t_d, info_d), (t_u, info_u) in zip(downs, ups): + r = t_d.shape[0] + w = info_d["weight"] + # We also apply sqrt(scale) to both to distribute the scale factor? + # Kohya just uses weights. But to be safe with different alphas, + # we apply scale to the final delta. + # For direct weight merge, we'll just use weights. + + if r < max_rank: + # Pad down: [r, in...] -> [max_rank, in...] + pad_d = [0] * (len(t_d.shape) * 2) + pad_d[-1] = max_rank - r # last dim in pad is first dim in tensor (reversed) + # Wait, F.pad uses reverse order of dims. + # For [r, in], padding is (0,0, 0, max_rank-r) + padding_d = [0, 0] * (len(t_d.shape) - 1) + [0, max_rank - r] + t_d = torch.nn.functional.pad(t_d, tuple(padding_d)) + + # Pad up: [out, r...] -> [out, max_rank...] + # For [out, r], padding is (0, max_rank-r, 0, 0) + padding_u = [0, max_rank - r] + [0, 0] * (len(t_u.shape) - 1) + t_u = torch.nn.functional.pad(t_u, tuple(padding_u)) + + merged_down += w * t_d + merged_up += w * t_u + + new_rank = max_rank + # Alpha is usually max of alphas or same as rank + new_alpha = float(max_rank) + + # Store + output_sd[f"{core}.lora_down.weight"] = merged_down.to(save_dtype).cpu().contiguous() + output_sd[f"{core}.lora_up.weight"] = merged_up.to(save_dtype).cpu().contiguous() + output_sd[f"{core}.alpha"] = torch.tensor(new_alpha, dtype=save_dtype) + + # Cleanup + del merged_down, merged_up + for d, _ in downs: del d + for u, _ in ups: del u + + pbar.update(1) + + # 3. Final Summary + if verbose: + matched_stats = {i: 0 for i in range(len(lora_paths))} + for indices in layer_map.values(): + for idx in indices: + matched_stats[idx] += 1 + + print(f"[LoRA Multi-Merge] --- Merge Summary ---") + for i, info in enumerate(lora_infos): + print(f"[LoRA Multi-Merge] LoRA {i+1}: {matched_stats[i]}/{len(info['pairs'])} layers used in merge") + print(f"[LoRA Multi-Merge] Output state dict has {len(output_sd)} tensors") + + # 4. Metadata + metadata = { + "ss_training_comment": f"Merged {len(lora_paths)} LoRAs via concatenation", + "ss_network_module": "networks.lora", + } + + # 4. Save + output_dir = folder_paths.get_folder_paths("loras")[0] + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors") + + save_file(output_sd, output_path, metadata) + if verbose: + print(f"[LoRA Multi-Merge] Saved merged LoRA to {output_path}") + + return output_path + + finally: + for h in handlers: + h.__exit__(None, None, None) + cleanup_after_operation() + + +class LoRAMultiMerge(io.ComfyNode): + """Merge multiple LoRAs into a single LoRA file.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoRAMultiMerge", + display_name="LoRA Multi-Merge", + category="ModelUtils/LoRA/Merge", + description="Merge 1-8 LoRAs into a single LoRA file. This resolves different ranks and naming conventions.", + inputs=[ + io.Combo.Input("lora_count", options=[str(i) for i in range(1, 9)], default="2", + tooltip="Number of LoRAs to merge"), + # LoRA 1 + io.Combo.Input("lora_1", options=folder_paths.get_filename_list("loras"), tooltip="First LoRA"), + io.Float.Input("weight_1", default=1.0, min=-10.0, max=10.0, step=0.01), + # LoRA 2 + io.Combo.Input("lora_2", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_2", default=1.0, min=-10.0, max=10.0, step=0.01), + # LoRA 3 + io.Combo.Input("lora_3", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_3", default=1.0, min=-10.0, max=10.0, step=0.01), + # LoRA 4 + io.Combo.Input("lora_4", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_4", default=1.0, min=-10.0, max=10.0, step=0.01), + # LoRA 5 + io.Combo.Input("lora_5", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_5", default=1.0, min=-10.0, max=10.0, step=0.01), + # LoRA 6 + io.Combo.Input("lora_6", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_6", default=1.0, min=-10.0, max=10.0, step=0.01), + # LoRA 7 + io.Combo.Input("lora_7", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_7", default=1.0, min=-10.0, max=10.0, step=0.01), + # LoRA 8 + io.Combo.Input("lora_8", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_8", default=1.0, min=-10.0, max=10.0, step=0.01), + + io.Combo.Input("merge_mode", options=["concatenate", "weighted_sum"], default="concatenate", + tooltip="concatenate: safe, increases rank. weighted_sum: fixed rank (to max input rank), mathematically lossy but standard in Kohya."), + io.String.Input("output_filename", default="merged_lora"), + io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"), + io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"), + ], + outputs=[io.String.Output(display_name="output_path")], + is_output_node=True, + ) + + @classmethod + def execute(cls, lora_count, + lora_1, weight_1, lora_2, weight_2, lora_3, weight_3, lora_4, weight_4, + lora_5, weight_5, lora_6, weight_6, lora_7, weight_7, lora_8, weight_8, + merge_mode, output_filename, save_dtype, device) -> io.NodeOutput: + + count = int(lora_count) + names = [lora_1, lora_2, lora_3, lora_4, lora_5, lora_6, lora_7, lora_8] + weights = [weight_1, weight_2, weight_3, weight_4, weight_5, weight_6, weight_7, weight_8] + + valid_paths = [] + valid_weights = [] + + for i in range(count): + if names[i] and names[i] != "None": + path = folder_paths.get_full_path_or_raise("loras", names[i]) + valid_paths.append(path) + valid_weights.append(weights[i]) + + if not valid_paths: + raise ValueError("No LoRAs selected for merging.") + + dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype] + + path = merge_multi_loras( + lora_paths=valid_paths, + lora_weights=valid_weights, + merge_mode=merge_mode, + device=device, + save_dtype=dtype, + output_filename=output_filename + ) + + return io.NodeOutput(path) + + +class LoRAMultiMergeDARE(io.ComfyNode): + """Merge multiple LoRAs into a single LoRA file using DARE-Ties method.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LoRAMultiMergeDARE", + display_name="LoRA Multi-Merge (DARE-Ties)", + category="ModelUtils/LoRA/Merge", + description="Merge 1-8 LoRAs into a single LoRA file using DARE-Ties. Drops small values and resolves sign conflicts.", + inputs=[ + io.Combo.Input("lora_count", options=[str(i) for i in range(1, 9)], default="2"), + # LoRA 1 + io.Combo.Input("lora_1", options=folder_paths.get_filename_list("loras"), tooltip="First LoRA"), + io.Float.Input("weight_1", default=1.0, min=-10.0, max=10.0, step=0.01), + # ... + io.Combo.Input("lora_2", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_2", default=1.0, min=-10.0, max=10.0, step=0.01), + io.Combo.Input("lora_3", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_3", default=1.0, min=-10.0, max=10.0, step=0.01), + io.Combo.Input("lora_4", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_4", default=1.0, min=-10.0, max=10.0, step=0.01), + io.Combo.Input("lora_5", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_5", default=1.0, min=-10.0, max=10.0, step=0.01), + io.Combo.Input("lora_6", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_6", default=1.0, min=-10.0, max=10.0, step=0.01), + io.Combo.Input("lora_7", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_7", default=1.0, min=-10.0, max=10.0, step=0.01), + io.Combo.Input("lora_8", options=["None"] + folder_paths.get_filename_list("loras"), default="None"), + io.Float.Input("weight_8", default=1.0, min=-10.0, max=10.0, step=0.01), + + io.Float.Input("drop_rate", default=0.1, min=0.0, max=1.0, step=0.01, tooltip="DARE drop rate"), + io.Float.Input("trim_quantile", default=0.2, min=0.0, max=1.0, step=0.01, tooltip="TIES trim quantile (drops smallest values)"), + io.Int.Input("seed", default=42, min=0, max=0xffffffffffffffff), + io.String.Input("output_filename", default="merged_lora_dare"), + io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"), + io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"), + ], + outputs=[io.String.Output(display_name="output_path")], + is_output_node=True, + ) + + @classmethod + def execute(cls, lora_count, + lora_1, weight_1, lora_2, weight_2, lora_3, weight_3, lora_4, weight_4, + lora_5, weight_5, lora_6, weight_6, lora_7, weight_7, lora_8, weight_8, + drop_rate, trim_quantile, seed, output_filename, save_dtype, device) -> io.NodeOutput: + + count = int(lora_count) + names = [lora_1, lora_2, lora_3, lora_4, lora_5, lora_6, lora_7, lora_8] + weights = [weight_1, weight_2, weight_3, weight_4, weight_5, weight_6, weight_7, weight_8] + + valid_paths = [] + valid_weights = [] + + for i in range(count): + if names[i] and names[i] != "None": + path = folder_paths.get_full_path_or_raise("loras", names[i]) + valid_paths.append(path) + valid_weights.append(weights[i]) + + if not valid_paths: + raise ValueError("No LoRAs selected for merging.") + + dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype] + + path = merge_multi_loras_dare( + lora_paths=valid_paths, + lora_weights=valid_weights, + drop_rate=drop_rate, + trim_quantile=trim_quantile, + seed=seed, + device=device, + save_dtype=dtype, + output_filename=output_filename + ) + + return io.NodeOutput(path) + +def merge_multi_loras_dare( + lora_paths: List[str], + lora_weights: List[float], + drop_rate: float, + trim_quantile: float, + seed: int, + device: str, + save_dtype: torch.dtype, + output_filename: str, + verbose: bool = True, +) -> str: + """ + Merge multiple LoRAs using DARE-Ties method applied to weights. + Different ranks are handled via zero-padding to the max rank. + """ + total_size_gb = sum(estimate_model_size(p) for p in lora_paths) + if verbose: + print(f"[LoRA Multi-Merge DARE] Preparing memory for {total_size_gb:.2f}GB operation...") + print(f"[LoRA Multi-Merge DARE] Drop rate: {drop_rate}, Trim quantile: {trim_quantile}") + + prepare_for_large_operation(total_size_gb * 2.5, torch.device(device)) + handlers = [MemoryEfficientSafeOpen(p) for p in lora_paths] + + rng = torch.Generator(device=device).manual_seed(seed) + + try: + # 1. Analyze + layer_map = {} + lora_infos = [] + for i, handler in enumerate(handlers): + keys = handler.keys() + fmt = detect_lora_format(keys) + pairs = extract_lora_pairs(keys, fmt) + rank, alpha = detect_lora_rank(handler, pairs) + info = { + "name": os.path.basename(lora_paths[i]), + "pairs": pairs, + "rank": rank, + "alpha": alpha, + "weight": lora_weights[i] + } + lora_infos.append(info) + + if verbose: + print(f"[LoRA Multi-Merge DARE] LoRA {i+1} ({info['name']}): {len(pairs)} layers, dim={rank}") + + for block_name in pairs.keys(): + core = _format_lora_key(block_name) + if core not in layer_map: layer_map[core] = [] + layer_map[core].append(i) + + if verbose: + print(f"[LoRA Multi-Merge DARE] Total unique layers after resolving naming: {len(layer_map)}") + + output_sd = {} + pbar = comfy.utils.ProgressBar(len(layer_map)) + + # 2. Merge + with torch.no_grad(): + for core, indices in tqdm(layer_map.items(), desc="Merging layers (DARE)", unit="layers"): + downs = [] + ups = [] + max_rank = 0 + + for idx in indices: + info = lora_infos[idx] + orig_block = next(b for b in info["pairs"].keys() if _format_lora_key(b) == core) + block_keys = info["pairs"][orig_block] + if "down" not in block_keys or "up" not in block_keys: continue + + t_d = handlers[idx].get_tensor(block_keys["down"]).to(device=device, dtype=torch.float32) + t_u = handlers[idx].get_tensor(block_keys["up"]).to(device=device, dtype=torch.float32) + + max_rank = max(max_rank, t_d.shape[0]) + downs.append((t_d, info["weight"])) + ups.append((t_u, info["weight"])) + + if not downs: + pbar.update(1) + continue + + def process_ties_dare(tensors, weights, dim_to_pad): + # Pad all to max_rank + padded = [] + for t, w in tensors: + if t.shape[dim_to_pad] < max_rank: + padding = [0] * (len(t.shape) * 2) + # dim_to_pad 0 (down) -> last pair in pad + # dim_to_pad 1 (up) -> second to last pair in pad + rev_dim = len(t.shape) - 1 - dim_to_pad + padding[rev_dim*2 + 1] = max_rank - t.shape[dim_to_pad] + t = torch.nn.functional.pad(t, tuple(padding)) + padded.append(t * w) + + # DARE + if drop_rate > 0: + for i in range(len(padded)): + mask = (torch.rand(padded[i].shape, generator=rng, device=device) > drop_rate).float() + padded[i] = (padded[i] * mask) / (1 - drop_rate) + + # TIES + # 1. Trim + if trim_quantile > 0: + for i in range(len(padded)): + flat = padded[i].abs().flatten() + k = int(len(flat) * trim_quantile) + if k > 0: + threshold = torch.kthvalue(flat, k).values + padded[i] = torch.where(padded[i].abs() < threshold, torch.zeros_like(padded[i]), padded[i]) + + # 2. Elect & Merge + stacked = torch.stack(padded) # [N, ...] + signs = torch.sign(stacked) + sum_signs = signs.sum(dim=0) + dominant_sign = torch.sign(sum_signs) + + # Filter those matching dominant sign + mask = (signs == dominant_sign) & (dominant_sign != 0) + filtered = torch.where(mask, stacked, torch.zeros_like(stacked)) + + # Average matching signs + count = mask.sum(dim=0) + result = filtered.sum(dim=0) / torch.clamp(count, min=1.0) + return result + + merged_down = process_ties_dare(downs, [1.0]*len(downs), 0) + merged_up = process_ties_dare(ups, [1.0]*len(ups), 1) + + output_sd[f"{core}.lora_down.weight"] = merged_down.to(save_dtype).cpu().contiguous() + output_sd[f"{core}.lora_up.weight"] = merged_up.to(save_dtype).cpu().contiguous() + output_sd[f"{core}.alpha"] = torch.tensor(float(max_rank), dtype=save_dtype) + + pbar.update(1) + + # Final Summary + if verbose: + matched_stats = {i: 0 for i in range(len(lora_paths))} + for indices in layer_map.values(): + for idx in indices: + matched_stats[idx] += 1 + + print(f"[LoRA Multi-Merge DARE] --- Merge Summary ---") + for i, info in enumerate(lora_infos): + print(f"[LoRA Multi-Merge DARE] LoRA {i+1}: {matched_stats[i]}/{len(info['pairs'])} layers used in merge") + print(f"[LoRA Multi-Merge DARE] Output state dict has {len(output_sd)} tensors") + + # Save + output_dir = folder_paths.get_folder_paths("loras")[0] + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors") + save_file(output_sd, output_path, {"ss_training_comment": "Merged via DARE-Ties"}) + return output_path + + finally: + for h in handlers: h.__exit__(None, None, None) + cleanup_after_operation() diff --git a/nodes/lora_resize.py b/nodes/lora_resize.py index b698a78..08c7157 100644 --- a/nodes/lora_resize.py +++ b/nodes/lora_resize.py @@ -26,6 +26,7 @@ from .lora_extract_svd import ( _svd_extract_conv, _compile_patterns, _matches_any_pattern, + _format_lora_key, ) @@ -452,9 +453,13 @@ def resize_lora_file( up_suffix = format_info["up_suffix"] alpha_suffix = format_info["alpha_suffix"] - output_sd[f"{block_name}{down_suffix}"] = result["lora_down"].to(save_dtype) - output_sd[f"{block_name}{up_suffix}"] = result["lora_up"].to(save_dtype) - output_sd[f"{block_name}{alpha_suffix}"] = torch.tensor([result["new_alpha"]]).to(save_dtype) + # Standardize prefix using heuristics if possible + new_block_name = _format_lora_key(block_name) + + output_sd[f"{new_block_name}{down_suffix}"] = result["lora_down"].to(save_dtype) + output_sd[f"{new_block_name}{up_suffix}"] = result["lora_up"].to(save_dtype) + if alpha_suffix: + output_sd[f"{new_block_name}{alpha_suffix}"] = torch.tensor(result["new_alpha"]).to(save_dtype) pbar.update(1) @@ -506,7 +511,7 @@ class LoRAResizeFixed(io.ComfyNode): inputs=[ io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"), tooltip="LoRA to resize"), - io.Int.Input("new_rank", default=64, min=1, max=1024, + io.Int.Input("new_rank", default=64, min=1, max=3072, tooltip="Target rank"), io.Int.Input("svd_niter", default=2, min=0, max=10, tooltip="SVD power iterations (higher = more accurate but slower)"), @@ -540,7 +545,7 @@ class LoRAResizeRatio(io.ComfyNode): inputs=[ io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"), tooltip="LoRA to resize"), - io.Int.Input("max_rank", default=128, min=1, max=1024, + io.Int.Input("max_rank", default=128, min=1, max=3072, tooltip="Maximum allowed rank"), io.Float.Input("ratio", default=2.0, min=1.0, max=100.0, step=0.1, tooltip="Keep SVs where S[i] > S[0]/ratio"), @@ -574,7 +579,7 @@ class LoRAResizeFrobenius(io.ComfyNode): inputs=[ io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"), tooltip="LoRA to resize"), - io.Int.Input("max_rank", default=128, min=1, max=1024, + io.Int.Input("max_rank", default=128, min=1, max=3072, tooltip="Maximum allowed rank"), io.Float.Input("target", default=0.9, min=0.1, max=1.0, step=0.01, tooltip="Target Frobenius norm retention (0.9 = 90%)"), @@ -608,7 +613,7 @@ class LoRAResizeCumulative(io.ComfyNode): inputs=[ io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"), tooltip="LoRA to resize"), - io.Int.Input("max_rank", default=128, min=1, max=1024, + io.Int.Input("max_rank", default=128, min=1, max=3072, tooltip="Maximum allowed rank"), io.Float.Input("target", default=0.9, min=0.1, max=1.0, step=0.01, tooltip="Target cumulative SV retention (0.9 = 90%)"), @@ -629,956 +634,6 @@ class LoRAResizeCumulative(io.ComfyNode): return io.NodeOutput(path) -# ============================================================================= -# Fast Resize via Base Model - Uses extraction code path for ~91x speedup -# ============================================================================= - -def resize_lora_via_base( - lora_path: str, - base_model_path: str, - mode: str, - linear_param: float, - conv_param: float, - device: str, - save_dtype: torch.dtype, - output_filename: str, - linear_max_rank: int = None, - conv_max_rank: int = None, - clamp_quantile: float = 0.99, - mismatch_mode: str = "skip", - skip_patterns_str: str = "", - svd_niter: int = 2, - verbose: bool = True, -) -> str: - """ - Resize LoRA by merging to base model weights and re-extracting. - - This uses the fast extraction code path for ~91x speed improvement over - the original merge-then-SVD approach. - - Args: - lora_path: Path to input LoRA - base_model_path: Path to base model the LoRA was trained on - mode: "fixed", "ratio", "sv_fro", "sv_cumulative" - linear_param: Rank for fixed mode, or mode parameter for dynamic - conv_param: Same for conv layers - device: Processing device - save_dtype: Output dtype - output_filename: Output filename (without extension) - linear_max_rank: Max rank for linear layers (dynamic modes) - conv_max_rank: Max rank for conv layers (dynamic modes) - clamp_quantile: Quantile for weight clamping - mismatch_mode: How to handle mismatches ("skip", "zeros", "error") - skip_patterns_str: Regex patterns for layers to skip - svd_niter: Power iterations for SVD accuracy - verbose: Print progress info - - Returns: - Path to saved resized LoRA - """ - # Prepare memory - lora_size_gb = estimate_model_size(lora_path) - base_size_gb = estimate_model_size(base_model_path) - total_size_gb = lora_size_gb + base_size_gb - - if verbose: - print(f"[LoRA Resize Fast] Preparing memory for {total_size_gb:.2f}GB operation...") - prepare_for_large_operation(total_size_gb * 1.5, torch.device(device)) - - # Open files with memory-efficient handlers - lora_handler = MemoryEfficientSafeOpen(lora_path) - base_handler = MemoryEfficientSafeOpen(base_model_path) - - try: - lora_metadata = lora_handler.metadata().copy() - lora_keys = lora_handler.keys() - base_keys = set(base_handler.keys()) - - # Detect LoRA format - format_info = detect_lora_format(lora_keys) - lora_pairs = extract_lora_pairs(lora_keys, format_info) - network_dim, network_alpha = detect_lora_rank(lora_handler, lora_pairs) - scale = network_alpha / network_dim if network_dim > 0 else 1.0 - - if verbose: - print(f"[LoRA Resize Fast] Format: {format_info['format']}, layers: {format_info['key_count']}") - print(f"[LoRA Resize Fast] Original dim={network_dim}, alpha={network_alpha:.1f}") - print(f"[LoRA Resize Fast] Mode: {mode}, linear_param={linear_param}, conv_param={conv_param}") - - # Compile skip patterns - skip_patterns = _compile_patterns(skip_patterns_str) - - # Build mapping from base model keys to LoRA block names - # LoRA keys are like: lora_unet_transformer_blocks_0_attn_to_q.lora_down.weight - # Base keys are like: transformer_blocks.0.attn.to_q.weight - down_suffix = format_info["down_suffix"] - up_suffix = format_info["up_suffix"] - alpha_suffix = format_info["alpha_suffix"] - - output_sd = {} - stats = {"extracted": 0, "skipped": 0, "no_match": 0, "full": 0} - - # Common prefixes to strip from base model keys - BASE_PREFIXES = ["model.diffusion_model.", "diffusion_model.", "transformer.", "model."] - - # Common prefixes to strip from LoRA block names (order matters - longer prefixes first) - LORA_PREFIXES = [ - "lora_unet_", "lora_transformer_", "lora_te1_", "lora_te2_", "lora_te_", - "lycoris_", "diffusion_model.", "transformer.", "unet." - ] - - def extract_core_layer_base(key: str) -> str: - """Extract core layer name from base model key (keeps dots).""" - # Strip .weight/.bias suffix - if key.endswith(".weight"): - key = key[:-7] - elif key.endswith(".bias"): - key = key[:-5] - # Strip common prefixes - for prefix in BASE_PREFIXES: - if key.startswith(prefix): - key = key[len(prefix):] - break - return key - - def extract_core_layer_lora(block_name: str) -> str: - """Extract core layer name from LoRA block name and normalize to underscored format.""" - # Strip common prefixes - for prefix in LORA_PREFIXES: - if block_name.startswith(prefix): - block_name = block_name[len(prefix):] - break - # Strip trailing .lora suffix (HuggingFace format leaves this after suffix removal) - if block_name.endswith(".lora"): - block_name = block_name[:-5] - # Convert dots to underscores (diffusers format uses dots) - return block_name.replace(".", "_") - - - # Build lookup table: core layer name (underscored) -> original base key - base_key_lookup = {} - for bk in base_handler.keys(): - if bk.endswith(".weight"): - core = extract_core_layer_base(bk) - # Convert dots to underscores for matching with LoRA format - core_underscored = core.replace(".", "_") - base_key_lookup[core_underscored] = bk - - if verbose: - print(f"[LoRA Resize Fast] Base model has {len(base_key_lookup)} weight keys") - # Debug: show first 3 samples from each side with their normalized forms - sample_lora = list(lora_pairs.keys())[:3] - sample_lora_normalized = [extract_core_layer_lora(b) for b in sample_lora] - sample_base = list(base_key_lookup.items())[:3] - print(f"[DEBUG] Sample LoRA blocks: {sample_lora}") - print(f"[DEBUG] Sample LoRA normalized: {sample_lora_normalized}") - print(f"[DEBUG] Sample base (underscored->original): {sample_base}") - - pbar = comfy.utils.ProgressBar(len(lora_pairs)) - - with torch.no_grad(): - for block_name, block_keys in tqdm(lora_pairs.items(), desc="Resizing via extraction", unit="layers"): - if "down" not in block_keys or "up" not in block_keys: - pbar.update(1) - continue - - # Check skip patterns on the LoRA block name - if _matches_any_pattern(block_name, skip_patterns): - stats["skipped"] += 1 - pbar.update(1) - continue - - # Extract core layer name from LoRA block (already underscored) - core_lora = extract_core_layer_lora(block_name) - - # Look up the base model key - if core_lora not in base_key_lookup: - if verbose and stats["no_match"] < 3: - print(f"[DEBUG] No match for LoRA core: {core_lora}") - stats["no_match"] += 1 - pbar.update(1) - continue - - base_key = base_key_lookup[core_lora] - - # Load tensors with pinned memory for CUDA - # Load base weight to RAM first - cpu_base = base_handler.get_tensor(base_key) - - # Transfer to GPU if cuda selected - if device == 'cuda': - base_weight = transfer_to_gpu_pinned(cpu_base, device, torch.float32) - else: - base_weight = cpu_base.to(device=device, dtype=torch.float32) - del cpu_base - - # Load LoRA weights - cpu_down = lora_handler.get_tensor(block_keys["down"]) - cpu_up = lora_handler.get_tensor(block_keys["up"]) - if device == 'cuda': - lora_down = transfer_to_gpu_pinned(cpu_down, device, torch.float32) - lora_up = transfer_to_gpu_pinned(cpu_up, device, torch.float32) - else: - lora_down = cpu_down.to(device=device, dtype=torch.float32) - lora_up = cpu_up.to(device=device, dtype=torch.float32) - del cpu_down, cpu_up - - # Get alpha for this layer - if "alpha" in block_keys: - alpha_tensor = lora_handler.get_tensor(block_keys["alpha"]) - layer_alpha = float(alpha_tensor.item()) - else: - layer_alpha = float(network_dim) - layer_scale = layer_alpha / network_dim if network_dim > 0 else 1.0 - - # Compute LoRA delta: scale * (up @ down) - is_conv = len(lora_down.shape) == 4 - - if is_conv: - # Conv merge: scale * (up @ down reshaped) - in_rank, in_size, kernel_size, k_ = lora_down.shape - out_size, out_rank, _, _ = lora_up.shape - lora_delta = lora_up.reshape(out_size, -1) @ lora_down.reshape(in_rank, -1) - lora_delta = lora_delta.reshape(out_size, in_size, kernel_size, kernel_size) - else: - # Linear merge: scale * (up @ down) - lora_delta = lora_up @ lora_down - del lora_down, lora_up - - # Apply scale and merge into base to get finetuned weight - finetuned_weight = base_weight + layer_scale * lora_delta - del lora_delta - - # Subtract base to get the weight difference for extraction - weight_diff = finetuned_weight - base_weight - del base_weight, finetuned_weight - - # Extract new LoRA from the weight difference - if is_conv: - result, mode_str = _svd_extract_conv( - weight_diff, mode, conv_param, device, conv_max_rank, clamp_quantile - ) - else: - result, mode_str = _svd_extract_linear( - weight_diff, mode, linear_param, device, linear_max_rank, clamp_quantile, svd_niter - ) - - # Cleanup GPU tensors - del weight_diff - - if mode_str == "full": - # Rank too high, store as full diff (rare) - stats["full"] += 1 - pbar.update(1) - continue - - new_lora_down, new_lora_up, _ = result - new_rank = new_lora_down.shape[0] - new_alpha = float(new_rank) # Standard: alpha = rank for 1.0 scale - - # Transfer extracted LoRA tensors back to RAM until saving - output_sd[f"{block_name}{down_suffix}"] = new_lora_down.to(save_dtype).cpu().contiguous() - output_sd[f"{block_name}{up_suffix}"] = new_lora_up.to(save_dtype).cpu().contiguous() - output_sd[f"{block_name}{alpha_suffix}"] = torch.tensor([new_alpha]).to(save_dtype) - - # Clear remaining tensors from this layer - del new_lora_down, new_lora_up, result - - stats["extracted"] += 1 - pbar.update(1) - - if verbose: - print(f"[LoRA Resize Fast] Done: {stats['extracted']} extracted, {stats['skipped']} skipped, {stats['full']} full") - - if not output_sd: - raise ValueError("No LoRA weights extracted. Check that the base model matches the LoRA.") - - # Update metadata - lora_metadata["ss_training_comment"] = f"Resized via base model extraction, mode={mode}" - if mode == "fixed": - lora_metadata["ss_network_dim"] = str(int(linear_param)) - lora_metadata["ss_network_alpha"] = str(int(linear_param)) - else: - lora_metadata["ss_network_dim"] = "Dynamic" - lora_metadata["ss_network_alpha"] = "Dynamic" - - # Save - output_dir = folder_paths.get_folder_paths("loras")[0] - os.makedirs(output_dir, exist_ok=True) - output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors") - - save_file(output_sd, output_path, lora_metadata) - print(f"[LoRA Resize Fast] Saved to {output_path}") - - return output_path - - finally: - lora_handler.__exit__(None, None, None) - base_handler.__exit__(None, None, None) - cleanup_after_operation() - - -# ============================================================================= -# Fast Resize Node Definitions -# ============================================================================= - -class LoRAResizeViaBaseFixed(io.ComfyNode): - """Fast LoRA resize using base model + extraction path.""" - - @classmethod - def define_schema(cls): - return io.Schema( - node_id="LoRAResizeViaBaseFixed", - display_name="LoRA Resize Fast (Fixed Rank)", - category="ModelUtils/LoRA/Resize", - description="Fast LoRA resize by merging to base model and re-extracting. ~91x faster than standard resize.", - inputs=[ - io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"), - tooltip="LoRA to resize"), - io.Combo.Input("base_model", options=folder_paths.get_filename_list("diffusion_models"), - tooltip="Base model the LoRA was trained on"), - io.Int.Input("linear_dim", default=64, min=1, max=4096, - tooltip="Target rank for linear/attention layers"), - io.Int.Input("conv_dim", default=32, min=1, max=4096, - tooltip="Target rank for conv layers"), - io.Int.Input("svd_niter", default=2, min=0, max=10, - tooltip="SVD power iterations (higher = more accurate)"), - io.Float.Input("clamp_quantile", default=0.99, min=0.5, max=1.0, step=0.01, - tooltip="Clamp outlier singular values"), - io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip", - tooltip="How to handle key mismatches"), - io.String.Input("skip_patterns", default="", multiline=True, - tooltip="Regex patterns for layers to skip"), - io.String.Input("output_filename", default="resized_lora_fast"), - io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"), - io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"), - ], - outputs=[io.String.Output(display_name="output_path")], - is_output_node=True, - ) - - @classmethod - def execute(cls, lora_name, base_model, linear_dim, conv_dim, svd_niter, - clamp_quantile, mismatch_mode, skip_patterns, - output_filename, save_dtype, device) -> io.NodeOutput: - lora_path = folder_paths.get_full_path_or_raise("loras", lora_name) - base_path = folder_paths.get_full_path_or_raise("diffusion_models", base_model) - dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype] - - path = resize_lora_via_base( - lora_path=lora_path, - base_model_path=base_path, - mode="fixed", - linear_param=linear_dim, - conv_param=conv_dim, - device=device, - save_dtype=dtype, - output_filename=output_filename, - linear_max_rank=linear_dim, - conv_max_rank=conv_dim, - clamp_quantile=clamp_quantile, - mismatch_mode=mismatch_mode, - skip_patterns_str=skip_patterns, - svd_niter=svd_niter, - ) - return io.NodeOutput(path) - - -class LoRAResizeViaBaseRatio(io.ComfyNode): - """Fast LoRA resize keeping SVs above ratio threshold.""" - - @classmethod - def define_schema(cls): - return io.Schema( - node_id="LoRAResizeViaBaseRatio", - display_name="LoRA Resize Fast (SV Ratio)", - category="ModelUtils/LoRA/Resize", - description="Fast dynamic resize keeping singular values where S[i] > S[0]/ratio.", - inputs=[ - io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"), - tooltip="LoRA to resize"), - io.Combo.Input("base_model", options=folder_paths.get_filename_list("diffusion_models"), - tooltip="Base model the LoRA was trained on"), - io.Int.Input("linear_max_rank", default=128, min=1, max=1024, - tooltip="Maximum rank for linear layers"), - io.Int.Input("conv_max_rank", default=128, min=1, max=1024, - tooltip="Maximum rank for conv layers"), - io.Float.Input("ratio", default=2.0, min=1.0, max=100.0, step=0.1, - tooltip="Keep SVs where S[i] > S[0]/ratio"), - io.Float.Input("clamp_quantile", default=0.99, min=0.5, max=1.0, step=0.01, - tooltip="Clamp outlier singular values"), - io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"), - io.String.Input("skip_patterns", default="", multiline=True, - tooltip="Regex patterns for layers to skip"), - io.String.Input("output_filename", default="resized_lora_ratio_fast"), - io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"), - io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"), - ], - outputs=[io.String.Output(display_name="output_path")], - is_output_node=True, - ) - - @classmethod - def execute(cls, lora_name, base_model, linear_max_rank, conv_max_rank, ratio, - clamp_quantile, mismatch_mode, skip_patterns, - output_filename, save_dtype, device) -> io.NodeOutput: - lora_path = folder_paths.get_full_path_or_raise("loras", lora_name) - base_path = folder_paths.get_full_path_or_raise("diffusion_models", base_model) - dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype] - - path = resize_lora_via_base( - lora_path=lora_path, - base_model_path=base_path, - mode="ratio", - linear_param=ratio, - conv_param=ratio, - device=device, - save_dtype=dtype, - output_filename=output_filename, - linear_max_rank=linear_max_rank, - conv_max_rank=conv_max_rank, - clamp_quantile=clamp_quantile, - mismatch_mode=mismatch_mode, - skip_patterns_str=skip_patterns, - ) - return io.NodeOutput(path) - - -class LoRAResizeViaBaseFrobenius(io.ComfyNode): - """Fast LoRA resize preserving Frobenius norm target.""" - - @classmethod - def define_schema(cls): - return io.Schema( - node_id="LoRAResizeViaBaseFrobenius", - display_name="LoRA Resize Fast (Frobenius)", - category="ModelUtils/LoRA/Resize", - description="Fast dynamic resize preserving target fraction of Frobenius norm.", - inputs=[ - io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"), - tooltip="LoRA to resize"), - io.Combo.Input("base_model", options=folder_paths.get_filename_list("diffusion_models"), - tooltip="Base model the LoRA was trained on"), - io.Int.Input("linear_max_rank", default=128, min=1, max=1024, - tooltip="Maximum rank for linear layers"), - io.Int.Input("conv_max_rank", default=128, min=1, max=1024, - tooltip="Maximum rank for conv layers"), - io.Float.Input("target", default=0.9, min=0.1, max=1.0, step=0.01, - tooltip="Target Frobenius norm retention (0.9 = 90%)"), - io.Float.Input("clamp_quantile", default=0.99, min=0.5, max=1.0, step=0.01, - tooltip="Clamp outlier singular values"), - io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"), - io.String.Input("skip_patterns", default="", multiline=True, - tooltip="Regex patterns for layers to skip"), - io.String.Input("output_filename", default="resized_lora_fro_fast"), - io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"), - io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"), - ], - outputs=[io.String.Output(display_name="output_path")], - is_output_node=True, - ) - - @classmethod - def execute(cls, lora_name, base_model, linear_max_rank, conv_max_rank, target, - clamp_quantile, mismatch_mode, skip_patterns, - output_filename, save_dtype, device) -> io.NodeOutput: - lora_path = folder_paths.get_full_path_or_raise("loras", lora_name) - base_path = folder_paths.get_full_path_or_raise("diffusion_models", base_model) - dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype] - - path = resize_lora_via_base( - lora_path=lora_path, - base_model_path=base_path, - mode="sv_fro", - linear_param=target, - conv_param=target, - device=device, - save_dtype=dtype, - output_filename=output_filename, - linear_max_rank=linear_max_rank, - conv_max_rank=conv_max_rank, - clamp_quantile=clamp_quantile, - mismatch_mode=mismatch_mode, - skip_patterns_str=skip_patterns, - ) - return io.NodeOutput(path) - - -class LoRAResizeViaBaseCumulative(io.ComfyNode): - """Fast LoRA resize preserving cumulative SV target.""" - - @classmethod - def define_schema(cls): - return io.Schema( - node_id="LoRAResizeViaBaseCumulative", - display_name="LoRA Resize Fast (Cumulative)", - category="ModelUtils/LoRA/Resize", - description="Fast dynamic resize preserving target fraction of cumulative singular values.", - inputs=[ - io.Combo.Input("lora_name", options=folder_paths.get_filename_list("loras"), - tooltip="LoRA to resize"), - io.Combo.Input("base_model", options=folder_paths.get_filename_list("diffusion_models"), - tooltip="Base model the LoRA was trained on"), - io.Int.Input("linear_max_rank", default=128, min=1, max=1024, - tooltip="Maximum rank for linear layers"), - io.Int.Input("conv_max_rank", default=128, min=1, max=1024, - tooltip="Maximum rank for conv layers"), - io.Float.Input("target", default=0.9, min=0.1, max=1.0, step=0.01, - tooltip="Target cumulative SV retention (0.9 = 90%)"), - io.Float.Input("clamp_quantile", default=0.99, min=0.5, max=1.0, step=0.01, - tooltip="Clamp outlier singular values"), - io.Combo.Input("mismatch_mode", options=["skip", "zeros", "error"], default="skip"), - io.String.Input("skip_patterns", default="", multiline=True, - tooltip="Regex patterns for layers to skip"), - io.String.Input("output_filename", default="resized_lora_cumulative_fast"), - io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"), - io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"), - ], - outputs=[io.String.Output(display_name="output_path")], - is_output_node=True, - ) - - @classmethod - def execute(cls, lora_name, base_model, linear_max_rank, conv_max_rank, target, - clamp_quantile, mismatch_mode, skip_patterns, - output_filename, save_dtype, device) -> io.NodeOutput: - lora_path = folder_paths.get_full_path_or_raise("loras", lora_name) - base_path = folder_paths.get_full_path_or_raise("diffusion_models", base_model) - dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype] - - path = resize_lora_via_base( - lora_path=lora_path, - base_model_path=base_path, - mode="sv_cumulative", - linear_param=target, - conv_param=target, - device=device, - save_dtype=dtype, - output_filename=output_filename, - linear_max_rank=linear_max_rank, - conv_max_rank=conv_max_rank, - clamp_quantile=clamp_quantile, - mismatch_mode=mismatch_mode, - skip_patterns_str=skip_patterns, - ) - return io.NodeOutput(path) - - -# ============================================================================= -# Multi-LoRA Merge to Single LoRA -# ============================================================================= - -def merge_multi_loras_via_base( - lora_paths: List[str], - lora_weights: List[float], - base_model_path: str, - mode: str, - linear_param: float, - conv_param: float, - device: str, - save_dtype: torch.dtype, - output_filename: str, - linear_max_rank: int = None, - conv_max_rank: int = None, - clamp_quantile: float = 0.99, - skip_patterns_str: str = "", - svd_niter: int = 2, - verbose: bool = True, -) -> str: - """ - Merge multiple LoRAs to base model and re-extract as a single LoRA. - - Args: - lora_paths: List of paths to LoRA files - lora_weights: List of weight strengths (0.0-1.0) for each LoRA - base_model_path: Path to base model - mode: "fixed", "ratio", "sv_fro", "sv_cumulative" - linear_param: Rank for fixed mode, or mode parameter for dynamic - conv_param: Same for conv layers - device: Processing device - save_dtype: Output dtype - output_filename: Output filename (without extension) - linear_max_rank: Max rank for linear layers (dynamic modes) - conv_max_rank: Max rank for conv layers (dynamic modes) - clamp_quantile: Clamp outlier singular values - skip_patterns_str: Regex patterns for layers to skip - svd_niter: Power iterations for SVD accuracy - verbose: Print progress info - - Returns: - Path to saved merged LoRA - """ - from .lora_extract_svd import _svd_extract_linear, _svd_extract_conv - - # Estimate memory and prepare - total_size_gb = estimate_model_size(base_model_path) - for lp in lora_paths: - total_size_gb += estimate_model_size(lp) - - if verbose: - print(f"[Multi-LoRA Merge] Preparing memory for {total_size_gb:.2f}GB operation...") - print(f"[Multi-LoRA Merge] Merging {len(lora_paths)} LoRAs with weights: {lora_weights}") - prepare_for_large_operation(total_size_gb * 1.5, torch.device(device)) - - # Open all files - base_handler = MemoryEfficientSafeOpen(base_model_path) - lora_handlers = [MemoryEfficientSafeOpen(lp) for lp in lora_paths] - - try: - # Detect format and extract pairs for each LoRA - lora_infos = [] - all_block_names = set() - - for i, handler in enumerate(lora_handlers): - keys = handler.keys() - format_info = detect_lora_format(keys) - pairs = extract_lora_pairs(keys, format_info) - network_dim, network_alpha = detect_lora_rank(handler, pairs) - lora_infos.append({ - "handler": handler, - "format_info": format_info, - "pairs": pairs, - "network_dim": network_dim, - "network_alpha": network_alpha, - "weight": lora_weights[i], - }) - all_block_names.update(pairs.keys()) - if verbose: - print(f"[Multi-LoRA Merge] LoRA {i+1}: {format_info['format']}, {len(pairs)} layers, dim={network_dim}") - - # Use first LoRA's format for output - output_format = lora_infos[0]["format_info"] - down_suffix = output_format["down_suffix"] - up_suffix = output_format["up_suffix"] - alpha_suffix = output_format["alpha_suffix"] - - # Common prefixes - BASE_PREFIXES = ["model.diffusion_model.", "diffusion_model.", "transformer.", "model."] - LORA_PREFIXES = [ - "lora_unet_", "lora_transformer_", "lora_te1_", "lora_te2_", "lora_te_", - "lycoris_", "diffusion_model.", "transformer.", "unet." - ] - - def extract_core_layer_base(key: str) -> str: - if key.endswith(".weight"): - key = key[:-7] - elif key.endswith(".bias"): - key = key[:-5] - for prefix in BASE_PREFIXES: - if key.startswith(prefix): - key = key[len(prefix):] - break - return key - - def extract_core_layer_lora(block_name: str) -> str: - for prefix in LORA_PREFIXES: - if block_name.startswith(prefix): - block_name = block_name[len(prefix):] - break - # Strip trailing .lora suffix (HuggingFace format leaves this after suffix removal) - if block_name.endswith(".lora"): - block_name = block_name[:-5] - return block_name.replace(".", "_") - - - # Build base key lookup - base_key_lookup = {} - for bk in base_handler.keys(): - if bk.endswith(".weight"): - core = extract_core_layer_base(bk) - core_underscored = core.replace(".", "_") - base_key_lookup[core_underscored] = bk - - if verbose: - print(f"[Multi-LoRA Merge] Base model has {len(base_key_lookup)} weight keys") - - # Compile skip patterns - skip_patterns = _compile_patterns(skip_patterns_str) - - output_sd = {} - stats = {"extracted": 0, "skipped": 0, "no_match": 0, "full": 0} - pbar = comfy.utils.ProgressBar(len(all_block_names)) - - with torch.no_grad(): - for block_name in tqdm(sorted(all_block_names), desc="Merging LoRAs", unit="layers"): - # Check skip patterns - if _matches_any_pattern(block_name, skip_patterns): - stats["skipped"] += 1 - pbar.update(1) - continue - - # Extract core layer name - core_lora = extract_core_layer_lora(block_name) - - # Find base model key - if core_lora not in base_key_lookup: - if verbose and stats["no_match"] < 3: - print(f"[DEBUG] No match for: {core_lora}") - stats["no_match"] += 1 - pbar.update(1) - continue - - base_key = base_key_lookup[core_lora] - - # Load base weight - cpu_base = base_handler.get_tensor(base_key) - if device == 'cuda': - base_weight = transfer_to_gpu_pinned(cpu_base, device, torch.float32) - else: - base_weight = cpu_base.to(device=device, dtype=torch.float32) - del cpu_base - - # Accumulate deltas from all LoRAs that have this block - combined_delta = None - is_conv = None - - for info in lora_infos: - if block_name not in info["pairs"]: - continue - - block_keys = info["pairs"][block_name] - is_full_diff = info["format_info"].get("is_full_diff", False) - - if is_full_diff: - # Full diff format: load .diff directly as the delta - if "diff" not in block_keys: - continue - - cpu_diff = info["handler"].get_tensor(block_keys["diff"]) - if device == 'cuda': - delta = transfer_to_gpu_pinned(cpu_diff, device, torch.float32) - else: - delta = cpu_diff.to(device=device, dtype=torch.float32) - del cpu_diff - - # Determine conv from shape - is_conv = len(delta.shape) == 4 - - # Apply LoRA weight strength (no alpha scaling for full diff) - effective_scale = info["weight"] - - else: - # Standard LoRA format: compute up @ down - if "down" not in block_keys or "up" not in block_keys: - continue - - # Load LoRA weights - cpu_down = info["handler"].get_tensor(block_keys["down"]) - cpu_up = info["handler"].get_tensor(block_keys["up"]) - if device == 'cuda': - lora_down = transfer_to_gpu_pinned(cpu_down, device, torch.float32) - lora_up = transfer_to_gpu_pinned(cpu_up, device, torch.float32) - else: - lora_down = cpu_down.to(device=device, dtype=torch.float32) - lora_up = cpu_up.to(device=device, dtype=torch.float32) - del cpu_down, cpu_up - - # Get alpha - if "alpha" in block_keys: - alpha_tensor = info["handler"].get_tensor(block_keys["alpha"]) - layer_alpha = float(alpha_tensor.item()) - else: - layer_alpha = float(info["network_dim"]) - layer_scale = layer_alpha / info["network_dim"] if info["network_dim"] > 0 else 1.0 - - # Apply LoRA weight strength - effective_scale = layer_scale * info["weight"] - - # Compute delta - is_conv = len(lora_down.shape) == 4 - if is_conv: - in_rank, in_size, kernel_size, k_ = lora_down.shape - out_size, out_rank, _, _ = lora_up.shape - delta = lora_up.reshape(out_size, -1) @ lora_down.reshape(in_rank, -1) - delta = delta.reshape(out_size, in_size, kernel_size, kernel_size) - else: - delta = lora_up @ lora_down - del lora_down, lora_up - - # Accumulate - if combined_delta is None: - combined_delta = effective_scale * delta - else: - combined_delta = combined_delta + effective_scale * delta - del delta - - if combined_delta is None: - # No LoRA had this block - del base_weight - pbar.update(1) - continue - - # Compute finetuned weight - finetuned_weight = base_weight + combined_delta - - # Compute weight difference for extraction - weight_diff = finetuned_weight - base_weight - del base_weight, finetuned_weight, combined_delta - - # Extract new LoRA - if is_conv: - result, mode_str = _svd_extract_conv( - weight_diff, mode, conv_param, device, conv_max_rank, clamp_quantile - ) - else: - result, mode_str = _svd_extract_linear( - weight_diff, mode, linear_param, device, linear_max_rank, clamp_quantile, svd_niter - ) - del weight_diff - - if mode_str == "full": - stats["full"] += 1 - pbar.update(1) - continue - - new_lora_down, new_lora_up, _ = result - new_rank = new_lora_down.shape[0] - new_alpha = float(new_rank) - - # Store result - output_sd[f"{block_name}{down_suffix}"] = new_lora_down.to(save_dtype).cpu().contiguous() - output_sd[f"{block_name}{up_suffix}"] = new_lora_up.to(save_dtype).cpu().contiguous() - output_sd[f"{block_name}{alpha_suffix}"] = torch.tensor([new_alpha]).to(save_dtype) - - del new_lora_down, new_lora_up, result - stats["extracted"] += 1 - pbar.update(1) - - if verbose: - print(f"[Multi-LoRA Merge] Done: {stats['extracted']} extracted, {stats['skipped']} skipped, {stats['no_match']} no_match") - - if not output_sd: - raise ValueError("No LoRA weights extracted. Check that base model matches the LoRAs.") - - # Build metadata - metadata = { - "ss_training_comment": f"Merged {len(lora_paths)} LoRAs via base model, mode={mode}", - "ss_network_dim": str(int(linear_param)) if mode == "fixed" else "Dynamic", - "ss_network_alpha": str(int(linear_param)) if mode == "fixed" else "Dynamic", - } - - # Save - output_dir = folder_paths.get_folder_paths("loras")[0] - os.makedirs(output_dir, exist_ok=True) - output_path = os.path.join(output_dir, f"{output_filename.strip()}.safetensors") - - save_file(output_sd, output_path, metadata) - print(f"[Multi-LoRA Merge] Saved to {output_path}") - - return output_path - - finally: - base_handler.__exit__(None, None, None) - for handler in lora_handlers: - handler.__exit__(None, None, None) - cleanup_after_operation() - - -class LoRAMultiMerge(io.ComfyNode): - """Merge multiple LoRAs to base model and extract as single LoRA.""" - - @classmethod - def define_schema(cls): - return io.Schema( - node_id="LoRAMultiMerge", - display_name="LoRA Multi-Merge", - category="ModelUtils/LoRA/Merge", - description="Merge 1-4 LoRAs to base model and re-extract as a single LoRA.", - inputs=[ - io.Combo.Input("base_model", options=folder_paths.get_filename_list("diffusion_models"), - tooltip="Base model the LoRAs were trained on"), - io.Combo.Input("lora_count", options=["1", "2", "3", "4"], default="2", - tooltip="Number of LoRAs to merge"), - # LoRA 1 - io.Combo.Input("lora_1", options=folder_paths.get_filename_list("loras"), - tooltip="First LoRA"), - io.Float.Input("weight_1", default=1.0, min=0.0, max=2.0, step=0.05, - tooltip="Weight strength for LoRA 1"), - # LoRA 2 - io.Combo.Input("lora_2", options=["None"] + folder_paths.get_filename_list("loras"), - default="None", tooltip="Second LoRA"), - io.Float.Input("weight_2", default=1.0, min=0.0, max=2.0, step=0.05, - tooltip="Weight strength for LoRA 2"), - # LoRA 3 - io.Combo.Input("lora_3", options=["None"] + folder_paths.get_filename_list("loras"), - default="None", tooltip="Third LoRA"), - io.Float.Input("weight_3", default=1.0, min=0.0, max=2.0, step=0.05, - tooltip="Weight strength for LoRA 3"), - # LoRA 4 - io.Combo.Input("lora_4", options=["None"] + folder_paths.get_filename_list("loras"), - default="None", tooltip="Fourth LoRA"), - io.Float.Input("weight_4", default=1.0, min=0.0, max=2.0, step=0.05, - tooltip="Weight strength for LoRA 4"), - # Extraction settings - io.Combo.Input("mode", options=["fixed", "sv_ratio", "sv_fro", "sv_cumulative"], default="fixed", - tooltip="Rank selection mode"), - io.Int.Input("linear_rank", default=64, min=1, max=1024, - tooltip="Target rank for linear layers (fixed) or max rank (dynamic)"), - io.Int.Input("conv_rank", default=32, min=1, max=512, - tooltip="Target rank for conv layers (fixed) or max rank (dynamic)"), - io.Float.Input("dynamic_param", default=0.9, min=0.1, max=1.0, step=0.01, - tooltip="Parameter for dynamic modes (ratio/target)"), - io.Float.Input("clamp_quantile", default=0.99, min=0.5, max=1.0, step=0.01, - tooltip="Clamp outlier singular values"), - io.String.Input("skip_patterns", default="", multiline=True, - tooltip="Regex patterns for layers to skip"), - io.String.Input("output_filename", default="merged_lora"), - io.Combo.Input("save_dtype", options=["fp16", "bf16", "fp32"], default="fp16"), - io.Combo.Input("device", options=["cuda", "cpu"], default="cuda"), - ], - outputs=[io.String.Output(display_name="output_path")], - is_output_node=True, - ) - - @classmethod - def execute(cls, base_model, lora_count, - lora_1, weight_1, lora_2, weight_2, lora_3, weight_3, lora_4, weight_4, - mode, linear_rank, conv_rank, dynamic_param, - clamp_quantile, skip_patterns, output_filename, save_dtype, device) -> io.NodeOutput: - - # Build LoRA list based on count - count = int(lora_count) - lora_names = [lora_1, lora_2, lora_3, lora_4][:count] - lora_weights = [weight_1, weight_2, weight_3, weight_4][:count] - - # Filter out "None" entries - valid_loras = [(name, weight) for name, weight in zip(lora_names, lora_weights) if name != "None"] - if not valid_loras: - raise ValueError("At least one LoRA must be selected") - - lora_names, lora_weights = zip(*valid_loras) - lora_paths = [folder_paths.get_full_path_or_raise("loras", name) for name in lora_names] - base_path = folder_paths.get_full_path_or_raise("diffusion_models", base_model) - dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32}[save_dtype] - - # Determine parameters based on mode - if mode == "fixed": - linear_param = float(linear_rank) - conv_param = float(conv_rank) - linear_max_rank = None - conv_max_rank = None - else: - linear_param = dynamic_param - conv_param = dynamic_param - linear_max_rank = linear_rank - conv_max_rank = conv_rank - - path = merge_multi_loras_via_base( - lora_paths=list(lora_paths), - lora_weights=list(lora_weights), - base_model_path=base_path, - mode=mode, - linear_param=linear_param, - conv_param=conv_param, - device=device, - save_dtype=dtype, - output_filename=output_filename, - linear_max_rank=linear_max_rank, - conv_max_rank=conv_max_rank, - clamp_quantile=clamp_quantile, - skip_patterns_str=skip_patterns, - ) - return io.NodeOutput(path) # ============================================================================= @@ -1840,22 +895,22 @@ class LoRAMergeToModel(io.ComfyNode): # LoRA 1 io.Combo.Input("lora_1", options=folder_paths.get_filename_list("loras"), tooltip="First LoRA"), - io.Float.Input("weight_1", default=1.0, min=0.0, max=2.0, step=0.05, + io.Float.Input("weight_1", default=1.0, min=-10.0, max=10.0, step=0.01, tooltip="Weight strength for LoRA 1"), # LoRA 2 io.Combo.Input("lora_2", options=["None"] + folder_paths.get_filename_list("loras"), default="None", tooltip="Second LoRA"), - io.Float.Input("weight_2", default=1.0, min=0.0, max=2.0, step=0.05, + io.Float.Input("weight_2", default=1.0, min=-10.0, max=10.0, step=0.01, tooltip="Weight strength for LoRA 2"), # LoRA 3 io.Combo.Input("lora_3", options=["None"] + folder_paths.get_filename_list("loras"), default="None", tooltip="Third LoRA"), - io.Float.Input("weight_3", default=1.0, min=0.0, max=2.0, step=0.05, + io.Float.Input("weight_3", default=1.0, min=-10.0, max=10.0, step=0.01, tooltip="Weight strength for LoRA 3"), # LoRA 4 io.Combo.Input("lora_4", options=["None"] + folder_paths.get_filename_list("loras"), default="None", tooltip="Fourth LoRA"), - io.Float.Input("weight_4", default=1.0, min=0.0, max=2.0, step=0.05, + io.Float.Input("weight_4", default=1.0, min=-10.0, max=10.0, step=0.01, tooltip="Weight strength for LoRA 4"), # Settings io.String.Input("skip_patterns", default="", multiline=True,