From 474dfe579c7cfb7154d5a5da415c609837b64973 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 16:41:01 +0100 Subject: [PATCH 01/19] feat: add LoRA Compatibility Analyzer node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standalone pre-merge planning tool that surfaces Pass 1 pairwise interaction data (cosine similarity, sign conflicts, magnitude ratios) as a readable report and N×N heatmap image. Clusters compatible LoRAs into recommended merge groups with auto-strength and strategy suggestions. - Extract _run_pass1_analysis() from optimize_merge() into shared method on _LoRAMergeBase (with optional progress_callback for AutoTuner) - Refactor LoRAAutoTuner.auto_tune() to use shared method (~80 lines removed) - Add LoRACompatibilityAnalyzer node: analyze() → (report, heatmap) - Greedy agglomerative clustering with compatibility threshold - Per-group auto-strength and merge strategy recommendations - PIL-based heatmap with group-reordered axes and contiguous borders - Same-name LoRA disambiguation in all output sections Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 808 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 647 insertions(+), 161 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index 04d4c8a..cc7a0ca 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -2216,6 +2216,112 @@ class _LoRAMergeBase: return normalized + def _run_pass1_analysis(self, active_loras, all_lora_prefixes, model_keys, clip_keys, model, clip, compute_device, progress_callback=None): + """ + Run Pass 1 streaming analysis over all LoRA prefixes. + + Args: + progress_callback: Optional callable invoked after each prefix is analyzed. + + Returns a dict with: + all_key_targets, per_lora_stats, pair_accum, all_magnitude_samples, + prefix_count, prefix_stats, skipped_keys, pairs + """ + use_gpu = compute_device.type != "cpu" + + all_key_targets = {} + skipped_keys = 0 + per_lora_stats = [{ + "name": item["name"], + "strength": item["strength"], + "ranks": [], + "key_count": 0, + "l2_norms": [], + } for item in active_loras] + + pairs = [(i, j) for i in range(len(active_loras)) + for j in range(i + 1, len(active_loras))] + pair_accum = {(i, j): [0, 0, 0.0, 0.0, 0.0] for i, j in pairs} + all_magnitude_samples = [] + prefix_count = 0 + prefix_stats = {} + + def _collect_analysis_result(result): + nonlocal skipped_keys, prefix_count + if result is None: + return + prefix, partial_stats, pair_conflicts, mag_samples, target_info, skips = result + skipped_keys += skips + if len(partial_stats) > 0: + all_key_targets[prefix] = target_info + prefix_count += 1 + for (idx, rank, l2) in partial_stats: + per_lora_stats[idx]["ranks"].append(rank) + per_lora_stats[idx]["key_count"] += 1 + per_lora_stats[idx]["l2_norms"].append(l2) + for (i, j), (ov, conf, dot, na_sq, nb_sq) in pair_conflicts.items(): + pair_accum[(i, j)][0] += ov + pair_accum[(i, j)][1] += conf + pair_accum[(i, j)][2] += dot + pair_accum[(i, j)][3] += na_sq + pair_accum[(i, j)][4] += nb_sq + all_magnitude_samples.extend(mag_samples) + + if len(partial_stats) > 0: + n_contributing = len(partial_stats) + pf_overlap = sum(ov for ov, conf, dot, na_sq, nb_sq in pair_conflicts.values()) + pf_conflict = sum(conf for ov, conf, dot, na_sq, nb_sq in pair_conflicts.values()) + pf_conflict_ratio = pf_conflict / pf_overlap if pf_overlap > 0 else 0.0 + pf_l2s = [l2 for _, _, l2 in partial_stats if l2 > 0] + if len(pf_l2s) >= 2: + pf_mag_ratio = max(pf_l2s) / min(pf_l2s) + else: + pf_mag_ratio = 1.0 + pf_cos_sims = [] + for (ov, conf, dot, na_sq, nb_sq) in pair_conflicts.values(): + denom = (na_sq ** 0.5) * (nb_sq ** 0.5) + if denom > 0: + pf_cos_sims.append(dot / denom) + avg_cos_sim = sum(pf_cos_sims) / len(pf_cos_sims) if pf_cos_sims else 0.0 + prefix_stats[prefix] = { + "n_loras": n_contributing, + "conflict_ratio": pf_conflict_ratio, + "magnitude_ratio": pf_mag_ratio, + "magnitude_samples": list(mag_samples), + "avg_cos_sim": avg_cos_sim, + } + + if use_gpu: + for lora_prefix in all_lora_prefixes: + result = self._analyze_prefix(lora_prefix, active_loras, + model_keys, clip_keys, model, clip, compute_device) + _collect_analysis_result(result) + if progress_callback: + progress_callback() + else: + max_workers = min(4, max(1, len(all_lora_prefixes))) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit(self._analyze_prefix, lora_prefix, active_loras, + model_keys, clip_keys, model, clip, compute_device): lora_prefix + for lora_prefix in all_lora_prefixes + } + for future in concurrent.futures.as_completed(futures): + _collect_analysis_result(future.result()) + if progress_callback: + progress_callback() + + return { + "all_key_targets": all_key_targets, + "per_lora_stats": per_lora_stats, + "pair_accum": pair_accum, + "all_magnitude_samples": all_magnitude_samples, + "prefix_count": prefix_count, + "prefix_stats": prefix_stats, + "skipped_keys": skipped_keys, + "pairs": pairs, + } + def _sample_conflict(self, diff_a, diff_b, device=None): """ Compute sign conflict ratio and cosine similarity components between @@ -3853,93 +3959,16 @@ class LoRAOptimizer(_LoRAMergeBase): f" ({'sequential' if use_gpu else 'threaded'})") t_pass1 = time.time() - # Lightweight accumulators (no full-rank diff tensors stored) - all_key_targets = {} # prefix -> (target_key, is_clip) - skipped_keys = 0 - per_lora_stats = [{ - "name": item["name"], - "strength": item["strength"], - "ranks": [], - "key_count": 0, - "l2_norms": [], - } for item in active_loras] - - pairs = [(i, j) for i in range(len(active_loras)) - for j in range(i + 1, len(active_loras))] - pair_accum = {(i, j): [0, 0, 0.0, 0.0, 0.0] for i, j in pairs} # [overlap, conflict, dot, norm_a_sq, norm_b_sq] - all_magnitude_samples = [] # list of small CPU tensors - prefix_count = 0 # number of prefixes with valid diffs - prefix_stats = {} # prefix -> {conflict_ratio, n_loras, magnitude_samples, magnitude_ratio} - - def _collect_analysis_result(result): - nonlocal skipped_keys, prefix_count - if result is None: - return - prefix, partial_stats, pair_conflicts, mag_samples, target_info, skips = result - skipped_keys += skips - if len(partial_stats) > 0: - all_key_targets[prefix] = target_info - prefix_count += 1 - for (idx, rank, l2) in partial_stats: - per_lora_stats[idx]["ranks"].append(rank) - per_lora_stats[idx]["key_count"] += 1 - per_lora_stats[idx]["l2_norms"].append(l2) - for (i, j), (ov, conf, dot, na_sq, nb_sq) in pair_conflicts.items(): - pair_accum[(i, j)][0] += ov - pair_accum[(i, j)][1] += conf - pair_accum[(i, j)][2] += dot - pair_accum[(i, j)][3] += na_sq - pair_accum[(i, j)][4] += nb_sq - all_magnitude_samples.extend(mag_samples) - - # Store per-prefix stats for per_prefix optimization mode - if len(partial_stats) > 0: - # Number of LoRAs contributing to this prefix - n_contributing = len(partial_stats) - - # Per-prefix conflict ratio - pf_overlap = sum(ov for ov, conf, dot, na_sq, nb_sq in pair_conflicts.values()) - pf_conflict = sum(conf for ov, conf, dot, na_sq, nb_sq in pair_conflicts.values()) - pf_conflict_ratio = pf_conflict / pf_overlap if pf_overlap > 0 else 0.0 - - # Per-prefix magnitude ratio (max/min L2 among contributing LoRAs) - pf_l2s = [l2 for _, _, l2 in partial_stats if l2 > 0] - if len(pf_l2s) >= 2: - pf_mag_ratio = max(pf_l2s) / min(pf_l2s) - else: - pf_mag_ratio = 1.0 - - # Per-prefix average cosine similarity - pf_cos_sims = [] - for (ov, conf, dot, na_sq, nb_sq) in pair_conflicts.values(): - denom = (na_sq ** 0.5) * (nb_sq ** 0.5) - if denom > 0: - pf_cos_sims.append(dot / denom) - avg_cos_sim = sum(pf_cos_sims) / len(pf_cos_sims) if pf_cos_sims else 0.0 - - prefix_stats[prefix] = { - "n_loras": n_contributing, - "conflict_ratio": pf_conflict_ratio, - "magnitude_ratio": pf_mag_ratio, - "magnitude_samples": list(mag_samples), # copy, not reference - "avg_cos_sim": avg_cos_sim, - } - - if use_gpu: - for lora_prefix in all_lora_prefixes: - result = self._analyze_prefix(lora_prefix, active_loras, - model_keys, clip_keys, model, clip, compute_device) - _collect_analysis_result(result) - else: - max_workers = min(4, max(1, len(all_lora_prefixes))) - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit(self._analyze_prefix, lora_prefix, active_loras, - model_keys, clip_keys, model, clip, compute_device): lora_prefix - for lora_prefix in all_lora_prefixes - } - for future in concurrent.futures.as_completed(futures): - _collect_analysis_result(future.result()) + analysis = self._run_pass1_analysis(active_loras, all_lora_prefixes, + model_keys, clip_keys, model, clip, compute_device) + all_key_targets = analysis["all_key_targets"] + per_lora_stats = analysis["per_lora_stats"] + pair_accum = analysis["pair_accum"] + all_magnitude_samples = analysis["all_magnitude_samples"] + prefix_count = analysis["prefix_count"] + prefix_stats = analysis["prefix_stats"] + skipped_keys = analysis["skipped_keys"] + pairs = analysis["pairs"] if prefix_count == 0: return (model, clip, "No compatible LoRA keys found. " @@ -4922,85 +4951,23 @@ class LoRAAutoTuner(LoRAOptimizer): use_gpu = compute_device.type != "cpu" # Run Pass 1 analysis - all_key_targets = {} - skipped_keys = 0 - per_lora_stats = [{ - "name": item["name"], "strength": item["strength"], - "ranks": [], "key_count": 0, "l2_norms": [], - } for item in active_loras] - - pairs = [(i, j) for i in range(len(active_loras)) - for j in range(i + 1, len(active_loras))] - pair_accum = {(i, j): [0, 0, 0.0, 0.0, 0.0] for i, j in pairs} - all_magnitude_samples = [] - prefix_count = 0 - prefix_stats = {} - - def _collect_analysis(result): - nonlocal skipped_keys, prefix_count - if result is None: - return - prefix, partial_stats, pair_conflicts, mag_samples, target_info, skips = result - skipped_keys += skips - if len(partial_stats) > 0: - all_key_targets[prefix] = target_info - prefix_count += 1 - for (idx, rank, l2) in partial_stats: - per_lora_stats[idx]["ranks"].append(rank) - per_lora_stats[idx]["key_count"] += 1 - per_lora_stats[idx]["l2_norms"].append(l2) - for (i, j), (ov, conf, dot, na_sq, nb_sq) in pair_conflicts.items(): - pair_accum[(i, j)][0] += ov - pair_accum[(i, j)][1] += conf - pair_accum[(i, j)][2] += dot - pair_accum[(i, j)][3] += na_sq - pair_accum[(i, j)][4] += nb_sq - all_magnitude_samples.extend(mag_samples) - if len(partial_stats) > 0: - n_contributing = len(partial_stats) - pf_overlap = sum(ov for ov, conf, dot, na_sq, nb_sq in pair_conflicts.values()) - pf_conflict = sum(conf for ov, conf, dot, na_sq, nb_sq in pair_conflicts.values()) - pf_conflict_ratio = pf_conflict / pf_overlap if pf_overlap > 0 else 0.0 - pf_l2s = [l2 for _, _, l2 in partial_stats if l2 > 0] - pf_mag_ratio = max(pf_l2s) / min(pf_l2s) if len(pf_l2s) >= 2 else 1.0 - pf_cos_sims = [] - for (ov, conf, dot, na_sq, nb_sq) in pair_conflicts.values(): - denom = (na_sq ** 0.5) * (nb_sq ** 0.5) - if denom > 0: - pf_cos_sims.append(dot / denom) - avg_cs = sum(pf_cos_sims) / len(pf_cos_sims) if pf_cos_sims else 0.0 - prefix_stats[prefix] = { - "n_loras": n_contributing, - "conflict_ratio": pf_conflict_ratio, - "magnitude_ratio": pf_mag_ratio, - "magnitude_samples": list(mag_samples), - "avg_cos_sim": avg_cs, - } - logging.info(f"[LoRA AutoTuner] Pass 1: Analyzing {len(all_lora_prefixes)} prefixes...") t_start = time.time() # Progress bar: analysis prefixes + top_n merges (+ 1 final merge when subsampling) n_pbar_merges = top_n + (1 if scoring_speed != "full" and top_n > 1 and output_mode != "tuning_only" else 0) pbar = comfy.utils.ProgressBar(len(all_lora_prefixes) + n_pbar_merges) - if use_gpu: - for lora_prefix in all_lora_prefixes: - result = self._analyze_prefix(lora_prefix, active_loras, - model_keys, clip_keys, model, clip, compute_device) - _collect_analysis(result) - pbar.update(1) - else: - max_workers = min(4, max(1, len(all_lora_prefixes))) - analyzed = 0 - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit(self._analyze_prefix, lora_prefix, active_loras, - model_keys, clip_keys, model, clip, compute_device): lora_prefix - for lora_prefix in all_lora_prefixes - } - for future in concurrent.futures.as_completed(futures): - _collect_analysis(future.result()) - analyzed += 1 - pbar.update(1) + + analysis = self._run_pass1_analysis(active_loras, all_lora_prefixes, + model_keys, clip_keys, model, clip, compute_device, + progress_callback=lambda: pbar.update(1)) + all_key_targets = analysis["all_key_targets"] + per_lora_stats = analysis["per_lora_stats"] + pair_accum = analysis["pair_accum"] + all_magnitude_samples = analysis["all_magnitude_samples"] + prefix_count = analysis["prefix_count"] + prefix_stats = analysis["prefix_stats"] + skipped_keys = analysis["skipped_keys"] + pairs = analysis["pairs"] if prefix_count == 0: return (model, clip, "No compatible LoRA keys found.", "", None, None) @@ -5993,6 +5960,523 @@ class MergedLoRAToWanVideo: return (new_model,) +class LoRACompatibilityAnalyzer(LoRAOptimizer): + """ + Standalone pre-merge planning tool. Analyzes pairwise LoRA interactions + (cosine similarity, sign conflicts, magnitude ratios) and recommends + which LoRAs to merge together vs. use independently. + + No merge happens — analysis only. Inherits from LoRAOptimizer to reuse + _analyze_prefix, _compute_auto_strengths, and _auto_select_params. + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model": ("MODEL", {"tooltip": "Base model for key mapping and target shapes."}), + "lora_stack": ("LORA_STACK", {"tooltip": "The LoRA stack to analyze for compatibility."}), + }, + "optional": { + "clip": ("CLIP", {"tooltip": "Text encoder. Connect for CLIP key analysis."}), + }, + } + + RETURN_TYPES = ("STRING", "IMAGE") + RETURN_NAMES = ("report", "compatibility_map") + FUNCTION = "analyze" + CATEGORY = "LoRA Optimizer" + DESCRIPTION = "Analyzes pairwise LoRA compatibility and recommends merge groups. No merge is performed — use this to plan which LoRAs to combine before merging." + + def analyze(self, model, lora_stack, clip=None): + if not lora_stack or len(lora_stack) == 0: + return ("No LoRAs in stack.", self._empty_image()) + + normalized_stack = self._normalize_stack(lora_stack) + active_loras = [item for item in normalized_stack if item["strength"] != 0] + + if len(active_loras) == 0: + return ("No active LoRAs in stack (all zero strength).", self._empty_image()) + + n_loras = len(active_loras) + detected_arch = getattr(self, '_detected_arch', None) or 'unknown' + preset_key, arch_preset = _resolve_arch_preset("auto", detected_arch) + + if n_loras == 1: + name = active_loras[0]["name"] + report = ( + "=" * 55 + "\n" + " LoRA Compatibility Analysis\n" + "=" * 55 + "\n\n" + f"Single LoRA: {name}\n" + "Nothing to compare — add more LoRAs to analyze compatibility.\n" + "\n" + "=" * 55 + ) + return (report, self._empty_image()) + + # --- Setup --- + logging.info(f"[Compatibility Analyzer] Analyzing {n_loras} LoRAs...") + t_start = time.time() + + model_keys = self._get_model_keys(model) + clip_keys = {} + if clip is not None: + clip_keys = comfy.lora.model_lora_keys_clip(clip.cond_stage_model, {}) + + all_lora_prefixes = set() + for item in active_loras: + for key in item["lora"].keys(): + for suffix in [".lora_up.weight", ".lora_down.weight", "_lora.up.weight", + "_lora.down.weight", ".lora_B.weight", ".lora_A.weight", + ".lora.up.weight", ".lora.down.weight", ".alpha"]: + if key.endswith(suffix): + prefix = key[:-len(suffix)] + all_lora_prefixes.add(prefix) + break + all_lora_prefixes = sorted(all_lora_prefixes) + compute_device = self._get_compute_device() + + # --- Pass 1 Analysis --- + analysis = self._run_pass1_analysis(active_loras, all_lora_prefixes, + model_keys, clip_keys, model, clip, compute_device) + prefix_count = analysis["prefix_count"] + + if prefix_count == 0: + return ("No compatible LoRA keys found. " + "LoRAs may be incompatible with this model architecture.", + self._empty_image()) + + # --- Finalize stats --- + per_lora_stats = analysis["per_lora_stats"] + pair_accum = analysis["pair_accum"] + pairs = analysis["pairs"] + + lora_stats = [] + l2_means = [] + for i, stat in enumerate(per_lora_stats): + avg_rank = sum(stat["ranks"]) / len(stat["ranks"]) if stat["ranks"] else 0 + l2_mean = sum(stat["l2_norms"]) / len(stat["l2_norms"]) if stat["l2_norms"] else 0 + l2_means.append(l2_mean) + lora_stats.append({ + "name": stat["name"], + "strength": stat["strength"], + "key_count": stat["key_count"], + "avg_rank": avg_rank, + "l2_mean": l2_mean, + }) + + total_overlap = 0 + total_conflict = 0 + pairwise_conflicts = [] + pairwise_similarities = {} + for i, j in pairs: + pair_overlap, pair_conflict, pair_dot, pair_na_sq, pair_nb_sq = pair_accum[(i, j)] + total_overlap += pair_overlap + total_conflict += pair_conflict + ratio = pair_conflict / pair_overlap if pair_overlap > 0 else 0 + denom = math.sqrt(pair_na_sq) * math.sqrt(pair_nb_sq) + cos_sim = pair_dot / denom if denom > 0 else 0.0 + pairwise_similarities[(i, j)] = cos_sim + name_i = active_loras[i]['name'] + name_j = active_loras[j]['name'] + if name_i == name_j: + pair_label = (f"{name_i} [#{i+1}, str={active_loras[i]['strength']}] vs " + f"{name_j} [#{j+1}, str={active_loras[j]['strength']}]") + else: + pair_label = f"{name_i} vs {name_j}" + pairwise_conflicts.append({ + "i": i, "j": j, + "pair": pair_label, + "overlap": pair_overlap, + "conflicts": pair_conflict, + "ratio": ratio, + "cosine_sim": cos_sim, + }) + + avg_conflict_ratio = total_conflict / total_overlap if total_overlap > 0 else 0 + valid_l2 = [m for m in l2_means if m > 0] + magnitude_ratio = max(valid_l2) / min(valid_l2) if len(valid_l2) >= 2 else 1.0 + avg_cos_sim = (sum(pairwise_similarities.values()) / len(pairwise_similarities) + if pairwise_similarities else 0.0) + + collection_stats = { + "n_loras": n_loras, + "total_keys": prefix_count, + "avg_conflict": avg_conflict_ratio, + "avg_cos_sim": avg_cos_sim, + "magnitude_ratio": magnitude_ratio, + } + + # --- Compatibility matrix --- + compat_matrix = self._compute_compat_matrix(pairwise_similarities, pairwise_conflicts, n_loras) + + # --- Clustering --- + groups = self._cluster_loras(compat_matrix, threshold=0.05) + + # --- Per-group recommendations --- + group_info = [] + for group in groups: + info = {"indices": group} + if len(group) == 1: + idx = group[0] + info["type"] = "solo" + info["strength"] = active_loras[idx]["strength"] + # Find opposing LoRAs + opposing = [] + for other_idx in range(n_loras): + if other_idx == idx: + continue + key = (min(idx, other_idx), max(idx, other_idx)) + cs = pairwise_similarities.get(key, 0.0) + if cs < -0.05: + opposing.append((active_loras[other_idx]["name"], cs)) + opposing.sort(key=lambda x: x[1]) + info["opposing"] = opposing + else: + info["type"] = "merge" + # Compute intra-group stats + conflict_by_pair = {(pc["i"], pc["j"]): pc["ratio"] for pc in pairwise_conflicts} + intra_cos = [] + intra_conflict = [] + for a_pos in range(len(group)): + for b_pos in range(a_pos + 1, len(group)): + key = (min(group[a_pos], group[b_pos]), max(group[a_pos], group[b_pos])) + cs = pairwise_similarities.get(key, 0.0) + cr = conflict_by_pair.get(key, 0.0) + intra_cos.append(cs) + intra_conflict.append(cr) + info["avg_cos_sim"] = sum(intra_cos) / len(intra_cos) if intra_cos else 0.0 + info["avg_conflict"] = sum(intra_conflict) / len(intra_conflict) if intra_conflict else 0.0 + avg_compat = sum(cs * (1.0 - cr) for cs, cr in zip(intra_cos, intra_conflict)) / len(intra_cos) if intra_cos else 0.0 + if avg_compat > 0.2: + info["confidence"] = "High" + elif avg_compat > 0.05: + info["confidence"] = "Moderate" + else: + info["confidence"] = "Low" + + # Auto-strengths for group + group_loras = [active_loras[idx] for idx in group] + group_lora_stats = [lora_stats[idx] for idx in group] + group_pw_sim = {} + for a_pos in range(len(group)): + for b_pos in range(a_pos + 1, len(group)): + orig_key = (min(group[a_pos], group[b_pos]), max(group[a_pos], group[b_pos])) + group_pw_sim[(a_pos, b_pos)] = pairwise_similarities.get(orig_key, 0.0) + new_strengths, _ = self._compute_auto_strengths( + group_loras, group_lora_stats, + pairwise_similarities=group_pw_sim, + arch_preset=arch_preset, detected_arch=detected_arch) + info["strengths"] = {group[k]: new_strengths[k] for k in range(len(group))} + + # Suggested merge strategy + group_avg_cos = info["avg_cos_sim"] + group_avg_conflict = info["avg_conflict"] + mag_samples = analysis["all_magnitude_samples"] + mode, _, _, _ = self._auto_select_params( + group_avg_conflict, + magnitude_ratio, + magnitude_samples=mag_samples, + avg_cos_sim=group_avg_cos, + arch_preset=arch_preset) + info["suggested_merge"] = mode + + group_info.append(info) + + # --- Warnings --- + def _disambiguate(idx_a, idx_b): + """Return display names, adding [#N] suffix when names collide.""" + na = active_loras[idx_a]["name"] + nb = active_loras[idx_b]["name"] + if na == nb: + na = f"{na} [#{idx_a+1}]" + nb = f"{nb} [#{idx_b+1}]" + return na, nb + + warnings = [] + for pc in pairwise_conflicts: + if pc["cosine_sim"] < -0.1: + na, nb = _disambiguate(pc["i"], pc["j"]) + warnings.append({ + "type": "opposing", + "name_i": na, + "name_j": nb, + "cos_sim": pc["cosine_sim"], + }) + # Magnitude imbalance warnings + for pc in pairwise_conflicts: + i, j = pc["i"], pc["j"] + l2_i = l2_means[i] + l2_j = l2_means[j] + if l2_i > 0 and l2_j > 0: + ratio = max(l2_i, l2_j) / min(l2_i, l2_j) + if ratio > 3.0: + na, nb = _disambiguate(i, j) + stronger = na if l2_i > l2_j else nb + weaker = nb if l2_i > l2_j else na + warnings.append({ + "type": "magnitude", + "stronger": stronger, + "weaker": weaker, + "ratio": ratio, + }) + + elapsed = time.time() - t_start + logging.info(f"[Compatibility Analyzer] Analysis complete ({elapsed:.1f}s)") + + # --- Build report --- + report = self._build_compatibility_report( + active_loras, groups, group_info, lora_stats, pairwise_conflicts, + collection_stats, warnings, detected_arch, prefix_count) + + # --- Generate heatmap --- + lora_names = [item["name"] for item in active_loras] + heatmap = self._generate_heatmap(compat_matrix, lora_names, groups) + + return (report, heatmap) + + @staticmethod + def _empty_image(): + """Return a minimal 1x1 black IMAGE tensor.""" + return torch.zeros(1, 1, 1, 3, dtype=torch.float32) + + @staticmethod + def _compute_compat_matrix(pairwise_similarities, pairwise_conflicts, n_loras): + """Compute N×N compatibility matrix: compat[i][j] = cos_sim * (1 - conflict_ratio).""" + conflict_by_pair = {} + for pc in pairwise_conflicts: + conflict_by_pair[(pc["i"], pc["j"])] = pc["ratio"] + + matrix = [[0.0] * n_loras for _ in range(n_loras)] + for (i, j), cos_sim in pairwise_similarities.items(): + conflict_ratio = conflict_by_pair.get((i, j), 0.0) + compat = cos_sim * (1.0 - conflict_ratio) + matrix[i][j] = compat + matrix[j][i] = compat + return matrix + + @staticmethod + def _cluster_loras(compat_matrix, threshold=0.05): + """Greedy agglomerative clustering. Returns list of groups (lists of indices).""" + n = len(compat_matrix) + groups = [[i] for i in range(n)] + + while len(groups) > 1: + best_score = -float('inf') + best_pair = None + for ga_idx in range(len(groups)): + for gb_idx in range(ga_idx + 1, len(groups)): + ga, gb = groups[ga_idx], groups[gb_idx] + total = 0.0 + count = 0 + for a in ga: + for b in gb: + total += compat_matrix[a][b] + count += 1 + avg_compat = total / count if count > 0 else 0.0 + if avg_compat > best_score: + best_score = avg_compat + best_pair = (ga_idx, gb_idx) + + if best_score < threshold: + break + + # Merge the best pair + ga_idx, gb_idx = best_pair + merged = groups[ga_idx] + groups[gb_idx] + groups = [g for idx, g in enumerate(groups) if idx != ga_idx and idx != gb_idx] + groups.append(merged) + + # Sort groups: multi-member first (by size desc), then singletons + groups.sort(key=lambda g: (-len(g), g[0])) + return groups + + @staticmethod + def _generate_heatmap(compat_matrix, lora_names, groups): + """Generate N×N heatmap as torch IMAGE tensor using PIL.""" + from PIL import Image, ImageDraw + import numpy as np + + n = len(compat_matrix) + if n == 0: + return torch.zeros(1, 1, 1, 3, dtype=torch.float32) + + # Reorder indices by group membership so group borders are contiguous + display_order = [] + for g in groups: + display_order.extend(g) + + cell_size = 80 + margin = 150 + img_size = margin + n * cell_size + + img = Image.new("RGB", (img_size, img_size), (30, 30, 30)) + draw = ImageDraw.Draw(img) + + # Color mapping: continuous red (-0.4) → yellow (0.05) → green (+0.5) + def compat_to_color(val): + # Normalize to 0..1 range: -0.4 → 0, +0.5 → 1 + t = max(0.0, min(1.0, (val + 0.4) / 0.9)) + if t < 0.5: + # Red to yellow (t: 0→0.5) + s = t / 0.5 # 0..1 + r = int(200 + 20 * s) + g = int(180 * s) + return (r, g, 0) + else: + # Yellow to green (t: 0.5→1) + s = (t - 0.5) / 0.5 # 0..1 + r = int(220 * (1 - s)) + g = int(180 + 20 * s) + return (r, g, 0) + + # Draw cells (using reordered indices) + for row in range(n): + for col in range(n): + orig_i = display_order[row] + orig_j = display_order[col] + x = margin + col * cell_size + y = margin + row * cell_size + if orig_i == orig_j: + color = (80, 80, 80) + val_text = "-" + else: + val = compat_matrix[orig_i][orig_j] + color = compat_to_color(val) + val_text = f"{val:.2f}" + + draw.rectangle([x, y, x + cell_size - 1, y + cell_size - 1], fill=color) + # Center text in cell + bbox = draw.textbbox((0, 0), val_text) + tw = bbox[2] - bbox[0] + th = bbox[3] - bbox[1] + tx = x + (cell_size - tw) // 2 + ty = y + (cell_size - th) // 2 + # Text color: white on dark, black on light + brightness = color[0] * 0.299 + color[1] * 0.587 + color[2] * 0.114 + text_color = (0, 0, 0) if brightness > 128 else (255, 255, 255) + draw.text((tx, ty), val_text, fill=text_color) + + # Draw labels (using reordered indices, truncated to 20 chars) + for pos in range(n): + label = lora_names[display_order[pos]][:20] + # Row labels (left) + y = margin + pos * cell_size + cell_size // 2 - 5 + draw.text((5, y), label, fill=(220, 220, 220)) + # Column labels (top, horizontal) + x = margin + pos * cell_size + 5 + draw.text((x, margin - 15), label[:10], fill=(220, 220, 220)) + + # Draw group borders (contiguous after reordering) + pos = 0 + for group in groups: + if len(group) > 1: + x0 = margin + pos * cell_size - 1 + y0 = margin + pos * cell_size - 1 + x1 = margin + (pos + len(group)) * cell_size + y1 = margin + (pos + len(group)) * cell_size + for offset in range(3): + draw.rectangle([x0 - offset, y0 - offset, x1 + offset, y1 + offset], + outline=(255, 255, 255)) + pos += len(group) + + # Convert PIL → torch tensor [1, H, W, 3] float32 + arr = np.array(img, dtype=np.float32) / 255.0 + tensor = torch.from_numpy(arr).unsqueeze(0) # [1, H, W, 3] + return tensor + + def _build_compatibility_report(self, active_loras, groups, group_info, + lora_stats, pairwise_conflicts, + collection_stats, warnings, + detected_arch, prefix_count): + """Build human-readable compatibility report.""" + n_loras = len(active_loras) + lines = [] + + lines.append("=" * 55) + lines.append(" LoRA Compatibility Analysis") + lines.append("=" * 55) + lines.append("") + lines.append(f"Analyzed: {n_loras} LoRAs across {prefix_count} prefixes") + arch_display = detected_arch.upper() if detected_arch and detected_arch != 'unknown' else "Unknown" + lines.append(f"Architecture: {arch_display}") + lines.append(f"Avg conflict: {collection_stats['avg_conflict']:.0%} | " + f"Avg cosine similarity: {collection_stats['avg_cos_sim']:.2f}") + lines.append("") + + # --- Recommended Groups --- + lines.append("-" * 30 + " Recommended Groups " + "-" * 30) + lines.append("") + + group_num = 0 + solo_entries = [] + for gi in group_info: + indices = gi["indices"] + if gi["type"] == "solo": + solo_entries.append(gi) + continue + + group_num += 1 + confidence = gi["confidence"] + lines.append(f"Group {group_num} -- Merge together (compatibility: {confidence})") + for idx in indices: + strength = gi["strengths"].get(idx, active_loras[idx]["strength"]) + name = active_loras[idx]["name"] + lines.append(f" * {name:<28s} strength {strength:.2f}") + lines.append(f" Avg cosine sim: {gi['avg_cos_sim']:.2f} | " + f"Avg conflict: {gi['avg_conflict']:.0%}") + lines.append(f" Suggested merge: {gi['suggested_merge']}") + lines.append("") + + if solo_entries: + lines.append("Solo -- Use independently") + for gi in solo_entries: + idx = gi["indices"][0] + name = active_loras[idx]["name"] + strength = gi["strength"] + lines.append(f" * {name:<28s} strength {strength:.2f}") + if gi["opposing"]: + opp_strs = [f"{oname} (cos: {cs:.2f})" for oname, cs in gi["opposing"][:3]] + lines.append(f" Opposes: {', '.join(opp_strs)}") + lines.append("") + + # --- Warnings --- + if warnings: + lines.append("-" * 30 + " Warnings " + "-" * 30) + lines.append("") + for w in warnings: + if w["type"] == "opposing": + lines.append(f"! {w['name_i']} vs {w['name_j']}: Opposing (cos_sim: {w['cos_sim']:.2f})") + lines.append(f" These cancel each other out -- using both will degrade quality.") + lines.append("") + elif w["type"] == "magnitude": + lines.append(f"! {w['stronger']} is {w['ratio']:.1f}x stronger than {w['weaker']}") + lines.append(f" {w['weaker']} may be overshadowed at equal strengths.") + lines.append("") + + # --- Pairwise Compatibility Table --- + lines.append("-" * 30 + " Pairwise Compatibility " + "-" * 30) + lines.append("") + + # Sort by compat score descending + sorted_pairs = sorted(pairwise_conflicts, key=lambda p: p["cosine_sim"] * (1.0 - p["ratio"]), reverse=True) + for pc in sorted_pairs: + cos = pc["cosine_sim"] + conflict = pc["ratio"] + compat = cos * (1.0 - conflict) + indicator = "" + if compat > 0.2: + indicator = " [OK]" + elif compat < -0.05: + indicator = " [!!]" + lines.append(f" {pc['pair']:<45s} " + f"cos:{cos:6.2f} conflict:{conflict:5.0%} compat:{compat:6.2f}{indicator}") + lines.append("") + lines.append("=" * 55) + + return "\n".join(lines) + + class LoRAConflictEditor(_LoRAMergeBase): """ Analyzes pairwise sign conflicts between LoRAs in a stack and enriches @@ -6410,6 +6894,7 @@ NODE_CLASS_MAPPINGS = { "LoRAMergeSelector": LoRAMergeSelector, "SaveTunerData": SaveTunerData, "LoadTunerData": LoadTunerData, + "LoRACompatibilityAnalyzer": LoRACompatibilityAnalyzer, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -6426,4 +6911,5 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LoRAMergeSelector": "Merge Selector", "SaveTunerData": "Save Tuner Data", "LoadTunerData": "Load Tuner Data", + "LoRACompatibilityAnalyzer": "LoRA Compatibility Analyzer", } From fc7825be99e95a9d75bcef0cc0037cc130eb054f Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 17:09:00 +0100 Subject: [PATCH 02/19] fix: improve Compatibility Analyzer output for orthogonal LoRA stacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Orthogonal LoRAs (sliders, concepts targeting different weights) were all shown as "Solo" because compat ≈ 0 fell below the clustering threshold. Four fixes: - Add +0.1 non-opposing baseline to compat formula so orthogonal pairs cluster together; use "Safe to combine" label for Low confidence groups - Track shared prefix count per pair; annotate (no shared keys) in report - Eliminate -0.00 display with +0.0 before formatting - Consolidate magnitude warnings: group by strong LoRA instead of N separate pairwise warnings Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 65 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index cc7a0ca..1e224bc 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -2242,6 +2242,7 @@ class _LoRAMergeBase: pairs = [(i, j) for i in range(len(active_loras)) for j in range(i + 1, len(active_loras))] pair_accum = {(i, j): [0, 0, 0.0, 0.0, 0.0] for i, j in pairs} + pair_shared_prefixes = {(i, j): 0 for i, j in pairs} all_magnitude_samples = [] prefix_count = 0 prefix_stats = {} @@ -2265,6 +2266,7 @@ class _LoRAMergeBase: pair_accum[(i, j)][2] += dot pair_accum[(i, j)][3] += na_sq pair_accum[(i, j)][4] += nb_sq + pair_shared_prefixes[(i, j)] += 1 all_magnitude_samples.extend(mag_samples) if len(partial_stats) > 0: @@ -2315,6 +2317,7 @@ class _LoRAMergeBase: "all_key_targets": all_key_targets, "per_lora_stats": per_lora_stats, "pair_accum": pair_accum, + "pair_shared_prefixes": pair_shared_prefixes, "all_magnitude_samples": all_magnitude_samples, "prefix_count": prefix_count, "prefix_stats": prefix_stats, @@ -6049,6 +6052,7 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): # --- Finalize stats --- per_lora_stats = analysis["per_lora_stats"] pair_accum = analysis["pair_accum"] + pair_shared_prefixes = analysis["pair_shared_prefixes"] pairs = analysis["pairs"] lora_stats = [] @@ -6091,6 +6095,7 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): "conflicts": pair_conflict, "ratio": ratio, "cosine_sim": cos_sim, + "shared_prefixes": pair_shared_prefixes[(i, j)], }) avg_conflict_ratio = total_conflict / total_overlap if total_overlap > 0 else 0 @@ -6203,7 +6208,17 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): "name_j": nb, "cos_sim": pc["cosine_sim"], }) - # Magnitude imbalance warnings + # Magnitude imbalance warnings — consolidated per strong LoRA + # Compute per-LoRA average magnitude ratio vs all others + lora_avg_ratios = {} # idx -> avg ratio relative to mean + if len(l2_means) >= 2: + valid_l2 = [m for m in l2_means if m > 0] + mean_l2 = sum(valid_l2) / len(valid_l2) if valid_l2 else 1.0 + for idx, l2 in enumerate(l2_means): + if l2 > 0 and mean_l2 > 0: + lora_avg_ratios[idx] = l2 / mean_l2 + + strong_loras = {} # strong_idx -> [(weak_name, ratio)] for pc in pairwise_conflicts: i, j = pc["i"], pc["j"] l2_i = l2_means[i] @@ -6211,15 +6226,23 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): if l2_i > 0 and l2_j > 0: ratio = max(l2_i, l2_j) / min(l2_i, l2_j) if ratio > 3.0: - na, nb = _disambiguate(i, j) - stronger = na if l2_i > l2_j else nb - weaker = nb if l2_i > l2_j else na - warnings.append({ - "type": "magnitude", - "stronger": stronger, - "weaker": weaker, - "ratio": ratio, - }) + strong_idx = i if l2_i > l2_j else j + weak_idx = j if l2_i > l2_j else i + weak_name = active_loras[weak_idx]["name"] + if strong_idx not in strong_loras: + strong_loras[strong_idx] = [] + if weak_name not in [w for w, _ in strong_loras[strong_idx]]: + strong_loras[strong_idx].append((weak_name, ratio)) + for strong_idx in sorted(strong_loras.keys()): + weak_list = strong_loras[strong_idx] + strong_name = active_loras[strong_idx]["name"] + avg_ratio = lora_avg_ratios.get(strong_idx, 0.0) + warnings.append({ + "type": "magnitude_group", + "stronger": strong_name, + "avg_ratio": avg_ratio, + "weaker_list": [w for w, _ in weak_list], + }) elapsed = time.time() - t_start logging.info(f"[Compatibility Analyzer] Analysis complete ({elapsed:.1f}s)") @@ -6242,7 +6265,11 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): @staticmethod def _compute_compat_matrix(pairwise_similarities, pairwise_conflicts, n_loras): - """Compute N×N compatibility matrix: compat[i][j] = cos_sim * (1 - conflict_ratio).""" + """Compute N×N compatibility matrix: compat[i][j] = cos_sim * (1 - conflict_ratio) + baseline. + + Non-opposing LoRAs (cos_sim > -0.1) get a +0.1 baseline because orthogonal + LoRAs are safe to merge — they don't interfere with each other. + """ conflict_by_pair = {} for pc in pairwise_conflicts: conflict_by_pair[(pc["i"], pc["j"])] = pc["ratio"] @@ -6251,6 +6278,8 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): for (i, j), cos_sim in pairwise_similarities.items(): conflict_ratio = conflict_by_pair.get((i, j), 0.0) compat = cos_sim * (1.0 - conflict_ratio) + if cos_sim > -0.1: + compat += 0.1 # non-opposing LoRAs are safe to merge matrix[i][j] = compat matrix[j][i] = compat return matrix @@ -6418,7 +6447,8 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): group_num += 1 confidence = gi["confidence"] - lines.append(f"Group {group_num} -- Merge together (compatibility: {confidence})") + label = "Safe to combine" if confidence == "Low" else "Merge together" + lines.append(f"Group {group_num} -- {label} (compatibility: {confidence})") for idx in indices: strength = gi["strengths"].get(idx, active_loras[idx]["strength"]) name = active_loras[idx]["name"] @@ -6449,6 +6479,11 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): lines.append(f"! {w['name_i']} vs {w['name_j']}: Opposing (cos_sim: {w['cos_sim']:.2f})") lines.append(f" These cancel each other out -- using both will degrade quality.") lines.append("") + elif w["type"] == "magnitude_group": + lines.append(f"! {w['stronger']} ({w['avg_ratio']:.1f}x avg) overshadows:") + lines.append(f" {', '.join(w['weaker_list'])}") + lines.append(f" Consider lowering its strength or raising weaker LoRAs.") + lines.append("") elif w["type"] == "magnitude": lines.append(f"! {w['stronger']} is {w['ratio']:.1f}x stronger than {w['weaker']}") lines.append(f" {w['weaker']} may be overshadowed at equal strengths.") @@ -6461,14 +6496,16 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): # Sort by compat score descending sorted_pairs = sorted(pairwise_conflicts, key=lambda p: p["cosine_sim"] * (1.0 - p["ratio"]), reverse=True) for pc in sorted_pairs: - cos = pc["cosine_sim"] + cos = pc["cosine_sim"] + 0.0 # avoid -0.0 conflict = pc["ratio"] - compat = cos * (1.0 - conflict) + compat = cos * (1.0 - conflict) + 0.0 # avoid -0.0 indicator = "" if compat > 0.2: indicator = " [OK]" elif compat < -0.05: indicator = " [!!]" + if pc.get("shared_prefixes", 1) == 0: + indicator += " (no shared keys)" lines.append(f" {pc['pair']:<45s} " f"cos:{cos:6.2f} conflict:{conflict:5.0%} compat:{compat:6.2f}{indicator}") lines.append("") From 00add7127c30d42b3aae4c819c3cc6a7b9a2c64a Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 17:12:44 +0100 Subject: [PATCH 03/19] fix: round before adding +0.0 to eliminate -0.00 display Values like -0.004 round to -0.00 with :.2f formatting. The +0.0 trick only fixes exact float -0.0. Round to display precision first. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index 1e224bc..6b46631 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -6496,9 +6496,9 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): # Sort by compat score descending sorted_pairs = sorted(pairwise_conflicts, key=lambda p: p["cosine_sim"] * (1.0 - p["ratio"]), reverse=True) for pc in sorted_pairs: - cos = pc["cosine_sim"] + 0.0 # avoid -0.0 conflict = pc["ratio"] - compat = cos * (1.0 - conflict) + 0.0 # avoid -0.0 + cos = round(pc["cosine_sim"], 2) + 0.0 # avoid -0.00 + compat = round(pc["cosine_sim"] * (1.0 - conflict), 2) + 0.0 # avoid -0.00 indicator = "" if compat > 0.2: indicator = " [OK]" From 13d4b07e0c9fa1cc3c39b42cb54b24e47583076e Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 17:15:35 +0100 Subject: [PATCH 04/19] fix: strip .safetensors from display names and right-align pairwise table Drop the .safetensors extension from all LoRA names in the compatibility report (groups, warnings, pairwise table). Right-align the numeric columns by computing max pair label width. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index 6b46631..ebe5501 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -6422,6 +6422,10 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): n_loras = len(active_loras) lines = [] + def _dn(name): + """Display name: strip .safetensors extension.""" + return name.removesuffix(".safetensors") + lines.append("=" * 55) lines.append(" LoRA Compatibility Analysis") lines.append("=" * 55) @@ -6451,7 +6455,7 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): lines.append(f"Group {group_num} -- {label} (compatibility: {confidence})") for idx in indices: strength = gi["strengths"].get(idx, active_loras[idx]["strength"]) - name = active_loras[idx]["name"] + name = _dn(active_loras[idx]["name"]) lines.append(f" * {name:<28s} strength {strength:.2f}") lines.append(f" Avg cosine sim: {gi['avg_cos_sim']:.2f} | " f"Avg conflict: {gi['avg_conflict']:.0%}") @@ -6462,11 +6466,11 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): lines.append("Solo -- Use independently") for gi in solo_entries: idx = gi["indices"][0] - name = active_loras[idx]["name"] + name = _dn(active_loras[idx]["name"]) strength = gi["strength"] lines.append(f" * {name:<28s} strength {strength:.2f}") if gi["opposing"]: - opp_strs = [f"{oname} (cos: {cs:.2f})" for oname, cs in gi["opposing"][:3]] + opp_strs = [f"{_dn(oname)} (cos: {cs:.2f})" for oname, cs in gi["opposing"][:3]] lines.append(f" Opposes: {', '.join(opp_strs)}") lines.append("") @@ -6476,17 +6480,17 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): lines.append("") for w in warnings: if w["type"] == "opposing": - lines.append(f"! {w['name_i']} vs {w['name_j']}: Opposing (cos_sim: {w['cos_sim']:.2f})") + lines.append(f"! {_dn(w['name_i'])} vs {_dn(w['name_j'])}: Opposing (cos_sim: {w['cos_sim']:.2f})") lines.append(f" These cancel each other out -- using both will degrade quality.") lines.append("") elif w["type"] == "magnitude_group": - lines.append(f"! {w['stronger']} ({w['avg_ratio']:.1f}x avg) overshadows:") - lines.append(f" {', '.join(w['weaker_list'])}") + lines.append(f"! {_dn(w['stronger'])} ({w['avg_ratio']:.1f}x avg) overshadows:") + lines.append(f" {', '.join(_dn(n) for n in w['weaker_list'])}") lines.append(f" Consider lowering its strength or raising weaker LoRAs.") lines.append("") elif w["type"] == "magnitude": - lines.append(f"! {w['stronger']} is {w['ratio']:.1f}x stronger than {w['weaker']}") - lines.append(f" {w['weaker']} may be overshadowed at equal strengths.") + lines.append(f"! {_dn(w['stronger'])} is {w['ratio']:.1f}x stronger than {_dn(w['weaker'])}") + lines.append(f" {_dn(w['weaker'])} may be overshadowed at equal strengths.") lines.append("") # --- Pairwise Compatibility Table --- @@ -6495,7 +6499,10 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): # Sort by compat score descending sorted_pairs = sorted(pairwise_conflicts, key=lambda p: p["cosine_sim"] * (1.0 - p["ratio"]), reverse=True) - for pc in sorted_pairs: + # Compute max pair label width for right-aligned columns + pair_labels = [_dn(pc["pair"]) for pc in sorted_pairs] + max_pair_w = max((len(p) for p in pair_labels), default=45) + for pc, pair_label in zip(sorted_pairs, pair_labels): conflict = pc["ratio"] cos = round(pc["cosine_sim"], 2) + 0.0 # avoid -0.00 compat = round(pc["cosine_sim"] * (1.0 - conflict), 2) + 0.0 # avoid -0.00 @@ -6506,7 +6513,7 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): indicator = " [!!]" if pc.get("shared_prefixes", 1) == 0: indicator += " (no shared keys)" - lines.append(f" {pc['pair']:<45s} " + lines.append(f" {pair_label:<{max_pair_w}s} " f"cos:{cos:6.2f} conflict:{conflict:5.0%} compat:{compat:6.2f}{indicator}") lines.append("") lines.append("=" * 55) From 51f30b981886cefa6ffc1dd6629752cc33e27b0d Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 17:17:43 +0100 Subject: [PATCH 05/19] fix: use replace instead of removesuffix for .safetensors stripping removesuffix only strips from the end of the string, so pair labels like "A.safetensors vs B.safetensors" kept the first occurrence. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index ebe5501..d651baf 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -6424,7 +6424,7 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): def _dn(name): """Display name: strip .safetensors extension.""" - return name.removesuffix(".safetensors") + return name.replace(".safetensors", "") lines.append("=" * 55) lines.append(" LoRA Compatibility Analysis") From e015be3ff1278e13cb93e7a1805f13a205740331 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 17:22:45 +0100 Subject: [PATCH 06/19] fix: heatmap uses raw compat values and shows full readable labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use raw compat matrix (without +0.1 baseline) for heatmap so cells show actual similarity differences instead of uniform ~0.10 - Strip path and extension from heatmap labels (just the LoRA name) - Auto-size left margin from longest label instead of fixed 150px - Rotate column headers 90° so full names are readable Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 68 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index d651baf..ab86bb5 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -6252,9 +6252,10 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): active_loras, groups, group_info, lora_stats, pairwise_conflicts, collection_stats, warnings, detected_arch, prefix_count) - # --- Generate heatmap --- - lora_names = [item["name"] for item in active_loras] - heatmap = self._generate_heatmap(compat_matrix, lora_names, groups) + # --- Generate heatmap (use raw compat without baseline for visual differentiation) --- + raw_matrix = self._compute_raw_compat_matrix(pairwise_similarities, pairwise_conflicts, n_loras) + display_names = [os.path.splitext(os.path.basename(item["name"]))[0] for item in active_loras] + heatmap = self._generate_heatmap(raw_matrix, display_names, groups) return (report, heatmap) @@ -6284,6 +6285,21 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): matrix[j][i] = compat return matrix + @staticmethod + def _compute_raw_compat_matrix(pairwise_similarities, pairwise_conflicts, n_loras): + """Compute N×N raw compatibility matrix without baseline (for heatmap display).""" + conflict_by_pair = {} + for pc in pairwise_conflicts: + conflict_by_pair[(pc["i"], pc["j"])] = pc["ratio"] + + matrix = [[0.0] * n_loras for _ in range(n_loras)] + for (i, j), cos_sim in pairwise_similarities.items(): + conflict_ratio = conflict_by_pair.get((i, j), 0.0) + compat = cos_sim * (1.0 - conflict_ratio) + matrix[i][j] = compat + matrix[j][i] = compat + return matrix + @staticmethod def _cluster_loras(compat_matrix, threshold=0.05): """Greedy agglomerative clustering. Returns list of groups (lists of indices).""" @@ -6323,7 +6339,7 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): @staticmethod def _generate_heatmap(compat_matrix, lora_names, groups): """Generate N×N heatmap as torch IMAGE tensor using PIL.""" - from PIL import Image, ImageDraw + from PIL import Image, ImageDraw, ImageFont import numpy as np n = len(compat_matrix) @@ -6336,10 +6352,17 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): display_order.extend(g) cell_size = 80 - margin = 150 - img_size = margin + n * cell_size - img = Image.new("RGB", (img_size, img_size), (30, 30, 30)) + # Compute margin from longest label + tmp_img = Image.new("RGB", (1, 1)) + tmp_draw = ImageDraw.Draw(tmp_img) + max_label_w = max(tmp_draw.textlength(name) for name in lora_names) if lora_names else 100 + margin = int(max_label_w) + 20 # padding + col_header_h = margin # same space for rotated column headers + img_w = margin + n * cell_size + img_h = col_header_h + n * cell_size + + img = Image.new("RGB", (img_w, img_h), (30, 30, 30)) draw = ImageDraw.Draw(img) # Color mapping: continuous red (-0.4) → yellow (0.05) → green (+0.5) @@ -6365,7 +6388,7 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): orig_i = display_order[row] orig_j = display_order[col] x = margin + col * cell_size - y = margin + row * cell_size + y = col_header_h + row * cell_size if orig_i == orig_j: color = (80, 80, 80) val_text = "-" @@ -6386,24 +6409,35 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): text_color = (0, 0, 0) if brightness > 128 else (255, 255, 255) draw.text((tx, ty), val_text, fill=text_color) - # Draw labels (using reordered indices, truncated to 20 chars) + # Draw row labels (full name, right-aligned against grid) for pos in range(n): - label = lora_names[display_order[pos]][:20] - # Row labels (left) - y = margin + pos * cell_size + cell_size // 2 - 5 + label = lora_names[display_order[pos]] + y = col_header_h + pos * cell_size + cell_size // 2 - 5 draw.text((5, y), label, fill=(220, 220, 220)) - # Column labels (top, horizontal) - x = margin + pos * cell_size + 5 - draw.text((x, margin - 15), label[:10], fill=(220, 220, 220)) + + # Draw column labels (rotated 90° for readability) + for pos in range(n): + label = lora_names[display_order[pos]] + # Render text onto a temporary image, rotate, paste + bbox = tmp_draw.textbbox((0, 0), label) + tw = bbox[2] - bbox[0] + 4 + th = bbox[3] - bbox[1] + 4 + txt_img = Image.new("RGBA", (tw, th), (30, 30, 30, 255)) + txt_draw = ImageDraw.Draw(txt_img) + txt_draw.text((2, 2), label, fill=(220, 220, 220, 255)) + rotated = txt_img.rotate(90, expand=True) + paste_x = margin + pos * cell_size + (cell_size - rotated.width) // 2 + paste_y = col_header_h - rotated.height - 5 + img.paste(rotated.convert("RGB"), (paste_x, max(0, paste_y))) # Draw group borders (contiguous after reordering) pos = 0 for group in groups: if len(group) > 1: x0 = margin + pos * cell_size - 1 - y0 = margin + pos * cell_size - 1 + y0 = col_header_h + pos * cell_size - 1 x1 = margin + (pos + len(group)) * cell_size - y1 = margin + (pos + len(group)) * cell_size + y1 = col_header_h + (pos + len(group)) * cell_size for offset in range(3): draw.rectangle([x0 - offset, y0 - offset, x1 + offset, y1 + offset], outline=(255, 255, 255)) From 9d65ab3f7b29748453e5837c21ee219034ea1aeb Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 17:24:42 +0100 Subject: [PATCH 07/19] feat: make Compatibility Analyzer an output node that always re-executes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add OUTPUT_NODE = True so it works as an endpoint without needing downstream connections. IS_CHANGED returns NaN to bypass ComfyUI's execution cache — the node always runs when queued. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lora_optimizer.py b/lora_optimizer.py index ab86bb5..e1b05c9 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -5989,8 +5989,14 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): RETURN_NAMES = ("report", "compatibility_map") FUNCTION = "analyze" CATEGORY = "LoRA Optimizer" + OUTPUT_NODE = True DESCRIPTION = "Analyzes pairwise LoRA compatibility and recommends merge groups. No merge is performed — use this to plan which LoRAs to combine before merging." + @classmethod + def IS_CHANGED(cls, **kwargs): + """Always re-execute — analysis-only node, no caching.""" + return float("NaN") + def analyze(self, model, lora_stack, clip=None): if not lora_stack or len(lora_stack) == 0: return ("No LoRAs in stack.", self._empty_image()) From 6a2389a3b420213037967ef6883774e19df0d4c0 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 17:26:18 +0100 Subject: [PATCH 08/19] feat: add enabled toggle so analyzer doesn't run during normal queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defaults to off — sits inert in the workflow. Flip to True when you want to analyze, then back to False. IS_CHANGED returns a stable value when disabled so ComfyUI caches it as a no-op. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index e1b05c9..fd7671c 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -5977,6 +5977,8 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): def INPUT_TYPES(cls): return { "required": { + "enabled": ("BOOLEAN", {"default": False, + "tooltip": "Toggle on to run the analysis. Off by default so it doesn't slow down normal queue execution."}), "model": ("MODEL", {"tooltip": "Base model for key mapping and target shapes."}), "lora_stack": ("LORA_STACK", {"tooltip": "The LoRA stack to analyze for compatibility."}), }, @@ -5993,11 +5995,15 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): DESCRIPTION = "Analyzes pairwise LoRA compatibility and recommends merge groups. No merge is performed — use this to plan which LoRAs to combine before merging." @classmethod - def IS_CHANGED(cls, **kwargs): - """Always re-execute — analysis-only node, no caching.""" - return float("NaN") + def IS_CHANGED(cls, enabled, **kwargs): + if not enabled: + return "disabled" # stable value → cached, effectively skipped + return float("NaN") # always re-execute when enabled + + def analyze(self, enabled, model, lora_stack, clip=None): + if not enabled: + return ("Analysis disabled. Toggle 'enabled' to run.", self._empty_image()) - def analyze(self, model, lora_stack, clip=None): if not lora_stack or len(lora_stack) == 0: return ("No LoRAs in stack.", self._empty_image()) From 00f6f315d1fd3a3049f64560b6317d5f395e405c Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 18:00:20 +0100 Subject: [PATCH 09/19] fix: SaveMergedLoRA produces correct rank and dtype for Z-Image/fused-QKV models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three compounding bugs caused saved LoRAs to be 2.3GB (instead of ~350MB) and produce absurd images: 1. QKV refusion expanded all SVD-compressed LoRAAdapters to full-rank float32 diffs, destroying all compression. SaveMergedLoRA saw 0 low-rank patches. 2. With no surviving LoRAAdapters, auto-rank fell back to 128 — too low for a merge of two rank-128 LoRAs (needs sum_rank=256), losing ~50% of signal. 3. _compress_to_lowrank returned float32 tensors (SVD needs float32 internally but never cast back), doubling file size vs expected fp16. Fixes: - _compress_to_lowrank now preserves input dtype (also halves merge-time RAM) - QKV refusion downcasts fused diffs back to original dtype - Optimizer passes sum_rank in lora_data so SaveMergedLoRA uses correct rank - SaveMergedLoRA detects native dtype and enforces it on all saved tensors Co-Authored-By: Claude Opus 4.6 --- js/lora_compatibility_analyzer.js | 105 ++++++++++++++++++++++++++++++ lora_optimizer.py | 55 +++++++++++++--- 2 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 js/lora_compatibility_analyzer.js diff --git a/js/lora_compatibility_analyzer.js b/js/lora_compatibility_analyzer.js new file mode 100644 index 0000000..8e10a4e --- /dev/null +++ b/js/lora_compatibility_analyzer.js @@ -0,0 +1,105 @@ +import { app } from "/scripts/app.js"; + +function findWidget(node, name) { + return node.widgets ? node.widgets.find((w) => w.name === name) : null; +} + +app.registerExtension({ + name: "LoRAOptimizer.CompatibilityAnalyzer", + nodeCreated(node) { + if (node.comfyClass !== "LoRACompatibilityAnalyzer") return; + + if (!node.properties) node.properties = {}; + if (!node.properties.createdGroupIds) node.properties.createdGroupIds = []; + + // Clean up child stack nodes when analyzer is deleted + const origOnRemoved = node.onRemoved; + node.onRemoved = function () { + const tracked = this.properties?.createdGroupIds || []; + for (const id of tracked) { + if (id != null) { + const child = app.graph.getNodeById(id); + if (child && child.comfyClass === "LoRAStackDynamic") { + app.graph.remove(child); + } + } + } + if (origOnRemoved) origOnRemoved.call(this); + }; + + const origOnExecuted = node.onExecuted; + node.onExecuted = function (message) { + if (origOnExecuted) origOnExecuted.call(this, message); + + const groups = message?.groups; + if (!groups) return; + + const tracked = node.properties.createdGroupIds || []; + const newTracked = []; + + // Update or create nodes for each group + for (let i = 0; i < groups.length; i++) { + const group = groups[i]; + let stackNode = null; + + // Try to reuse existing tracked node + if (i < tracked.length && tracked[i] != null) { + stackNode = app.graph.getNodeById(tracked[i]); + } + + if (stackNode && stackNode.comfyClass === "LoRAStackDynamic") { + // Update existing node + populateStackNode(stackNode, group, i); + newTracked.push(stackNode.id); + } else { + // Create new node + const created = LiteGraph.createNode("LoRAStackDynamic"); + if (!created) { + console.warn("[CompatibilityAnalyzer] LoRAStackDynamic node type not found"); + continue; + } + app.graph.add(created); + created.pos = [ + node.pos[0] + node.size[0] + 50, + node.pos[1] + i * 250, + ]; + created.title = `Group ${i + 1} (Analyzer)`; + + // Delay population to let widgets initialize (must exceed + // the 100ms initial-visibility timeout in lora_stack_dynamic.js) + setTimeout(() => populateStackNode(created, group, i), 200); + newTracked.push(created.id); + } + } + + // Remove orphaned nodes (groups shrank) + for (let i = groups.length; i < tracked.length; i++) { + if (tracked[i] != null) { + const orphan = app.graph.getNodeById(tracked[i]); + if (orphan && orphan.comfyClass === "LoRAStackDynamic") { + app.graph.remove(orphan); + } + } + } + + node.properties.createdGroupIds = newTracked; + app.canvas?.setDirty?.(true, true); + }; + }, +}); + +function populateStackNode(stackNode, group, groupIndex) { + const modeWidget = findWidget(stackNode, "input_mode"); + if (modeWidget) modeWidget.value = "text"; + + const countWidget = findWidget(stackNode, "lora_count"); + if (countWidget) countWidget.value = group.loras.length; + + for (let i = 0; i < group.loras.length; i++) { + const textWidget = findWidget(stackNode, `lora_name_text_${i + 1}`); + if (textWidget) textWidget.value = group.loras[i].name; + + const strengthWidget = findWidget(stackNode, `strength_${i + 1}`); + if (strengthWidget) strengthWidget.value = group.loras[i].strength; + } +} diff --git a/lora_optimizer.py b/lora_optimizer.py index fd7671c..f210bb5 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -1085,19 +1085,33 @@ class _LoRAMergeBase: # Independent decompositions (e.g., SVD-compressed) — # expand to full-rank diffs, then fuse as a single diff patch parts = [] + store_dtype = q_data[0].dtype for comp_data in [q_data, k_data, v_data]: alpha = comp_data[2] if comp_data[2] is not None else float(comp_data[1].shape[0]) rank = comp_data[1].shape[0] diff = torch.mm(comp_data[0].float(), comp_data[1].float()) * (alpha / rank) parts.append(diff) fused_diff = torch.cat(parts, dim=0) + # Downcast back to original dtype to avoid doubling memory/file size + if store_dtype not in (torch.float32, torch.float64): + fused_diff = fused_diff.to(store_dtype) fused[fused_key] = ("diff", (fused_diff,)) elif any(hasattr(p, 'weights') for p in [q_patch, k_patch, v_patch]): # LoKr/LoHa or mixed adapter types: expand to diff, then fuse parts = [] + # Try to detect the native dtype before float32 expansion + store_dtype = torch.float16 + for p in [q_patch, k_patch, v_patch]: + if hasattr(p, 'weights') and p.weights[0] is not None: + dt = p.weights[0].dtype + if dt not in (torch.float32, torch.float64): + store_dtype = dt + break for comp_patch in [q_patch, k_patch, v_patch]: parts.append(self._expand_patch_to_diff(comp_patch)) fused_diff = torch.cat(parts, dim=0) + if store_dtype not in (torch.float32, torch.float64): + fused_diff = fused_diff.to(store_dtype) fused[fused_key] = ("diff", (fused_diff,)) else: # Unknown patch format — pass through unfused @@ -1727,16 +1741,19 @@ class _LoRAMergeBase: @staticmethod @torch.no_grad() - def _compress_to_lowrank(diff, rank, svd_device=None): + def _compress_to_lowrank(diff, rank, svd_device=None, output_dtype=None): """ Re-compress a full-rank diff tensor to low-rank via truncated SVD. Returns ("lora", (mat_up, mat_down, alpha=rank, None)) so ComfyUI computes up @ down * (rank/rank) = up @ down (no extra scaling). svd_device: where to run SVD. GPU is ~10-50x faster. CPU if None. + output_dtype: cast output to this dtype. None = same as input (fp16 stays fp16). For a [4096, 4096] diff at rank 128: 64MB → 2MB (~32x reduction). """ original_shape = diff.shape + if output_dtype is None: + output_dtype = diff.dtype # Reshape to 2D for SVD: [out_features, in_features] mat = diff.reshape(original_shape[0], -1).float() rank = min(rank, min(mat.shape)) @@ -1750,8 +1767,8 @@ class _LoRAMergeBase: # Reconstruct as: mat_up = U * sqrt(S), mat_down = sqrt(S) * V^T # Return on CPU for storage (ComfyUI moves to device when applying) sqrt_S = S.sqrt() - mat_up = (U * sqrt_S.unsqueeze(0)).cpu() # [out, rank] - mat_down = ((V * sqrt_S.unsqueeze(0)).T).cpu() # [rank, in] + mat_up = (U * sqrt_S.unsqueeze(0)).to(output_dtype).cpu() # [out, rank] + mat_down = ((V * sqrt_S.unsqueeze(0)).T).to(output_dtype).cpu() # [rank, in] del U, S, V, sqrt_S # alpha=rank so ComfyUI computes: up @ down * (rank/rank) = up @ down return LoRAAdapter(set(), (mat_up, mat_down, float(rank), None, None, None)) @@ -4125,9 +4142,9 @@ class LoRAOptimizer(_LoRAMergeBase): torch.cuda.empty_cache() # Resolve compress_patches rank (sum of input LoRA ranks) + sum_rank = sum(int(stat["avg_rank"]) for stat in lora_stats if stat["avg_rank"] > 0) compress_rank = 0 # 0 = disabled if compress_patches in ("non_ties", "all"): - sum_rank = sum(int(stat["avg_rank"]) for stat in lora_stats if stat["avg_rank"] > 0) compress_rank = max(sum_rank, 64) # floor at 64 logging.info(f"[LoRA Optimizer] Patch compression: {compress_patches} (rank {compress_rank} from sum of input LoRA ranks)") @@ -4667,6 +4684,7 @@ class LoRAOptimizer(_LoRAMergeBase): "output_strength": output_strength, "clip_strength": clip_strength_out, "suggested_max_strength": suggested_max_strength, + "sum_rank": max(sum_rank, 64), } # Cache patches for re-use (single entry to limit memory) @@ -5716,23 +5734,44 @@ class SaveMergedLoRA: auto_rank = save_rank == 0 - # Auto mode: collect ranks from existing LoRAAdapter patches to use - # as the fallback rank for any full-rank diffs that need compression + # Auto mode: determine rank for compressing full-rank diffs. + # Priority: 1) existing LoRAAdapter ranks, 2) sum_rank from optimizer, 3) fallback 128 if auto_rank: existing_ranks = [] for patch in list(model_patches.values()) + list(clip_patches.values()): if isinstance(patch, LoRAAdapter): existing_ranks.append(patch.weights[1].shape[0]) # mat_down rows = rank # LoKr/LoHa factor ranks (1-8) are not comparable to LoRA ranks — - # skip them so the 128 fallback kicks in instead - # Use the most common rank, or 128 as a safe default + # skip them so the fallback kicks in instead if existing_ranks: fallback_rank = max(set(existing_ranks), key=existing_ranks.count) + elif lora_data.get("sum_rank", 0) > 0: + # Use sum of input LoRA ranks from the optimizer (e.g., QKV refusion + # may have expanded all LoRAAdapters to full-rank diffs) + fallback_rank = lora_data["sum_rank"] else: fallback_rank = 128 logging.info(f"[Save Merged LoRA] Auto rank: using rank {fallback_rank} for full-rank patches " f"(from {len(existing_ranks)} low-rank patches)") + # Detect native storage dtype (fp16/bf16) from patches for output. + # SVD compression produces float32 internally, but saved LoRAs should + # match the original weight precision to avoid doubling file size. + save_dtype = None + for patch in list(model_patches.values()) + list(clip_patches.values()): + if isinstance(patch, tuple) and patch[0] == "diff": + dt = patch[1][0].dtype + elif hasattr(patch, 'weights') and patch.weights[0] is not None: + dt = patch.weights[0].dtype + else: + continue + if dt not in (torch.float32, torch.float64): + save_dtype = dt + break + if save_dtype is None: + save_dtype = torch.float16 + logging.info(f"[Save Merged LoRA] Output dtype: {save_dtype}") + state_dict = {} for is_clip, patches in [(False, model_patches), (True, clip_patches)]: From b7003fb31bb84713e106c171324237d3caaac5f2 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 18:01:30 +0100 Subject: [PATCH 10/19] feat: analyzer sends group data via ui dict for auto-creating LoRA Stack nodes Change analyzer return from bare tuple to {"ui": {"groups": [...]}, "result": (...)} so the JS extension can auto-create LoRAStackDynamic nodes pre-populated with each merge group's LoRAs and strengths. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index f210bb5..a8310c2 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -5803,8 +5803,8 @@ class SaveMergedLoRA: strength = clip_strength if is_clip else output_strength alpha *= strength - state_dict[f"{lora_prefix}.lora_up.weight"] = mat_up.contiguous() - state_dict[f"{lora_prefix}.lora_down.weight"] = mat_down.contiguous() + state_dict[f"{lora_prefix}.lora_up.weight"] = mat_up.to(save_dtype).contiguous() + state_dict[f"{lora_prefix}.lora_down.weight"] = mat_down.to(save_dtype).contiguous() state_dict[f"{lora_prefix}.alpha"] = torch.tensor(alpha) save_file(state_dict, save_path) @@ -6041,16 +6041,16 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): def analyze(self, enabled, model, lora_stack, clip=None): if not enabled: - return ("Analysis disabled. Toggle 'enabled' to run.", self._empty_image()) + return {"ui": {"groups": []}, "result": ("Analysis disabled. Toggle 'enabled' to run.", self._empty_image())} if not lora_stack or len(lora_stack) == 0: - return ("No LoRAs in stack.", self._empty_image()) + return {"ui": {"groups": []}, "result": ("No LoRAs in stack.", self._empty_image())} normalized_stack = self._normalize_stack(lora_stack) active_loras = [item for item in normalized_stack if item["strength"] != 0] if len(active_loras) == 0: - return ("No active LoRAs in stack (all zero strength).", self._empty_image()) + return {"ui": {"groups": []}, "result": ("No active LoRAs in stack (all zero strength).", self._empty_image())} n_loras = len(active_loras) detected_arch = getattr(self, '_detected_arch', None) or 'unknown' @@ -6066,7 +6066,7 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): "Nothing to compare — add more LoRAs to analyze compatibility.\n" "\n" + "=" * 55 ) - return (report, self._empty_image()) + return {"ui": {"groups": []}, "result": (report, self._empty_image())} # --- Setup --- logging.info(f"[Compatibility Analyzer] Analyzing {n_loras} LoRAs...") @@ -6096,9 +6096,10 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): prefix_count = analysis["prefix_count"] if prefix_count == 0: - return ("No compatible LoRA keys found. " - "LoRAs may be incompatible with this model architecture.", - self._empty_image()) + return {"ui": {"groups": []}, "result": ( + "No compatible LoRA keys found. " + "LoRAs may be incompatible with this model architecture.", + self._empty_image())} # --- Finalize stats --- per_lora_stats = analysis["per_lora_stats"] @@ -6308,7 +6309,21 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): display_names = [os.path.splitext(os.path.basename(item["name"]))[0] for item in active_loras] heatmap = self._generate_heatmap(raw_matrix, display_names, groups) - return (report, heatmap) + # --- Build group data for UI (auto-create LoRA Stack nodes) --- + groups_for_ui = [] + for gi in group_info: + if gi["type"] != "merge": + continue + groups_for_ui.append({ + "loras": [ + {"name": active_loras[idx]["name"], "strength": round(gi["strengths"][idx], 4)} + for idx in gi["indices"] + ], + "suggested_merge": gi.get("suggested_merge", "weighted_average"), + "confidence": gi.get("confidence", "Low"), + }) + + return {"ui": {"groups": groups_for_ui}, "result": (report, heatmap)} @staticmethod def _empty_image(): From 4aa46a8d700ae220259247b5906494e7b4a868c2 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 18:30:53 +0100 Subject: [PATCH 11/19] fix: SaveMergedLoRA loses Q/K data for models with fused QKV weights MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For Z-Image and similar architectures with fused QKV attention weights, the normalization step splits fused QKV LoRA keys into separate to_q/to_k/to_v entries, each targeting a different SLICE of the same model weight (via offset tuples). The reverse_key_map stripped the offset with target_key[0], causing all three Q/K/V entries to collide — only V survived, losing Q and K data entirely. This produced garbage output from saved LoRAs. Fix: use the full target_key (including offset tuple) as the reverse_key_map dict key, and look up by full target_key in save_lora. Each Q/K/V slice now gets its own LoRA prefix in the saved file, which standard loaders handle. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index a8310c2..3481d3e 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -4609,10 +4609,12 @@ class LoRAOptimizer(_LoRAMergeBase): # Build reverse key map: target_key → lora_prefix # (used by SaveMergedLoRA to reconstruct standard LoRA key names) + # Use the FULL target_key (including offset tuple) as the dict key so + # split Q/K/V patches targeting different slices of the same fused weight + # don't overwrite each other (e.g., Z-Image fused QKV). reverse_key_map = {} for lora_prefix, (target_key, is_clip) in all_key_targets.items(): - tkey = target_key[0] if isinstance(target_key, tuple) else target_key - reverse_key_map[tkey] = lora_prefix + reverse_key_map[target_key] = lora_prefix # Auto output strength: -1 = use suggested max strength auto_output_strength = False @@ -5776,8 +5778,12 @@ class SaveMergedLoRA: for is_clip, patches in [(False, model_patches), (True, clip_patches)]: for target_key, patch in patches.items(): - tkey = target_key[0] if isinstance(target_key, tuple) else target_key - lora_prefix = key_map.get(tkey, tkey) + # Look up by full target_key first (preserves offset info for split + # Q/K/V patches), then fall back to base key string for non-tuple keys. + lora_prefix = key_map.get(target_key) + if lora_prefix is None: + tkey = target_key[0] if isinstance(target_key, tuple) else target_key + lora_prefix = key_map.get(tkey, tkey) if isinstance(patch, (LoKrAdapter, LoHaAdapter)): # Expand LoKr/LoHa to full diff and compress to standard LoRA format From 8a47dc33b4bf4cad3a7f11399e6d1f967ab3d246 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sat, 7 Mar 2026 23:59:56 +0100 Subject: [PATCH 12/19] fix: ensure saved tensors on CPU + add save validation diagnostics - Add .cpu() before saving to prevent safetensors errors when patches are on GPU (LoRAAdapter patches kept in VRAM by the optimizer) - Add comprehensive save-time validation: detect unmapped keys (fallback mapping), NaN tensors, all-zero tensors, and log sample prefixes - These diagnostics will help identify why saved LoRAs may produce garbage output on load Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 48 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index 3481d3e..a5f86d5 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -5809,12 +5809,54 @@ class SaveMergedLoRA: strength = clip_strength if is_clip else output_strength alpha *= strength - state_dict[f"{lora_prefix}.lora_up.weight"] = mat_up.to(save_dtype).contiguous() - state_dict[f"{lora_prefix}.lora_down.weight"] = mat_down.to(save_dtype).contiguous() + state_dict[f"{lora_prefix}.lora_up.weight"] = mat_up.to(save_dtype).cpu().contiguous() + state_dict[f"{lora_prefix}.lora_down.weight"] = mat_down.to(save_dtype).cpu().contiguous() state_dict[f"{lora_prefix}.alpha"] = torch.tensor(alpha) + # Validate saved data before writing + n_keys = len(state_dict) // 3 + unmapped_keys = [] + nan_keys = [] + zero_keys = [] + shape_info = [] + for is_clip, patches in [(False, model_patches), (True, clip_patches)]: + for target_key, patch in patches.items(): + direct = key_map.get(target_key) + if direct is None: + tkey = target_key[0] if isinstance(target_key, tuple) else target_key + fallback = key_map.get(tkey) + label = f"{'CLIP' if is_clip else 'MODEL'} {tkey}" + if fallback is None: + unmapped_keys.append(f"{label} (using raw key as prefix)") + else: + unmapped_keys.append(f"{label} (fallback to base key)") + + for skey, tensor in state_dict.items(): + if skey.endswith('.alpha'): + continue + if torch.isnan(tensor).any(): + nan_keys.append(skey) + if tensor.abs().max().item() == 0: + zero_keys.append(skey) + + if unmapped_keys: + logging.warning(f"[Save Merged LoRA] {len(unmapped_keys)} keys fell through to fallback mapping:") + for k in unmapped_keys[:5]: + logging.warning(f" {k}") + if nan_keys: + logging.error(f"[Save Merged LoRA] {len(nan_keys)} tensors contain NaN!") + for k in nan_keys[:5]: + logging.error(f" {k}") + if zero_keys: + logging.warning(f"[Save Merged LoRA] {len(zero_keys)} tensors are all zeros") + + # Log first few saved key prefixes for debugging + prefixes = sorted(set(k.rsplit('.lora_', 1)[0] for k in state_dict if '.lora_' in k)) + if prefixes: + logging.info(f"[Save Merged LoRA] Sample prefixes: {prefixes[:3]} ... ({len(prefixes)} total)") + save_file(state_dict, save_path) - logging.info(f"[Save Merged LoRA] Saved {len(state_dict) // 3} LoRA keys to {save_path}") + logging.info(f"[Save Merged LoRA] Saved {n_keys} LoRA keys to {save_path}") return (save_path,) From bce8e2b47a10f753e14ce2a18bddc0f51492fe5c Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sun, 8 Mar 2026 00:16:03 +0100 Subject: [PATCH 13/19] feat: add SVD reconstruction error check in save validation Compares each saved lora_up @ lora_down against the original merged diff to quantify how much information SVD compression loses. Logs avg/max reconstruction error. High error would indicate rank too low. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index a5f86d5..7dd1a64 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -5818,7 +5818,6 @@ class SaveMergedLoRA: unmapped_keys = [] nan_keys = [] zero_keys = [] - shape_info = [] for is_clip, patches in [(False, model_patches), (True, clip_patches)]: for target_key, patch in patches.items(): direct = key_map.get(target_key) @@ -5855,6 +5854,44 @@ class SaveMergedLoRA: if prefixes: logging.info(f"[Save Merged LoRA] Sample prefixes: {prefixes[:3]} ... ({len(prefixes)} total)") + # Roundtrip SVD quality check: compare reconstructed diffs against originals + svd_errors = [] + for is_clip, patches in [(False, model_patches), (True, clip_patches)]: + for target_key, patch in patches.items(): + lora_prefix = key_map.get(target_key) + if lora_prefix is None: + continue + up_key = f"{lora_prefix}.lora_up.weight" + down_key = f"{lora_prefix}.lora_down.weight" + alpha_key = f"{lora_prefix}.alpha" + if up_key not in state_dict: + continue + # Reconstruct diff from saved up/down + saved_up = state_dict[up_key].float() + saved_down = state_dict[down_key].float() + saved_alpha = state_dict[alpha_key].item() + rank = saved_down.shape[0] + scale = saved_alpha / rank + reconstructed = torch.mm(saved_up, saved_down) * scale + # Get original diff for comparison + if isinstance(patch, tuple) and patch[0] == "diff": + orig_diff = patch[1][0].float() + strength = clip_strength if is_clip else output_strength + if bake_strength: + orig_with_strength = orig_diff * strength + else: + orig_with_strength = orig_diff + orig_norm = orig_with_strength.norm().item() + if orig_norm > 0: + error = (reconstructed - orig_with_strength).norm().item() / orig_norm + svd_errors.append(error) + if svd_errors: + avg_err = sum(svd_errors) / len(svd_errors) + max_err = max(svd_errors) + logging.info(f"[Save Merged LoRA] SVD reconstruction error: " + f"avg={avg_err:.4f}, max={max_err:.4f} " + f"({len(svd_errors)} diffs checked)") + save_file(state_dict, save_path) logging.info(f"[Save Merged LoRA] Saved {n_keys} LoRA keys to {save_path}") From 02d4a7a368f247b013daff4925457500d6d19ecb Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sun, 8 Mar 2026 00:38:53 +0100 Subject: [PATCH 14/19] fix: adaptive SVD rank estimation to prevent lossy LoRA compression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SaveMergedLoRA was using sum_rank (sum of input LoRA ranks) as the SVD compression rank, but enhanced merge quality operations (TALL-masks, orthogonalization, KnOTS) expand the effective rank far beyond sum_rank by adding selfish_additions back after merging. This caused 34% average reconstruction error, producing garbage output from saved LoRAs. - Add _estimate_save_rank() that samples diffs and iteratively increases rank until reconstruction error < 5% - Add SVD oversampling (20%) and increase niter from 2→4 in _compress_to_lowrank for better randomized SVD convergence - Increase save_rank max from 1024→2048 for manual override Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 75 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index 7dd1a64..ab00615 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -1761,8 +1761,15 @@ class _LoRAMergeBase: # Move to requested device for SVD (GPU is much faster for matmul-heavy randomized SVD) if svd_device is not None and mat.device != svd_device: mat = mat.to(svd_device) - U, S, V = torch.svd_lowrank(mat, q=rank) + # Oversample by 20% for better randomized SVD accuracy, then truncate. + # Without oversampling, svd_lowrank struggles when effective rank ≈ q. + q_oversample = min(rank + max(20, rank // 5), min(mat.shape)) + U, S, V = torch.svd_lowrank(mat, q=q_oversample, niter=4) del mat + # Truncate to target rank (oversampled columns improve accuracy but aren't kept) + U = U[:, :rank] + S = S[:rank] + V = V[:, :rank] # U: [out, rank], S: [rank], V: [in, rank] # Reconstruct as: mat_up = U * sqrt(S), mat_down = sqrt(S) * V^T # Return on CPU for storage (ComfyUI moves to device when applying) @@ -1773,6 +1780,52 @@ class _LoRAMergeBase: # alpha=rank so ComfyUI computes: up @ down * (rank/rank) = up @ down return LoRAAdapter(set(), (mat_up, mat_down, float(rank), None, None, None)) + @staticmethod + @torch.no_grad() + def _estimate_save_rank(initial_rank, model_patches, clip_patches, + max_error=0.05, max_rank_cap=1024, n_samples=3): + """ + Adaptively estimate the SVD rank needed to reconstruct merged diffs + within `max_error` relative error. Samples a few diffs, tests at + initial_rank, and doubles until error is acceptable or cap is reached. + """ + # Collect sample full-rank diffs + samples = [] + for patch in list(model_patches.values()) + list(clip_patches.values()): + if isinstance(patch, tuple) and patch[0] == "diff": + samples.append(patch[1][0]) + if len(samples) >= n_samples: + break + if not samples: + return max(initial_rank, 64) + + rank = max(initial_rank, 64) + for sample in samples: + mat = sample.reshape(sample.shape[0], -1).float() + dim_cap = min(mat.shape) + test_rank = rank + while test_rank < min(dim_cap, max_rank_cap): + q_over = min(test_rank + max(20, test_rank // 5), dim_cap) + U, S, V = torch.svd_lowrank(mat, q=q_over, niter=4) + reconstructed = (U[:, :test_rank] * S[:test_rank].unsqueeze(0)) @ V[:, :test_rank].T + orig_norm = mat.norm().item() + if orig_norm > 0: + error = (reconstructed - mat).norm().item() / orig_norm + else: + error = 0.0 + del U, S, V, reconstructed + if error <= max_error: + break + old = test_rank + test_rank = min(int(test_rank * 1.5), dim_cap, max_rank_cap) + if test_rank == old: + break + logging.info(f"[Save Merged LoRA] Rank {old} → {test_rank} " + f"(sample error {error:.4f} > {max_error})") + rank = max(rank, test_rank) + del mat + return rank + @torch.no_grad() def _merge_diffs(self, diffs_with_weights, mode, density=0.5, majority_sign_method="frequency", compute_device=None, sparsification="disabled", @@ -5698,8 +5751,8 @@ class SaveMergedLoRA: "save_folder": (folder_choices, {"tooltip": "Which loras folder to save into. Lists all configured lora paths (from extra_model_paths.yaml and defaults)."}), "filename": ("STRING", {"default": "merged_lora", "tooltip": "Name for the saved file. Subdirectories allowed (e.g. 'merged/my_lora'). Extension .safetensors is added automatically."}), "save_rank": ("INT", { - "default": 0, "min": 0, "max": 1024, "step": 4, - "tooltip": "0 = auto (uses each layer's existing rank from the merge — recommended). Non-zero = force this rank for any layers that need compression. Higher values = more accurate but larger file." + "default": 0, "min": 0, "max": 2048, "step": 4, + "tooltip": "0 = auto (adaptively finds the rank needed for <5%% reconstruction error — recommended). Non-zero = force this rank for any layers that need compression. Higher values = more accurate but larger file." }), "bake_strength": ("BOOLEAN", { "default": True, @@ -5737,7 +5790,7 @@ class SaveMergedLoRA: auto_rank = save_rank == 0 # Auto mode: determine rank for compressing full-rank diffs. - # Priority: 1) existing LoRAAdapter ranks, 2) sum_rank from optimizer, 3) fallback 128 + # Priority: 1) existing LoRAAdapter ranks, 2) adaptive from sample diffs, 3) sum_rank, 4) fallback 128 if auto_rank: existing_ranks = [] for patch in list(model_patches.values()) + list(clip_patches.values()): @@ -5747,14 +5800,14 @@ class SaveMergedLoRA: # skip them so the fallback kicks in instead if existing_ranks: fallback_rank = max(set(existing_ranks), key=existing_ranks.count) - elif lora_data.get("sum_rank", 0) > 0: - # Use sum of input LoRA ranks from the optimizer (e.g., QKV refusion - # may have expanded all LoRAAdapters to full-rank diffs) - fallback_rank = lora_data["sum_rank"] + logging.info(f"[Save Merged LoRA] Auto rank: {fallback_rank} from {len(existing_ranks)} low-rank patches") else: - fallback_rank = 128 - logging.info(f"[Save Merged LoRA] Auto rank: using rank {fallback_rank} for full-rank patches " - f"(from {len(existing_ranks)} low-rank patches)") + # No low-rank patches — all diffs are full-rank (from merge). + # Estimate needed rank adaptively: sample a diff, compress, check error. + initial_rank = lora_data.get("sum_rank", 128) + fallback_rank = LoRAOptimizer._estimate_save_rank(initial_rank, model_patches, clip_patches) + logging.info(f"[Save Merged LoRA] Auto rank: {fallback_rank} " + f"(initial estimate {initial_rank}, adapted from sample diffs)") # Detect native storage dtype (fp16/bf16) from patches for output. # SVD compression produces float32 internally, but saved LoRAs should From bc2f74c2bd25d7a343300189ad6b12cf5b99b17f Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sun, 8 Mar 2026 00:42:29 +0100 Subject: [PATCH 15/19] fix: always run adaptive rank estimation when diff patches exist The previous logic checked existing_ranks first (from LoRAAdapters compressed during merge). With compress_patches="non_ties" (default), SLERP prefixes are LoRAAdapters but TIES prefixes remain as full-rank diffs. The existing_ranks branch would use rank 400 from the SLERP LoRAAdapters, never running adaptive estimation for the TIES diffs that actually needed it. Now checks for diff patches directly and always runs adaptive estimation when they exist. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index ab00615..099cc82 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -5790,24 +5790,26 @@ class SaveMergedLoRA: auto_rank = save_rank == 0 # Auto mode: determine rank for compressing full-rank diffs. - # Priority: 1) existing LoRAAdapter ranks, 2) adaptive from sample diffs, 3) sum_rank, 4) fallback 128 + # LoRAAdapter patches keep their existing rank (already compressed during merge). + # Full-rank diff patches (e.g. TIES) need adaptive estimation — element-wise + # operations like TIES trim/elect destroy low-rank structure, so sum_rank + # is not a reliable estimate of the effective rank. if auto_rank: - existing_ranks = [] - for patch in list(model_patches.values()) + list(clip_patches.values()): - if isinstance(patch, LoRAAdapter): - existing_ranks.append(patch.weights[1].shape[0]) # mat_down rows = rank - # LoKr/LoHa factor ranks (1-8) are not comparable to LoRA ranks — - # skip them so the fallback kicks in instead - if existing_ranks: - fallback_rank = max(set(existing_ranks), key=existing_ranks.count) - logging.info(f"[Save Merged LoRA] Auto rank: {fallback_rank} from {len(existing_ranks)} low-rank patches") - else: - # No low-rank patches — all diffs are full-rank (from merge). - # Estimate needed rank adaptively: sample a diff, compress, check error. + # Check if there are any full-rank diffs that need compression + has_diffs = any( + isinstance(patch, tuple) and patch[0] == "diff" + for patch in list(model_patches.values()) + list(clip_patches.values()) + ) + if has_diffs: initial_rank = lora_data.get("sum_rank", 128) fallback_rank = LoRAOptimizer._estimate_save_rank(initial_rank, model_patches, clip_patches) - logging.info(f"[Save Merged LoRA] Auto rank: {fallback_rank} " + logging.info(f"[Save Merged LoRA] Auto rank for diffs: {fallback_rank} " f"(initial estimate {initial_rank}, adapted from sample diffs)") + else: + # All patches are already low-rank (LoRAAdapter) — fallback_rank + # is only used for LoKr/LoHa expansion, use sum_rank or 128 + fallback_rank = lora_data.get("sum_rank", 128) + logging.info(f"[Save Merged LoRA] Auto rank: {fallback_rank} (no full-rank diffs to compress)") # Detect native storage dtype (fp16/bf16) from patches for output. # SVD compression produces float32 internally, but saved LoRAs should From 9d31de315875da6c671c71cdb6625314a033d562 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sun, 8 Mar 2026 00:48:49 +0100 Subject: [PATCH 16/19] fix: use exact SVD for rank estimation and high-rank compression TIES element-wise operations (trim/elect) destroy low-rank structure, making merged diffs effectively full-rank. The previous iterative approach with randomized SVD hit a 1024 cap while still at 26% error. - _estimate_save_rank: use torch.linalg.svdvals (exact) to find the precise rank needed for <5% error with no arbitrary cap - _compress_to_lowrank: use full SVD (torch.linalg.svd) when rank > dim/2, which is both faster and more accurate than randomized SVD at high rank ratios Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 79 ++++++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index 099cc82..9a92060 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -1761,15 +1761,23 @@ class _LoRAMergeBase: # Move to requested device for SVD (GPU is much faster for matmul-heavy randomized SVD) if svd_device is not None and mat.device != svd_device: mat = mat.to(svd_device) - # Oversample by 20% for better randomized SVD accuracy, then truncate. - # Without oversampling, svd_lowrank struggles when effective rank ≈ q. - q_oversample = min(rank + max(20, rank // 5), min(mat.shape)) - U, S, V = torch.svd_lowrank(mat, q=q_oversample, niter=4) - del mat - # Truncate to target rank (oversampled columns improve accuracy but aren't kept) - U = U[:, :rank] - S = S[:rank] - V = V[:, :rank] + # When rank > dim/2, full SVD + truncate is both faster and more accurate + # than randomized SVD (which would oversample beyond dim anyway). + if rank > min(mat.shape) // 2: + U, S, Vh = torch.linalg.svd(mat, full_matrices=False) + del mat + U = U[:, :rank] + S = S[:rank] + V = Vh[:rank, :].T # Vh is [min(dim), n], transpose slice to [n, rank] + del Vh + else: + # Oversample by 20% for better randomized SVD accuracy, then truncate. + q_oversample = min(rank + max(20, rank // 5), min(mat.shape)) + U, S, V = torch.svd_lowrank(mat, q=q_oversample, niter=4) + del mat + U = U[:, :rank] + S = S[:rank] + V = V[:, :rank] # U: [out, rank], S: [rank], V: [in, rank] # Reconstruct as: mat_up = U * sqrt(S), mat_down = sqrt(S) * V^T # Return on CPU for storage (ComfyUI moves to device when applying) @@ -1783,13 +1791,14 @@ class _LoRAMergeBase: @staticmethod @torch.no_grad() def _estimate_save_rank(initial_rank, model_patches, clip_patches, - max_error=0.05, max_rank_cap=1024, n_samples=3): + max_error=0.05, n_samples=3): """ - Adaptively estimate the SVD rank needed to reconstruct merged diffs - within `max_error` relative error. Samples a few diffs, tests at - initial_rank, and doubles until error is acceptable or cap is reached. + Determine the exact SVD rank needed to reconstruct merged diffs within + `max_error` relative Frobenius error. Uses exact SVD on a few sample + diffs to find the minimum rank where the truncation error drops below + the threshold. No arbitrary cap — rank is bounded only by min(dim). """ - # Collect sample full-rank diffs + # Collect sample full-rank diffs (prefer larger ones for conservative estimate) samples = [] for patch in list(model_patches.values()) + list(clip_patches.values()): if isinstance(patch, tuple) and patch[0] == "diff": @@ -1802,28 +1811,28 @@ class _LoRAMergeBase: rank = max(initial_rank, 64) for sample in samples: mat = sample.reshape(sample.shape[0], -1).float() - dim_cap = min(mat.shape) - test_rank = rank - while test_rank < min(dim_cap, max_rank_cap): - q_over = min(test_rank + max(20, test_rank // 5), dim_cap) - U, S, V = torch.svd_lowrank(mat, q=q_over, niter=4) - reconstructed = (U[:, :test_rank] * S[:test_rank].unsqueeze(0)) @ V[:, :test_rank].T - orig_norm = mat.norm().item() - if orig_norm > 0: - error = (reconstructed - mat).norm().item() / orig_norm - else: - error = 0.0 - del U, S, V, reconstructed - if error <= max_error: + # Exact SVD gives precise singular values — no randomized noise + S = torch.linalg.svdvals(mat) + total_sq = (S ** 2).sum().item() + if total_sq == 0: + continue + # Find minimum rank where residual/total < max_error + # ||A - A_k||_F / ||A||_F < max_error ⟺ residual_sq/total_sq < max_error^2 + threshold_sq = max_error * max_error * total_sq + cumulative_sq = 0.0 + needed = len(S) + for r in range(len(S)): + cumulative_sq += S[r].item() ** 2 + if total_sq - cumulative_sq <= threshold_sq: + needed = r + 1 break - old = test_rank - test_rank = min(int(test_rank * 1.5), dim_cap, max_rank_cap) - if test_rank == old: - break - logging.info(f"[Save Merged LoRA] Rank {old} → {test_rank} " - f"(sample error {error:.4f} > {max_error})") - rank = max(rank, test_rank) - del mat + if needed > rank: + error_at_initial = math.sqrt(max(0, total_sq - sum(s.item()**2 for s in S[:rank])) / total_sq) + logging.info(f"[Save Merged LoRA] Sample diff ({mat.shape[0]}x{mat.shape[1]}): " + f"need rank {needed} for <{max_error:.0%} error " + f"(rank {rank} has {error_at_initial:.1%} error)") + rank = needed + del S, mat return rank @torch.no_grad() From 686af8d27da98a50742b4a89e8451b8aa01dffa3 Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sun, 8 Mar 2026 01:14:04 +0100 Subject: [PATCH 17/19] feat: add tuner_data output to LoRA Optimizer, save_folder to SaveTunerData - LoRA Optimizer (Advanced) now outputs tuner_data (4th output, before lora_data) matching AutoTuner output order - LoRA Optimizer Simple accepts optional tuner_data input and removes lora_data output - SaveTunerData gets save_folder dropdown (like SaveMergedLoRA) and defaults to .tuner extension - All subclass unpackings updated for new 5-tuple return from optimize_merge Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 63 ++++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index 9a92060..4e7b092 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -2942,8 +2942,8 @@ class LoRAOptimizer(_LoRAMergeBase): } } - RETURN_TYPES = ("MODEL", "CLIP", "STRING", "LORA_DATA") - RETURN_NAMES = ("model", "clip", "analysis_report", "lora_data") + RETURN_TYPES = ("MODEL", "CLIP", "STRING", "TUNER_DATA", "LORA_DATA") + RETURN_NAMES = ("model", "clip", "analysis_report", "tuner_data", "lora_data") FUNCTION = "execute_node" CATEGORY = "LoRA Optimizer" DESCRIPTION = "Auto-analyzes LoRA stack and selects optimal merge strategy per weight group. Outputs merged model + analysis report. Best for style/character LoRAs — apply edit, distillation (LCM/Turbo/Hyper), or DPO LoRAs via a standard Load LoRA node instead." @@ -3876,7 +3876,7 @@ class LoRAOptimizer(_LoRAMergeBase): logging.info(f"[AutoTuner Bridge] Passthrough mode — model already merged by AutoTuner") # Return model as-is + UI data for JS widget sync - return {"result": (model, clip, report, None), + return {"result": (model, clip, report, tuner_data, None), "ui": {"applied_settings": [json.dumps(config)]}} # --- manual: active merge mode --- @@ -3914,13 +3914,13 @@ class LoRAOptimizer(_LoRAMergeBase): """ # Normalize stack format (standard tuples or LoRAStack dicts) if not lora_stack or len(lora_stack) == 0: - return (model, clip, "No LoRAs in stack.", None) + return (model, clip, "No LoRAs in stack.", None, None) normalized_stack = self._normalize_stack(lora_stack, normalize_keys=normalize_keys) active_loras = [item for item in normalized_stack if item["strength"] != 0] if len(active_loras) == 0: - return (model, clip, "No LoRAs in stack (all zero strength or malformed).", None) + return (model, clip, "No LoRAs in stack (all zero strength or malformed).", None, None) # Resolve architecture preset from override or auto-detection preset_key, arch_preset = _resolve_arch_preset( @@ -3990,7 +3990,7 @@ class LoRAOptimizer(_LoRAMergeBase): new_clip.add_patches(clip_patches, clip_strength_out) self._update_model_size(new_clip, clip_patches) logging.info(f"[LoRA Optimizer] Using cached merge result ({len(model_patches)} model + {len(clip_patches)} CLIP patches)") - return (new_model, new_clip, report, lora_data) + return (new_model, new_clip, report, None, lora_data) logging.info(f"[LoRA Optimizer] Starting analysis of {len(active_loras)} LoRAs") t_start = time.time() @@ -4054,7 +4054,7 @@ class LoRAOptimizer(_LoRAMergeBase): if prefix_count == 0: return (model, clip, "No compatible LoRA keys found. " - "LoRAs may be incompatible with this model architecture.", None) + "LoRAs may be incompatible with this model architecture.", None, None) # Log per-LoRA summaries for i, stat in enumerate(per_lora_stats): @@ -4770,7 +4770,7 @@ class LoRAOptimizer(_LoRAMergeBase): logging.info(f"[LoRA Optimizer] Done! {processed_keys} keys processed ({time.time() - t_start:.1f}s total)") - return (new_model, new_clip, report, lora_data) + return (new_model, new_clip, report, None, lora_data) class LoRAOptimizerSimple(LoRAOptimizer): @@ -4803,12 +4803,18 @@ class LoRAOptimizerSimple(LoRAOptimizer): "default": 1.0, "min": 0.0, "max": 10.0, "step": 0.05, "tooltip": "Multiplier for CLIP LoRA strengths (stacks with per-LoRA clip_strength when provided)." }), + "tuner_data": ("TUNER_DATA", { + "tooltip": "Connect from a Load Tuner Data or LoRA AutoTuner node. When connected, uses the best AutoTuner config instead of defaults." + }), }, } + RETURN_TYPES = ("MODEL", "CLIP", "STRING") + RETURN_NAMES = ("model", "clip", "analysis_report") FUNCTION = "optimize_merge" DESCRIPTION = ( "Simplified LoRA Optimizer — merges a LoRA stack with good defaults. " + "Accepts optional tuner_data to use AutoTuner results. " "Use LoRA Optimizer (Advanced) for sparsification, merge quality, " "SVD device, and other fine-tuning options." ) @@ -4833,19 +4839,26 @@ class LoRAOptimizerSimple(LoRAOptimizer): ) def optimize_merge(self, model, lora_stack, output_strength, - clip=None, clip_strength_multiplier=1.0, **_kwargs): - return super().optimize_merge( + clip=None, clip_strength_multiplier=1.0, + tuner_data=None, **_kwargs): + settings_source = "from_autotuner" if tuner_data is not None else "manual" + result = super().optimize_merge( model, lora_stack, output_strength, clip=clip, clip_strength_multiplier=clip_strength_multiplier, + tuner_data=tuner_data, settings_source=settings_source, **self._SIMPLE_DEFAULTS, ) + return result[:3] @classmethod def IS_CHANGED(cls, model, lora_stack, output_strength, - clip=None, clip_strength_multiplier=1.0, **_kwargs): + clip=None, clip_strength_multiplier=1.0, + tuner_data=None, **_kwargs): return LoRAOptimizer.IS_CHANGED( model, lora_stack, output_strength, clip=clip, clip_strength_multiplier=clip_strength_multiplier, + tuner_data=tuner_data, + settings_source="from_autotuner" if tuner_data is not None else "manual", **cls._SIMPLE_DEFAULTS, ) @@ -4976,7 +4989,7 @@ class LoRAAutoTuner(LoRAOptimizer): # Single LoRA: nothing to tune, delegate directly if output_mode == "tuning_only": return (model, clip, "Single LoRA detected -- tuning_only passthrough.", "", None, None) - merged_model, merged_clip, report, lora_data = super().optimize_merge( + merged_model, merged_clip, report, _, lora_data = super().optimize_merge( model, lora_stack, output_strength, clip=clip, clip_strength_multiplier=clip_strength_multiplier, normalize_keys=normalize_keys, behavior_profile="v1.2", @@ -5185,7 +5198,7 @@ class LoRAAutoTuner(LoRAOptimizer): # merge_strategy_override would force one mode everywhere, defeating per-prefix logic. strategy_override = config["merge_mode"] if config["optimization_mode"] == "global" else "" - merged_model, merged_clip, _report, lora_data = super().optimize_merge( + merged_model, merged_clip, _report, _, lora_data = super().optimize_merge( model, lora_stack, output_strength, clip=clip, clip_strength_multiplier=clip_strength_multiplier, @@ -5267,7 +5280,7 @@ class LoRAAutoTuner(LoRAOptimizer): f"({best_config['merge_mode']}, {best_config['merge_quality']})...") t_final = time.time() strategy_override = best_config["merge_mode"] if best_config["optimization_mode"] == "global" else "" - best_model, best_clip, best_analysis_report, best_lora_data = super().optimize_merge( + best_model, best_clip, best_analysis_report, _, best_lora_data = super().optimize_merge( model, lora_stack, output_strength, clip=clip, clip_strength_multiplier=clip_strength_multiplier, @@ -5614,7 +5627,7 @@ class LoRAMergeSelector(LoRAOptimizer): # In per_prefix mode, let the optimizer auto-select strategy per prefix. strategy_override = config["merge_mode"] if config["optimization_mode"] == "global" else "" - merged_model, merged_clip, _report, _lora_data = super().optimize_merge( + merged_model, merged_clip, _report, _, _lora_data = super().optimize_merge( model, lora_stack, output_strength, clip=clip, clip_strength_multiplier=clip_strength_multiplier, @@ -5727,7 +5740,7 @@ class WanVideoLoRAOptimizer(LoRAOptimizer): kwargs.pop("clip", None) kwargs.pop("clip_strength_multiplier", None) kwargs.pop("free_vram_between_passes", None) - model_out, _clip, report, lora_data = super().optimize_merge( + model_out, _clip, report, _, lora_data = super().optimize_merge( model, lora_stack, output_strength, clip=None, clip_strength_multiplier=1.0, **kwargs ) @@ -5963,14 +5976,17 @@ class SaveMergedLoRA: class SaveTunerData: - """Saves AutoTuner results (TUNER_DATA) to a JSON file for later reuse.""" + """Saves AutoTuner results (TUNER_DATA) to a .tuner file for later reuse.""" @classmethod def INPUT_TYPES(cls): + tuner_folders = folder_paths.get_folder_paths("tuner_data") + folder_choices = tuner_folders if tuner_folders else [TUNER_DATA_DIR] return { "required": { "tuner_data": ("TUNER_DATA", {"tooltip": "Connect the tuner_data output from LoRA AutoTuner."}), - "filename": ("STRING", {"default": "tuner_data", "tooltip": "Filename (saved to models/tuner_data/). Absolute path saves to that location instead."}), + "save_folder": (folder_choices, {"tooltip": "Which tuner_data folder to save into. Lists all configured paths (from extra_model_paths.yaml and defaults)."}), + "filename": ("STRING", {"default": "tuner_data", "tooltip": "Filename. Subdirectories allowed (e.g. 'results/experiment1'). Extension .tuner is added automatically (.json also accepted)."}), } } @@ -5979,14 +5995,15 @@ class SaveTunerData: FUNCTION = "save_tuner_data" CATEGORY = "LoRA Optimizer" OUTPUT_NODE = True - DESCRIPTION = "Saves AutoTuner results to a JSON file so they can be reloaded later without re-running the tuner." + DESCRIPTION = "Saves AutoTuner results to a .tuner file so they can be reloaded later without re-running the tuner." - def save_tuner_data(self, tuner_data, filename): + def save_tuner_data(self, tuner_data, save_folder, filename): if tuner_data is None: return ("",) - base = filename if filename.endswith(".json") else f"{filename}.json" - save_path = os.path.join(TUNER_DATA_DIR, base) - if not os.path.realpath(save_path).startswith(os.path.realpath(TUNER_DATA_DIR) + os.sep): + save_dir = save_folder + base = filename if filename.endswith((".json", ".tuner")) else f"{filename}.tuner" + save_path = os.path.join(save_dir, base) + if not os.path.realpath(save_path).startswith(os.path.realpath(save_dir) + os.sep): raise ValueError(f"[Save Tuner Data] Path escapes tuner_data directory: {filename}") os.makedirs(os.path.dirname(save_path), exist_ok=True) with open(save_path, "w") as f: From 1f7c03e81f24fb66b3b03fee39eeca2f627fb54c Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sun, 8 Mar 2026 01:49:05 +0100 Subject: [PATCH 18/19] feat: add full-rank LoRA aware merge path for rank 256+ LoRAs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full-rank LoRAs (rank 800+) produce garbage when merged because the merge algorithm's SLERP upgrade, conflict interpretation, and auto-strength floor were tuned for low-rank LoRAs. This adds a full-rank detection gate (avg rank >= 256) that switches to safer strategies: disables SLERP upgrade, uses weighted_sum for orthogonal patches, and raises the auto-strength floor to 1.0 — all without touching existing low-rank thresholds. Also adds overwrite toggle to SaveTunerData node. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 86 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index 4e7b092..261d5e2 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -50,6 +50,12 @@ _ARCH_PRESETS = { "suggested_max_strength_cap": 3.0, "auto_strength_orthogonal_floor": 0.85, "display_name": "SD/SDXL UNet", + "full_rank": { + "rank_threshold": 256, + "disable_slerp_upgrade": True, + "prefer_sum_orthogonal": True, + "auto_strength_floor": 1.0, + }, }, "dit": { "density_noise_floor_ratio": 0.05, @@ -66,6 +72,12 @@ _ARCH_PRESETS = { "suggested_max_strength_cap": 5.0, "auto_strength_orthogonal_floor": 0.85, "display_name": "DiT (Flux/WAN/Z-Image/LTX/HunyuanVideo)", + "full_rank": { + "rank_threshold": 256, + "disable_slerp_upgrade": True, + "prefer_sum_orthogonal": True, + "auto_strength_floor": 1.0, + }, }, "llm": { "density_noise_floor_ratio": 0.15, @@ -82,6 +94,12 @@ _ARCH_PRESETS = { "suggested_max_strength_cap": 3.0, "auto_strength_orthogonal_floor": 0.9, "display_name": "LLM (Qwen/LLaMA)", + "full_rank": { + "rank_threshold": 256, + "disable_slerp_upgrade": True, + "prefer_sum_orthogonal": True, + "auto_strength_floor": 1.0, + }, }, } @@ -3316,7 +3334,7 @@ class LoRAOptimizer(_LoRAMergeBase): return max(arch_preset["density_clamp_min"], min(arch_preset["density_clamp_max"], above_noise)) - def _compute_auto_strengths(self, active_loras, lora_stats, pairwise_similarities=None, arch_preset=None, detected_arch=None, auto_strength_floor=-1.0): + def _compute_auto_strengths(self, active_loras, lora_stats, pairwise_similarities=None, arch_preset=None, detected_arch=None, auto_strength_floor=-1.0, is_full_rank=False): """ Compute reduced per-LoRA strengths using interference-aware energy normalization. Uses pairwise cosine similarity to account for @@ -3395,6 +3413,10 @@ class LoRAOptimizer(_LoRAMergeBase): if abs(avg_cos) <= alignment_thresh: if auto_strength_floor >= 0: floor = auto_strength_floor + elif is_full_rank: + # Full-rank patches already represent the complete weight delta; + # scaling them down loses information. + floor = arch_preset.get("full_rank", {}).get("auto_strength_floor", 1.0) else: floor = _VIDEO_ARCH_ORTHOGONAL_FLOOR.get( detected_arch, @@ -3415,7 +3437,10 @@ class LoRAOptimizer(_LoRAMergeBase): reasoning.append(f"Scale factor: {scale:.4f}") if floor_applied: arch_label = detected_arch or "unknown" - reasoning.append(f" (orthogonal floor {floor:.2f} applied for {arch_label} — preserving independent contributions)") + if is_full_rank: + reasoning.append(f" (full-rank orthogonal floor {floor:.2f} applied — preserving complete weight deltas)") + else: + reasoning.append(f" (orthogonal floor {floor:.2f} applied for {arch_label} — preserving independent contributions)") if pairwise_similarities: avg_cos = sum(pairwise_similarities.values()) / len(pairwise_similarities) orthogonal_energy = math.sqrt(orthogonal_energy_sq) @@ -3575,7 +3600,8 @@ class LoRAOptimizer(_LoRAMergeBase): merge_quality="standard", compatibility_warnings=None, behavior_profile="v1.2", - architecture_preset=None): + architecture_preset=None, + is_full_rank=False): """Format analysis as a multi-line report string.""" lines = [] lines.append("=" * 50) @@ -3613,6 +3639,13 @@ class LoRAOptimizer(_LoRAMergeBase): lines.append(f"Key normalization: enabled") lines.append("") + if is_full_rank: + global_avg_rank = (sum(s["avg_rank"] for s in lora_stats) / len(lora_stats)) if lora_stats else 0 + lines.append("Full-rank LoRAs detected (avg rank {:.0f})".format(global_avg_rank)) + lines.append(" Using full-rank merge path: SLERP upgrade disabled,") + lines.append(" orthogonal patches use weighted_sum, auto-strength floor raised") + lines.append("") + if compatibility_warnings: lines.append("") lines.append("!!! COMPATIBILITY WARNING !!!") @@ -4084,6 +4117,19 @@ class LoRAOptimizer(_LoRAMergeBase): "key_filter": active_loras[i].get("key_filter", "all"), }) + # Full-rank detection: when average rank exceeds threshold, switch to + # safer merge strategies that preserve information in high-dimensional patches. + fr_preset = arch_preset.get("full_rank", {}) + fr_rank_threshold = fr_preset.get("rank_threshold", 256) + global_avg_rank = (sum(s["avg_rank"] for s in lora_stats) / len(lora_stats)) if lora_stats else 0 + is_full_rank = global_avg_rank >= fr_rank_threshold + if is_full_rank: + logging.info(f"[LoRA Optimizer] Full-rank LoRAs detected (avg rank {global_avg_rank:.0f} >= {fr_rank_threshold})") + if fr_preset.get("disable_slerp_upgrade", False): + logging.info(f"[LoRA Optimizer] SLERP upgrade disabled for full-rank patches") + if fr_preset.get("prefer_sum_orthogonal", False): + logging.info(f"[LoRA Optimizer] Orthogonal full-rank patches will use weighted_sum (additive)") + # Pairwise conflict stats and cosine similarity from accumulated counts total_overlap = 0 total_conflict = 0 @@ -4174,7 +4220,8 @@ class LoRAOptimizer(_LoRAMergeBase): active_loras, lora_stats, pairwise_similarities=pairwise_similarities, arch_preset=arch_preset, detected_arch=getattr(self, '_detected_arch', None), - auto_strength_floor=auto_strength_floor) + auto_strength_floor=auto_strength_floor, + is_full_rank=is_full_rank) for i in range(len(active_loras)): orig = active_loras[i]["strength"] @@ -4303,10 +4350,19 @@ class LoRAOptimizer(_LoRAMergeBase): pf_raw_cos = pf.get("avg_cos_sim", 0.0) pf_orthogonal = abs(pf_raw_cos) < arch_preset["orthogonal_cos_sim_max"] pf_opposing = pf_raw_cos < 0 + # Full-rank gate: skip SLERP upgrade — for full-rank patches the + # information is spread across all dimensions, and SLERP's + # hypersphere interpolation loses signal from both LoRAs. if (pf_mode == "weighted_average" and pf["n_loras"] >= 2 and behavior_profile == "v1.2" - and not pf_opposing): + and not pf_opposing + and not (is_full_rank and fr_preset.get("disable_slerp_upgrade", False))): pf_mode = "slerp" + # Full-rank gate: orthogonal full-rank patches → weighted_sum. + # Independent full-rank deltas should be added, not averaged. + if (is_full_rank and fr_preset.get("prefer_sum_orthogonal", False) + and pf_mode == "weighted_average" and pf_orthogonal): + pf_mode = "weighted_sum" # Apply merge strategy override from Conflict Editor (takes priority over auto-selection) # Skip when user explicitly chose weighted_sum_only (protects DPO/edit LoRAs) @@ -4738,6 +4794,7 @@ class LoRAOptimizer(_LoRAMergeBase): compatibility_warnings=compatibility_warnings, behavior_profile=behavior_profile, architecture_preset=preset_key, + is_full_rank=is_full_rank, ) # Bundle LORA_DATA for optional downstream saving @@ -5987,6 +6044,7 @@ class SaveTunerData: "tuner_data": ("TUNER_DATA", {"tooltip": "Connect the tuner_data output from LoRA AutoTuner."}), "save_folder": (folder_choices, {"tooltip": "Which tuner_data folder to save into. Lists all configured paths (from extra_model_paths.yaml and defaults)."}), "filename": ("STRING", {"default": "tuner_data", "tooltip": "Filename. Subdirectories allowed (e.g. 'results/experiment1'). Extension .tuner is added automatically (.json also accepted)."}), + "overwrite": ("BOOLEAN", {"default": True, "tooltip": "When enabled, overwrites existing files. When disabled, appends _001, _002, etc. to avoid overwriting."}), } } @@ -5997,7 +6055,7 @@ class SaveTunerData: OUTPUT_NODE = True DESCRIPTION = "Saves AutoTuner results to a .tuner file so they can be reloaded later without re-running the tuner." - def save_tuner_data(self, tuner_data, save_folder, filename): + def save_tuner_data(self, tuner_data, save_folder, filename, overwrite=True): if tuner_data is None: return ("",) save_dir = save_folder @@ -6005,6 +6063,12 @@ class SaveTunerData: save_path = os.path.join(save_dir, base) if not os.path.realpath(save_path).startswith(os.path.realpath(save_dir) + os.sep): raise ValueError(f"[Save Tuner Data] Path escapes tuner_data directory: {filename}") + if not overwrite and os.path.exists(save_path): + stem, ext = os.path.splitext(save_path) + counter = 1 + while os.path.exists(save_path): + save_path = f"{stem}_{counter:03d}{ext}" + counter += 1 os.makedirs(os.path.dirname(save_path), exist_ok=True) with open(save_path, "w") as f: json.dump(tuner_data, f, indent=2) @@ -6287,6 +6351,13 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): "l2_mean": l2_mean, }) + # Full-rank detection for compatibility analyzer + ca_fr_preset = arch_preset.get("full_rank", {}) + ca_global_avg_rank = (sum(s["avg_rank"] for s in lora_stats) / len(lora_stats)) if lora_stats else 0 + ca_is_full_rank = ca_global_avg_rank >= ca_fr_preset.get("rank_threshold", 256) + if ca_is_full_rank: + logging.info(f"[Compatibility Analyzer] Full-rank LoRAs detected (avg rank {ca_global_avg_rank:.0f})") + total_overlap = 0 total_conflict = 0 pairwise_conflicts = [] @@ -6389,7 +6460,8 @@ class LoRACompatibilityAnalyzer(LoRAOptimizer): new_strengths, _ = self._compute_auto_strengths( group_loras, group_lora_stats, pairwise_similarities=group_pw_sim, - arch_preset=arch_preset, detected_arch=detected_arch) + arch_preset=arch_preset, detected_arch=detected_arch, + is_full_rank=ca_is_full_rank) info["strengths"] = {group[k]: new_strengths[k] for k in range(len(group))} # Suggested merge strategy From 813b3772ce598661dd91a0ce1f54f7827ff71a8a Mon Sep 17 00:00:00 2001 From: Ethanfel Date: Sun, 8 Mar 2026 01:51:26 +0100 Subject: [PATCH 19/19] fix: raise full-rank detection threshold from 256 to 512 256 is only ~25-33% of ZIT's actual full rank (~768-1024) and would incorrectly trigger on mid-rank LoRAs that merge fine. 512 better targets genuinely full-rank LoRAs where the merge path matters. Co-Authored-By: Claude Opus 4.6 --- lora_optimizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lora_optimizer.py b/lora_optimizer.py index 261d5e2..3efb6bb 100644 --- a/lora_optimizer.py +++ b/lora_optimizer.py @@ -51,7 +51,7 @@ _ARCH_PRESETS = { "auto_strength_orthogonal_floor": 0.85, "display_name": "SD/SDXL UNet", "full_rank": { - "rank_threshold": 256, + "rank_threshold": 512, "disable_slerp_upgrade": True, "prefer_sum_orthogonal": True, "auto_strength_floor": 1.0, @@ -73,7 +73,7 @@ _ARCH_PRESETS = { "auto_strength_orthogonal_floor": 0.85, "display_name": "DiT (Flux/WAN/Z-Image/LTX/HunyuanVideo)", "full_rank": { - "rank_threshold": 256, + "rank_threshold": 512, "disable_slerp_upgrade": True, "prefer_sum_orthogonal": True, "auto_strength_floor": 1.0, @@ -95,7 +95,7 @@ _ARCH_PRESETS = { "auto_strength_orthogonal_floor": 0.9, "display_name": "LLM (Qwen/LLaMA)", "full_rank": { - "rank_threshold": 256, + "rank_threshold": 512, "disable_slerp_upgrade": True, "prefer_sum_orthogonal": True, "auto_strength_floor": 1.0,