feat: interference-aware auto-strength, save to loras folder, dynamic stacker JS
- Auto-strength now uses pairwise cosine similarity for exact vector-sum energy normalization instead of assuming orthogonal LoRAs - Save Merged LoRA defaults to ComfyUI loras folder; supports absolute paths - Add JS extension for LoRA Stack (Dynamic) widget show/hide - Fix LTX architecture detection for middle_block/mid_block keys - Update banner SVG with new feature tags Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ethanfel
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
7d77079acb
commit
e9304617f4
@@ -332,7 +332,7 @@ Connect the `LORA_DATA` output from LoRA Optimizer to this node.
|
||||
|
||||
| Option | Default | Effect |
|
||||
|--------|---------|--------|
|
||||
| `filename` | `merged_lora` | Name for the saved file (saved to `output/loras_merged/`) |
|
||||
| `filename` | `merged_lora` | Plain name saves to your ComfyUI loras folder. Absolute path (e.g. `/path/to/my_lora`) saves to that location |
|
||||
| `save_rank` | 0 (auto) | 0 = use each layer's existing rank from the merge. Non-zero = force this rank for layers that need compression |
|
||||
| `bake_strength` | enabled | When on, the saved LoRA reproduces your exact merge at strength 1.0. When off, strengths are not baked in |
|
||||
|
||||
|
||||
+3
-1
@@ -2,4 +2,6 @@
|
||||
|
||||
from .lora_optimizer import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
|
||||
WEB_DIRECTORY = "./js"
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
+18
-13
@@ -5,10 +5,6 @@
|
||||
<stop offset="50%" style="stop-color:#1e1b4b"/>
|
||||
<stop offset="100%" style="stop-color:#0f172a"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="bar-fill" x1="0%" y1="0%" x2="100%" y2="0%">
|
||||
<stop offset="0%" style="stop-color:#e94560"/>
|
||||
<stop offset="100%" style="stop-color:#a78bfa"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="accent" x1="0%" y1="0%" x2="100%" y2="0%">
|
||||
<stop offset="0%" style="stop-color:#e94560"/>
|
||||
<stop offset="100%" style="stop-color:#a78bfa"/>
|
||||
@@ -21,7 +17,6 @@
|
||||
|
||||
<!-- Optimizer icon: analysis bars -->
|
||||
<g transform="translate(56, 52)">
|
||||
<!-- Bar chart representing analysis -->
|
||||
<rect x="0" y="50" width="14" height="46" rx="3" fill="#e94560" opacity="0.8"/>
|
||||
<rect x="20" y="28" width="14" height="68" rx="3" fill="#c026d3" opacity="0.8"/>
|
||||
<rect x="40" y="10" width="14" height="86" rx="3" fill="#a78bfa" opacity="0.8"/>
|
||||
@@ -35,17 +30,27 @@
|
||||
|
||||
<!-- Title -->
|
||||
<text x="180" y="88" font-family="Segoe UI, Arial, sans-serif" font-size="44" font-weight="700" fill="#ffffff" letter-spacing="-0.5">LoRA Optimizer</text>
|
||||
<text x="180" y="120" font-family="Segoe UI, Arial, sans-serif" font-size="17" fill="#94a3b8" letter-spacing="0.3">Analyze. Decide. Merge. Automatically.</text>
|
||||
<text x="180" y="120" font-family="Segoe UI, Arial, sans-serif" font-size="17" fill="#94a3b8" letter-spacing="0.3">Analyze. Decide. Sparsify. Merge. Automatically.</text>
|
||||
|
||||
<!-- Tags -->
|
||||
<rect x="180" y="136" width="120" height="26" rx="13" fill="#a78bfa" opacity="0.12"/>
|
||||
<text x="240" y="154" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#a78bfa" text-anchor="middle" font-weight="600">TIES Merging</text>
|
||||
<!-- Tags row 1 -->
|
||||
<rect x="180" y="134" width="120" height="24" rx="12" fill="#a78bfa" opacity="0.12"/>
|
||||
<text x="240" y="150" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#a78bfa" text-anchor="middle" font-weight="600">TIES Merging</text>
|
||||
|
||||
<rect x="310" y="136" width="120" height="26" rx="13" fill="#64ffda" opacity="0.12"/>
|
||||
<text x="370" y="154" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#64ffda" text-anchor="middle" font-weight="600">Auto-Params</text>
|
||||
<rect x="308" y="134" width="130" height="24" rx="12" fill="#f59e0b" opacity="0.12"/>
|
||||
<text x="373" y="150" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#f59e0b" text-anchor="middle" font-weight="600">DARE / DELLA</text>
|
||||
|
||||
<rect x="440" y="136" width="140" height="26" rx="13" fill="#e94560" opacity="0.12"/>
|
||||
<text x="510" y="154" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#e94560" text-anchor="middle" font-weight="600">SVD Compression</text>
|
||||
<rect x="446" y="134" width="140" height="24" rx="12" fill="#e94560" opacity="0.12"/>
|
||||
<text x="516" y="150" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#e94560" text-anchor="middle" font-weight="600">SVD Compression</text>
|
||||
|
||||
<!-- Tags row 2 -->
|
||||
<rect x="180" y="162" width="120" height="24" rx="12" fill="#64ffda" opacity="0.12"/>
|
||||
<text x="240" y="178" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#64ffda" text-anchor="middle" font-weight="600">Auto-Params</text>
|
||||
|
||||
<rect x="308" y="162" width="158" height="24" rx="12" fill="#22c55e" opacity="0.12"/>
|
||||
<text x="387" y="178" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#22c55e" text-anchor="middle" font-weight="600">Key Normalization</text>
|
||||
|
||||
<rect x="474" y="162" width="130" height="24" rx="12" fill="#c026d3" opacity="0.12"/>
|
||||
<text x="539" y="178" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#c026d3" text-anchor="middle" font-weight="600">Per-Prefix Adaptive</text>
|
||||
|
||||
<!-- ComfyUI badge -->
|
||||
<text x="862" y="26" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#64748b" text-anchor="end" font-weight="600">ComfyUI</text>
|
||||
|
||||
|
Before Width: | Height: | Size: 2.9 KiB After Width: | Height: | Size: 3.4 KiB |
@@ -0,0 +1,94 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 340">
|
||||
<defs>
|
||||
<linearGradient id="knbg" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#0f172a"/>
|
||||
<stop offset="100%" style="stop-color:#1e1b4b"/>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<rect width="900" height="340" rx="12" fill="url(#knbg)"/>
|
||||
|
||||
<text x="450" y="28" font-family="Segoe UI, Arial, sans-serif" font-size="14" fill="#22c55e" text-anchor="middle" font-weight="700" letter-spacing="2">ARCHITECTURE-AWARE KEY NORMALIZATION</text>
|
||||
|
||||
<!-- Problem: different trainer formats -->
|
||||
<rect x="30" y="46" width="260" height="160" rx="10" fill="#1e293b" stroke="#e94560" stroke-width="1.5"/>
|
||||
<rect x="30" y="46" width="260" height="26" rx="10" fill="#e94560" opacity="0.12"/>
|
||||
<rect x="30" y="62" width="260" height="10" fill="#1e293b"/>
|
||||
<text x="160" y="64" font-family="Segoe UI, Arial, sans-serif" font-size="12" fill="#e94560" text-anchor="middle" font-weight="700">PROBLEM: Key Mismatch</text>
|
||||
|
||||
<text x="48" y="88" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#f87171" font-weight="600">Kohya:</text>
|
||||
<text x="95" y="88" font-family="monospace, Consolas" font-size="9" fill="#94a3b8">lora_unet_down_blocks_0_...</text>
|
||||
<text x="48" y="106" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#c084fc" font-weight="600">AI-Toolkit:</text>
|
||||
<text x="115" y="106" font-family="monospace, Consolas" font-size="9" fill="#94a3b8">transformer.transformer_...</text>
|
||||
<text x="48" y="124" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#3b82f6" font-weight="600">LyCORIS:</text>
|
||||
<text x="105" y="124" font-family="monospace, Consolas" font-size="9" fill="#94a3b8">lycoris_unet_blocks_0_...</text>
|
||||
<text x="48" y="142" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#22c55e" font-weight="600">Diffusers:</text>
|
||||
<text x="110" y="142" font-family="monospace, Consolas" font-size="9" fill="#94a3b8">unet.down_blocks.0.attn...</text>
|
||||
|
||||
<line x1="45" y1="154" x2="275" y2="154" stroke="#334155" stroke-width="0.5"/>
|
||||
<text x="160" y="172" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#e94560" text-anchor="middle">Same weights, different names</text>
|
||||
<text x="160" y="186" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#e94560" text-anchor="middle">= keys don't overlap = no merge</text>
|
||||
|
||||
<!-- Arrow -->
|
||||
<line x1="290" y1="126" x2="328" y2="126" stroke="#94a3b8" stroke-width="1.5"/>
|
||||
<polygon points="328,121 338,126 328,131" fill="#94a3b8"/>
|
||||
|
||||
<!-- Auto-detect + normalize -->
|
||||
<rect x="340" y="46" width="230" height="160" rx="10" fill="#1e293b" stroke="#22c55e" stroke-width="1.5"/>
|
||||
<rect x="340" y="46" width="230" height="26" rx="10" fill="#22c55e" opacity="0.1"/>
|
||||
<rect x="340" y="62" width="230" height="10" fill="#1e293b"/>
|
||||
<text x="455" y="64" font-family="Segoe UI, Arial, sans-serif" font-size="12" fill="#22c55e" text-anchor="middle" font-weight="700">AUTO-DETECT + NORMALIZE</text>
|
||||
|
||||
<text x="358" y="88" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#ccd6f6">1. Detect architecture from keys</text>
|
||||
<text x="358" y="106" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#ccd6f6">2. Apply format-specific remap</text>
|
||||
<text x="358" y="124" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#ccd6f6">3. Unified canonical format</text>
|
||||
<line x1="355" y1="134" x2="555" y2="134" stroke="#334155" stroke-width="0.5"/>
|
||||
<text x="455" y="152" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#22c55e" text-anchor="middle">All keys now match</text>
|
||||
<text x="455" y="168" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#64748b" text-anchor="middle">Proper overlap detection</text>
|
||||
<text x="455" y="184" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#64748b" text-anchor="middle">Correct conflict analysis</text>
|
||||
|
||||
<!-- Arrow -->
|
||||
<line x1="570" y1="126" x2="608" y2="126" stroke="#94a3b8" stroke-width="1.5"/>
|
||||
<polygon points="608,121 618,126 608,131" fill="#94a3b8"/>
|
||||
|
||||
<!-- Result -->
|
||||
<rect x="620" y="46" width="250" height="160" rx="10" fill="#1e293b" stroke="#a78bfa" stroke-width="1.5"/>
|
||||
<rect x="620" y="46" width="250" height="26" rx="10" fill="#a78bfa" opacity="0.1"/>
|
||||
<rect x="620" y="62" width="250" height="10" fill="#1e293b"/>
|
||||
<text x="745" y="64" font-family="Segoe UI, Arial, sans-serif" font-size="12" fill="#a78bfa" text-anchor="middle" font-weight="700">MERGE CORRECTLY</text>
|
||||
|
||||
<text x="638" y="88" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#ccd6f6">Keys aligned across trainers</text>
|
||||
<text x="638" y="106" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#ccd6f6">Full overlap detected</text>
|
||||
<text x="638" y="124" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#ccd6f6">Per-prefix analysis works correctly</text>
|
||||
<line x1="635" y1="134" x2="855" y2="134" stroke="#334155" stroke-width="0.5"/>
|
||||
<text x="638" y="152" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#a78bfa">Z-Image: QKV split + re-fuse</text>
|
||||
<text x="638" y="168" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#64748b">FLUX: AI-Toolkit/Kohya unified</text>
|
||||
<text x="638" y="184" font-family="Segoe UI, Arial, sans-serif" font-size="10" fill="#64748b">Wan: LyCORIS/diffusers unified</text>
|
||||
|
||||
<!-- Supported architectures row -->
|
||||
<text x="450" y="232" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#94a3b8" text-anchor="middle" font-weight="600">Supported Architectures</text>
|
||||
|
||||
<rect x="70" y="246" width="105" height="28" rx="14" fill="#22c55e" opacity="0.12"/>
|
||||
<text x="122" y="264" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#22c55e" text-anchor="middle" font-weight="600">Z-Image</text>
|
||||
|
||||
<rect x="188" y="246" width="80" height="28" rx="14" fill="#a78bfa" opacity="0.12"/>
|
||||
<text x="228" y="264" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#a78bfa" text-anchor="middle" font-weight="600">FLUX</text>
|
||||
|
||||
<rect x="281" y="246" width="80" height="28" rx="14" fill="#f59e0b" opacity="0.12"/>
|
||||
<text x="321" y="264" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#f59e0b" text-anchor="middle" font-weight="600">Wan</text>
|
||||
|
||||
<rect x="374" y="246" width="80" height="28" rx="14" fill="#e94560" opacity="0.12"/>
|
||||
<text x="414" y="264" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#e94560" text-anchor="middle" font-weight="600">SDXL</text>
|
||||
|
||||
<rect x="467" y="246" width="100" height="28" rx="14" fill="#3b82f6" opacity="0.12"/>
|
||||
<text x="517" y="264" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#3b82f6" text-anchor="middle" font-weight="600">LTX Video</text>
|
||||
|
||||
<rect x="580" y="246" width="120" height="28" rx="14" fill="#c084fc" opacity="0.12"/>
|
||||
<text x="640" y="264" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#c084fc" text-anchor="middle" font-weight="600">Qwen-Image</text>
|
||||
|
||||
<rect x="713" y="246" width="115" height="28" rx="14" fill="#64748b" opacity="0.12"/>
|
||||
<text x="770" y="264" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#64748b" text-anchor="middle" font-weight="600">SD 1.5 / others</text>
|
||||
|
||||
<!-- Bottom notes -->
|
||||
<text x="450" y="300" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#94a3b8" text-anchor="middle">Enable with <tspan font-family="monospace" fill="#22c55e">normalize_keys: enabled</tspan> when mixing LoRAs from different trainers</text>
|
||||
<text x="450" y="320" font-family="Segoe UI, Arial, sans-serif" font-size="11" fill="#64748b" text-anchor="middle">Auto-detects architecture from LoRA key patterns. No manual configuration needed.</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 7.6 KiB |
@@ -0,0 +1,92 @@
|
||||
import { app } from "/scripts/app.js";
|
||||
|
||||
// Widget hide/show via type-mangling (standard ComfyUI pattern)
|
||||
const HIDDEN_TAG = "loraopt_hidden";
|
||||
const origProps = {};
|
||||
|
||||
function toggleWidget(node, widget, show) {
|
||||
if (!widget) return;
|
||||
|
||||
if (!origProps[widget.name]) {
|
||||
origProps[widget.name] = {
|
||||
origType: widget.type,
|
||||
origComputeSize: widget.computeSize,
|
||||
};
|
||||
}
|
||||
|
||||
widget.type = show ? origProps[widget.name].origType : HIDDEN_TAG;
|
||||
widget.computeSize = show
|
||||
? origProps[widget.name].origComputeSize
|
||||
: () => [0, -4];
|
||||
|
||||
if (widget.linkedWidgets) {
|
||||
for (const w of widget.linkedWidgets) {
|
||||
toggleWidget(node, w, show);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function findWidget(node, name) {
|
||||
return node.widgets ? node.widgets.find((w) => w.name === name) : null;
|
||||
}
|
||||
|
||||
function updateVisibility(node) {
|
||||
const modeWidget = findWidget(node, "mode");
|
||||
const countWidget = findWidget(node, "lora_count");
|
||||
if (!modeWidget || !countWidget) return;
|
||||
|
||||
const isSimple = modeWidget.value === "simple";
|
||||
const count = countWidget.value;
|
||||
const MAX = 10;
|
||||
|
||||
for (let i = 1; i <= MAX; i++) {
|
||||
const visible = i <= count;
|
||||
|
||||
toggleWidget(node, findWidget(node, `lora_name_${i}`), visible);
|
||||
toggleWidget(node, findWidget(node, `strength_${i}`), visible && isSimple);
|
||||
toggleWidget(node, findWidget(node, `model_strength_${i}`), visible && !isSimple);
|
||||
toggleWidget(node, findWidget(node, `clip_strength_${i}`), visible && !isSimple);
|
||||
}
|
||||
|
||||
const newHeight = node.computeSize()[1];
|
||||
node.setSize([node.size[0], newHeight]);
|
||||
}
|
||||
|
||||
app.registerExtension({
|
||||
name: "LoRAOptimizer.LoRAStackDynamic",
|
||||
nodeCreated(node) {
|
||||
if (node.comfyClass !== "LoRAStackDynamic") return;
|
||||
|
||||
// Intercept mode and lora_count changes to update visibility
|
||||
for (const w of node.widgets || []) {
|
||||
if (w.name !== "mode" && w.name !== "lora_count") continue;
|
||||
|
||||
let widgetValue = w.value;
|
||||
const originalDescriptor =
|
||||
Object.getOwnPropertyDescriptor(w, "value") ||
|
||||
Object.getOwnPropertyDescriptor(
|
||||
Object.getPrototypeOf(w),
|
||||
"value"
|
||||
);
|
||||
|
||||
Object.defineProperty(w, "value", {
|
||||
get() {
|
||||
return originalDescriptor?.get
|
||||
? originalDescriptor.get.call(w)
|
||||
: widgetValue;
|
||||
},
|
||||
set(newVal) {
|
||||
if (originalDescriptor?.set) {
|
||||
originalDescriptor.set.call(w, newVal);
|
||||
} else {
|
||||
widgetValue = newVal;
|
||||
}
|
||||
updateVisibility(node);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Initial visibility update (after a tick so widgets are fully initialized)
|
||||
requestAnimationFrame(() => updateVisibility(node));
|
||||
},
|
||||
});
|
||||
+86
-30
@@ -209,6 +209,7 @@ class _LoRAMergeBase:
|
||||
'transformer_blocks' in k and ('attn1' in k or 'attn2' in k)
|
||||
and 'input_blocks' not in k and 'output_blocks' not in k
|
||||
and 'down_blocks' not in k and 'up_blocks' not in k
|
||||
and 'middle_block' not in k and 'mid_block' not in k
|
||||
for k in keys):
|
||||
return 'ltx'
|
||||
|
||||
@@ -1439,7 +1440,9 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
}),
|
||||
"sparsification_density": ("FLOAT", {
|
||||
"default": 0.7, "min": 0.01, "max": 1.0, "step": 0.05,
|
||||
"tooltip": "What percentage of weights to keep (0.7 = keep 70%, drop 30%). Lower values drop more weights — reduces interference but may lose detail. Only matters when sparsification is enabled."
|
||||
"tooltip": "What percentage of weights to keep (0.7 = keep 70%, drop 30%). Lower values drop more weights — reduces interference but may lose detail. "
|
||||
"At 1.0, no weights are dropped (equivalent to disabled). "
|
||||
"Note: in TIES mode, sparsification replaces the trim step — setting density to 1.0 disables both sparsification AND trimming."
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -1512,9 +1515,9 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
|
||||
def _sample_conflict(self, diff_a, diff_b, device=None):
|
||||
"""
|
||||
Compute sign conflict ratio between two diff tensors.
|
||||
Samples up to 100k positions for large tensors.
|
||||
Returns (n_overlap, n_conflict).
|
||||
Compute sign conflict ratio and cosine similarity components between
|
||||
two diff tensors. Samples up to 100k positions for large tensors.
|
||||
Returns (n_overlap, n_conflict, dot, norm_a_sq, norm_b_sq).
|
||||
"""
|
||||
# Avoid unnecessary .float() copies — diffs from _analyze_prefix are already float32
|
||||
flat_a = diff_a.flatten()
|
||||
@@ -1527,7 +1530,7 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
flat_b = flat_b.float()
|
||||
|
||||
if flat_a.numel() != flat_b.numel():
|
||||
return (0, 0)
|
||||
return (0, 0, 0.0, 0.0, 0.0)
|
||||
|
||||
n = flat_a.numel()
|
||||
if n > 100000:
|
||||
@@ -1541,10 +1544,18 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
mask = (flat_a != 0) & (flat_b != 0)
|
||||
n_overlap = mask.sum().item()
|
||||
if n_overlap == 0:
|
||||
return (0, 0)
|
||||
return (0, 0, 0.0, 0.0, 0.0)
|
||||
|
||||
n_conflict = (flat_a[mask].sign() != flat_b[mask].sign()).sum().item()
|
||||
return (n_overlap, n_conflict)
|
||||
a_overlap = flat_a[mask]
|
||||
b_overlap = flat_b[mask]
|
||||
n_conflict = (a_overlap.sign() != b_overlap.sign()).sum().item()
|
||||
|
||||
# Cosine similarity components (on overlap region)
|
||||
dot = (a_overlap * b_overlap).sum().item()
|
||||
norm_a_sq = (a_overlap * a_overlap).sum().item()
|
||||
norm_b_sq = (b_overlap * b_overlap).sum().item()
|
||||
|
||||
return (n_overlap, n_conflict, dot, norm_a_sq, norm_b_sq)
|
||||
|
||||
def _process_prefix(self, lora_prefix, active_loras, model_keys, clip_keys,
|
||||
model, clip, device):
|
||||
@@ -1645,7 +1656,7 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
target_info, skip_count) or None if prefix should be skipped.
|
||||
|
||||
partial_stats: list of (lora_index, rank, l2_norm)
|
||||
pair_conflicts: dict mapping (i, j) -> (overlap, conflict)
|
||||
pair_conflicts: dict mapping (i, j) -> (overlap, conflict, dot, norm_a_sq, norm_b_sq)
|
||||
magnitude_samples: list of small 1D CPU float tensors
|
||||
"""
|
||||
# --- Resolve target key and shape (same as _process_prefix) ---
|
||||
@@ -1753,8 +1764,8 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
i, j = lora_indices[ai], lora_indices[bi]
|
||||
diff_i = diffs[i] if eff_strengths[i] >= 0 else -diffs[i]
|
||||
diff_j = diffs[j] if eff_strengths[j] >= 0 else -diffs[j]
|
||||
ov, conf = self._sample_conflict(diff_i, diff_j, device=device)
|
||||
pair_conflicts[(i, j)] = (ov, conf)
|
||||
ov, conf, dot, na_sq, nb_sq = self._sample_conflict(diff_i, diff_j, device=device)
|
||||
pair_conflicts[(i, j)] = (ov, conf, dot, na_sq, nb_sq)
|
||||
|
||||
# --- Magnitude sampling (sample on device, free each diff eagerly) ---
|
||||
magnitude_samples = []
|
||||
@@ -1832,11 +1843,14 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
|
||||
return max(0.1, min(0.9, above_noise))
|
||||
|
||||
def _compute_auto_strengths(self, active_loras, lora_stats):
|
||||
def _compute_auto_strengths(self, active_loras, lora_stats, pairwise_similarities=None):
|
||||
"""
|
||||
Compute reduced per-LoRA strengths using L2-aware energy normalization.
|
||||
Scales strengths so the total combined energy matches the strongest
|
||||
single LoRA's contribution, preventing overexposure from stacking.
|
||||
Compute reduced per-LoRA strengths using interference-aware energy
|
||||
normalization. Uses pairwise cosine similarity to account for
|
||||
directional alignment between LoRAs:
|
||||
- Aligned LoRAs (cos≈1) → more aggressive scaling (they reinforce)
|
||||
- Orthogonal LoRAs (cos≈0) → same as classic L2 norm
|
||||
- Opposing LoRAs (cos≈-1) → less scaling (they cancel out)
|
||||
|
||||
Returns (new_strengths, reasoning_lines) where new_strengths is a list
|
||||
of floats (one per active LoRA) and reasoning_lines is a list of strings.
|
||||
@@ -1867,8 +1881,21 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
reasoning.append("Single contributing LoRA or none — no strength adjustment needed")
|
||||
return (list(original_strengths), reasoning)
|
||||
|
||||
# Current combined energy: sqrt(sum(effective[i]^2))
|
||||
current_energy = math.sqrt(sum(e * e for e in effective))
|
||||
# Interference-aware combined energy using the vector-sum formula:
|
||||
# ||sum(v_i)||^2 = sum(||v_i||^2) + 2 * sum_{i<j}(||v_i|| * ||v_j|| * cos(v_i, v_j))
|
||||
# This is the exact squared magnitude of the sum of vectors given their
|
||||
# pairwise angles. When cos=0 (orthogonal), reduces to sum of squares.
|
||||
energy_sq = sum(e * e for e in effective)
|
||||
orthogonal_energy_sq = energy_sq # save for reporting
|
||||
|
||||
if pairwise_similarities:
|
||||
for (i, j), cos_sim in pairwise_similarities.items():
|
||||
if effective[i] > 0 and effective[j] > 0:
|
||||
energy_sq += 2.0 * effective[i] * effective[j] * cos_sim
|
||||
|
||||
# Clamp to avoid sqrt of negative (possible if strong opposing LoRAs)
|
||||
energy_sq = max(energy_sq, 0.0)
|
||||
current_energy = math.sqrt(energy_sq)
|
||||
|
||||
# Reference energy: what the strongest single LoRA contributes alone
|
||||
reference_energy = max(effective)
|
||||
@@ -1889,7 +1916,20 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
|
||||
# Build reasoning
|
||||
reasoning.append(f"Scale factor: {scale:.4f}")
|
||||
reasoning.append("Method: L2-aware energy normalization")
|
||||
if pairwise_similarities:
|
||||
avg_cos = sum(pairwise_similarities.values()) / len(pairwise_similarities)
|
||||
orthogonal_energy = math.sqrt(orthogonal_energy_sq)
|
||||
if avg_cos > 0.1:
|
||||
alignment_desc = "mostly aligned (reinforcing)"
|
||||
elif avg_cos < -0.1:
|
||||
alignment_desc = "mostly opposing (cancelling)"
|
||||
else:
|
||||
alignment_desc = "mostly orthogonal (independent)"
|
||||
reasoning.append(f"Method: interference-aware energy normalization")
|
||||
reasoning.append(f" Avg pairwise cosine similarity: {avg_cos:.3f} ({alignment_desc})")
|
||||
reasoning.append(f" Interference-aware energy: {current_energy:.4f} (orthogonal assumption: {orthogonal_energy:.4f})")
|
||||
else:
|
||||
reasoning.append("Method: L2-aware energy normalization (no similarity data)")
|
||||
for i in range(n):
|
||||
if effective[i] > 0 and abs(scale - 1.0) > 1e-9:
|
||||
reasoning.append(f" {active_loras[i]['name']}: {original_strengths[i]} -> {new_strengths[i]:.4f}")
|
||||
@@ -2046,6 +2086,8 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
lines.append(f" {pc['pair']}:")
|
||||
lines.append(f" Overlapping positions: {pc['overlap']}")
|
||||
lines.append(f" Sign conflicts: {pc['conflicts']} ({pc['ratio']:.1%})")
|
||||
if 'cosine_sim' in pc:
|
||||
lines.append(f" Cosine similarity: {pc['cosine_sim']:.3f}")
|
||||
|
||||
# Collection Statistics
|
||||
lines.append("")
|
||||
@@ -2297,7 +2339,7 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
|
||||
pairs = [(i, j) for i in range(len(active_loras))
|
||||
for j in range(i + 1, len(active_loras))]
|
||||
pair_accum = {(i, j): [0, 0] for i, j in pairs} # [overlap, conflict]
|
||||
pair_accum = {(i, j): [0, 0, 0.0, 0.0, 0.0] for i, j in pairs} # [overlap, conflict, dot, norm_a_sq, norm_b_sq]
|
||||
all_magnitude_samples = [] # list of small CPU tensors
|
||||
prefix_count = 0 # number of prefixes with valid diffs
|
||||
prefix_stats = {} # prefix -> {conflict_ratio, n_loras, magnitude_samples, magnitude_ratio}
|
||||
@@ -2315,9 +2357,12 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
per_lora_stats[idx]["ranks"].append(rank)
|
||||
per_lora_stats[idx]["key_count"] += 1
|
||||
per_lora_stats[idx]["l2_norms"].append(l2)
|
||||
for (i, j), (ov, conf) in pair_conflicts.items():
|
||||
for (i, j), (ov, conf, dot, na_sq, nb_sq) in pair_conflicts.items():
|
||||
pair_accum[(i, j)][0] += ov
|
||||
pair_accum[(i, j)][1] += conf
|
||||
pair_accum[(i, j)][2] += dot
|
||||
pair_accum[(i, j)][3] += na_sq
|
||||
pair_accum[(i, j)][4] += nb_sq
|
||||
all_magnitude_samples.extend(mag_samples)
|
||||
|
||||
# Store per-prefix stats for per_prefix optimization mode
|
||||
@@ -2326,8 +2371,8 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
n_contributing = len(partial_stats)
|
||||
|
||||
# Per-prefix conflict ratio
|
||||
pf_overlap = sum(ov for ov, _ in pair_conflicts.values())
|
||||
pf_conflict = sum(conf for _, conf in pair_conflicts.values())
|
||||
pf_overlap = sum(ov for ov, conf, dot, na_sq, nb_sq in pair_conflicts.values())
|
||||
pf_conflict = sum(conf for ov, conf, dot, na_sq, nb_sq in pair_conflicts.values())
|
||||
pf_conflict_ratio = pf_conflict / pf_overlap if pf_overlap > 0 else 0.0
|
||||
|
||||
# Per-prefix magnitude ratio (max/min L2 among contributing LoRAs)
|
||||
@@ -2390,15 +2435,19 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
"l2_mean": l2_mean,
|
||||
})
|
||||
|
||||
# Pairwise conflict stats from accumulated counts
|
||||
# Pairwise conflict stats and cosine similarity from accumulated counts
|
||||
total_overlap = 0
|
||||
total_conflict = 0
|
||||
pairwise_conflicts = []
|
||||
pairwise_similarities = {}
|
||||
for i, j in pairs:
|
||||
pair_overlap, pair_conflict = pair_accum[(i, j)]
|
||||
pair_overlap, pair_conflict, pair_dot, pair_na_sq, pair_nb_sq = pair_accum[(i, j)]
|
||||
total_overlap += pair_overlap
|
||||
total_conflict += pair_conflict
|
||||
ratio = pair_conflict / pair_overlap if pair_overlap > 0 else 0
|
||||
denom = math.sqrt(pair_na_sq) * math.sqrt(pair_nb_sq)
|
||||
cos_sim = pair_dot / denom if denom > 0 else 0.0
|
||||
pairwise_similarities[(i, j)] = cos_sim
|
||||
name_i = active_loras[i]['name']
|
||||
name_j = active_loras[j]['name']
|
||||
if name_i == name_j:
|
||||
@@ -2410,8 +2459,9 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
"overlap": pair_overlap,
|
||||
"conflicts": pair_conflict,
|
||||
"ratio": ratio,
|
||||
"cosine_sim": cos_sim,
|
||||
})
|
||||
logging.info(f"[LoRA Optimizer] {pair_label} -> {ratio:.1%} conflict")
|
||||
logging.info(f"[LoRA Optimizer] {pair_label} -> {ratio:.1%} conflict, cos_sim={cos_sim:.3f}")
|
||||
|
||||
avg_conflict_ratio = total_conflict / total_overlap if total_overlap > 0 else 0
|
||||
logging.info(f"[LoRA Optimizer] Average conflict ratio: {avg_conflict_ratio:.1%}")
|
||||
@@ -2445,7 +2495,8 @@ class LoRAOptimizer(_LoRAMergeBase):
|
||||
auto_strength_info = None
|
||||
scale_ratios = {}
|
||||
if auto_strength == "enabled":
|
||||
new_strengths, strength_reasoning = self._compute_auto_strengths(active_loras, lora_stats)
|
||||
new_strengths, strength_reasoning = self._compute_auto_strengths(
|
||||
active_loras, lora_stats, pairwise_similarities=pairwise_similarities)
|
||||
|
||||
for i in range(len(active_loras)):
|
||||
orig = active_loras[i]["strength"]
|
||||
@@ -2818,7 +2869,7 @@ class SaveMergedLoRA:
|
||||
return {
|
||||
"required": {
|
||||
"lora_data": ("LORA_DATA", {"tooltip": "Connect the lora_data output from LoRA Optimizer here."}),
|
||||
"filename": ("STRING", {"default": "merged_lora", "tooltip": "Name for the saved file. The file will be saved to your output/loras_merged/ folder as 'name.safetensors'."}),
|
||||
"filename": ("STRING", {"default": "merged_lora", "tooltip": "Name or path for the saved file. Plain name (e.g. 'merged_lora') saves to your ComfyUI loras folder. Absolute path (e.g. '/path/to/my_lora') saves to that location. Extension .safetensors is added automatically."}),
|
||||
"save_rank": ("INT", {
|
||||
"default": 0, "min": 0, "max": 1024, "step": 4,
|
||||
"tooltip": "0 = auto (uses each layer's existing rank from the merge — recommended). Non-zero = force this rank for any layers that need compression. Higher values = more accurate but larger file."
|
||||
@@ -2839,10 +2890,15 @@ class SaveMergedLoRA:
|
||||
|
||||
def save_lora(self, lora_data, filename, save_rank=0, bake_strength=True):
|
||||
# Determine save path
|
||||
output_dir = folder_paths.get_output_directory()
|
||||
save_dir = os.path.join(output_dir, "loras_merged")
|
||||
if os.path.isabs(filename) or os.sep in filename or '/' in filename:
|
||||
# Absolute or relative path provided — use as-is
|
||||
save_path = filename if filename.endswith('.safetensors') else f"{filename}.safetensors"
|
||||
save_dir = os.path.dirname(save_path)
|
||||
else:
|
||||
# Plain filename — save to primary ComfyUI loras folder
|
||||
save_dir = folder_paths.get_folder_paths("loras")[0]
|
||||
save_path = os.path.join(save_dir, f"{filename}.safetensors")
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
save_path = os.path.join(save_dir, f"{filename}.safetensors")
|
||||
|
||||
model_patches = lora_data["model_patches"]
|
||||
clip_patches = lora_data["clip_patches"]
|
||||
|
||||
Reference in New Issue
Block a user