From eab8b20ef19f7f31191a2d1f27815672821cc638 Mon Sep 17 00:00:00 2001 From: silveroxides Date: Mon, 5 Oct 2026 23:02:43 +0200 Subject: [PATCH] Clarify merge mode documentation and parameter strength --- docs/merger_2_model_modes.md | 218 +++++++++++++++++++++-------------- docs/merger_3_model_modes.md | 120 +++++++++++-------- nodes/merger_ops.py | 19 ++- 3 files changed, 215 insertions(+), 142 deletions(-) diff --git a/docs/merger_2_model_modes.md b/docs/merger_2_model_modes.md index b72484a..8343c18 100644 --- a/docs/merger_2_model_modes.md +++ b/docs/merger_2_model_modes.md @@ -2,154 +2,194 @@ --- +### Core Convention Across All 2-Model Modes +- **Model A is ALWAYS the base / anchor model.** +- **Model B is ALWAYS the donor / incoming model being merged into Model A.** +- When interpolating or blending between weights: + $$\text{Output} = A \cdot (1 - \text{mask}) + B \cdot \text{mask}$$ +- A blend factor / weight of `0.0` yields **100% Model A** (pure Model A). +- A blend factor / weight of `1.0` yields **100% Model B** (pure Model B). +- Intermediate values control how much of Model B replaces or alters Model A. + +--- + ## Weight-Sum -> `A * (1 - α) + B * α`. A simple linear interpolation between two models. +> $\text{Output} = A \cdot (1 - \alpha) + B \cdot \alpha$. Direct linear interpolation between Model A and Model B. + +**Models Used:** A (Base), B (Incoming) -**Models Used:** A, B **Parameters:** -- **Alpha:** Interpolation weight. `0.0` is 100% Model A, `1.0` is 100% Model B. +- **Alpha:** The blend ratio determining what fraction of **Model B** replaces **Model A**. + - `0.0`: **100% Model A, 0% Model B** (exact Model A; Model B is completely ignored). + - `0.25`: **75% Model A, 25% Model B** (mostly Model A, with 25% of Model B blended in). + - `0.50`: **50% Model A, 50% Model B** (equal 50/50 mix of both models). + - `0.75`: **25% Model A, 75% Model B** (mostly Model B, retaining 25% of Model A). + - `1.0`: **0% Model A, 100% Model B** (exact Model B; Model A is completely replaced). --- ## Comparative-Interpolation -> Interpolates between A and B based on the relative differences in their tensor values. Creates a unique blend by deciding how much of each model to use for each individual weight. +> Interpolates between A and B based on the relative differences in their tensor values, creating a selective blend rather than a flat uniform mix. + +**Models Used:** A (Base), B (Incoming) -**Models Used:** A, B **Parameters:** -- **Alpha:** Controls the interpolation curve's strength and shape. -- **Beta:** Switches the interpolation style. `0.0` focuses on similarity, `1.0` focuses on difference. -- **Gamma:** Mixes between a randomized (binomial) interpolation at `0.0` and a smooth (linear) one at `1.0`. +- **Alpha (Blend Intensity):** Controls the overall pull of **Model B into Model A**. + - `0.0`: Heavily favors Model A (virtually no Model B incorporated). + - `0.5`: Balanced curve where weights with moderate differences blend equally. + - `1.0`: Strongly pulls in Model B wherever weights differ. +- **Beta (Feature Focus):** Controls whether the merge targets similar or divergent weights. + - `0.0`: Focuses blending on weights where Model A and Model B are already similar. + - `1.0`: Focuses blending on weights where Model A and Model B differ most. +- **Gamma (Transition Style):** Mixes between discrete selection and smooth gradation. + - `0.0`: Stochastic / binomial selection (individual weights are chosen probabilistically as either pure A or pure B). + - `1.0`: Smooth linear interpolation across all weights. --- ## Power-Up (DARE) -> Adds the unique capabilities of Model B to Model A using the Drop and Rescale (DARE) technique. This implementation handles shape mismatches between models by padding and uses a randomized dropout mask. +> Adds the unique capabilities of Model B onto Model A using the Drop and Rescale (DARE) technique. Computes the task delta $(B - A)$, randomly drops a fraction of the delta parameters, rescales the survivors to maintain expected magnitude, and adds them to Model A. + +**Models Used:** A (Base), B (Donor) -**Models Used:** A, B **Parameters:** -- **Alpha:** The dropout rate ($p$). This is the proportion of delta parameters from Model B that are randomly set to zero. -- **Beta:** A final multiplier for the rescaled difference before it's added to Model A. -- **Rescaling Logic:** Remaining weights are automatically rescaled by $1/(1-p)$ as per the DARE paper to approximate the original embeddings. +- **Alpha (Dropout Rate $p$):** Proportion of Model B's delta parameters to randomly zero out. + - Range: `0.0` to `1.0`. + - Typical: `0.7` to `0.9`. For example, `0.85` means 85% of Model B's delta values are discarded, keeping only 15% of the most distinct changes. +- **Beta (Scale Multiplier):** Final strength multiplier for Model B's rescaled delta before adding to Model A. + - `1.0`: Standard 100% strength addition of Model B's surviving features. + - `0.5`: Half-strength addition of Model B's features. + - `1.2`: Amplified addition of Model B's features. +- **Rescaling Logic:** Surviving parameters are automatically scaled by $1 / (1 - p)$ per the DARE paper to preserve the expected total update magnitude. --- ## Power-Up (DARE+TIES) -> Combines DARE (Drop and Rescale) with TIES (Trim, Elect Sign), the current best-practice algorithm for merging fine-tuned models, particularly LoRA-tuned ones. DARE sparsifies the task vector via random dropout and rescaling; TIES then removes residual low-magnitude noise and enforces sign consistency. The result is a cleaner, more targeted capability transfer from B to A with less interference. +> Combines DARE (Drop and Rescale) with TIES (Trim, Elect Sign) to merge fine-tuned models cleanly. DARE sparsifies the delta; TIES trims low-magnitude background noise and enforces sign consensus. + +**Models Used:** A (Base), B (Donor) -**Models Used:** A, B **Parameters:** -- **Alpha:** DARE drop rate. Fraction of delta parameters randomly zeroed. Higher values produce a sparser, more targeted delta. Typical range: `0.5`–`0.9`. -- **Beta:** TIES trim quantile. Fraction of the smallest-magnitude delta parameters zeroed after DARE, removing residual noise. `0.0` disables trimming. Typical range: `0.1`–`0.3`. -- **Gamma:** Lambda scale. Final multiplier applied to the merged delta before adding to Model A. Equivalent to the task arithmetic scaling coefficient. Typical range: `0.5`–`1.5`. -- **Seed:** Random seed for the DARE dropout mask. Controls reproducibility. +- **Alpha (DARE Drop Rate):** Fraction of Model B's delta parameters randomly dropped (`0.5`–`0.9`). Higher values produce a sparser capability transfer. +- **Beta (TIES Trim Quantile):** Fraction of the smallest-magnitude delta parameters eliminated after DARE to remove noise. `0.0` disables trimming; `0.2` trims the lowest 20% by absolute magnitude. Typical: `0.1`–`0.3`. +- **Gamma (Lambda Scale / Strength):** Multiplier for the filtered delta added to Model A. + - `1.0`: Standard 100% strength of Model B's filtered capability added to Model A. + - `0.25`: Gentle 25% addition of Model B's capabilities. + - `1.5`: Amplified capability transfer. +- **Seed:** Random seed for reproducible DARE dropout masks. **Algorithm:** -1. Compute task vector `δ = B − A` -2. **DARE:** apply random binary mask with keep probability `1 − alpha`; rescale survivors by `1/(1−alpha)` -3. **TIES trim:** zero out parameters below the `beta`-quantile magnitude threshold -4. **TIES elect:** determine dominant sign per position; zero out disagreeing parameters -5. Return `A + gamma × δ_filtered` - -**Guidance:** -- Start with `alpha=0.9, beta=0.2, gamma=1.0` for LoRA-derived models -- Lower `alpha` (e.g. `0.5`) for full fine-tunes where more delta mass should be retained -- Increase `gamma` beyond `1.0` to amplify B's influence; decrease below `1.0` to soften it +1. Compute task vector $\delta = B - A$ +2. **DARE:** Apply random binary mask with keep probability $1 - \alpha$; rescale survivors by $1 / (1 - \alpha)$ +3. **TIES trim:** Zero out parameters below the $\beta$-quantile magnitude threshold +4. **TIES elect:** Determine dominant sign direction per position; zero out disagreeing parameters +5. Return $A + \gamma \cdot \delta_{\text{filtered}}$ --- -## Enhanced Man Interp -> Sophisticated interpolation between values from A and B depending on their difference relative to other values, with manual threshold control. +## Enhanced Man Interp (Enhanced Manual Interpolation) +> Selective interpolation between Model A and Model B based on normalized pairwise differences, constrained within a user-defined threshold band. + +**Models Used:** A (Base), B (Incoming) + +**Formula:** +$$\text{diff} = \frac{\max(|A - B|) - |A - B|}{\max(|A - B|)}$$ +$$\text{mask}_{\text{threshold}} = (\beta < \text{mean}(\text{diff}) < \gamma)$$ +$$\text{blend\_strength} = \text{diff}^{(1/\alpha - 1)} \cdot \text{mask}_{\text{threshold}}$$ +$$\text{Output} = A \cdot (1 - \text{interpolated\_mask}) + B \cdot \text{interpolated\_mask}$$ -**Models Used:** A, B **Parameters:** -- **Alpha:** Interpolation strength. -- **Beta:** Lower mean threshold for filtering differences. -- **Gamma:** Upper mean threshold for filtering differences. -- **Delta:** Smoothness factor (mix between randomized mask and powered differences). +- **Alpha (Model B Pull Strength):** Controls how strongly Model B is pulled into Model A. + - Mathematically, the exponent applied to similarity is $(1/\alpha - 1)$. + - **`Alpha = 0.25`**: Exponent is $(1/0.25 - 1) = 3$. This aggressively suppresses Model B's contribution toward zero. Only weights that are virtually identical between both models get even a minor blend of Model B; the final result remains overwhelmingly **Model A** (>90% Model A). + - **`Alpha = 0.50`**: Exponent is $(1/0.50 - 1) = 1$ (linear). Weights with high similarity incorporate Model B proportionally up to 50%. + - **`Alpha = 0.75`**: Exponent is $0.33$. Flattens similarity, pulling in substantially more of Model B wherever differences meet the threshold. + - **`Alpha = 1.0`**: Exponent is $0$. Full maximum pull of Model B across all qualifying weights. +- **Beta (Lower Mean Threshold):** Minimum normalized similarity required before Model B can be blended into Model A. Layers/weights with similarity below $\beta$ remain 100% Model A. +- **Gamma (Upper Mean Threshold):** Maximum normalized similarity allowed for blending. Layers/weights with similarity above $\gamma$ remain 100% Model A. Together, $[\beta, \gamma]$ acts as a bandpass filter targeting only weights with moderate differences. +- **Delta (Smoothness Factor):** Blends between stochastic selection (`0.0`, a Bernoulli coin-flip choosing either pure A or pure B per weight) and smooth continuous blending (`1.0`, smooth linear weighting between A and B). --- -## Enhanced Auto Interp -> Automated version of the enhanced interpolation mode that dynamically calculates thresholds based on mean differences. +## Enhanced Auto Interp (Enhanced Automatic Interpolation) +> Automated version of Enhanced Interpolation that dynamically computes threshold boundaries centered on the layer's mean difference. + +**Models Used:** A (Base), B (Incoming) + +**Formula:** +$$\text{threshold\_band} = [\text{mean}(\text{diff}) \cdot (1 - \beta), \;\; \text{mean}(\text{diff}) \cdot (1 + \beta)]$$ +$$\text{Output} = A \cdot (1 - \text{interpolated\_mask}) + B \cdot \text{interpolated\_mask}$$ -**Models Used:** A, B **Parameters:** -- **Alpha:** Interpolation strength. -- **Beta:** Threshold adjustment factor. -- **Gamma:** Smoothness factor. +- **Alpha (Model B Pull Strength):** Controls how much of Model B replaces Model A inside the active band. + - `0.25`: Low Model B contribution; output remains mostly Model A. + - `0.50`: Moderate, balanced blend of Model B into Model A. + - `1.0`: Maximum Model B influence within the threshold band. +- **Beta (Threshold Width Around Mean):** Defines how wide the window of affected weights is around the layer's average difference. + - `0.1`: Narrow window ($\pm 10\%$ around mean difference). + - `0.5`: Broad window ($\pm 50\%$ around mean difference). +- **Gamma (Smoothness Factor):** Blends between stochastic Bernoulli selection (`0.0`) and smooth continuous linear interpolation (`1.0`). --- ## Weight-Sum Cutoff -> A linear interpolation mode that only applies the merge to weights whose differences fall within a specific threshold range. +> Linear interpolation that only blends Model B into Model A for weights whose relative difference falls within an explicit cutoff window. + +**Models Used:** A (Base), B (Incoming) + +**Formula:** +$$\text{Output} = A \cdot (1 - \alpha \cdot \text{mask}) + B \cdot (\alpha \cdot \text{mask})$$ -**Models Used:** A, B **Parameters:** -- **Alpha:** Interpolation weight (multiplier for the difference). -- **Beta:** Upper threshold for the difference cutoff. -- **Gamma:** Lower threshold for the difference cutoff. +- **Alpha (Model B Mix Ratio):** The percentage of Model B blended into Model A for weights inside the window. + - `0.0`: 0% Model B (100% Model A everywhere). + - `0.25`: 25% Model B, 75% Model A for weights inside the cutoff window; weights outside remain 100% Model A. + - `0.50`: 50% Model B, 50% Model A inside the window. + - `1.0`: 100% Model B inside the window (complete replacement of qualifying weights). +- **Beta (Upper Difference Threshold):** Upper cutoff for normalized difference. +- **Gamma (Lower Difference Threshold):** Lower cutoff for normalized difference. Weights with difference between $\gamma$ and $\beta$ receive the blend; all other weights stay 100% Model A. --- ## SVD LoRA Extraction -> Extracts the difference between Model B and Model A and saves it as a new LoRA file. **This mode outputs a LoRA, not a full model.** The output file will be saved to your `loras` directory. +> Computes the difference between Model A and Model B and extracts it as a new LoRA file using Singular Value Decomposition (SVD). **Outputs a LoRA safetensors file into your `loras` directory, not a full model.** + +**Models Used:** A (Tuned / Modified Model), B (Base Model) +- Computes delta: $\Delta W = A - B$ +- Low-rank approximation: $\Delta W \approx U S V^T \implies \text{lora\_B} \cdot \text{lora\_A}$ -**Models Used:** A (Tuned Model), B (Base Model) **Parameters:** -- **Alpha:** The Rank (dimension) for the LoRA's standard layers. -- **Beta:** The Rank (dimension) specifically for 3x3 convolution layers. -- **Gamma:** The clamp quantile for weight values. `0.99` is a typical value to prevent outlier weights from dominating. +- **Alpha (Linear Rank):** Target rank (dimension) for 2D dense/linear layer LoRA factors (e.g. `32`, `64`, `128`). +- **Beta (Conv Rank):** Target rank specifically for 3x3 convolution layer LoRA factors (e.g. `16`, `32`). +- **Gamma (Clamp Quantile):** Outlier clamping quantile (e.g. `0.99` clips the top 1% extreme values before SVD to prevent outlier distortion). --- -## Layer Mismatch Handling +## Layer Mismatch Handling (`mismatch_mode`) -When merging models with different layer structures (e.g., fine-tuned models with added/removed layers, or LoRAs with partial layer coverage), the `mismatch_mode` parameter controls behavior: +When Model A and Model B have differing layer structures or when merging partial LoRAs: | Mode | Behavior | |------|----------| -| `skip` | Missing layers in B use A's values **(default)** | -| `zeros` | Missing layers in B are treated as zeros | -| `error` | Fail if any layer is missing | +| `skip` | Missing layers in Model B use Model A's values unchanged **(default)** | +| `zeros` | Missing layers in Model B are treated as zeros | +| `error` | Raise an error and stop execution if any layer is missing in Model B | -**Common scenarios:** -- Merging a fine-tuned model with added/removed layers -- Combining LoRA-extracted differences with base models -- Cross-architecture experiments - -**Note:** Extra layers in Model B that don't exist in Model A are currently ignored. The output will always have the same layer structure as Model A. +**Note:** Extra layers in Model B that do not exist in Model A are ignored. Output topology always mirrors Model A. --- ## Dtype Preservation & Override -By default, the merger preserves the highest precision `dtype` for each key found across all active source models (Model A and Model B) and the requested `save_dtype`. - -- **Preserve Higher Precision**: If a key is `float32` in any of the source models, but you requested `save_dtype="bf16"` or `"fp16"`, that key will be saved as `float32` anyway. This prevents degrading critical high-precision layers (such as normalization layers or layers with restored outliers). -- **Explicit Override**: If you prefer to force the entire model to be saved strictly as the selected `save_dtype` (e.g. for complete downcasting to `bf16`), toggle `override_dtype` to `True`. +By default, the merger preserves the highest precision `dtype` for each key found across active source models and the requested `save_dtype`: +- **Preserve Higher Precision (default, `override_dtype = False`)**: If a key is `float32` in Model A or Model B, but `save_dtype="bf16"` is selected, that key remains `float32` to protect sensitive normalization layers. +- **Explicit Override (`override_dtype = True`)**: Forces all floating tensors strictly to `save_dtype`. --- -## Layer Filtering (Regex Patterns) +## Layer Filtering -Use regex patterns to control which layers are merged: - -### `exclude_patterns` -Layers matching any pattern will **keep Model A's values only** (no merge). - -### `discard_patterns` -Layers matching any pattern will be **removed entirely** from the output. - -**Pattern format:** -- Whitespace-separated regex patterns -- Patterns use **substring matching** (not full match) -- Example: `text_model lora` matches any key containing "text_model" OR "lora" -- Example: `layer\.[0-5]\.` matches layers 0-5 using regex syntax - -**Pattern Examples:** -| Pattern | Matches | -|---------|---------| -| `text_model` | All text encoder layers | -| `\.norm` | All normalization layers | -| `attn\.(q\|k\|v)` | Query, key, value attention weights | -| `block\.[0-9]\.` | Blocks 0-9 | +- **`exclude_patterns`**: Newline-separated patterns. Matching layers are **not merged** and keep Model A's values unchanged (unless `include_mode` is enabled). +- **`include_mode`**: Inverts `exclude_patterns` into a whitelist: only matching layers are merged; all non-matching layers keep Model A's values unchanged. +- **`discard_patterns`**: Newline-separated patterns. Matching layers are **removed completely** from the saved output. +- **`glob_patterns`**: When enabled, uses shell wildcards (`*`, `?`). When disabled, uses regex substring matching. diff --git a/docs/merger_3_model_modes.md b/docs/merger_3_model_modes.md index 658d0dd..377b563 100644 --- a/docs/merger_3_model_modes.md +++ b/docs/merger_3_model_modes.md @@ -2,91 +2,111 @@ --- +### Core Convention Across All 3-Model Modes +- **Model A is ALWAYS the base / receiving model** that receives the merge modifications. +- **Model B is the donor / target fine-tune model.** +- **Model C is the reference base model of Model B.** +- In difference merging, $(B - C)$ calculates the isolated delta (the specific training / fine-tune changes introduced in Model B relative to its base Model C). +- That delta is then added onto Model A with scaling $\alpha$: + $$\text{Output} = A + (B - C) \cdot \alpha$$ + +--- + ## Add-Difference -> `A + (B - C) * α`. This classic technique applies the "difference" between models B and C to model A. For example, it can be used to add a style (B) to a base model (A) while subtracting the style's original base (C). +> $\text{Output} = A + (B - C) \cdot \alpha$. Extracts the isolated fine-tune / style delta from $(B - C)$ and applies it directly to Model A. + +**Models Used:** A (Receiving Base), B (Donor Fine-Tune), C (Donor's Base Reference) -**Models Used:** A, B, C **Parameters:** -- **Alpha:** A multiplier for the `(B - C)` difference vector before it's added to A. -- **Beta:** A smoothing toggle. `0` for Off, `1` for On. When On, applies a median and gaussian filter to the difference, which can result in a smoother merge. +- **Alpha (Delta Multiplier / Strength):** Scaling factor for how strongly the $(B - C)$ difference is added to Model A. + - `0.0`: **0% addition** (Output is pure Model A; no changes added). + - `0.25`: **25% strength addition** (gentle, subtle application of Model B's style/features onto Model A). + - `0.50`: **50% strength addition** (moderate application). + - `1.0`: **100% full-strength addition** (applies the exact full delta of B relative to C onto Model A). + - `1.5`: **150% over-amplified addition** (exaggerates the difference; can introduce visual artifacts if pushed too high). +- **Beta (Smoothing Toggle):** + - `0`: Off (raw mathematical delta added directly). + - `1`: On (applies median and Gaussian smoothing to the delta before adding, reducing harsh pixel noise and high-frequency artifacts). --- ## Train-Difference -> A variation of Add-Difference from the SuperMerger extension that uses a more complex scaling factor based on the relative distances between the tensors of A, B, and C. +> A variation of Add-Difference that calculates an adaptive scaling factor based on the relative vector distances between Model A, Model B, and Model C. + +**Models Used:** A (Receiving Base), B (Donor Fine-Tune), C (Donor's Base Reference) -**Models Used:** A, B, C **Parameters:** -- **Alpha:** A multiplier for the final calculated "training" difference. +- **Alpha (Difference Multiplier):** Multiplier for the calculated distance-weighted difference added to Model A. + - `0.0`: Pure Model A. + - `1.0`: Standard 100% strength addition of the adaptive training delta. + - `< 1.0`: Softened application. + - `> 1.0`: Amplified application. --- ## Extract-Features -> A powerful mode that identifies features present in both `(B - A)` and `(C - A)` and adds them to A. It uses per-vector cosine similarity to decide how much of each feature to keep, allowing for fine-grained control over combining aspects. +> Analyzes feature vectors present in both $(B - A)$ and $(C - A)$, using cosine similarity to isolate shared features and blend them into Model A. + +**Models Used:** A (Base), B (Donor 1), C (Donor 2) -**Models Used:** A, B, C **Parameters:** -- **Alpha:** Weights the merge between Model B (`0.0`) and Model C (`1.0`). -- **Beta:** Controls the focus on similarity (`0.0`) versus dissimilarity (`1.0`). -- **Gamma:** A bias exponent for similarity calculation. -- **Delta:** A final multiplier for the extracted features before they are added to Model A. +- **Alpha (B vs C Contribution Balance):** Weights the source contribution between Model B and Model C. + - `0.0`: 100% Model B's delta features. + - `0.5`: Equal mix of features from both Model B and Model C. + - `1.0`: 100% Model C's delta features. +- **Beta (Similarity vs Dissimilarity):** + - `0.0`: Focuses exclusively on features where B and C agree / are similar relative to A. + - `1.0`: Focuses on features where B and C diverge / are dissimiliar relative to A. +- **Gamma (Similarity Bias Exponent):** Exponential bias curve shaping how sharply similar features are separated from dissimilar features. +- **Delta (Feature Multiplier / Strength):** Final scaling factor for the extracted features before adding them onto Model A. + - `0.0`: No features added (pure Model A). + - `1.0`: Standard strength addition of extracted features onto Model A. --- ## Add-Dissimilarities -> Identifies features that are dissimilar between Model B and Model C (relative to A) and adds them to Model A. Useful for combining unique aspects of two different models. +> Identifies features that are dissimilar between Model B and Model C (relative to Model A) and transfers them into Model A. Ideal for combining non-overlapping, unique aspects from two different fine-tunes without conflicting duplicate features. + +**Models Used:** A (Base), B (Donor 1), C (Donor 2) -**Models Used:** A, B, C **Parameters:** -- **Alpha:** Weights the comparison between Model B (`0.0`) and Model C (`1.0`). -- **Beta:** A multiplier for the final extracted dissimilar features. -- **Gamma:** A bias exponent for the similarity calculation. +- **Alpha (B vs C Balance):** Balance weighting between Model B and Model C when extracting differences. + - `0.0`: 100% weight to Model B's dissimilarities. + - `0.5`: Balanced extraction from both models. + - `1.0`: 100% weight to Model C's dissimilarities. +- **Beta (Feature Multiplier / Strength):** Final scaling multiplier for the dissimilar features added to Model A. + - `0.0`: No features added (pure Model A). + - `1.0`: Standard strength addition onto Model A. +- **Gamma (Similarity Bias Exponent):** Exponent controlling the cutoff sensitivity between similar and dissimilar directions. --- -## Layer Mismatch Handling +## Layer Mismatch Handling (`mismatch_mode`) -When merging models with different layer structures, the `mismatch_mode` parameter controls behavior: +When merging models with different layer topologies: | Mode | Behavior | |------|----------| -| `skip` | Missing layers in B/C use A's values **(default)** | -| `zeros` | Missing layers in B/C are treated as zeros | -| `error` | Fail if any layer is missing | +| `skip` | Missing layers in B or C use Model A's values unchanged **(default)** | +| `zeros` | Missing layers in B or C are treated as zero tensors | +| `error` | Raise an error and abort if any layer is missing across models | -**Special case for B - C subtraction (Add-Difference, Train-Difference):** -- If **both** B and C have the layer: Normal subtraction is performed -- If **either** B or C is missing the layer: The difference is treated as `None` (no contribution from this layer), and Model A's value is used - -**Common scenarios:** -- Merging a fine-tuned model with added/removed layers -- Extracting style differences between models with slightly different architectures -- Combining partial LoRAs or adapters - -**Note:** Extra layers in Models B/C that don't exist in Model A are currently ignored. The output will always have the same layer structure as Model A. +**Special rule for $(B - C)$ difference operations (Add-Difference, Train-Difference):** +- If **both** B and C have the layer: Normal difference $(B - C)$ is computed and applied to A. +- If **either** B or C is missing the layer: The difference is treated as zero, and Model A's value is preserved untouched. --- ## Dtype Preservation & Override -By default, the merger preserves the highest precision `dtype` for each key found across all active source models (Model A, B, and C) and the requested `save_dtype`. - -- **Preserve Higher Precision**: If a key is `float32` in any of the active source models, but you requested `save_dtype="bf16"` or `"fp16"`, that key will be saved as `float32` anyway. This prevents degrading critical high-precision layers (such as normalization layers or layers with restored outliers). -- **Explicit Override**: If you prefer to force the entire model to be saved strictly as the selected `save_dtype` (e.g. for complete downcasting to `bf16`), toggle `override_dtype` to `True`. +- **Preserve Higher Precision (default, `override_dtype = False`)**: If a key is `float32` in any source model (A, B, or C), it is saved as `float32` regardless of `save_dtype`. +- **Explicit Override (`override_dtype = True`)**: Forces all floating tensors strictly to `save_dtype`. --- -## Layer Filtering (Regex Patterns) +## Layer Filtering -Use regex patterns to control which layers are merged: - -### `exclude_patterns` -Layers matching any pattern will **keep Model A's values only** (no merge). - -### `discard_patterns` -Layers matching any pattern will be **removed entirely** from the output. - -**Pattern format:** -- Whitespace-separated regex patterns -- Patterns use **substring matching** (not full match) -- Example: `text_model lora` matches any key containing "text_model" OR "lora" +- **`exclude_patterns`**: Newline-separated patterns. Matching layers keep Model A's values untouched. +- **`include_mode`**: Inverts `exclude_patterns` into a whitelist: only matching layers receive the merge; all other layers keep Model A's values. +- **`discard_patterns`**: Newline-separated patterns. Matching layers are omitted completely from the output model. +- **`glob_patterns`**: Enables shell wildcards (`*`, `?`) when checked; uses regex substring matching when unchecked. diff --git a/nodes/merger_ops.py b/nodes/merger_ops.py index 91d8c1f..ef757a4 100644 --- a/nodes/merger_ops.py +++ b/nodes/merger_ops.py @@ -692,7 +692,12 @@ class ManEnhInterpDifference(CalcMode): name = 'Enhanced Man Interp' description = 'Enhanced interpolation between each pair of values from A and B depending on their difference relative to other values.' models_used = ['A', 'B'] - param_docs = {'alpha': 'Interpolation strength.', 'beta': 'Lower mean threshold.', 'gamma': 'Upper mean threshold.', 'delta': 'Smoothness factor.'} + param_docs = { + 'alpha': 'Model B pull strength (exponent 1/alpha - 1). 0.25 strongly suppresses B; 0.50 is linear; 1.0 is full B.', + 'beta': 'Lower mean threshold. Only weights with normalized difference > beta blend B.', + 'gamma': 'Upper mean threshold. Only weights with normalized difference < gamma blend B.', + 'delta': 'Smoothness factor. 0.0 is stochastic Bernoulli choice; 1.0 is smooth linear blend.', + } def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) alignment_mode = kwargs.get('alignment_mode', 'pad/crop') @@ -717,7 +722,11 @@ class AutoEnhInterpDifference(CalcMode): name = 'Enhanced Auto Interp' description = 'Interpolates between each pair of values from A and B depending on their difference relative to other values.' models_used = ['A', 'B'] - param_docs = {'alpha': 'Interpolation strength.', 'beta': 'Threshold adjustment factor.', 'gamma': 'Smoothness factor.'} + param_docs = { + 'alpha': 'Model B pull strength (exponent 1/alpha - 1). Lower keeps output close to Model A; higher pulls in Model B.', + 'beta': 'Threshold width around the layer mean difference (+/- beta).', + 'gamma': 'Smoothness factor. 0.0 is stochastic Bernoulli choice; 1.0 is smooth linear blend.', + } def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) alignment_mode = kwargs.get('alignment_mode', 'pad/crop') @@ -742,7 +751,11 @@ class WeightSumCutoffMode(CalcMode): name = 'Weight-Sum Cutoff' description = 'Weight-sum with cutoff based on value differences.' models_used = ['A', 'B'] - param_docs = {'alpha': 'Interpolation weight.', 'beta': 'Upper threshold.', 'gamma': 'Lower threshold.'} + param_docs = { + 'alpha': 'Model B blend fraction inside the threshold window. 0.0 is pure Model A; 0.25 is 25% B / 75% A; 1.0 is 100% B.', + 'beta': 'Upper difference threshold cutoff.', + 'gamma': 'Lower difference threshold cutoff.', + } def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) alignment_mode = kwargs.get('alignment_mode', 'pad/crop')