diff --git a/.gitignore b/.gitignore index 0a19790..c2ad863 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Repo specific +AGENTS.MD +custom_nodes.examples.md + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/docs/merger_2_model_modes.md b/docs/merger_2_model_modes.md index 0a6d5f8..d1100df 100644 --- a/docs/merger_2_model_modes.md +++ b/docs/merger_2_model_modes.md @@ -39,4 +39,49 @@ **Parameters:** - **Alpha:** The Rank (dimension) for the LoRA's standard layers. - **Beta:** The Rank (dimension) specifically for 3x3 convolution layers. -- **Gamma:** The clamp quantile for weight values. `0.99` is a typical value to prevent outlier weights from dominating. \ No newline at end of file +- **Gamma:** The clamp quantile for weight values. `0.99` is a typical value to prevent outlier weights from dominating. + +--- + +## Layer Mismatch Handling + +When merging models with different layer structures (e.g., fine-tuned models with added/removed layers, or LoRAs with partial layer coverage), the `mismatch_mode` parameter controls behavior: + +| Mode | Behavior | +|------|----------| +| `skip` | Missing layers in B use A's values **(default)** | +| `zeros` | Missing layers in B are treated as zeros | +| `error` | Fail if any layer is missing | + +**Common scenarios:** +- Merging a fine-tuned model with added/removed layers +- Combining LoRA-extracted differences with base models +- Cross-architecture experiments + +**Note:** Extra layers in Model B that don't exist in Model A are currently ignored. The output will always have the same layer structure as Model A. + +--- + +## Layer Filtering (Regex Patterns) + +Use regex patterns to control which layers are merged: + +### `exclude_patterns` +Layers matching any pattern will **keep Model A's values only** (no merge). + +### `discard_patterns` +Layers matching any pattern will be **removed entirely** from the output. + +**Pattern format:** +- Whitespace-separated regex patterns +- Patterns use **substring matching** (not full match) +- Example: `text_model lora` matches any key containing "text_model" OR "lora" +- Example: `layer\.[0-5]\.` matches layers 0-5 using regex syntax + +**Pattern Examples:** +| Pattern | Matches | +|---------|---------| +| `text_model` | All text encoder layers | +| `\.norm` | All normalization layers | +| `attn\.(q\|k\|v)` | Query, key, value attention weights | +| `block\.[0-9]\.` | Blocks 0-9 | \ No newline at end of file diff --git a/docs/merger_3_model_modes.md b/docs/merger_3_model_modes.md index 06b273e..ba5020d 100644 --- a/docs/merger_3_model_modes.md +++ b/docs/merger_3_model_modes.md @@ -40,4 +40,44 @@ **Parameters:** - **Alpha:** Weights the comparison between Model B (`0.0`) and Model C (`1.0`). - **Beta:** A multiplier for the final extracted dissimilar features. -- **Gamma:** A bias exponent for the similarity calculation. \ No newline at end of file +- **Gamma:** A bias exponent for the similarity calculation. + +--- + +## Layer Mismatch Handling + +When merging models with different layer structures, the `mismatch_mode` parameter controls behavior: + +| Mode | Behavior | +|------|----------| +| `skip` | Missing layers in B/C use A's values **(default)** | +| `zeros` | Missing layers in B/C are treated as zeros | +| `error` | Fail if any layer is missing | + +**Special case for B - C subtraction (Add-Difference, Train-Difference):** +- If **both** B and C have the layer: Normal subtraction is performed +- If **either** B or C is missing the layer: The difference is treated as `None` (no contribution from this layer), and Model A's value is used + +**Common scenarios:** +- Merging a fine-tuned model with added/removed layers +- Extracting style differences between models with slightly different architectures +- Combining partial LoRAs or adapters + +**Note:** Extra layers in Models B/C that don't exist in Model A are currently ignored. The output will always have the same layer structure as Model A. + +--- + +## Layer Filtering (Regex Patterns) + +Use regex patterns to control which layers are merged: + +### `exclude_patterns` +Layers matching any pattern will **keep Model A's values only** (no merge). + +### `discard_patterns` +Layers matching any pattern will be **removed entirely** from the output. + +**Pattern format:** +- Whitespace-separated regex patterns +- Patterns use **substring matching** (not full match) +- Example: `text_model lora` matches any key containing "text_model" OR "lora" \ No newline at end of file diff --git a/nodes/merger.py b/nodes/merger.py index 77cf366..5888107 100644 --- a/nodes/merger.py +++ b/nodes/merger.py @@ -1,11 +1,13 @@ import os +import re import torch import folder_paths import comfy.utils from safetensors.torch import save_file -from .merger_ops import TWO_MODEL_MODES, THREE_MODEL_MODES +from .merger_ops import TWO_MODEL_MODES, THREE_MODEL_MODES, MissingTensorBehavior, MissingTensorError from .merger_utils import MemoryEfficientSafeOpen + def load_documentation_from_file(filename): """Loads documentation from a markdown file in the ../docs/ directory.""" current_dir = os.path.dirname(os.path.realpath(__file__)) @@ -16,6 +18,33 @@ def load_documentation_from_file(filename): except FileNotFoundError: return f"# Documentation File Not Found\n\nPlease ensure `{filename}` exists in the `docs` directory of the custom node." + +def _compile_patterns(pattern_string): + """Compiles whitespace-separated regex patterns into a list of compiled patterns. + + Returns empty list if pattern_string is empty or whitespace-only. + Raises ValueError if any pattern is invalid regex. + """ + if not pattern_string or not pattern_string.strip(): + return [] + + patterns = [] + for pattern in pattern_string.split(): + try: + patterns.append(re.compile(pattern)) + except re.error as e: + raise ValueError(f"Invalid regex pattern '{pattern}': {e}") + return patterns + + +def _matches_any_pattern(key, patterns): + """Returns True if key matches any of the compiled regex patterns (substring search).""" + for pattern in patterns: + if pattern.search(key): + return True + return False + + # --- UI Dictionary Helper Functions --- def _create_two_model_inputs(model_folder_name): return {"required": { @@ -23,6 +52,7 @@ def _create_two_model_inputs(model_folder_name): "model_a": (["None"] + folder_paths.get_filename_list(model_folder_name),), "model_b": (["None"] + folder_paths.get_filename_list(model_folder_name),), "calc_mode": ([m.name for m in TWO_MODEL_MODES],), + "mismatch_mode": (["skip", "zeros", "error"], {"default": "skip"}), "alpha": ("FLOAT", {"default": 0.5, "min": -2.0, "max": 3.0, "step": 0.01}), "beta": ("FLOAT", {"default": 0.5, "min": -2.0, "max": 3.0, "step": 0.01}), "gamma": ("FLOAT", {"default": 0.99, "min": 0.0, "max": 1.0, "step": 0.001}), @@ -30,8 +60,11 @@ def _create_two_model_inputs(model_folder_name): "output_filename": ("STRING", {"default": "merged_2_model"}), "save_dtype": (["fp32", "fp16", "bf16"],), "process_device": (["cuda", "cpu"],), + "exclude_patterns": ("STRING", {"default": "", "multiline": True, "placeholder": "Regex patterns (space-separated) - matching layers keep Model A only"}), + "discard_patterns": ("STRING", {"default": "", "multiline": True, "placeholder": "Regex patterns (space-separated) - matching layers removed from output"}), }} + def _create_three_model_inputs(model_folder_name): return {"required": { "execution_mode": (["MERGE", "DOCUMENTATION ONLY"],), @@ -39,6 +72,7 @@ def _create_three_model_inputs(model_folder_name): "model_b": (["None"] + folder_paths.get_filename_list(model_folder_name),), "model_c": (["None"] + folder_paths.get_filename_list(model_folder_name),), "calc_mode": ([m.name for m in THREE_MODEL_MODES],), + "mismatch_mode": (["skip", "zeros", "error"], {"default": "skip"}), "alpha": ("FLOAT", {"default": 0.5, "min": -2.0, "max": 3.0, "step": 0.01}), "beta": ("FLOAT", {"default": 0.5, "min": -2.0, "max": 3.0, "step": 0.01}), "gamma": ("FLOAT", {"default": 0.5, "min": -2.0, "max": 3.0, "step": 0.01}), @@ -47,8 +81,11 @@ def _create_three_model_inputs(model_folder_name): "output_filename": ("STRING", {"default": "merged_3_model"}), "save_dtype": (["fp32", "fp16", "bf16"],), "process_device": (["cuda", "cpu"],), + "exclude_patterns": ("STRING", {"default": "", "multiline": True, "placeholder": "Regex patterns (space-separated) - matching layers keep Model A only"}), + "discard_patterns": ("STRING", {"default": "", "multiline": True, "placeholder": "Regex patterns (space-separated) - matching layers removed from output"}), }} + # --- Base Classes for Backend Logic ONLY --- class MergerLogic: def _execute_merge(self, model_names, calc_mode, all_modes, recipe_params, model_type): @@ -56,26 +93,108 @@ class MergerLogic: if not calc_mode_class: raise ValueError(f"Calc mode '{calc_mode}' not found.") primary_model_name = model_names.get('model_a') if not primary_model_name or primary_model_name == "None": raise ValueError("Model A is required to run the merge.") + handlers = {} for name in model_names.values(): if name and name != "None": path = folder_paths.get_full_path(model_type, name) if not path: raise FileNotFoundError(f"Model '{name}' not found.") handlers[name] = MemoryEfficientSafeOpen(path) - all_keys = handlers[primary_model_name].keys() - metadata = handlers[primary_model_name].header.get("__metadata__", {}) + + primary_handler = handlers[primary_model_name] + all_keys = primary_handler.keys() + metadata = primary_handler.header.get("__metadata__", {}) + + # Convert mismatch_mode string to enum + mismatch_mode_str = recipe_params.get('mismatch_mode', 'skip') + mismatch_mode = MissingTensorBehavior(mismatch_mode_str) + recipe_params['mismatch_mode'] = mismatch_mode + + # Compile filter patterns + exclude_patterns = _compile_patterns(recipe_params.get('exclude_patterns', '')) + discard_patterns = _compile_patterns(recipe_params.get('discard_patterns', '')) + + # Pre-compute key differences for logging + primary_keys = set(all_keys) + for name, handler in handlers.items(): + if name != primary_model_name: + secondary_keys = set(handler.keys()) + missing = primary_keys - secondary_keys + extra = secondary_keys - primary_keys + if missing: + print(f"[Merger] {name} is missing {len(missing)} keys present in Model A") + if extra: + print(f"[Merger] {name} has {len(extra)} extra keys not in Model A (ignored)") + merged_state_dict = {} pbar = comfy.utils.ProgressBar(len(all_keys)) save_dtype = recipe_params.pop('save_dtype') save_torch_dtype = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16}.get(save_dtype) - recipe_params.update({"handlers": handlers}) # Add handlers, device and dtype are already in + recipe_params.update({"handlers": handlers}) + process_device = recipe_params.get('device', 'cpu') + process_dtype = recipe_params.get('dtype', torch.float32) + + skipped_keys = 0 + excluded_keys = 0 + discarded_keys = 0 + error_keys = [] + for key in all_keys: - recipe = calc_mode_class.create_recipe(key=key, **recipe_params) - result = recipe.merge() + # Check discard patterns first - skip entirely + if _matches_any_pattern(key, discard_patterns): + discarded_keys += 1 + pbar.update(1) + continue + + # Pre-load Model A's tensor + tensor_a = primary_handler.get_tensor(key).to(device=process_device, dtype=process_dtype) + + # Check exclude patterns - use Model A only, no merge + if _matches_any_pattern(key, exclude_patterns): + merged_state_dict[key] = tensor_a.to(save_torch_dtype).cpu().clone() + excluded_keys += 1 + pbar.update(1) + continue + + # Pass tensor_a metadata to recipes for zeros mode and fallback + recipe_params['_tensor_a'] = tensor_a + recipe_params['_tensor_a_shape'] = tensor_a.shape + recipe_params['_tensor_a_dtype'] = tensor_a.dtype + + try: + recipe = calc_mode_class.create_recipe(key=key, **recipe_params) + result = recipe.merge() + except MissingTensorError as e: + if mismatch_mode == MissingTensorBehavior.ERROR: + raise ValueError(f"Layer mismatch error (mismatch_mode='error'): {e}") + result = None + error_keys.append(key) + + # Handle None result (mismatch occurred with skip mode) + if result is None: + result = tensor_a + skipped_keys += 1 + if isinstance(result, dict): - for r_key, r_tensor in result.items(): merged_state_dict[r_key] = r_tensor.to(save_torch_dtype).cpu().clone() - else: merged_state_dict[key] = result.to(save_torch_dtype).cpu().clone() + for r_key, r_tensor in result.items(): + merged_state_dict[r_key] = r_tensor.to(save_torch_dtype).cpu().clone() + else: + merged_state_dict[key] = result.to(save_torch_dtype).cpu().clone() + + # Clean up tensor_a reference to allow GC + del recipe_params['_tensor_a'] pbar.update(1) + + # Log summary + if excluded_keys > 0: + print(f"[Merger] Excluded {excluded_keys} keys from merge (kept Model A only)") + if discarded_keys > 0: + print(f"[Merger] Discarded {discarded_keys} keys from output") + if skipped_keys > 0: + print(f"[Merger] Used Model A's values for {skipped_keys} keys due to mismatches") + if error_keys: + print(f"[Merger] Unexpected errors on {len(error_keys)} keys: {error_keys[:5]}{'...' if len(error_keys) > 5 else ''}") + for handler in handlers.values(): handler.__exit__(None, None, None) output_folder = "loras" if calc_mode == "SVD LoRA Extraction" else model_type output_dir = folder_paths.get_folder_paths(output_folder)[0] @@ -85,44 +204,51 @@ class MergerLogic: save_file(merged_state_dict, output_path, metadata=metadata) return (f"{output_filename}.safetensors",) + class BaseTwoMerger(MergerLogic): FUNCTION = "merge"; CATEGORY = "ModelUtils/Merging" RETURN_TYPES = ("STRING", "STRING",); RETURN_NAMES = ("output_filename", "documentation",) - def merge(self, execution_mode, model_a, model_b, calc_mode, alpha, beta, gamma, seed, output_filename, save_dtype, process_device): + def merge(self, execution_mode, model_a, model_b, calc_mode, mismatch_mode, alpha, beta, gamma, seed, output_filename, save_dtype, process_device, exclude_patterns, discard_patterns): doc = load_documentation_from_file('merger_2_model_modes.md') if execution_mode == "DOCUMENTATION ONLY": return ("Documentation mode active. No merge performed.", doc) recipe_params = { "model_a": model_a, "model_b": model_b, "calc_mode": calc_mode, + "mismatch_mode": mismatch_mode, "alpha": alpha, "beta": beta, "gamma": gamma, "seed": seed, "output_filename": output_filename, "save_dtype": save_dtype, - "device": process_device, "dtype": torch.float32 + "device": process_device, "dtype": torch.float32, + "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns, } model_names = {"model_a": model_a, "model_b": model_b} filename, = self._execute_merge(model_names, calc_mode, TWO_MODEL_MODES, recipe_params, self.MODEL_TYPE) return (filename, doc) + class BaseThreeMerger(MergerLogic): FUNCTION = "merge"; CATEGORY = "ModelUtils/Merging" RETURN_TYPES = ("STRING", "STRING",); RETURN_NAMES = ("output_filename", "documentation",) - def merge(self, execution_mode, model_a, model_b, model_c, calc_mode, alpha, beta, gamma, delta, seed, output_filename, save_dtype, process_device): + def merge(self, execution_mode, model_a, model_b, model_c, calc_mode, mismatch_mode, alpha, beta, gamma, delta, seed, output_filename, save_dtype, process_device, exclude_patterns, discard_patterns): doc = load_documentation_from_file('merger_3_model_modes.md') if execution_mode == "DOCUMENTATION ONLY": return ("Documentation mode active. No merge performed.", doc) recipe_params = { "model_a": model_a, "model_b": model_b, "model_c": model_c, "calc_mode": calc_mode, + "mismatch_mode": mismatch_mode, "alpha": alpha, "beta": beta, "gamma": gamma, "delta": delta, "seed": seed, "output_filename": output_filename, "save_dtype": save_dtype, - "device": process_device, "dtype": torch.float32 + "device": process_device, "dtype": torch.float32, + "exclude_patterns": exclude_patterns, "discard_patterns": discard_patterns, } model_names = {"model_a": model_a, "model_b": model_b, "model_c": model_c} filename, = self._execute_merge(model_names, calc_mode, THREE_MODEL_MODES, recipe_params, self.MODEL_TYPE) return (filename, doc) + # --- Concrete 2-Model Nodes --- class CheckpointTwoMerger(BaseTwoMerger): MODEL_TYPE = "checkpoints" diff --git a/nodes/merger_ops.py b/nodes/merger_ops.py index 9664f2e..be93a92 100644 --- a/nodes/merger_ops.py +++ b/nodes/merger_ops.py @@ -1,6 +1,19 @@ import torch import scipy.ndimage import torch.nn.functional as F +from enum import Enum + + +class MissingTensorBehavior(Enum): + """Controls behavior when a tensor key is missing from a model.""" + ERROR = "error" # Raise exception (strict mode) + SKIP = "skip" # Return None, let caller handle + ZEROS = "zeros" # Return zeros tensor matching shape from primary model + + +class MissingTensorError(Exception): + """Raised when a tensor is missing and behavior is ERROR.""" + pass # ################# # # MERGE OPERATORS @@ -34,29 +47,62 @@ class Operation: return self.merge_func(self) class LoadTensor(Operation): - def __init__(self, key, model_name, handlers, device, dtype): + def __init__(self, key, model_name, handlers, device, dtype, + on_missing=MissingTensorBehavior.ERROR, fallback_shape=None, fallback_dtype=None): super().__init__(key) self.model_name = model_name self.handlers = handlers self.device = device self.dtype = dtype + self.on_missing = on_missing + self.fallback_shape = fallback_shape + self.fallback_dtype = fallback_dtype def merge(self) -> torch.Tensor: if self.model_name not in self.handlers: raise ValueError(f"Model '{self.model_name}' is required for this mode but was not provided.") - return self.handlers[self.model_name].get_tensor(self.key).to(device=self.device, dtype=self.dtype) + + handler = self.handlers[self.model_name] + if self.key not in handler.keys(): + if self.on_missing == MissingTensorBehavior.ERROR: + raise MissingTensorError( + f"Key '{self.key}' not found in model '{self.model_name}'" + ) + elif self.on_missing == MissingTensorBehavior.SKIP: + return None + elif self.on_missing == MissingTensorBehavior.ZEROS: + if self.fallback_shape is None: + raise ValueError( + f"Cannot create zeros for '{self.key}' without fallback_shape" + ) + dtype = self.fallback_dtype if self.fallback_dtype else self.dtype + return torch.zeros(self.fallback_shape, device=self.device, dtype=dtype) + + return handler.get_tensor(self.key).to(device=self.device, dtype=self.dtype) class Multiply(Operation): def __init__(self, key, alpha, *sources): super().__init__(key, *sources) self.alpha = alpha def oper(self, a) -> torch.Tensor: + if a is None: + return None return a * self.alpha + + class Add(Operation): def oper(self, a, b) -> torch.Tensor: + if a is None: + return b + if b is None: + return a return a + b + + class Sub(Operation): def oper(self, a, b) -> torch.Tensor: + if a is None or b is None: + return None # Cannot compute difference with missing operand return a - b class SVD(Operation): @@ -64,6 +110,8 @@ class SVD(Operation): super().__init__(key, *sources) self.alpha, self.beta, self.gamma = int(alpha), int(beta), gamma def oper(self, a, b) -> torch.Tensor: + if a is None or b is None: + return None diff, weights, conv2d = a - b, {}, len(a.size()) == 4 kernel_size = None if not conv2d else a.size()[2:4] conv2d_3x3 = conv2d and kernel_size != (1, 1) @@ -84,11 +132,15 @@ class SVD(Operation): class Smooth(Operation): def oper(self, a) -> torch.Tensor: + if a is None: + return None filtered = scipy.ndimage.median_filter(a.detach().cpu().to(torch.float32).numpy(), size=3) filtered = scipy.ndimage.gaussian_filter(filtered, sigma=1) return torch.tensor(filtered, dtype=a.dtype, device=a.device) class TrainDiff(Operation): def oper(self, a, b, c) -> torch.Tensor: + if a is None or b is None or c is None: + return None if torch.allclose(b.float(), c.float()): return torch.zeros_like(a) diff_bc = b.float() - c.float() dist_bc, dist_ab = torch.abs(diff_bc), torch.abs(b.float() - a.float()) @@ -99,27 +151,40 @@ class ExtractOp(Operation): super().__init__(key, *args) self.alpha, self.beta, self.gamma = alpha, beta, gamma def oper(self, base: torch.Tensor, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + if a is None or b is None: + return None base_f = base.float() if base is not None else 0 a_f, b_f = a.float() - base_f, b.float() - base_f c = torch.cosine_similarity(a_f.flatten(), b_f.flatten(), 0).clamp(-1, 1) d = ((c + 1) / 2)**self.gamma return (torch.lerp(a_f, b_f, self.alpha) * torch.lerp(d, 1 - d, self.beta)).to(a.dtype) + + class Similarities(ExtractOp): - def oper(self, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: return super().oper(None, a, b) + def oper(self, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + if a is None or b is None: + return None + return super().oper(None, a, b) class PowerUpOp(Operation): def __init__(self, key, alpha, seed, *sources): super().__init__(key, *sources) self.alpha, self.seed = alpha, seed def oper(self, a, b): + if a is None or b is None: + return None delta = b - a rng = torch.Generator(device=delta.device).manual_seed(self.seed) m = torch.bernoulli(torch.full_like(delta, 1 - self.alpha), generator=rng) return (m * delta) / (1 - self.alpha) if (1 - self.alpha) > 0 else torch.zeros_like(delta) + + class InterpolateDifference(Operation): def __init__(self, key, alpha, beta, gamma, seed, *sources): super().__init__(key, *sources) self.alpha, self.beta, self.gamma, self.seed = alpha, beta, gamma, seed def oper(self, a, b): + if a is None or b is None: + return None alpha, delta = max(self.alpha, 0.001), torch.abs(a - b) max_delta = torch.max(delta) if max_delta == 0: return a @@ -135,6 +200,34 @@ class InterpolateDifference(Operation): class CalcMode: name = 'calcmode'; description = 'description'; models_used = []; param_docs = {} def create_recipe(self, key, **kwargs) -> Operation: raise NotImplementedError + + @staticmethod + def _get_secondary_loader_args(kwargs, mismatch_mode): + """Helper to build loader args for secondary models (B, C) with fallback support.""" + base_args = { + "handlers": kwargs['handlers'], + "device": kwargs['device'], + "dtype": kwargs['dtype'], + "on_missing": mismatch_mode, + } + # Add fallback info for zeros mode + if '_tensor_a_shape' in kwargs: + base_args['fallback_shape'] = kwargs['_tensor_a_shape'] + base_args['fallback_dtype'] = kwargs.get('_tensor_a_dtype', kwargs['dtype']) + return base_args + + +class PreloadedTensor(Operation): + """Returns a pre-loaded tensor instead of loading from file. + + Used when Model A's tensor is pre-loaded in the merge loop. + """ + def __init__(self, key, tensor): + super().__init__(key) + self.tensor = tensor + + def merge(self) -> torch.Tensor: + return self.tensor class WeightSum(CalcMode): name = 'Weight-Sum' @@ -142,38 +235,65 @@ class WeightSum(CalcMode): models_used = ['A', 'B'] param_docs = {'alpha': 'Interpolation weight. 0.0 is 100% Model A, 1.0 is 100% Model B.'} def create_recipe(self, key, **kwargs): - loader_args = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - a = LoadTensor(key, kwargs['model_a'], **loader_args) - b = LoadTensor(key, kwargs['model_b'], **loader_args) + mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) + + # Use pre-loaded tensor_a if available + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode) + b = LoadTensor(key, kwargs['model_b'], **loader_args_b) + if kwargs['alpha'] >= 1: return b if kwargs['alpha'] <= 0: return a c = Multiply(key, 1 - kwargs['alpha'], a) d = Multiply(key, kwargs['alpha'], b) return Add(key, c, d) + + class AddDifference(CalcMode): name = 'Add-Difference' description = 'A + (B - C) * α. Applies the difference between B and C to A.' models_used = ['A', 'B', 'C'] param_docs = {'alpha': 'Multiplier for the (B - C) difference.', 'beta': 'Smoothing (0=Off, 1=On).'} def create_recipe(self, key, **kwargs): - loader_args = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - a = LoadTensor(key, kwargs['model_a'], **loader_args) - b = LoadTensor(key, kwargs['model_b'], **loader_args) - c = LoadTensor(key, kwargs['model_c'], **loader_args) + mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode) + b = LoadTensor(key, kwargs['model_b'], **loader_args_bc) + c = LoadTensor(key, kwargs['model_c'], **loader_args_bc) diff = Sub(key, b, c) if kwargs['beta'] == 1: diff = Smooth(key, diff) diffm = Multiply(key, kwargs['alpha'], diff) return Add(key, a, diffm) + + class TrainDifference(CalcMode): name = 'Train-Difference' description = 'A + (B - C) * α, using tensor distances.' models_used = ['A', 'B', 'C'] param_docs = {'alpha': 'Multiplier for the calculated difference.'} def create_recipe(self, key, **kwargs): - loader_args = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - a = LoadTensor(key, kwargs['model_a'], **loader_args) - b = LoadTensor(key, kwargs['model_b'], **loader_args) - c = LoadTensor(key, kwargs['model_c'], **loader_args) + mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode) + b = LoadTensor(key, kwargs['model_b'], **loader_args_bc) + c = LoadTensor(key, kwargs['model_c'], **loader_args_bc) diff = TrainDiff(key, a, b, c) diffm = Multiply(key, kwargs['alpha'], diff) return Add(key, a, diffm) @@ -183,59 +303,106 @@ class InterpDifference(CalcMode): models_used = ['A', 'B'] param_docs = {'alpha': 'Curve strength.', 'beta': 'Style (0=Similarity, 1=Difference).', 'gamma': 'Mix (0=Random, 1=Linear).'} def create_recipe(self, key, **kwargs): - loader_args = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - if key.startswith('cond_stage_model.transformer.text_model.embeddings'): return LoadTensor(key, kwargs['model_a'], **loader_args) - a = LoadTensor(key, kwargs['model_a'], **loader_args) - b = LoadTensor(key, kwargs['model_b'], **loader_args) + mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + # Skip certain keys entirely + if key.startswith('cond_stage_model.transformer.text_model.embeddings'): + return a + + loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode) + b = LoadTensor(key, kwargs['model_b'], **loader_args_b) return InterpolateDifference(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['seed'], a, b) + + class Extract(CalcMode): name = 'Extract-Features' description = 'Adds features from B and C to A, based on their similarity.' models_used = ['A', 'B', 'C'] param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Similarity vs dissimilarity.', 'gamma': 'Similarity bias.', 'delta': 'Multiplier for final features.'} def create_recipe(self, key, **kwargs): - loader_args = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - a = LoadTensor(key, kwargs['model_a'], **loader_args) - b = LoadTensor(key, kwargs['model_b'], **loader_args) - c = LoadTensor(key, kwargs['model_c'], **loader_args) + mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode) + b = LoadTensor(key, kwargs['model_b'], **loader_args_bc) + c = LoadTensor(key, kwargs['model_c'], **loader_args_bc) extracted = ExtractOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'] * 15, a, b, c) multiplied = Multiply(key, kwargs['delta'], extracted) return Add(key, a, multiplied) + + class AddDisimilarity(CalcMode): name = 'Add-Dissimilarities' description = 'Adds dissimilar features between B and C to model A.' models_used = ['A', 'B', 'C'] param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Multiplier for final features.', 'gamma': 'Similarity bias.'} def create_recipe(self, key, **kwargs): - loader_args = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - a = LoadTensor(key, kwargs['model_a'], **loader_args) - b = LoadTensor(key, kwargs['model_b'], **loader_args) - c = LoadTensor(key, kwargs['model_c'], **loader_args) + mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode) + b = LoadTensor(key, kwargs['model_b'], **loader_args_bc) + c = LoadTensor(key, kwargs['model_c'], **loader_args_bc) extracted = Similarities(key, kwargs['alpha'], 1, kwargs['gamma'] * 15, b, c) multiplied = Multiply(key, kwargs['beta'], extracted) return Add(key, a, multiplied) + + class PowerUp(CalcMode): name = 'Power-Up (DARE)' description = 'Adds capabilities of B to A using Drop and Rescale (DARE).' models_used = ['A', 'B'] param_docs = {'alpha': 'Dropout rate (proportion of weights from B to *drop*).', 'beta': 'Multiplier for the final difference.'} def create_recipe(self, key, **kwargs): - loader_args = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - a = LoadTensor(key, kwargs['model_a'], **loader_args) - b = LoadTensor(key, kwargs['model_b'], **loader_args) + mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode) + b = LoadTensor(key, kwargs['model_b'], **loader_args_b) deltahat = PowerUpOp(key, kwargs['alpha'], kwargs['seed'], a, b) res = Multiply(key, kwargs['beta'], deltahat) return Add(key, a, res) -class SVD(CalcMode): + + +class SVDMode(CalcMode): name = 'SVD LoRA Extraction' description = 'Extracts the difference between B and A into a LoRA file.' models_used = ['A', 'B'] param_docs = {'alpha': 'Rank for standard layers.', 'beta': 'Rank for 3x3 conv layers.', 'gamma': 'Clamp quantile for weights.'} def create_recipe(self, key, **kwargs): - loader_args = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - a = LoadTensor(key, kwargs['model_a'], **loader_args) - b = LoadTensor(key, kwargs['model_b'], **loader_args) + mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode) + b = LoadTensor(key, kwargs['model_b'], **loader_args_b) return SVD(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], a, b) -TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), SVD()] + +TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), SVDMode()] THREE_MODEL_MODES = [AddDifference(), TrainDifference(), Extract(), AddDisimilarity()] \ No newline at end of file diff --git a/nodes/merger_utils.py b/nodes/merger_utils.py index e18f872..59a0c4a 100644 --- a/nodes/merger_utils.py +++ b/nodes/merger_utils.py @@ -77,4 +77,30 @@ class MemoryEfficientSafeOpen: elif dtype_str == "F8_E4M3" and hasattr(torch, "float8_e4m3fn"): return byte_tensor.view(torch.float8_e4m3fn).reshape(shape) else: - raise ValueError(f"Unsupported float8 type: {dtype_str}. Your PyTorch version may be too old.") \ No newline at end of file + raise ValueError(f"Unsupported float8 type: {dtype_str}. Your PyTorch version may be too old.") + + +def get_union_keys(*handlers): + """Returns the union of all tensor keys across multiple model handlers. + + Useful when you want to iterate over all unique keys present in any model. + """ + all_keys = set() + for handler in handlers: + if handler is not None: + all_keys.update(handler.keys()) + return sorted(all_keys) + + +def get_intersection_keys(*handlers): + """Returns only keys present in ALL model handlers. + + Useful when you want to iterate only over keys that exist in every model. + """ + handlers = [h for h in handlers if h is not None] + if not handlers: + return [] + all_keys = set(handlers[0].keys()) + for handler in handlers[1:]: + all_keys &= set(handler.keys()) + return sorted(all_keys) \ No newline at end of file