diff --git a/nodes/merger.py b/nodes/merger.py index 9c7046d..37001fb 100644 --- a/nodes/merger.py +++ b/nodes/merger.py @@ -3,7 +3,7 @@ import torch import folder_paths import comfy.utils from safetensors.torch import save_file -from .merger_ops import TWO_MODEL_MODES, THREE_MODEL_MODES, MissingTensorBehavior +from .merger_ops import TWO_MODEL_MODES, THREE_MODEL_MODES, MissingTensorBehavior, MissingTensorError from .merger_utils import MemoryEfficientSafeOpen def load_documentation_from_file(filename): @@ -66,25 +66,28 @@ class MergerLogic: if not path: raise FileNotFoundError(f"Model '{name}' not found.") handlers[name] = MemoryEfficientSafeOpen(path) - all_keys = handlers[primary_model_name].keys() - metadata = handlers[primary_model_name].header.get("__metadata__", {}) + primary_handler = handlers[primary_model_name] + all_keys = primary_handler.keys() + metadata = primary_handler.header.get("__metadata__", {}) # Convert mismatch_mode string to enum mismatch_mode_str = recipe_params.get('mismatch_mode', 'skip') mismatch_mode = MissingTensorBehavior(mismatch_mode_str) recipe_params['mismatch_mode'] = mismatch_mode - # Log mismatched keys for debugging + # Pre-compute key differences for logging primary_keys = set(all_keys) + mismatch_report = {} for name, handler in handlers.items(): if name != primary_model_name: secondary_keys = set(handler.keys()) - missing_in_secondary = primary_keys - secondary_keys - extra_in_secondary = secondary_keys - primary_keys - if missing_in_secondary: - print(f"[Merger] {name} is missing {len(missing_in_secondary)} keys present in Model A") - if extra_in_secondary: - print(f"[Merger] {name} has {len(extra_in_secondary)} extra keys not in Model A (ignored)") + missing = primary_keys - secondary_keys + extra = secondary_keys - primary_keys + mismatch_report[name] = {"missing": missing, "extra": extra} + if missing: + print(f"[Merger] {name} is missing {len(missing)} keys present in Model A") + if extra: + print(f"[Merger] {name} has {len(extra)} extra keys not in Model A (ignored)") merged_state_dict = {} pbar = comfy.utils.ProgressBar(len(all_keys)) @@ -92,18 +95,34 @@ class MergerLogic: save_torch_dtype = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16}.get(save_dtype) recipe_params.update({"handlers": handlers}) process_device = recipe_params.get('device', 'cpu') + process_dtype = recipe_params.get('dtype', torch.float32) skipped_keys = 0 + error_keys = [] + for key in all_keys: - recipe = calc_mode_class.create_recipe(key=key, **recipe_params) - result = recipe.merge() + # Pre-load Model A's tensor - this is always required and serves as reference + tensor_a = primary_handler.get_tensor(key).to(device=process_device, dtype=process_dtype) + + # Pass tensor_a metadata to recipes for zeros mode and fallback + recipe_params['_tensor_a'] = tensor_a # For operations that need reference + recipe_params['_tensor_a_shape'] = tensor_a.shape + recipe_params['_tensor_a_dtype'] = tensor_a.dtype + + try: + recipe = calc_mode_class.create_recipe(key=key, **recipe_params) + result = recipe.merge() + except MissingTensorError as e: + if mismatch_mode == MissingTensorBehavior.ERROR: + # Re-raise to fail the entire merge + raise ValueError(f"Layer mismatch error (mismatch_mode='error'): {e}") + # This shouldn't happen in skip/zeros mode, but handle gracefully + result = None + error_keys.append(key) # Handle None result (mismatch occurred with skip mode) if result is None: - # Fall back to Model A's value - result = handlers[primary_model_name].get_tensor(key).to( - device=process_device, dtype=torch.float32 - ) + result = tensor_a # Use pre-loaded Model A tensor skipped_keys += 1 if isinstance(result, dict): @@ -111,10 +130,15 @@ class MergerLogic: merged_state_dict[r_key] = r_tensor.to(save_torch_dtype).cpu().clone() else: merged_state_dict[key] = result.to(save_torch_dtype).cpu().clone() + + # Clean up tensor_a reference to allow GC + del recipe_params['_tensor_a'] pbar.update(1) if skipped_keys > 0: print(f"[Merger] Used Model A's values for {skipped_keys} keys due to mismatches") + if error_keys: + print(f"[Merger] Unexpected errors on {len(error_keys)} keys: {error_keys[:5]}{'...' if len(error_keys) > 5 else ''}") for handler in handlers.values(): handler.__exit__(None, None, None) output_folder = "loras" if calc_mode == "SVD LoRA Extraction" else model_type diff --git a/nodes/merger_ops.py b/nodes/merger_ops.py index 7dcc748..be93a92 100644 --- a/nodes/merger_ops.py +++ b/nodes/merger_ops.py @@ -110,6 +110,8 @@ class SVD(Operation): super().__init__(key, *sources) self.alpha, self.beta, self.gamma = int(alpha), int(beta), gamma def oper(self, a, b) -> torch.Tensor: + if a is None or b is None: + return None diff, weights, conv2d = a - b, {}, len(a.size()) == 4 kernel_size = None if not conv2d else a.size()[2:4] conv2d_3x3 = conv2d and kernel_size != (1, 1) @@ -130,11 +132,15 @@ class SVD(Operation): class Smooth(Operation): def oper(self, a) -> torch.Tensor: + if a is None: + return None filtered = scipy.ndimage.median_filter(a.detach().cpu().to(torch.float32).numpy(), size=3) filtered = scipy.ndimage.gaussian_filter(filtered, sigma=1) return torch.tensor(filtered, dtype=a.dtype, device=a.device) class TrainDiff(Operation): def oper(self, a, b, c) -> torch.Tensor: + if a is None or b is None or c is None: + return None if torch.allclose(b.float(), c.float()): return torch.zeros_like(a) diff_bc = b.float() - c.float() dist_bc, dist_ab = torch.abs(diff_bc), torch.abs(b.float() - a.float()) @@ -145,27 +151,40 @@ class ExtractOp(Operation): super().__init__(key, *args) self.alpha, self.beta, self.gamma = alpha, beta, gamma def oper(self, base: torch.Tensor, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + if a is None or b is None: + return None base_f = base.float() if base is not None else 0 a_f, b_f = a.float() - base_f, b.float() - base_f c = torch.cosine_similarity(a_f.flatten(), b_f.flatten(), 0).clamp(-1, 1) d = ((c + 1) / 2)**self.gamma return (torch.lerp(a_f, b_f, self.alpha) * torch.lerp(d, 1 - d, self.beta)).to(a.dtype) + + class Similarities(ExtractOp): - def oper(self, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: return super().oper(None, a, b) + def oper(self, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: + if a is None or b is None: + return None + return super().oper(None, a, b) class PowerUpOp(Operation): def __init__(self, key, alpha, seed, *sources): super().__init__(key, *sources) self.alpha, self.seed = alpha, seed def oper(self, a, b): + if a is None or b is None: + return None delta = b - a rng = torch.Generator(device=delta.device).manual_seed(self.seed) m = torch.bernoulli(torch.full_like(delta, 1 - self.alpha), generator=rng) return (m * delta) / (1 - self.alpha) if (1 - self.alpha) > 0 else torch.zeros_like(delta) + + class InterpolateDifference(Operation): def __init__(self, key, alpha, beta, gamma, seed, *sources): super().__init__(key, *sources) self.alpha, self.beta, self.gamma, self.seed = alpha, beta, gamma, seed def oper(self, a, b): + if a is None or b is None: + return None alpha, delta = max(self.alpha, 0.001), torch.abs(a - b) max_delta = torch.max(delta) if max_delta == 0: return a @@ -181,6 +200,34 @@ class InterpolateDifference(Operation): class CalcMode: name = 'calcmode'; description = 'description'; models_used = []; param_docs = {} def create_recipe(self, key, **kwargs) -> Operation: raise NotImplementedError + + @staticmethod + def _get_secondary_loader_args(kwargs, mismatch_mode): + """Helper to build loader args for secondary models (B, C) with fallback support.""" + base_args = { + "handlers": kwargs['handlers'], + "device": kwargs['device'], + "dtype": kwargs['dtype'], + "on_missing": mismatch_mode, + } + # Add fallback info for zeros mode + if '_tensor_a_shape' in kwargs: + base_args['fallback_shape'] = kwargs['_tensor_a_shape'] + base_args['fallback_dtype'] = kwargs.get('_tensor_a_dtype', kwargs['dtype']) + return base_args + + +class PreloadedTensor(Operation): + """Returns a pre-loaded tensor instead of loading from file. + + Used when Model A's tensor is pre-loaded in the merge loop. + """ + def __init__(self, key, tensor): + super().__init__(key) + self.tensor = tensor + + def merge(self) -> torch.Tensor: + return self.tensor class WeightSum(CalcMode): name = 'Weight-Sum' @@ -189,15 +236,24 @@ class WeightSum(CalcMode): param_docs = {'alpha': 'Interpolation weight. 0.0 is 100% Model A, 1.0 is 100% Model B.'} def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) - loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - loader_args_b = {**loader_args_a, "on_missing": mismatch_mode} - a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + # Use pre-loaded tensor_a if available + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode) b = LoadTensor(key, kwargs['model_b'], **loader_args_b) + if kwargs['alpha'] >= 1: return b if kwargs['alpha'] <= 0: return a c = Multiply(key, 1 - kwargs['alpha'], a) d = Multiply(key, kwargs['alpha'], b) return Add(key, c, d) + + class AddDifference(CalcMode): name = 'Add-Difference' description = 'A + (B - C) * α. Applies the difference between B and C to A.' @@ -205,15 +261,22 @@ class AddDifference(CalcMode): param_docs = {'alpha': 'Multiplier for the (B - C) difference.', 'beta': 'Smoothing (0=Off, 1=On).'} def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) - loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - loader_args_bc = {**loader_args_a, "on_missing": mismatch_mode} - a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode) b = LoadTensor(key, kwargs['model_b'], **loader_args_bc) c = LoadTensor(key, kwargs['model_c'], **loader_args_bc) diff = Sub(key, b, c) if kwargs['beta'] == 1: diff = Smooth(key, diff) diffm = Multiply(key, kwargs['alpha'], diff) return Add(key, a, diffm) + + class TrainDifference(CalcMode): name = 'Train-Difference' description = 'A + (B - C) * α, using tensor distances.' @@ -221,9 +284,14 @@ class TrainDifference(CalcMode): param_docs = {'alpha': 'Multiplier for the calculated difference.'} def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) - loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - loader_args_bc = {**loader_args_a, "on_missing": mismatch_mode} - a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode) b = LoadTensor(key, kwargs['model_b'], **loader_args_bc) c = LoadTensor(key, kwargs['model_c'], **loader_args_bc) diff = TrainDiff(key, a, b, c) @@ -236,12 +304,22 @@ class InterpDifference(CalcMode): param_docs = {'alpha': 'Curve strength.', 'beta': 'Style (0=Similarity, 1=Difference).', 'gamma': 'Mix (0=Random, 1=Linear).'} def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) - loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - loader_args_b = {**loader_args_a, "on_missing": mismatch_mode} - if key.startswith('cond_stage_model.transformer.text_model.embeddings'): return LoadTensor(key, kwargs['model_a'], **loader_args_a) - a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + # Skip certain keys entirely + if key.startswith('cond_stage_model.transformer.text_model.embeddings'): + return a + + loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode) b = LoadTensor(key, kwargs['model_b'], **loader_args_b) return InterpolateDifference(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['seed'], a, b) + + class Extract(CalcMode): name = 'Extract-Features' description = 'Adds features from B and C to A, based on their similarity.' @@ -249,14 +327,21 @@ class Extract(CalcMode): param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Similarity vs dissimilarity.', 'gamma': 'Similarity bias.', 'delta': 'Multiplier for final features.'} def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) - loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - loader_args_bc = {**loader_args_a, "on_missing": mismatch_mode} - a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode) b = LoadTensor(key, kwargs['model_b'], **loader_args_bc) c = LoadTensor(key, kwargs['model_c'], **loader_args_bc) extracted = ExtractOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'] * 15, a, b, c) multiplied = Multiply(key, kwargs['delta'], extracted) return Add(key, a, multiplied) + + class AddDisimilarity(CalcMode): name = 'Add-Dissimilarities' description = 'Adds dissimilar features between B and C to model A.' @@ -264,14 +349,21 @@ class AddDisimilarity(CalcMode): param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Multiplier for final features.', 'gamma': 'Similarity bias.'} def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) - loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - loader_args_bc = {**loader_args_a, "on_missing": mismatch_mode} - a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode) b = LoadTensor(key, kwargs['model_b'], **loader_args_bc) c = LoadTensor(key, kwargs['model_c'], **loader_args_bc) extracted = Similarities(key, kwargs['alpha'], 1, kwargs['gamma'] * 15, b, c) multiplied = Multiply(key, kwargs['beta'], extracted) return Add(key, a, multiplied) + + class PowerUp(CalcMode): name = 'Power-Up (DARE)' description = 'Adds capabilities of B to A using Drop and Rescale (DARE).' @@ -279,13 +371,20 @@ class PowerUp(CalcMode): param_docs = {'alpha': 'Dropout rate (proportion of weights from B to *drop*).', 'beta': 'Multiplier for the final difference.'} def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) - loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - loader_args_b = {**loader_args_a, "on_missing": mismatch_mode} - a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode) b = LoadTensor(key, kwargs['model_b'], **loader_args_b) deltahat = PowerUpOp(key, kwargs['alpha'], kwargs['seed'], a, b) res = Multiply(key, kwargs['beta'], deltahat) return Add(key, a, res) + + class SVDMode(CalcMode): name = 'SVD LoRA Extraction' description = 'Extracts the difference between B and A into a LoRA file.' @@ -293,11 +392,17 @@ class SVDMode(CalcMode): param_docs = {'alpha': 'Rank for standard layers.', 'beta': 'Rank for 3x3 conv layers.', 'gamma': 'Clamp quantile for weights.'} def create_recipe(self, key, **kwargs): mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP) - loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} - loader_args_b = {**loader_args_a, "on_missing": mismatch_mode} - a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + if '_tensor_a' in kwargs: + a = PreloadedTensor(key, kwargs['_tensor_a']) + else: + loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']} + a = LoadTensor(key, kwargs['model_a'], **loader_args_a) + + loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode) b = LoadTensor(key, kwargs['model_b'], **loader_args_b) return SVD(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], a, b) + TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), SVDMode()] THREE_MODEL_MODES = [AddDifference(), TrainDifference(), Extract(), AddDisimilarity()] \ No newline at end of file