fix handling of mismatched layers

nodes/merger.py

@@ -3,7 +3,7 @@ import torch
 import folder_paths
 import comfy.utils
 from safetensors.torch import save_file
-from .merger_ops import TWO_MODEL_MODES, THREE_MODEL_MODES, MissingTensorBehavior
+from .merger_ops import TWO_MODEL_MODES, THREE_MODEL_MODES, MissingTensorBehavior, MissingTensorError
 from .merger_utils import MemoryEfficientSafeOpen
 
 def load_documentation_from_file(filename):
@@ -66,25 +66,28 @@ class MergerLogic:
                 if not path: raise FileNotFoundError(f"Model '{name}' not found.")
                 handlers[name] = MemoryEfficientSafeOpen(path)
         
-        all_keys = handlers[primary_model_name].keys()
-        metadata = handlers[primary_model_name].header.get("__metadata__", {})
+        primary_handler = handlers[primary_model_name]
+        all_keys = primary_handler.keys()
+        metadata = primary_handler.header.get("__metadata__", {})
         
         # Convert mismatch_mode string to enum
         mismatch_mode_str = recipe_params.get('mismatch_mode', 'skip')
         mismatch_mode = MissingTensorBehavior(mismatch_mode_str)
         recipe_params['mismatch_mode'] = mismatch_mode
         
-        # Log mismatched keys for debugging
+        # Pre-compute key differences for logging
         primary_keys = set(all_keys)
+        mismatch_report = {}
         for name, handler in handlers.items():
             if name != primary_model_name:
                 secondary_keys = set(handler.keys())
-                missing_in_secondary = primary_keys - secondary_keys
-                extra_in_secondary = secondary_keys - primary_keys
-                if missing_in_secondary:
-                    print(f"[Merger] {name} is missing {len(missing_in_secondary)} keys present in Model A")
-                if extra_in_secondary:
-                    print(f"[Merger] {name} has {len(extra_in_secondary)} extra keys not in Model A (ignored)")
+                missing = primary_keys - secondary_keys
+                extra = secondary_keys - primary_keys
+                mismatch_report[name] = {"missing": missing, "extra": extra}
+                if missing:
+                    print(f"[Merger] {name} is missing {len(missing)} keys present in Model A")
+                if extra:
+                    print(f"[Merger] {name} has {len(extra)} extra keys not in Model A (ignored)")
         
         merged_state_dict = {}
         pbar = comfy.utils.ProgressBar(len(all_keys))
@@ -92,18 +95,34 @@ class MergerLogic:
         save_torch_dtype = {"fp32": torch.float32, "fp16": torch.float16, "bf16": torch.bfloat16}.get(save_dtype)
         recipe_params.update({"handlers": handlers})
         process_device = recipe_params.get('device', 'cpu')
+        process_dtype = recipe_params.get('dtype', torch.float32)
         
         skipped_keys = 0
+        error_keys = []
+        
         for key in all_keys:
-            recipe = calc_mode_class.create_recipe(key=key, **recipe_params)
-            result = recipe.merge()
+            # Pre-load Model A's tensor - this is always required and serves as reference
+            tensor_a = primary_handler.get_tensor(key).to(device=process_device, dtype=process_dtype)
+            
+            # Pass tensor_a metadata to recipes for zeros mode and fallback
+            recipe_params['_tensor_a'] = tensor_a  # For operations that need reference
+            recipe_params['_tensor_a_shape'] = tensor_a.shape
+            recipe_params['_tensor_a_dtype'] = tensor_a.dtype
+            
+            try:
+                recipe = calc_mode_class.create_recipe(key=key, **recipe_params)
+                result = recipe.merge()
+            except MissingTensorError as e:
+                if mismatch_mode == MissingTensorBehavior.ERROR:
+                    # Re-raise to fail the entire merge
+                    raise ValueError(f"Layer mismatch error (mismatch_mode='error'): {e}")
+                # This shouldn't happen in skip/zeros mode, but handle gracefully
+                result = None
+                error_keys.append(key)
             
             # Handle None result (mismatch occurred with skip mode)
             if result is None:
-                # Fall back to Model A's value
-                result = handlers[primary_model_name].get_tensor(key).to(
-                    device=process_device, dtype=torch.float32
-                )
+                result = tensor_a  # Use pre-loaded Model A tensor
                 skipped_keys += 1
             
             if isinstance(result, dict):
@@ -111,10 +130,15 @@ class MergerLogic:
                     merged_state_dict[r_key] = r_tensor.to(save_torch_dtype).cpu().clone()
             else:
                 merged_state_dict[key] = result.to(save_torch_dtype).cpu().clone()
+            
+            # Clean up tensor_a reference to allow GC
+            del recipe_params['_tensor_a']
             pbar.update(1)
         
         if skipped_keys > 0:
             print(f"[Merger] Used Model A's values for {skipped_keys} keys due to mismatches")
+        if error_keys:
+            print(f"[Merger] Unexpected errors on {len(error_keys)} keys: {error_keys[:5]}{'...' if len(error_keys) > 5 else ''}")
         
         for handler in handlers.values(): handler.__exit__(None, None, None)
         output_folder = "loras" if calc_mode == "SVD LoRA Extraction" else model_type

nodes/merger_ops.py

@@ -110,6 +110,8 @@ class SVD(Operation):
         super().__init__(key, *sources)
         self.alpha, self.beta, self.gamma = int(alpha), int(beta), gamma
     def oper(self, a, b) -> torch.Tensor:
+        if a is None or b is None:
+            return None
         diff, weights, conv2d = a - b, {}, len(a.size()) == 4
         kernel_size = None if not conv2d else a.size()[2:4]
         conv2d_3x3 = conv2d and kernel_size != (1, 1)
@@ -130,11 +132,15 @@ class SVD(Operation):
 
 class Smooth(Operation):
     def oper(self, a) -> torch.Tensor:
+        if a is None:
+            return None
         filtered = scipy.ndimage.median_filter(a.detach().cpu().to(torch.float32).numpy(), size=3)
         filtered = scipy.ndimage.gaussian_filter(filtered, sigma=1)
         return torch.tensor(filtered, dtype=a.dtype, device=a.device)
 class TrainDiff(Operation):
     def oper(self, a, b, c) -> torch.Tensor:
+        if a is None or b is None or c is None:
+            return None
         if torch.allclose(b.float(), c.float()): return torch.zeros_like(a)
         diff_bc = b.float() - c.float()
         dist_bc, dist_ab = torch.abs(diff_bc), torch.abs(b.float() - a.float())
@@ -145,27 +151,40 @@ class ExtractOp(Operation):
         super().__init__(key, *args)
         self.alpha, self.beta, self.gamma = alpha, beta, gamma
     def oper(self, base: torch.Tensor, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
+        if a is None or b is None:
+            return None
         base_f = base.float() if base is not None else 0
         a_f, b_f = a.float() - base_f, b.float() - base_f
         c = torch.cosine_similarity(a_f.flatten(), b_f.flatten(), 0).clamp(-1, 1)
         d = ((c + 1) / 2)**self.gamma
         return (torch.lerp(a_f, b_f, self.alpha) * torch.lerp(d, 1 - d, self.beta)).to(a.dtype)
+
+
 class Similarities(ExtractOp):
-    def oper(self, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: return super().oper(None, a, b)
+    def oper(self, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
+        if a is None or b is None:
+            return None
+        return super().oper(None, a, b)
 class PowerUpOp(Operation):
     def __init__(self, key, alpha, seed, *sources):
         super().__init__(key, *sources)
         self.alpha, self.seed = alpha, seed
     def oper(self, a, b):
+        if a is None or b is None:
+            return None
         delta = b - a
         rng = torch.Generator(device=delta.device).manual_seed(self.seed)
         m = torch.bernoulli(torch.full_like(delta, 1 - self.alpha), generator=rng)
         return (m * delta) / (1 - self.alpha) if (1 - self.alpha) > 0 else torch.zeros_like(delta)
+
+
 class InterpolateDifference(Operation):
     def __init__(self, key, alpha, beta, gamma, seed, *sources):
         super().__init__(key, *sources)
         self.alpha, self.beta, self.gamma, self.seed = alpha, beta, gamma, seed
     def oper(self, a, b):
+        if a is None or b is None:
+            return None
         alpha, delta = max(self.alpha, 0.001), torch.abs(a - b)
         max_delta = torch.max(delta)
         if max_delta == 0: return a
@@ -181,6 +200,34 @@ class InterpolateDifference(Operation):
 class CalcMode:
     name = 'calcmode'; description = 'description'; models_used = []; param_docs = {}
     def create_recipe(self, key, **kwargs) -> Operation: raise NotImplementedError
+    
+    @staticmethod
+    def _get_secondary_loader_args(kwargs, mismatch_mode):
+        """Helper to build loader args for secondary models (B, C) with fallback support."""
+        base_args = {
+            "handlers": kwargs['handlers'],
+            "device": kwargs['device'],
+            "dtype": kwargs['dtype'],
+            "on_missing": mismatch_mode,
+        }
+        # Add fallback info for zeros mode
+        if '_tensor_a_shape' in kwargs:
+            base_args['fallback_shape'] = kwargs['_tensor_a_shape']
+            base_args['fallback_dtype'] = kwargs.get('_tensor_a_dtype', kwargs['dtype'])
+        return base_args
+
+
+class PreloadedTensor(Operation):
+    """Returns a pre-loaded tensor instead of loading from file.
+    
+    Used when Model A's tensor is pre-loaded in the merge loop.
+    """
+    def __init__(self, key, tensor):
+        super().__init__(key)
+        self.tensor = tensor
+    
+    def merge(self) -> torch.Tensor:
+        return self.tensor
 
 class WeightSum(CalcMode):
     name = 'Weight-Sum'
@@ -189,15 +236,24 @@ class WeightSum(CalcMode):
     param_docs = {'alpha': 'Interpolation weight. 0.0 is 100% Model A, 1.0 is 100% Model B.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-        loader_args_b = {**loader_args_a, "on_missing": mismatch_mode}
-        a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        # Use pre-loaded tensor_a if available
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
+        
         if kwargs['alpha'] >= 1: return b
         if kwargs['alpha'] <= 0: return a
         c = Multiply(key, 1 - kwargs['alpha'], a)
         d = Multiply(key, kwargs['alpha'], b)
         return Add(key, c, d)
+
+
 class AddDifference(CalcMode):
     name = 'Add-Difference'
     description = 'A + (B - C) * α. Applies the difference between B and C to A.'
@@ -205,15 +261,22 @@ class AddDifference(CalcMode):
     param_docs = {'alpha': 'Multiplier for the (B - C) difference.', 'beta': 'Smoothing (0=Off, 1=On).'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-        loader_args_bc = {**loader_args_a, "on_missing": mismatch_mode}
-        a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         diff = Sub(key, b, c)
         if kwargs['beta'] == 1: diff = Smooth(key, diff)
         diffm = Multiply(key, kwargs['alpha'], diff)
         return Add(key, a, diffm)
+
+
 class TrainDifference(CalcMode):
     name = 'Train-Difference'
     description = 'A + (B - C) * α, using tensor distances.'
@@ -221,9 +284,14 @@ class TrainDifference(CalcMode):
     param_docs = {'alpha': 'Multiplier for the calculated difference.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-        loader_args_bc = {**loader_args_a, "on_missing": mismatch_mode}
-        a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         diff = TrainDiff(key, a, b, c)
@@ -236,12 +304,22 @@ class InterpDifference(CalcMode):
     param_docs = {'alpha': 'Curve strength.', 'beta': 'Style (0=Similarity, 1=Difference).', 'gamma': 'Mix (0=Random, 1=Linear).'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-        loader_args_b = {**loader_args_a, "on_missing": mismatch_mode}
-        if key.startswith('cond_stage_model.transformer.text_model.embeddings'): return LoadTensor(key, kwargs['model_a'], **loader_args_a)
-        a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        # Skip certain keys entirely
+        if key.startswith('cond_stage_model.transformer.text_model.embeddings'):
+            return a
+        
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
         return InterpolateDifference(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], kwargs['seed'], a, b)
+
+
 class Extract(CalcMode):
     name = 'Extract-Features'
     description = 'Adds features from B and C to A, based on their similarity.'
@@ -249,14 +327,21 @@ class Extract(CalcMode):
     param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Similarity vs dissimilarity.', 'gamma': 'Similarity bias.', 'delta': 'Multiplier for final features.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-        loader_args_bc = {**loader_args_a, "on_missing": mismatch_mode}
-        a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         extracted = ExtractOp(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'] * 15, a, b, c)
         multiplied = Multiply(key, kwargs['delta'], extracted)
         return Add(key, a, multiplied)
+
+
 class AddDisimilarity(CalcMode):
     name = 'Add-Dissimilarities'
     description = 'Adds dissimilar features between B and C to model A.'
@@ -264,14 +349,21 @@ class AddDisimilarity(CalcMode):
     param_docs = {'alpha': 'Weighting B vs C.', 'beta': 'Multiplier for final features.', 'gamma': 'Similarity bias.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-        loader_args_bc = {**loader_args_a, "on_missing": mismatch_mode}
-        a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        loader_args_bc = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_bc)
         c = LoadTensor(key, kwargs['model_c'], **loader_args_bc)
         extracted = Similarities(key, kwargs['alpha'], 1, kwargs['gamma'] * 15, b, c)
         multiplied = Multiply(key, kwargs['beta'], extracted)
         return Add(key, a, multiplied)
+
+
 class PowerUp(CalcMode):
     name = 'Power-Up (DARE)'
     description = 'Adds capabilities of B to A using Drop and Rescale (DARE).'
@@ -279,13 +371,20 @@ class PowerUp(CalcMode):
     param_docs = {'alpha': 'Dropout rate (proportion of weights from B to *drop*).', 'beta': 'Multiplier for the final difference.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-        loader_args_b = {**loader_args_a, "on_missing": mismatch_mode}
-        a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
         deltahat = PowerUpOp(key, kwargs['alpha'], kwargs['seed'], a, b)
         res = Multiply(key, kwargs['beta'], deltahat)
         return Add(key, a, res)
+
+
 class SVDMode(CalcMode):
     name = 'SVD LoRA Extraction'
     description = 'Extracts the difference between B and A into a LoRA file.'
@@ -293,11 +392,17 @@ class SVDMode(CalcMode):
     param_docs = {'alpha': 'Rank for standard layers.', 'beta': 'Rank for 3x3 conv layers.', 'gamma': 'Clamp quantile for weights.'}
     def create_recipe(self, key, **kwargs):
         mismatch_mode = kwargs.get('mismatch_mode', MissingTensorBehavior.SKIP)
-        loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
-        loader_args_b = {**loader_args_a, "on_missing": mismatch_mode}
-        a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        if '_tensor_a' in kwargs:
+            a = PreloadedTensor(key, kwargs['_tensor_a'])
+        else:
+            loader_args_a = {"handlers": kwargs['handlers'], "device": kwargs['device'], "dtype": kwargs['dtype']}
+            a = LoadTensor(key, kwargs['model_a'], **loader_args_a)
+        
+        loader_args_b = self._get_secondary_loader_args(kwargs, mismatch_mode)
         b = LoadTensor(key, kwargs['model_b'], **loader_args_b)
         return SVD(key, kwargs['alpha'], kwargs['beta'], kwargs['gamma'], a, b)
 
+
 TWO_MODEL_MODES = [WeightSum(), InterpDifference(), PowerUp(), SVDMode()]
 THREE_MODEL_MODES = [AddDifference(), TrainDifference(), Extract(), AddDisimilarity()]