Cast SparseCtrl/SVD control models to unet dtype after loading

Setting controlnet_config["dtype"] alone is not enough: comfy's lazy/zero-copy
state_dict loading (Windows + aimdo path in disable_weight_init) assigns the
on-disk fp32 tensors directly as parameters and ignores the configured dtype.
With the disable_weight_init ops (no runtime weight casting), the control model
then runs fp32 weights against fp16 activations, raising "mat1 and mat2 must have
the same dtype, but got Half and Float" at the first time_embed Linear.

Explicitly cast the control model to unet_dtype after load_state_dict (mirroring
the motion model load and AnimateDiff-Evolved #573) so the weights always match
the activation dtype at runtime.

Verified end-to-end with v3_sd15_sparsectrl_rgb.ckpt on an fp16 SD1.5 model.

Amp-Thread-ID: https://ampcode.com/threads/T-019e947a-9fd3-76df-a847-5eb68d7f18de
Co-authored-by: Amp <amp@ampcode.com>

adv_control/control.py

@@ -811,6 +811,7 @@ def load_sparsectrl(ckpt_path: str, controlnet_data: dict[str, Tensor]=None, tim
         controlnet_config["operations"] = manual_cast_clean_groupnorm
     else:
         controlnet_config["operations"] = disable_weight_init_clean_groupnorm
+    controlnet_config["dtype"] = unet_dtype
     controlnet_config.pop("out_channels")
     # get proper hint channels
     if use_simplified_conditioning_embedding:
@@ -845,6 +846,10 @@ def load_sparsectrl(ckpt_path: str, controlnet_data: dict[str, Tensor]=None, tim
         missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False)
     if len(missing) > 0 or len(unexpected) > 0:
         logger.info(f"SparseCtrl ControlNet: {missing}, {unexpected}")
+    # cast control_model to the intended dtype; load_state_dict can leave weights
+    # in their on-disk dtype (e.g. comfy's lazy/zero-copy state dict loading), which
+    # would otherwise mismatch the activations at runtime
+    control_model = control_model.to(unet_dtype)
 
     global_average_pooling = False
     filename = os.path.splitext(ckpt_path)[0]
@@ -944,6 +949,7 @@ def load_svdcontrolnet(ckpt_path: str, controlnet_data: dict[str, Tensor]=None,
     manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device)
     if manual_cast_dtype is not None:
         controlnet_config["operations"] = comfy.ops.manual_cast
+    controlnet_config["dtype"] = unet_dtype
     controlnet_config.pop("out_channels")
     controlnet_config["hint_channels"] = controlnet_data["{}input_hint_block.0.weight".format(prefix)].shape[1]
     control_model = SVDControlNet(**controlnet_config)
@@ -972,6 +978,10 @@ def load_svdcontrolnet(ckpt_path: str, controlnet_data: dict[str, Tensor]=None,
         missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False)
     if len(missing) > 0 or len(unexpected) > 0:
         logger.info(f"SVD-ControlNet: {missing}, {unexpected}")
+    # cast control_model to the intended dtype; load_state_dict can leave weights
+    # in their on-disk dtype (e.g. comfy's lazy/zero-copy state dict loading), which
+    # would otherwise mismatch the activations at runtime
+    control_model = control_model.to(unet_dtype)
 
     global_average_pooling = False
     filename = os.path.splitext(ckpt_path)[0]

adv_control/utils.py

@@ -338,8 +338,10 @@ class AbstractPreprocWrapper:
 class disable_weight_init_clean_groupnorm(comfy.ops.disable_weight_init):
     class GroupNorm(comfy.ops.disable_weight_init.GroupNorm):
         def forward_comfy_cast_weights(self, input):
-            weight, bias = comfy.ops.cast_bias_weight(self, input)
-            return torch.nn.functional.group_norm(input, self.num_groups, weight, bias, self.eps)
+            weight, bias, offload_stream = comfy.ops.cast_bias_weight(self, input, offloadable=True)
+            x = torch.nn.functional.group_norm(input, self.num_groups, weight, bias, self.eps)
+            comfy.ops.uncast_bias_weight(self, weight, bias, offload_stream)
+            return x
 
         def forward(self, input):
             if self.comfy_cast_weights:

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-advanced-controlnet"
 description = "Nodes for scheduling ControlNet strength across timesteps and batched latents, as well as applying custom weights and attention masks."
-version = "1.5.6"
+version = "1.5.7"
 license = { file = "LICENSE" }
 dependencies = []
 
@@ -13,3 +13,4 @@ Repository = "https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet"
 PublisherId = "kosinkadink"
 DisplayName = "ComfyUI-Advanced-ControlNet"
 Icon = ""
+requires-comfyui = ">=0.3.68"