From 9538b054cdb6dad525abd2af63293612c66d563a Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Thu, 4 Jun 2026 15:10:48 -0700 Subject: [PATCH] Cast SparseCtrl/SVD control models to unet dtype after loading Setting controlnet_config["dtype"] alone is not enough: comfy's lazy/zero-copy state_dict loading (Windows + aimdo path in disable_weight_init) assigns the on-disk fp32 tensors directly as parameters and ignores the configured dtype. With the disable_weight_init ops (no runtime weight casting), the control model then runs fp32 weights against fp16 activations, raising "mat1 and mat2 must have the same dtype, but got Half and Float" at the first time_embed Linear. Explicitly cast the control model to unet_dtype after load_state_dict (mirroring the motion model load and AnimateDiff-Evolved #573) so the weights always match the activation dtype at runtime. Verified end-to-end with v3_sd15_sparsectrl_rgb.ckpt on an fp16 SD1.5 model. Amp-Thread-ID: https://ampcode.com/threads/T-019e947a-9fd3-76df-a847-5eb68d7f18de Co-authored-by: Amp --- adv_control/control.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/adv_control/control.py b/adv_control/control.py index 9ddb701..5c09140 100644 --- a/adv_control/control.py +++ b/adv_control/control.py @@ -846,6 +846,10 @@ def load_sparsectrl(ckpt_path: str, controlnet_data: dict[str, Tensor]=None, tim missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False) if len(missing) > 0 or len(unexpected) > 0: logger.info(f"SparseCtrl ControlNet: {missing}, {unexpected}") + # cast control_model to the intended dtype; load_state_dict can leave weights + # in their on-disk dtype (e.g. comfy's lazy/zero-copy state dict loading), which + # would otherwise mismatch the activations at runtime + control_model = control_model.to(unet_dtype) global_average_pooling = False filename = os.path.splitext(ckpt_path)[0] @@ -974,6 +978,10 @@ def load_svdcontrolnet(ckpt_path: str, controlnet_data: dict[str, Tensor]=None, missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False) if len(missing) > 0 or len(unexpected) > 0: logger.info(f"SVD-ControlNet: {missing}, {unexpected}") + # cast control_model to the intended dtype; load_state_dict can leave weights + # in their on-disk dtype (e.g. comfy's lazy/zero-copy state dict loading), which + # would otherwise mismatch the activations at runtime + control_model = control_model.to(unet_dtype) global_average_pooling = False filename = os.path.splitext(ckpt_path)[0]