diff --git a/utils.py b/utils.py
index 7531b5b..104b627 100644
--- a/utils.py
+++ b/utils.py
@@ -73,7 +73,10 @@ def apply_lora(model, device_to, transformer_load_device, params_to_keep=None, d
                         set_module_tensor_to_device(model.model.diffusion_model, key, device=transformer_load_device, dtype=dtype_to_use, value=state_dict[key])
                     except:
                         continue
-                model.patch_weight_to_device("{}.{}".format(name, param), device_to=device_to)
+                if low_mem_load:
+                    model.patch_weight_to_device("{}.{}".format(name, param), device_to=device_to, inplace_update=True)
+                else:
+                    model.patch_weight_to_device("{}.{}".format(name, param), device_to=device_to)
                 if low_mem_load:
                     try:
                         set_module_tensor_to_device(model.model.diffusion_model, key, device=transformer_load_device, dtype=dtype_to_use, value=model.model.diffusion_model.state_dict()[key])
@@ -237,4 +240,4 @@ def fourier_filter(x, scale_low=1.0, scale_high=1.5, freq_cutoff=20):
 def is_image_black(image, threshold=1e-3):
     if image.min() < 0:
         image = (image + 1) / 2
-    return torch.all(image < threshold).item()
\ No newline at end of file
+    return torch.all(image < threshold).item()