diff --git a/nodes.py b/nodes.py index 062aafe..094d8e9 100644 --- a/nodes.py +++ b/nodes.py @@ -10,7 +10,7 @@ import comfy.model_management import comfy.model_patcher import folder_paths -from .ops import GGMLTensor, GGMLOps, move_patch_to_cuda +from .ops import GGMLTensor, GGMLOps, move_patch_to_device from .dequant import dequantize_tensor # Add a custom keys for files ending in .gguf @@ -108,8 +108,7 @@ class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): qtype = getattr(weight, "tensor_type", None) if qtype not in (None, gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16): out_weight = weight.to(device_to) - if self.patch_on_device: - patches = move_patch_to_cuda(patches, self.load_device) + patches = move_patch_to_device(patches, self.load_device if self.patch_on_device else self.offload_device) out_weight.patches.append((calculate_weight, patches, key)) else: inplace_update = self.weight_inplace_update or inplace_update diff --git a/ops.py b/ops.py index bc17c3a..82a3e8e 100644 --- a/ops.py +++ b/ops.py @@ -91,7 +91,7 @@ class GGMLLayer(torch.nn.Module): patch_list = [] device = tensor.device for function, patches, key in getattr(tensor, "patches", []): - patch_list += move_patch_to_cuda(patches, device) + patch_list += move_patch_to_device(patches, device) # dequantize tensor while patches load weight = dequantize_tensor(tensor, dtype, self.dequant_dtype) @@ -149,12 +149,12 @@ class GGMLOps(comfy.ops.manual_cast): input, weight, self.padding_idx, self.max_norm, self.norm_type, self.scale_grad_by_freq, self.sparse ).to(dtype=output_dtype) -def move_patch_to_cuda(item, device): +def move_patch_to_device(item, device): if isinstance(item, torch.Tensor): return item.to(device, non_blocking=True) elif isinstance(item, tuple): - return tuple(move_patch_to_cuda(x, device) for x in item) + return tuple(move_patch_to_device(x, device) for x in item) elif isinstance(item, list): - return [move_patch_to_cuda(x, device) for x in item] + return [move_patch_to_device(x, device) for x in item] else: return item