101 lines
3.3 KiB
Python
101 lines
3.3 KiB
Python
# (c) City96 || Apache-2.0 (apache.org/licenses/LICENSE-2.0)
|
|
import torch
|
|
import gguf
|
|
import types
|
|
import logging
|
|
import numpy as np
|
|
|
|
import comfy.sd
|
|
import comfy.utils
|
|
import comfy.model_management
|
|
import folder_paths
|
|
|
|
from .ops import GGMLTensor, GGMLOps
|
|
|
|
# TODO: This causes gguf files to show up in the main unet loader
|
|
folder_paths.folder_names_and_paths["unet"][1].add(".gguf")
|
|
|
|
def gguf_sd_loader(path):
|
|
"""
|
|
Read state dict as fake tensors
|
|
"""
|
|
reader = gguf.GGUFReader(path)
|
|
sd = {}
|
|
dt = {}
|
|
for tensor in reader.tensors:
|
|
sd[str(tensor.name)] = GGMLTensor(
|
|
torch.from_numpy(tensor.data), # mmap
|
|
tensor_type = tensor.tensor_type,
|
|
tensor_shape = torch.Size(
|
|
np.flip(list(tensor.shape))
|
|
)
|
|
)
|
|
dt[str(tensor.tensor_type)] = dt.get(str(tensor.tensor_type), 0) + 1
|
|
|
|
# sanity check debug print
|
|
print("\nggml_sd_loader:")
|
|
for k,v in dt.items():
|
|
print(f" {k:30}{v:3}")
|
|
print("\n")
|
|
return sd
|
|
|
|
class UnetLoaderGGUF:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
unet_names = [x for x in folder_paths.get_filename_list("unet") if x.endswith(".gguf")]
|
|
return {
|
|
"required": {
|
|
"unet_name": (unet_names,),
|
|
}
|
|
}
|
|
|
|
RETURN_TYPES = ("MODEL",)
|
|
FUNCTION = "load_unet"
|
|
CATEGORY = "bootleg"
|
|
TITLE = "Unet Loader (GGUF)"
|
|
|
|
def load_unet(self, unet_name):
|
|
unet_path = folder_paths.get_full_path("unet", unet_name)
|
|
sd = gguf_sd_loader(unet_path)
|
|
model = comfy.sd.load_diffusion_model_state_dict(
|
|
sd, model_options={"custom_operations": GGMLOps}
|
|
)
|
|
if model is None:
|
|
logging.error("ERROR UNSUPPORTED UNET {}".format(unet_path))
|
|
raise RuntimeError("ERROR: Could not detect model type of: {}".format(unet_path))
|
|
self.patch_calculate_weight(model)
|
|
return (model,)
|
|
|
|
def patch_calculate_weight(self, model):
|
|
def calculate_weight(self, patches, weight, key):
|
|
if isinstance(weight, GGMLTensor):
|
|
qtype = weight.tensor_type
|
|
# TODO: don't even store these in a custom format
|
|
if qtype in [gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16]:
|
|
return self.calculate_weight_orig(patches, weight, key)
|
|
else:
|
|
weight.patches.append((self.calculate_weight_orig, patches, key))
|
|
return weight
|
|
else:
|
|
return self.calculate_weight_orig(patches, weight, key)
|
|
|
|
# make sure custom patch logic is used as to not mangle quants
|
|
model.calculate_weight_orig = model.calculate_weight
|
|
model.calculate_weight = types.MethodType(calculate_weight, model)
|
|
|
|
# make sure this logic is preserved for cloned patchers
|
|
def clone(self, *args, **kwargs):
|
|
new = self.clone_orig(*args, **kwargs)
|
|
new.calculate_weight_orig = new.calculate_weight
|
|
new.calculate_weight = types.MethodType(calculate_weight, model)
|
|
return new
|
|
|
|
model.clone_orig = model.clone
|
|
model.clone = types.MethodType(clone, model)
|
|
|
|
return model
|
|
|
|
NODE_CLASS_MAPPINGS = {
|
|
"UnetLoaderGGUF": UnetLoaderGGUF,
|
|
}
|