nodes/city96-ComfyUI-GGUF
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cleanup

Browse Source 
Perf difference from `torch.tensor(tensor.data)` doesn't seem realistically measurable and the warning is annoying.
This commit is contained in:
City
2024-08-30 03:47:52 +02:00
parent ec24e75d89
commit c231239fdf
2 changed files with 10 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dequant.py
+5 -6
View File
@@ -4,20 +4,19 @@ import torch
from tqdm import tqdm
def dequantize_tensor(tensor, dtype=None, dequant_dtype=None):
data = torch.tensor(tensor.data)
qtype = getattr(tensor, "tensor_type", None)
oshape = getattr(tensor, "tensor_shape", data.shape)
oshape = getattr(tensor, "tensor_shape", tensor.shape)
if qtype in (None, gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16):
return data.to(dtype)
return tensor.to(dtype)
elif qtype in dequantize_functions:
dequant_dtype = dtype if dequant_dtype == "target" else dequant_dtype
return dequantize(data, qtype, oshape, dtype=dequant_dtype).to(dtype)
return dequantize(tensor, qtype, oshape, dtype=dequant_dtype).to(dtype)
else:
# this is incredibly slow
tqdm.write(f"Falling back to numpy dequant for qtype: {qtype}")
new = gguf.quants.dequantize(data.cpu().numpy(), qtype)
return torch.from_numpy(new).to(data.device, dtype=dtype)
new = gguf.quants.dequantize(tensor.cpu().numpy(), qtype)
return torch.from_numpy(new).to(tensor.device, dtype=dtype)
def dequantize(data, qtype, oshape, dtype=None):
"""