From d4c75a69c9a7777fae7e26ee945435133a878320 Mon Sep 17 00:00:00 2001
From: City <125218114+city96@users.noreply.github.com>
Date: Fri, 16 Aug 2024 01:26:57 +0200
Subject: [PATCH] Change dequant logic to FP16

Benefit of FP32 for dequantization seems less noticable and feels more like a sidegrade due to differences in numerical precision. Was causing issues with memory not being freed.
#16
---
 dequant.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/dequant.py b/dequant.py
index 920c237..8c38d30 100644
--- a/dequant.py
+++ b/dequant.py
@@ -16,13 +16,9 @@ def dequantize_tensor(tensor, dtype=torch.float16):
     elif qtype == gguf.GGMLQuantizationType.F16:
         return data.to(dtype)
     elif qtype in dequantize_functions:
-        # this is the main pytorch op
-        if False and dtype == torch.bfloat16: # disabled, lower quality
-            # dequant directly in native BF16
-            return dequantize(data, qtype, oshape, dtype) 
-        else:
-            # dequant in fp32, convert to target
-            return dequantize(data, qtype, oshape, dtype=torch.float32).to(dtype)
+        # dequantize in fp16 then convert instead of keeping FP32
+        out = dequantize(data, qtype, oshape, dtype=None)
+        return out.to(dtype) if out.dtype != dtype else out # why is .to() not a no-op?
     else:
         # this is incredibly slow
         new = gguf.quants.dequantize(data.cpu().numpy(), qtype)