From 3d9869240bd4ff786030ad85b09441045bb3fda9 Mon Sep 17 00:00:00 2001
From: City <125218114+city96@users.noreply.github.com>
Date: Wed, 28 Aug 2024 17:12:24 +0200
Subject: [PATCH 1/3] Always patch weight

Should help with #33
---
 nodes.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/nodes.py b/nodes.py
index 0e61662..1deb9ec 100644
--- a/nodes.py
+++ b/nodes.py
@@ -140,6 +140,10 @@ class GGUFModelPatcher(comfy.model_patcher.ModelPatcher):
         else:
             comfy.utils.set_attr_param(self.model, key, out_weight)
 
+    def load(self, *args, force_patch_weights=False, **kwargs):
+        # always call `patch_weight_to_device` even for lowvram
+        return super().load(*args, force_patch_weights=True, **kwargs)
+
     def clone(self, *args, **kwargs):
         n = GGUFModelPatcher(self.model, self.load_device, self.offload_device, self.size, weight_inplace_update=self.weight_inplace_update)
         n.patches = {}

From abe5c6a6c90a3f8a53a097614bf749de506c6df7 Mon Sep 17 00:00:00 2001
From: City <125218114+city96@users.noreply.github.com>
Date: Wed, 28 Aug 2024 20:23:08 +0200
Subject: [PATCH 2/3] Fix weights not being copied for lowvram patch

This can probably be improved since the weights themselves never need to be copied as they cannot be changed anyway.
---
 nodes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nodes.py b/nodes.py
index 1deb9ec..70823f4 100644
--- a/nodes.py
+++ b/nodes.py
@@ -120,7 +120,7 @@ class GGUFModelPatcher(comfy.model_patcher.ModelPatcher):
             if device_to is not None:
                 out_weight = weight.to(device_to, copy=True)
             else:
-                out_weight = weight.clone()
+                out_weight = weight.to(weight.device, copy=True) # make sure patches are copied
 
             if self.patch_on_device:
                 patches = move_patch_to_cuda(patches, self.load_device)

From 7f3ced6c3ed2b23abf4105e0210da03ad6015b87 Mon Sep 17 00:00:00 2001
From: City <125218114+city96@users.noreply.github.com>
Date: Sun, 1 Sep 2024 00:56:30 +0200
Subject: [PATCH 3/3] Consider FP8 as source

#91
---
 tools/convert.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/convert.py b/tools/convert.py
index a49ef1c..63bded9 100644
--- a/tools/convert.py
+++ b/tools/convert.py
@@ -109,6 +109,9 @@ def handle_tensors(args, writer, state_dict):
 
         if data.dtype == torch.bfloat16:
             data = data.to(torch.float32).numpy()
+        # this is so we don't break torch 2.0.X
+        elif data.dtype in [getattr(torch, "float8_e4m3fn", "_invalid"), getattr(torch, "float8_e5m2", "_invalid")]:
+            data = data.to(torch.float16).numpy()
         else:
             data = data.numpy()