Merge branch 'main' into convert_refactor_new

loader.py

@@ -5,7 +5,7 @@ import gguf
 from .ops import GGMLTensor
 from .dequant import is_quantized
 
-IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "ltxv", "hyvid"}
+IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "ltxv", "hyvid", "wan"}
 TXT_ARCH_LIST = {"t5", "t5encoder", "llama"}
 
 def get_orig_shape(reader, tensor_name):

ops.py

@@ -6,6 +6,15 @@ import comfy.ops
 import comfy.model_management
 from .dequant import dequantize_tensor, is_quantized
 
+# to avoid breaking really old pytorch versions
+if hasattr(torch, "compiler") and hasattr(torch.compiler, "disable"):
+    torch_compiler_disable = torch.compiler.disable
+else:
+    def torch_compiler_disable(*args, **kwargs):
+        def noop(x):
+            return x
+        return noop
+
 class GGMLTensor(torch.Tensor):
     """
     Main tensor-like class for storing quantized weights
@@ -152,6 +161,7 @@ class GGMLLayer(torch.nn.Module):
                 weight = function(patch_list, weight, key, patch_dtype)
         return weight
 
+    @torch_compiler_disable()
     def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None):
         if input is not None:
             if dtype is None:

tools/README.md

@@ -1,24 +1,44 @@
-This needs the llama.cpp version of gguf-py to work at the moment, not the pip one as that one does not have the python quantization code yet.
+## STEP 1 (Patch files with Unix (LF) line endings
+
+Solution to fix lines endings of the patch files from Windows (CRLF) to Unix (LF)
+
+```
+python fix_lines_ending.py
+```
+## STEP 2 (Clone llama.cpp version of gguf-py)
+
+Git clone llama.cpp into the current folder. You may also install gguf-py from the llama.cpp repo directly, though the one specified in `requirements.txt` should also work on recent versions.
 
 ```
 git clone https://github.com/ggerganov/llama.cpp
 pip install llama.cpp/gguf-py
 ```
 
+## STEP 3 (Convert to FP16 or BF16)
 
 To convert your initial source model to FP16 (or BF16), run the following command:
 ```
 python convert.py --src E:\models\unet\flux1-dev.safetensors
 ```
+## STEP 4 (Patch llama.cpp)
 
-
-To quantize the model, first apply the provided patch to the llama.cpp repo you've just cloned. If you get a "corrupt patch" error, you may have to [change the line endings in the patch file](https://github.com/city96/ComfyUI-GGUF/issues/90#issuecomment-2323011648).
+- To quantize the model, first apply the provided patch to the llama.cpp repo you've just cloned.
 ```
 cd llama.cpp
 git checkout tags/b3962
 git apply ..\lcpp.patch
 ```
 
+- To quantize **SD3** or **AuraFlow** models, you should use the patch  `lcpp_sd3.patch` and target `tags/b3962` instead.
+- There is a [WIP PR for other model architectures](https://github.com/city96/ComfyUI-GGUF/pull/216)
+```
+cd llama.cpp
+git checkout tags/b3962
+git apply ..\lcpp_sd3.patch
+```
+
+
+## STEP 5 (Compile llama-quantize binary)
 
 Then, compile the llama-quantize binary. This example uses cmake, on linux you can just use make.
 ```
@@ -30,7 +50,7 @@ cd ..
 cd ..
 ```
 
-
+## STEP 6 (Quantization)
 Now you can use the newly build binary to quantize your model to the desired format:
 ```
 llama.cpp\build\bin\Debug\llama-quantize.exe E:\models\unet\flux1-dev-BF16.gguf E:\models\unet\flux1-dev-Q4_K_S.gguf Q4_K_S

tools/fix_lines_ending.py

@@ -0,0 +1,31 @@
+import os
+
+files = ["lcpp.patch", "lcpp_sd3.patch"]
+
+def has_unix_line_endings(file_path):
+    try:
+        with open(file_path, 'rb') as file:
+            content = file.read()
+        return b'\r\n' not in content
+    except Exception as e:
+        print(f"Error checking '{file_path}': {e}")
+        return False
+
+def convert_to_linux_format(file_path):
+    try:
+        with open(file_path, 'rb') as file:
+            content = file.read().replace(b'\r\n', b'\n')
+        with open(file_path, 'wb') as file:
+            file.write(content)
+        print(f"'{file_path}' converted to Linux line endings (LF).")
+    except Exception as e:
+        print(f"Error processing '{file_path}': {e}")
+
+for file in files:
+    if os.path.exists(file):
+        if has_unix_line_endings(file):
+            print(f"'{file}' already has Unix line endings (LF). No conversion needed.")
+        else:
+            convert_to_linux_format(file)
+    else:
+        print(f"File '{file}' does not exist.")