diff --git a/loader.py b/loader.py index 0f983f9..894191e 100644 --- a/loader.py +++ b/loader.py @@ -5,7 +5,7 @@ import gguf from .ops import GGMLTensor from .dequant import is_quantized -IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "ltxv", "hyvid"} +IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "ltxv", "hyvid", "wan"} TXT_ARCH_LIST = {"t5", "t5encoder", "llama"} def get_orig_shape(reader, tensor_name): diff --git a/ops.py b/ops.py index 422ef2e..cc82b08 100644 --- a/ops.py +++ b/ops.py @@ -6,6 +6,15 @@ import comfy.ops import comfy.model_management from .dequant import dequantize_tensor, is_quantized +# to avoid breaking really old pytorch versions +if hasattr(torch, "compiler") and hasattr(torch.compiler, "disable"): + torch_compiler_disable = torch.compiler.disable +else: + def torch_compiler_disable(*args, **kwargs): + def noop(x): + return x + return noop + class GGMLTensor(torch.Tensor): """ Main tensor-like class for storing quantized weights @@ -152,6 +161,7 @@ class GGMLLayer(torch.nn.Module): weight = function(patch_list, weight, key, patch_dtype) return weight + @torch_compiler_disable() def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None): if input is not None: if dtype is None: diff --git a/tools/README.md b/tools/README.md index 5b040ba..2bee67b 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,24 +1,44 @@ -This needs the llama.cpp version of gguf-py to work at the moment, not the pip one as that one does not have the python quantization code yet. +## STEP 1 (Patch files with Unix (LF) line endings + +Solution to fix lines endings of the patch files from Windows (CRLF) to Unix (LF) + +``` +python fix_lines_ending.py +``` +## STEP 2 (Clone llama.cpp version of gguf-py) + +Git clone llama.cpp into the current folder. You may also install gguf-py from the llama.cpp repo directly, though the one specified in `requirements.txt` should also work on recent versions. ``` git clone https://github.com/ggerganov/llama.cpp pip install llama.cpp/gguf-py ``` +## STEP 3 (Convert to FP16 or BF16) To convert your initial source model to FP16 (or BF16), run the following command: ``` python convert.py --src E:\models\unet\flux1-dev.safetensors ``` +## STEP 4 (Patch llama.cpp) - -To quantize the model, first apply the provided patch to the llama.cpp repo you've just cloned. If you get a "corrupt patch" error, you may have to [change the line endings in the patch file](https://github.com/city96/ComfyUI-GGUF/issues/90#issuecomment-2323011648). +- To quantize the model, first apply the provided patch to the llama.cpp repo you've just cloned. ``` cd llama.cpp git checkout tags/b3962 git apply ..\lcpp.patch ``` +- To quantize **SD3** or **AuraFlow** models, you should use the patch `lcpp_sd3.patch` and target `tags/b3962` instead. +- There is a [WIP PR for other model architectures](https://github.com/city96/ComfyUI-GGUF/pull/216) +``` +cd llama.cpp +git checkout tags/b3962 +git apply ..\lcpp_sd3.patch +``` + + +## STEP 5 (Compile llama-quantize binary) Then, compile the llama-quantize binary. This example uses cmake, on linux you can just use make. ``` @@ -30,7 +50,7 @@ cd .. cd .. ``` - +## STEP 6 (Quantization) Now you can use the newly build binary to quantize your model to the desired format: ``` llama.cpp\build\bin\Debug\llama-quantize.exe E:\models\unet\flux1-dev-BF16.gguf E:\models\unet\flux1-dev-Q4_K_S.gguf Q4_K_S diff --git a/tools/fix_lines_ending.py b/tools/fix_lines_ending.py new file mode 100644 index 0000000..346e350 --- /dev/null +++ b/tools/fix_lines_ending.py @@ -0,0 +1,31 @@ +import os + +files = ["lcpp.patch", "lcpp_sd3.patch"] + +def has_unix_line_endings(file_path): + try: + with open(file_path, 'rb') as file: + content = file.read() + return b'\r\n' not in content + except Exception as e: + print(f"Error checking '{file_path}': {e}") + return False + +def convert_to_linux_format(file_path): + try: + with open(file_path, 'rb') as file: + content = file.read().replace(b'\r\n', b'\n') + with open(file_path, 'wb') as file: + file.write(content) + print(f"'{file_path}' converted to Linux line endings (LF).") + except Exception as e: + print(f"Error processing '{file_path}': {e}") + +for file in files: + if os.path.exists(file): + if has_unix_line_endings(file): + print(f"'{file}' already has Unix line endings (LF). No conversion needed.") + else: + convert_to_linux_format(file) + else: + print(f"File '{file}' does not exist.")