From 0ca2aeb797d20b45e75ba25aa23d37a36954b9e6 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Tue, 19 Aug 2025 17:50:55 +0200 Subject: [PATCH 1/7] Qwen2-VL vision support #317 --- loader.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/loader.py b/loader.py index 56bbd57..af894d9 100644 --- a/loader.py +++ b/loader.py @@ -3,12 +3,14 @@ import warnings import logging import torch import gguf +import os from .ops import GGMLTensor from .dequant import is_quantized, dequantize_tensor IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image"} TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl"} +VIS_TYPE_LIST = {"clip-vision"} def get_orig_shape(reader, tensor_name): field_key = f"comfy.gguf.orig_shape.{tensor_name}" @@ -70,6 +72,7 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", return_arch=Fal # detect and verify architecture compat = None arch_str = get_field(reader, "general.architecture", str) + type_str = get_field(reader, "general.type", str) if arch_str in [None, "pig"]: if is_text_model: raise ValueError(f"This text model is incompatible with llama.cpp!\nConsider using the safetensors version\n({path})") @@ -81,7 +84,8 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", return_arch=Fal except Exception as e: raise ValueError(f"This model is not currently supported - ({e})") elif arch_str not in TXT_ARCH_LIST and is_text_model: - raise ValueError(f"Unexpected text model architecture type in GGUF file: {arch_str!r}") + if type_str not in VIS_TYPE_LIST: + raise ValueError(f"Unexpected text model architecture type in GGUF file: {arch_str!r}") elif arch_str not in IMG_ARCH_LIST and not is_text_model: raise ValueError(f"Unexpected architecture type in GGUF file: {arch_str!r}") @@ -165,6 +169,19 @@ LLAMA_SD_MAP = { "output.weight": "lm_head.weight", } +CLIP_VISION_SD_MAP = { + "mm.": "visual.merger.mlp.", + "v.post_ln.": "visual.merger.ln_q.", + "v.patch_embd": "visual.patch_embed.proj", + "v.blk.": "visual.blocks.", + "ffn_up": "mlp.up_proj", + "ffn_down": "mlp.down_proj", + "ffn_gate": "mlp.gate_proj", + "attn_out.": "attn.proj.", + "ln1.": "norm1.", + "ln2.": "norm2.", +} + def sd_map_replace(raw_sd, key_map): sd = {} for k,v in raw_sd.items(): @@ -185,6 +202,76 @@ def llama_permute(raw_sd, n_head, n_head_kv): sd[k] = v return sd +def gguf_mmproj_loader(path): + # Reverse version of Qwen2VLVisionModel.modify_tensors + logging.info("Attenpting to find mmproj file for text encoder...") + + # get name to match w/o quant suffix + tenc_fname = os.path.basename(path) + tenc = os.path.splitext(tenc_fname)[0].lower() + for q in [x.name for x in gguf.GGMLQuantizationType]: + if q.lower() in tenc: + tenc = tenc.rsplit(q.lower(), 1)[0] + break + tenc = tenc[:-1] # dash/underscore/etc + + # try and find matching mmproj + target = [] + root = os.path.dirname(path) + for fname in os.listdir(root): + name, ext = os.path.splitext(fname) + if ext.lower() != ".gguf": + continue + if "mmproj" not in name.lower(): + continue + if tenc in name.lower(): + target.append(fname) + + if len(target) == 0: + logging.error(f"Error: Can't find mmproj file for '{tenc_fname}'! Qwen-Image-Edit will be broken!") + return {} + if len(target) > 1: + logging.error(f"Ambiguous mmproj for text encoder '{tenc_fname}', will use first match.") + + logging.info(f"Using mmproj '{target[0]}' for text encoder '{tenc_fname}'.") + target = os.path.join(root, target[0]) + vsd = gguf_sd_loader(target, is_text_model=True) + + # concat 4D to 5D + if "v.patch_embd.weight.1" in vsd: + w1 = dequantize_tensor(vsd.pop("v.patch_embd.weight"), dtype=torch.float32) + w2 = dequantize_tensor(vsd.pop("v.patch_embd.weight.1"), dtype=torch.float32) + vsd["v.patch_embd.weight"] = torch.stack([w1, w2], dim=2) + + # run main replacement + vsd = sd_map_replace(vsd, CLIP_VISION_SD_MAP) + + # handle split Q/K/V + if "visual.blocks.0.attn_q.weight" in vsd: + attns = {} + # filter out attentions + group + for k,v in vsd.items(): + if any(x in k for x in ["attn_q", "attn_k", "attn_v"]): + k_attn, k_name = k.rsplit(".attn_", 1) + k_attn += ".attn.qkv." + k_name.split(".")[-1] + if k_attn not in attns: + attns[k_attn] = {} + attns[k_attn][k_name] = dequantize_tensor( + v, dtype=(torch.bfloat16 if is_quantized(v) else torch.float16) + ) + + # recombine + for k,v in attns.items(): + suffix = k.split(".")[-1] + vsd[k] = torch.cat([ + v[f"q.{suffix}"], + v[f"k.{suffix}"], + v[f"v.{suffix}"], + ], dim=0) + del attns + + return vsd + def gguf_tokenizer_loader(path, temb_shape): # convert gguf tokenizer to spiece logging.info("Attempting to recreate sentencepiece tokenizer from GGUF file metadata...") @@ -254,6 +341,9 @@ def gguf_clip_loader(path): sd = sd_map_replace(sd, LLAMA_SD_MAP) if arch == "llama": sd = llama_permute(sd, 32, 8) # L3 + if arch == "qwen2vl": + vsd = gguf_mmproj_loader(path) + sd.update(vsd) else: pass return sd From d247022e3fa66851c5084cc251b076aab816423d Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Wed, 20 Aug 2025 00:51:28 +0200 Subject: [PATCH 2/7] Better mmproj detection logic #317 --- loader.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/loader.py b/loader.py index af894d9..fd35e13 100644 --- a/loader.py +++ b/loader.py @@ -3,6 +3,7 @@ import warnings import logging import torch import gguf +import re import os from .ops import GGMLTensor @@ -202,6 +203,13 @@ def llama_permute(raw_sd, n_head, n_head_kv): sd[k] = v return sd +def strip_quant_suffix(name): + pattern = r"[-_]?(?:ud-)?i?q[0-9]_[a-z0-9_\-]{1,8}$" + match = re.search(pattern, name, re.IGNORECASE) + if match: + name = name[:match.start()] + return name + def gguf_mmproj_loader(path): # Reverse version of Qwen2VLVisionModel.modify_tensors logging.info("Attenpting to find mmproj file for text encoder...") @@ -209,11 +217,7 @@ def gguf_mmproj_loader(path): # get name to match w/o quant suffix tenc_fname = os.path.basename(path) tenc = os.path.splitext(tenc_fname)[0].lower() - for q in [x.name for x in gguf.GGMLQuantizationType]: - if q.lower() in tenc: - tenc = tenc.rsplit(q.lower(), 1)[0] - break - tenc = tenc[:-1] # dash/underscore/etc + tenc = strip_quant_suffix(tenc) # try and find matching mmproj target = [] @@ -228,7 +232,7 @@ def gguf_mmproj_loader(path): target.append(fname) if len(target) == 0: - logging.error(f"Error: Can't find mmproj file for '{tenc_fname}'! Qwen-Image-Edit will be broken!") + logging.error(f"Error: Can't find mmproj file for '{tenc_fname}' (matching:'{tenc}')! Qwen-Image-Edit will be broken!") return {} if len(target) > 1: logging.error(f"Ambiguous mmproj for text encoder '{tenc_fname}', will use first match.") From 6b778afdc938427116cb72e683d28e42baf67082 Mon Sep 17 00:00:00 2001 From: blepping <157360029+blepping@users.noreply.github.com> Date: Mon, 8 Sep 2025 23:07:45 -0600 Subject: [PATCH 3/7] Fix attempt to create a tensor with dtype "target" (#330) * Fix attempt to create a tensor with dtype "target" * Check if dequant_dtype is None before string comparison with target --- ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops.py b/ops.py index 41d42f5..ff9b960 100644 --- a/ops.py +++ b/ops.py @@ -153,7 +153,7 @@ class GGMLLayer(torch.nn.Module): # Take into account space required for dequantizing the largest tensor if self.largest_layer: shape = getattr(self.weight, "tensor_shape", self.weight.shape) - dtype = self.dequant_dtype or torch.float16 + dtype = (self.dequant_dtype and self.dequant_dtype != "target") or torch.float16 temp = torch.empty(*shape, device=torch.device("meta"), dtype=dtype) destination[prefix + "temp.weight"] = temp From be2a08330d7ec232d684e50ab938870d7529471e Mon Sep 17 00:00:00 2001 From: blepping <157360029+blepping@users.noreply.github.com> Date: Sun, 14 Sep 2025 15:10:50 -0600 Subject: [PATCH 4/7] Fix dequant_dtype handling (#334) * Fix dequant_dtype handling * Simplify dequant dtype selection expression --- ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops.py b/ops.py index ff9b960..88a352e 100644 --- a/ops.py +++ b/ops.py @@ -153,7 +153,7 @@ class GGMLLayer(torch.nn.Module): # Take into account space required for dequantizing the largest tensor if self.largest_layer: shape = getattr(self.weight, "tensor_shape", self.weight.shape) - dtype = (self.dequant_dtype and self.dequant_dtype != "target") or torch.float16 + dtype = self.dequant_dtype if self.dequant_dtype and self.dequant_dtype != "target" else torch.float16 temp = torch.empty(*shape, device=torch.device("meta"), dtype=dtype) destination[prefix + "temp.weight"] = temp From 100c06cda840b0406f96fd7c3c57076d443f82b6 Mon Sep 17 00:00:00 2001 From: rattus <46076784+rattus128@users.noreply.github.com> Date: Tue, 4 Nov 2025 00:57:29 +1000 Subject: [PATCH 5/7] nodes: mmap detach weights before they are pinned (#355) Comfy core recently introduced a feature where weights may be pinned when loading, particularly for the case of offloading. Intercept this, and immediately detached each weight before the pinning. This avoids a crash that at least some users are experiencing. Use a little dict on the modules to keep track of whats already done, and when the catch-all detacher loop comes through, use this dict (which has already done modules removed) as the iterator basis. --- nodes.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index 4159142..08849df 100644 --- a/nodes.py +++ b/nodes.py @@ -76,8 +76,21 @@ class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): # TODO: Find another way to not unload after patches return super().unpatch_model(device_to=device_to, unpatch_weights=unpatch_weights) + + def pin_weight_to_device(self, key): + op_key = key.rsplit('.', 1)[0] + if self.named_modules_to_munmap is not None and op_key in self.named_modules_to_munmap: + # TODO: possible to OOM, find better way to detach + self.named_modules_to_munmap[op_key].to(self.load_device).to(self.offload_device) + del self.named_modules_to_munmap[op_key] + super().pin_weight_to_device(key) + mmap_released = False + def load(self, *args, force_patch_weights=False, **kwargs): + if not self.mmap_released: + self.named_modules_to_munmap = dict(self.model.named_modules()) + # always call `patch_weight_to_device` even for lowvram super().load(*args, force_patch_weights=True, **kwargs) @@ -85,7 +98,7 @@ class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): if not self.mmap_released: linked = [] if kwargs.get("lowvram_model_memory", 0) > 0: - for n, m in self.model.named_modules(): + for n, m in self.named_modules_to_munmap.items(): if hasattr(m, "weight"): device = getattr(m.weight, "device", None) if device == self.offload_device: @@ -102,6 +115,7 @@ class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): # TODO: possible to OOM, find better way to detach m.to(self.load_device).to(self.offload_device) self.mmap_released = True + self.named_modules_to_munmap = None def clone(self, *args, **kwargs): src_cls = self.__class__ From d4fbdb01390230d02d4b33c9b4ad721321f22d67 Mon Sep 17 00:00:00 2001 From: City <125218114+city96@users.noreply.github.com> Date: Tue, 4 Nov 2025 07:04:11 +0100 Subject: [PATCH 6/7] Add type string for older mmproj files --- loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader.py b/loader.py index fd35e13..032c739 100644 --- a/loader.py +++ b/loader.py @@ -11,7 +11,7 @@ from .dequant import is_quantized, dequantize_tensor IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image"} TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl"} -VIS_TYPE_LIST = {"clip-vision"} +VIS_TYPE_LIST = {"clip-vision", "mmproj"} def get_orig_shape(reader, tensor_name): field_key = f"comfy.gguf.orig_shape.{tensor_name}" From 02dac863ee1b65852d39ce6b9180bf5d9bc8a636 Mon Sep 17 00:00:00 2001 From: rattus <46076784+rattus128@users.noreply.github.com> Date: Sat, 8 Nov 2025 08:31:20 +1000 Subject: [PATCH 7/7] add default named_modules_to_munmap variable (#357) * add default named_modules_to_munmap variable Default this to handle to case of cloning a GGUFModelPatcher which then expects it to exist on the clone. I could never reproduce this condition, however 4 users have the error. This error path is very sensitive to VRAM condtions and workflow and I never got the details so its a bit of a guess. But from a code point of view, its the right thing anyway * nodes: carry the mmap_released state through a clone() There are workflows where the same underlying model will be loaded twice by two different ModelPatchers. With the current code as-is, the first patcher will load fine and intercept the pins while removing already done pins from the free modules list. It then will load again with the second MP however, this one will not get the callbacks for the already pinned stuff and then rip through and .to().to() them all. This destroys the pinning setup while comfy core still keeps the modules registered as unpinned. This causes a crash on einval when core goes to unpin the not pinned tensors. Comfy core now has a guard against unpinning the not-pinned, however what I think is happening, is the RAM of the pin in freed in the .to().to() and then malloc re-allocates it to new tensors with cuda keeping its records of previous pinning. einval can happen when you try and unpin a tensor in the middle of the block which is definately possible on this use after free pattern. --- nodes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nodes.py b/nodes.py index 08849df..ff5aaf0 100644 --- a/nodes.py +++ b/nodes.py @@ -79,13 +79,14 @@ class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): def pin_weight_to_device(self, key): op_key = key.rsplit('.', 1)[0] - if self.named_modules_to_munmap is not None and op_key in self.named_modules_to_munmap: + if not self.mmap_released and op_key in self.named_modules_to_munmap: # TODO: possible to OOM, find better way to detach self.named_modules_to_munmap[op_key].to(self.load_device).to(self.offload_device) del self.named_modules_to_munmap[op_key] super().pin_weight_to_device(key) mmap_released = False + named_modules_to_munmap = {} def load(self, *args, force_patch_weights=False, **kwargs): if not self.mmap_released: @@ -115,7 +116,7 @@ class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): # TODO: possible to OOM, find better way to detach m.to(self.load_device).to(self.offload_device) self.mmap_released = True - self.named_modules_to_munmap = None + self.named_modules_to_munmap = {} def clone(self, *args, **kwargs): src_cls = self.__class__ @@ -125,6 +126,7 @@ class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): self.__class__ = src_cls # GGUF specific clone values below n.patch_on_device = getattr(self, "patch_on_device", False) + n.mmap_released = getattr(self, "mmap_released", False) if src_cls != GGUFModelPatcher: n.size = 0 # force recalc return n