From e8771b68c5c0a7e95cef813950fb0e91b4a0654c Mon Sep 17 00:00:00 2001 From: Level Pixel Dev Date: Sun, 27 Jul 2025 22:52:29 +0600 Subject: [PATCH] Improvements to the multimodal generator --- README.md | 19 +-- install_init.py | 16 ++- node_list.json | 2 +- .../where_to_download_the_library.txt | 14 --- nodes/vlm/llava_LP.py | 115 +++--------------- pyproject.toml | 5 +- requirements.txt | 1 - 7 files changed, 37 insertions(+), 135 deletions(-) delete mode 100644 nodes/vlm/llama-mtmd-cli/where_to_download_the_library.txt diff --git a/README.md b/README.md index 8b5e65f..a66992e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The purpose of this package is to collect the most necessary and atomic nodes for working with LLM and VLM models with GGUF format. Installation and maintenance of LLM and VLM nodes based on LLaVA is more complex, so this node package should now be installed separately from the main Level Pixel node package. -*[Our dream is to see object-oriented programming in ComfyUI. We will try to get closer to it.](https://www.patreon.com/LevelPixel)* +*[Our dream is to see the possibilities for convenient creation of full automation in ComfyUI workflows. We will try to get closer to it.](https://www.patreon.com/LevelPixel)* **In this Level Pixel Advanced node pack you will find:** LLM nodes, LLaVa and other VLM nodes, Autotagger, RAM autotagger, WD Autotagger @@ -41,14 +41,6 @@ Before running ComfyUI with this node package, you should make sure that you hav - C++ Cmake tools for Windows - C++ Cmake tools for Linux and Mac 3. Download [CMAKE official distribution](https://cmake.org/download) - install the latest version. -4. Download [llama-mtmd-cli](https://github.com/ggml-org/llama.cpp/releases/tag/b5317) - if you want to use Qwen2.5-VL gguf in Multimodal Generator Advanced node.\ - Choose the archive that suits your system. Probably, for you it will be `llama-b5317-bin-win--cu12.4-x64.zip`\ - IMPORTANT - download only `b5317` realese. There are still some output errors on new versions\ - After downloading, go to the path:\ - `ComfyUI\custom_nodes\ComfyUI-LevelPixel-Advanced\nodes\vlm\llama-mtmd-cli`\ - And unzip the downloaded archive here.\ - The llama-mtmd-cli folder should contain the llama-mtmd-cli.exe file and other libraries from the archive:\ - `ComfyUI\custom_nodes\ComfyUI-LevelPixel-Advanced\nodes\vlm\llama-mtmd-cli\llama-mtmd-cli.exe`\ ### Installation package using ComfyUI Manager (recommended): @@ -114,8 +106,8 @@ All nodes Level Pixel: ### Multimodal Generator node -Multimodal Generator Advanced - New node on new technology of multimodal neural models based on GGUF. Supports Qwen2.5-VL of GGUF format. -Experimental node! It is not recommended to use it in fully autonomous workflows yet. +Multimodal Generator Advanced - New node on new technology of multimodal neural models based on GGUF. +Supports Qwen2.5-VL of GGUF format. How to use Multimodal Generator node: @@ -142,7 +134,7 @@ Our LLava nodes support the latest LLM and VLM models, and should support future The core functionality is taken from [ComfyUI_VLM_nodes](https://github.com/gokayfem/ComfyUI_VLM_nodes) and belongs to its authors. -Mainly supports only Mistral (with clip for images) and Llama (with clip for images) models. +Mainly supports Qwen2.5-VL (with clip for images), Mistral (with clip for images) and Llama (with clip for images) models. At the moment the nodes are obsolete (but still in support status), instead of them it is supposed to develop "Multimodal Generator nodes" based on llama-mtmd for using Qwen2.5-VL, Bagel and other multimodal neural networks. @@ -197,10 +189,11 @@ This is an image recognition node for [ComfyUI](https://github.com/comfyanonymou Furthermore you need to download the [RAM](https://huggingface.co/xinyu1205/recognize_anything_model/resolve/main/ram_swin_large_14m.pth), [RAM++](https://huggingface.co/xinyu1205/recognize-anything-plus-model/resolve/main/ram_plus_swin_large_14m.pth) and [tag2text](https://huggingface.co/xinyu1205/recognize_anything_model/resolve/main/tag2text_swin_14m.pth) models and place it in the /ComfyUI/models/rams/ folder or use the [ComfyUI-Manager](https://github.com/Comfy-Org/ComfyUI-Manager) model downloader. -You can also configure the location in 'extra\_model\_paths.yaml' in the Confyui folder. +You can also configure the location in 'extra\_model\_paths.yaml' in the ComfyUI folder. ## Update History +27-07-2025 - Multimodal Generator Advanced stabilized by natively using the latest version of the llama-cpp-python library 30-05-2025 - Added new node Multimodal Generation Advanced for neural models of multimodal type (for example, for Qwen2.5-VL) The license for this package has been changed from Apache 2.0 to GNU GPLv3 diff --git a/install_init.py b/install_init.py index a6d4786..33dd615 100644 --- a/install_init.py +++ b/install_init.py @@ -13,6 +13,10 @@ import subprocess import sys from importlib import metadata +package_name_llama_cpp_python = "llama-cpp-python" +min_version_llama_cpp_python = "0.3.14" +version_llama_cpp_python = f"{package_name_llama_cpp_python}>={min_version_llama_cpp_python}" + def verify_python_support(): version = tuple(map(int, platform.python_version_tuple()[:2])) if version < (3, 8): @@ -120,7 +124,7 @@ def install_llama(system_info): if not verify_pypy_support(system_info): print("WARNING: Unsupported PyPy configuration") - imported = package_is_installed("llama-cpp-python") or package_is_installed("llama_cpp") + imported = is_installed(package_name_llama_cpp_python, min_version_llama_cpp_python) # or package_is_installed("llama_cpp") if imported: print("llama-cpp installed") return True @@ -129,7 +133,7 @@ def install_llama(system_info): if system_info['os'] == 'linux': try: print("Installing llama-cpp-python via pip") - install_package("llama-cpp-python") + install_package(version_llama_cpp_python) return True except Exception as e: print(f"Installation failed: {e}") @@ -158,7 +162,7 @@ def install_llama(system_info): if system_info.get('metal', False): print("Building llama-cpp-python from source with Metal support") os.environ['CMAKE_ARGS'] = "-DGGML_METAL=on" - install_package("llama-cpp-python") + install_package(version_llama_cpp_python) return True elif system_info['gpu']: if system_info.get('cuda_version'): @@ -169,12 +173,12 @@ def install_llama(system_info): os.environ['CMAKE_ARGS'] = "-DGGML_CUDA=on -DGGML_CUDA_ZLUDA=on" else: os.environ['CMAKE_ARGS'] = "-DGGML_CUDA=on" - install_package("llama-cpp-python") + install_package(version_llama_cpp_python) return True elif system_info.get('rocm_version'): print("Building llama-cpp-python from source with ROCm support") os.environ['CMAKE_ARGS'] = "-DGGML_HIPBLAS=on" - install_package("llama-cpp-python") + install_package(version_llama_cpp_python) return True except Exception as e: print(f"Accelerated build failed: {e}") @@ -183,7 +187,7 @@ def install_llama(system_info): # Final fallback - basic CPU version try: print("Installing CPU-only version") - install_package("llama-cpp-python") + install_package(version_llama_cpp_python) return True except Exception as e: print(f"CPU installation failed: {e}") diff --git a/node_list.json b/node_list.json index a22167d..002a1be 100644 --- a/node_list.json +++ b/node_list.json @@ -14,5 +14,5 @@ "Image Remove Background (BiRefNet) [LP]":"Removes background for images with user-defined model type of BiRefNet. Recommended for fast and high-quality background removal", "Image Remove Background (RMBG) [LP]":"Removes background for images with user-defined model. This node uses the RMBG models. Recommended for ultra-high precision", "Color Input [LP]":"Color Input for remove background nodes.", - "Multimodal Generator Advanced [LP]":"Node for Multimodal neural models based on GGUF. Supports Qwen2.5-VL of GGUF format. Based on llama-mtmd." + "Multimodal Generator Advanced [LP]":"Node for Multimodal neural models based on GGUF. Supports Qwen2.5-VL of GGUF format." } \ No newline at end of file diff --git a/nodes/vlm/llama-mtmd-cli/where_to_download_the_library.txt b/nodes/vlm/llama-mtmd-cli/where_to_download_the_library.txt deleted file mode 100644 index 96bdd20..0000000 --- a/nodes/vlm/llama-mtmd-cli/where_to_download_the_library.txt +++ /dev/null @@ -1,14 +0,0 @@ -To make this node work, you need to download the Llama.cpp libraries manually from this link: -https://github.com/ggml-org/llama.cpp/releases - -Download the archive that suits your system. Probably, for you it will be llama-b5317-bin-win--cu12.4-x64.zip - -After downloading, go to the path: -ComfyUI\custom_nodes\ComfyUI-LevelPixel-Advanced\nodes\vlm\llama-mtmd-cli\ - -And UNZIP the downloaded archive here. - -The llama-mtmd-cli folder should contain the llama-mtmd-cli.exe file and other libraries from the archive: -ComfyUI\custom_nodes\ComfyUI-LevelPixel-Advanced\nodes\vlm\llama-mtmd-cli\llama-mtmd-cli.exe - -After that, run the generation again. \ No newline at end of file diff --git a/nodes/vlm/llava_LP.py b/nodes/vlm/llava_LP.py index 4556374..433aa1f 100644 --- a/nodes/vlm/llava_LP.py +++ b/nodes/vlm/llava_LP.py @@ -2,13 +2,12 @@ import folder_paths import os from io import BytesIO from llama_cpp import Llama -from llama_cpp.llama_chat_format import Llava16ChatHandler +from llama_cpp.llama_chat_format import Llava16ChatHandler, Qwen25VLChatHandler import base64 from torchvision.transforms import ToPILImage import gc import torch - supported_LLava_extensions = set(['.gguf']) try: @@ -292,14 +291,18 @@ class LLavaAdvanced: presence_penalty, repeat_penalty, seed, unload): clip_path = folder_paths.get_full_path("LLavacheckpoints", clip_name) - self.clip = Llava16ChatHandler(clip_model_path=clip_path, verbose=False) - + + if "qwen2.5-vl" in str(clip_name).lower(): + self.clip = Qwen25VLChatHandler(clip_model_path=clip_path, verbose=False) + else: + self.clip = Llava16ChatHandler(clip_model_path=clip_path, verbose=False) + ckpt_path = folder_paths.get_full_path("LLavacheckpoints", ckpt_name) self.llm = Llama(model_path = ckpt_path, chat_handler=self.clip, offload_kqv=True, f16_kv=True, use_mlock=False, embedding=False, n_batch=1024, last_n_tokens_size=1024, verbose=True, seed=42, n_ctx = max_ctx, n_gpu_layers=gpu_layers, n_threads=n_threads, logits_all=True, echo=False) - + pil_image = ToPILImage()(image[0].permute(2, 0, 1)) buffer = BytesIO() @@ -336,7 +339,6 @@ class LLavaAdvanced: self.llm = None gc.collect() torch.cuda.empty_cache() - if unload and self.clip is not None: self.clip._exit_stack.close() # info https://github.com/abetlen/llama-cpp-python/issues/1746 @@ -347,10 +349,9 @@ class LLavaAdvanced: return (f"{response['choices'][0]['message']['content']}", ) -class MultimodalGeneratorAdvanced: +class MultimodalGeneratorAdvanced(LLavaAdvanced): def __init__(self): - self.llm = None - self.clip = None + super().__init__() @classmethod def INPUT_TYPES(cls): @@ -370,9 +371,9 @@ class MultimodalGeneratorAdvanced: "presence_penalty": ("FLOAT", {"default": 0.0, "step": 0.01}), "repeat_penalty": ("FLOAT", {"default": 1.1, "step": 0.01}), "seed": ("INT", {"default": 42, "step": 1}), - #"unload": ("BOOLEAN", {"default": False}), + "unload": ("BOOLEAN", {"default": False}), "prompt": ("STRING", {"multiline": True, "default": ""}), - #"system_msg": ("STRING", {"multiline": True, "default": "You are an assistant who perfectly describes images."}), + "system_msg": ("STRING", {"multiline": True, "default": "You are an assistant who perfectly describes images."}), } } @@ -381,95 +382,15 @@ class MultimodalGeneratorAdvanced: CATEGORY = "LevelPixel/VLM" def multimodal_generator_advanced(self, ckpt_name, clip_name, max_ctx, gpu_layers, n_threads, image, - #system_msg, - prompt, max_tokens, temperature, top_p, top_k, frequency_penalty, - presence_penalty, repeat_penalty, seed): #, unload): + system_msg, prompt, max_tokens, temperature, top_p, top_k, frequency_penalty, + presence_penalty, repeat_penalty, seed, unload): - clip_path = folder_paths.get_full_path("LLavacheckpoints", clip_name) - ckpt_path = folder_paths.get_full_path("LLavacheckpoints", ckpt_name) - - import tempfile - - pil_image = ToPILImage()(image[0].permute(2, 0, 1)) - - buffer = BytesIO() - pil_image.save(buffer, format="PNG") - - with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp: - temp_image_path = tmp.name - pil_image.save(temp_image_path) - - import subprocess - from pathlib import Path - - relative_cli_path="llama-mtmd-cli\\llama-mtmd-cli.exe" - cli = Path(__file__).parent / relative_cli_path - if not cli.is_file(): - raise FileNotFoundError(f"""Library llama-mtmd-cli.exe not founded: {cli} - - SOLUTION: - - To make this node work, you need to download the Llama.cpp libraries manually from this link: - https://github.com/ggml-org/llama.cpp/releases - - Download the archive that suits your system. Probably, for you it will be llama-b5317-bin-win--cu12.4-x64.zip - - After downloading, go to the path: - ComfyUI\\custom_nodes\\ComfyUI-LevelPixel-Advanced\\nodes\\vlm\\llama-mtmd-cli - - And unzip the downloaded archive here. - - The llama-mtmd-cli folder should contain the llama-mtmd-cli.exe file and other libraries from the archive. - - After that, run the generation again.""") - - cmd = [ - str(cli), - '-m', ckpt_path, - '--mmproj', clip_path, - '-p', prompt, - '--image', temp_image_path, - '--ctx-size', max_ctx, - '--gpu-layers', gpu_layers, - '--threads', n_threads, - '--n-predict', max_tokens, - '--temp', temperature, - '--top-p', top_p, - '--top-k', top_k, - '--frequency-penalty', frequency_penalty, - '--presence-penalty', presence_penalty, - '--repeat-penalty', repeat_penalty, - '--seed', seed, - #'-sys', system_msg - ] - - cmd = [str(x) for x in cmd] - - print(cmd) - - print("Generate by llama-mtmd-cli") - result = subprocess.run( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True + return self.generate_text_full_advanced( + ckpt_name, clip_name, max_ctx, gpu_layers, n_threads, image, + system_msg, prompt, max_tokens, temperature, top_p, top_k, frequency_penalty, + presence_penalty, repeat_penalty, seed, unload ) - output = result.stdout.decode('utf-8').strip() - - def parse_output(output: str) -> str: - lines = output.strip().splitlines() - if len(lines) > 1: - return '\n'.join(lines[1:]).strip() - return output.strip() - - clean_output = parse_output(output) - - if os.path.exists(temp_image_path): - os.remove(temp_image_path) - - return (clean_output, ) - NODE_CLASS_MAPPINGS = { "LLavaLoader|LP": LLavaLoader, "LLavaClipLoader|LP": LLavaClipLoader, diff --git a/pyproject.toml b/pyproject.toml index 6b20280..c97f5da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-levelpixel-advanced" -description = "Advanced nodes of the Level Pixel company (aka levelpixel, LP). Includes convenient advanced nodes for working with LLM и VLM models (LLaVa, Multimodal Generator) with GGUF format. Qwen2.5-VL and Qwen2.5 supported, based on llama-mtmd. Also included is a node for the RAM model. Nodes have the ability to automatically unload models from VRAM. This package also includes Autotagger (WD), Image Remove Background (based on RMBG-2.0, on BiRefNet, on BiRefNet-HR-Matter, on RemBG). Licensed under GNU GPLv3. Repository: https://github.com/LevelPixel/ComfyUI-LevelPixel-Advanced" -version = "1.3.1" +description = "Advanced nodes of the Level Pixel company (aka levelpixel, LP). Includes convenient advanced nodes for working with LLM и VLM models (LLaVa, Multimodal Generator) with GGUF format. Qwen2.5-VL and Qwen2.5 supported. Also included is a node for the RAM model. Nodes have the ability to automatically unload models from VRAM. This package also includes Autotagger (WD), Image Remove Background (based on RMBG-2.0, on BiRefNet, on BiRefNet-HR-Matter, on RemBG). Licensed under GNU GPLv3. Repository: https://github.com/LevelPixel/ComfyUI-LevelPixel-Advanced" +version = "1.3.2" license = { file = "LICENSE.txt" } dependencies = [ "torch>=2.0.1", @@ -14,7 +14,6 @@ dependencies = [ "scikit-build-core>=0.10.7", "transparent-background>=1.2.4", "timm>=0.4.12", - "tqdm>=4.65.0", "fairscale>=0.4.4", "rembg>=2.0.66", "diskcache>=5.6.1", diff --git a/requirements.txt b/requirements.txt index 64340de..eb8d192 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ safetensors>=0.3.0 scikit-build-core>=0.10.7 transparent-background>=1.2.4 timm>=0.4.12 -tqdm>=4.65.0 fairscale>=0.4.4 rembg>=2.0.66 matplotlib