From 450e39e7a0fa00836626b33da0863896ffdad7e1 Mon Sep 17 00:00:00 2001 From: dseditor Date: Fri, 30 Jan 2026 00:26:18 +0800 Subject: [PATCH] PR merged and fix for different llama_cpp version release VRAM method (need test) --- gguf_inference2.py | 1497 -------------------------------------------- 1 file changed, 1497 deletions(-) delete mode 100644 gguf_inference2.py diff --git a/gguf_inference2.py b/gguf_inference2.py deleted file mode 100644 index 2f383cd..0000000 --- a/gguf_inference2.py +++ /dev/null @@ -1,1497 +0,0 @@ -import os -import sys -import gc -import time -import re -import subprocess -import platform -import base64 -import io -import random -import json -import numpy as np -import folder_paths -import urllib.request -from typing import Optional, Tuple, List - -# Import JSON prompt extractor for automatic prompt list extraction -from .json_prompt_extractor import extract_prompts_from_json - -# Suggested models for download -SUGGESTED_MODELS = { - "Download: Z-Image": "https://huggingface.co/unsloth/Qwen3-4B-GGUF/resolve/main/Qwen3-4B-Q5_K_M.gguf", - "Download: Z-Image (Abliterated)": "https://huggingface.co/Mungert/Qwen3-4B-abliterated-GGUF/resolve/main/Qwen3-4B-abliterated-q4_k_m.gguf", - "Download: Qwen": "https://huggingface.co/unsloth/Qwen2.5-VL-7B-Instruct-GGUF/resolve/main/Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf", - "Download: Qwen (Abliterated)": "https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-abliterated-GGUF/resolve/main/Qwen2.5-VL-7B-Instruct-abliterated.Q4_K_M.gguf", - "Download: QwenVL": "https://huggingface.co/mradermacher/Huihui-Qwen3-VL-4B-Instruct-abliterated-GGUF/resolve/main/Huihui-Qwen3-VL-4B-Instruct-abliterated.Q4_K_M.gguf", - "Download: QwenVL(2B)": "https://huggingface.co/second-state/Qwen2-VL-2B-Instruct-GGUF/resolve/main/Qwen2-VL-2B-Instruct-Q4_K_M.gguf", -} - -SUGGESTED_MMPROJ = { - "Download: mmproj": "https://huggingface.co/unsloth/Qwen2.5-VL-7B-Instruct-GGUF/resolve/main/mmproj-F16.gguf", - "Download: QwenVL mmproj": "https://huggingface.co/noctrex/Huihui-Qwen3-VL-4B-Instruct-abliterated-i1-GGUF/resolve/main/mmproj-F16.gguf", -} - -class GGUFInference: - """ - GGUF Model Inference Node with llama-cpp-python - Supports both text-only and vision models with optional mmproj files - Auto-searches GGUF files from text_encoders and clip folders - """ - - def __init__(self): - self.model = None - self.current_model_path = None - self.current_mmproj_path = None - self.clip_model_array = None - self.llama_cpp_available = False - self.failed_versions = [] # Track failed installation versions - self.ggml_error_retry_count = 0 # Track retry attempts for ggml errors - self.max_ggml_retries = 2 # Maximum retry attempts - self._check_llama_cpp() - - def _check_llama_cpp(self): - """Check if llama-cpp-python is available""" - try: - import llama_cpp - self.llama_cpp_available = True - print("llama-cpp-python is available") - except ImportError: - self.llama_cpp_available = False - print("llama-cpp-python is not installed") - - @classmethod - def _download_file(cls, url: str, destination: str) -> bool: - """Download file from URL with progress reporting""" - try: - def report_progress(block_num, block_size, total_size): - downloaded = block_num * block_size - if total_size > 0: - percent = min(downloaded * 100 / total_size, 100) - downloaded_mb = downloaded / (1024 * 1024) - total_mb = total_size / (1024 * 1024) - print(f"⬇️ {percent:.0f}% ({downloaded_mb:.0f}MB/{total_mb:.0f}MB)", end='\r') - - urllib.request.urlretrieve(url, destination, reporthook=report_progress) - print(f"\n✓ Downloaded: {os.path.basename(destination)}") - return True - except Exception as e: - print(f"\n⚠️ Download failed: {e}") - return False - - @classmethod - def _get_gguf_files(cls): - """Get all GGUF files from text_encoders and clip folders""" - gguf_files = [] - - # Search in text_encoders folder - try: - text_encoder_paths = folder_paths.get_folder_paths("text_encoders") - for path in text_encoder_paths: - if os.path.exists(path): - for file in os.listdir(path): - if file.lower().endswith('.gguf'): - full_path = os.path.join(path, file) - if full_path not in gguf_files: - gguf_files.append(full_path) - except: - pass - - # Search in clip folder - try: - clip_paths = folder_paths.get_folder_paths("clip") - for path in clip_paths: - if os.path.exists(path): - for file in os.listdir(path): - if file.lower().endswith('.gguf'): - full_path = os.path.join(path, file) - if full_path not in gguf_files: - gguf_files.append(full_path) - except: - pass - - # Add suggested models to the list - suggested_models = list(SUGGESTED_MODELS.keys()) - - if not gguf_files: - # If no local models, only show suggested models - return suggested_models - - # Return local models first, then suggested models - return sorted(gguf_files) + suggested_models - - @classmethod - def _get_mmproj_files(cls): - """Get all mmproj files from text_encoders and clip folders""" - mmproj_files = [] - - # Search in text_encoders folder - try: - text_encoder_paths = folder_paths.get_folder_paths("text_encoders") - for path in text_encoder_paths: - if os.path.exists(path): - for file in os.listdir(path): - if 'mmproj' in file.lower() and file.lower().endswith('.gguf'): - full_path = os.path.join(path, file) - if full_path not in mmproj_files: - mmproj_files.append(full_path) - except: - pass - - # Search in clip folder - try: - clip_paths = folder_paths.get_folder_paths("clip") - for path in clip_paths: - if os.path.exists(path): - for file in os.listdir(path): - if 'mmproj' in file.lower() and file.lower().endswith('.gguf'): - full_path = os.path.join(path, file) - if full_path not in mmproj_files: - mmproj_files.append(full_path) - except: - pass - - # Add suggested mmproj to the list - suggested_mmproj = list(SUGGESTED_MMPROJ.keys()) - - if not mmproj_files: - # If no local mmproj files, show "No mmproj files" and suggested option - return ["No mmproj files"] + suggested_mmproj - - # Return local mmproj files first, then suggested option - return sorted(mmproj_files) + suggested_mmproj - - @classmethod - def _get_prompt_templates(cls): - """Get all .md template files from Prompt folder""" - current_dir = os.path.dirname(os.path.abspath(__file__)) - prompt_dir = os.path.join(current_dir, "Prompt") - - templates = [] - - if os.path.exists(prompt_dir): - for file in os.listdir(prompt_dir): - if file.lower().endswith('.md'): - templates.append(file) - - if not templates: - return ["No Template"] - - return sorted(templates) - - @classmethod - def _load_template_content(cls, template_name): - """Load template content""" - if template_name == "No Template" or template_name == "Custom": - return "" - - current_dir = os.path.dirname(os.path.abspath(__file__)) - template_path = os.path.join(current_dir, "Prompt", template_name) - - if os.path.exists(template_path): - try: - with open(template_path, 'r', encoding='utf-8') as f: - return f.read() - except: - return "" - - return "" - - @classmethod - def INPUT_TYPES(cls): - gguf_files = cls._get_gguf_files() - # Get file names only for dropdown - gguf_names = [os.path.basename(f) if f != "No GGUF files found" else f for f in gguf_files] - - # Get mmproj files - mmproj_files = cls._get_mmproj_files() - mmproj_names = [os.path.basename(f) if f != "No mmproj files" else f for f in mmproj_files] - - # Get prompt templates - templates = cls._get_prompt_templates() - template_options = ["Custom"] + templates - - # Set default model to QwenVL download option if available - default_model = "Download: QwenVL" if "Download: QwenVL" in gguf_names else (gguf_names[0] if gguf_names else "No GGUF files found") - - # Set default mmproj to QwenVL mmproj download option if available - default_mmproj = "Download: QwenVL mmproj" if "Download: QwenVL mmproj" in mmproj_names else (mmproj_names[0] if mmproj_names else "No mmproj files") - - # Set default template to image_to_prompt.md if available, otherwise Custom - default_template = "image_to_prompt.md" if "image_to_prompt.md" in template_options else (template_options[0] if template_options else "Custom") - - return { - "required": { - "model": (gguf_names, { - "default": default_model - }), - "prompt": ("STRING", { - "multiline": True, - "default": "" - }), - "prompt_template": (template_options, { - "default": default_template - }), - "system_prompt": ("STRING", { - "multiline": True, - "default": "" - }), - "max_tokens": ("INT", { - "default": 3072, - "min": 1, - "max": 8192, - "step": 1 - }), - "temperature": ("FLOAT", { - "default": 0.7, - "min": 0.0, - "max": 2.0, - "step": 0.1 - }), - "top_p": ("FLOAT", { - "default": 0.9, - "min": 0.0, - "max": 1.0, - "step": 0.05 - }), - "top_k": ("INT", { - "default": 40, - "min": 0, - "max": 100, - "step": 1 - }), - }, - "optional": { - "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "control_after_generate": True}), - "keep_model_loaded": ("BOOLEAN", { - "default": False, - "tooltip": "Keep model in memory after inference" - }), - "mmproj_file": (mmproj_names, { - "default": default_mmproj, - "tooltip": "Vision model mmproj file (auto-enabled when any image is provided and model is VL type)" - }), - "image_1": ("IMAGE", { - "tooltip": "First input image for vision model (auto-enables vision mode for VL models)" - }), - "image_2": ("IMAGE", { - "tooltip": "Second input image for vision model (optional, for multi-image analysis)" - }), - "image_3": ("IMAGE", { - "tooltip": "Third input image for vision model (optional, for multi-image analysis)" - }), - "auto_install_llama_cpp": ("BOOLEAN", { - "default": True, - "tooltip": "Auto-install/repair llama-cpp-python (supports all platforms/CUDA versions, prioritizes Basic version for compatibility)" - }), - } - } - - RETURN_TYPES = ("STRING", "INT", "STRING") - RETURN_NAMES = ("text", "used_seed", "prompts") - OUTPUT_IS_LIST = (False, False, True) # prompts 是列表 - FUNCTION = "inference" - CATEGORY = "ListHelper/LLM" - - def _free_memory(self): - """Free memory with robust handling for Vision Handlers & Zombie Wrappers""" - print("🧹 Releasing resources...") - try: - import llama_cpp - - # 1. Vision Cleanup (Qwen2.5-VL / Llava) - if self.clip_model_array is not None: - # Capture pointer (legacy handlers) - clip_ptr = None - if hasattr(self.clip_model_array, 'clip_model'): - clip_ptr = self.clip_model_array.clip_model - elif hasattr(self.clip_model_array, 'vision_model'): - clip_ptr = self.clip_model_array.vision_model - - # A. Close Exit Stack (Crucial for Qwen25VLChatHandler) - if hasattr(self.clip_model_array, '_exit_stack') and self.clip_model_array._exit_stack: - try: self.clip_model_array._exit_stack.close() - except: pass - - # B. Standard Close - if hasattr(self.clip_model_array, 'close'): - try: self.clip_model_array.close() - except: pass - - # C. Force Kill (Legacy/Direct Pointer) - if clip_ptr: - try: llama_cpp.llama_free_model(clip_ptr) - except: pass - - del self.clip_model_array - self.clip_model_array = None - - # 2. Main Model Cleanup - if self.model is not None: - # --- CAPTURE RAW POINTERS --- - raw_model_ptr = None - raw_ctx_ptr = None - - if hasattr(self.model, '_model') and self.model._model: - if hasattr(self.model._model, 'model'): - raw_model_ptr = self.model._model.model - else: - raw_model_ptr = self.model._model - elif hasattr(self.model, 'model') and self.model.model: - raw_model_ptr = self.model.model - - if hasattr(self.model, '_ctx') and self.model._ctx: - if hasattr(self.model._ctx, 'ctx'): - raw_ctx_ptr = self.model._ctx.ctx - else: - raw_ctx_ptr = self.model._ctx - - # A. Try Standard Close - if hasattr(self.model, 'close'): - try: self.model.close() - except: pass - - # B. FORCE KILL using Captured Pointers - if raw_model_ptr: - try: llama_cpp.llama_free_model(raw_model_ptr) - except: pass - - if raw_ctx_ptr: - try: llama_cpp.llama_free(raw_ctx_ptr) - except: pass - - # C. Nullify Wrappers - if hasattr(self.model, '_model'): self.model._model = None - if hasattr(self.model, '_ctx'): self.model._ctx = None - - del self.model - self.model = None - - self.current_model_path = None - self.current_mmproj_path = None - - # 3. System Cleanup - import gc - gc.collect() - gc.collect() - try: - import torch - if torch.cuda.is_available(): - torch.cuda.empty_cache() - torch.cuda.ipc_collect() - except: pass - - print("✓ Resources released") - - except Exception as e: - print(f"⚠️ Error freeing memory: {e}") - - def _free_image_memory(self): - """Free image-related memory without unloading model""" - try: - # Run garbage collection multiple times - for _ in range(2): - gc.collect() - - # Try to free CUDA memory if available - try: - import torch - if torch.cuda.is_available(): - torch.cuda.empty_cache() - torch.cuda.synchronize() - # Empty cache again after sync - torch.cuda.empty_cache() - except: - pass - - # Image memory freed silently - pass - except Exception as e: - print(f"⚠️ Error freeing image memory: {e}") - - def _is_ggml_error(self, exception) -> bool: - """Detect if exception is related to ggml/CUDA compatibility issues - - Args: - exception: The exception object to check - - Returns: - True if this is a ggml-related error that might be fixed by reinstalling - """ - error_str = str(exception).lower() - - # Check for common ggml-related error patterns - ggml_patterns = [ - 'ggml', - 'cannot load library', - 'dll load failed', - 'cuda', - 'shared library', - 'importerror', - 'oserror', - ] - - return any(pattern in error_str for pattern in ggml_patterns) - - def _handle_ggml_error_and_reinstall(self) -> Tuple[bool, Optional[str]]: - """Handle ggml error by detecting system and reinstalling compatible version - - This method is called when a ggml-related error occurs during model loading. - It will: - 1. Re-detect system configuration (Python/CUDA versions) - 2. Uninstall current llama-cpp-python - 3. Search for and install compatible version - - Returns: - Tuple of (success: bool, message: str or None) - """ - print("=" * 70) - print("GGML ERROR DETECTED - Attempting automatic recovery") - print("=" * 70) - - # Increment retry count - self.ggml_error_retry_count += 1 - - # Check if we've exceeded max retries - if self.ggml_error_retry_count > self.max_ggml_retries: - error_msg = ( - f"Maximum retry attempts ({self.max_ggml_retries}) reached.\n\n" - "Unable to automatically resolve the ggml compatibility issue.\n" - "This usually means:\n" - "1. Your CUDA/Python version changed and no compatible wheel is available\n" - "2. System libraries are missing or corrupted\n\n" - "Please manually install from:\n" - "https://github.com/JamePeng/llama-cpp-python/releases" - ) - print(error_msg) - print("=" * 70) - return (False, error_msg) - - print(f"Retry attempt {self.ggml_error_retry_count}/{self.max_ggml_retries}") - print("Re-detecting system configuration...") - - # Get current installed version to add to skip list - try: - import pkg_resources - current_version = pkg_resources.get_distribution("llama-cpp-python").version - # Extract tag from version if possible - # Version format like: 0.3.18+cu128 - if current_version: - print(f"Current version: {current_version}") - # Add current version to failed list if not already there - # We need to find the matching tag, but for now just track by version - pass - except: - current_version = None - - # Uninstall current version - print("Uninstalling incompatible version...") - if not self._uninstall_llama_cpp(): - error_msg = ( - "Failed to uninstall current llama-cpp-python.\n" - "Please manually uninstall: pip uninstall llama-cpp-python" - ) - print(error_msg) - print("=" * 70) - return (False, error_msg) - - # Try to install compatible version, skipping previously failed versions - print("Searching for compatible version...") - print("System will be re-detected automatically during installation") - print("=" * 70) - - success, new_version = self._install_llama_cpp(skip_versions=set(self.failed_versions)) - - if success and new_version: - # Add this version to failed list (since we got here, it had issues) - if new_version not in self.failed_versions: - self.failed_versions.append(new_version) - - success_msg = ( - f"Successfully installed llama-cpp-python {new_version}\n\n" - "IMPORTANT: Please restart ComfyUI to use the new version.\n" - "The current session may still have compatibility issues." - ) - print("=" * 70) - print(success_msg) - print("=" * 70) - return (True, success_msg) - else: - # Add to failed list even if we couldn't install - if new_version and new_version not in self.failed_versions: - self.failed_versions.append(new_version) - - error_msg = ( - "Failed to find/install compatible llama-cpp-python version.\n\n" - "Please visit: https://github.com/JamePeng/llama-cpp-python/releases\n" - "And manually install a version compatible with:\n" - f"- Python {sys.version_info.major}.{sys.version_info.minor}\n" - ) - - # Add CUDA info if available - cuda_version = self._detect_cuda_version() - if cuda_version: - error_msg += f"- CUDA {cuda_version}\n" - - print(error_msg) - print("=" * 70) - return (False, error_msg) - - def _detect_cuda_version(self): - """Detect CUDA version from system""" - try: - # Try to get CUDA version from torch - import torch - if torch.cuda.is_available(): - cuda_version = torch.version.cuda - if cuda_version: - # Convert 12.8 to 128, 12.1 to 121, etc. - major, minor = cuda_version.split('.') - return f"cu{major}{minor}" - except: - pass - - # Try to get CUDA version from nvcc - try: - result = subprocess.run(['nvcc', '--version'], - capture_output=True, - text=True, - timeout=5) - if result.returncode == 0: - # Parse output like "Cuda compilation tools, release 12.8, V12.8.89" - match = re.search(r'release (\d+)\.(\d+)', result.stdout) - if match: - major, minor = match.groups() - return f"cu{major}{minor}" - except: - pass - - return None - - def _parse_release_info(self, release): - """Parse release tag to extract CUDA version, AVX support, and OS info - - Example tag formats: - - v0.3.18-cu130-AVX2-win-20251220 -> CUDA 13.0, AVX2, Windows - - v0.3.18-cu128-AVX2-linux-20251220 -> CUDA 12.8, AVX2, Linux - - v0.3.16+cu128avx2 -> CUDA 12.8, AVX2 - - v0.3.16+cu121 -> CUDA 12.1, no AVX - - v0.3.16+avx2 -> no CUDA, AVX2 - - v0.3.16 -> no CUDA, no AVX - """ - tag_name = release.get('tag_name', '') - - # Extract CUDA version (e.g., cu128, cu121, cu130) - # Support both formats: -cu130- and +cu128 - cuda_match = re.search(r'[-+]cu(\d+)', tag_name) - cuda_version = f"cu{cuda_match.group(1)}" if cuda_match else None - - # Check for AVX2 support (case-insensitive) - has_avx2 = 'avx2' in tag_name.lower() - - # Check for AVX (non-AVX2) - has_avx = 'avx' in tag_name.lower() and not has_avx2 - - return { - 'tag_name': tag_name, - 'cuda_version': cuda_version, - 'has_avx2': has_avx2, - 'has_avx': has_avx, - 'release': release - } - - def _get_github_releases(self, max_releases=15): - """Get recent releases from GitHub""" - try: - api_url = f"https://api.github.com/repos/JamePeng/llama-cpp-python/releases?per_page={max_releases}" - - # Create request with User-Agent header - req = urllib.request.Request(api_url) - req.add_header('User-Agent', 'ComfyUI-GGUFInference/1.0') - - with urllib.request.urlopen(req, timeout=10) as response: - data = json.loads(response.read().decode('utf-8')) - return data # Returns list of releases - except Exception as e: - print(f"Failed to fetch GitHub releases: {e}") - return None - - def _find_matching_release(self, releases_list, skip_versions=None): - """Find matching release and wheel URL from releases list based on system configuration - - Args: - releases_list: List of GitHub releases - skip_versions: Set of version tags to skip (previously failed versions) - """ - if not releases_list: - return None - - if skip_versions is None: - skip_versions = set() - - # Get system info - system = platform.system().lower() # 'windows', 'linux', 'darwin' - py_version = sys.version_info - py_ver = f"cp{py_version.major}{py_version.minor}" - - # Detect CUDA version - cuda_version = self._detect_cuda_version() - - print("=" * 70) - print("Detecting system configuration:") - print(f" OS: {system}") - print(f" Python: {py_version.major}.{py_version.minor} ({py_ver})") - print(f" CUDA: {cuda_version if cuda_version else 'Not detected'}") - if skip_versions: - print(f" Skipping versions: {', '.join(sorted(skip_versions))}") - print("=" * 70) - - # Determine platform suffix - if system == 'windows': - platform_suffix = 'win_amd64.whl' - elif system == 'linux': - platform_suffix = 'linux_x86_64.whl' - elif system == 'darwin': - # macOS - try both arm64 and x86_64 - import platform as plt - machine = plt.machine().lower() - if 'arm' in machine or 'aarch64' in machine: - platform_suffix = 'macosx_11_0_arm64.whl' - else: - platform_suffix = 'macosx_10_9_x86_64.whl' - else: - print(f"Unsupported OS: {system}") - return None - - # Build priority list for release matching - # Priority: CUDA Basic > CPU Basic - # Note: Only use Basic versions (no AVX/AVX2) for maximum compatibility - # AVX and AVX2 versions are skipped due to compatibility issues - print("=" * 70) - print("Searching through releases...") - print("Note: Skipping AVX/AVX2 versions for better compatibility") - print("=" * 70) - - # Try each priority level - priority_filters = [] - - if cuda_version and system != 'darwin': - # Priority 1: CUDA exact version match (Basic only - no AVX/AVX2) - priority_filters.append({ - 'name': f'CUDA {cuda_version} Basic (exact match)', - 'cuda': cuda_version, - 'require_basic': True, # Only match Basic (no AVX, no AVX2) - 'allow_cuda_upgrade': False - }) - # Priority 2: CUDA compatible version (Basic only - allows newer CUDA) - # CUDA is forward compatible, e.g., cu130 wheels work on cu128 systems - priority_filters.append({ - 'name': f'CUDA {cuda_version}+ Basic (compatible)', - 'cuda': cuda_version, - 'require_basic': True, - 'allow_cuda_upgrade': True - }) - - # Priority 3: CPU Basic version (no AVX, no AVX2) - priority_filters.append({ - 'name': 'CPU Basic', - 'cuda': None, - 'require_basic': True - }) - - # Search through releases with priority - for priority in priority_filters: - print(f"Trying priority: {priority['name']}") - - for release in releases_list: - release_info = self._parse_release_info(release) - - # Skip this version if it's in the skip list - if release_info['tag_name'] in skip_versions: - continue - - # Check if this release matches current priority - if priority.get('allow_cuda_upgrade', False): - # Allow CUDA version upgrade (e.g., cu130 wheel on cu128 system) - # Extract numeric CUDA version for comparison - if priority['cuda'] and release_info['cuda_version']: - try: - required_cuda = int(priority['cuda'][2:]) # cu128 -> 128 - release_cuda = int(release_info['cuda_version'][2:]) # cu130 -> 130 - # Match if release CUDA >= required CUDA (forward compatibility) - cuda_match = release_cuda >= required_cuda - except: - cuda_match = False - else: - cuda_match = (priority['cuda'] is None and release_info['cuda_version'] is None) - else: - # Exact CUDA version match - cuda_match = (priority['cuda'] is None and release_info['cuda_version'] is None) or \ - (priority['cuda'] == release_info['cuda_version']) - - # Handle require_basic flag: skip all AVX and AVX2 versions - if priority.get('require_basic', False): - # Only match Basic versions (no AVX, no AVX2) - basic_match = not release_info['has_avx'] and not release_info['has_avx2'] - else: - # Old logic for backward compatibility (not used in new priority system) - basic_match = True - - if cuda_match and basic_match: - # Found matching release, now find wheel for this Python version - print(f" Found matching release: {release_info['tag_name']}") - - # Search for wheel with matching Python version - for asset in release.get('assets', []): - name = asset.get('name', '') - download_url = asset.get('browser_download_url', '') - - if not name.endswith('.whl'): - continue - - # Check if wheel matches Python version and platform - if f"-{py_ver}-{py_ver}-" in name and platform_suffix in name: - print("=" * 70) - print(f"SUCCESS: Found matching wheel!") - print(f" Release: {release_info['tag_name']}") - print(f" Wheel: {name}") - print(f" Download URL: {download_url}") - print("=" * 70) - return (download_url, release_info['tag_name']) - - # Release matched but no wheel for this Python version - print(f" No wheel found for Python {py_ver} in this release") - - # No matching release found - print("=" * 70) - print("No matching release found for your system configuration") - print("Available releases:") - for release in releases_list[:5]: # Show first 5 releases - release_info = self._parse_release_info(release) - print(f" - {release_info['tag_name']} (CUDA: {release_info['cuda_version']}, AVX: {release_info['has_avx']}, AVX2: {release_info['has_avx2']})") - print("=" * 70) - - return None - - def _uninstall_llama_cpp(self): - """Uninstall existing llama-cpp-python""" - print("=" * 70) - print("Uninstalling existing llama-cpp-python...") - print("=" * 70) - - try: - # Use pip uninstall with -y flag to auto-confirm - result = subprocess.run([ - sys.executable, "-m", "pip", "uninstall", "llama-cpp-python", "-y" - ], capture_output=True, text=True) - - if result.returncode == 0: - print("=" * 70) - print("SUCCESS: llama-cpp-python uninstalled successfully") - print("=" * 70) - return True - else: - print("=" * 70) - print(f"Uninstall completed with warnings:") - print(result.stdout) - print(result.stderr) - print("=" * 70) - return True # Still consider it successful - - except Exception as e: - print("=" * 70) - print(f"ERROR: Failed to uninstall llama-cpp-python: {e}") - print("Please manually uninstall using: pip uninstall llama-cpp-python") - print("=" * 70) - return False - - def _install_llama_cpp(self, skip_versions=None): - """Auto-installation: Auto-detect system and download from GitHub releases - - Supports all platforms (Windows/Linux/macOS) and CUDA versions. - Only uses Basic versions (no AVX/AVX2) for maximum compatibility. - - Args: - skip_versions: Set of version tags to skip (previously failed versions) - - Returns: - Tuple of (success: bool, installed_version: str or None) - """ - if skip_versions is None: - skip_versions = set() - - print("=" * 70) - print("Auto-Installing llama-cpp-python") - print("Detecting system configuration and searching through releases...") - print("=" * 70) - - # Get recent releases (up to 15) - releases_list = self._get_github_releases(max_releases=15) - if not releases_list: - print("=" * 70) - print("ERROR: Failed to fetch releases from GitHub") - print("Please visit: https://github.com/JamePeng/llama-cpp-python/releases") - print("=" * 70) - return (False, None) - - # Find matching release and wheel - result = self._find_matching_release(releases_list, skip_versions=skip_versions) - if not result: - print("=" * 70) - print("ERROR: Could not find a matching wheel for your system") - print("Please visit: https://github.com/JamePeng/llama-cpp-python/releases") - print("And manually download the appropriate wheel for your system") - print("=" * 70) - return (False, None) - - wheel_url, version_tag = result - - # Download and install - try: - print("=" * 70) - print("Installing llama-cpp-python...") - print(f"Version: {version_tag}") - print("=" * 70) - - subprocess.check_call([ - sys.executable, "-m", "pip", "install", wheel_url - ]) - - print("=" * 70) - print("SUCCESS: llama-cpp-python installed successfully") - print("IMPORTANT: Please restart ComfyUI to use the GGUF node") - print("=" * 70) - return (True, version_tag) - - except Exception as e: - print("=" * 70) - print(f"ERROR: Failed to install llama-cpp-python: {e}") - print("") - print("Please visit: https://github.com/JamePeng/llama-cpp-python/releases") - print("=" * 70) - return (False, version_tag) - - - def _is_vision_model(self, model_path: str) -> bool: - """Check if model is a vision model based on filename""" - model_name = os.path.basename(model_path).lower() - return 'vl' in model_name - - def _load_model(self, model_path: str, enable_vision: bool = False, mmproj_path: Optional[str] = None, force_reload: bool = False) -> bool: - """Load GGUF model with llama-cpp-python - - Args: - model_path: Path to GGUF model file - enable_vision: Whether to enable vision mode - mmproj_path: Path to mmproj file (for vision models) - force_reload: Force reload even if model is already loaded - """ - try: - # Check if model and mmproj are already loaded - if (not force_reload and - self.model is not None and - self.current_model_path == model_path and - self.current_mmproj_path == mmproj_path): - # Model already loaded, skip verbose output - return True - - # Unload previous model if model or mmproj changed, or force reload - if self.model is not None: - # Unload silently - self._free_memory() - # Wait a bit for memory to be fully released - time.sleep(0.5) - - if not self.llama_cpp_available: - return False - - from llama_cpp import Llama - from llama_cpp.llama_chat_format import Llava15ChatHandler - - # Check if this is a vision model - is_vision_model = self._is_vision_model(model_path) - - # Only print model name for initial load - print(f"Loading: {os.path.basename(model_path)}") - - load_start = time.time() - - # Load model with appropriate settings - load_kwargs = { - "model_path": model_path, - "n_ctx": 8192, - "n_gpu_layers": -1, # Use GPU if available - "verbose": False, - "use_mmap": False, # Disable mmap for clean unload - "use_mlock": False, - } - - # Load vision model if it's a VL model and vision is enabled - if is_vision_model and enable_vision and mmproj_path and mmproj_path != "No mmproj files": - try: - chat_handler = Llava15ChatHandler(clip_model_path=mmproj_path) - load_kwargs["chat_handler"] = chat_handler - self.clip_model_array = chat_handler - except Exception as e: - print(f"⚠️ Failed to load mmproj: {e}") - - self.model = Llama(**load_kwargs) - self.current_model_path = model_path - self.current_mmproj_path = mmproj_path - - load_time = time.time() - load_start - print(f"✓ Loaded ({load_time:.1f}s)") - return True - - except Exception as e: - print(f"⚠️ Load failed: {e}") - - # Clean up all resources on load failure (silently) - self._free_memory() - - # Check if this is a ggml-related error and raise it for handling - if self._is_ggml_error(e): - # Re-raise with ggml indicator for upstream handling - raise RuntimeError(f"GGML_ERROR: {str(e)}") from e - - return False - - def _remove_thinking_tags(self, text: str) -> str: - """Remove ... tags and their content""" - cleaned_text = re.sub(r'.*?', '', text, flags=re.DOTALL) - cleaned_text = re.sub(r'\n\s*\n', '\n', cleaned_text) - return cleaned_text.strip() - - def _detect_excessive_repetition(self, text: str, max_repeat_ratio: float = 0.3) -> Tuple[bool, str]: - """Detect if text has excessive repetition - - Returns: - Tuple of (has_repetition, cleaned_text) - """ - if not text or len(text) < 100: - return (False, text) - - # Split into sentences/lines - lines = [line.strip() for line in text.split('\n') if line.strip()] - - if len(lines) < 3: - return (False, text) - - # Count line repetitions - line_counts = {} - for line in lines: - if len(line) > 10: # Only count substantial lines - line_counts[line] = line_counts.get(line, 0) + 1 - - # Check for excessive repetition - total_lines = len(lines) - max_repeats = max(line_counts.values()) if line_counts else 0 - repeat_ratio = max_repeats / total_lines if total_lines > 0 else 0 - - if repeat_ratio > max_repeat_ratio and max_repeats > 2: - # Remove duplicate lines, keeping only first occurrence - seen = set() - cleaned_lines = [] - for line in lines: - if line not in seen: - cleaned_lines.append(line) - seen.add(line) - elif len(line) <= 10: # Keep short lines (like separators) - cleaned_lines.append(line) - - cleaned_text = '\n'.join(cleaned_lines) - print(f"⚠️ Repetition detected, cleaned {len(lines)} → {len(cleaned_lines)} lines") - return (True, cleaned_text) - - return (False, text) - - def _tensor_to_base64(self, image_tensor) -> str: - """Convert ComfyUI IMAGE tensor to base64 string (single image)""" - try: - from PIL import Image - - # ComfyUI IMAGE format: [B, H, W, C] with values in [0, 1] - # Take first image if batch - if len(image_tensor.shape) == 4: - image_tensor = image_tensor[0] - - # Convert from [0, 1] to [0, 255] - image_np = (image_tensor.cpu().numpy() * 255).astype(np.uint8) - - # Create PIL Image - pil_image = Image.fromarray(image_np) - - # Convert to JPEG bytes - buffered = io.BytesIO() - pil_image.save(buffered, format="JPEG", quality=95) - img_bytes = buffered.getvalue() - - # Encode to base64 - img_base64 = base64.b64encode(img_bytes).decode('utf-8') - - return f"data:image/jpeg;base64,{img_base64}" - - except Exception as e: - print(f"Error converting image to base64: {e}") - import traceback - traceback.print_exc() - return None - - def _tensors_to_base64_list(self, image_tensor) -> List[str]: - """Convert ComfyUI IMAGE tensor(s) to list of base64 strings - - Args: - image_tensor: ComfyUI IMAGE tensor [B, H, W, C] or [H, W, C] - - Returns: - List of base64 encoded image URLs - """ - try: - from PIL import Image - - # Handle single image [H, W, C] - if len(image_tensor.shape) == 3: - image_tensor = image_tensor.unsqueeze(0) # Add batch dimension - - # ComfyUI IMAGE format: [B, H, W, C] with values in [0, 1] - batch_size = image_tensor.shape[0] - base64_list = [] - - for i in range(batch_size): - # Get single image from batch - single_image = image_tensor[i] - - # Convert from [0, 1] to [0, 255] - image_np = (single_image.cpu().numpy() * 255).astype(np.uint8) - - # Create PIL Image - pil_image = Image.fromarray(image_np) - - # Convert to JPEG bytes - buffered = io.BytesIO() - pil_image.save(buffered, format="JPEG", quality=95) - img_bytes = buffered.getvalue() - - # Encode to base64 - img_base64 = base64.b64encode(img_bytes).decode('utf-8') - base64_list.append(f"data:image/jpeg;base64,{img_base64}") - - return base64_list - - except Exception as e: - print(f"Error converting images to base64: {e}") - import traceback - traceback.print_exc() - return None - - def inference( - self, - model: str, - prompt: str, - prompt_template: str, - system_prompt: str, - max_tokens: int, - temperature: float, - top_p: float, - top_k: int, - seed: int = 0, - keep_model_loaded: bool = False, - mmproj_file: str = "No mmproj files", - image_1 = None, - image_2 = None, - image_3 = None, - auto_install_llama_cpp: bool = True, - ) -> Tuple[str, int]: - """Execute GGUF model inference""" - - # Check if llama-cpp-python is available - if not self.llama_cpp_available: - if auto_install_llama_cpp: - print("llama-cpp-python not found, attempting auto-installation...") - success, version = self._install_llama_cpp(skip_versions=set(self.failed_versions)) - if success: - error_msg = "llama-cpp-python installed successfully!\n\nPlease restart ComfyUI to use the GGUF node." - else: - error_msg = "Failed to install llama-cpp-python.\n\nPlease visit: https://github.com/JamePeng/llama-cpp-python/releases" - else: - error_msg = "ERROR: llama-cpp-python is not installed.\n\nPlease either:\n1. Enable 'auto_install_llama_cpp' option (recommended)\n2. Install manually: pip install llama-cpp-python" - print(error_msg) - return (error_msg, seed) - - # Check if model is a suggested download - if model in SUGGESTED_MODELS: - download_url = SUGGESTED_MODELS[model] - filename = os.path.basename(download_url) - - # Get clip folder path for downloading - try: - clip_paths = folder_paths.get_folder_paths("clip") - if clip_paths and len(clip_paths) > 0: - download_dir = clip_paths[0] - else: - error_msg = "Error: Cannot find clip folder for downloading model." - print(error_msg) - return (error_msg, seed) - except: - error_msg = "Error: Cannot access clip folder for downloading model." - print(error_msg) - return (error_msg, seed) - - model_path = os.path.join(download_dir, filename) - - # Check if file already exists - if not os.path.exists(model_path): - print(f"Downloading model...") - if not self._download_file(download_url, model_path): - error_msg = f"⚠️ Download failed: {download_url}" - return (error_msg, seed) - else: - # Get full path from model name - gguf_files = self._get_gguf_files() - model_path = None - for full_path in gguf_files: - if os.path.basename(full_path) == model: - model_path = full_path - break - - if model_path is None: - error_msg = f"Error: Model not found: {model}\nPlease place GGUF files in text_encoders or clip folder." - print(error_msg) - return (error_msg, seed) - - if not os.path.exists(model_path): - error_msg = f"Error: Model file does not exist: {model_path}" - print(error_msg) - return (error_msg, seed) - - # Check if this is a vision model - is_vision_model = self._is_vision_model(model_path) - - # Collect all provided images into a list - images = [] - if image_1 is not None: - images.append(image_1) - if image_2 is not None: - images.append(image_2) - if image_3 is not None: - images.append(image_3) - - # Auto-detect vision mode: enable if any image is provided and model is VL type - enable_vision = False - if len(images) > 0 and is_vision_model: - enable_vision = True - print(f"📸 Detected {len(images)} image(s) for vision analysis") - elif len(images) > 0 and not is_vision_model: - print(f"⚠️ {len(images)} image(s) ignored (model is text-only)") - - # Get mmproj path if vision is enabled - mmproj_path = None - if enable_vision: - if mmproj_file == "No mmproj files": - print("⚠️ No mmproj file, falling back to text-only mode") - enable_vision = False - else: - # Check if mmproj is a suggested download - if mmproj_file in SUGGESTED_MMPROJ: - download_url = SUGGESTED_MMPROJ[mmproj_file] - filename = os.path.basename(download_url) - - # Get clip folder path for downloading - try: - clip_paths = folder_paths.get_folder_paths("clip") - if clip_paths and len(clip_paths) > 0: - download_dir = clip_paths[0] - else: - print("⚠️ Cannot find clip folder") - enable_vision = False - mmproj_path = None - except: - print("⚠️ Cannot access clip folder") - enable_vision = False - mmproj_path = None - - if enable_vision: - mmproj_path = os.path.join(download_dir, filename) - - # Check if file already exists - if not os.path.exists(mmproj_path): - print(f"Downloading mmproj...") - if not self._download_file(download_url, mmproj_path): - print(f"⚠️ Download failed") - enable_vision = False - mmproj_path = None - else: - # Find full path for mmproj from text_encoders and clip folders - mmproj_files = self._get_mmproj_files() - for full_path in mmproj_files: - if os.path.basename(full_path) == mmproj_file: - mmproj_path = full_path - break - - if mmproj_path is None or not os.path.exists(mmproj_path): - print(f"⚠️ mmproj not found: {mmproj_file}") - enable_vision = False - - # Load model with ggml error handling - load_result = False - ggml_error_occurred = False - - try: - # First attempt to load model - load_result = self._load_model(model_path, enable_vision, mmproj_path) - except RuntimeError as e: - # Check if this is a ggml error - if "GGML_ERROR" in str(e): - ggml_error_occurred = True - print("⚠️ GGML compatibility error detected during model loading") - else: - # Re-raise if not a ggml error - raise - - # If loading failed (but not ggml error), try force reload once - if not load_result and not ggml_error_occurred: - try: - print("⚠️ Retrying with force reload...") - load_result = self._load_model(model_path, enable_vision, mmproj_path, force_reload=True) - except RuntimeError as e: - if "GGML_ERROR" in str(e): - ggml_error_occurred = True - print("⚠️ GGML compatibility error detected during force reload") - else: - raise - - # --- NEW: Clear KV Cache to prevent context overflow on re-runs --- - try: - import llama_cpp - # Attempt to find the low-level context pointer - ctx_ptr = getattr(self.model, 'ctx', None) - if not ctx_ptr and hasattr(self.model, '_ctx'): - ctx_ptr = getattr(self.model._ctx, 'ctx', self.model._ctx) - - if ctx_ptr: - # Strategy A: Modern API (v0.3+) - if hasattr(llama_cpp, 'llama_get_memory'): - mem = llama_cpp.llama_get_memory(ctx_ptr) - llama_cpp.llama_memory_seq_rm(mem, -1, 0, -1) - # Strategy B: Legacy API - elif hasattr(llama_cpp, 'llama_kv_cache_seq_rm'): - llama_cpp.llama_kv_cache_seq_rm(ctx_ptr, -1, 0, -1) - except Exception as e: - print(f"Warning: Could not clear KV cache at start: {e}") - # --------------------------------------------------------------- - - # Handle ggml error with auto-reinstall - if ggml_error_occurred and auto_install_llama_cpp: - print("Attempting automatic recovery from GGML error...") - success, message = self._handle_ggml_error_and_reinstall() - # Return the message to user (success or failure) - return (message, seed, []) - - if not load_result: - # Model loading failed - get the error details from the exception - if auto_install_llama_cpp: - # Check for specific error types - import traceback - error_trace = traceback.format_exc() - - # Check for WinError - has_winerror = 'WinError' in error_trace or 'WindowsError' in error_trace - - # Check for ggml.dll error - has_ggml_dll_error = 'ggml.dll' in error_trace.lower() or 'cannot load library' in error_trace.lower() - - print("=" * 70) - print("ERROR: Model loading failed!") - - if has_ggml_dll_error: - # Special handling for ggml.dll error - error_msg = ( - "ERROR: Cannot load ggml.dll\n\n" - "This error typically occurs when:\n" - "1. Your CUDA version is incompatible with the installed llama-cpp-python\n" - "2. CUDA runtime libraries are missing or outdated\n\n" - "Recommended solutions:\n" - "1. Update your NVIDIA GPU drivers and CUDA toolkit\n" - "2. Or manually install a compatible llama-cpp-python version from:\n" - " https://github.com/JamePeng/llama-cpp-python/releases\n\n" - "The auto-installer will now try to find an older compatible version..." - ) - print(error_msg) - print("=" * 70) - - # Get current installed version info - try: - import pkg_resources - current_version = pkg_resources.get_distribution("llama-cpp-python").version - print(f"Current llama-cpp-python version: {current_version}") - except: - current_version = None - - if has_winerror or has_ggml_dll_error: - print("Detected compatibility issue - will try previous version") - print("Attempting to uninstall and install previous version...") - else: - print("This may indicate llama-cpp-python is incompatible or corrupted.") - print("Attempting to uninstall and reinstall llama-cpp-python...") - print("=" * 70) - - # Try to uninstall and reinstall with previous version - if self._uninstall_llama_cpp(): - print("\nNow attempting to install compatible version...") - - # Try to install, skipping failed versions - success, new_version = self._install_llama_cpp(skip_versions=set(self.failed_versions)) - - if success and new_version: - # Track this version if it was just installed - if new_version not in self.failed_versions: - self.failed_versions.append(new_version) - - error_msg = f"llama-cpp-python has been reinstalled (version: {new_version})!\n\nIMPORTANT: Please restart ComfyUI to use the new version." - print("=" * 70) - print(error_msg) - print("=" * 70) - return (error_msg, seed) - else: - # Installation failed - add to failed versions if we got a version - if new_version and new_version not in self.failed_versions: - self.failed_versions.append(new_version) - - error_msg = ( - "Failed to install compatible llama-cpp-python version.\n\n" - "Please visit: https://github.com/JamePeng/llama-cpp-python/releases\n" - "And manually install the appropriate version for your system.\n\n" - ) - if has_ggml_dll_error: - error_msg += ( - "Note: For ggml.dll errors, ensure your CUDA toolkit is up to date:\n" - "- Update NVIDIA GPU drivers\n" - "- Install latest CUDA toolkit from: https://developer.nvidia.com/cuda-downloads" - ) - else: - error_msg = ( - "Failed to uninstall llama-cpp-python.\n\n" - "Please manually uninstall using: pip uninstall llama-cpp-python\n" - "Then enable auto_install_llama_cpp option to reinstall." - ) - else: - error_msg = ( - "Error: Model loading failed.\n\n" - "This may indicate llama-cpp-python is incompatible.\n" - "Please enable 'auto_install_llama_cpp' to automatically fix this,\n" - "or manually reinstall llama-cpp-python." - ) - - print(error_msg) - return (error_msg, seed) - - try: - # Load and apply template - template_content = "" - if prompt_template != "Custom": - template_content = self._load_template_content(prompt_template) - - # If template content exists, replace system_prompt with template - if template_content: - system_prompt = template_content - - # Prepare messages - messages = [] - if system_prompt and system_prompt.strip(): - messages.append({"role": "system", "content": system_prompt}) - - # Add image(s) if vision is enabled and model supports it - if enable_vision and is_vision_model and len(images) > 0: - # Build content with text first - content = [{"type": "text", "text": prompt}] - - # Convert each image to base64 and add to content - for idx, img in enumerate(images, 1): - # Convert single image tensor to base64 - image_url = self._tensor_to_base64(img) - if image_url is None: - error_msg = f"Error: Failed to convert image_{idx} to base64 format." - print(error_msg) - return (error_msg, seed) - content.append({"type": "image_url", "image_url": image_url}) - - # Log number of images being processed - print(f"🎬 Processing {len(images)} image(s) with vision model") - - messages.append({ - "role": "user", - "content": content - }) - else: - messages.append({"role": "user", "content": prompt}) - - # Run inference (silently) - inference_start = time.time() - - # Prepare generation parameters - # Set repeat_penalty internally to prevent repetitive output (1.15 is a good balance) - gen_params = { - "messages": messages, - "max_tokens": max_tokens, - "temperature": temperature, - "top_p": top_p, - "top_k": top_k, - "repeat_penalty": 1.15, # Internal setting to reduce repetition - "stop": ["<|im_end|>", "<|endoftext|>", "", "\n\n\n"], # Common stop sequences - } - - # Only add seed if it's set (> 0) - if seed > 0: - gen_params["seed"] = seed - - response = self.model.create_chat_completion(**gen_params) - - # Extract response text - response_text = response["choices"][0]["message"]["content"] - - # Remove thinking tags - response_text = self._remove_thinking_tags(response_text) - - # Detect and clean excessive repetition - has_repetition, response_text = self._detect_excessive_repetition(response_text) - - inference_time = time.time() - inference_start - tokens_generated = response["usage"]["completion_tokens"] - tokens_per_sec = tokens_generated / inference_time if inference_time > 0 else 0 - - print(f"✓ Done ({inference_time:.1f}s, {tokens_generated} tokens, {tokens_per_sec:.0f}t/s)") - - # Always free image-related memory after inference - if len(images) > 0: - self._free_image_memory() - - # Unload model if requested - if not keep_model_loaded: - self._free_memory() - - # 自動提取提示詞列表（如果輸出包含 JSON 格式的雜誌數據） - try: - prompts = extract_prompts_from_json(response_text) - if prompts: - print(f"✅ 自動提取到 {len(prompts)} 個圖片提示詞") - except Exception as e: - print(f"ℹ️ 提示詞提取失敗（這是正常的，如果輸出不是雜誌 JSON）: {e}") - prompts = [] - - return (response_text, seed, prompts) - - except Exception as e: - import traceback - error_msg = f"⚠️ Inference failed: {str(e)}" - print(error_msg) - - # Clean up resources on error (silently) - if len(images) > 0: - self._free_image_memory() - - # If error suggests memory/loading issue, unload model completely - error_str = str(e).lower() - if any(keyword in error_str for keyword in ['memory', 'load', 'allocation', 'cuda', 'out of memory']): - self._free_memory() - - return (error_msg, seed, []) # 錯誤時返回空的提示詞列表