Fix HF tokenizer backend fallback

AILab_QwenVL_PromptEnhancer.py

@@ -13,6 +13,8 @@ from enum import Enum
 from pathlib import Path
 
 import torch
+import subprocess
+import sys
 from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 
 from AILab_OutputCleaner import OutputCleanConfig, clean_model_output, prompt_output_guard
@@ -347,7 +349,35 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
             load_kwargs["torch_dtype"] = torch.float16 if device == "cuda" else torch.float32
 
         print(f"[QwenVL] Loading text model {model_name} ({quantization})")
-        self.text_tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
+        tokenizer_kwargs = {"trust_remote_code": True}
+        try:
+            self.text_tokenizer = AutoTokenizer.from_pretrained(repo_id, **tokenizer_kwargs)
+        except ValueError as exc:
+            message = str(exc)
+            missing_backend = "sentencepiece or tiktoken" in message.lower()
+            if missing_backend:
+                print("[QwenVL] Tokenizer backend missing; installing sentencepiece and tiktoken into the active ComfyUI Python environment.")
+                install_cmd = [
+                    sys.executable,
+                    "-m",
+                    "pip",
+                    "install",
+                    "--upgrade",
+                    "sentencepiece",
+                    "tiktoken",
+                ]
+                result = subprocess.run(install_cmd, capture_output=True, text=True)
+                if result.returncode != 0:
+                    stdout = (result.stdout or "").strip()
+                    stderr = (result.stderr or "").strip()
+                    raise RuntimeError(
+                        "[QwenVL] Failed to install tokenizer backends automatically. "
+                        f"Command: {' '.join(install_cmd)}\nstdout:\n{stdout or '<empty>'}\nstderr:\n{stderr or '<empty>'}"
+                    ) from exc
+            try:
+                self.text_tokenizer = AutoTokenizer.from_pretrained(repo_id, use_fast=False, **tokenizer_kwargs)
+            except Exception:
+                raise exc
         self.text_model = AutoModelForCausalLM.from_pretrained(repo_id, trust_remote_code=True, **load_kwargs).eval()
         self.text_model.to(device)
         ensure_cuda_vram_headroom("QwenVL PromptEnhancer HF", min_free_gb=1.0, min_free_ratio=0.08)

README.md

@@ -52,6 +52,8 @@ This is the recommended default path.
 
 The HF path is the most straightforward option for Windows and Linux when you want the fewest install complications.
 
+HF tokenizer note: some Gemma and Qwen variants need `sentencepiece` or `tiktoken` available in the ComfyUI Python environment. ThinkingLLM now declares those dependencies and will attempt to install them automatically if a tokenizer backend is missing at first load.
+
 ### GGUF / llama.cpp nodes
 
 - `ThinkingLLM (GGUF)`

__init__.py

@@ -3,7 +3,7 @@
     "name": "ComfyUI-ThinkingLLM",
     "description": "ThinkingLLM multimodal fork with HF and GGUF paths. GPL-3.0 lineage preserved from Deaquay/ComfyUI-Qwen3.5-Uncensored, huchukato/ComfyUI-QwenVL-Mod, and 1038lab/ComfyUI-QwenVL.",
     "author": "goodguy1963",
-    "version": "2.2.7",
+    "version": "2.2.8",
     "url": "https://github.com/goodguy1963/ComfyUI-ThinkingLLM",
   "category": "image"
 }

custom_nodes.json

@@ -9,5 +9,5 @@
   "description": "A multimodal ComfyUI AI node with Qwen3.5, Qwen3-VL, Qwen2.5-VL, Qwen3, and Gemma 4 integrations. Features live thinking in the terminal to see what the LLM is doing in real time.",
   "category": "image",
   "tags": ["vision", "language", "multimodal", "thinkingllm", "qwen", "gguf", "flash_attention", "performance", "wan2.2", "video", "i2v", "t2v", "storyboard", "caching", "fork", "gpl-3.0"],
-  "version": "2.2.7"
+  "version": "2.2.8"
 }

pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "ComfyUI-ThinkingLLM"
 description = "ThinkingLLM multimodal ComfyUI fork with Hugging Face and GGUF backends, WAN 2.2 workflows, prompt tooling, and preserved GPL fork lineage from the Qwen3.5-Uncensored branch."
-version = "2.2.7"
+version = "2.2.8"
 license = {file = "LICENSE"}
-dependencies = ["transformers>=5.2.0", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "llama-cpp-python"]
+dependencies = ["transformers>=5.2.0", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "sentencepiece", "tiktoken", "llama-cpp-python"]
 
 [project.urls]
 Repository = "https://github.com/goodguy1963/ComfyUI-ThinkingLLM"

requirements.txt

@@ -11,6 +11,8 @@ Pillow
 opencv-python
 bitsandbytes
 accelerate
+sentencepiece
+tiktoken
 
 # For running GGUF models with vision support
 # The [server] extra installs necessary components for multimodal models