From de5d3e5da382d2bf467c5100d18f3d97b129a12c Mon Sep 17 00:00:00 2001 From: Marcel Hofer <49859266+goodguy1963@users.noreply.github.com> Date: Tue, 9 Jun 2026 00:21:54 +0200 Subject: [PATCH] Release 2.3.5 audio model updates --- AILab_LlamaCppInstaller.py | 4 +- AILab_QwenVL.py | 11 +- AILab_QwenVL_GGUF.py | 557 +++++++++++++++++++++++++++++---- AILab_QwenVL_PromptEnhancer.py | 3 +- AILab_WhisperASR.py | 262 ++++++++++++++++ README.md | 45 ++- __init__.py | 4 +- custom_nodes.json | 2 +- gguf_models.json | 44 ++- hf_models.json | 24 +- pyproject.toml | 4 +- requirements.txt | 1 + tests/test_new_features.py | 398 +++++++++++++++++++++++ web/js/appearance.js | 48 ++- web/model_recommendations.json | 60 +++- 15 files changed, 1386 insertions(+), 81 deletions(-) create mode 100644 AILab_WhisperASR.py diff --git a/AILab_LlamaCppInstaller.py b/AILab_LlamaCppInstaller.py index c6fd163..ea20a58 100644 --- a/AILab_LlamaCppInstaller.py +++ b/AILab_LlamaCppInstaller.py @@ -45,7 +45,7 @@ KNOWN_LINUX_WHEEL_SPECS = { "llama_cpp_python-0.3.34+cu130.basic-cp312-cp312-linux_x86_64.whl" ), } -VISION_HANDLER_NAMES = ("Qwen3VLChatHandler", "Qwen25VLChatHandler") +VISION_HANDLER_NAMES = ("Qwen3VLChatHandler", "Qwen25VLChatHandler", "Gemma4ChatHandler", "MTMDChatHandler") _LAST_BACKEND_INFO: dict | None = None _PRINTED_BACKEND_LINES: set[str] = set() @@ -460,4 +460,4 @@ def ensure_llama_cpp_backend(require_vision_handlers: bool = False): "[QwenVL] llama_cpp was installed, but the backend is still incompatible. " "Set THINKINGLLM_LLAMA_CPP_LINUX_WHEEL_URL to a matching JamePeng wheel or " "THINKINGLLM_LLAMA_CPP_LINUX_SPEC to an explicit package spec, then restart ComfyUI." - ) from second_exc \ No newline at end of file + ) from second_exc diff --git a/AILab_QwenVL.py b/AILab_QwenVL.py index e57a686..0949cd9 100644 --- a/AILab_QwenVL.py +++ b/AILab_QwenVL.py @@ -901,6 +901,13 @@ if not HF_ALL_MODELS: load_model_configs() +def _default_model_from_config(models: dict[str, dict], fallback: str) -> str: + for name, info in models.items(): + if isinstance(info, dict) and info.get("default"): + return name + return next(iter(models.keys()), fallback) + + def read_hf_model_type(model_dir: str) -> str | None: """Read model_type from a HuggingFace model's config.json.""" try: @@ -1895,7 +1902,7 @@ class ThinkingLLM_QwenVL(QwenVLBase): @classmethod def INPUT_TYPES(cls): models = list(HF_VL_MODELS.keys()) - default_model = models[0] if models else "Qwen3-VL-4B-Instruct" + default_model = _default_model_from_config(HF_VL_MODELS, "Qwen3-VL-4B-Instruct") prompts = PRESET_PROMPTS or [NO_PRESET_PROMPT, "Describe this image in detail."] preferred_prompt = "🖼️ Detailed Description" default_prompt = preferred_prompt if preferred_prompt in prompts else prompts[0] @@ -1943,7 +1950,7 @@ class ThinkingLLM_QwenVL_Advanced(QwenVLBase): @classmethod def INPUT_TYPES(cls): models = list(HF_VL_MODELS.keys()) - default_model = models[0] if models else "Qwen3-VL-4B-Instruct" + default_model = _default_model_from_config(HF_VL_MODELS, "Qwen3-VL-4B-Instruct") prompts = PRESET_PROMPTS or [NO_PRESET_PROMPT, "Describe this image in detail."] preferred_prompt = "🖼️ Detailed Description" default_prompt = preferred_prompt if preferred_prompt in prompts else prompts[0] diff --git a/AILab_QwenVL_GGUF.py b/AILab_QwenVL_GGUF.py index e3666ec..b72a416 100644 --- a/AILab_QwenVL_GGUF.py +++ b/AILab_QwenVL_GGUF.py @@ -16,10 +16,14 @@ import io import inspect import json import os +import re +import shutil import struct +import subprocess import sys import time import weakref +import wave from dataclasses import dataclass from pathlib import Path @@ -350,6 +354,10 @@ def _model_name_to_filename_candidates(model_name: str) -> set[str]: return candidates +def _is_removed_qwen3_asr_filename(filename: str) -> bool: + return "qwen3-asr" in str(filename or "").lower() + + def _scan_local_gguf_models(base_dir: Path, existing_filenames: set[str]) -> dict[str, dict]: """Scan the GGUF base directory for locally available .gguf files not already in the JSON catalog.""" local_models: dict[str, dict] = {} @@ -375,6 +383,8 @@ def _scan_local_gguf_models(base_dir: Path, existing_filenames: set[str]) -> dic mmproj_path = mmproj_files[0] if mmproj_files else None for model_file in model_files: + if _is_removed_qwen3_asr_filename(model_file.name): + continue # Skip if this filename is already in the JSON catalog if model_file.name in existing_filenames: continue @@ -427,6 +437,7 @@ def _load_gguf_vl_catalog(): defaults = repo.get("defaults") or {} mmproj_file = repo.get("mmproj_file") + mmproj_files = repo.get("mmproj_files") or {} model_files = repo.get("model_files") or [] quant_sizes = _parse_repo_quant_sizes(repo_key) @@ -446,13 +457,40 @@ def _load_gguf_vl_catalog(): "repo_id": repo_id, "alt_repo_ids": alt_repo_ids, "filename": model_file, - "mmproj_filename": mmproj_file, + "mmproj_filename": mmproj_files.get(model_file) or mmproj_file, } legacy_models = data.get("models") or {} for name, entry in legacy_models.items(): if isinstance(entry, dict): - flattened[name] = entry + model_files = entry.get("model_files") or [] + if model_files and not entry.get("filename"): + defaults = entry.get("defaults") or {} + mmproj_file = entry.get("mmproj_file") + mmproj_files = entry.get("mmproj_files") or {} + author = entry.get("author") or entry.get("publisher") + repo_name = entry.get("repo_name") or entry.get("repo_name_override") or name + repo_id = entry.get("repo_id") + alt_repo_ids = entry.get("alt_repo_ids") or [] + quant_sizes = _parse_repo_quant_sizes(name) + for model_file in model_files: + display = Path(model_file).name + q = _quant_from_filename(model_file) + if q and q in quant_sizes: + display = f"{display} [~{quant_sizes[q]}]" + if display in flattened: + display = f"{display} ({name})" + flattened[display] = { + **defaults, + "author": author, + "repo_dirname": repo_name, + "repo_id": repo_id, + "alt_repo_ids": alt_repo_ids, + "filename": model_file, + "mmproj_filename": mmproj_files.get(model_file) or mmproj_file, + } + else: + flattened[name] = entry # Scan filesystem for locally available models not in JSON config # Collect all directories to scan: the configured base_dir + any extra LLM paths from ComfyUI @@ -483,6 +521,8 @@ GGUF_VL_CATALOG = _load_gguf_vl_catalog() GGUF_TOOLTIPS = { "model_name": "GGUF vision model from gguf_models.json or auto-detected local files. First run downloads the selected GGUF and mmproj files when they are not already on disk.", + "audio_model_name": "Gemma 4 audio-capable GGUF model. Only Gemma 4 E2B, E4B, and 12B are listed; 26B/31B variants are image/text-only for this purpose.", + "audio_file_path": "Optional local audio file path. M4A, MP3, WAV, FLAC, and other FFmpeg-readable files are decoded to 16 kHz mono WAV before inference.", "device": "auto prefers CUDA when PyTorch sees an NVIDIA GPU. If RAW_TRACE says GPU offload is no or unknown, verify your llama-cpp-python CUDA wheel before blaming the model.", "max_tokens": "Maximum new tokens to generate. Larger values give more room for reasoning but increase runtime and memory use.", "keep_model_loaded": "Keep the GGUF model in RAM/VRAM after the run so repeated prompts skip model loading. Disable if you need memory back for other nodes.", @@ -504,6 +544,32 @@ GGUF_TOOLTIPS = { "hf_token": "Optional Hugging Face access token for private or gated GGUF/mmproj downloads. It is passed only to the download call, never logged or cached, and the in-memory copy is dropped after the download attempt. Clear this field before saving or sharing workflows.", } +GEMMA4_AUDIO_DEFAULT_PROMPT = ( + "Audio analysis: transcribe the speech in the original language, then summarize the important points. " + "If there is no clear speech, describe the audible scene and relevant sounds." +) + +GGUF_AUDIO_TARGET_SAMPLE_RATE = 16000 + + +def _is_gemma4_audio_model_name(model_name: str) -> bool: + lowered = str(model_name or "").lower() + if "gemma-4" not in lowered and "gemma4" not in lowered: + return False + if "26b" in lowered or "31b" in lowered or "a4b" in lowered: + return False + return any(token in lowered for token in ("12b", "e2b", "e4b")) + + +def _gguf_audio_model_keys() -> list[str]: + all_models = GGUF_VL_CATALOG.get("models") or {} + keys = [ + key + for key, entry in all_models.items() + if (entry or {}).get("mmproj_filename") and _is_gemma4_audio_model_name(key) + ] + return sorted(keys) + def _filter_kwargs_for_callable(fn, kwargs: dict) -> dict: try: @@ -522,6 +588,239 @@ def _filter_kwargs_for_callable(fn, kwargs: dict) -> dict: return {k: v for k, v in kwargs.items() if k in allowed} +def _unexpected_kwarg_names_from_type_error(exc: TypeError) -> list[str]: + message = str(exc or "") + matches = re.findall(r"unexpected keyword argument(?:\(s\))?\s+'([^']+)'", message) + names: list[str] = [] + for match in matches: + for name in str(match).split(","): + cleaned = name.strip() + if cleaned: + names.append(cleaned) + return names + + +def _filter_mmproj_handler_kwargs(handler_cls, kwargs: dict) -> dict: + filtered = _filter_kwargs_for_callable(getattr(handler_cls, "__init__", handler_cls), kwargs) + handler_name = getattr(handler_cls, "__name__", "") + if "MTMD" in handler_name: + filtered.pop("force_reasoning", None) + return filtered + + +def _select_mmproj_handler_class(model_name: str, arch: str | None): + """Pick the llama-cpp-python multimodal chat handler for the selected model.""" + try: + from llama_cpp import llama_chat_format + except ImportError as exc: + raise RuntimeError( + "[QwenVL] Missing llama_cpp chat handlers. Install a multimodal-capable llama-cpp-python build. " + "See docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md" + ) from exc + + name = (model_name or "").lower() + architecture = (arch or "").lower() + if "gemma-4" in name or "gemma4" in name or architecture == "gemma4": + preferred = ("Gemma4ChatHandler",) + elif "qwen2.5" in name or "qwen25" in name or architecture in ("qwen2vl", "qwen25vl"): + preferred = ("Qwen25VLChatHandler", "Qwen3VLChatHandler") + else: + preferred = ("Qwen3VLChatHandler", "Qwen25VLChatHandler") + + for handler_name in preferred: + handler_cls = getattr(llama_chat_format, handler_name, None) + if handler_cls is not None: + return handler_cls + + raise RuntimeError( + "[QwenVL] Missing multimodal chat handler for this model. " + f"Tried: {', '.join(preferred)}. Install/update llama-cpp-python." + ) + + +def _resample_mono_audio(samples, source_rate: int, target_rate: int = GGUF_AUDIO_TARGET_SAMPLE_RATE): + samples = np.asarray(samples, dtype=np.float32).reshape(-1) + if samples.size == 0 or source_rate == target_rate: + return samples + target_length = max(1, int(round(float(samples.size) * float(target_rate) / float(source_rate)))) + if target_length == samples.size: + return samples + if samples.size == 1: + return np.full((target_length,), float(samples[0]), dtype=np.float32) + old_positions = np.linspace(0.0, float(samples.size - 1), num=samples.size, dtype=np.float64) + new_positions = np.linspace(0.0, float(samples.size - 1), num=target_length, dtype=np.float64) + return np.interp(new_positions, old_positions, samples).astype(np.float32) + + +def _clean_audio_file_path(audio_file_path) -> Path | None: + if audio_file_path is None: + return None + raw_path = str(audio_file_path).strip().strip("\"'") + if not raw_path: + return None + expanded = os.path.expandvars(raw_path) + return Path(expanded).expanduser() + + +def _audio_file_path_to_wav_base64(audio_file_path) -> list[str]: + path = _clean_audio_file_path(audio_file_path) + if path is None: + return [] + if not path.exists() or not path.is_file(): + raise FileNotFoundError(f"audio_file_path does not exist or is not a file: {path}") + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + raise RuntimeError("ffmpeg was not found on PATH; install FFmpeg or connect a decoded ComfyUI AUDIO input") + + result = subprocess.run( + [ + ffmpeg, + "-hide_banner", + "-v", + "error", + "-i", + str(path), + "-vn", + "-ac", + "1", + "-ar", + str(GGUF_AUDIO_TARGET_SAMPLE_RATE), + "-acodec", + "pcm_s16le", + "-f", + "s16le", + "pipe:1", + ], + check=False, + capture_output=True, + ) + if result.returncode != 0: + stderr = result.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(f"ffmpeg failed to decode audio_file_path: {stderr or 'unknown ffmpeg error'}") + if not result.stdout: + raise RuntimeError("ffmpeg decoded no audio samples from audio_file_path") + + if len(result.stdout) % 2 != 0: + raise RuntimeError("ffmpeg produced malformed 16-bit PCM audio") + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(GGUF_AUDIO_TARGET_SAMPLE_RATE) + wav.writeframes(result.stdout) + return [base64.b64encode(buffer.getvalue()).decode("ascii")] + + +def _audio_to_wav_base64(audio) -> list[str]: + if audio is None: + return [] + try: + if not isinstance(audio, dict): + raise ValueError("expected ComfyUI AUDIO dict with waveform/sample_rate") + waveform = audio.get("waveform") + if waveform is None: + raise ValueError("missing waveform") + sample_rate = int(audio.get("sample_rate") or 0) + if sample_rate <= 0: + raise ValueError(f"invalid sample_rate={audio.get('sample_rate')!r}") + tensor = waveform.detach().cpu() if hasattr(waveform, "detach") else waveform + tensor = tensor.cpu() if hasattr(tensor, "cpu") else tensor + array = tensor.numpy() if hasattr(tensor, "numpy") else np.asarray(tensor) + array = np.asarray(array, dtype=np.float32) + if array.size == 0: + raise ValueError("empty waveform") + if array.ndim == 3: + array = array[0] + if array.ndim == 1: + array = array[None, :] + if array.ndim != 2: + raise ValueError(f"unsupported waveform shape={getattr(array, 'shape', None)}") + if array.shape[0] > array.shape[-1]: + array = array.T + if not np.all(np.isfinite(array)): + array = np.nan_to_num(array, nan=0.0, posinf=1.0, neginf=-1.0) + mono = np.mean(array, axis=0) + mono = _resample_mono_audio(mono, sample_rate, GGUF_AUDIO_TARGET_SAMPLE_RATE) + pcm = np.clip(mono, -1.0, 1.0) + pcm16 = (pcm * 32767.0).astype(np.int16) + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(GGUF_AUDIO_TARGET_SAMPLE_RATE) + wav.writeframes(pcm16.tobytes()) + return [base64.b64encode(buffer.getvalue()).decode("ascii")] + except Exception as exc: + print(f"[QwenVL] Warning: failed to encode AUDIO input as WAV; audio will be ignored. Cause: {exc}") + return [] + + +def _audio_inputs_to_wav_base64(audio=None, audio_file_path="") -> tuple[list[str], list[str]]: + items: list[str] = [] + notes: list[str] = [] + if audio is not None: + converted = _audio_to_wav_base64(audio) + if converted: + items.extend(converted) + else: + notes.append("Connected ComfyUI AUDIO input could not be converted to 16 kHz WAV.") + + clean_path = _clean_audio_file_path(audio_file_path) + if clean_path is not None: + try: + converted = _audio_file_path_to_wav_base64(str(clean_path)) + if converted: + items.extend(converted) + print(f"[QwenVL] Audio file decoded: {clean_path}") + except Exception as exc: + note = f"audio_file_path failed: {exc}" + notes.append(note) + print(f"[QwenVL] Warning: {note}") + + return items, notes + + +def _get_audio_hash(audio) -> str | None: + if audio is None or not isinstance(audio, dict): + return None + waveform = audio.get("waveform") + sample_rate = audio.get("sample_rate") + if waveform is None: + return None + try: + shape = tuple(getattr(waveform, "shape", ())) + tensor = waveform.detach().cpu() if hasattr(waveform, "detach") else waveform + array = tensor.numpy() if hasattr(tensor, "numpy") else np.asarray(tensor) + sample = array.reshape(-1)[:32].tolist() + content = json.dumps({"shape": shape, "sample_rate": sample_rate, "sample": sample}, sort_keys=True) + return hashlib.md5(content.encode()).hexdigest()[:16] + except Exception: + return hashlib.md5(f"{sample_rate}|{getattr(waveform, 'shape', '')}".encode()).hexdigest()[:16] + + +def _get_audio_file_hash(audio_file_path) -> str | None: + path = _clean_audio_file_path(audio_file_path) + if path is None: + return None + try: + stat = path.stat() + content = f"{path.resolve()}|{stat.st_size}|{stat.st_mtime_ns}" + return hashlib.md5(content.encode("utf-8", errors="replace")).hexdigest()[:16] + except Exception: + return hashlib.md5(str(path).encode("utf-8", errors="replace")).hexdigest()[:16] + + +def _missing_audio_result(notes: list[str] | None = None) -> tuple[str, str]: + message = ( + "[QwenVL GGUF] No usable audio input. Connect a decoded ComfyUI AUDIO output " + "or set audio_file_path to an existing M4A/MP3/WAV/FLAC file." + ) + trace_lines = ["[AUDIO]", message] + if notes: + trace_lines.extend(str(note) for note in notes if note) + return message, "\n".join(trace_lines) + + def _env_bool(name: str, default: bool = False) -> bool: value = os.getenv(name) if value is None: @@ -762,11 +1061,27 @@ class QwenVLGGUFBase: def _create_chat_completion(self, *, enable_thinking: bool, **kwargs): self._sync_live_chat_template_kwargs(enable_thinking) + kwargs = dict(kwargs) if self._uses_chat_template_thinking(): - kwargs = dict(kwargs) kwargs["chat_template_kwargs"] = {"enable_thinking": bool(enable_thinking)} - kwargs = _filter_kwargs_for_callable(getattr(self.llm, "create_chat_completion"), kwargs) - return self.llm.create_chat_completion(**kwargs) + completion_fn = getattr(self.llm, "create_chat_completion") + kwargs = _filter_kwargs_for_callable(completion_fn, kwargs) + + for _ in range(4): + try: + return completion_fn(**kwargs) + except TypeError as exc: + unexpected = [name for name in _unexpected_kwarg_names_from_type_error(exc) if name in kwargs] + if not unexpected: + raise + for name in unexpected: + kwargs.pop(name, None) + print( + "[QwenVL GGUF] llama-cpp-python rejected chat completion kwarg(s): " + f"{', '.join(unexpected)}; retrying without them." + ) + + return completion_fn(**kwargs) def clear(self): print(f"[QwenVL GGUF DEBUG] Starting VRAM cleanup...") @@ -979,22 +1294,16 @@ class QwenVLGGUFBase: torch.cuda.synchronize() time.sleep(0.1) # Brief pause for cleanup to complete + # Detect architecture from GGUF metadata before selecting a multimodal handler. + arch = read_gguf_architecture(model_path) + self.gguf_arch = arch + is_qwen35 = arch in ("qwen35", "qwen35moe") if arch else "qwen3.5-" in model_name.lower() + self.supports_qwen_soft_think = arch == "qwen3" if arch else "qwen3-" in model_name.lower() + self.uses_qwen_template_thinking = bool(is_qwen35 or arch == "qwen3" or arch == "qwen") + self.chat_handler = None if has_mmproj: - handler_cls = None - try: - from llama_cpp.llama_chat_format import Qwen3VLChatHandler - - handler_cls = Qwen3VLChatHandler - except ImportError: - try: - from llama_cpp.llama_chat_format import Qwen25VLChatHandler - - handler_cls = Qwen25VLChatHandler - except ImportError: - raise RuntimeError( - "[QwenVL] Missing Qwen VL chat handler in llama_cpp. Install the correct fork/wheel. See docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md" - ) + handler_cls = _select_mmproj_handler_class(model_name, arch) mmproj_kwargs = { "clip_model_path": str(mmproj_path), @@ -1002,7 +1311,7 @@ class QwenVLGGUFBase: "force_reasoning": False, "verbose": False, } - mmproj_kwargs = _filter_kwargs_for_callable(getattr(handler_cls, "__init__", handler_cls), mmproj_kwargs) + mmproj_kwargs = _filter_mmproj_handler_kwargs(handler_cls, mmproj_kwargs) if "image_max_tokens" not in mmproj_kwargs: print( "[QwenVL] Warning: installed llama_cpp chat handler does not support image_max_tokens; " @@ -1028,12 +1337,6 @@ class QwenVLGGUFBase: "ctx_checkpoints": ctx_checkpoints_val, } - # Detect architecture from GGUF metadata instead of relying on model name - arch = read_gguf_architecture(model_path) - self.gguf_arch = arch - is_qwen35 = arch in ("qwen35", "qwen35moe") if arch else "qwen3.5-" in model_name.lower() - self.supports_qwen_soft_think = arch == "qwen3" if arch else "qwen3-" in model_name.lower() - self.uses_qwen_template_thinking = bool(is_qwen35 or arch == "qwen3" or arch == "qwen") # Thinking toggle: for Qwen3.5/4 we control enable_thinking via the chat template kwarg. # For non-Qwen backends (Gemma, LLaMA) the flag is advisory and may be silently ignored. if self.uses_qwen_template_thinking: @@ -1115,6 +1418,7 @@ class QwenVLGGUFBase: system_prompt: str, user_prompt: str, images_b64: list[str], + audio_b64: list[str] | None, max_tokens: int, temperature: float, top_p: float, @@ -1142,6 +1446,7 @@ class QwenVLGGUFBase: system_prompt_text: str, user_prompt_text: str, images_for_call: list[str], + audio_for_call: list[str], *, stage_label: str, seed_value: int, @@ -1153,12 +1458,18 @@ class QwenVLGGUFBase: current_enable_thinking, supports_soft_switch=supports_soft_think, ) - if images_for_call: + if effective_user_prompt is None: + effective_user_prompt = user_prompt_text + if images_for_call or audio_for_call: content = [{"type": "text", "text": effective_user_prompt}] for img in images_for_call: if not img: continue content.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}) + for aud in audio_for_call: + if not aud: + continue + content.append({"type": "input_audio", "input_audio": {"data": aud, "format": "wav"}}) messages = [ {"role": "system", "content": system_prompt_text}, {"role": "user", "content": content}, @@ -1170,6 +1481,7 @@ class QwenVLGGUFBase: ] start = time.perf_counter() + stop_tokens = ["<|im_end|>", "<|im_start|>"] sampler_kwargs = { "top_k": int(top_k) if top_k is not None else 60, "min_p": 0.01, @@ -1191,7 +1503,7 @@ class QwenVLGGUFBase: top_p=float(top_p), repeat_penalty=float(repetition_penalty), seed=int(seed_value), - stop=["<|im_end|>", "<|im_start|>"], + stop=stop_tokens, stream=True, **sampler_kwargs, ) @@ -1221,7 +1533,13 @@ class QwenVLGGUFBase: if stopped_for_degenerate_stream: print(f"[QwenVL GGUF] stopped repeated-token loop: {stopped_for_degenerate_stream}") cleaned = clean_model_output(raw_full, OutputCleanConfig(mode="text")) - return cleaned.strip(), raw_full + raw_for_trace = raw_full + if stopped_for_degenerate_stream: + raw_for_trace = ( + f"[stopped repeated-token loop: {stopped_for_degenerate_stream}]\n" + f"{raw_full}" + ).strip() + return cleaned.strip(), raw_for_trace # Thread-based interrupt: always stream, poll interrupt on main thread import threading, queue as qmod @@ -1243,7 +1561,7 @@ class QwenVLGGUFBase: top_p=float(top_p), repeat_penalty=float(repetition_penalty), seed=int(seed_value), - stop=["<|im_end|>", "<|im_start|>"], + stop=stop_tokens, stream=True, **sampler_kwargs, ) @@ -1293,12 +1611,19 @@ class QwenVLGGUFBase: if stopped_for_degenerate_stream: print(f"[QwenVL GGUF] stopped repeated-token loop: {stopped_for_degenerate_stream}") cleaned = clean_model_output(raw_content, OutputCleanConfig(mode="text")) - return cleaned.strip(), raw_content + raw_for_trace = raw_content + if stopped_for_degenerate_stream: + raw_for_trace = ( + f"[stopped repeated-token loop: {stopped_for_degenerate_stream}]\n" + f"{raw_content}" + ).strip() + return cleaned.strip(), raw_for_trace cleaned_text, raw_text = _run_completion( system_prompt, user_prompt, images_b64, + audio_b64 or [], stage_label="STREAMING", seed_value=seed, ) @@ -1329,6 +1654,7 @@ class QwenVLGGUFBase: retry_system, retry_user, [], + [], stage_label=stage_label, seed_value=int(seed) + 999 + attempt_number, attempt_enable_thinking=False, @@ -1395,6 +1721,8 @@ class QwenVLGGUFBase: seed, keep_model_loaded, device, + audio=None, + audio_file_path="", ctx=None, n_batch=None, n_ubatch=None, @@ -1417,12 +1745,16 @@ class QwenVLGGUFBase: print(f"[QwenVL GGUF DEBUG] Starting run with seed={seed}") image_hash = get_image_hash(image) video_hash = get_video_hash(video) + audio_hash = _get_audio_hash(audio) + audio_file_hash = _get_audio_file_hash(audio_file_path) input_signature = build_node_input_signature( model_name=model_name, preset_prompt=preset_prompt, custom_prompt=custom_prompt, image_hash=image_hash, video_hash=video_hash, + audio_hash=audio_hash, + audio_file_hash=audio_file_hash, frame_count=frame_count, ctx=ctx, n_batch=n_batch, @@ -1443,7 +1775,15 @@ class QwenVLGGUFBase: top_p=top_p, repetition_penalty=repetition_penalty, ) - if not stream_to_terminal: + requires_audio_input = node_class in {"ThinkingLLM_Gemma4_Audio_GGUF"} + skip_prompt_persistence = requires_audio_input + audio_b64, audio_notes = _audio_inputs_to_wav_base64(audio=audio, audio_file_path=audio_file_path) + if requires_audio_input and not audio_b64: + return _missing_audio_result(audio_notes) + if audio_b64: + print(f"[QwenVL GGUF DEBUG] Audio processed: {len(audio_b64)} audio item(s)") + + if not stream_to_terminal and not skip_prompt_persistence: saved = get_node_saved_prompt_with_seed( node_class, unique_id, @@ -1527,7 +1867,7 @@ class QwenVLGGUFBase: estimated_prompt_tokens = estimate_qwen_text_tokens( "You are a helpful vision-language assistant.", prompt, - ) + len(images_b64) * max(1024, int(attempt_image_max_tokens or 1024)) + ) + len(images_b64) * max(1024, int(attempt_image_max_tokens or 1024)) + len(audio_b64) * 2048 effective_thinking = resolve_qwen_thinking_mode( enable_thinking, max_tokens, @@ -1558,8 +1898,8 @@ class QwenVLGGUFBase: ) hf_token = "" print(f"[QwenVL GGUF DEBUG] Model loaded successfully") - if images_b64 and self.chat_handler is None: - print("[QwenVL] Warning: images provided but this model entry has no mmproj_file; images will be ignored") + if (images_b64 or audio_b64) and self.chat_handler is None: + print("[QwenVL] Warning: media provided but this model entry has no multimodal handler; media will be ignored") print(f"[QwenVL GGUF DEBUG] Starting generation...") text, raw_trace = self._invoke( system_prompt=( @@ -1570,6 +1910,7 @@ class QwenVLGGUFBase: ), user_prompt=prompt, images_b64=images_b64 if self.chat_handler is not None else [], + audio_b64=audio_b64 if self.chat_handler is not None else [], max_tokens=max_tokens, temperature=temperature, top_p=top_p, @@ -1602,25 +1943,31 @@ class QwenVLGGUFBase: print(f"[QwenVL GGUF DEBUG] Generation completed. Text length: {len(text) if text else 0}") print(f"[QwenVL GGUF DEBUG] Generated text: {text[:100] if text else 'EMPTY'}...") - # Cache the generated text - PROMPT_CACHE[cache_key] = { - "text": text, - "timestamp": None, # GGUF doesn't have CUDA events - "model": model_name, - "preset": preset_prompt, - "seed": int(seed), - "image_hash": image_hash, - "video_hash": video_hash - } - save_prompt_cache() # Save cache to file + if not skip_prompt_persistence: + # Cache the generated text + PROMPT_CACHE[cache_key] = { + "text": text, + "timestamp": None, # GGUF doesn't have CUDA events + "model": model_name, + "preset": preset_prompt, + "seed": int(seed), + "image_hash": image_hash, + "video_hash": video_hash + } + save_prompt_cache() # Save cache to file - print(f"[QwenVL GGUF] Cached new prompt for seed {seed}: {cache_key[:8]}...") + print(f"[QwenVL GGUF] Cached new prompt for seed {seed}: {cache_key[:8]}...") + else: + print("[QwenVL GGUF] Audio node result was not stored in the prompt cache") print(f"[QwenVL GGUF DEBUG] Returning tuple with text...") - # Save the generated prompt for future per-node keep-last-prompt - set_node_saved_prompt(node_class, unique_id, extra_pnginfo, text, raw_trace=raw_trace, seed=int(seed), max_tokens=max_tokens, temperature=temperature, top_p=top_p, repetition_penalty=repetition_penalty, input_signature=input_signature) - print(f"[QwenVL GGUF] Saved per-node prompt: {text[:50]}...") + if not skip_prompt_persistence: + # Save the generated prompt for future per-node keep-last-prompt + set_node_saved_prompt(node_class, unique_id, extra_pnginfo, text, raw_trace=raw_trace, seed=int(seed), max_tokens=max_tokens, temperature=temperature, top_p=top_p, repetition_penalty=repetition_penalty, input_signature=input_signature) + print(f"[QwenVL GGUF] Saved per-node prompt: {text[:50]}...") + else: + print("[QwenVL GGUF] Audio node result was not stored as a saved prompt") return (text, raw_trace) finally: @@ -1644,7 +1991,7 @@ class ThinkingLLM_QwenVL_GGUF(QwenVLGGUFBase): "model_name": (model_keys, {"default": default_model, "tooltip": GGUF_TOOLTIPS["model_name"]}), "preset_prompt": (prompts, {"default": default_prompt, "tooltip": "Select 'No preset' to use only the custom prompt or image input."}), "custom_prompt": ("STRING", {"default": "", "multiline": True, "tooltip": "Additional user input that gets combined with the preset template. Leave empty to use only the template."}), - "max_tokens": ("INT", {"default": 8192, "min": 64, "max": 8192, "tooltip": GGUF_TOOLTIPS["max_tokens"]}), + "max_tokens": ("INT", {"default": 8192, "min": 64, "max": 32768, "tooltip": GGUF_TOOLTIPS["max_tokens"]}), "keep_model_loaded": ("BOOLEAN", {"default": False, "tooltip": GGUF_TOOLTIPS["keep_model_loaded"]}), "seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1, "tooltip": GGUF_TOOLTIPS["seed"]}), "stream_tokens_to_terminal": ("BOOLEAN", {"default": False, "tooltip": GGUF_TOOLTIPS["stream_tokens_to_terminal"]}), @@ -1654,6 +2001,8 @@ class ThinkingLLM_QwenVL_GGUF(QwenVLGGUFBase): "optional": { "image": ("IMAGE",), "video": ("IMAGE",), + "audio": ("AUDIO",), + "audio_file_path": ("STRING", {"default": "", "multiline": False, "tooltip": GGUF_TOOLTIPS["audio_file_path"]}), }, "hidden": { "unique_id": "UNIQUE_ID", @@ -1661,8 +2010,8 @@ class ThinkingLLM_QwenVL_GGUF(QwenVLGGUFBase): }, } - RETURN_TYPES = ("STRING",) - RETURN_NAMES = ("RESPONSE",) + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("RESPONSE", "RAW_TRACE") FUNCTION = "process" CATEGORY = "ThinkingLLM" @@ -1676,6 +2025,8 @@ class ThinkingLLM_QwenVL_GGUF(QwenVLGGUFBase): seed, image=None, video=None, + audio=None, + audio_file_path="", unique_id=None, extra_pnginfo=None, stream_tokens_to_terminal=False, @@ -1689,6 +2040,8 @@ class ThinkingLLM_QwenVL_GGUF(QwenVLGGUFBase): image=image, video=video, frame_count=16, + audio=audio, + audio_file_path=audio_file_path, max_tokens=max_tokens, temperature=0.6, top_p=0.9, @@ -1716,7 +2069,93 @@ class ThinkingLLM_QwenVL_GGUF(QwenVLGGUFBase): hf_token=hf_token, ) hf_token = "" - return (result[0],) + return result + + +class ThinkingLLM_Gemma4_Audio_GGUF(QwenVLGGUFBase): + @classmethod + def INPUT_TYPES(cls): + model_keys = _gguf_audio_model_keys() or ["(no Gemma 4 audio GGUF models found)"] + default_model = next((key for key in model_keys if "gemma-4-12b" in key.lower()), model_keys[0]) + + return { + "required": { + "model_name": (model_keys, {"default": default_model, "tooltip": GGUF_TOOLTIPS["audio_model_name"]}), + "custom_prompt": ("STRING", {"default": GEMMA4_AUDIO_DEFAULT_PROMPT, "multiline": True, "tooltip": "Audio instruction sent with the AUDIO input. Gemma 4 audio works best with short 16 kHz mono WAV-style input; Comfy AUDIO is converted to WAV before inference."}), + "max_tokens": ("INT", {"default": 2048, "min": 64, "max": 32768, "tooltip": GGUF_TOOLTIPS["max_tokens"]}), + "keep_model_loaded": ("BOOLEAN", {"default": False, "tooltip": GGUF_TOOLTIPS["keep_model_loaded"]}), + "seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1, "tooltip": GGUF_TOOLTIPS["seed"]}), + "stream_tokens_to_terminal": ("BOOLEAN", {"default": False, "tooltip": GGUF_TOOLTIPS["stream_tokens_to_terminal"]}), + "enable_thinking": ("BOOLEAN", {"default": False, "tooltip": "Gemma 4 can reason, but audio transcription and short analysis are usually clearer with thinking disabled."}), + "hf_token": ("STRING", {"default": "", "multiline": False, "tooltip": GGUF_TOOLTIPS["hf_token"]}), + }, + "optional": { + "audio": ("AUDIO",), + "audio_file_path": ("STRING", {"default": "", "multiline": False, "tooltip": GGUF_TOOLTIPS["audio_file_path"]}), + }, + "hidden": { + "unique_id": "UNIQUE_ID", + "extra_pnginfo": "EXTRA_PNGINFO", + }, + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("RESPONSE", "RAW_TRACE") + FUNCTION = "process" + CATEGORY = "ThinkingLLM" + + def process( + self, + model_name, + custom_prompt, + max_tokens, + keep_model_loaded, + seed, + stream_tokens_to_terminal=False, + enable_thinking=False, + hf_token="", + audio=None, + audio_file_path="", + unique_id=None, + extra_pnginfo=None, + ): + result = self.run( + model_name=model_name, + preset_prompt="🚫 No preset (image-only)", + custom_prompt=custom_prompt, + image=None, + video=None, + frame_count=1, + audio=audio, + audio_file_path=audio_file_path, + max_tokens=max_tokens, + temperature=1.0, + top_p=0.95, + repetition_penalty=1.0, + seed=seed, + keep_model_loaded=keep_model_loaded, + device="auto", + ctx=None, + n_batch=None, + n_ubatch=None, + gpu_layers=None, + image_max_tokens=None, + top_k=None, + pool_size=None, + n_threads=None, + n_threads_batch=None, + flash_attn=True, + offload_kqv=True, + ctx_checkpoints=0, + unique_id=unique_id, + extra_pnginfo=extra_pnginfo, + node_class="ThinkingLLM_Gemma4_Audio_GGUF", + stream_to_terminal=stream_tokens_to_terminal, + enable_thinking=enable_thinking, + hf_token=hf_token, + ) + hf_token = "" + return result class ThinkingLLM_QwenVL_GGUF_Advanced(QwenVLGGUFBase): @@ -1740,7 +2179,7 @@ class ThinkingLLM_QwenVL_GGUF_Advanced(QwenVLGGUFBase): "device": (device_options, {"default": "auto", "tooltip": GGUF_TOOLTIPS["device"]}), "preset_prompt": (prompts, {"default": default_prompt, "tooltip": "Select 'No preset' to use only the custom prompt or image input."}), "custom_prompt": ("STRING", {"default": "", "multiline": True, "tooltip": "Additional user input that gets combined with the preset template. Leave empty to use only the template."}), - "max_tokens": ("INT", {"default": 8192, "min": 64, "max": 8192, "tooltip": GGUF_TOOLTIPS["max_tokens"]}), + "max_tokens": ("INT", {"default": 8192, "min": 64, "max": 32768, "tooltip": GGUF_TOOLTIPS["max_tokens"]}), "temperature": ("FLOAT", {"default": 0.6, "min": 0.0, "max": 2.0, "tooltip": "Sampling randomness. Lower values are more deterministic; higher values are more varied."}), "top_p": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0, "tooltip": "Nucleus sampling cutoff. Lower values restrict token choice; 0.9 is a balanced default."}), "repetition_penalty": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0, "tooltip": "Values above 1.0 reduce repeated phrases; 1.0 leaves repetition unmodified."}), @@ -1768,6 +2207,8 @@ class ThinkingLLM_QwenVL_GGUF_Advanced(QwenVLGGUFBase): "optional": { "image": ("IMAGE",), "video": ("IMAGE",), + "audio": ("AUDIO",), + "audio_file_path": ("STRING", {"default": "", "multiline": False, "tooltip": GGUF_TOOLTIPS["audio_file_path"]}), }, "hidden": { "unique_id": "UNIQUE_ID", @@ -1809,6 +2250,8 @@ class ThinkingLLM_QwenVL_GGUF_Advanced(QwenVLGGUFBase): ctx_checkpoints, image=None, video=None, + audio=None, + audio_file_path="", unique_id=None, extra_pnginfo=None, stream_tokens_to_terminal=False, @@ -1824,6 +2267,8 @@ class ThinkingLLM_QwenVL_GGUF_Advanced(QwenVLGGUFBase): image=image, video=video, frame_count=frame_count, + audio=audio, + audio_file_path=audio_file_path, max_tokens=max_tokens, temperature=temperature, top_p=top_p, @@ -1857,6 +2302,7 @@ class ThinkingLLM_QwenVL_GGUF_Advanced(QwenVLGGUFBase): NODE_CLASS_MAPPINGS = { "ThinkingLLM_QwenVL_GGUF": ThinkingLLM_QwenVL_GGUF, "ThinkingLLM_QwenVL_GGUF_Advanced": ThinkingLLM_QwenVL_GGUF_Advanced, + "ThinkingLLM_Gemma4_Audio_GGUF": ThinkingLLM_Gemma4_Audio_GGUF, "AILab_QwenVL_GGUF": ThinkingLLM_QwenVL_GGUF, "AILab_QwenVL_GGUF_Advanced": ThinkingLLM_QwenVL_GGUF_Advanced, } @@ -1864,6 +2310,7 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { "ThinkingLLM_QwenVL_GGUF": "ThinkingLLM (GGUF)", "ThinkingLLM_QwenVL_GGUF_Advanced": "ThinkingLLM Advanced (GGUF)", + "ThinkingLLM_Gemma4_Audio_GGUF": "ThinkingLLM Gemma 4 Audio (GGUF)", "AILab_QwenVL_GGUF": "ThinkingLLM (GGUF)", "AILab_QwenVL_GGUF_Advanced": "ThinkingLLM Advanced (GGUF)", } diff --git a/AILab_QwenVL_PromptEnhancer.py b/AILab_QwenVL_PromptEnhancer.py index e1d07b1..2363c31 100644 --- a/AILab_QwenVL_PromptEnhancer.py +++ b/AILab_QwenVL_PromptEnhancer.py @@ -30,6 +30,7 @@ from AILab_QwenVL import ( HF_VL_MODELS, NODE_PROMPT_STATE, PROMPT_CACHE, + _default_model_from_config, _make_node_state_key, apply_qwen_soft_thinking_directive, build_node_input_signature, @@ -155,7 +156,7 @@ class ThinkingLLM_QwenVL_PromptEnhancer(QwenVLBase): @classmethod def INPUT_TYPES(cls): models = list(HF_ALL_MODELS.keys()) - default_model = models[0] if models else "Qwen3-VL-4B-Instruct" + default_model = _default_model_from_config(HF_ALL_MODELS, "Qwen3-VL-4B-Instruct") styles = list(cls.STYLES.keys()) preferred_style = "📝 Enhance" default_style = preferred_style if preferred_style in styles else (styles[0] if styles else "📝 Enhance") diff --git a/AILab_WhisperASR.py b/AILab_WhisperASR.py new file mode 100644 index 0000000..00ef29a --- /dev/null +++ b/AILab_WhisperASR.py @@ -0,0 +1,262 @@ +import json +import os +import shutil +import subprocess +import tempfile +import wave +from pathlib import Path + +import numpy as np + + +WHISPER_MODEL_CHOICES = [ + "small", + "base", + "tiny", + "medium", + "large-v3", + "distil-large-v3", +] + +WHISPER_LANGUAGE_CHOICES = [ + "auto", + "en", + "de", + "fr", + "es", + "it", + "pt", + "nl", + "pl", + "tr", + "ja", + "ko", + "zh", +] + +WHISPER_INSTALL_MESSAGE = ( + "[ThinkingLLM Whisper ASR] faster-whisper is not installed in this ComfyUI Python. " + "Install it in the active ComfyUI Python with: python -m pip install faster-whisper" +) + + +def _clean_audio_file_path(audio_file_path) -> Path | None: + if audio_file_path is None: + return None + raw_path = str(audio_file_path).strip().strip("\"'") + if not raw_path: + return None + return Path(os.path.expandvars(raw_path)).expanduser() + + +def _ffmpeg_path() -> str: + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + raise RuntimeError("ffmpeg was not found on PATH; install FFmpeg or connect a decoded ComfyUI AUDIO input") + return ffmpeg + + +def _decode_audio_file_to_wav(audio_file_path, target_path: Path) -> str: + source_path = _clean_audio_file_path(audio_file_path) + if source_path is None: + raise ValueError("audio_file_path is empty") + if not source_path.exists() or not source_path.is_file(): + raise FileNotFoundError(f"audio_file_path does not exist or is not a file: {source_path}") + + result = subprocess.run( + [ + _ffmpeg_path(), + "-hide_banner", + "-v", + "error", + "-i", + str(source_path), + "-vn", + "-ac", + "1", + "-ar", + "16000", + "-acodec", + "pcm_s16le", + str(target_path), + ], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"ffmpeg failed to decode audio_file_path: {result.stderr.strip() or 'unknown ffmpeg error'}") + return str(source_path) + + +def _audio_to_wav_file(audio, target_path: Path) -> str: + if not isinstance(audio, dict): + raise ValueError("expected ComfyUI AUDIO dict with waveform/sample_rate") + waveform = audio.get("waveform") + if waveform is None: + raise ValueError("missing waveform") + sample_rate = int(audio.get("sample_rate") or 0) + if sample_rate <= 0: + raise ValueError(f"invalid sample_rate={audio.get('sample_rate')!r}") + + tensor = waveform.detach().cpu() if hasattr(waveform, "detach") else waveform + tensor = tensor.cpu() if hasattr(tensor, "cpu") else tensor + array = tensor.numpy() if hasattr(tensor, "numpy") else np.asarray(tensor) + array = np.asarray(array, dtype=np.float32) + if array.size == 0: + raise ValueError("empty waveform") + if array.ndim == 3: + array = array[0] + if array.ndim == 1: + array = array[None, :] + if array.ndim != 2: + raise ValueError(f"unsupported waveform shape={getattr(array, 'shape', None)}") + if array.shape[0] > array.shape[-1]: + array = array.T + if not np.all(np.isfinite(array)): + array = np.nan_to_num(array, nan=0.0, posinf=1.0, neginf=-1.0) + + mono = np.mean(array, axis=0).astype(np.float32) + if sample_rate != 16000: + target_length = max(1, int(round(float(mono.size) * 16000.0 / float(sample_rate)))) + old_positions = np.linspace(0.0, float(mono.size - 1), num=mono.size, dtype=np.float64) + new_positions = np.linspace(0.0, float(mono.size - 1), num=target_length, dtype=np.float64) + mono = np.interp(new_positions, old_positions, mono).astype(np.float32) + + pcm16 = (np.clip(mono, -1.0, 1.0) * 32767.0).astype(np.int16) + with wave.open(str(target_path), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(pcm16.tobytes()) + return "ComfyUI AUDIO input" + + +def _resolve_whisper_device(device: str) -> str: + if device == "auto": + try: + import torch + + return "cuda" if torch.cuda.is_available() else "cpu" + except Exception: + return "cpu" + return device + + +def _resolve_compute_type(device: str, compute_type: str) -> str: + if compute_type != "auto": + return compute_type + return "float16" if device == "cuda" else "int8" + + +def _missing_audio_result(notes: list[str] | None = None) -> tuple[str, str, str]: + message = ( + "[ThinkingLLM Whisper ASR] No usable audio input. Connect a ComfyUI AUDIO output " + "or set audio_file_path to an existing M4A/MP3/WAV/FLAC file." + ) + trace_lines = ["[AUDIO]", message] + if notes: + trace_lines.extend(str(note) for note in notes if note) + return message, "[]", "\n".join(trace_lines) + + +class ThinkingLLM_Whisper_ASR: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model_size": (WHISPER_MODEL_CHOICES, {"default": "small", "tooltip": "faster-whisper model size. small is a reliable Windows-friendly starting point; large-v3 is higher quality but downloads and runs much larger."}), + "language": (WHISPER_LANGUAGE_CHOICES, {"default": "auto", "tooltip": "Audio language. Use auto to let Whisper detect it."}), + "task": (["transcribe", "translate"], {"default": "transcribe", "tooltip": "transcribe preserves the original language; translate returns English."}), + "device": (["cpu", "auto", "cuda"], {"default": "cpu", "tooltip": "cpu is the most reliable Windows default; choose cuda when your faster-whisper/CTranslate2 CUDA runtime is working."}), + "compute_type": (["int8", "auto", "float16", "int8_float16", "float32"], {"default": "int8", "tooltip": "int8 is reliable on CPU. Use float16 or int8_float16 for CUDA."}), + "beam_size": ("INT", {"default": 5, "min": 1, "max": 10, "tooltip": "Higher values can improve accuracy but run slower."}), + "vad_filter": ("BOOLEAN", {"default": True, "tooltip": "Skip long silence and non-speech sections before transcription."}), + }, + "optional": { + "audio": ("AUDIO",), + "audio_file_path": ("STRING", {"default": "", "multiline": False, "tooltip": "Optional local audio file path. M4A, MP3, WAV, FLAC, and other FFmpeg-readable files are decoded to 16 kHz mono WAV before Whisper."}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "STRING") + RETURN_NAMES = ("TRANSCRIPT", "SEGMENTS_JSON", "RAW_TRACE") + FUNCTION = "process" + CATEGORY = "ThinkingLLM/Audio" + + def process( + self, + model_size: str, + language: str, + task: str, + device: str, + compute_type: str, + beam_size: int, + vad_filter: bool, + audio=None, + audio_file_path="", + ): + notes: list[str] = [] + clean_path = _clean_audio_file_path(audio_file_path) + if audio is None and clean_path is None: + return _missing_audio_result() + + try: + from faster_whisper import WhisperModel + except Exception as exc: + trace = "\n".join(["[WHISPER]", WHISPER_INSTALL_MESSAGE, f"import_error={exc}"]) + return WHISPER_INSTALL_MESSAGE, "[]", trace + + with tempfile.TemporaryDirectory(prefix="thinkingllm-whisper-") as temp_dir: + wav_path = Path(temp_dir) / "audio.wav" + try: + if clean_path is not None: + source_label = _decode_audio_file_to_wav(str(clean_path), wav_path) + else: + source_label = _audio_to_wav_file(audio, wav_path) + except Exception as exc: + return _missing_audio_result([str(exc)]) + + resolved_device = _resolve_whisper_device(str(device)) + resolved_compute_type = _resolve_compute_type(resolved_device, str(compute_type)) + model = WhisperModel(str(model_size), device=resolved_device, compute_type=resolved_compute_type) + segments_iter, info = model.transcribe( + str(wav_path), + language=None if language == "auto" else str(language), + task=str(task), + beam_size=int(beam_size), + vad_filter=bool(vad_filter), + ) + segments = [ + { + "start": float(segment.start), + "end": float(segment.end), + "text": str(segment.text).strip(), + } + for segment in segments_iter + ] + + transcript = "\n".join(segment["text"] for segment in segments if segment["text"]).strip() + segments_json = json.dumps(segments, ensure_ascii=False, indent=2) + detected_language = getattr(info, "language", None) + detected_probability = getattr(info, "language_probability", None) + duration = getattr(info, "duration", None) + trace = "\n".join( + [ + "[WHISPER]", + f"source={source_label}", + f"model_size={model_size}; device={resolved_device}; compute_type={resolved_compute_type}; task={task}; vad_filter={bool(vad_filter)}", + f"language={detected_language}; language_probability={detected_probability}; duration={duration}", + f"segments={len(segments)}; transcript_chars={len(transcript)}", + ] + ) + return transcript, segments_json, trace + + +NODE_CLASS_MAPPINGS = { + "ThinkingLLM_Whisper_ASR": ThinkingLLM_Whisper_ASR, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "ThinkingLLM_Whisper_ASR": "ThinkingLLM Whisper ASR", +} diff --git a/README.md b/README.md index 3c46577..e16620d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ComfyUI-ThinkingLLM -**ThinkingLLM** is a local-LLM custom-node pack for [ComfyUI](https://github.com/comfyanonymous/ComfyUI) that wraps Qwen3.5, Qwen3, and Qwen2.5-VL models behind a cleaner node interface with thinking-mode control, live token streaming, and a raw trace output for debugging. +**ThinkingLLM** is a local-LLM custom-node pack for [ComfyUI](https://github.com/comfyanonymous/ComfyUI) that wraps Qwen3.5, Qwen3, Qwen-VL, Gemma 4, and Whisper ASR behind a cleaner node interface with thinking-mode control, audio/vision workflows, live token streaming, and a raw trace output for debugging. 📺 **[Watch the demo](https://youtu.be/K1JsvnzujOw)** for a better viewing experience. @@ -13,6 +13,15 @@ ## What ThinkingLLM adds +### Model guidance box + +ThinkingLLM nodes add a read-only `recommended_settings` info box. + +- It explains the selected model family and recommended generation settings. +- It does not change your saved widget values. +- For GGUF audio-capable nodes, it tells you whether the selected model actually supports audio. +- In Advanced GGUF nodes, if you connect or expose audio for a model without known audio support, the box warns you instead of silently implying that audio will work. + ### The Thinking Toggle Every ThinkingLLM node exposes an `enable_thinking` toggle. @@ -40,6 +49,15 @@ Each node returns a second string output named `RAW_TRACE`. That makes it possible to inspect what the model thought about the prompt while still keeping the primary output clean. +### Audio support + +Audio is available through dedicated nodes so the model list stays honest: + +- `ThinkingLLM Gemma 4 Audio (GGUF)` for Gemma 4 audio-capable models. +- `ThinkingLLM Whisper ASR` for reliable speech-to-text transcription through `faster-whisper`. + +The Advanced GGUF node also exposes optional `audio`, but this is for power users. Use the info box before assuming a model can hear audio. Normal Qwen/Qwen-VL models are not audio models. + ## Nodes ### Transformers / HF nodes @@ -58,15 +76,17 @@ HF tokenizer note: some Gemma and Qwen variants need `sentencepiece` or `tiktoke - `ThinkingLLM (GGUF)` - `ThinkingLLM (GGUF Advanced)` +- `ThinkingLLM Gemma 4 Audio (GGUF)` - `ThinkingLLM Prompt Enhancer (GGUF)` -The GGUF path requires a vision-capable `llama-cpp-python` build. The normal PyPI package is not enough for Qwen vision handlers. On Linux, ThinkingLLM now auto-checks this at first GGUF use and attempts an automatic install of a matching JamePeng backend. Use the setup notes in [docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md) if you need to override the wheel/package source. +The GGUF path requires a multimodal-capable `llama-cpp-python` build. The normal PyPI package may not include the chat handlers needed for Qwen vision, Gemma 4, or MTMD audio. On Linux, ThinkingLLM auto-checks this at first GGUF use and attempts an automatic install of a matching JamePeng backend. Use the setup notes in [docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md) if you need to override the wheel/package source. ## Workflow tips - **Pre-process input images** — use a resize or scale node before ThinkingLLM so large images don't blow up the context window. - **Display the output** — connect RESPONSE or ENHANCED_OUTPUT to a **Show Text** or **Show Anything** node to see the generated text in the UI. - **Inspect reasoning** — connect the RAW_TRACE output to a second Show Text node to see what the model was thinking. +- **Use the dedicated audio nodes first** — Gemma 4 Audio is for audio understanding; Whisper ASR is for transcription. The Advanced GGUF audio input is intentionally more flexible, but the info box will warn you when the selected model has no curated audio support. ## Supported models @@ -94,17 +114,36 @@ ThinkingLLM supports these model families out of the box, with pre-configured en ### Gemma 4 — Vision-language (HF) - Gemma-4-E2B-it / Uncensored - Gemma-4-E4B-it / Uncensored +- Gemma-4-12B - Gemma-4-26B-A4B-it / Heretic - Gemma-4-31B-it ### Gemma 4 — Vision-language (GGUF) - Gemma-4-E2B / E4B (Q4 K M through BF16, uncensored) +- Gemma-4-12B-it (Q4/Q5/Q6/Q8/BF16/UD-Q4) - Gemma-4-26B-A4B-it (Q4 K M through BF16) - Gemma-4-31B-it (Q4 K M through BF16) +### Gemma 4 — Audio (GGUF) +- Gemma-4-E2B / E4B when present in the GGUF multimodal catalog +- Gemma-4-12B-it via `unsloth/gemma-4-12b-it-GGUF` + +Use `ThinkingLLM Gemma 4 Audio (GGUF)` for the clean audio-only interface. Gemma 4 audio currently depends on a recent multimodal `llama-cpp-python`/`Gemma4ChatHandler` build, so check `RAW_TRACE` if audio behaves unexpectedly. + +### Whisper — ASR +- tiny +- base +- small +- medium +- large-v3 +- distil-large-v3 + +Use `ThinkingLLM Whisper ASR` for transcription. It accepts a connected ComfyUI `AUDIO` input or an `audio_file_path` pointing to M4A, MP3, WAV, FLAC, or any FFmpeg-readable audio file. The node uses `faster-whisper`; first use may download the selected Whisper model into the Hugging Face cache. The default is `small` on CPU with `int8`, which is slower than CUDA but reliable on Windows; switch to CUDA once the local CTranslate2 CUDA runtime is confirmed working. + ### Gemma 4 — Text-only (HF) - Gemma-4-E2B-it / Uncensored - Gemma-4-E4B-it / Uncensored +- Gemma-4-12B - Gemma-4-26B-A4B-it / Heretic - Gemma-4-31B-it @@ -141,6 +180,8 @@ For local GGUF models, keep the matching mmproj file beside the model file. - HF is the recommended default path and is the simplest cross-platform option. - GGUF on Windows needs a matching `win_amd64` vision-capable wheel. - GGUF on Linux now auto-installs a matching backend on first use when possible. Override with `THINKINGLLM_LLAMA_CPP_LINUX_WHEEL_URL` or `THINKINGLLM_LLAMA_CPP_LINUX_SPEC` when your server needs a different build. +- GGUF audio needs an audio-capable multimodal backend. `Gemma4ChatHandler` is used for Gemma 4. +- Whisper ASR needs `faster-whisper` and FFmpeg. The node returns an install hint if `faster-whisper` is missing. - Flash Attention support is best on Linux. The nodes fall back when it is unavailable. - The thinking toggle works best with Qwen3.5 and Qwen3 style models. Other architectures may ignore the steering. diff --git a/__init__.py b/__init__.py index f4404e3..21a65a3 100644 --- a/__init__.py +++ b/__init__.py @@ -3,7 +3,7 @@ "name": "ComfyUI-ThinkingLLM", "description": "A multimodal ComfyUI AI node with Qwen3.5, Qwen3-VL, Qwen2.5-VL, Qwen3, and Gemma 4 integrations. Features live thinking in the terminal to see what the LLM is doing in real time.", "author": "goodguy1963", - "version": "2.3.4", + "version": "2.3.5", "url": "https://github.com/goodguy1963/ComfyUI-ThinkingLLM", "category": "image" } @@ -56,4 +56,4 @@ NODE_DISPLAY_NAME_MAPPINGS = dict(sorted(NODE_DISPLAY_NAME_MAPPINGS.items(), key __all__ = [ "NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS" -] \ No newline at end of file +] diff --git a/custom_nodes.json b/custom_nodes.json index ef5d749..b111461 100644 --- a/custom_nodes.json +++ b/custom_nodes.json @@ -9,5 +9,5 @@ "description": "A multimodal ComfyUI AI node with Qwen3.5, Qwen3-VL, Qwen2.5-VL, Qwen3, and Gemma 4 integrations. Features live thinking in the terminal to see what the LLM is doing in real time.", "category": "image", "tags": ["vision", "language", "multimodal", "thinkingllm", "qwen", "gguf", "flash_attention", "performance", "wan2.2", "video", "i2v", "t2v", "storyboard", "caching", "fork", "gpl-3.0"], - "version": "2.3.4" + "version": "2.3.5" } diff --git a/gguf_models.json b/gguf_models.json index e29c68a..e4500da 100644 --- a/gguf_models.json +++ b/gguf_models.json @@ -251,6 +251,24 @@ "context_length": 262144 } }, + "Gemma-4-12B-it-GGUF [Q4:7.5GB|Q5:9GB|Q6:11GB|Q8:13GB|BF16:24GB|VRAM:~12GB]": { + "author": "unsloth", + "repo_name": "gemma-4-12b-it-GGUF", + "repo_id": "unsloth/gemma-4-12b-it-GGUF", + "model_files": [ + "gemma-4-12b-it-Q4_K_M.gguf", + "gemma-4-12b-it-Q4_K_S.gguf", + "gemma-4-12b-it-Q5_K_M.gguf", + "gemma-4-12b-it-Q5_K_S.gguf", + "gemma-4-12b-it-Q6_K.gguf", + "gemma-4-12b-it-Q8_0.gguf", + "gemma-4-12b-it-BF16.gguf", + "gemma-4-12b-it-UD-Q4_K_XL.gguf" + ], + "defaults": { + "context_length": 262144 + } + }, "Gemma-4-31B-it-GGUF [Q4:20GB|Q8:32GB|VRAM:~22GB]": { "author": "unsloth", "repo_name": "gemma-4-31B-it-GGUF", @@ -441,6 +459,30 @@ "pool_size": 4194304 } }, + "Gemma-4-12B-it-GGUF [Q4:7.5GB|Q5:9GB|Q6:11GB|Q8:13GB|BF16:24GB|VRAM:~12GB]": { + "author": "unsloth", + "repo_name": "gemma-4-12b-it-GGUF", + "repo_id": "unsloth/gemma-4-12b-it-GGUF", + "mmproj_file": "mmproj-F16.gguf", + "model_files": [ + "gemma-4-12b-it-Q4_K_M.gguf", + "gemma-4-12b-it-Q4_K_S.gguf", + "gemma-4-12b-it-Q5_K_M.gguf", + "gemma-4-12b-it-Q5_K_S.gguf", + "gemma-4-12b-it-Q6_K.gguf", + "gemma-4-12b-it-Q8_0.gguf", + "gemma-4-12b-it-BF16.gguf", + "gemma-4-12b-it-UD-Q4_K_XL.gguf" + ], + "defaults": { + "context_length": 262144, + "image_max_tokens": 4096, + "n_batch": 512, + "gpu_layers": -1, + "top_k": 64, + "pool_size": 4194304 + } + }, "Qwen3-VL-4B-Instruct-c_abliterated-v2-GGUF [Q4:2.4GB|Q5:2.8GB|Q6:3.2GB|Q8:4.2GB|F16:8GB|VRAM:~4GB]": { "author": "mradermacher", "repo_name": "Qwen3-VL-4B-Instruct-c_abliterated-v2-GGUF", @@ -579,4 +621,4 @@ } } } -} \ No newline at end of file +} diff --git a/hf_models.json b/hf_models.json index 1e59a79..4b68395 100644 --- a/hf_models.json +++ b/hf_models.json @@ -2,7 +2,7 @@ "hf_vl_models": { "Qwen3.5-4B-heretic-v2 [DL: 7.5GB, VRAM: 9.0GB]": { "repo_id": "tvall43/Qwen3.5-4B-heretic-v2", - "default": true, + "default": false, "quantized": false, "vram_requirement": { "full": 9.0 @@ -142,6 +142,16 @@ "4bit": 16.0 } }, + "Gemma-4-12B [DL: 24GB, VRAM: 24.0GB]": { + "repo_id": "google/gemma-4-12B", + "default": false, + "quantized": false, + "vram_requirement": { + "full": 24.0, + "8bit": 13.0, + "4bit": 8.0 + } + }, "Gemma-4-E4B-Uncensored [DL: 16GB, VRAM: 7.0GB]": { "repo_id": "prithivMLmods/gemma-4-E4B-it-Uncensored-MAX", "default": false, @@ -294,6 +304,16 @@ "4bit": 16.0 } }, + "Gemma-4-12B [DL: 24GB, VRAM: 24.0GB]": { + "repo_id": "google/gemma-4-12B", + "default": false, + "quantized": false, + "vram_requirement": { + "full": 24.0, + "8bit": 13.0, + "4bit": 8.0 + } + }, "Gemma-4-E4B-Uncensored [DL: 16GB, VRAM: 7.0GB]": { "repo_id": "prithivMLmods/gemma-4-E4B-it-Uncensored-MAX", "default": false, @@ -345,4 +365,4 @@ } } } -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 4afe7bf..69eca7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "ComfyUI-ThinkingLLM" description = "A multimodal ComfyUI AI node with Qwen3.5, Qwen3-VL, Qwen2.5-VL, Qwen3, and Gemma 4 integrations. Features live thinking in the terminal to see what the LLM is doing in real time." -version = "2.3.4" +version = "2.3.5" license = {file = "LICENSE"} -dependencies = ["transformers>=5.2.0", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "sentencepiece", "tiktoken", "llama-cpp-python"] +dependencies = ["transformers>=5.2.0", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "sentencepiece", "tiktoken", "faster-whisper", "llama-cpp-python"] [project.urls] Repository = "https://github.com/goodguy1963/ComfyUI-ThinkingLLM" diff --git a/requirements.txt b/requirements.txt index 8d45c70..bee4dbc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ bitsandbytes accelerate sentencepiece tiktoken +faster-whisper # For running GGUF models with vision support # The [server] extra installs necessary components for multimodal models diff --git a/tests/test_new_features.py b/tests/test_new_features.py index c7fefd2..6e86429 100644 --- a/tests/test_new_features.py +++ b/tests/test_new_features.py @@ -8,16 +8,21 @@ plain shell without ML dependencies. """ import ast +import base64 import contextlib import importlib.util import inspect +import io import json import os import re +import shutil +import subprocess import sys import tempfile import types import unittest +import wave from pathlib import Path from unittest import mock @@ -150,6 +155,14 @@ def build_loader_test_stubs() -> dict[str, types.ModuleType]: PROMPT_CACHE={}, _make_node_state_key=lambda *args, **kwargs: "node-key", _build_workflow_fingerprint=lambda *args, **kwargs: "workflow-fingerprint", + _default_model_from_config=lambda models, fallback: next( + ( + name + for name, info in models.items() + if isinstance(info, dict) and info.get("default") + ), + next(iter(models.keys()), fallback), + ), apply_qwen_soft_thinking_directive=lambda *args, **kwargs: None, build_node_input_signature=lambda *args, **kwargs: {}, download_hf_file_to_path=lambda *args, **kwargs: None, @@ -361,6 +374,18 @@ class TestModelRecommendations(unittest.TestCase): with self.subTest(rule=rule.get("id")): self.assertTrue(rule.get("sources"), "each recommendation rule should record its research source") + def test_recommendation_notes_are_creative_work_focused(self): + payload = load_model_recommendations() + banned_terms = ("coding", "programming", "benchmark", "webdev", "math competition") + for rule in payload.get("rules") or []: + for note in rule.get("notes") or []: + lowered = note.lower() + with self.subTest(rule=rule.get("id"), note=note): + self.assertFalse( + any(term in lowered for term in banned_terms), + "recommendation notes should focus on creative prompt generation rather than coding/benchmarking", + ) + def test_recommendations_cover_all_catalog_models(self): payload = load_model_recommendations() rules = payload.get("rules") or [] @@ -381,14 +406,90 @@ class TestModelRecommendations(unittest.TestCase): self.assertGreaterEqual(len(text_catalog[text_key].get("model_files") or []), 11) self.assertGreaterEqual(len(vision_catalog[vision_key].get("model_files") or []), 11) + def test_gemma4_12b_is_available_for_hf_and_gguf_vision_and_text(self): + hf_payload = json.loads((PKG / "hf_models.json").read_text(encoding="utf-8")) + gguf_payload = json.loads((PKG / "gguf_models.json").read_text(encoding="utf-8")) + + self.assertTrue(any("Gemma-4-12B" in key for key in hf_payload.get("hf_vl_models", {}))) + self.assertTrue(any("Gemma-4-12B" in key for key in hf_payload.get("hf_text_models", {}))) + + gguf_text = gguf_payload.get("Qwen_model") or {} + gguf_vision = gguf_payload.get("qwenVL_model") or {} + text_key = next((key for key in gguf_text if "Gemma-4-12B" in key), None) + vision_key = next((key for key in gguf_vision if "Gemma-4-12B" in key), None) + + self.assertIsNotNone(text_key) + self.assertIsNotNone(vision_key) + self.assertIn("mmproj-F16.gguf", gguf_vision[vision_key]["mmproj_file"]) + self.assertGreaterEqual(len(gguf_text[text_key].get("model_files") or []), 8) + self.assertGreaterEqual(len(gguf_vision[vision_key].get("model_files") or []), 8) + + def test_qwen3_asr_gguf_catalog_is_removed(self): + gguf_payload = json.loads((PKG / "gguf_models.json").read_text(encoding="utf-8")) + vision_catalog = gguf_payload.get("qwenVL_model") or {} + asr_entries = {key: value for key, value in vision_catalog.items() if "Qwen3-ASR" in key} + + self.assertEqual({}, asr_entries) + + def test_legacy_gguf_model_file_entries_expand_to_runnable_models(self): + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF.py", + "thinkingllm_legacy_gguf_model_files_expand_test", + ) + + models = module.GGUF_VL_CATALOG.get("models") or {} + self.assertIn("gemma-4-12b-it-Q4_K_M.gguf [~7.5GB]", models) + resolved = module._resolve_model_entry("gemma-4-12b-it-Q4_K_M.gguf [~7.5GB]") + + self.assertEqual(resolved.model_filename, "gemma-4-12b-it-Q4_K_M.gguf") + self.assertEqual(resolved.mmproj_filename, "mmproj-F16.gguf") + + def test_hf_default_flag_controls_default_model(self): + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + package = load_thinkingllm_loader_subset(["AILab_QwenVL.py"]) + required = package.NODE_CLASS_MAPPINGS["ThinkingLLM_QwenVL"].INPUT_TYPES()["required"] + _, meta = required["model_name"] + self.assertEqual(meta["default"], "Qwen3-VL-4B-Instruct-Abliterated [DL: 7.5GB, VRAM: 6.0GB]") + def test_appearance_js_loads_recommendation_widget(self): source = read_source("web/js/appearance.js") self.assertIn("model_recommendations.json", source) self.assertIn("ComfyWidgets", source) self.assertIn("recommended_settings", source) + self.assertIn("support_notes?.audio", source) + self.assertIn("AUDIO_UNSUPPORTED_MESSAGE", source) + self.assertIn("else if (hasAudioInput(node))", source) + self.assertIn("ThinkingLLM_Gemma4_Audio_GGUF", source) + self.assertIn("ThinkingLLM_Whisper_ASR", source) + self.assertIn("Whisper ASR transcription", source) self.assertNotIn("Prompt enhancer compact terminal progress is active", source) self.assertIn("queueRecommendationRefresh", source) + def test_appearance_js_styles_all_visible_plugin_nodes(self): + source = read_source("web/js/appearance.js") + expected_node_names = [ + "AILab_QwenVL", + "AILab_QwenVL_Advanced", + "AILab_QwenVL_PromptEnhancer", + "AILab_QwenVL_GGUF", + "AILab_QwenVL_GGUF_Advanced", + "AILab_QwenVL_GGUF_PromptEnhancer", + "ThinkingLLM_QwenVL", + "ThinkingLLM_QwenVL_Advanced", + "ThinkingLLM_QwenVL_PromptEnhancer", + "ThinkingLLM_QwenVL_GGUF", + "ThinkingLLM_QwenVL_GGUF_Advanced", + "ThinkingLLM_Gemma4_Audio_GGUF", + "ThinkingLLM_Whisper_ASR", + "ThinkingLLM_QwenVL_GGUF_PromptEnhancer", + "VRAMCleanup", + "StorySplitNode", + ] + for node_name in expected_node_names: + with self.subTest(node_name=node_name): + self.assertIn(f'"{node_name}"', source) + def test_gguf_prompt_enhancer_caps_context_and_safe_constructs_llama(self): source = read_source("AILab_QwenVL_GGUF_PromptEnhancer.py") self.assertIn("_PROMPT_ENHANCER_MAX_CONTEXT_LENGTH = 32768", source) @@ -699,6 +800,7 @@ class TestLoaderModuleRegistration(unittest.TestCase): package = load_thinkingllm_loader_subset( [ "AILab_QwenVL_GGUF.py", + "AILab_WhisperASR.py", "AILab_QwenVL_PromptEnhancer.py", "AILab_QwenVL_GGUF_PromptEnhancer.py", ] @@ -707,11 +809,14 @@ class TestLoaderModuleRegistration(unittest.TestCase): for node_name in [ "ThinkingLLM_QwenVL_GGUF", "ThinkingLLM_QwenVL_GGUF_Advanced", + "ThinkingLLM_Gemma4_Audio_GGUF", + "ThinkingLLM_Whisper_ASR", "ThinkingLLM_QwenVL_PromptEnhancer", "ThinkingLLM_QwenVL_GGUF_PromptEnhancer", ]: with self.subTest(node_name=node_name): self.assertIn(node_name, package.NODE_CLASS_MAPPINGS) + self.assertNotIn("ThinkingLLM_Qwen3_ASR_GGUF", package.NODE_CLASS_MAPPINGS) class TestLegacyNodeNameCompatibility(unittest.TestCase): @@ -851,6 +956,265 @@ class TestGGUFAdvancedWorkflowCompatibility(unittest.TestCase): self.assertIn("if n_ubatch_val <= 0:", source) self.assertIn("n_ubatch_val = min(n_batch_val, 512)", source) + def test_gguf_vision_nodes_accept_optional_audio_without_legacy_order_change(self): + node_cls = self._load_gguf_advanced_class() + input_types = node_cls.INPUT_TYPES() + + self.assertEqual(input_types["optional"]["audio"], ("AUDIO",)) + + simple_cls = load_thinkingllm_loader_subset(["AILab_QwenVL_GGUF.py"]).NODE_CLASS_MAPPINGS["ThinkingLLM_QwenVL_GGUF"] + self.assertEqual(simple_cls.INPUT_TYPES()["optional"]["audio"], ("AUDIO",)) + + def test_gguf_vision_advanced_allows_larger_reasoning_output_budget(self): + node_cls = self._load_gguf_advanced_class() + max_tokens_type, max_tokens_meta = node_cls.INPUT_TYPES()["required"]["max_tokens"] + + self.assertEqual(max_tokens_type, "INT") + self.assertGreaterEqual(max_tokens_meta["max"], 32768) + + def test_simple_gguf_node_exposes_raw_trace(self): + node_cls = load_thinkingllm_loader_subset(["AILab_QwenVL_GGUF.py"]).NODE_CLASS_MAPPINGS["ThinkingLLM_QwenVL_GGUF"] + + self.assertEqual(node_cls.RETURN_TYPES, ("STRING", "STRING")) + self.assertEqual(node_cls.RETURN_NAMES, ("RESPONSE", "RAW_TRACE")) + + def test_gemma4_audio_node_is_audio_only_and_model_filtered(self): + node_cls = load_thinkingllm_loader_subset(["AILab_QwenVL_GGUF.py"]).NODE_CLASS_MAPPINGS["ThinkingLLM_Gemma4_Audio_GGUF"] + input_types = node_cls.INPUT_TYPES() + + optional = input_types["optional"] + self.assertEqual(optional["audio"], ("AUDIO",)) + self.assertIn("audio_file_path", optional) + self.assertNotIn("image", optional) + self.assertNotIn("video", optional) + + model_keys, model_meta = input_types["required"]["model_name"] + self.assertTrue(model_keys, "audio node should expose at least one Gemma 4 audio model") + self.assertTrue(all("gemma-4" in key.lower() for key in model_keys)) + self.assertTrue(all("26b" not in key.lower() and "31b" not in key.lower() for key in model_keys)) + self.assertIn("gemma-4-12b", model_meta["default"].lower()) + self.assertIn("Audio", input_types["required"]["custom_prompt"][1]["default"]) + + def test_qwen3_asr_node_is_not_registered(self): + package = load_thinkingllm_loader_subset(["AILab_QwenVL_GGUF.py", "AILab_WhisperASR.py"]) + + self.assertNotIn("ThinkingLLM_Qwen3_ASR_GGUF", package.NODE_CLASS_MAPPINGS) + + def test_whisper_asr_node_is_audio_only(self): + node_cls = load_thinkingllm_loader_subset(["AILab_WhisperASR.py"]).NODE_CLASS_MAPPINGS["ThinkingLLM_Whisper_ASR"] + input_types = node_cls.INPUT_TYPES() + + self.assertEqual(input_types["optional"]["audio"], ("AUDIO",)) + self.assertIn("audio_file_path", input_types["optional"]) + model_keys, model_meta = input_types["required"]["model_size"] + self.assertIn("base", model_keys) + self.assertIn("large-v3", model_keys) + self.assertEqual(model_meta["default"], "small") + self.assertEqual(input_types["required"]["device"][1]["default"], "cpu") + self.assertEqual(input_types["required"]["compute_type"][1]["default"], "int8") + self.assertNotIn("image", input_types["optional"]) + self.assertNotIn("video", input_types["optional"]) + + def test_whisper_asr_returns_install_message_when_backend_missing(self): + node_cls = load_thinkingllm_loader_subset(["AILab_WhisperASR.py"]).NODE_CLASS_MAPPINGS["ThinkingLLM_Whisper_ASR"] + with mock.patch.dict(sys.modules, {"faster_whisper": None}, clear=False): + transcript, segments_json, raw_trace = node_cls().process( + model_size="base", + language="auto", + task="transcribe", + device="auto", + compute_type="auto", + beam_size=5, + vad_filter=True, + audio_file_path="dummy.wav", + ) + + self.assertIn("faster-whisper is not installed", transcript) + self.assertEqual(segments_json, "[]") + self.assertIn("pip install faster-whisper", raw_trace) + + def test_whisper_asr_transcribes_with_mock_backend(self): + module = load_module_from_file( + "AILab_WhisperASR.py", + "thinkingllm_whisper_mock_backend_test", + ) + + class FakeSegment: + def __init__(self, start, end, text): + self.start = start + self.end = end + self.text = text + + fake_info = types.SimpleNamespace(language="de", language_probability=0.99, duration=1.0) + captured = {} + + class FakeWhisperModel: + def __init__(self, model_size, device, compute_type): + captured["init"] = (model_size, device, compute_type) + + def transcribe(self, path, language, task, beam_size, vad_filter): + captured["transcribe"] = (Path(path).suffix, language, task, beam_size, vad_filter) + return [FakeSegment(0.0, 1.0, " Hallo Welt. ")], fake_info + + fake_backend = build_stub_module("faster_whisper", WhisperModel=FakeWhisperModel) + with tempfile.TemporaryDirectory() as temp_dir: + source = Path(temp_dir) / "input.wav" + with wave.open(str(source), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b"\0\0" * 1600) + + with mock.patch.dict(sys.modules, {"faster_whisper": fake_backend}, clear=False): + transcript, segments_json, raw_trace = module.ThinkingLLM_Whisper_ASR().process( + model_size="base", + language="de", + task="transcribe", + device="cpu", + compute_type="int8", + beam_size=3, + vad_filter=True, + audio_file_path=str(source), + ) + + self.assertEqual(transcript, "Hallo Welt.") + self.assertEqual(json.loads(segments_json)[0]["text"], "Hallo Welt.") + self.assertEqual(captured["init"], ("base", "cpu", "int8")) + self.assertEqual(captured["transcribe"], (".wav", "de", "transcribe", 3, True)) + self.assertIn("language=de", raw_trace) + + def test_audio_to_wav_base64_resamples_to_16khz_mono(self): + import numpy as real_numpy + + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF.py", + "thinkingllm_audio_resample_test", + ) + module.np = real_numpy + + waveform = real_numpy.stack([ + real_numpy.linspace(-0.5, 0.5, 800, dtype=real_numpy.float32), + real_numpy.linspace(0.5, -0.5, 800, dtype=real_numpy.float32), + ]) + + encoded = module._audio_to_wav_base64({"waveform": waveform, "sample_rate": 8000}) + + self.assertEqual(len(encoded), 1) + with wave.open(io.BytesIO(base64.b64decode(encoded[0])), "rb") as wav: + self.assertEqual(wav.getframerate(), 16000) + self.assertEqual(wav.getnchannels(), 1) + self.assertEqual(wav.getsampwidth(), 2) + self.assertEqual(wav.getnframes(), 1600) + + def test_audio_to_wav_base64_rejects_invalid_audio_metadata(self): + import numpy as real_numpy + + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF.py", + "thinkingllm_audio_validation_test", + ) + module.np = real_numpy + + self.assertEqual(module._audio_to_wav_base64({"waveform": real_numpy.zeros((1, 10)), "sample_rate": 0}), []) + self.assertEqual(module._audio_to_wav_base64({"sample_rate": 16000}), []) + + def test_audio_file_path_to_wav_base64_decodes_m4a_with_ffmpeg(self): + if shutil.which("ffmpeg") is None: + self.skipTest("ffmpeg is required for audio file path decoding") + + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF.py", + "thinkingllm_audio_file_path_test", + ) + + with tempfile.TemporaryDirectory(prefix="thinkingllm audio ") as temp_dir: + m4a_path = Path(temp_dir) / "sample voice memo.m4a" + subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-v", + "error", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=0.25", + "-c:a", + "aac", + str(m4a_path), + ], + check=True, + capture_output=True, + ) + + encoded = module._audio_file_path_to_wav_base64(str(m4a_path)) + + self.assertEqual(len(encoded), 1) + with wave.open(io.BytesIO(base64.b64decode(encoded[0])), "rb") as wav: + self.assertEqual(wav.getframerate(), 16000) + self.assertEqual(wav.getnchannels(), 1) + self.assertEqual(wav.getsampwidth(), 2) + self.assertGreater(wav.getnframes(), 3000) + self.assertLess(wav.getnframes(), 5000) + + def test_audio_nodes_skip_saved_prompt_cache(self): + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF.py", + "thinkingllm_audio_nodes_skip_saved_prompt_cache_test", + ) + + class DummyLoader(module.QwenVLGGUFBase): + def _load_model(self, *args, **kwargs): + self.chat_handler = object() + self.last_backend_trace = "" + + def _encode_media(self, image, video, frame_count): + return [] + + def _invoke(self, *args, **kwargs): + return "real transcript", "raw transcript" + + with mock.patch.object( + module, + "get_node_saved_prompt_with_seed", + side_effect=AssertionError("audio nodes must not reuse saved prompts"), + ), mock.patch.object( + module, + "set_node_saved_prompt", + side_effect=AssertionError("audio nodes must not store saved prompts"), + ), mock.patch.object( + module, + "_audio_inputs_to_wav_base64", + return_value=(["BASE64WAV"], []), + ), mock.patch.object( + module, + "_resolve_model_entry", + return_value=types.SimpleNamespace(context_length=4096), + ): + text, raw_trace = DummyLoader().run( + model_name="Gemma-4-12B-it-Q4_K_M.gguf [~7.5GB]", + preset_prompt="🚫 No preset (image-only)", + custom_prompt="Analyze the audio.", + image=None, + video=None, + frame_count=1, + max_tokens=64, + temperature=0.0, + top_p=1.0, + repetition_penalty=1.0, + seed=1, + keep_model_loaded=False, + device="auto", + node_class="ThinkingLLM_Gemma4_Audio_GGUF", + ) + + self.assertEqual(text, "real transcript") + self.assertEqual(raw_trace, "raw transcript") + class TestLlamaCppInstaller(unittest.TestCase): def _resolve_linux_install_spec(self, installer, *, py_minor, cuda_version, machine="x86_64"): @@ -1144,6 +1508,40 @@ class TestGGUFWindllRuntimeFallback(unittest.TestCase): self.assertEqual(loader.llm.chat_template_kwargs["enable_thinking"], False) self.assertEqual(captured_kwargs[0]["chat_template_kwargs"], {"enable_thinking": False}) + def test_create_chat_completion_retries_without_unexpected_kwarg(self): + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF.py", + "thinkingllm_gguf_unexpected_kwarg_retry_test", + ) + + captured_kwargs = [] + + class DummyLlama: + def create_chat_completion(self, **kwargs): + captured_kwargs.append(dict(kwargs)) + if "repeat_last_n" in kwargs: + raise TypeError("Llama.create_chat_completion() got an unexpected keyword argument 'repeat_last_n'") + return {"choices": [{"message": {"content": "ok"}}]} + + loader = module.QwenVLGGUFBase() + loader.llm = DummyLlama() + loader.uses_qwen_template_thinking = False + + result = loader._create_chat_completion( + enable_thinking=False, + messages=[], + max_tokens=1, + temperature=0.0, + top_p=1.0, + stream=True, + repeat_last_n=48, + ) + + self.assertEqual(result["choices"][0]["message"]["content"], "ok") + self.assertIn("repeat_last_n", captured_kwargs[0]) + self.assertNotIn("repeat_last_n", captured_kwargs[1]) + class _GlobalCheck(ast.NodeVisitor): def __init__(self): diff --git a/web/js/appearance.js b/web/js/appearance.js index c1d2246..59a22eb 100644 --- a/web/js/appearance.js +++ b/web/js/appearance.js @@ -15,7 +15,17 @@ const NODE_COLORS = { "AILab_QwenVL_GGUF": "QwenVLGGUF", "AILab_QwenVL_GGUF_Advanced": "QwenVLGGUF", "AILab_QwenVL_GGUF_PromptEnhancer": "Enhancer", + "ThinkingLLM_QwenVL": "QwenVL", + "ThinkingLLM_QwenVL_Advanced": "QwenVL", + "ThinkingLLM_QwenVL_PromptEnhancer": "Enhancer", + "ThinkingLLM_QwenVL_GGUF": "QwenVLGGUF", + "ThinkingLLM_QwenVL_GGUF_Advanced": "QwenVLGGUF", + "ThinkingLLM_Gemma4_Audio_GGUF": "QwenVLGGUF", + "ThinkingLLM_Whisper_ASR": "QwenVLGGUF", + "ThinkingLLM_QwenVL_GGUF_PromptEnhancer": "Enhancer", "AILab_QwenVL_PromptLibrary": "Tools", + "VRAMCleanup": "Tools", + "StorySplitNode": "Tools", }; const THINKINGLLM_NODE_NAMES = new Set([ @@ -30,9 +40,20 @@ const THINKINGLLM_NODE_NAMES = new Set([ "ThinkingLLM_QwenVL_PromptEnhancer", "ThinkingLLM_QwenVL_GGUF", "ThinkingLLM_QwenVL_GGUF_Advanced", + "ThinkingLLM_Gemma4_Audio_GGUF", + "ThinkingLLM_Whisper_ASR", "ThinkingLLM_QwenVL_GGUF_PromptEnhancer", ]); +const AUDIO_CAPABLE_NODE_NAMES = new Set([ + "ThinkingLLM_Gemma4_Audio_GGUF", + "ThinkingLLM_Whisper_ASR", + "AILab_QwenVL_GGUF", + "AILab_QwenVL_GGUF_Advanced", + "ThinkingLLM_QwenVL_GGUF", + "ThinkingLLM_QwenVL_GGUF_Advanced", +]); + const GGUF_ADVANCED_INTERNAL_WIDGETS = new Set([ "legacy_seed_mode", "legacy_unload_after_run", @@ -48,6 +69,7 @@ const RECOMMENDATIONS_URL = new URL("../model_recommendations.json", import.meta const RECOMMENDATION_WIDGET_NAME = "recommended_settings"; const RECOMMENDATION_PLACEHOLDER = "Select a model to see provider/community recommended settings. This note never changes your saved widget values."; const RECOMMENDATION_FIELDS = ["temperature", "top_p", "top_k", "max_tokens", "ctx_min", "n_batch", "image_max_tokens"]; +const AUDIO_UNSUPPORTED_MESSAGE = "No curated audio support for the selected model. Use Gemma 4 Audio for audio understanding or Whisper ASR for transcription."; let recommendationsPromise = null; @@ -132,6 +154,8 @@ function recommendationScope(node) { "AILab_QwenVL_GGUF_Advanced", "ThinkingLLM_QwenVL_GGUF", "ThinkingLLM_QwenVL_GGUF_Advanced", + "ThinkingLLM_Gemma4_Audio_GGUF", + "ThinkingLLM_Whisper_ASR", "AILab_QwenVL_GGUF_PromptEnhancer", "ThinkingLLM_QwenVL_GGUF_PromptEnhancer", ].includes(node.comfyClass)) { @@ -162,6 +186,11 @@ function hasWidget(node, widgetName) { return Boolean(findWidget(node, widgetName)); } +function hasAudioInput(node) { + return AUDIO_CAPABLE_NODE_NAMES.has(node.comfyClass) + || Boolean(node.inputs?.some((input) => input?.name === "audio")); +} + function formatSetting(fieldName, value) { if (typeof value !== "number") { return null; @@ -198,6 +227,17 @@ function buildModeSummary(node, settings) { } function buildRecommendationText(node, rules) { + if (node.comfyClass === "ThinkingLLM_Whisper_ASR") { + const modelSize = String(findWidget(node, "model_size")?.value || "small").trim() || "small"; + return [ + "Whisper ASR transcription", + `Model: ${modelSize}`, + "Audio: Use for speech-to-text from AUDIO or audio_file_path. M4A/MP3/WAV/FLAC are decoded with FFmpeg.", + "Settings: small + CPU/int8 is the reliable Windows default; large-v3 is higher quality; vad_filter skips silence.", + "Dependency: faster-whisper must be installed in the active ComfyUI Python.", + ].join("\n"); + } + const modelName = String(findWidget(node, "model_name")?.value || "").trim(); if (!modelName || modelName.startsWith("(no ")) { return RECOMMENDATION_PLACEHOLDER; @@ -227,6 +267,12 @@ function buildRecommendationText(node, rules) { if (supportNote) { lines.push(`Toggle: ${supportNote}`); } + const audioNote = rule.support_notes?.audio; + if (audioNote && hasAudioInput(node)) { + lines.push(`Audio: ${audioNote}`); + } else if (hasAudioInput(node)) { + lines.push(`Audio: ${AUDIO_UNSUPPORTED_MESSAGE}`); + } lines.push(`Thinking ON: ${(thinkingEnabled ? activeSummary : alternateSummary).join(" | ") || "use the provider defaults"}`); lines.push(`Thinking OFF: ${(thinkingEnabled ? alternateSummary : activeSummary).join(" | ") || "use the provider defaults"}`); @@ -348,4 +394,4 @@ const ext = { } }; -app.registerExtension(ext); \ No newline at end of file +app.registerExtension(ext); diff --git a/web/model_recommendations.json b/web/model_recommendations.json index ac5bf55..678b51e 100644 --- a/web/model_recommendations.json +++ b/web/model_recommendations.json @@ -32,9 +32,9 @@ "ctx_min": 131072 }, "notes": [ - "Provider says keep at least 128K context to preserve thinking quality.", + "For creative prompt generation, use the general-task sampler: thinking ON for richer scene planning, thinking OFF for shorter direct rewrites.", "Q4_K_P is the best quality-size starting point; move to Q6_K_P or Q8_K_P when VRAM allows.", - "IQ quants are fallback options when you cannot fit Q4_K_M or Q4_K_P." + "Keep a large context when expanding long image/video prompt sequences so character and scene continuity are not lost." ] }, { @@ -68,9 +68,9 @@ "ctx_min": 131072 }, "notes": [ - "For precise coding, Qwen recommends temperature 0.6 with thinking still enabled.", - "For hard non-thinking reasoning tasks, Qwen recommends top_p 1.0 and top_k 40 when the backend exposes them.", - "Provider says keep at least 128K context to preserve thinking capability and use 32K+ output budget when possible." + "For creative prompt generation, Qwen's general-task settings are the right fit: thinking ON for richer planning, non-thinking for concise prompt rewrites.", + "Use the larger output budget for multi-shot scene descriptions, storyboard continuity, or detailed style transfer prompts.", + "If generations loop or over-elaborate, prefer a shorter custom prompt and lower max_tokens before changing the sampler." ] }, { @@ -114,9 +114,49 @@ "ctx_min": 32768 }, "notes": [ - "Qwen warns against greedy decoding for thinking mode because it can repeat or degrade quality.", + "For creative prompt drafting, use thinking ON when you want structured scene planning and thinking OFF when you want a fast final prompt.", "Provider recommends 32K output budget for most tasks and only enabling YaRN when you really need longer than the native 32K context.", - "For harder reasoning benchmarks, Qwen suggests stretching output budget beyond 32K if your backend allows it." + "Avoid greedy sampling for creative generation; the recommended sampler gives more varied but still coherent prompt language." + ] + }, + { + "id": "gemma4_12b_unified", + "priority": 65, + "match_any": [ + "Gemma-4-12B", + "gemma-4-12b" + ], + "family_label": "Gemma 4 12B Unified", + "sources": [ + "Google Gemma 4 model overview", + "Unsloth Gemma 4 12B GGUF model card" + ], + "support_notes": { + "hf": "Gemma 4 12B is a unified multimodal model. HF support depends on current Transformers processor support for image/audio/video inputs.", + "gguf": "Use Gemma4ChatHandler in a recent multimodal llama-cpp-python build. Pair the model with mmproj-F16.gguf.", + "audio": "Audio input is intended for Gemma 4 E2B, E4B, and 12B only. Use a recent Gemma4ChatHandler build; 16 kHz mono WAV-style audio is safest.", + "soft_switch": "Gemma uses configurable thinking controls instead of Qwen-style /think or /no_think soft switches." + }, + "thinking_on": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 64, + "max_tokens": 1024, + "ctx_min": 262144, + "image_max_tokens": 4096 + }, + "thinking_off": { + "temperature": 1.0, + "top_p": 0.95, + "top_k": 64, + "max_tokens": 1024, + "ctx_min": 262144, + "image_max_tokens": 4096 + }, + "notes": [ + "For creative prompt generation, Gemma 4's default temperature/top_p is intentionally open enough for varied visual language.", + "For audio analysis, 16 kHz mono WAV is the safest input shape.", + "Prefer Q4_K_M or UD-Q4_K_XL first; move up to Q6/Q8/BF16 when VRAM allows." ] }, { @@ -152,7 +192,7 @@ "ctx_min": 262144 }, "notes": [ - "Gemma 4 uses one standardized sampler for both thinking and non-thinking modes.", + "For creative prompt generation, keep Gemma 4 near its official sampler defaults before tuning style or length.", "Official context target is 256K on 26B A4B and 31B variants.", "If the backend does not expose top_k, keep temperature and top_p at the official defaults." ] @@ -190,10 +230,10 @@ "ctx_min": 131072 }, "notes": [ - "Gemma 4 uses one standardized sampler for both thinking and non-thinking modes.", + "For creative prompt generation, keep Gemma 4 near its official sampler defaults before tuning style or length.", "Official context target is 128K on E2B and E4B variants.", "Gemma recommends higher visual token budgets only for OCR or fine detail tasks; lower budgets are faster for captioning or video summaries." ] } ] -} \ No newline at end of file +}