chore: refresh API model catalogs for v2.5.8

.gitignore

@@ -55,6 +55,7 @@ Thumbs.db
 # Logs
 *.log
 logs/
+.firecrawl/
 
 # ComfyUI specific
 models/

AILab_QwenVL_GGUF.py

@@ -104,6 +104,14 @@ load_node_prompt_state()
 _GEMMA4_MULTIMODAL_MAX_CTX = 32768
 _GEMMA4_MULTIMODAL_IMAGE_TOKENS = 1120
 _GEMMA4_MULTIMODAL_MAX_BATCH = 2048
+_GEMMA4_AUDIO_MIN_BACKEND_VERSION = (0, 3, 36)
+
+
+def _version_tuple(version: str) -> tuple[int, int, int] | None:
+    match = re.match(r"\s*(\d+)\.(\d+)\.(\d+)", str(version or ""))
+    if match is None:
+        return None
+    return tuple(int(part) for part in match.groups())
 
 
 def read_gguf_architecture(filepath: Path) -> str | None:
@@ -1166,6 +1174,7 @@ class QwenVLGGUFBase:
         self.current_ubatch_size = None
         self.current_image_token_budget = None
         self.current_context_length = None
+        self.current_backend_info = None
         self.gguf_arch = None
         register_active_gguf_loader(self)
 
@@ -1286,6 +1295,7 @@ class QwenVLGGUFBase:
     ):
         Llama = self._load_backend()
         backend_info = get_last_llama_cpp_backend_info()
+        self.current_backend_info = dict(backend_info) if backend_info else None
 
         resolved = _resolve_model_entry(model_name)
         access_info = {"commercial_status": resolved.commercial_status}
@@ -1595,11 +1605,35 @@ class QwenVLGGUFBase:
         if arch != "gemma4":
             return
         if audio_b64:
-            raise RuntimeError(
-                "Gemma 4 GGUF audio input is disabled because the installed llama.cpp backend can abort the "
-                "ComfyUI process for Gemma 4 audio. Use image/text input with Gemma 4 GGUF, or use a backend "
-                "that explicitly supports Gemma 4 audio."
-            )
+            backend_info = getattr(self, "current_backend_info", None) or {}
+            backend_version = str(backend_info.get("version") or "unknown")
+            parsed_version = _version_tuple(backend_version)
+            if parsed_version is None or parsed_version < _GEMMA4_AUDIO_MIN_BACKEND_VERSION:
+                minimum = ".".join(str(part) for part in _GEMMA4_AUDIO_MIN_BACKEND_VERSION)
+                raise RuntimeError(
+                    "Gemma 4 GGUF audio requires llama-cpp-python "
+                    f">={minimum}; detected {backend_version}. Older or unknown backends remain blocked because "
+                    "their native Gemma 4 audio path can abort ComfyUI. Update the multimodal backend and restart ComfyUI."
+                )
+
+            initialize_mtmd = getattr(self.chat_handler, "_init_mtmd_context", None)
+            if not callable(initialize_mtmd):
+                raise RuntimeError(
+                    "The installed Gemma4ChatHandler cannot verify native audio support. "
+                    "Update llama-cpp-python and restart ComfyUI."
+                )
+            try:
+                initialize_mtmd(self.llm)
+            except Exception as exc:
+                raise RuntimeError(
+                    "Gemma 4 GGUF audio could not initialize the selected multimodal projector: "
+                    f"{exc}"
+                ) from exc
+            if getattr(self.chat_handler, "is_support_audio", None) is not True:
+                raise RuntimeError(
+                    "The selected Gemma 4 multimodal projector does not report audio support. "
+                    "Use an audio-capable mmproj (BF16 is recommended for Gemma 4 E2B/E4B)."
+                )
         context_length = int(getattr(self, "current_context_length", 0) or 0)
         batch_size = int(getattr(self, "current_batch_size", 0) or 0)
         ubatch_size = int(getattr(self, "current_ubatch_size", 0) or 0)

README.md

@@ -1,26 +1,35 @@
 # ComfyUI-ThinkingLLM
 
-**ThinkingLLM** is a local-LLM custom-node pack for [ComfyUI](https://github.com/comfyanonymous/ComfyUI) that wraps Qwen3.5, Qwen3, Qwen-VL, Gemma 4, and Whisper ASR behind a cleaner node interface with thinking-mode control, audio/vision workflows, live token streaming, a raw trace output for debugging — and a secure OpenAI-compatible API node for remote providers (OpenRouter, OrcaRouter, …).
+**ThinkingLLM** is a multimodal local-LLM node pack for [ComfyUI](https://github.com/comfyanonymous/ComfyUI). It provides clean workflows for Qwen, Gemma 4, GGUF, Whisper ASR, prompt enhancement, image/video understanding, audio analysis, and secure OpenAI-compatible providers.
 
-📺 **[Watch the demo](https://youtu.be/K1JsvnzujOw)** for a better viewing experience.
+[Watch the demo on YouTube](https://youtu.be/K1JsvnzujOw)
 
 > GPL-3.0 fork of Deaquay/ComfyUI-Qwen3.5-Uncensored, itself derived from huchukato/ComfyUI-QwenVL-Mod and 1038lab/ComfyUI-QwenVL. See [LICENSE](LICENSE).
 
 ![ThinkingLLM Demo](docs/DemoThinkingLLM%20(1).gif)
 ![Node screenshot](docs/Screenshot%20of%20Notes.png)
 
-> 🎥 Prefer a narrated version? **[Watch the demo on YouTube](https://youtu.be/K1JsvnzujOw)**
+## Highlights
 
----
+- Local Transformers/Hugging Face and GGUF inference
+- Image and video understanding with optional mask-focused analysis
+- Gemma 4 audio understanding and Whisper speech-to-text
+- Prompt enhancers for image and video generation workflows
+- Thinking-mode control, live terminal streaming, and a separate `RAW_TRACE` output
+- Curated model recommendations and preset-specific generation guidance
+- Secure OpenAI-compatible API node for approved remote providers
+- Existing compatible models are discovered instead of downloaded twice
 
 ## Quick install
 
 ### ComfyUI Manager (recommended)
-1. Open **ComfyUI Manager** → **Install via Registry**
-2. Search for `ThinkingLLM`
-3. Click **Install**
+
+1. Open **ComfyUI Manager** → **Install via Registry**.
+2. Search for `ThinkingLLM`.
+3. Click **Install** and restart ComfyUI.
 
 ### Manual install
+
 ```bash
 cd ComfyUI/custom_nodes
 git clone https://github.com/goodguy1963/ComfyUI-ThinkingLLM.git
@@ -28,269 +37,78 @@ cd ComfyUI-ThinkingLLM
 pip install -r requirements.txt
 ```
 
-For GGUF vision support, follow [docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md).
-
----
+For GGUF vision or Gemma 4 audio, also follow the [llama.cpp backend installation guide](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md).
 
 ## Nodes at a glance
 
-| Node | Path | Purpose |
+| Node | Backend | Purpose |
 | --- | --- | --- |
-| `ThinkingLLM` | Transformers / HF (recommended) | Vision/video analysis with text output |
-| `ThinkingLLM (Advanced)` | Transformers / HF | Vision/video + full sampling controls, `MASK_PREVIEW` |
+| `ThinkingLLM` | Transformers / HF | Image and video understanding |
+| `ThinkingLLM (Advanced)` | Transformers / HF | Full controls and `MASK_PREVIEW` |
 | `ThinkingLLM Prompt Enhancer` | Transformers / HF | Text prompt enhancement |
-| `ThinkingLLM (GGUF)` | GGUF / llama.cpp | Vision/video via GGUF models |
-| `ThinkingLLM (GGUF Advanced)` | GGUF / llama.cpp | GGUF vision/video + advanced controls, `MASK_PREVIEW` |
-| `ThinkingLLM Gemma 4 Audio (GGUF)` | GGUF / llama.cpp | Audio understanding (Gemma 4) |
-| `ThinkingLLM Whisper ASR` | ASR | Speech-to-text transcription |
-| `ThinkingLLM Prompt Enhancer (GGUF)` | GGUF / llama.cpp | Text prompt enhancement via GGUF |
-| `ThinkingLLM System Prompt Preset` | Utility | Reusable system-prompt presets as `STRING` |
-| `ThinkingLLM API (OpenAI Compatible)` | ThinkingLLM/API | Secure calls to OpenAI-compatible remote providers |
+| `ThinkingLLM (GGUF)` | llama.cpp | GGUF image and video understanding |
+| `ThinkingLLM (GGUF Advanced)` | llama.cpp | GGUF advanced controls and `MASK_PREVIEW` |
+| `ThinkingLLM Gemma 4 Audio (GGUF)` | llama.cpp | Gemma 4 audio understanding |
+| `ThinkingLLM Whisper ASR` | faster-whisper | Speech-to-text transcription |
+| `ThinkingLLM Prompt Enhancer (GGUF)` | llama.cpp | GGUF prompt enhancement |
+| `ThinkingLLM System Prompt Preset` | Utility | Reusable system prompts as `STRING` |
+| `ThinkingLLM API (OpenAI Compatible)` | Remote API | Approved OpenAI-compatible providers |
 
-> **Need a remote/cloud model?** The **API node** calls approved OpenAI-compatible backends (OpenRouter, OrcaRouter, OpenAI, QwenCloud, Groq, …) with credentials kept server-side. See [docs/OPENAI_COMPATIBLE_API.md](docs/OPENAI_COMPATIBLE_API.md).
+The Transformers/HF nodes are the recommended default when you want the simplest installation. Choose GGUF when you specifically need quantized llama.cpp models or Gemma 4 audio.
 
----
+## Start a workflow
 
-## Feature overview
+1. Add the ThinkingLLM node that matches your backend and task.
+2. Select a model or an already installed local model.
+3. Connect image, video, audio, mask, or text inputs as needed.
+4. Connect `RESPONSE` or `ENHANCED_OUTPUT` to a text display or downstream prompt input.
+5. Connect `RAW_TRACE` when you want generation diagnostics.
 
-### Model guidance box
+Useful defaults:
 
-ThinkingLLM nodes add a read-only `recommended_settings` info box.
+- Start with `enable_thinking=false` for transcription and short direct answers.
+- Enable `stream_tokens_to_terminal` when diagnosing a slow or unhelpful generation.
+- Resize very large images before inference to reduce context and memory pressure.
+- Use the dedicated Gemma 4 Audio or Whisper node instead of assuming every multimodal model supports audio.
 
-- It explains the selected model family and recommended generation settings.
-- For every LTX 2.3, MiniMax H3, and Wan 2.2 video preset, it adds a preset-specific `enable_thinking` and `max_tokens` recommendation with a short reason.
-- Model sampling guidance and video-preset output budgets stay separate, so selecting a preset never replaces the selected model family's sampler advice.
-- It does not change your saved widget values.
-- For GGUF audio-capable nodes, it tells you whether the selected model actually supports audio.
-- In Advanced GGUF nodes, if you connect or expose audio for a model without known audio support, the box warns you instead of silently implying that audio will work.
+For explanations of thinking mode, model guidance, raw traces, audio, masks, and object-removal prompting, see [Features and workflows](docs/FEATURES_AND_WORKFLOWS.md).
 
-See [video preset settings](docs/VIDEO_PRESET_SETTINGS.md) for the complete recommendation table and source rationale.
+## Model families
 
-### The Thinking Toggle
+ThinkingLLM includes curated entries for:
 
-Every ThinkingLLM node exposes an `enable_thinking` toggle.
+- Qwen3.8, Qwen3.5, Qwen3-VL, Qwen2.5-VL, and Qwen3
+- Gemma 4 vision, text, and audio-capable GGUF variants
+- Whisper `tiny` through `large-v3` and `distil-large-v3`
+- Local compatible Transformers checkpoints and GGUF/mmproj pairs
 
-- When it is on, the model may reason internally before answering. That can make a run take longer.
-- For easy prompts, the model can still decide that explicit reasoning is unnecessary and answer directly. That is not a bug. It is a feature.
-- When it is off, the node sends a `/no_think` style directive to discourage visible reasoning and push a more direct one-shot answer.
+See [Models and backends](docs/MODELS_AND_BACKENDS.md) for exact families, model locations, local discovery, platform notes, and backend requirements.
 
-### Live token streaming
+## Remote API node
 
-Enable `stream_tokens_to_terminal` to print tokens into the ComfyUI terminal as they arrive.
+`ThinkingLLM API (OpenAI Compatible)` supports approved OpenAI-compatible providers while keeping credentials on the ComfyUI server. Provider keys are not stored in workflows.
 
-This is useful when:
+- [API setup, profiles, keys, and security](docs/OPENAI_COMPATIBLE_API.md)
+- [API image inputs](docs/OPENAI_COMPATIBLE_API_IMAGES.md)
 
-- a prompt seems stuck and you want to see whether the model is actually working
-- you want to debug a bad prompt while it is going wrong
-- you want to understand whether the model is thinking or simply producing a long answer
+## Documentation
 
-### Raw trace output
-
-Each node returns a second string output named `RAW_TRACE`.
-
-- The main output is the cleaned answer for downstream workflow use.
-- `RAW_TRACE` preserves the raw generation stream, including visible thinking blocks when the model emits them.
-
-That makes it possible to inspect what the model thought about the prompt while still keeping the primary output clean.
-
-### Audio support
-
-Audio is available through dedicated nodes so the model list stays honest:
-
-- `ThinkingLLM Gemma 4 Audio (GGUF)` for Gemma 4 audio-capable models.
-- `ThinkingLLM Whisper ASR` for reliable speech-to-text transcription through `faster-whisper`.
-
-The Advanced GGUF node also exposes optional `audio`, but this is for power users. Use the info box before assuming a model can hear audio. Normal Qwen/Qwen-VL models are not audio models.
-
-### Mask-focused image analysis and object removal
-
-The four ThinkingLLM vision nodes (`ThinkingLLM`, `ThinkingLLM (Advanced)`, `ThinkingLLM (GGUF)`, `ThinkingLLM (GGUF Advanced)`) accept an optional ComfyUI `MASK` input. White mask pixels select the area to analyze; black pixels preserve surrounding context, and soft mask values create a gradual transition. ThinkingLLM keeps the selected area at normal brightness while dimming everything outside it, helping the vision model focus on the mask without losing the scene's geometry, lighting, or background context.
-
-**Object-removal workflow**
-1. Connect the source image to `image` and the mask to `mask`.
-2. Select `👁️ Remove (Mask Area)` from `preset_prompt`.
-3. Optionally describe the desired replacement in `custom_prompt` (e.g. `continue the brick wall` or `fill with matching wooden floor`).
-4. Connect `RESPONSE` to the positive prompt of your diffusion inpainting workflow.
-5. Send the original image and mask separately to the diffusion model's inpainting inputs.
-
-ThinkingLLM does not edit the image itself — it examines the masked area and surrounding scene, then generates a positive description of what should appear inside the mask. The downstream diffusion model performs the actual replacement.
-
-**Previewing the LLM mask view**
-
-`ThinkingLLM (Advanced)` and `ThinkingLLM (GGUF Advanced)` return a third output `MASK_PREVIEW` — connect it to a ComfyUI **Preview Image** node to inspect the exact image used for analysis. The GGUF nodes also expose `mask_mode`: `focus` keeps the selected area at full brightness and dims the surroundings to 20%, while `reconstruct` conceals the selected area and asks the model to infer a natural continuation from the visible surroundings. With no mask connected, the preview remains the original image.
-
-Without a custom instruction, the preset infers the most natural background continuation — matching surfaces, perspective, materials, texture, color, lighting, shadows, reflections, and image style, while avoiding phrases such as "remove the object" or "erase this area."
-
-Example output:
-
-> Continuous warm beige plaster wall with subtle uneven texture, matching the existing perspective, soft window illumination, natural tonal variation, and the floor-edge shadows.
-
-**Notes and limitations**
-
-- Masks apply to still images only, not video input.
-- Mask dimensions are resized automatically to match the image.
-- An empty mask or a mask without an image produces a clear error.
-- Image batches continue to use the first image and first mask.
-- Precise masks with a small amount of surrounding context generally produce the best reconstruction prompt.
-
-## Nodes
-
-### Transformers / HF nodes
-
-This is the recommended default path: `ThinkingLLM`, `ThinkingLLM (Advanced)`, `ThinkingLLM Prompt Enhancer`. The HF path is the most straightforward option for Windows and Linux when you want the fewest install complications.
-
-HF tokenizer note: some Gemma and Qwen variants need `sentencepiece` or `tiktoken` in the ComfyUI Python environment. ThinkingLLM declares those dependencies and will attempt to install them automatically if a tokenizer backend is missing at first load.
-
-#### Shared ComfyUI text encoders
-
-Compatible single-file checkpoints already in `ComfyUI/models/text_encoders` appear as `[local] filename (text_encoders)` in every applicable model selector. ThinkingLLM loads them through ComfyUI, so it does not download or duplicate a Transformers snapshot. Older workflows saved with the former `[ComfyUI]` label remain compatible.
-
-- Qwen3-VL 4B and 8B `.safetensors`, including `qwen3vl_8b_fp8_scaled.safetensors`
-- Gemma 3 12B `.safetensors`, including `gemma-3-12b-it-heretic-v2_int8.safetensors`
-
-The MiniMax H3/Qwen3-VL 32B conditioning checkpoint is intentionally excluded because it is not a complete text-generation model. The valid SafeTensors extension is `.safetensors` (plural). ComfyUI-native generation supports one-beam sampling; terminal output is printed after generation rather than streamed token by token.
-
-### GGUF / llama.cpp nodes
-
-- `ThinkingLLM (GGUF)`
-- `ThinkingLLM (GGUF Advanced)`
-- `ThinkingLLM Gemma 4 Audio (GGUF)`
-- `ThinkingLLM Prompt Enhancer (GGUF)`
-
-The GGUF path requires a multimodal-capable `llama-cpp-python` build. The normal PyPI package may not include the chat handlers needed for Qwen vision, Gemma 4, or MTMD audio. On Linux, ThinkingLLM auto-checks this at first GGUF use and attempts an automatic install of a matching JamePeng backend. Use the setup notes in [docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md) if you need to override the wheel/package source.
-
-Model labels are consistent across the standard, Advanced, audio, and Prompt Enhancer selectors: `[installed]` marks a catalog model found in any configured `LLM`, `GGUF`, or `Qwen-VL` location, while `[local]` marks an uncatalogued local model. These labels are display-only and do not change saved model identities.
-
-### API node (OpenAI Compatible)
-
-`ThinkingLLM API (OpenAI Compatible)` calls approved OpenAI-compatible backends (OpenRouter, OrcaRouter, OpenAI, QwenCloud, Groq, Together, Fireworks, DeepInfra, Featherless) with credentials and endpoints bound server-side. `model_name` offers the full curated per-provider list; use `custom_model_name` for anything not listed.
-
-- Provider keys live in **server/user environment variables**, never in the workflow, profile JSON, or Git.
-- Full setup, security model, and per-provider key names: [docs/OPENAI_COMPATIBLE_API.md](docs/OPENAI_COMPATIBLE_API.md).
-- Image input: [docs/OPENAI_COMPATIBLE_API_IMAGES.md](docs/OPENAI_COMPATIBLE_API_IMAGES.md).
-
-## Workflow tips
-
-- **Pre-process input images** — use a resize or scale node before ThinkingLLM so large images don't blow up the context window.
-- **Display the output** — connect RESPONSE or ENHANCED_OUTPUT to a **Show Text** or **Show Anything** node to see the generated text in the UI.
-- **Inspect reasoning** — connect the RAW_TRACE output to a second Show Text node to see what the model was thinking.
-- **Use the dedicated audio nodes first** — Gemma 4 Audio is for audio understanding; Whisper ASR is for transcription. The Advanced GGUF audio input is intentionally more flexible, but the info box will warn you when the selected model has no curated audio support.
-
-### Duration-aware video prompts
-
-The four vision nodes and both Prompt Enhancer nodes expose an optional `duration_seconds` input. It becomes visible only for registered LTX 2.3 and MiniMax H3 video presets; all other presets ignore it. Connect the same requested duration value to ThinkingLLM and your downstream video generator so prompt timing and generated length stay aligned.
-
-Switching from a duration-aware preset back to `Custom Only` or another non-video preset hides the control without changing its ComfyUI serialization slot. The last valid duration is preserved; if an older workflow contains a corrupted non-numeric value in that slot, the frontend repairs it to the last valid value or `5.0` seconds. After updating ThinkingLLM, restart ComfyUI and use `Ctrl+F5` once so the corrected frontend code is loaded.
-
-- LTX 2.3 uses the requested value directly and returns one chronological paragraph with a natural total-duration phrase.
-- MiniMax H3 mirrors ComfyUI's 24 fps `17k+5` frame grid. For example, `5.0` seconds becomes 124 frames (`5.17` seconds), `8.0` stays 192 frames (`8.00` seconds), and `12.0` becomes 294 frames (`12.25` seconds). The effective duration is internal planning context: T2VA, I2VA, and full-reference outputs do not get an invented `Target duration:` field. FL2VA and L2VA use it only in the official image-alignment line.
-- The duration input overrides conflicting duration wording in the free prompt for duration-aware presets. Older saved nodes that do not contain the optional input use `5.0` seconds.
-
-See the [duration-aware video prompt guide](docs/VIDEO_PROMPT_DURATION.md) for node coverage, MiniMax frame normalization, Base and Full-Reference structures, cache behavior, best-effort normalization, and troubleshooting.
-
-For LTX, use `📖 LTX 2.3 NSFW T2V Scene` in a text Prompt Enhancer, or choose the matching I2V, First/Last-to-Video, or Reference-to-Video preset in a vision node. The First/Last preset treats the first two connected pictures as exact boundary frames; the Reference preset treats connected media as reusable identity, style, prop, environment, motion, or camera anchors.
-
-### MiniMax H3 prompt preset
-
-Select the dedicated T2VA, I2VA, FL2VA, L2VA, or Reference-to-Video preset that matches the connected media. Vision nodes expose all five formats; text-only Prompt Enhancer nodes expose only T2VA. The preset instructs the LLM to use the official three-field Base or six-section Full-Reference structure. After generation, ThinkingLLM only applies safe best-effort cleanup: it removes obsolete `Target duration:` declarations, normalizes recognizable cut timestamps, separates uniquely identifiable fields, and converts clear `<d>English ...</d>` variants to `<d>[English] ...</d>`. It does not reject the result when fields, speaker IDs, dialogue length, or timing remain imperfect.
-
-Standalone quoted dialogue blocks are presented to the LLM as verbatim source material. When the input is a longer master script, the preset asks the model to select one coherent moment that fits instead of compressing the complete story. The selected moment may be visual-only. This is prompt guidance, not a blocking post-generation validator; the downstream MiniMax pipeline receives the cleaned model output even if the model deviates slightly.
-
-For example, a 99-word multi-scene script with `duration_seconds=8.0` is valid input. ThinkingLLM selects one feasible short scene, with or without dialogue; it does not require all 99 dialogue words to play in eight seconds and does not compress the full narrative into rapid cuts.
-
-The effective duration remains authoritative planning context for the LLM. ThinkingLLM does not launch a schema-repair generation or block output because of an out-of-range timestamp, missing speaker tag, unexpected field, or estimated dialogue length; MiniMax performs its own downstream prompt interpretation. The only automatic retry is a direct finalization retry when generation ends before any usable final prompt, such as inside an unfinished `<think>` block.
-
-Reference-to-Video requires a custom prompt that describes the target video and assigns each reference a role such as identity, style, motion, camera, or voice. To animate one image as the first frame, use Text-to-Video instead. Start with `enable_thinking=false` and `max_tokens=2048` for Base formats or `max_tokens=3072` for Reference-to-Video. If a model still spends the initial budget inside an unfinished `<think>` block, ThinkingLLM discards that incomplete reasoning and automatically performs a direct schema-prefilled retry instead of forwarding it as the video prompt.
-
-See the official [MiniMax H3 ComfyUI workflows](https://docs.comfy.org/tutorials/video/minimax/minimax-h3), [base prompt guide](https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/docs/VIDEO_PROMPT_WRITING_GUIDE_base_en.md), and [full-reference prompt guide](https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/docs/VIDEO_PROMPT_WRITING_GUIDE_ref_en.md).
-
-## Supported models
-
-ThinkingLLM ships pre-configured entries in `hf_models.json` and `gguf_models.json`. Local GGUF models in `models/LLM/GGUF/` with a matching mmproj file are auto-discovered.
-
-Commercial-release dropdowns group entries by rights status and accept only reviewed, locked models. Community dropdowns keep normal model names and download behavior; gated Hugging Face repositories still require the model owner's access terms and a read token.
-
-### Qwen — Vision-language
-- **HF:** Qwen3.5 4B/9B · Qwen3-VL 4B/8B (Instruct, abliterated, unredacted) · Qwen3-VL 32B Heretic · Qwen2.5-VL 3B/7B
-- **GGUF:** Qwen3.8 27B Blackfrost Abliterated (Q2–Q8, image/video) · Qwen3.5 4B/9B/27B (Q4 K M–BF16) · Qwen3-VL 4B/8B/32B (Instruct, abliterated, thinking) · Qwen2.5-VL 3B/7B
-
-### Qwen — Text-only
-- **HF:** Qwen3 0.6B/4B/8B · Qwen3.5 4B/9B (base + heretic + uncensored)
-- **GGUF:** Qwen3.8 27B Blackfrost Abliterated (Q2–Q8) · Qwen3 4B/8B (abliterated, Josiefied, base) · Qwen3.5 4B/9B/27B (base + uncensored HauhauCS)
-
-### Gemma 4 — Vision-language
-- **HF:** Gemma-4-E2B/E4B-it/Uncensored · Gemma-4-12B · Gemma-4-26B-A4B-it/Heretic · Gemma-4-31B-it
-- **GGUF:** Gemma-4-E2B/E4B (Q4 K M–BF16, uncensored) · Gemma-4-12B-it (Q4/Q5/Q6/Q8/BF16/UD-Q4) · Gemma-4-26B-A4B-it · Gemma-4-31B-it (Q4 K M–BF16)
-
-### Gemma 4 — Text-only (HF)
-- Gemma-4-E2B-it / Uncensored · Gemma-4-E4B-it / Uncensored · Gemma-4-12B · Gemma-4-26B-A4B-it / Heretic · Gemma-4-31B-it
-
-### Gemma 4 — Audio (GGUF)
-- Gemma-4-E2B / E4B when present in the GGUF multimodal catalog · Gemma-4-12B-it via `unsloth/gemma-4-12b-it-GGUF`
-
-Use `ThinkingLLM Gemma 4 Audio (GGUF)` for the clean audio-only interface. Audio depends on a recent multimodal `llama-cpp-python`/`Gemma4ChatHandler` build — check `RAW_TRACE` if audio behaves unexpectedly.
-
-### Whisper — ASR
-tiny · base · small · medium · large-v3 · distil-large-v3
-
-Use `ThinkingLLM Whisper ASR` with a connected ComfyUI `AUDIO` input or an `audio_file_path` (M4A/MP3/WAV/FLAC/FFmpeg-readable). The node uses `faster-whisper`; first use may download the model into the Hugging Face cache. Default `small` on CPU/int8 is reliable on Windows; switch to CUDA once the local CTranslate2 CUDA runtime is confirmed working.
-
-## Model locations
-
-Models are discovered automatically.
-
-- HF models: `ComfyUI/models/LLM/Qwen-VL/`
-- GGUF models: `ComfyUI/models/LLM/GGUF/`
-
-For local GGUF models, keep the matching mmproj file beside the model file.
-
-## Platform notes
-
-- **HF is the recommended default path** and the simplest cross-platform option.
-- **Shared ComfyUI text encoders**: compatible single-file checkpoints in `ComfyUI/models/text_encoders` appear as `[local] filename (text_encoders)` in every applicable model selector and load through ComfyUI (no duplicate Transformers snapshot). Qwen3-VL 4B/8B and Gemma 3 12B `.safetensors` are supported. The MiniMax H3/Qwen3-VL 32B conditioning checkpoint is intentionally excluded (not a complete text-generation model). Valid extension is `.safetensors`; native generation uses one-beam sampling and prints terminal output after generation.
-- **GGUF on Windows** needs a matching `win_amd64` vision-capable wheel.
-- **GGUF on Linux** auto-installs a matching backend on first use when possible. Override with `THINKINGLLM_LLAMA_CPP_LINUX_WHEEL_URL` or `THINKINGLLM_LLAMA_CPP_LINUX_SPEC` when your server needs a different build.
-- **GGUF audio** needs an audio-capable multimodal backend. `Gemma4ChatHandler` is used for Gemma 4.
-- **Whisper ASR** needs `faster-whisper` and FFmpeg. The node returns an install hint if `faster-whisper` is missing.
-- **Flash Attention** support is best on Linux. The nodes fall back when it is unavailable.
-- **The thinking toggle** works best with Qwen3.5 and Qwen3 style models. Other architectures may ignore the steering.
+| Topic | Guide |
+| --- | --- |
+| Thinking, streaming, traces, audio, masks, and workflow behavior | [Features and workflows](docs/FEATURES_AND_WORKFLOWS.md) |
+| Supported models, discovery, locations, and backend selection | [Models and backends](docs/MODELS_AND_BACKENDS.md) |
+| GGUF / llama.cpp installation on Windows and Linux | [llama.cpp backend installation](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md) |
+| Duration-aware LTX and MiniMax H3 prompting | [Duration-aware video prompts](docs/VIDEO_PROMPT_DURATION.md) |
+| Video preset recommendations and rationale | [Video preset settings](docs/VIDEO_PRESET_SETTINGS.md) |
+| OpenAI-compatible API node | [API setup and security](docs/OPENAI_COMPATIBLE_API.md) |
+| OpenAI-compatible image inputs | [API image inputs](docs/OPENAI_COMPATIBLE_API_IMAGES.md) |
+| Maintainer and fork notes | [Maintainer notes](docs/MAINTAINER_NOTES.md) |
 
 ## Production deployment
 
-This repository provides ComfyUI nodes, not a complete production runtime. If you plan to expose a workflow through an API on RunPod Serverless, review the dependency, validation, testing, and rollback requirements in the [ComfyUI on RunPod production-readiness checklist](https://comfyrail.dev/production-readiness).
+This repository provides ComfyUI nodes, not a complete production runtime. For exposed or serverless workflows, review dependency locking, validation, secrets, cost controls, monitoring, and rollback before deployment. The [ComfyUI on RunPod production-readiness checklist](https://comfyrail.dev/production-readiness) is a useful starting point.
 
-## FAQ
-
-**Why does the node sometimes answer quickly and sometimes take longer?**
-
-When thinking is enabled, the model may decide to reason before answering. For easy prompts it may skip explicit reasoning. That is expected behavior.
-
-**How do I see what the model was thinking?**
-
-Use the second `RAW_TRACE` output, or enable live terminal token streaming.
-
-**How do I debug prompts that seem bad or stuck?**
-
-Turn on `stream_tokens_to_terminal`. It is there specifically so you can see the generation live and catch bad prompt behavior sooner.
-
-**Can I call remote/cloud models?**
-
-Yes — the **ThinkingLLM API (OpenAI Compatible)** node calls approved OpenAI-compatible providers with credentials kept server-side. See [docs/OPENAI_COMPATIBLE_API.md](docs/OPENAI_COMPATIBLE_API.md).
-
-## Documentation index
-
-| Topic | Doc |
-| --- | --- |
-| OpenAI-compatible API node (setup, keys, security) | [docs/OPENAI_COMPATIBLE_API.md](docs/OPENAI_COMPATIBLE_API.md) |
-| API node image inputs | [docs/OPENAI_COMPATIBLE_API_IMAGES.md](docs/OPENAI_COMPATIBLE_API_IMAGES.md) |
-| GGUF / llama.cpp vision install (Windows + Linux) | [docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md) |
-| Video preset recommendation table | [docs/VIDEO_PRESET_SETTINGS.md](docs/VIDEO_PRESET_SETTINGS.md) |
-| Duration-aware video prompts | [docs/VIDEO_PROMPT_DURATION.md](docs/VIDEO_PROMPT_DURATION.md) |
-| Maintainer notes (fork lineage, publishability) | [docs/MAINTAINER_NOTES.md](docs/MAINTAINER_NOTES.md) |
-
-## Credits and fork lineage
-
-This fork preserves the GPL-3.0 lineage of its predecessors.
+## Credits
 
 - Deaquay/ComfyUI-Qwen3.5-Uncensored
 - huchukato/ComfyUI-QwenVL-Mod
@@ -299,8 +117,7 @@ This fork preserves the GPL-3.0 lineage of its predecessors.
 - JamePeng/llama-cpp-python
 - comfyanonymous/ComfyUI
 
-Maintainer: goodguy1963  
-Planned repo: [goodguy1963/ComfyUI-ThinkingLLM](https://github.com/goodguy1963/ComfyUI-ThinkingLLM)
+Maintainer: [goodguy1963](https://github.com/goodguy1963)
 
 ## License
 

__init__.py

@@ -3,7 +3,7 @@
     "name": "ComfyUI-ThinkingLLM",
     "description": "A multimodal ComfyUI AI node with Qwen3.5, Qwen3-VL, Qwen2.5-VL, Qwen3, and Gemma 4 integrations. Features live thinking in the terminal to see what the LLM is doing in real time.",
     "author": "goodguy1963",
-    "version": "2.5.6",
+    "version": "2.5.8",
     "url": "https://github.com/goodguy1963/ComfyUI-ThinkingLLM",
     "category": "image"
 }

custom_nodes.json

@@ -9,5 +9,5 @@
   "description": "A multimodal ComfyUI AI node with Qwen3.5, Qwen3-VL, Qwen2.5-VL, Qwen3, and Gemma 4 integrations. Features live thinking in the terminal to see what the LLM is doing in real time.",
   "category": "image",
   "tags": ["vision", "language", "multimodal", "thinkingllm", "qwen", "gguf", "flash_attention", "performance", "wan2.2", "video", "i2v", "t2v", "storyboard", "caching", "fork", "gpl-3.0"],
-  "version": "2.5.6"
+  "version": "2.5.8"
 }

docs/FEATURES_AND_WORKFLOWS.md

@@ -0,0 +1,109 @@
+# Features and workflows
+
+This guide explains how ThinkingLLM's user-facing controls affect inference and how to choose the right workflow. For installation and the short node list, start with the [README](../README.md).
+
+## Model guidance box
+
+ThinkingLLM nodes add a read-only `recommended_settings` box. It explains the selected model family and suggests generation settings without changing saved widget values.
+
+For LTX 2.3, MiniMax H3, and Wan 2.2 presets, the box also provides preset-specific `enable_thinking` and `max_tokens` guidance. Model sampling recommendations and video output budgets remain separate so a video preset does not silently overwrite model-family advice.
+
+Audio-capable GGUF nodes use the same box to distinguish models with curated audio support. The Advanced GGUF node warns when audio is connected to a model that is not known to support it.
+
+The complete video recommendation table and its rationale are in [Video preset settings](VIDEO_PRESET_SETTINGS.md).
+
+## Thinking mode
+
+Every inference node exposes `enable_thinking`.
+
+- When enabled, a compatible model may reason before answering. This can improve difficult responses but uses more time and output budget.
+- A model may still answer directly when it considers the task simple.
+- When disabled, ThinkingLLM applies the appropriate direct-answer steering for the selected model family, such as a `/no_think` directive where supported.
+- For transcription, short classification, and concise prompt enhancement, begin with thinking disabled.
+
+ThinkingLLM keeps incomplete or visible reasoning out of the cleaned primary response. When supported, reasoning remains available in `RAW_TRACE` for diagnostics.
+
+## Live terminal streaming
+
+Enable `stream_tokens_to_terminal` to display generation in the ComfyUI terminal as tokens arrive. This is useful when a run appears stuck, when you want to catch a poor prompt early, or when you need to distinguish model reasoning from slow loading.
+
+Terminal streaming is diagnostic output. Downstream nodes should consume the normal response output instead.
+
+## Response and raw trace
+
+Inference nodes return a cleaned answer and a second string named `RAW_TRACE`.
+
+- `RESPONSE` or `ENHANCED_OUTPUT` is intended for display and downstream workflow use.
+- `RAW_TRACE` preserves the raw generation stream and backend diagnostics, including visible reasoning when the model emits it.
+
+Connect `RAW_TRACE` to a separate text display when diagnosing empty output, unexpected thinking, backend selection, model loading, or generation formatting.
+
+## Audio workflows
+
+### Gemma 4 audio understanding
+
+Use `ThinkingLLM Gemma 4 Audio (GGUF)` for audio questions, descriptions, classification, and transcription with a compatible Gemma 4 model.
+
+The node accepts a ComfyUI `AUDIO` input or an `audio_file_path`. Audio is normalized to 16 kHz mono WAV before inference. Native Gemma 4 audio requires:
+
+- multimodal `llama-cpp-python` 0.3.36 or newer;
+- `Gemma4ChatHandler`;
+- an mmproj that positively reports audio support.
+
+ThinkingLLM verifies these conditions before sending audio to inference. Older or unknown native paths remain blocked because they can terminate the ComfyUI process instead of returning a Python exception. BF16 mmproj is recommended for Gemma 4 E2B/E4B audio.
+
+### Whisper speech-to-text
+
+Use `ThinkingLLM Whisper ASR` when the primary goal is dependable transcription. It accepts ComfyUI audio and common FFmpeg-readable paths such as M4A, MP3, WAV, and FLAC.
+
+The node uses `faster-whisper`. The first run may download the selected model. The `small` model on CPU/int8 is a practical Windows starting point; use CUDA after confirming that the local CTranslate2 CUDA runtime works.
+
+### Advanced GGUF audio input
+
+The Advanced GGUF node exposes optional audio for expert workflows, but most Qwen and Qwen-VL models are not audio models. Prefer the dedicated nodes unless the selected model and mmproj explicitly support audio.
+
+## Mask-focused image analysis
+
+The standard and Advanced HF/GGUF vision nodes accept an optional ComfyUI `MASK` input.
+
+- White pixels select the area of interest.
+- Black pixels preserve surrounding context.
+- Soft values create a gradual transition.
+- Mask dimensions are resized to the image automatically.
+
+In `focus` mode, ThinkingLLM keeps the selected area at normal brightness and dims the surroundings. This directs the vision model without removing spatial, lighting, or scene context.
+
+In `reconstruct` mode, the selected area is concealed and the model is asked to infer a natural continuation from visible surroundings. This is useful for producing a positive inpainting prompt.
+
+### Object-removal prompt workflow
+
+1. Connect the original image and mask to a ThinkingLLM vision node.
+2. Select `Remove (Mask Area)` from `preset_prompt`.
+3. Optionally describe the desired replacement in `custom_prompt`.
+4. Connect `RESPONSE` to the positive prompt of a diffusion inpainting workflow.
+5. Send the original image and mask separately to the diffusion model's inpainting inputs.
+
+ThinkingLLM does not edit the image. It describes what should naturally appear inside the masked region; the downstream diffusion model performs the replacement.
+
+Advanced nodes return `MASK_PREVIEW`. Connect it to **Preview Image** to inspect exactly what the LLM receives.
+
+Mask limitations:
+
+- Masks apply to still images, not video.
+- An empty mask or a mask without an image returns a clear error.
+- Image batches use the first image and first mask.
+- Precise masks with a modest amount of surrounding context generally produce the best reconstruction prompt.
+
+## Video prompt workflows
+
+ThinkingLLM provides preset-specific guidance for LTX 2.3, MiniMax H3, and Wan 2.2 workflows. The optional `duration_seconds` input appears only for registered duration-aware presets.
+
+Use the preset that matches the downstream workflow and connected media. MiniMax H3 offers separate T2VA, I2VA, FL2VA, L2VA, and Reference-to-Video contracts; text-only Prompt Enhancer nodes expose T2VA.
+
+Detailed duration resolution, MiniMax frame-grid normalization, output structures, long-script handling, best-effort cleanup, and recommended budgets are documented in [Duration-aware video prompts](VIDEO_PROMPT_DURATION.md).
+
+## Shared model files and caching
+
+ThinkingLLM reuses compatible model files discovered in configured ComfyUI locations. Fixed-seed local generations may reuse saved prompt output when all relevant inputs match. Audio-node results skip prompt persistence because the external audio content must not be mistaken for a reusable text-only result.
+
+For supported file layouts and model labels, see [Models and backends](MODELS_AND_BACKENDS.md).

docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md

@@ -4,6 +4,7 @@ This plugin’s **QwenVL (GGUF)** vision nodes require a `llama-cpp-python` buil
 
 - `Qwen3VLChatHandler`
 - `Qwen25VLChatHandler`
+- `Gemma4ChatHandler` (`llama-cpp-python` 0.3.36 or newer for Gemma 4 audio)
 
 The upstream `llama-cpp-python` from PyPI often does **not** include these vision handlers. Use a fork/build that provides them (e.g. JamePeng’s fork) and install a **Release wheel**.
 

docs/MODELS_AND_BACKENDS.md

@@ -0,0 +1,120 @@
+# Models and backends
+
+This guide describes ThinkingLLM's inference paths, curated model families, local discovery, and platform requirements.
+
+## Choose a backend
+
+### Transformers / Hugging Face
+
+The HF path is the recommended default for Windows and Linux when you want the fewest installation complications. It powers:
+
+- `ThinkingLLM`
+- `ThinkingLLM (Advanced)`
+- `ThinkingLLM Prompt Enhancer`
+
+Some Gemma and Qwen tokenizers require `sentencepiece` or `tiktoken`. ThinkingLLM declares both dependencies and attempts to install a missing tokenizer backend at first load.
+
+### GGUF / llama.cpp
+
+The GGUF path powers:
+
+- `ThinkingLLM (GGUF)`
+- `ThinkingLLM (GGUF Advanced)`
+- `ThinkingLLM Gemma 4 Audio (GGUF)`
+- `ThinkingLLM Prompt Enhancer (GGUF)`
+
+Multimodal GGUF inference requires a llama.cpp Python build with the chat handlers used by the selected architecture. The regular PyPI package may not include the required Qwen vision, Gemma 4, or MTMD audio support. Follow [Install llama-cpp-python](LLAMA_CPP_PYTHON_VISION_INSTALL.md) for platform-specific setup and verification.
+
+### Whisper ASR
+
+`ThinkingLLM Whisper ASR` uses `faster-whisper` for speech-to-text. It is independent of the Gemma 4 GGUF audio path and requires FFmpeg for general audio-file decoding.
+
+### OpenAI-compatible API
+
+The API node calls approved remote providers with credentials bound on the ComfyUI server. See [ThinkingLLM API](OPENAI_COMPATIBLE_API.md) for setup and security boundaries.
+
+## Curated model families
+
+ThinkingLLM ships model metadata in `hf_models.json` and `gguf_models.json`.
+
+### Qwen vision-language
+
+- **HF:** Qwen3.5 4B/9B; Qwen3-VL 4B/8B/32B; Qwen2.5-VL 3B/7B
+- **GGUF:** Qwen3.8 27B; Qwen3.5 4B/9B/27B; Qwen3-VL 4B/8B/32B; Qwen2.5-VL 3B/7B
+
+Catalog variants include selected Instruct, thinking, abliterated, heretic, unredacted, and uncensored releases.
+
+### Qwen text-only
+
+- **HF:** Qwen3 0.6B/4B/8B; Qwen3.5 4B/9B
+- **GGUF:** Qwen3.8 27B; Qwen3 4B/8B; Qwen3.5 4B/9B/27B
+
+### Gemma 4 vision-language
+
+- **HF:** Gemma 4 E2B/E4B, 12B, 26B-A4B, and 31B families
+- **GGUF:** Gemma 4 E2B/E4B, 12B, 26B-A4B, and 31B families across selected quantizations
+
+### Gemma 4 text-only
+
+Selected E2B, E4B, 12B, 26B-A4B, and 31B HF variants are available in text workflows.
+
+### Gemma 4 audio
+
+The audio selector includes curated audio-capable Gemma 4 E2B, E4B, and 12B GGUF entries. Audio also depends on the selected mmproj and a compatible `Gemma4ChatHandler`; a model name alone does not guarantee audio support.
+
+### Whisper
+
+Available sizes include `tiny`, `base`, `small`, `medium`, `large-v3`, and `distil-large-v3`.
+
+The catalog files are the authoritative current list. Dropdown contents can evolve between releases as models and compatibility information change.
+
+## Model locations
+
+ThinkingLLM searches configured ComfyUI model paths and common defaults, including:
+
+- HF snapshots: `ComfyUI/models/LLM/Qwen-VL/`
+- GGUF models: `ComfyUI/models/LLM/GGUF/`
+- compatible single-file checkpoints: `ComfyUI/models/text_encoders/`
+
+Keep a GGUF model and its matching mmproj together. Local GGUF files with an mmproj are discovered even when they are not in the bundled catalog.
+
+## Model labels
+
+- `[installed]` means a catalog model was found in a configured `LLM`, `GGUF`, or `Qwen-VL` location.
+- `[local]` means an uncatalogued compatible local model was discovered.
+
+Labels are display-only. They do not alter saved model identities, and older saved display labels remain compatible where supported.
+
+## Shared ComfyUI text encoders
+
+Compatible checkpoints in `ComfyUI/models/text_encoders` appear in applicable selectors and are loaded through ComfyUI rather than copied into a second Transformers snapshot.
+
+Supported examples include:
+
+- Qwen3-VL 4B and 8B `.safetensors`, including `qwen3vl_8b_fp8_scaled.safetensors`
+- Gemma 3 12B `.safetensors`, including `gemma-3-12b-it-heretic-v2_int8.safetensors`
+
+The MiniMax H3/Qwen3-VL 32B conditioning checkpoint is excluded because it is not a complete text-generation model. ComfyUI-native generation supports one-beam sampling and prints terminal output after generation rather than streaming token-by-token.
+
+## Community and commercial catalogs
+
+Community dropdowns preserve normal model names and download behavior. Gated Hugging Face repositories still require acceptance of the model owner's terms and a read token.
+
+Commercial-release dropdowns include only reviewed, locked models and group entries by rights status. This restriction is intentionally separate from Community behavior.
+
+## Platform notes
+
+- **Windows GGUF:** install a matching `win_amd64` multimodal wheel for the Python version bundled with ComfyUI.
+- **Linux GGUF:** ThinkingLLM checks the backend at first use and can install a verified matching JamePeng wheel. Environment overrides are documented in the installation guide.
+- **Gemma 4 audio:** requires multimodal `llama-cpp-python` 0.3.36 or newer and an mmproj that reports audio support.
+- **Whisper:** requires `faster-whisper`; file-path decoding also requires FFmpeg.
+- **Hugging Face attention:** set `attention_mode` to `sage` to request SageAttention instead of Flash Attention 2. `auto` tries SageAttention, then Flash Attention 2, then PyTorch SDPA. On ROCm, SageAttention 1.0.6 uses its Triton backend when available in the same Python environment as ComfyUI. Unsupported models or attention inputs use SDPA. The GGUF node's `flash_attn` setting belongs to llama.cpp and does not select SageAttention.
+- **Flash Attention:** support is strongest on Linux and falls back when unavailable.
+- **Thinking controls:** Qwen3/Qwen3.5-style models generally respond most consistently to explicit thinking steering; other architectures may interpret it differently.
+
+## Troubleshooting direction
+
+- Missing or incompatible GGUF handlers: use [the llama.cpp installation guide](LLAMA_CPP_PYTHON_VISION_INSTALL.md).
+- Unexpected response versus reasoning output: inspect `RAW_TRACE` and [Features and workflows](FEATURES_AND_WORKFLOWS.md).
+- Remote provider setup or security errors: use [the API guide](OPENAI_COMPATIBLE_API.md).
+- Duration or MiniMax structure questions: use [Duration-aware video prompts](VIDEO_PROMPT_DURATION.md).

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "ComfyUI-ThinkingLLM"
 description = "A multimodal ComfyUI AI node with Qwen3.5, Qwen3-VL, Qwen2.5-VL, Qwen3, and Gemma 4 integrations. Features live thinking in the terminal to see what the LLM is doing in real time."
-version = "2.5.6"
+version = "2.5.8"
 license = {file = "LICENSE"}
 dependencies = ["transformers>=5.2.0", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "sentencepiece", "tiktoken", "faster-whisper", "llama-cpp-python"]
 

sageattention_patch.py

@@ -1,144 +1,129 @@
-"""
-SageAttention patching functions for QwenVL models.
-This module contains the safe SageAttention implementation with proper error handling.
-"""
+"""Model-local SageAttention patching for supported Qwen attention layers."""
 
 import torch
 
 
 def set_sage_attention(model):
-    """
-    Apply SageAttention patching to the model.
-    Patches Qwen2Attention and Qwen3VLTextAttention modules to use SageAttention kernels.
-    """
-    from thinkingllm_core.hf_models import get_sage_attention_config, sage_attn_available
-    
-    if not sage_attn_available():
-        raise ImportError("SageAttention library is not installed or GPU doesn't support it.")
+    """Patch supported Qwen attention layers, falling back per call when needed."""
+    from thinkingllm_core.hf_models import get_sage_attention_config
 
-    SAGE_ATTN_FUNC, QK_QUANT_GRAN, PV_ACCUM_DTYPE = get_sage_attention_config()
-    if SAGE_ATTN_FUNC is None:
-        raise RuntimeError("No compatible SageAttention kernel found for this GPU.")
+    attn_func, qk_quant_gran, pv_accum_dtype = get_sage_attention_config()
+    if attn_func is None:
+        raise RuntimeError("No compatible SageAttention kernel found for this GPU")
 
-    # Try to import different attention classes for different Qwen models
     attention_classes = []
-
-    # Qwen2 models
     try:
-        from transformers.models.qwen2.modeling_qwen2 import Qwen2Attention, apply_rotary_pos_emb as qwen2_apply_rotary
-        attention_classes.append((Qwen2Attention, qwen2_apply_rotary))
+        from transformers.models.qwen2.modeling_qwen2 import Qwen2Attention, apply_rotary_pos_emb
+        attention_classes.append((Qwen2Attention, apply_rotary_pos_emb))
+    except ImportError:
+        pass
+    try:
+        from transformers.models.qwen3.modeling_qwen3 import Qwen3Attention, apply_rotary_pos_emb
+        attention_classes.append((Qwen3Attention, apply_rotary_pos_emb))
+    except ImportError:
+        pass
+    try:
+        from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLTextAttention, apply_rotary_pos_emb
+        attention_classes.append((Qwen3VLTextAttention, apply_rotary_pos_emb))
     except ImportError:
         pass
 
-    # Qwen3 models (Qwen3-VL, etc.)
-    try:
-        from transformers.models.qwen3.modeling_qwen3 import Qwen3Attention, apply_rotary_pos_emb as qwen3_apply_rotary
-        attention_classes.append((Qwen3Attention, qwen3_apply_rotary))
-    except ImportError:
-        pass
-
-    # Qwen3-VL specific
-    try:
-        from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLTextAttention, apply_rotary_pos_emb as qwen3vl_apply_rotary
-        attention_classes.append((Qwen3VLTextAttention, qwen3vl_apply_rotary))
-    except ImportError:
-        pass
-
-    if not attention_classes:
-        print("[QwenVL] Could not import any attention classes for SageAttention patching")
-        return
-
-    def make_sage_forward(AttentionClass, apply_rotary_pos_emb_func):
-        def sage_attention_forward(
+    def make_sage_forward(original_forward, apply_rotary):
+        def sage_forward(
             self,
             hidden_states: torch.Tensor,
             position_embeddings: tuple = None,
             attention_mask: torch.Tensor = None,
             past_key_values=None,
             cache_position: torch.LongTensor = None,
-            position_ids: torch.LongTensor = None,
             **kwargs,
         ):
-            original_dtype = hidden_states.dtype
+            def fallback():
+                return original_forward(
+                    hidden_states,
+                    position_embeddings=position_embeddings,
+                    attention_mask=attention_mask,
+                    past_key_values=past_key_values,
+                    cache_position=cache_position,
+                    **kwargs,
+                )
 
-            # Determine target dtype
-            is_4bit = hasattr(self.q_proj, 'quant_state')
-            if is_4bit:
-                target_dtype = torch.bfloat16
-            else:
-                target_dtype = self.q_proj.weight.dtype
-
-            if hidden_states.dtype != target_dtype:
-                hidden_states = hidden_states.to(target_dtype)
+            # The Triton v1 kernel supports these dimensions and causal
+            # attention only when query and key sequence lengths agree.
+            if (
+                hidden_states.ndim != 3
+                or hidden_states.shape[1] < 1
+                or (self.head_dim not in (64, 96, 128) if qk_quant_gran is None
+                    else not 0 < self.head_dim <= 128)
+                or attention_mask is not None
+                or position_embeddings is None
+                or len(position_embeddings) != 2
+                or getattr(self, "sliding_window", None) is not None
+                or self.training
+                or any(name not in ("position_ids", "use_cache") for name in kwargs)
+                or (kwargs.get("use_cache") is False and past_key_values is not None)
+                or (past_key_values is not None and hidden_states.shape[1] != 1)
+                or hidden_states.dtype not in (torch.float16, torch.bfloat16)
+            ):
+                return fallback()
 
             input_shape = hidden_states.shape[:-1]
             hidden_shape = (*input_shape, -1, self.head_dim)
-            bsz, q_len = input_shape[0], input_shape[1] if len(input_shape) > 1 else hidden_states.size(1)
+            query_states = self.q_proj(hidden_states).view(hidden_shape)
+            key_states = self.k_proj(hidden_states).view(hidden_shape)
+            value_states = self.v_proj(hidden_states).view(hidden_shape)
+            if hasattr(self, "q_norm"):
+                query_states = self.q_norm(query_states)
+            if hasattr(self, "k_norm"):
+                key_states = self.k_norm(key_states)
+            query_states = query_states.transpose(1, 2)
+            key_states = key_states.transpose(1, 2)
+            value_states = value_states.transpose(1, 2)
+            cos, sin = position_embeddings
+            query_states, key_states = apply_rotary(query_states, key_states, cos, sin)
 
-            # Handle q_norm and k_norm for Qwen3-VL
-            query_states = self.q_proj(hidden_states)
-            key_states = self.k_proj(hidden_states)
-            value_states = self.v_proj(hidden_states)
-
-            # Apply normalization if available (Qwen3-VL specific)
-            if hasattr(self, 'q_norm'):
-                query_states = self.q_norm(query_states.view(hidden_shape)).transpose(1, 2)
-            else:
-                query_states = query_states.view(hidden_shape).transpose(1, 2)
-
-            if hasattr(self, 'k_norm'):
-                key_states = self.k_norm(key_states.view(hidden_shape)).transpose(1, 2)
-            else:
-                key_states = key_states.view(hidden_shape).transpose(1, 2)
-
-            value_states = value_states.view(hidden_shape).transpose(1, 2)
-
-            # Apply rotary embeddings
-            if position_embeddings is not None:
-                cos, sin = position_embeddings
-                query_states, key_states = apply_rotary_pos_emb_func(query_states, key_states, cos, sin)
+            # This check precedes cache.update(), since calling the original
+            # forward after an update would append the same token twice.
+            if (query_states.dtype != key_states.dtype or key_states.dtype != value_states.dtype
+                    or query_states.dtype not in (torch.float16, torch.bfloat16)):
+                return fallback()
 
             if past_key_values is not None:
-                cache_kwargs = {"sin": sin if position_embeddings else None, "cos": cos if position_embeddings else None, "cache_position": cache_position}
-                key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)
-
-            is_causal = attention_mask is None and q_len > 1
-
-            attn_output = SAGE_ATTN_FUNC(
-                query_states.to(target_dtype),
-                key_states.to(target_dtype),
-                value_states.to(target_dtype),
-                tensor_layout="HND",
-                is_causal=is_causal,
-                qk_quant_gran=QK_QUANT_GRAN,
-                pv_accum_dtype=PV_ACCUM_DTYPE,
-            )
+                cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+                key_states, value_states = past_key_values.update(
+                    key_states, value_states, self.layer_idx, cache_kwargs
+                )
 
+            is_causal = input_shape[1] > 1
+            if qk_quant_gran is None:
+                # SageAttention 1 smooths K in place, including cached K.
+                attn_output = attn_func(
+                    query_states, key_states.clone(), value_states,
+                    tensor_layout="HND", is_causal=is_causal,
+                    sm_scale=getattr(self, "scaling", self.head_dim ** -0.5),
+                )
+            else:
+                attn_output = attn_func(
+                    query_states, key_states, value_states,
+                    tensor_layout="HND", is_causal=is_causal,
+                    qk_quant_gran=qk_quant_gran, pv_accum_dtype=pv_accum_dtype,
+                )
             if isinstance(attn_output, tuple):
                 attn_output = attn_output[0]
+            attn_output = attn_output.transpose(1, 2).contiguous().reshape(*input_shape, -1)
+            return self.o_proj(attn_output), None
 
-            attn_output = attn_output.transpose(1, 2).contiguous()
-            attn_output = attn_output.reshape(*input_shape, -1)
+        return sage_forward
 
-            attn_output = self.o_proj(attn_output)
-
-            if attn_output.dtype != original_dtype:
-                attn_output = attn_output.to(original_dtype)
-
-            return attn_output, None
-
-        return sage_attention_forward
-
-    # Apply patching to all supported attention modules
     patched_count = 0
-    for AttentionClass, apply_rotary_func in attention_classes:
-        sage_forward = make_sage_forward(AttentionClass, apply_rotary_func)
-        for module in model.modules():
-            if isinstance(module, AttentionClass):
-                module.forward = sage_forward.__get__(module, AttentionClass)
+    for module in model.modules():
+        for attention_class, apply_rotary in attention_classes:
+            if isinstance(module, attention_class):
+                sage_forward = make_sage_forward(module.forward, apply_rotary)
+                module.forward = sage_forward.__get__(module, attention_class)
                 patched_count += 1
+                break
 
-    if patched_count > 0:
-        print(f"[QwenVL] SageAttention: Patched {patched_count} attention layers")
-    else:
-        print("[QwenVL] SageAttention: No compatible attention layers found to patch")
+    if not patched_count:
+        raise RuntimeError("SageAttention: No compatible attention layers found to patch")
+    print(f"[QwenVL] SageAttention: Patched {patched_count} attention layers")

tests/test_new_features.py

@@ -4174,8 +4174,9 @@ class TestGGUFWindllRuntimeFallback(unittest.TestCase):
         loader.current_batch_size = 1120
         loader.current_ubatch_size = 1120
         loader.current_image_token_budget = 1120
+        loader.current_backend_info = {"version": "0.3.35"}
 
-        with self.assertRaisesRegex(RuntimeError, "Gemma 4 GGUF audio input is disabled"):
+        with self.assertRaisesRegex(RuntimeError, "requires llama-cpp-python >=0.3.36"):
             loader._invoke(
                 system_prompt="You are helpful.",
                 user_prompt="Transcribe this audio.",
@@ -4193,6 +4194,87 @@ class TestGGUFWindllRuntimeFallback(unittest.TestCase):
                 top_k=64,
             )
 
+    def test_gemma4_audio_runs_when_backend_and_mmproj_report_support(self):
+        with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False):
+            module = load_module_from_file(
+                "AILab_QwenVL_GGUF.py",
+                "thinkingllm_gemma4_audio_supported_backend_test",
+            )
+
+        captured_messages = []
+
+        class DummyChatHandler:
+            def __init__(self):
+                self.is_support_audio = None
+                self.initialized_with = None
+
+            def _init_mtmd_context(self, llama):
+                self.initialized_with = llama
+                self.is_support_audio = True
+
+        class DummyLlama:
+            def reset(self):
+                return None
+
+            def create_chat_completion(self, **kwargs):
+                captured_messages.append(kwargs["messages"])
+                return iter([{"choices": [{"delta": {"content": "audio understood"}}]}])
+
+        loader = module.QwenVLGGUFBase()
+        loader.llm = DummyLlama()
+        loader.chat_handler = DummyChatHandler()
+        loader.gguf_arch = "gemma4"
+        loader.current_backend_info = {"version": "0.3.36+cu130"}
+        loader.current_context_length = 32768
+        loader.current_batch_size = 1120
+        loader.current_ubatch_size = 1120
+        loader.current_image_token_budget = 1120
+
+        text, _raw_trace = loader._invoke(
+            system_prompt="You are helpful.",
+            user_prompt="Transcribe this audio.",
+            images_b64=[],
+            audio_b64=["BASE64WAV"],
+            max_tokens=256,
+            temperature=0.0,
+            top_p=1.0,
+            repetition_penalty=1.0,
+            seed=1,
+            model_name="google_gemma-4-E4B-it-Q8_0.gguf",
+            stream_to_terminal=False,
+            enable_thinking=False,
+            auto_finalization_retry=False,
+            top_k=64,
+        )
+
+        self.assertEqual(text, "audio understood")
+        self.assertIs(loader.chat_handler.initialized_with, loader.llm)
+        audio_item = captured_messages[0][1]["content"][1]
+        self.assertEqual(audio_item["type"], "input_audio")
+        self.assertEqual(audio_item["input_audio"], {"data": "BASE64WAV", "format": "wav"})
+
+    def test_gemma4_audio_guard_blocks_mmproj_without_audio_support(self):
+        with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False):
+            module = load_module_from_file(
+                "AILab_QwenVL_GGUF.py",
+                "thinkingllm_gemma4_audio_unsupported_mmproj_test",
+            )
+
+        class DummyChatHandler:
+            is_support_audio = None
+
+            def _init_mtmd_context(self, _llama):
+                self.is_support_audio = False
+
+        loader = module.QwenVLGGUFBase()
+        loader.llm = object()
+        loader.chat_handler = DummyChatHandler()
+        loader.gguf_arch = "gemma4"
+        loader.current_backend_info = {"version": "0.3.36"}
+
+        with self.assertRaisesRegex(RuntimeError, "does not report audio support"):
+            loader._raise_if_unsafe_native_multimodal_path([], ["BASE64WAV"])
+
     def test_gguf_thinking_toggle_updates_without_reload_signature(self):
         with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False):
             module = load_module_from_file(

tests/test_sage_attention.py

@@ -0,0 +1,164 @@
+"""Regression checks for SageAttention selection and model-local fallback."""
+
+import ast
+import importlib.util
+import sys
+import types
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import torch
+
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def load_attention_selector(*, hip, sage_version="1.0.6", triton_kernel=None, cuda_kernels=False):
+    tree = ast.parse((ROOT / "thinkingllm_core/hf_models.py").read_text(encoding="utf-8"))
+    names = {"sage_attn_available", "get_sage_attention_config", "resolve_attention_mode"}
+    functions = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in names]
+    cuda = types.SimpleNamespace(
+        is_available=lambda: True,
+        get_device_capability=(
+            (lambda: (_ for _ in ()).throw(AssertionError("HIP queried CUDA SM")))
+            if hip else (lambda: (8, 6))
+        ),
+    )
+    metadata = types.SimpleNamespace(
+        version=lambda package: sage_version,
+        PackageNotFoundError=Exception,
+    )
+    namespace = {
+        "torch": types.SimpleNamespace(cuda=cuda, version=types.SimpleNamespace(hip=hip)),
+        "importlib": types.SimpleNamespace(metadata=metadata),
+        "sageattn_triton": triton_kernel,
+        "SAGE_ATTENTION_AVAILABLE": cuda_kernels,
+        "sageattn_qk_int8_pv_fp16_cuda": triton_kernel if cuda_kernels else None,
+        "sageattn_qk_int8_pv_fp8_cuda": None,
+        "sageattn_qk_int8_pv_fp8_cuda_sm90": None,
+        "flash_attn_available": lambda: False,
+    }
+    exec(compile(ast.Module(body=functions, type_ignores=[]), "hf_models.py", "exec"), namespace)
+    return namespace
+
+
+class TestSageBackendSelection(unittest.TestCase):
+    def test_rocm_v1_selects_triton_without_nvidia_capability_query(self):
+        kernel = lambda *args, **kwargs: None
+        selector = load_attention_selector(hip="7.0", triton_kernel=kernel)
+        self.assertIs(selector["get_sage_attention_config"]()[0], kernel)
+        self.assertEqual("sage", selector["resolve_attention_mode"]("sage"))
+        self.assertEqual("sage", selector["resolve_attention_mode"]("auto"))
+
+    def test_rocm_unavailable_or_unsupported_version_falls_back(self):
+        for version, kernel in (("2.2.0", lambda: None), ("1.0.6", None)):
+            with self.subTest(version=version, installed=kernel is not None):
+                selector = load_attention_selector(hip="7.0", sage_version=version, triton_kernel=kernel)
+                self.assertEqual("sdpa", selector["resolve_attention_mode"]("sage"))
+
+    def test_forced_sdpa_still_wins(self):
+        selector = load_attention_selector(hip="7.0", triton_kernel=lambda: None)
+        self.assertEqual("sdpa", selector["resolve_attention_mode"]("sage", force_sdpa=True))
+
+    def test_nvidia_cuda_kernel_selection_is_preserved(self):
+        kernel = lambda *args, **kwargs: None
+        selector = load_attention_selector(hip=None, triton_kernel=kernel, cuda_kernels=True)
+        self.assertEqual((kernel, "per_warp", "fp32"), selector["get_sage_attention_config"]())
+
+
+class FakeAttention(torch.nn.Module):
+    def __init__(self, head_dim=64):
+        super().__init__()
+        self.head_dim = head_dim
+        self.layer_idx = 0
+        self.scaling = head_dim ** -0.5
+        self.q_proj = torch.nn.Linear(head_dim, head_dim, bias=False)
+        self.k_proj = torch.nn.Linear(head_dim, head_dim, bias=False)
+        self.v_proj = torch.nn.Linear(head_dim, head_dim, bias=False)
+        self.o_proj = torch.nn.Linear(head_dim, head_dim, bias=False)
+        self.original_calls = 0
+
+    def forward(self, hidden_states, **kwargs):
+        self.original_calls += 1
+        return hidden_states, None
+
+
+class FakeCache:
+    def __init__(self):
+        self.keys = None
+
+    def update(self, keys, values, layer_idx, cache_kwargs):
+        self.keys = keys
+        return keys, values
+
+
+class TestSageModelPatch(unittest.TestCase):
+    def setUp(self):
+        self.calls = []
+
+        def kernel(q, k, v, **kwargs):
+            self.calls.append(kwargs)
+            k.add_(1)  # SageAttention 1 smooths K in place.
+            return q
+
+        hf_models = types.ModuleType("thinkingllm_core.hf_models")
+        hf_models.get_sage_attention_config = lambda: (kernel, None, None)
+        qwen2 = types.ModuleType("transformers.models.qwen2.modeling_qwen2")
+        qwen2.Qwen2Attention = FakeAttention
+        qwen2.apply_rotary_pos_emb = lambda q, k, cos, sin: (q, k)
+        self.modules = mock.patch.dict(sys.modules, {
+            "thinkingllm_core.hf_models": hf_models,
+            "transformers.models.qwen2.modeling_qwen2": qwen2,
+        })
+        self.modules.start()
+        self.addCleanup(self.modules.stop)
+        spec = importlib.util.spec_from_file_location("_test_sageattention_patch", ROOT / "sageattention_patch.py")
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        self.patch_model = module.set_sage_attention
+
+    @staticmethod
+    def embeddings():
+        return torch.zeros(1, 1, 64), torch.zeros(1, 1, 64)
+
+    def test_unmasked_prefill_and_cached_decode_use_sage_without_mutating_cache(self):
+        layer = FakeAttention().half().eval()
+        self.patch_model(torch.nn.Sequential(layer))
+        prefill = torch.randn(1, 2, 64, dtype=torch.float16)
+        output, weights = layer(prefill, position_embeddings=self.embeddings(), use_cache=True, position_ids=None)
+        self.assertEqual((1, 2, 64), tuple(output.shape))
+        self.assertIsNone(weights)
+        self.assertTrue(self.calls[-1]["is_causal"])
+        self.assertEqual(0, layer.original_calls)
+
+        cache = FakeCache()
+        token = torch.randn(1, 1, 64, dtype=torch.float16)
+        layer(token, position_embeddings=self.embeddings(), past_key_values=cache, use_cache=True)
+        self.assertFalse(self.calls[-1]["is_causal"])
+        self.assertEqual(2, len(self.calls))
+        self.assertIsNotNone(cache.keys)
+        expected_keys = layer.k_proj(token).view(1, 1, 1, 64).transpose(1, 2)
+        torch.testing.assert_close(cache.keys, expected_keys)
+
+    def test_mask_and_unsupported_head_size_call_original_forward(self):
+        layer = FakeAttention().half().eval()
+        self.patch_model(torch.nn.Sequential(layer))
+        inputs = torch.randn(1, 2, 64, dtype=torch.float16)
+        layer(inputs, position_embeddings=self.embeddings(), attention_mask=torch.zeros(1, 1, 2, 2))
+        self.assertEqual(1, layer.original_calls)
+        self.assertEqual([], self.calls)
+
+        unsupported = FakeAttention(head_dim=80).half().eval()
+        self.patch_model(torch.nn.Sequential(unsupported))
+        unsupported(torch.randn(1, 2, 80, dtype=torch.float16), position_embeddings=self.embeddings())
+        self.assertEqual(1, unsupported.original_calls)
+        self.assertEqual([], self.calls)
+
+    def test_no_supported_layers_is_an_error(self):
+        with self.assertRaisesRegex(RuntimeError, "No compatible attention layers"):
+            self.patch_model(torch.nn.Sequential(torch.nn.Linear(2, 2)))
+
+
+if __name__ == "__main__":
+    unittest.main()

thinkingllm_core/hf_models.py

@@ -1,6 +1,7 @@
 """Hugging Face model configuration, discovery, device, and attention support."""
 
 import gc
+import importlib.metadata
 import json
 import os
 import platform
@@ -39,6 +40,11 @@ except Exception:
     sageattn_qk_int8_pv_fp8_cuda_sm90 = None
     SAGE_ATTENTION_AVAILABLE = False
 
+try:
+    from sageattention import sageattn as sageattn_triton
+except Exception:
+    sageattn_triton = None
+
 NODE_DIR = Path(__file__).resolve().parent.parent
 CONFIG_PATH = NODE_DIR / ("hf_models.commercial.json" if COMMERCIAL_RELEASE else "hf_models.json")
 SYSTEM_PROMPTS_PATH = NODE_DIR / "AILab_System_Prompts.json"
@@ -53,7 +59,7 @@ PRESET_PROMPTS: list[str] = [NO_PRESET_PROMPT, "Describe this image in detail."]
 TOOLTIPS = {
     "model_name": "Pick the checkpoint. [installed] means a catalog model is available in a configured LLM location; [local] means an uncatalogued local model, including compatible ComfyUI text_encoders. Missing catalog models download on first use.",
     "quantization": "Precision vs VRAM. FP16 gives the best quality if memory allows; 8-bit suits 8–16 GB GPUs; 4-bit fits 6 GB or lower but is slower.",
-    "attention_mode": "auto tries SageAttention → FlashAttention 2 → SDPA in order. SDPA is stable and recommended. Only override when debugging attention backends.",
+    "attention_mode": "auto tries compatible SageAttention (including Triton Sage 1 on ROCm) → FlashAttention 2 → SDPA in order. SDPA is stable and recommended. Only override when debugging attention backends.",
     "preset_prompt": "Built-in instruction describing how Qwen-VL should analyze the media input.",
     "custom_prompt": "Additional user input that gets combined with the preset template. Leave empty to use only the template.",
     "max_tokens": "Maximum number of new tokens to decode. Larger values yield longer answers but consume more time and memory.",
@@ -195,8 +201,11 @@ ATTENTION_MODES = ["auto", "sage", "flash_attention_2", "sdpa"]
 # Debug: Check SageAttention availability
 print(f"[QwenVL Debug] SAGE_ATTENTION_AVAILABLE: {SAGE_ATTENTION_AVAILABLE}")
 if torch.cuda.is_available():
-    major, minor = torch.cuda.get_device_capability()
-    print(f"[QwenVL Debug] CUDA capability: {major}.{minor}")
+    if getattr(torch.version, "hip", None):
+        print(f"[QwenVL Debug] ROCm/HIP: {torch.version.hip}")
+    else:
+        major, minor = torch.cuda.get_device_capability()
+        print(f"[QwenVL Debug] CUDA capability: {major}.{minor}")
 else:
     print("[QwenVL Debug] CUDA not available")
 print(f"[QwenVL Debug] Final ATTENTION_MODES: {ATTENTION_MODES}")
@@ -549,23 +558,30 @@ def flash_attn_available():
 
 def sage_attn_available():
     """Check if SageAttention is available and GPU supports it."""
-    if not SAGE_ATTENTION_AVAILABLE:
-        return False
-    if not torch.cuda.is_available():
-        return False
-    major, _ = torch.cuda.get_device_capability()
-    if major < 8:
-        return False
-    return True
+    return get_sage_attention_config()[0] is not None
 
 
 def get_sage_attention_config():
     """Get the appropriate SageAttention kernel based on GPU architecture."""
-    if not sage_attn_available():
+    if not torch.cuda.is_available():
+        return None, None, None
+
+    if getattr(torch.version, "hip", None):
+        try:
+            is_v1 = importlib.metadata.version("sageattention").split(".", 1)[0] == "1"
+        except importlib.metadata.PackageNotFoundError:
+            is_v1 = False
+        if is_v1 and callable(sageattn_triton):
+            return sageattn_triton, None, None
+        return None, None, None
+
+    if not SAGE_ATTENTION_AVAILABLE:
         return None, None, None
 
     major, minor = torch.cuda.get_device_capability()
     arch_code = major * 10 + minor
+    if arch_code < 80:
+        return None, None, None
 
     attn_func = None
     pv_accum_dtype = "fp32"
@@ -590,7 +606,7 @@ def get_sage_attention_config():
         print(f"[QwenVL] SageAttention not supported on SM{arch_code}")
         return None, None, None
 
-    return attn_func, "per_warp", pv_accum_dtype
+    return (attn_func, "per_warp", pv_accum_dtype) if callable(attn_func) else (None, None, None)
 
 def is_fp8_model(model_name: str) -> bool:
     """Check if model name indicates it's a pre-quantized FP8 model."""

update.md

@@ -1,5 +1,32 @@
 # ComfyUI-QwenVL Update Log
 
+## Version 2.5.8 (2026/09/27) - ROCm SageAttention 1 Support
+
+### 2.5.8 Changes
+
+- Added a SageAttention 1.x Triton path for Hugging Face Qwen nodes on ROCm, including SageAttention 1.0.6. Select `attention_mode=sage`, or use `auto` to try Sage before Flash Attention 2 and SDPA.
+- Retained NVIDIA CUDA kernel selection. Unsupported attention masks, head sizes, and model layers use the original SDPA forward.
+- Protected the KV cache from SageAttention 1's in-place key smoothing and documented that the GGUF `flash_attn` setting is separate from Hugging Face attention selection.
+
+### 2.5.8 Validation
+
+- Seven focused SageAttention tests and all 153 feature and compatibility tests pass on the local Windows CPU test path.
+- End-to-end inference on ROCm hardware remains unverified.
+
+## Version 2.5.7 (2026/08/26) - Safe Gemma 4 GGUF Audio
+
+### 2.5.7 Changes
+
+- Fixed the dedicated Gemma 4 Audio GGUF node so supported audio no longer hits the obsolete unconditional safety block reported in issue #18.
+- Kept the native crash protection for older or unknown llama.cpp backends, and now require `llama-cpp-python` 0.3.36 or newer.
+- Added a positive runtime capability check: the selected `Gemma4ChatHandler` must initialize its multimodal context and the mmproj must report audio support before inference begins.
+- Reorganized the README into a concise project entry point and moved detailed feature, workflow, model, backend, and troubleshooting explanations into dedicated documentation guides.
+
+### 2.5.7 Validation
+
+- Added regression coverage for legacy backends, modern audio-capable backends, the OpenAI-compatible `input_audio` payload, and mmproj files without audio support.
+- All 153 feature and compatibility tests pass in the bundled ComfyUI Python 3.12 runtime.
+
 ## Version 2.4.0 (2026/08/12) - Duration-Aware Video Prompting and QwenVL Core
 
 ### 2.4.0 Changes