diff --git a/AILab_QwenVL_GGUF.py b/AILab_QwenVL_GGUF.py index f845366..dee3a55 100644 --- a/AILab_QwenVL_GGUF.py +++ b/AILab_QwenVL_GGUF.py @@ -76,11 +76,15 @@ from thinkingllm_core.model_access import ( resolve_model_catalog_name, ) from thinkingllm_core.prompt_contracts import ( + QWEN_MAX_REASONING_BUDGET_TOKENS, + QWEN_MIN_FINAL_ANSWER_TOKENS, + QWEN_THINKING_BUDGET_POLICY_VERSION, VIDEO_DURATION_INPUT, apply_qwen_soft_thinking_directive, apply_video_duration_context, ensure_video_prompt_duration, estimate_qwen_text_tokens, + resolve_qwen_output_token_plan, resolve_qwen_thinking_mode, resolve_video_duration, validate_minimax_reference_request, @@ -682,6 +686,35 @@ def _filter_kwargs_for_callable(fn, kwargs: dict) -> dict: return {k: v for k, v in kwargs.items() if k in allowed} +def _supports_reasoning_budget(llm) -> bool: + """Return true only when llama-cpp exposes the full per-call reasoning budget API.""" + completion_fn = getattr(llm, "create_chat_completion", None) + if not callable(completion_fn): + return False + try: + parameters = inspect.signature(completion_fn).parameters + except Exception: + return False + required = { + "reasoning_budget", + "reasoning_start", + "reasoning_end", + "reasoning_start_in_prompt", + } + return all( + name in parameters + and parameters[name].kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY) + for name in required + ) + + +def _require_nonempty_final_text(text: str, *, source: str) -> str: + """Reject empty final output before it can be persisted as a successful result.""" + if not isinstance(text, str) or not text.strip(): + raise RuntimeError(f"[{source}] Generation produced no usable final text; empty output was not saved or cached.") + return text + + def _unexpected_kwarg_names_from_type_error(exc: TypeError) -> list[str]: message = str(exc or "") matches = re.findall(r"unexpected keyword argument(?:\(s\))?\s+'([^']+)'", message) @@ -714,7 +747,9 @@ def _select_mmproj_handler_class(model_name: str, arch: str | None): name = (model_name or "").lower() architecture = (arch or "").lower() - if "gemma-4" in name or "gemma4" in name or architecture == "gemma4": + if architecture in ("qwen35", "qwen35moe") or "qwen3.5-" in name: + preferred = ("Qwen35ChatHandler", "Qwen3VLChatHandler") + elif "gemma-4" in name or "gemma4" in name or architecture == "gemma4": preferred = ("Gemma4ChatHandler",) elif "qwen2.5" in name or "qwen25" in name or architecture in ("qwen2vl", "qwen25vl"): preferred = ("Qwen25VLChatHandler", "Qwen3VLChatHandler") @@ -732,6 +767,25 @@ def _select_mmproj_handler_class(model_name: str, arch: str | None): ) +def _set_chat_handler_thinking(chat_handler, enable_thinking: bool) -> None: + """Update the state read by llama-cpp-python's model-specific Jinja handler.""" + if chat_handler is None: + return + enabled = bool(enable_thinking) + if hasattr(chat_handler, "enable_thinking"): + try: + chat_handler.enable_thinking = enabled + except Exception: + pass + template_arguments = getattr(chat_handler, "extra_template_arguments", None) + if isinstance(template_arguments, dict): + template_arguments["enable_thinking"] = enabled + for attr_name in ("chat_template_kwargs", "_chat_template_kwargs"): + current = getattr(chat_handler, attr_name, None) + if isinstance(current, dict): + current["enable_thinking"] = enabled + + def _resample_mono_audio(samples, source_rate: int, target_rate: int = GGUF_AUDIO_TARGET_SAMPLE_RATE): samples = np.asarray(samples, dtype=np.float32).reshape(-1) if samples.size == 0 or source_rate == target_rate: @@ -1195,6 +1249,8 @@ class QwenVLGGUFBase: if self.llm is None or not self._uses_chat_template_thinking(): return template_kwargs = {"enable_thinking": bool(enable_thinking)} + _set_chat_handler_thinking(self.chat_handler, enable_thinking) + _set_chat_handler_thinking(getattr(self.llm, "chat_handler", None), enable_thinking) for attr_name in ("chat_template_kwargs", "_chat_template_kwargs"): try: current = getattr(self.llm, attr_name, None) @@ -1426,6 +1482,7 @@ class QwenVLGGUFBase: arch = read_gguf_architecture(model_path) self.gguf_arch = arch is_qwen35 = arch in ("qwen35", "qwen35moe") if arch else "qwen3.5-" in model_name.lower() + self.is_qwen35 = bool(is_qwen35) self.supports_qwen_soft_think = ( bool(is_qwen35 or arch == "qwen3") if arch @@ -1496,6 +1553,8 @@ class QwenVLGGUFBase: "force_reasoning": False, "verbose": False, } + if is_qwen35: + mmproj_kwargs["enable_thinking"] = bool(enable_thinking) mmproj_kwargs = _filter_mmproj_handler_kwargs(handler_cls, mmproj_kwargs) if "image_max_tokens" not in mmproj_kwargs: print( @@ -1532,8 +1591,9 @@ class QwenVLGGUFBase: state_label = "enabled" if thinking_enabled else "disabled" print(f"[QwenVL] Qwen architecture detected (arch={arch}): Thinking {state_label} via chat template.") - if has_mmproj and self.chat_handler is not None: + if self.chat_handler is not None: llm_kwargs["chat_handler"] = self.chat_handler + if has_mmproj and self.chat_handler is not None: llm_kwargs["image_min_tokens"] = 1024 llm_kwargs["image_max_tokens"] = img_max @@ -1680,6 +1740,7 @@ class QwenVLGGUFBase: model_name: str = "", stream_to_terminal: bool = False, enable_thinking: bool = True, + reasoning_budget: int = 0, auto_finalization_retry: bool = False, top_k: int | None = None, preset_name: str = "", @@ -1689,10 +1750,22 @@ class QwenVLGGUFBase: ensure_cuda_vram_headroom("QwenVL GGUF", min_free_gb=1.0, min_free_ratio=0.08) self._raise_if_unsafe_native_multimodal_path(images_b64, audio_b64 or []) supports_soft_think = getattr(self, "supports_qwen_soft_think", False) + supports_reasoning_budget = supports_soft_think and _supports_reasoning_budget(self.llm) + try: + reasoning_budget = max(0, int(reasoning_budget or 0)) + except (TypeError, ValueError, OverflowError): + reasoning_budget = 0 if supports_soft_think: directive = "/think" if enable_thinking else "/no_think" if not stream_to_terminal: print(f"[QwenVL] Qwen3 GGUF detected: Thinking {'enabled' if enable_thinking else 'disabled'} via chat template and {directive}.") + if enable_thinking and supports_reasoning_budget and reasoning_budget > 0: + print(f"[QwenVL GGUF] Per-request reasoning budget enforced: {reasoning_budget} tokens.") + elif enable_thinking: + print( + "[QwenVL GGUF] Backend has no explicit per-request reasoning budget; " + "the shared preflight cutoff applies, but the estimated answer reserve is not a hard cap." + ) def _run_completion( system_prompt_text: str, user_prompt_text: str, @@ -1704,10 +1777,35 @@ class QwenVLGGUFBase: attempt_enable_thinking: bool | None = None, ) -> tuple[str, str]: current_enable_thinking = enable_thinking if attempt_enable_thinking is None else bool(attempt_enable_thinking) + handler = self.chat_handler or getattr(self.llm, "chat_handler", None) + handler_controls_thinking = hasattr(handler, "enable_thinking") + if ( + getattr(self, "is_qwen35", False) + and not current_enable_thinking + and not supports_reasoning_budget + and not handler_controls_thinking + ): + raise RuntimeError( + "[QwenVL GGUF] Cannot reliably disable Qwen3.5 thinking for this request: " + "the loaded llama.cpp backend exposes no reasoning-budget control and the active chat " + "handler exposes no enable_thinking template switch. Upgrade llama-cpp-python or use " + "a Qwen3.5 chat handler that supports enable_thinking." + ) + reasoning_kwargs = {} + if supports_reasoning_budget: + handler_opens_reasoning = bool( + current_enable_thinking and hasattr(handler, "enable_thinking") + ) + reasoning_kwargs = { + "reasoning_budget": reasoning_budget if current_enable_thinking else 0, + "reasoning_start": "", + "reasoning_end": "", + "reasoning_start_in_prompt": handler_opens_reasoning, + } effective_user_prompt = apply_qwen_soft_thinking_directive( user_prompt_text, current_enable_thinking, - supports_soft_switch=supports_soft_think, + supports_soft_switch=(supports_soft_think and not getattr(self, "is_qwen35", False)), ) if self.llm is not None and hasattr(self.llm, "reset"): try: @@ -1776,6 +1874,7 @@ class QwenVLGGUFBase: stop=stop_tokens, stream=True, **sampler_kwargs, + **reasoning_kwargs, ) for chunk in result: throw_exception_if_processing_interrupted() @@ -1840,6 +1939,7 @@ class QwenVLGGUFBase: stop=stop_tokens, stream=True, **sampler_kwargs, + **reasoning_kwargs, ) for chunk in result: if abort_event.is_set(): @@ -2068,6 +2168,7 @@ class QwenVLGGUFBase: offload_kqv=bool(offload_kqv), ctx_checkpoints=ctx_checkpoints, enable_thinking=bool(enable_thinking), + thinking_budget_policy_version=QWEN_THINKING_BUDGET_POLICY_VERSION, auto_finalization_retry=bool(auto_finalization_retry), mask_mode=mask_mode, max_tokens=max_tokens, @@ -2119,6 +2220,7 @@ class QwenVLGGUFBase: top_p=top_p, repetition_penalty=repetition_penalty, enable_thinking=enable_thinking, + thinking_budget_policy_version=QWEN_THINKING_BUDGET_POLICY_VERSION, effective_duration_seconds=effective_duration_seconds, video_prompt_contract_version=resolved_duration.get("contract_version") if resolved_duration else None, mask_prompt_contract_version=MASK_PROMPT_CONTRACT_VERSION if mask_hash else None, @@ -2206,14 +2308,31 @@ class QwenVLGGUFBase: "You are a helpful vision-language assistant.", prompt, ) + len(images_b64) * max(1024, int(attempt_image_max_tokens or 1024)) + len(audio_b64) * 2048 + available_output, answer_reserve, reasoning_budget = resolve_qwen_output_token_plan( + max_tokens, + prompt_tokens=estimated_prompt_tokens, + context_window=context_window, + min_answer_reserve_tokens=QWEN_MIN_FINAL_ANSWER_TOKENS, + max_reasoning_tokens=QWEN_MAX_REASONING_BUDGET_TOKENS, + ) effective_thinking = resolve_qwen_thinking_mode( enable_thinking, max_tokens, label="QwenVL GGUF", prompt_tokens=estimated_prompt_tokens, context_window=context_window, + answer_reserve_tokens=answer_reserve, quiet=stream_to_terminal, ) + if not effective_thinking: + reasoning_budget = 0 + if not stream_to_terminal: + print( + f"[QwenVL GGUF] Token preflight: " + f"available_output={available_output}, " + f"reasoning_cap={reasoning_budget}, " + f"final_answer_reserve={answer_reserve}." + ) print(f"[QwenVL GGUF DEBUG] Loading model...") self._load_model( model_name=model_name, @@ -2257,6 +2376,7 @@ class QwenVLGGUFBase: model_name=model_name, stream_to_terminal=stream_to_terminal, enable_thinking=effective_thinking, + reasoning_budget=reasoning_budget, auto_finalization_retry=auto_finalization_retry, top_k=top_k, preset_name=preset_prompt, @@ -2292,6 +2412,7 @@ class QwenVLGGUFBase: duration_seconds, source_prompt_text=custom_prompt, ) + text = _require_nonempty_final_text(text, source="QwenVL GGUF") print(f"[QwenVL GGUF DEBUG] Generation completed. Text length: {len(text) if text else 0}") print(f"[QwenVL GGUF DEBUG] Generated text: {text[:100] if text else 'EMPTY'}...") diff --git a/AILab_QwenVL_GGUF_PromptEnhancer.py b/AILab_QwenVL_GGUF_PromptEnhancer.py index 7dd457d..53fddf4 100644 --- a/AILab_QwenVL_GGUF_PromptEnhancer.py +++ b/AILab_QwenVL_GGUF_PromptEnhancer.py @@ -51,11 +51,16 @@ from thinkingllm_core.model_access import ( resolve_model_catalog_name, ) from thinkingllm_core.prompt_contracts import ( + QWEN_MAX_REASONING_BUDGET_TOKENS, + QWEN_MIN_FINAL_ANSWER_TOKENS, + QWEN_MIN_REASONING_BUDGET_TOKENS, + QWEN_THINKING_BUDGET_POLICY_VERSION, VIDEO_DURATION_INPUT, apply_qwen_soft_thinking_directive, apply_video_duration_context, ensure_video_prompt_duration, estimate_qwen_text_tokens, + resolve_qwen_output_token_plan, resolve_qwen_thinking_mode, resolve_video_duration, validate_minimax_source_duration, @@ -82,6 +87,7 @@ from AILab_QwenVL_GGUF import ( _filter_kwargs_for_callable, _find_existing_local_file_in_dirs, _gguf_search_dirs, + _supports_reasoning_budget, ) @@ -90,7 +96,7 @@ GGUF_PROMPT_TOOLTIPS = { "prompt_text": "Type your scene idea or video description here; no separate text node is needed. The selected style transforms this text into a generation prompt. For image-to-video styles, include any source-frame details in your text because this node does not receive images.", "preset_system_prompt": "Preset enhancement style. Use Custom Only when you want custom_system_prompt to fully control the instruction.", "custom_system_prompt": "Optional extra instruction. Required when using Custom Only; otherwise it is prepended to the selected style.", - "max_tokens": "Maximum new tokens for the enhanced prompt. Increase only when the model truncates useful detail.", + "max_tokens": "Maximum new tokens for the enhanced prompt. The node dynamically caps Qwen reasoning to preserve at least two thirds of the available output for the final prompt.", "temperature": "Sampling randomness. Lower is more stable; higher is more varied.", "top_p": "Nucleus sampling cutoff. Lower values restrict token choice; 0.9 is a balanced default.", "repetition_penalty": "Values above 1.0 reduce repeated phrases in the enhanced prompt.", @@ -100,8 +106,8 @@ GGUF_PROMPT_TOOLTIPS = { "seed": "Sampling seed. Reusing it with identical inputs can reuse the saved prompt result.", "keep_last_prompt": "Return the saved per-node prompt instead of generating a new one when available.", "stream_tokens_to_terminal": "Show clean wrapped generated tokens in the ComfyUI terminal. Streaming bypasses fixed-seed reuse for a fresh run.", - "enable_thinking": "Enable model reasoning/thinking when supported. The node still returns the cleaned final prompt, so reasoning may be hidden or empty.", - "auto_finalization_retry": "If enabled, runs an extra LLM completion when the first output is empty or reasoning-only. Disabled by default so one node execution performs one generation pass.", + "enable_thinking": "Request model reasoning/thinking when supported. The node calculates a reasoning cap before generation, reserves room for the final prompt, and disables thinking automatically when the available budget is too small or the backend cannot cap it.", + "auto_finalization_retry": "If enabled, allows up to two finalization retries after an empty or reasoning-only response. The node automatically makes one direct-answer recovery attempt for reasoning-only output.", "hf_token": "Optional Hugging Face access token for private or gated GGUF downloads. It is passed only to the download call, never logged or cached, and the in-memory copy is dropped after the download attempt. Clear this field before saving or sharing workflows.", } @@ -429,6 +435,7 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: def __init__(self): self.llm = None self.current_signature = None + self.is_qwen35 = False self.gguf_models = self.load_gguf_models() self.styles = STYLES register_active_gguf_loader(self) @@ -789,10 +796,14 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: arch = read_gguf_architecture(resolved) self.gguf_arch = arch is_qwen = (arch and any(a in arch for a in ("qwen35", "qwen35moe", "qwen3", "qwen"))) or "qwen3.5-" in model_name.lower() or "qwen3-" in model_name.lower() + normalized_arch = re.sub(r"[^a-z0-9]", "", str(arch or "").lower()) + model_name_lower = model_name.lower() + is_qwen35 = normalized_arch in {"qwen35", "qwen35moe"} or "qwen3.5-" in model_name_lower + self.is_qwen35 = bool(is_qwen35) self.supports_qwen_soft_think = ( - bool(arch and (arch == "qwen3" or arch.startswith("qwen35"))) - if arch - else ("qwen3-" in model_name.lower() or "qwen3.5-" in model_name.lower()) + normalized_arch in {"qwen3", "qwen35", "qwen35moe", "qwen3vl"} + or "qwen3-" in model_name_lower + or "qwen3.5-" in model_name_lower ) if is_qwen: thinking_state = bool(enable_thinking) @@ -831,6 +842,8 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: stream_to_terminal=False, initial_stage_label="INITIAL GENERATION", enable_thinking=True, + reasoning_budget=0, + answer_reserve_tokens=0, auto_finalization_retry=False, preset_name="", ): @@ -838,10 +851,45 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: stream_display = TerminalStreamDisplay("QwenVL GGUF", suppress_planning=True, compact=True) if stream_to_terminal else None supports_soft_think = getattr(self, "supports_qwen_soft_think", False) is_video_preset = bool(resolve_video_duration(preset_name, 5.0)) + supports_reasoning_budget = supports_soft_think and _supports_reasoning_budget(self.llm) + reasoning_budget = max(0, int(reasoning_budget or 0)) + if enable_thinking and supports_soft_think and reasoning_budget <= 0: + enable_thinking = False + print( + "[QwenVL PromptEnhancer GGUF] The preflight left no safe reasoning budget; " + "disabling thinking before generation." + ) + elif ( + enable_thinking + and supports_soft_think + and not supports_reasoning_budget + and reasoning_budget < QWEN_MIN_REASONING_BUDGET_TOKENS + ): + enable_thinking = False + reasoning_budget = 0 + print( + "[QwenVL PromptEnhancer GGUF] The backend cannot enforce a per-request reasoning cap, " + f"and the preflight allowance is below {QWEN_MIN_REASONING_BUDGET_TOKENS} tokens; " + "disabling thinking to preserve answer space." + ) if supports_soft_think: directive = "/think" if enable_thinking else "/no_think" if not stream_to_terminal: - print(f"[QwenVL] Qwen3 GGUF prompt enhancer: Thinking {'enabled' if enable_thinking else 'disabled'} via chat template and {directive}.") + if enable_thinking: + if supports_reasoning_budget: + print( + f"[QwenVL] Qwen GGUF prompt enhancer: Thinking enabled with an enforced " + f"{reasoning_budget}-token cap and a {int(answer_reserve_tokens)}-token " + f"final-answer reserve via chat template and {directive}." + ) + else: + print( + f"[QwenVL] Qwen GGUF prompt enhancer: Thinking enabled with a preflight " + f"allowance of {reasoning_budget} tokens and a {int(answer_reserve_tokens)}-token " + "answer reserve. This backend cannot hard-cap reasoning; the reserve is a heuristic." + ) + else: + print(f"[QwenVL] Qwen GGUF prompt enhancer: Thinking disabled via chat template and {directive}.") def _call( system: str, @@ -857,7 +905,7 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: effective_user_prompt = apply_qwen_soft_thinking_directive( user, current_enable_thinking, - supports_soft_switch=supports_soft_think, + supports_soft_switch=(supports_soft_think and not self.is_qwen35), ) ensure_cuda_vram_headroom("QwenVL PromptEnhancer GGUF", min_free_gb=1.0, min_free_ratio=0.08) if self.llm is not None and hasattr(self.llm, "reset"): @@ -872,8 +920,11 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: {"role": "system", "content": system}, {"role": "user", "content": effective_user_prompt}, ] - if assistant_prefill: - messages.append({"role": "assistant", "content": assistant_prefill}) + effective_assistant_prefill = assistant_prefill + if supports_soft_think and self.is_qwen35 and not current_enable_thinking: + effective_assistant_prefill = f"\n\n\n{effective_assistant_prefill}" + if effective_assistant_prefill: + messages.append({"role": "assistant", "content": effective_assistant_prefill}) log_llm_input( "QwenVL PromptEnhancer GGUF", stage_label, @@ -912,12 +963,22 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: "seed": seed_val, "stop": ["<|im_end|>", "<|im_start|>"], "stream": True, + "assistant_prefill": bool(effective_assistant_prefill), } if supports_soft_think: completion_kwargs["chat_template_kwargs"] = { "enable_thinking": current_enable_thinking, "thinking_mode": "enabled" if current_enable_thinking else "disabled", } + if supports_soft_think and supports_reasoning_budget: + completion_kwargs.update( + { + "reasoning_budget": reasoning_budget if current_enable_thinking else 0, + "reasoning_start": "", + "reasoning_end": "", + "reasoning_start_in_prompt": False, + } + ) completion_kwargs = _filter_kwargs_for_callable(getattr(llm, "create_chat_completion"), completion_kwargs) response = llm.create_chat_completion(**completion_kwargs) for chunk in response: @@ -1001,14 +1062,30 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: best_cleaned = cleaned.strip() if _preset_output_is_usable(cleaned.strip(), preset_name) else "" if best_cleaned: return best_cleaned, "\n\n".join(raw_trace_parts) - if not (auto_finalization_retry or is_video_preset): + reasoning_only = bool( + not best_cleaned + and re.match(r"^\s*. " + "Cover subject, environment, lighting, camera settings, composition, color/texture, and style. Output only the prompt." + ) + translation_prompt_tokens = estimate_qwen_text_tokens(translation_system_prompt, enhanced) + translation_available, translation_answer_reserve, translation_reasoning_budget = ( + resolve_qwen_output_token_plan( + max_tokens, + prompt_tokens=translation_prompt_tokens, + context_window=effective_context_window, + min_answer_reserve_tokens=QWEN_MIN_FINAL_ANSWER_TOKENS, + max_reasoning_tokens=QWEN_MAX_REASONING_BUDGET_TOKENS, + ) + ) + translation_effective_thinking = resolve_qwen_thinking_mode( + effective_thinking, + max_tokens, + label="QwenVL PromptEnhancer GGUF translation", + prompt_tokens=translation_prompt_tokens, + context_window=effective_context_window, + min_reasoning_tokens=QWEN_MIN_REASONING_BUDGET_TOKENS, + answer_reserve_tokens=translation_answer_reserve, + quiet=stream_tokens_to_terminal, + ) + if not translation_effective_thinking: + translation_reasoning_budget = 0 + if not stream_tokens_to_terminal: + print( + f"[QwenVL PromptEnhancer GGUF] Translation token preflight: " + f"available_output={translation_available}, " + f"reasoning_cap={translation_reasoning_budget}, " + f"final_answer_reserve={translation_answer_reserve}." + ) translated, trans_trace = self._invoke_llama( - system_prompt=( - PROMPT_CONFIG.get("translation_prompt") - or "Return a single English paragraph (150-300 words). No prefixes, bullets, JSON, or . " - "Cover subject, environment, lighting, camera settings, composition, color/texture, and style. Output only the prompt." - ), + system_prompt=translation_system_prompt, user_prompt=enhanced, max_tokens=max_tokens, temperature=0.3, @@ -1203,7 +1332,9 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: seed=seed + 1, stream_to_terminal=stream_tokens_to_terminal, initial_stage_label="TRANSLATION STAGE", - enable_thinking=effective_thinking, + enable_thinking=translation_effective_thinking, + reasoning_budget=translation_reasoning_budget, + answer_reserve_tokens=translation_answer_reserve, auto_finalization_retry=auto_finalization_retry, preset_name=preset_system_prompt, ) @@ -1218,6 +1349,12 @@ class ThinkingLLM_QwenVL_GGUF_PromptEnhancer: source_prompt_text=prompt_text, ) + if not final.strip(): + raise RuntimeError( + "QwenVL GGUF PromptEnhancer produced no usable final prompt after finalization; " + "the empty result was not cached. Check the model output and token budget." + ) + # Cache the generated text PROMPT_CACHE[cache_key] = { "text": final, diff --git a/AILab_QwenVL_PromptEnhancer.py b/AILab_QwenVL_PromptEnhancer.py index 4ae3577..90c286e 100644 --- a/AILab_QwenVL_PromptEnhancer.py +++ b/AILab_QwenVL_PromptEnhancer.py @@ -43,6 +43,7 @@ from thinkingllm_core.model_access import ( ) from thinkingllm_core.hf_trust import trust_remote_code_for_snapshot from thinkingllm_core.prompt_contracts import ( + QWEN_THINKING_BUDGET_POLICY_VERSION, VIDEO_DURATION_INPUT, apply_qwen_soft_thinking_directive, apply_video_duration_context, @@ -253,6 +254,7 @@ class ThinkingLLM_QwenVL_PromptEnhancer(QwenVLBase): enhancement_style=enhancement_style, custom_system_prompt=custom_system_prompt, enable_thinking=bool(enable_thinking), + thinking_budget_policy_version=QWEN_THINKING_BUDGET_POLICY_VERSION, output_cleaner_version=OUTPUT_CLEANER_VERSION, max_tokens=max_tokens, temperature=temperature, @@ -341,6 +343,11 @@ class ThinkingLLM_QwenVL_PromptEnhancer(QwenVLBase): duration_seconds, source_prompt_text=prompt_text, ) + if not final.strip(): + raise RuntimeError( + "[QwenVL PromptEnhancer HF] Model returned no usable prompt after removing reasoning. " + "Try disabling thinking or increasing the output token limit." + ) # Persist per-node for future keep_last_prompt=True set_node_saved_prompt(node_class, unique_id, extra_pnginfo, final, raw_trace=raw_trace, seed=seed, max_tokens=max_tokens, temperature=temperature, top_p=top_p, repetition_penalty=repetition_penalty, input_signature=input_signature) @@ -566,10 +573,15 @@ class ThinkingLLM_QwenVL_PromptEnhancer(QwenVLBase): messages = [{"role": "user", "content": effective_prompt}] local_template_kwargs = dict(template_kwargs) if is_qwen35 or supports_soft_think: - local_template_kwargs["chat_template_kwargs"] = {"enable_thinking": thinking_enabled} + local_template_kwargs["enable_thinking"] = thinking_enabled try: formatted_prompt = self.text_tokenizer.apply_chat_template(messages, **local_template_kwargs) - except Exception: + except Exception as exc: + if is_qwen35 or supports_soft_think: + raise RuntimeError( + "[QwenVL PromptEnhancer HF] Could not apply the tokenizer chat template with " + f"enable_thinking={thinking_enabled}; refusing to continue without the requested thinking control." + ) from exc formatted_prompt = effective_prompt inputs = self.text_tokenizer(formatted_prompt, return_tensors="pt").to(device_choice) return effective_prompt, formatted_prompt, inputs @@ -646,7 +658,9 @@ class ThinkingLLM_QwenVL_PromptEnhancer(QwenVLBase): if not full_streamed.strip(): raise RuntimeError("[QwenVL] HF streaming returned empty response") raw_text = full_streamed.strip() - result = clean_model_output(raw_text, OutputCleanConfig(mode="prompt")) or raw_text + result = clean_model_output(raw_text, OutputCleanConfig(mode="prompt")) + if not result.strip(): + raise RuntimeError("[QwenVL PromptEnhancer HF] Streaming produced no usable prompt after removing reasoning.") if not keep_model_loaded: self.text_model = None self.text_processor = None @@ -667,7 +681,9 @@ class ThinkingLLM_QwenVL_PromptEnhancer(QwenVLBase): input_length = inputs["input_ids"].shape[1] generated_tokens = outputs[0][input_length:] raw_text = self.text_tokenizer.decode(generated_tokens, skip_special_tokens=True).strip() - result = clean_model_output(raw_text, OutputCleanConfig(mode="prompt")) or raw_text + result = clean_model_output(raw_text, OutputCleanConfig(mode="prompt")) + if not result.strip(): + raise RuntimeError("[QwenVL PromptEnhancer HF] Generation produced no usable prompt after removing reasoning.") if not keep_model_loaded: self.text_model = None diff --git a/__init__.py b/__init__.py index 3597d68..e0fa863 100644 --- a/__init__.py +++ b/__init__.py @@ -3,7 +3,7 @@ "name": "ComfyUI-ThinkingLLM", "description": "A multimodal ComfyUI AI node pack with Qwen, Gemma 4, GGUF, Whisper, image/text decision routing, typed decision fields, and live thinking output.", "author": "goodguy1963", - "version": "3.1.2", + "version": "3.1.3", "url": "https://github.com/goodguy1963/ComfyUI-ThinkingLLM", "category": "image" } diff --git a/custom_nodes.json b/custom_nodes.json index 336f090..b0fe38d 100644 --- a/custom_nodes.json +++ b/custom_nodes.json @@ -9,5 +9,5 @@ "description": "A multimodal ComfyUI AI node pack with Qwen, Gemma 4, GGUF, Whisper, image and text decision routing, typed Boolean and Integer decision fields, and live thinking output.", "category": "image", "tags": ["vision", "language", "multimodal", "thinkingllm", "decision-router", "decision-fields", "qwen", "gguf", "flash_attention", "performance", "wan2.2", "video", "i2v", "t2v", "storyboard", "caching", "fork", "gpl-3.0"], - "version": "3.1.2" + "version": "3.1.3" } diff --git a/docs/MAINTAINER_NOTES.md b/docs/MAINTAINER_NOTES.md index 7e09d17..a4c80cf 100644 --- a/docs/MAINTAINER_NOTES.md +++ b/docs/MAINTAINER_NOTES.md @@ -23,6 +23,10 @@ - Keep the same version mirrored in the `__init__.py` metadata block and `custom_nodes.json` so local ComfyUI metadata and manager-facing metadata stay aligned. - README still contains historical upstream links and changelog references beyond the top-level fork identity; that is acceptable for attribution, but a docs pass should normalize publish-facing instructions before release. +## ThinkingLLM 3.1.3 release record + +Version metadata is synchronized at **3.1.3** in `pyproject.toml`, `__init__.py`, and `custom_nodes.json`. This release unifies Qwen thinking preflight, fixes Qwen3.5 template control, and prevents empty or reasoning-only results from being stored as successful output. See `update.md` for validation details. + ## ThinkingLLM 3.1.1 release record Version metadata is synchronized at **3.1.1** in `pyproject.toml`, `__init__.py`, and `custom_nodes.json`. This release expands audio, mask-focused image, and video prompt workflows, and adds security controls for audio paths and resource use, pinned Transformers remote code, and terminal output. See `update.md` for the change summary and focused validation. diff --git a/nodes/openai_compatible_api.py b/nodes/openai_compatible_api.py index c8d654b..0b87c24 100644 --- a/nodes/openai_compatible_api.py +++ b/nodes/openai_compatible_api.py @@ -21,6 +21,11 @@ from pathlib import Path from AILab_OutputCleaner import OutputCleanConfig, clean_model_output from AILab_StreamDisplay import compose_streamed_model_output, extract_stream_token, get_thinking_stream_display +from thinkingllm_core.prompt_contracts import ( + QWEN_MIN_REASONING_BUDGET_TOKENS, + resolve_qwen_output_token_plan, + resolve_qwen_thinking_mode, +) PACKAGE_DIR = Path(__file__).resolve().parent.parent DEFAULT_PROFILE_FILE = PACKAGE_DIR / "thinkingllm_api_profiles.json" @@ -513,6 +518,36 @@ def _validate_thinking_budget(profile: dict, value: int) -> int: return value +def _resolve_api_qwen_thinking(profile: dict, max_tokens: int, enabled: bool, requested_budget: int): + """Apply the shared cutoff when the profile's output limit includes reasoning.""" + if ( + profile.get("thinking_mode") != "qwen" + or profile.get("max_tokens_field") != "max_completion_tokens" + ): + return bool(enabled), requested_budget, None + + available_output, answer_reserve, reasoning_budget = resolve_qwen_output_token_plan(max_tokens) + minimum_reasoning = QWEN_MIN_REASONING_BUDGET_TOKENS + if requested_budget > 0: + minimum_reasoning = min(minimum_reasoning, requested_budget) + effective_thinking = resolve_qwen_thinking_mode( + enabled, + available_output, + label="ThinkingLLM API", + min_reasoning_tokens=minimum_reasoning, + answer_reserve_tokens=answer_reserve, + ) + if effective_thinking: + effective_budget = ( + min(reasoning_budget, requested_budget) + if requested_budget > 0 + else reasoning_budget + ) + else: + effective_budget = 0 + return effective_thinking, effective_budget, (available_output, answer_reserve) + + def _validate_input_text(profile: dict, system_prompt, prompt) -> tuple[str, str]: if system_prompt is not None and not isinstance(system_prompt, str): raise ValueError("system_prompt must be a string.") @@ -792,6 +827,21 @@ class ThinkingLLMOpenAICompatibleAPI: timeout = _validate_timeout(profile, timeout_seconds) temperature, top_p, seed, thinking_budget = _validate_sampling(temperature, top_p, seed, thinking_budget) thinking_budget = _validate_thinking_budget(profile, thinking_budget) + effective_thinking, effective_thinking_budget, qwen_token_plan = _resolve_api_qwen_thinking( + profile, + max_out, + enable_thinking, + thinking_budget, + ) + if qwen_token_plan is not None and ( + effective_thinking != enable_thinking or effective_thinking_budget != thinking_budget + ): + available_output, answer_reserve = qwen_token_plan + print( + f"[ThinkingLLM API] Qwen output preflight: available_output={available_output}, " + f"reasoning_budget={effective_thinking_budget}, answer_reserve={answer_reserve}, " + f"thinking={'enabled' if effective_thinking else 'disabled'}." + ) system_text, prompt_text = _validate_input_text(profile, system_prompt, prompt) user_content, image_supplied = _build_user_content(profile, prompt_text, image=image, image_url=image_url) @@ -810,9 +860,9 @@ class ThinkingLLMOpenAICompatibleAPI: if profile.get("send_seed"): request_body["seed"] = seed if profile.get("thinking_mode") == "qwen": - request_body["enable_thinking"] = enable_thinking - if thinking_budget > 0: - request_body["thinking_budget"] = thinking_budget + request_body["enable_thinking"] = effective_thinking + if effective_thinking_budget > 0: + request_body["thinking_budget"] = effective_thinking_budget request_body.update(_parse_extra_json(extra_body_json, profile.get("allowed_extra_body_fields", []))) request_body["stream"] = True @@ -910,6 +960,12 @@ class ThinkingLLMOpenAICompatibleAPI: strip_json_wrappers=False, strip_leading_preamble=False, strip_planning=False, ), ) + if not response_text.strip(): + raise RuntimeError( + f"ThinkingLLM API profile {profile['name']!r} returned no final answer " + "(empty or reasoning-only output). Check the provider's thinking settings " + "and completion token limit before retrying." + ) return (response_text, raw_trace) diff --git a/pyproject.toml b/pyproject.toml index deb0ee7..9738f28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "ComfyUI-ThinkingLLM" description = "A multimodal ComfyUI AI node pack with Qwen, Gemma 4, GGUF, Whisper, image/text decision routing, typed decision fields, and live thinking output." -version = "3.1.2" +version = "3.1.3" license = {file = "LICENSE"} dependencies = ["transformers>=5.2.0", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "sentencepiece", "tiktoken", "faster-whisper", "av>=17,<19", "llama-cpp-python"] diff --git a/tests/test_hf_thinking_control.py b/tests/test_hf_thinking_control.py new file mode 100644 index 0000000..409ae98 --- /dev/null +++ b/tests/test_hf_thinking_control.py @@ -0,0 +1,77 @@ +import ast +import unittest +from pathlib import Path + +from AILab_OutputCleaner import OutputCleanConfig, clean_model_output + + +ROOT = Path(__file__).resolve().parents[1] + + +class HFThinkingControlTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.enhancer_source = (ROOT / "AILab_QwenVL_PromptEnhancer.py").read_text(encoding="utf-8") + cls.enhancer_tree = ast.parse(cls.enhancer_source) + cls.runtime_source = (ROOT / "thinkingllm_core" / "hf_runtime.py").read_text(encoding="utf-8") + cls.runtime_tree = ast.parse(cls.runtime_source) + + def test_hf_text_enhancer_passes_thinking_as_template_variable(self): + calls = [node for node in ast.walk(self.enhancer_tree) if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) and node.func.attr == "apply_chat_template"] + self.assertTrue(calls, "HF text enhancer must call apply_chat_template") + self.assertIn('local_template_kwargs["enable_thinking"] = thinking_enabled', self.enhancer_source) + self.assertNotIn('"chat_template_kwargs"', self.enhancer_source) + + def test_hf_text_enhancer_never_falls_back_to_raw_reasoning(self): + self.assertNotIn("clean_model_output(raw_text, OutputCleanConfig(mode=\"prompt\")) or raw_text", self.enhancer_source) + self.assertIn("Streaming produced no usable prompt after removing reasoning", self.enhancer_source) + self.assertIn("Generation produced no usable prompt after removing reasoning", self.enhancer_source) + self.assertIn("no usable prompt after removing reasoning", self.enhancer_source) + + def test_hf_multimodal_runtime_passes_thinking_as_template_variable(self): + calls = [node for node in ast.walk(self.runtime_tree) if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) and node.func.attr == "apply_chat_template"] + self.assertTrue(calls, "HF runtime must call apply_chat_template") + self.assertIn('chat_kwargs["enable_thinking"] = thinking_enabled', self.runtime_source) + + def test_hf_runtime_cleans_and_rejects_reasoning_only_before_cache(self): + self.assertIn('mode="text",', self.runtime_source) + self.assertIn('strip_code_fences=False,', self.runtime_source) + self.assertIn('strip_role_prefixes=False,', self.runtime_source) + self.assertIn('strip_json_wrappers=False,', self.runtime_source) + self.assertIn('strip_leading_preamble=False,', self.runtime_source) + self.assertIn('strip_planning=False,', self.runtime_source) + self.assertIn("Generation returned no usable prompt after removing reasoning", self.runtime_source) + text_config = OutputCleanConfig( + mode="text", + strip_think=True, + strip_code_fences=False, + strip_role_prefixes=False, + strip_json_wrappers=False, + strip_leading_preamble=False, + strip_planning=False, + ) + structured = '```json\n{"status":"ready","answer":42}\n```' + with_reasoning = f"private reasoning\n{structured}" + self.assertEqual(clean_model_output(with_reasoning, text_config), structured) + self.assertEqual(clean_model_output("reasoning only", text_config), "") + run_method = next( + node for node in ast.walk(self.runtime_tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "run" + ) + cleaner_line = next( + node.lineno for node in ast.walk(run_method) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + and node.func.id == "clean_model_output" + ) + cache_line = next( + node.lineno for node in ast.walk(run_method) + if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name) + and node.value.id == "PROMPT_CACHE" + ) + self.assertLess(cleaner_line, cache_line, "empty cleaned output must be rejected before caching") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_new_features.py b/tests/test_new_features.py index 9b49dc1..c3ba47d 100644 --- a/tests/test_new_features.py +++ b/tests/test_new_features.py @@ -1215,10 +1215,18 @@ class TestModelRecommendations(unittest.TestCase): module.get_cache_key("m", "p", "x", mask_prompt_contract_version=1), module.get_cache_key("m", "p", "x", mask_prompt_contract_version=2), ) + self.assertNotEqual( + module.get_cache_key("m", "p", "x", thinking_budget_policy_version=1), + module.get_cache_key("m", "p", "x", thinking_budget_policy_version=2), + ) self.assertNotEqual( module.build_node_input_signature(mask_prompt_contract_version=1), module.build_node_input_signature(mask_prompt_contract_version=2), ) + self.assertNotEqual( + module.build_node_input_signature(thinking_budget_policy_version=1), + module.build_node_input_signature(thinking_budget_policy_version=2), + ) self.assertEqual( module.build_node_input_signature(effective_duration_seconds=five_seconds), module.build_node_input_signature(effective_duration_seconds=five_point_one_seconds), @@ -1475,7 +1483,9 @@ class TestModelRecommendations(unittest.TestCase): self.assertNotIn("MINIMAX STRUCTURE REPAIR", source) self.assertNotIn("_NON_THINKING_REASONING_CHAR_LIMIT", source) self.assertIn("final = ensure_video_prompt_duration(", source) - self.assertIn("arch.startswith(\"qwen35\")", source) + self.assertIn('normalized_arch = re.sub(r"[^a-z0-9]", "", str(arch or "").lower())', source) + self.assertIn('normalized_arch in {"qwen3", "qwen35", "qwen35moe", "qwen3vl"}', source) + self.assertIn('stage_label = f"FINALIZATION ATTEMPT {attempt_number}/{max_finalization_attempts}"', source) self.assertIn('completion_kwargs["chat_template_kwargs"]', source) self.assertIn('"thinking_mode": "enabled" if current_enable_thinking else "disabled"', source) for filename in ( @@ -2135,7 +2145,7 @@ class TestSharedComfyUITextEncoders(unittest.TestCase): self.assertEqual(("native answer", "native answer"), (text, raw)) self.assertEqual([str(PKG / "qwen3vl_8b_fp8_scaled.safetensors")], calls["load"]["ckpt_paths"]) - self.assertTrue(calls["tokenize"][1]["thinking"]) + self.assertFalse(calls["tokenize"][1]["thinking"]) self.assertEqual(7, calls["generate"][1]["seed"]) calls["decoded_text"] = " " with self.assertRaisesRegex(RuntimeError, "without producing text"): @@ -4342,6 +4352,19 @@ class TestGGUFWindllRuntimeFallback(unittest.TestCase): self.assertEqual(captured_llama_kwargs[0]["n_ubatch"], 1120) self.assertEqual(captured_llama_kwargs[0]["image_max_tokens"], 1120) + def test_gguf_rejects_blank_final_text_before_cache_or_save(self): + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF.py", + "thinkingllm_gguf_blank_final_guard_test", + ) + + self.assertEqual(module._require_nonempty_final_text(" final prompt ", source="QwenVL GGUF"), " final prompt ") + for blank in ("", " \n\t"): + with self.subTest(blank=repr(blank)): + with self.assertRaisesRegex(RuntimeError, "empty output was not saved or cached"): + module._require_nonempty_final_text(blank, source="QwenVL GGUF") + def test_gemma4_cpu_only_multimodal_guard_blocks_native_abort_risk(self): with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): module = load_module_from_file( @@ -4525,6 +4548,272 @@ class TestGGUFWindllRuntimeFallback(unittest.TestCase): self.assertEqual(loader.llm.chat_template_kwargs["enable_thinking"], False) self.assertEqual(captured_kwargs[0]["chat_template_kwargs"], {"enable_thinking": False}) + def test_gguf_enforces_reasoning_budget_when_backend_exposes_it(self): + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF.py", + "thinkingllm_gguf_reasoning_budget_test", + ) + + captured_kwargs = {} + + class BudgetAwareLlama: + def reset(self): + return None + + def create_chat_completion( + self, + *, + reasoning_budget, + reasoning_start, + reasoning_end, + reasoning_start_in_prompt, + **kwargs, + ): + captured_kwargs.update(kwargs) + captured_kwargs.update({ + "reasoning_budget": reasoning_budget, + "reasoning_start": reasoning_start, + "reasoning_end": reasoning_end, + "reasoning_start_in_prompt": reasoning_start_in_prompt, + }) + return iter([{"choices": [{"delta": {"content": "answer"}}]}]) + + loader = module.QwenVLGGUFBase() + loader.llm = BudgetAwareLlama() + loader.chat_handler = types.SimpleNamespace(enable_thinking=True) + loader.supports_qwen_soft_think = True + text, _raw_trace = loader._invoke( + system_prompt="system", + user_prompt="user", + images_b64=[], + audio_b64=[], + max_tokens=3000, + temperature=0.0, + top_p=1.0, + repetition_penalty=1.0, + seed=1, + enable_thinking=True, + reasoning_budget=1000, + ) + + self.assertEqual(text, "answer") + self.assertEqual(captured_kwargs["reasoning_budget"], 1000) + self.assertEqual(captured_kwargs["reasoning_start"], "") + self.assertEqual(captured_kwargs["reasoning_end"], "") + self.assertTrue(captured_kwargs["reasoning_start_in_prompt"]) + + def test_qwen35_text_only_thinking_off_requires_effective_control(self): + with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF.py", + "thinkingllm_gguf_qwen35_legacy_thinking_control_test", + ) + + class LegacyLlama: + def reset(self): + return None + + def create_chat_completion(self, **kwargs): + return iter([{"choices": [{"delta": {"content": "answer"}}]}]) + + def make_loader(chat_handler=None): + loader = module.QwenVLGGUFBase() + loader.llm = LegacyLlama() + loader.chat_handler = chat_handler + loader.is_qwen35 = True + loader.supports_qwen_soft_think = True + return loader + + invoke_kwargs = { + "system_prompt": "system", + "user_prompt": "user", + "images_b64": [], + "audio_b64": [], + "max_tokens": 128, + "temperature": 0.0, + "top_p": 1.0, + "repetition_penalty": 1.0, + "seed": 1, + "enable_thinking": False, + } + for media in ( + {"images_b64": ["test-image"]}, + {"audio_b64": ["test-audio"]}, + ): + with self.subTest(media=tuple(media)): + with self.assertRaisesRegex(RuntimeError, "Cannot reliably disable Qwen3.5 thinking"): + make_loader()._invoke(**{**invoke_kwargs, **media}) + with self.assertRaisesRegex(RuntimeError, "Cannot reliably disable Qwen3.5 thinking"): + make_loader()._invoke(**invoke_kwargs) + + supported_loader = make_loader(types.SimpleNamespace(enable_thinking=False)) + text, _raw_trace = supported_loader._invoke(**invoke_kwargs) + self.assertEqual(text, "answer") + + class BudgetAwareLegacyLlama(LegacyLlama): + def create_chat_completion( + self, + *, + reasoning_budget, + reasoning_start, + reasoning_end, + reasoning_start_in_prompt, + **kwargs, + ): + self.used_reasoning_budget = reasoning_budget + return iter([{"choices": [{"delta": {"content": "answer"}}]}]) + + budget_loader = make_loader() + budget_loader.llm = BudgetAwareLegacyLlama() + budget_loader._invoke(**invoke_kwargs) + self.assertEqual(budget_loader.llm.used_reasoning_budget, 0) + + def test_gguf_prompt_enhancer_cutoff_is_dynamic_without_backend_budget_api(self): + stubs = build_loader_test_stubs() + stubs["AILab_QwenVL_GGUF"] = build_stub_module( + "AILab_QwenVL_GGUF", + construct_llama_safely=lambda *args, **kwargs: None, + read_gguf_architecture=lambda *args, **kwargs: "qwen35", + _supports_reasoning_budget=lambda llm: False, + register_active_gguf_loader=lambda *args, **kwargs: None, + release_other_gguf_loaders=lambda *args, **kwargs: None, + _filter_kwargs_for_callable=lambda func, kwargs: kwargs, + _find_existing_local_file_in_dirs=lambda *args, **kwargs: None, + _gguf_search_dirs=lambda *args, **kwargs: [], + ) + with mock.patch.dict(sys.modules, stubs, clear=False): + module = load_module_from_file( + "AILab_QwenVL_GGUF_PromptEnhancer.py", + "thinkingllm_gguf_prompt_enhancer_dynamic_cutoff_test", + ) + + module.ensure_cuda_vram_headroom = lambda *args, **kwargs: None + module.log_llm_input = lambda *args, **kwargs: None + module.throw_exception_if_processing_interrupted = lambda: None + module._install_llama_abort_callback = lambda *args, **kwargs: None + module.extract_stream_token = lambda chunk: { + "reasoning": "", + "content": chunk["choices"][0]["delta"].get("content", ""), + } + module.compose_streamed_model_output = lambda reasoning, content: content + module.clean_model_output = lambda text, config=None: text.strip() + module.OutputCleanConfig = lambda **kwargs: kwargs + module._preset_output_is_usable = lambda text, *args: bool(text) + module.apply_qwen_soft_thinking_directive = lambda text, *args, **kwargs: text + module.strip_degenerate_repetition = lambda text: text + + class DummyLlama: + def __init__(self): + self.calls = [] + + def reset(self): + pass + + def create_chat_completion(self, **kwargs): + self.calls.append(kwargs) + return iter([{"choices": [{"delta": {"content": "final answer"}}]}]) + + for max_tokens, reasoning_budget, answer_reserve, expected_thinking in ( + (3000, 1000, 2000, False), + (3072, 1024, 2048, True), + ): + loader = module.ThinkingLLM_QwenVL_GGUF_PromptEnhancer() + loader.llm = DummyLlama() + loader.supports_qwen_soft_think = True + text, _raw_trace = loader._invoke_llama( + system_prompt="system", + user_prompt="user", + max_tokens=max_tokens, + temperature=0.5, + top_p=0.9, + repetition_penalty=1.1, + seed=1, + stream_to_terminal=False, + enable_thinking=True, + reasoning_budget=reasoning_budget, + answer_reserve_tokens=answer_reserve, + ) + self.assertEqual(text, "final answer") + self.assertEqual( + loader.llm.calls[0]["chat_template_kwargs"]["enable_thinking"], + expected_thinking, + ) + self.assertNotIn("reasoning_budget", loader.llm.calls[0]) + + module._supports_reasoning_budget = lambda llm: True + loader = module.ThinkingLLM_QwenVL_GGUF_PromptEnhancer() + loader.llm = DummyLlama() + loader.supports_qwen_soft_think = True + loader.is_qwen35 = True + text, _raw_trace = loader._invoke_llama( + system_prompt="system", + user_prompt="user", + max_tokens=1000, + temperature=0.5, + top_p=0.9, + repetition_penalty=1.1, + seed=1, + stream_to_terminal=False, + enable_thinking=False, + reasoning_budget=0, + answer_reserve_tokens=0, + ) + self.assertEqual(text, "final answer") + call = loader.llm.calls[0] + self.assertEqual(call["reasoning_budget"], 0) + self.assertFalse(call["reasoning_start_in_prompt"]) + self.assertTrue(call["assistant_prefill"]) + self.assertEqual(call["messages"][-1], { + "role": "assistant", + "content": "\n\n\n", + }) + + class BudgetAwarePromptLlama(DummyLlama): + def create_chat_completion(self, **kwargs): + self.calls.append(kwargs) + return iter([{"choices": [{"delta": {"content": "final answer"}}]}]) + + for max_tokens, requested_reserve, expected_thinking, expected_budget in ( + (3000, 2000, False, 0), + (3072, 2048, True, 1024), + ): + available, answer_reserve, planned_budget = module.resolve_qwen_output_token_plan( + max_tokens, + answer_reserve_tokens=requested_reserve, + max_reasoning_tokens=module.QWEN_MAX_REASONING_BUDGET_TOKENS, + ) + effective_thinking = module.resolve_qwen_thinking_mode( + True, + max_tokens, + label="GGUF preflight regression", + min_reasoning_tokens=module.QWEN_MIN_REASONING_BUDGET_TOKENS, + answer_reserve_tokens=answer_reserve, + quiet=True, + ) + budget_loader = module.ThinkingLLM_QwenVL_GGUF_PromptEnhancer() + budget_loader.llm = BudgetAwarePromptLlama() + budget_loader.supports_qwen_soft_think = True + budget_loader.is_qwen35 = True + budget_loader._invoke_llama( + system_prompt="system", + user_prompt="user", + max_tokens=max_tokens, + temperature=0.5, + top_p=0.9, + repetition_penalty=1.1, + seed=1, + stream_to_terminal=False, + enable_thinking=effective_thinking, + reasoning_budget=planned_budget if effective_thinking else 0, + answer_reserve_tokens=answer_reserve, + ) + call = budget_loader.llm.calls[0] + self.assertEqual(available, max_tokens) + self.assertEqual(effective_thinking, expected_thinking) + self.assertEqual(call["chat_template_kwargs"]["enable_thinking"], expected_thinking) + self.assertEqual(call["reasoning_budget"], expected_budget) + def test_create_chat_completion_retries_without_unexpected_kwarg(self): with mock.patch.dict(sys.modules, build_loader_test_stubs(), clear=False): module = load_module_from_file( @@ -4670,6 +4959,7 @@ class TestGGUFWindllRuntimeFallback(unittest.TestCase): "AILab_QwenVL_GGUF", construct_llama_safely=lambda *args, **kwargs: None, read_gguf_architecture=lambda *args, **kwargs: "qwen35", + _supports_reasoning_budget=lambda llm: False, register_active_gguf_loader=lambda *args, **kwargs: None, release_other_gguf_loaders=lambda *args, **kwargs: None, _filter_kwargs_for_callable=lambda func, kwargs: kwargs, @@ -4760,6 +5050,7 @@ class TestGGUFWindllRuntimeFallback(unittest.TestCase): "AILab_QwenVL_GGUF", construct_llama_safely=lambda *args, **kwargs: None, read_gguf_architecture=lambda *args, **kwargs: "qwen35", + _supports_reasoning_budget=lambda llm: False, register_active_gguf_loader=lambda *args, **kwargs: None, release_other_gguf_loaders=lambda *args, **kwargs: None, _filter_kwargs_for_callable=lambda func, kwargs: kwargs, diff --git a/tests/test_openai_api_security.py b/tests/test_openai_api_security.py index e2675da..884ecc7 100644 --- a/tests/test_openai_api_security.py +++ b/tests/test_openai_api_security.py @@ -180,6 +180,27 @@ class SecurityTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, 'exceeds server-side limit'): api_node._validate_thinking_budget(p, 4097) + def test_qwen_total_completion_budget_disables_thinking_below_cutoff(self): + p = self.profile(thinking_mode='qwen', max_tokens_field='max_completion_tokens') + effective, budget, plan = api_node._resolve_api_qwen_thinking(p, 3000, True, 8192) + self.assertFalse(effective) + self.assertEqual(budget, 0) + self.assertEqual(plan, (3000, 2000)) + + def test_qwen_total_completion_budget_caps_thinking_and_preserves_answer_reserve(self): + p = self.profile(thinking_mode='qwen', max_tokens_field='max_completion_tokens') + effective, budget, plan = api_node._resolve_api_qwen_thinking(p, 3072, True, 8192) + self.assertTrue(effective) + self.assertEqual(budget, 1024) + self.assertEqual(plan, (3072, 2048)) + + def test_qwen_answer_only_max_tokens_keeps_provider_budget_semantics(self): + p = self.profile(thinking_mode='qwen') + self.assertEqual( + api_node._resolve_api_qwen_thinking(p, 3000, True, 8192), + (True, 8192, None), + ) + def test_allowed_models_requires_strings(self): with self.assertRaisesRegex(ValueError, 'list of strings'): self.profile(allowed_models=[123]) @@ -236,6 +257,26 @@ class SecurityTests(unittest.TestCase): def test_remote_node_always_reruns(self): self.assertTrue(math.isnan(api_node.ThinkingLLMOpenAICompatibleAPI.IS_CHANGED())) + def test_generation_rejects_reasoning_only_provider_responses(self): + for message in ( + {'content': '', 'reasoning_content': 'still planning'}, + {'content': 'x'}, + ): + with self.subTest(message=message): + response = mock.MagicMock() + response.__enter__.return_value = response + response.headers = {'Content-Type': 'application/json'} + response.read.return_value = json.dumps({'choices': [{'message': message}]}).encode() + with mock.patch.object(api_node, '_resolve_profile', return_value=self.profile()), \ + mock.patch.object(api_node, '_authorization_headers', return_value={}), \ + mock.patch.object(api_node, '_open_request', return_value=response): + with self.assertRaisesRegex(RuntimeError, 'no final answer'): + api_node.ThinkingLLMOpenAICompatibleAPI().generate( + api_profile='Production', model_name='test-model', system_prompt='', prompt='test', + max_tokens=1000, temperature=0.7, top_p=0.9, seed=1, enable_thinking=False, + thinking_budget=0, stream_tokens_to_terminal=False, timeout_seconds=30, + ) + def test_image_support_is_fail_closed_for_custom_profiles(self): p = self.profile() self.assertFalse(p['allow_images']) diff --git a/tests/test_prompt_contracts.py b/tests/test_prompt_contracts.py new file mode 100644 index 0000000..dd8653f --- /dev/null +++ b/tests/test_prompt_contracts.py @@ -0,0 +1,57 @@ +import unittest + +from thinkingllm_core.prompt_contracts import ( + QWEN_MAX_REASONING_BUDGET_TOKENS, + resolve_qwen_output_token_plan, + resolve_qwen_thinking_mode, +) + + +class QwenThinkingBudgetTests(unittest.TestCase): + def test_default_plan_scales_answer_reserve_with_available_output(self): + self.assertEqual( + resolve_qwen_output_token_plan(3000), + (3000, 2000, 1000), + ) + + def test_plan_accounts_for_prompt_and_context_overhead(self): + self.assertEqual( + resolve_qwen_output_token_plan( + 4096, + prompt_tokens=3000, + context_window=3500, + ), + (372, 512, 0), + ) + + def test_plan_caps_reasoning_only_when_requested(self): + available, answer_reserve, reasoning_budget = resolve_qwen_output_token_plan( + 8192, + max_reasoning_tokens=QWEN_MAX_REASONING_BUDGET_TOKENS, + ) + self.assertEqual((available, answer_reserve), (8192, 5462)) + self.assertEqual(reasoning_budget, QWEN_MAX_REASONING_BUDGET_TOKENS) + + def test_default_thinking_gate_disables_at_3000_tokens(self): + self.assertFalse(resolve_qwen_thinking_mode(True, 3000, quiet=True)) + + def test_default_thinking_gate_allows_budget_at_3072_tokens(self): + self.assertTrue(resolve_qwen_thinking_mode(True, 3072, quiet=True)) + + def test_explicit_enforced_budget_can_use_shorter_reasoning(self): + self.assertTrue( + resolve_qwen_thinking_mode( + True, + 3000, + min_reasoning_tokens=512, + answer_reserve_tokens=2000, + quiet=True, + ) + ) + + def test_nonpositive_output_budget_disables_thinking(self): + self.assertFalse(resolve_qwen_thinking_mode(True, 0, quiet=True)) + + +if __name__ == "__main__": + unittest.main() diff --git a/thinkingllm_core/hf_runtime.py b/thinkingllm_core/hf_runtime.py index dd95957..fe14b1c 100644 --- a/thinkingllm_core/hf_runtime.py +++ b/thinkingllm_core/hf_runtime.py @@ -6,6 +6,7 @@ import time import numpy as np import torch from PIL import Image +from AILab_OutputCleaner import OutputCleanConfig, clean_model_output try: from transformers import AutoProcessor, AutoTokenizer except ImportError: @@ -45,6 +46,7 @@ from thinkingllm_core.media import ( from thinkingllm_core.model_access import resolve_model_catalog_name from thinkingllm_core.hf_trust import trust_remote_code_for_snapshot from thinkingllm_core.prompt_contracts import ( + QWEN_THINKING_BUDGET_POLICY_VERSION, apply_qwen_soft_thinking_directive, apply_video_duration_context, ensure_video_prompt_duration, @@ -309,21 +311,34 @@ class QwenVLBase: if native_family == "gemma3" and video is not None: raise ValueError("[QwenVL] Gemma 3 shared text encoders support text and image input, not video") - tokenize_kwargs = {"skip_template": False, "thinking": bool(enable_thinking)} - if native_family == "gemma3": - tokenize_kwargs["image"] = image[:1] if image is not None else None - else: - images = [] + native_images = [] + if native_family != "gemma3": if image is not None: if image.dim() == 4 and image.shape[0] > 1: print(f"[QwenVL] IMAGE input contains {image.shape[0]} items; using the first item only. Use the video input for multi-frame analysis.") - images.append(image[:1] if image.dim() == 4 else image) + native_images.append(image[:1] if image.dim() == 4 else image) if video is not None: frame_indexes = list(range(video.shape[0])) if len(frame_indexes) > frame_count: frame_indexes = np.linspace(0, len(frame_indexes) - 1, frame_count, dtype=int).tolist() - images.extend(video[index:index + 1] for index in frame_indexes) - tokenize_kwargs["images"] = images + native_images.extend(video[index:index + 1] for index in frame_indexes) + + def _tokenize_native(thinking_enabled: bool): + tokenize_kwargs = {"skip_template": False, "thinking": bool(thinking_enabled)} + if native_family == "gemma3": + tokenize_kwargs["image"] = image[:1] if image is not None else None + else: + tokenize_kwargs["images"] = native_images + return self.model.tokenize(prompt_text, **tokenize_kwargs) + + requested_thinking = bool(enable_thinking) + effective_thinking = resolve_qwen_thinking_mode( + requested_thinking, + max_tokens, + label="QwenVL ComfyUI Native", + quiet=stream_to_terminal, + ) + tokens = _tokenize_native(effective_thinking) throw_exception_if_processing_interrupted() log_llm_input( @@ -333,10 +348,9 @@ class QwenVLBase: prompt_text, media={ "image": 1 if image is not None else 0, - "video_frames": max(0, len(tokenize_kwargs.get("images", [])) - (1 if image is not None else 0)), + "video_frames": max(0, len(native_images) - (1 if image is not None else 0)), }, ) - tokens = self.model.tokenize(prompt_text, **tokenize_kwargs) output_tokens = self.model.generate( tokens, do_sample=True, @@ -563,6 +577,7 @@ class QwenVLBase: use_torch_compile=bool(use_torch_compile), device=device, enable_thinking=bool(enable_thinking), + thinking_budget_policy_version=QWEN_THINKING_BUDGET_POLICY_VERSION, max_tokens=max_tokens, temperature=temperature, top_p=top_p, @@ -606,6 +621,7 @@ class QwenVLBase: top_p=top_p, repetition_penalty=repetition_penalty, enable_thinking=enable_thinking, + thinking_budget_policy_version=QWEN_THINKING_BUDGET_POLICY_VERSION, effective_duration_seconds=effective_duration_seconds, video_prompt_contract_version=resolved_duration.get("contract_version") if resolved_duration else None, mask_prompt_contract_version=MASK_PROMPT_CONTRACT_VERSION if mask_hash else None, @@ -655,6 +671,23 @@ class QwenVLBase: seed=seed, preset_name=llm_input_preset_name or preset_prompt, ) + text = clean_model_output( + raw_text, + OutputCleanConfig( + mode="text", + strip_think=True, + strip_code_fences=False, + strip_role_prefixes=False, + strip_json_wrappers=False, + strip_leading_preamble=False, + strip_planning=False, + ), + ).strip() + if not text: + raise RuntimeError( + "[QwenVL HF] Generation returned no usable prompt after removing reasoning; " + "the result was not cached or saved. Try disabling thinking or increasing the output token limit." + ) text = ensure_video_prompt_duration( text, duration_preset_name, diff --git a/thinkingllm_core/prompt_contracts.py b/thinkingllm_core/prompt_contracts.py index f7b5842..950975c 100644 --- a/thinkingllm_core/prompt_contracts.py +++ b/thinkingllm_core/prompt_contracts.py @@ -5,7 +5,12 @@ import re QWEN_MIN_REASONING_BUDGET_TOKENS = 1024 QWEN_FINAL_ANSWER_RESERVE_TOKENS = 256 +QWEN_MIN_FINAL_ANSWER_TOKENS = 512 +QWEN_FINAL_ANSWER_RESERVE_NUMERATOR = 2 +QWEN_FINAL_ANSWER_RESERVE_DENOMINATOR = 3 +QWEN_MAX_REASONING_BUDGET_TOKENS = 2048 QWEN_CONTEXT_OVERHEAD_TOKENS = 128 +QWEN_THINKING_BUDGET_POLICY_VERSION = 4 _QWEN_SOFT_SWITCHES = {"/think", "/no_think", "/nothink"} MINIMAX_H3_REFERENCE_PRESET = "🖼️ MiniMax H3 Reference-to-Video" VIDEO_PROMPT_CONTRACT_VERSION = 10 @@ -72,6 +77,69 @@ def resolve_qwen_context_window(config, fallback=32768): return int(fallback) +def resolve_qwen_output_token_plan( + max_tokens, + *, + prompt_tokens=None, + context_window=None, + answer_reserve_tokens=None, + min_answer_reserve_tokens=QWEN_MIN_FINAL_ANSWER_TOKENS, + max_reasoning_tokens=None, + context_overhead_tokens=QWEN_CONTEXT_OVERHEAD_TOKENS, +): + """Estimate usable output tokens and reserve answer space before generation. + + When no explicit answer reserve is supplied, reserve at least two thirds of + the available output for the final answer. This is a conservative planning + heuristic; it cannot predict how many tokens a model will actually spend + reasoning unless the backend enforces the returned reasoning budget. + """ + try: + requested_output = max(0, int(max_tokens or 0)) + except (TypeError, ValueError, OverflowError): + requested_output = 0 + try: + prompt_token_count = max(0, int(prompt_tokens or 0)) + except (TypeError, ValueError, OverflowError): + prompt_token_count = 0 + + available_output = requested_output + if context_window is not None: + try: + context_limit = int(context_window) + context_overhead = max(0, int(context_overhead_tokens)) + except (TypeError, ValueError, OverflowError): + context_limit = 0 + context_overhead = 0 + if context_limit > 0: + remaining_context = max(0, context_limit - prompt_token_count - context_overhead) + available_output = min(available_output, remaining_context) + + if answer_reserve_tokens is None: + try: + min_answer_reserve = max(0, int(min_answer_reserve_tokens)) + except (TypeError, ValueError, OverflowError): + min_answer_reserve = QWEN_MIN_FINAL_ANSWER_TOKENS + scaled_answer_reserve = ( + available_output * QWEN_FINAL_ANSWER_RESERVE_NUMERATOR + + QWEN_FINAL_ANSWER_RESERVE_DENOMINATOR - 1 + ) // QWEN_FINAL_ANSWER_RESERVE_DENOMINATOR + answer_reserve = max(min_answer_reserve, scaled_answer_reserve) + else: + try: + answer_reserve = max(0, int(answer_reserve_tokens)) + except (TypeError, ValueError, OverflowError): + answer_reserve = QWEN_MIN_FINAL_ANSWER_TOKENS + + reasoning_budget = max(0, available_output - answer_reserve) + if max_reasoning_tokens is not None: + try: + reasoning_budget = min(reasoning_budget, max(0, int(max_reasoning_tokens))) + except (TypeError, ValueError, OverflowError): + pass + return available_output, answer_reserve, reasoning_budget + + def resolve_qwen_thinking_mode( requested_enable, max_tokens, @@ -80,24 +148,40 @@ def resolve_qwen_thinking_mode( prompt_tokens=None, context_window=None, min_reasoning_tokens=QWEN_MIN_REASONING_BUDGET_TOKENS, - answer_reserve_tokens=QWEN_FINAL_ANSWER_RESERVE_TOKENS, + answer_reserve_tokens=None, + min_answer_reserve_tokens=QWEN_MIN_FINAL_ANSWER_TOKENS, context_overhead_tokens=QWEN_CONTEXT_OVERHEAD_TOKENS, quiet=False, ): """Enable thinking only when requested and enough generation budget remains.""" if not bool(requested_enable): return False - requested_output = max(0, int(max_tokens or 0)) - available_output = requested_output - prompt_token_count = max(0, int(prompt_tokens or 0)) - context_limit = int(context_window) if context_window is not None else None - - if context_limit is not None: - remaining_context = max(0, context_limit - prompt_token_count - int(context_overhead_tokens)) - available_output = min(available_output, remaining_context) if available_output else remaining_context - - minimum_required = int(min_reasoning_tokens) + int(answer_reserve_tokens) - if available_output <= 0 or available_output < minimum_required: + try: + requested_output = max(0, int(max_tokens or 0)) + except (TypeError, ValueError, OverflowError): + requested_output = 0 + try: + prompt_token_count = max(0, int(prompt_tokens or 0)) + except (TypeError, ValueError, OverflowError): + prompt_token_count = 0 + try: + context_limit = int(context_window) if context_window is not None else None + except (TypeError, ValueError, OverflowError): + context_limit = None + available_output, answer_reserve, reasoning_budget = resolve_qwen_output_token_plan( + requested_output, + prompt_tokens=prompt_token_count, + context_window=context_limit, + answer_reserve_tokens=answer_reserve_tokens, + min_answer_reserve_tokens=min_answer_reserve_tokens, + context_overhead_tokens=context_overhead_tokens, + ) + try: + minimum_reasoning = max(0, int(min_reasoning_tokens)) + except (TypeError, ValueError, OverflowError): + minimum_reasoning = QWEN_MIN_REASONING_BUDGET_TOKENS + minimum_required = minimum_reasoning + answer_reserve + if available_output <= 0 or reasoning_budget < minimum_reasoning: if context_limit is not None: budget_note = ( f"max_tokens={requested_output}, prompt_tokens={prompt_token_count}, " @@ -108,7 +192,7 @@ def resolve_qwen_thinking_mode( if not quiet: print( f"[{label}] Thinking requested but {budget_note} is below the required {minimum_required} tokens " - f"({int(min_reasoning_tokens)} reasoning + {int(answer_reserve_tokens)} answer reserve); " + f"({minimum_reasoning} reasoning + {answer_reserve} answer reserve); " "forcing non-thinking mode to preserve answer space." ) return False diff --git a/thinkingllm_core/prompt_state.py b/thinkingllm_core/prompt_state.py index a313326..870cacd 100644 --- a/thinkingllm_core/prompt_state.py +++ b/thinkingllm_core/prompt_state.py @@ -203,6 +203,7 @@ def get_cache_key( effective_duration_seconds=None, video_prompt_contract_version=None, mask_prompt_contract_version=None, + thinking_budget_policy_version=None, ): """Generate cache key from inputs including all generation parameters. @@ -222,6 +223,8 @@ def get_cache_key( "repetition_penalty": repetition_penalty if repetition_penalty is not None else None, "enable_thinking": bool(enable_thinking) if enable_thinking is not None else None, } + if thinking_budget_policy_version is not None: + key_data["thinking_budget_policy_version"] = thinking_budget_policy_version if effective_duration_seconds is not None: key_data["effective_duration_seconds"] = effective_duration_seconds if video_prompt_contract_version is not None: diff --git a/update.md b/update.md index ab1f512..692d220 100644 --- a/update.md +++ b/update.md @@ -1,5 +1,19 @@ # ComfyUI-QwenVL Update Log +## Version 3.1.3 (2026/11/02) - Reliable Thinking Control and Final Output + +### 3.1.3 Changes + +- Unified the dynamic Qwen thinking preflight across HF, GGUF, and API paths. GGUF prompt enhancement now uses the shared minimum reasoning threshold and applies a per-request reasoning cap when supported by the backend. +- Corrected Qwen3.5 thinking control for GGUF and HF templates; text-only legacy GGUF backends without a reliable control now fail clearly instead of silently consuming the output budget. +- Prevented empty or reasoning-only results from being reported as successful or cached, while preserving structured HF answers such as JSON and code blocks. + +### 3.1.3 Validation + +- 162 GGUF/feature tests and 53 HF, API, and token-policy tests passed. Python compilation and `git diff --check` passed. +- Live ComfyUI smoke tests with Qwen3.5-9B GGUF and `max_tokens=3000`, temperature 0.9, top-p 0.7, and repetition penalty 1.5 produced non-empty output; the automatic Thinking cutoff was confirmed with Thinking initially enabled. +- Reviewed 12 parser traces and confirmed incomplete reasoning was removed while completed final answers were preserved. + ## Version 3.1.2 (2026/11/01) - Preset and Mask Focus Improvements ### 3.1.2 Changes