remove thinking
This commit is contained in:
KingManiya
@@ -11,7 +11,6 @@ This node is intentionally small: it discovers local `.gguf` models from
|
||||
- Qwen text generation with GGUF models
|
||||
- Optional image input through llama.cpp multimodal support
|
||||
- Separate `RESPONSE` and `THINKING` outputs
|
||||
- `/think` and `/no_think` control for Qwen reasoning models
|
||||
- System prompt presets from text files
|
||||
- Recursive model discovery from `ComfyUI/models/LLM`
|
||||
- Automatic llama.cpp binary download for Windows x64 CUDA 13
|
||||
@@ -131,7 +130,6 @@ Each top-level `.txt` file appears in the `system_prompt` dropdown. Choose
|
||||
| `n_gpu_layers` | Used in GPU layer modes. Passes `--gpu-layers` / `-ngl`. |
|
||||
| `n_cpu_moe_layers` | Used in CPU MoE modes. Passes `--n-cpu-moe`. |
|
||||
| `seed` | Random seed. Use `-1` for a random seed. |
|
||||
| `enable_thinking` | Uses `/think` when enabled and `/no_think` when disabled. |
|
||||
| `timeout_seconds` | Maximum runtime before the subprocess is stopped. |
|
||||
| `image` | Optional ComfyUI image input. Uses the first image in a batch. |
|
||||
| `extra_args` | Advanced llama.cpp CLI flags appended to the command. |
|
||||
@@ -143,7 +141,7 @@ Every input includes an in-node tooltip.
|
||||
| Output | Description |
|
||||
| --- | --- |
|
||||
| `RESPONSE` | Final model response with Qwen thinking blocks removed. |
|
||||
| `THINKING` | Extracted `<think>...</think>` reasoning when thinking is enabled. |
|
||||
| `THINKING` | Extracted `<think>...</think>` reasoning when present in model output. |
|
||||
| `PERF` | llama.cpp `llama_perf_context_print` lines. |
|
||||
|
||||
Both outputs include ComfyUI output tooltips.
|
||||
|
||||
+6
-21
@@ -52,14 +52,10 @@ def _write_temp_text_file(prefix: str, text: str) -> Path:
|
||||
return text_path
|
||||
|
||||
|
||||
def _write_prompt_file(system_prompt: str, prompt: str, enable_thinking: bool) -> Path:
|
||||
# Qwen chat templates understand these control tokens across llama.cpp builds
|
||||
# more reliably than version-specific template kwargs.
|
||||
think_prefix = "/think" if enable_thinking else "/no_think"
|
||||
def _write_prompt_file(system_prompt: str, prompt: str) -> Path:
|
||||
chunks = []
|
||||
if system_prompt:
|
||||
chunks.append(system_prompt.strip())
|
||||
chunks.append(think_prefix)
|
||||
chunks.append(prompt.strip())
|
||||
|
||||
return _write_temp_text_file("qwen-gguf-prompt-", "\n\n".join(chunks))
|
||||
@@ -99,7 +95,6 @@ def build_command(
|
||||
n_gpu_layers: int,
|
||||
n_cpu_moe_layers: int,
|
||||
seed: int,
|
||||
text_only: bool,
|
||||
extra_args: str = "",
|
||||
) -> list[str]:
|
||||
if memory_mode not in MEMORY_MODES:
|
||||
@@ -126,12 +121,6 @@ def build_command(
|
||||
if memory_mode in {"cpu_moe_layers", "gpu_and_cpu_moe_layers"}:
|
||||
command.extend(["--n-cpu-moe", str(n_cpu_moe_layers)])
|
||||
|
||||
if text_only:
|
||||
command.extend([
|
||||
"--jinja",
|
||||
"--single-turn",
|
||||
"--no-display-prompt",
|
||||
])
|
||||
command.extend(["-f", str(prompt_path)])
|
||||
|
||||
if mmproj_path is not None:
|
||||
@@ -160,26 +149,23 @@ def run_llama_cli(
|
||||
n_gpu_layers: int,
|
||||
n_cpu_moe_layers: int,
|
||||
seed: int,
|
||||
enable_thinking: bool,
|
||||
timeout_seconds: int,
|
||||
extra_args: str = "",
|
||||
) -> tuple[str, str]:
|
||||
) -> tuple[str, str, str]:
|
||||
image_path = None
|
||||
prompt_path = None
|
||||
process = None
|
||||
try:
|
||||
effective_mmproj_path = None
|
||||
effective_cli_path = cli_paths.text
|
||||
text_only = True
|
||||
if image is not None:
|
||||
if mmproj_path is None:
|
||||
raise ValueError("Image input requires a selected mmproj GGUF file.")
|
||||
effective_cli_path = cli_paths.multimodal
|
||||
effective_mmproj_path = mmproj_path
|
||||
text_only = False
|
||||
image_path = tensor_to_temp_png(image)
|
||||
|
||||
prompt_path = _write_prompt_file(system_prompt, prompt, enable_thinking)
|
||||
prompt_path = _write_prompt_file(system_prompt, prompt)
|
||||
|
||||
command = build_command(
|
||||
cli_path=effective_cli_path,
|
||||
@@ -197,7 +183,6 @@ def run_llama_cli(
|
||||
n_gpu_layers=n_gpu_layers,
|
||||
n_cpu_moe_layers=n_cpu_moe_layers,
|
||||
seed=seed,
|
||||
text_only=text_only,
|
||||
extra_args=extra_args,
|
||||
)
|
||||
|
||||
@@ -231,8 +216,9 @@ def run_llama_cli(
|
||||
raise RuntimeError(
|
||||
f"llama.cpp inference failed with exit code {result.returncode}:\n{stderr[-2000:]}"
|
||||
)
|
||||
response, thinking = extract_thinking(result.stdout)
|
||||
perf = extract_perf(result.stderr)
|
||||
return result.stdout, perf
|
||||
return response, thinking, perf
|
||||
|
||||
|
||||
def _stop_process(process: subprocess.Popen) -> None:
|
||||
@@ -272,7 +258,7 @@ def extract_perf(stderr: str) -> str:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def extract_thinking(text: str, enable_thinking: bool) -> tuple[str, str]:
|
||||
def extract_thinking(text: str) -> tuple[str, str]:
|
||||
text = str(text or "")
|
||||
thinking = ""
|
||||
|
||||
@@ -295,5 +281,4 @@ def extract_thinking(text: str, enable_thinking: bool) -> tuple[str, str]:
|
||||
text = text.replace(token, "")
|
||||
|
||||
response = text.strip()
|
||||
thinking = thinking if enable_thinking else ""
|
||||
return response, thinking
|
||||
|
||||
@@ -10,7 +10,7 @@ from .folder_registry import (
|
||||
system_prompt_options,
|
||||
)
|
||||
from .llama_binary import ensure_llama_cli_paths
|
||||
from .llama_cli import MAX_LLAMA_SEED, MEMORY_MODES, extract_thinking, run_llama_cli
|
||||
from .llama_cli import MAX_LLAMA_SEED, MEMORY_MODES, run_llama_cli
|
||||
|
||||
|
||||
class QwenGGUF:
|
||||
@@ -99,10 +99,6 @@ class QwenGGUF:
|
||||
"max": MAX_LLAMA_SEED,
|
||||
"tooltip": "Random seed used by llama.cpp. Use -1 for a random seed.",
|
||||
}),
|
||||
"enable_thinking": ("BOOLEAN", {
|
||||
"default": False,
|
||||
"tooltip": "Use /think and return extracted reasoning in the THINKING output.",
|
||||
}),
|
||||
"timeout_seconds": ("INT", {
|
||||
"default": 300,
|
||||
"min": 10,
|
||||
@@ -126,7 +122,7 @@ class QwenGGUF:
|
||||
RETURN_NAMES = ("RESPONSE", "THINKING", "PERF")
|
||||
OUTPUT_TOOLTIPS = (
|
||||
"Final model response with Qwen thinking blocks removed.",
|
||||
"Extracted <think> reasoning when thinking mode is enabled.",
|
||||
"Extracted <think> reasoning when present in model output.",
|
||||
"llama.cpp llama_perf_context_print lines.",
|
||||
)
|
||||
FUNCTION = "generate"
|
||||
@@ -149,7 +145,6 @@ class QwenGGUF:
|
||||
n_gpu_layers: int,
|
||||
n_cpu_moe_layers: int,
|
||||
seed: int,
|
||||
enable_thinking: bool,
|
||||
timeout_seconds: int,
|
||||
image=None,
|
||||
extra_args: str = "",
|
||||
@@ -162,7 +157,7 @@ class QwenGGUF:
|
||||
# workflows do not touch the network until this node actually runs.
|
||||
cli_paths = ensure_llama_cli_paths()
|
||||
|
||||
raw_output, perf = run_llama_cli(
|
||||
response, thinking, perf = run_llama_cli(
|
||||
cli_paths=cli_paths,
|
||||
model_path=model_path,
|
||||
mmproj_path=mmproj_path,
|
||||
@@ -179,11 +174,9 @@ class QwenGGUF:
|
||||
n_gpu_layers=n_gpu_layers,
|
||||
n_cpu_moe_layers=n_cpu_moe_layers,
|
||||
seed=seed,
|
||||
enable_thinking=enable_thinking,
|
||||
timeout_seconds=timeout_seconds,
|
||||
extra_args=extra_args,
|
||||
)
|
||||
response, thinking = extract_thinking(raw_output, enable_thinking)
|
||||
return (response, thinking, perf)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user