nodes/SeargeDP-ComfyUI_Searge_LLM
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Forked, refactored, and simplified the LLM_Node into Searge_LLM_Node

Browse Source
This commit is contained in:
Searge
2024-08-26 10:57:58 +02:00
parent 6a5ef0576a
commit 92a454f35e
12 changed files with 895 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/publish.yml
-22
View File
@@ -1,22 +0,0 @@
name: Publish to Comfy registry
on:
workflow_dispatch:
push:
branches:
- main
- master
paths:
- "pyproject.toml"
jobs:
publish-node:
name: Publish Custom Node to registry
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Publish Custom Node
uses: Comfy-Org/publish-node-action@main
with:
## Add your own personal access token to your Github Repository secrets and reference it here.
personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}
LICENSE
+1
View File
@@ -1,6 +1,7 @@
MIT License
Copyright (c) 2024 Big-Idea-Technology
Copyright (c) 2024 Searge
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
LLM_Node.py
-410
View File
@@ -1,410 +0,0 @@
from transformers import (
AutoTokenizer,
AutoModelForSequenceClassification,
AutoModelForCausalLM,
AutoModelForSeq2SeqLM,
AutoConfig,
BitsAndBytesConfig
)
from llama_cpp import Llama
import torch
import os
import folder_paths
import re
import subprocess
import datetime
import shutil
GLOBAL_MODELS_DIR = os.path.join(folder_paths.models_dir, "LLM_checkpoints")
WEB_DIRECTORY = "./web/assets/js"
class AnyType(str):
"""A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
def __ne__(self, __value: object) -> bool:
return False
any = AnyType("*")
class LLM_Node:
def __init__(self, device="cuda"):
self.device = device
self.custom_nodes_folder = folder_paths.folder_names_and_paths['custom_nodes'][0][0]
self.comfy_ui_llm_node_path = os.path.join(self.custom_nodes_folder, "ComfyUI_LLM_Node")
# Check if bfloat16 is supported by the device
self.supports_bfloat16 = 'cuda' in device and torch.cuda.is_bf16_supported()
self.fixmeused = False
@classmethod
def INPUT_TYPES(cls):
# Get a list of directories in the checkpoints_path
model_options = []
for name in os.listdir(GLOBAL_MODELS_DIR):
dir_path = os.path.join(GLOBAL_MODELS_DIR, name)
if os.path.isdir(dir_path):
if "GGUF" in name:
gguf_files = [os.path.join(name, file) for file in os.listdir(dir_path) if file.endswith('.gguf')]
model_options.extend(gguf_files)
else:
model_options.append(name)
return {
"required": {
"text": ("STRING", {"multiline": True, "default": ""}),
"seed": ("INT", {"default": 777}),
"model": (model_options, ),
"max_tokens": ("INT", {"default": 2000, "min": 1}),
"apply_chat_template": ("BOOLEAN", {"default": False}),
},
"optional": {
"AdvOptionsConfig": ("ADVOPTIONSCONFIG",),
"QuantizationConfig": ("QUANTIZATIONCONFIG",),
"CodingConfig": ("CODINGCONFIG",),
}
}
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("string",)
OUTPUT_NODE = False
FUNCTION = "main"
CATEGORY = "LLM"
def fixme(self, filename, tokenizer, model_to_use, generate_kwargs, apply_chat_template, text):
self.fixmeused = True
generated_text = "\n\n**************************************** FIX ME loop ****************************************\n\n"
generated_text += text+"\n\n\n"
file_path = os.path.join(self.comfy_ui_llm_node_path, "tmp", filename)
try:
generated_text += self.generate_text(text, tokenizer, model_to_use, generate_kwargs, apply_chat_template)
except Exception as e:
print(f"Failed to generate new text for {filename}: {e}")
return
match = re.search(r'```python\s*([\s\S]+?)\s*```', generated_text)
if not match:
print(f"No code block found in generated text for {filename}.")
return
new_code = match.group(1)
# Write the new content back to the file
try:
with open(file_path, 'w') as file:
file.write(new_code)
print(f"File {filename} has been updated successfully.")
return generated_text
except IOError as e:
print(f"Failed to write new content to file {filename}: {e}")
return
def extract_files_and_code(self, text):
files_code = []
# First pattern to check for filenames and associated Python code
primary_pattern = r'\*\*(.+?\.py):\*\*\s*```python\s*([\s\S]+?)```'
matches = re.findall(primary_pattern, text)
if matches:
for filename, code in matches:
files_code.append((filename, code.strip()))
else:
# Secondary pattern to check for Python code blocks without filenames
secondary_pattern = r'```python\s*([\s\S]+?)\s*```'
code_blocks = re.findall(secondary_pattern, text)
for code in code_blocks:
files_code.append(("main.py", code.strip()))
return files_code
def write_files_to_folder(self, files_code, folder_path):
for filename, code in files_code:
file_path = os.path.join(folder_path, filename)
with open(file_path, 'w') as file:
file.write(code)
def log_history(self, user_text, generated_text):
history_dir = os.path.join(self.comfy_ui_llm_node_path, "history")
timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
filename = datetime.datetime.now().strftime('%Y-%m-%d') + '.txt'
filepath = os.path.join(history_dir, filename)
with open(filepath, 'a') as file:
file.write(f"{timestamp} - User: {user_text}\n")
file.write(f"{timestamp} - Generated: {generated_text}\n\n")
def generate_text(self, text, tokenizer, model_to_use, generate_kwargs, apply_chat_template):
if apply_chat_template:
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": text}
]
text = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True
)
input_ids = tokenizer([text], return_tensors="pt").input_ids.to(self.device)
outputs = model_to_use.generate(input_ids, **generate_kwargs)
if apply_chat_template:
generated_ids = [output_ids[len(input_id):] for input_id, output_ids in zip(input_ids, outputs)]
generated_text = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
else:
generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
return generated_text
def main(self, text, seed, model, max_tokens, apply_chat_template, AdvOptionsConfig=None, QuantizationConfig=None, CodingConfig=None):
model_path = os.path.join(GLOBAL_MODELS_DIR, model)
generated_text = None
if "GGUF" in model:
generate_kwargs = {'max_tokens': max_tokens}
if AdvOptionsConfig:
for option in ['temperature', 'top_p', 'top_k', 'repetition_penalty']:
if option in AdvOptionsConfig:
if (option == 'repetition_penalty'):
option1 = 'repeat_penalty'
else:
option1 = option
generate_kwargs[option1] = AdvOptionsConfig[option]
model_to_use = Llama(
model_path=model_path,
n_gpu_layers=-1,
seed=seed,
# n_ctx=2048, # Uncomment to increase the context window
)
generated_text = model_to_use(text, **generate_kwargs)
self.log_history(text, generated_text['choices'][0]['text'])
return (generated_text['choices'][0]['text'],)
else:
torch.manual_seed(seed)
model_kwargs = {
'device_map': 'auto',
'quantization_config': QuantizationConfig
}
if AdvOptionsConfig:
if 'trust_remote_code' in AdvOptionsConfig:
model_kwargs['trust_remote_code'] = AdvOptionsConfig['trust_remote_code']
if 'torch_dtype' in AdvOptionsConfig and hasattr(torch, AdvOptionsConfig['torch_dtype']):
model_kwargs['torch_dtype'] = getattr(torch, AdvOptionsConfig['torch_dtype'])
config = AutoConfig.from_pretrained(model_path, **model_kwargs)
tokenizer = AutoTokenizer.from_pretrained(model_path)
if config.model_type == "t5":
model_to_use = AutoModelForSeq2SeqLM.from_pretrained(model_path, **model_kwargs)
elif config.model_type in ["gpt2", "gpt_refact", "gemma", "llama", "mistral", "qwen2"]:
model_to_use = AutoModelForCausalLM.from_pretrained(model_path, **model_kwargs)
elif config.model_type == "bert":
model_to_use = AutoModelForSequenceClassification.from_pretrained(model_path, **model_kwargs)
else:
raise ValueError(f"Unsupported model type: {config.model_type}")
generate_kwargs = {'max_length': max_tokens}
if AdvOptionsConfig:
for option in ['temperature', 'top_p', 'top_k', 'repetition_penalty']:
if option in AdvOptionsConfig:
generate_kwargs[option] = AdvOptionsConfig[option]
if config.model_type in ["t5", "gpt2", "gpt_refact", "gemma", "llama", "mistral", "qwen2"]:
generated_text = self.generate_text(text, tokenizer, model_to_use, generate_kwargs, apply_chat_template)
chat_story = generated_text
if CodingConfig and CodingConfig.get('execute_code'):
files_code = self.extract_files_and_code(generated_text)
execution_attempts = 0
while execution_attempts < 10:
if files_code:
tmp_folder_path = os.path.join(self.comfy_ui_llm_node_path, "tmp")
if self.fixmeused == False:
os.makedirs(tmp_folder_path, exist_ok=True)
self.write_files_to_folder(files_code, tmp_folder_path)
pycache_path = os.path.join(tmp_folder_path, "__pycache__")
if os.path.exists(pycache_path):
shutil.rmtree(pycache_path)
files = os.listdir(tmp_folder_path)
files.sort(key=lambda x: x == 'main.py')
for filename in files:
file_path = os.path.join(tmp_folder_path, filename)
try:
env = os.environ.copy()
env["SDL_VIDEODRIVER"] = "dummy"
command = ['python', file_path]
try:
subprocess.run(command, capture_output=True, text=True, check=True, env=env, timeout=1) # no output only check for errors
print(f"Successful Execution: {filename}")
except subprocess.TimeoutExpired:
print("Execution timed out after 1 second. Terminating subprocess.")
except subprocess.CalledProcessError as e:
print(f"Execution failed, retrying... Error: {e.stderr}")
with open(file_path, 'r') as file:
file_data = file.read()
text = f"Error encountered: {e.stderr}\nFix the code. Write all code. Don't take shortcut. Don't write 'same as your original code'!\n{file_data}"
chat_story += self.fixme(filename, tokenizer, model_to_use, generate_kwargs, apply_chat_template, text)
execution_attempts += 1
break
else: # no more files to check
break
else: # no code found
break
self.log_history(text, chat_story)
if (CodingConfig.get('project_folder')):
for filename in files:
source_file_path = os.path.join(tmp_folder_path, filename)
destination_file_path = os.path.join(CodingConfig.get('project_folder'), filename)
shutil.copy(source_file_path, destination_file_path)
main_py_path = os.path.join(CodingConfig.get('project_folder'), 'main.py')
subprocess.run(['python', main_py_path], capture_output=True, text=True, check=True)
if os.path.exists(tmp_folder_path):
shutil.rmtree(tmp_folder_path)
return (chat_story,)
else:
self.log_history(text, chat_story)
return (chat_story,)
elif config.model_type == "bert":
return ("BERT model detected; specific task handling not implemented in this example.",)
class Output_Node:
def __init__(self):
pass
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"text": (any, {}),
}
}
OUTPUT_NODE = True
FUNCTION = "main"
CATEGORY = "LLM"
RETURN_TYPES = ()
def main(self, text):
return {"ui": {"text": (text,)}}
class QuantizationConfig_Node:
def __init__(self):
pass
@classmethod
def INPUT_TYPES(cls):
quantization_modes = ["none", "load_in_8bit", "load_in_4bit"]
return {
"required": {
"quantization_mode": (quantization_modes, {"default": "none"}),
"llm_int8_threshold": ("FLOAT", {"default": 6.0}),
"llm_int8_skip_modules": ("STRING", {"default": ""}),
"llm_int8_enable_fp32_cpu_offload": ("BOOLEAN", {"default": False}),
"llm_int8_has_fp16_weight": ("BOOLEAN", {"default": False}),
"bnb_4bit_compute_dtype": ("STRING", {"default": "float32"}),
"bnb_4bit_quant_type": ("STRING", {"default": "fp4"}),
"bnb_4bit_use_double_quant": ("BOOLEAN", {"default": False}),
"bnb_4bit_quant_storage": ("STRING", {"default": "uint8"}),
}
}
FUNCTION = "main"
CATEGORY = "LLM"
RETURN_TYPES = ("QUANTIZATIONCONFIG",)
RETURN_NAMES = ("QuantizationConfig",)
def main(self, quantization_mode, llm_int8_threshold: float = 6.0, llm_int8_skip_modules="", llm_int8_enable_fp32_cpu_offload=False, llm_int8_has_fp16_weight=False, bnb_4bit_compute_dtype="float32", bnb_4bit_quant_type="fp4", bnb_4bit_use_double_quant=False, bnb_4bit_quant_storage="uint8"):
llm_int8_skip_modules_list = llm_int8_skip_modules.split(',') if llm_int8_skip_modules else []
quantization_config = BitsAndBytesConfig(
load_in_8bit=quantization_mode == "load_in_8bit",
load_in_4bit=quantization_mode == "load_in_4bit",
llm_int8_threshold=float(llm_int8_threshold),
llm_int8_skip_modules=llm_int8_skip_modules_list,
llm_int8_enable_fp32_cpu_offload=llm_int8_enable_fp32_cpu_offload,
llm_int8_has_fp16_weight=llm_int8_has_fp16_weight,
bnb_4bit_compute_dtype=getattr(torch, bnb_4bit_compute_dtype, torch.float32),
bnb_4bit_quant_type=bnb_4bit_quant_type,
bnb_4bit_use_double_quant=bnb_4bit_use_double_quant,
bnb_4bit_quant_storage=bnb_4bit_quant_storage,
)
return (quantization_config,)
class AdvOptionsNode:
def __init__(self):
pass
@classmethod
def INPUT_TYPES(cls):
dtype_options = ["auto", "float32", "bfloat16", "float16", "float64"]
return {
"required": {
"temperature": ("FLOAT", {"default": 1.0, "min": 0.1, "step": 0.1}),
"top_p": ("FLOAT", {"default": 0.9, "min": 0.1, "step": 0.1}),
"top_k": ("INT", {"default": 50, "min": 0}),
"repetition_penalty": ("FLOAT", {"default": 1.2, "min": 0.1, "step": 0.1}),
"trust_remote_code": ("BOOLEAN", {"default": False}),
"torch_dtype": (dtype_options, {"default": "auto"}),
}
}
FUNCTION = "main"
CATEGORY = "LLM"
RETURN_TYPES = ("ADVOPTIONSCONFIG",)
RETURN_NAMES = ("AdvOptionsConfig",)
def main(self, temperature=1.0, top_p=0.9, top_k=50, repetition_penalty=1.2, trust_remote_code=False, torch_dtype="auto"):
options_config = {
"temperature": temperature,
"top_p": top_p,
"top_k": top_k,
"repetition_penalty": repetition_penalty,
"trust_remote_code": trust_remote_code,
"torch_dtype": torch_dtype,
}
return (options_config,)
class CodingOptionsNode:
def __init__(self):
pass
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"execute_code": ("BOOLEAN", {"default": False}),
"project_folder": ("STRING", {"default": "/projects/LLM"}),
}
}
FUNCTION = "main"
CATEGORY = "LLM"
RETURN_TYPES = ("CODINGCONFIG",)
RETURN_NAMES = ("CodingConfig",)
def main(self, execute_code, project_folder):
return ({"execute_code":execute_code,"project_folder":project_folder},)
NODE_CLASS_MAPPINGS = {
"LLM_Node": LLM_Node,
"Output_Node": Output_Node,
"QuantizationConfig_Node": QuantizationConfig_Node,
"AdvOptions_Node": AdvOptionsNode,
"CodingOptionsNode": CodingOptionsNode,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"LLM_Node": "LLM Node",
"Output_Node": "Output Node",
"QuantizationConfig_Node": "Quantization Config Node",
"AdvOptions_Node": "Advanced Options Node",
"CodingOptionsNode": "Code Config Node",
}
README.md
+58 -49
View File
@@ -1,66 +1,75 @@
## LLM_Node for ComfyUI
The `LLM_Node` enhances ComfyUI by integrating advanced language model capabilities, enabling a wide range of NLP tasks such as text generation, content summarization, question answering, and more. This flexibility is powered by various transformer model architectures from the transformers library, allowing for the deployment of models like T5, GPT-2, and others based on your project's needs.
# Searge-LLM for ComfyUI v1.0
## Features
- **Versatile Text Generation:** Leverage state-of-the-art transformer models for dynamic text generation, adaptable to a wide range of NLP tasks.
- **Customizable Model and Tokenizer Paths:** Specify paths within the `models/LLM_checkpoints` directory for using specialized models tailored to specific tasks.
- **Dynamic Token Limit and Generation Parameters:** Control the length of generated content and fine-tune generation with parameters such as `temperature`, `top_p`, `top_k`, and `repetition_penalty`.
- **Device-Specific Optimizations:** Automatically utilize `bfloat16` on compatible CUDA devices for enhanced performance.
- **Seamless Integration:** Designed for easy integration into ComfyUI workflows, enriching applications with powerful NLP functionalities.
A prompt-generator or prompt-improvement node for ComfyUI, utilizing the power of a language model to turn a provided
text-to-image prompt into a more detailed and improved prompt.
## Installation
This Node is designed for use within ComfyUI. Ensure ComfyUI is installed and operational in your environment.
## Install the language model
- Create a new folder called `llm_gguf` in the `ComfyUI/models` directory.
- Download the file `Mistral-7B-Instruct-v0.3.Q4_K_M.gguf` **(4.37 GB)**.
from the repository `MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF` on HuggingFace.
- [download link to the gguf model](https://huggingface.co/MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF/resolve/main/Mistral-7B-Instruct-v0.3.Q4_K_M.gguf)
- place `Mistral-7B-Instruct-v0.3.Q4_K_M.gguf` in the `ComfyUI/models/llm_gguf` directory.
1. **Prepare the Models Directory:**
- Create a `LLM_checkpoints` directory within the `models` directory of your ComfyUI environment.
- Place your transformer model directories in `LLM_checkpoints`. Each directory should contain the necessary model and tokenizer files.
### Note
- Currently the node requires the language model as a `gguf` file and only works with models that are
supported by `llama-cpp-python`.
2. **Node Integration:**
- Copy the `LLM_Node` class file into the `custom_nodes` directory accessible by your ComfyUI project.
## Potential problems
(this was only tested this on Windows)
## Configuration
Configure the LLM_Node with the necessary parameters within your ComfyUI project to utilize its capabilities fully:
If you get error message about missing `llama-cpp`, try these manual steps:
- These instruction are assuming that you use the portable version of ComfyUI, otherwise make sure to run these commands
in the pything v-env that you're using for ComfyUI.
- Open a command line interface in the directory `ComfyUI_windows_portable/python_embeded`.
- It's important to run these commands in the `ComfyUI_windows_portable/python_embeded` directory.
- Run the following commands:
```
python -m pip install https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.89+cpuavx2-cp311-cp311-win_amd64.whl
python -m pip install https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.89+cu121-cp311-cp311-win_amd64.whl
```
## Searge LLM Node
Configure the `Searge_LLM_Node` with the necessary parameters within your ComfyUI project to utilize its capabilities
fully:
- `text`: The input text for the language model to process.
- `model`: The directory name of the model within `models/LLM_checkpoints` you wish to use.
- `model`: The directory name of the model within `models/llm_gguf` you wish to use.
- `max_tokens`: Maximum number of tokens for the generated text, adjustable according to your needs.
- `apply_instructions`:
- `instructions`: The instructions for the language model to generate a prompt. It supports the placeholder
`{prompt}` to insert the prompt from the `text` input.
**Example:** `Generate a prompt from "{prompt}"`
- Advanced Options Node (`AdvOptions`):
- Fine-tune the generation process with parameters such as `temperature`, `top_p`, `top_k`, and `repetition_penalty`.
- Control security via `trust_remote_code` and performance through `torch_dtype`.
## Advanced Options Node
The `Searge_AdvOptionsNode` offers a range of configurable parameters allowing for precise control over the text
generation process and model behavior.
## Advanced Configuration Parameters
*The default values on this node are also the defaults that `Searge_LLM_Node`*
*uses when no `Searge_AdvOptionsNode` is connected to it.*
The `LLM_Node` offers a range of configurable parameters allowing for precise control over the text generation process and model behavior. Below is a detailed overview of these parameters:
Below is a detailed overview of these parameters:
- **Temperature (`temperature`):** Controls the randomness in the text generation process. Lower values make the model more confident in its predictions, leading to less variability in output. Higher values increase diversity but can also introduce more randomness. Default: `1.0`.
- **Temperature (`temperature`):** Controls the randomness in the text generation process. Lower values make the model
more confident in its predictions, leading to less variability in output. Higher values increase diversity but can
also introduce more randomness. Default: `1.0`.
- **Top-p (`top_p`):** Also known as nucleus sampling, this parameter controls the cumulative probability distribution
cutoff. The model will only consider the top p% of tokens with the highest probabilities for sampling. Reducing this
value helps in controlling the generation quality by avoiding low-probability tokens. Default: `0.9`.
- **Top-k (`top_k`):** Limits the number of highest probability tokens considered for each step of the generation. A
value of `0` means no limit. This parameter can prevent the model from focusing too narrowly on the top choices,
promoting diversity in the generated text. Default: `50`.
- **Repetition Penalty (`repetition_penalty`):** Adjusts the likelihood of tokens that have already appeared in the
output, discouraging repetition. Values greater than `1` penalize tokens that have been used, making them less likely
to appear again. Default: `1.2`.
- **Top-p (`top_p`):** Also known as nucleus sampling, this parameter controls the cumulative probability distribution cutoff. The model will only consider the top p% of tokens with the highest probabilities for sampling. Reducing this value helps in controlling the generation quality by avoiding low-probability tokens. Default: `0.9`.
- **Top-k (`top_k`):** Limits the number of highest probability tokens considered for each step of the generation. A value of `0` means no limit. This parameter can prevent the model from focusing too narrowly on the top choices, promoting diversity in the generated text. Default: `50`.
- **Repetition Penalty (`repetition_penalty`):** Adjusts the likelihood of tokens that have already appeared in the output, discouraging repetition. Values greater than `1` penalize tokens that have been used, making them less likely to appear again. Default: `1.2`.
- **Trust Remote Code (`trust_remote_code`):** A security parameter that allows or prevents the execution of remote code within loaded models. It is crucial for safely using models from untrusted or unknown sources. Setting this to `True` may introduce security risks. Default: `False`.
- **Torch Data Type (`torch_dtype`):** Specifies the tensor data type for calculations within the model. Options include `"float32"`, `"bfloat16"`, `"float16"`, `"float64"`, or `"auto"` for automatic selection based on device capabilities. Using `"bfloat16"` or `"float16"` can significantly reduce memory usage and increase computation speed on compatible hardware. Default: `"auto"`.
These parameters provide granular control over the text generation capabilities of the `LLM_Node`, allowing users to fine-tune the behavior of the underlying models to best fit their application requirements.
## Quantization Config Node
The Quantization Config Node offers options for model quantization, balancing performance and precision. For configurations and usage, check https://huggingface.co/docs/transformers/main_classes/quantization#transformers.BitsAndBytesConfig
## Output Node
The Output Node is a new addition that formats the generated text to ensure it is presented in a clear and readable manner. It handles the arrangement of text, spacing, and alignment, significantly improving the output's usability for further processing or display.
## Contributing
Contributions to enhance the LLM_Node, add support for more models, or improve functionality are welcome. Please adhere to the project's contribution guidelines when submitting pull requests.
These parameters provide granular control over the text generation capabilities of the `Searge_LLM_Node`, allowing
users to fine-tune the behavior of the underlying models to best fit their application requirements.
## License
The LLM_Node is released under the MIT License. Feel free to use and modify it for your personal or commercial projects.
The Searge_LLM_Node is released under the MIT License. Feel free to use and modify it for your personal or commercial
projects.
## Acknowledgments
- Special thanks to the open-source community and the developers behind the transformers library for providing the foundational tools that make this Node possible.
- Appreciation to the ComfyUI team for their support and contributions to integrating complex NLP functionalities seamlessly.
- Based on the LLM_Node custom extension by Big-Idea-Technology, found
[here on Github](https://github.com/Big-Idea-Technology/ComfyUI_LLM_Node)
Searge_LLM_Node.py
+239
View File
@@ -0,0 +1,239 @@
import importlib
import os
import folder_paths
GLOBAL_MODELS_DIR = os.path.join(folder_paths.models_dir, "llm_gguf")
WEB_DIRECTORY = "./web/assets/js"
DEFAULT_INSTRUCTIONS = 'Generate a prompt from "{prompt}"'
try:
Llama = importlib.import_module("llama_cpp_cuda").Llama
except ImportError:
Llama = importlib.import_module("llama_cpp").Llama
class AnyType(str):
"""A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
def __ne__(self, __value: object) -> bool:
return False
anytype = AnyType("*")
class Searge_LLM_Node:
@classmethod
def INPUT_TYPES(cls):
model_options = []
if os.path.isdir(GLOBAL_MODELS_DIR):
gguf_files = [file for file in os.listdir(GLOBAL_MODELS_DIR) if file.endswith('.gguf')]
model_options.extend(gguf_files)
return {
"required": {
"text": ("STRING", {"multiline": True, "dynamicPrompts": True, "default": ""}),
"random_seed": ("INT", {"default": 1234567890, "min": 0, "max": 0xffffffffffffffff}),
"model": (model_options,),
"max_tokens": ("INT", {"default": 4096, "min": 1, "max": 8192}),
"apply_instructions": ("BOOLEAN", {"default": True}),
"instructions": ("STRING", {"multiline": False, "default": DEFAULT_INSTRUCTIONS}),
},
"optional": {
"adv_options_config": ("SRGADVOPTIONSCONFIG",),
}
}
CATEGORY = "Searge-LLM"
FUNCTION = "main"
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("string",)
def main(self, text, random_seed, model, max_tokens, apply_instructions, instructions, adv_options_config=None):
model_path = os.path.join(GLOBAL_MODELS_DIR, model)
if model.endswith(".gguf"):
generate_kwargs = {'max_tokens': max_tokens, 'temperature': 1.0, 'top_p': 0.9, 'top_k': 50,
'repeat_penalty': 1.2}
if adv_options_config:
for option in ['temperature', 'top_p', 'top_k', 'repeat_penalty']:
if option in adv_options_config:
generate_kwargs[option] = adv_options_config[option]
model_to_use = Llama(
model_path=model_path,
n_gpu_layers=-1,
seed=random_seed,
verbose=False,
n_ctx=2048,
)
if apply_instructions:
req = instructions.replace("{prompt}", text) if "{prompt}" in instructions else f"{instructions} {text}"
messages = [
{"role": "system",
"content": f"You are a helpful assistant."},
{"role": "user",
"content": f"An image generation prompt is a single paragraph summary to describe the subject and "
f"style of an image. It includes a description of the kind of image, the subject of "
f"the image, and some description of the image medium and style in the form of short "
f"keyword.\n\nCreate an image generation prompt for the subject \"a creepy creature "
f"shadow in the dark in a dimly lit tunnel\" in the style \"digital art illustration "
f"with intricate details\"."},
{"role": "assistant",
"content": f"Image Description: A digitally crafted illustration portrays a chilling scene within "
f"a dimly lit, cavernous tunnel. The dominant subject of the image is a mysterious "
f"creature, its form subtly discernible only as a menacing shadow on the walls of the "
f"tunnel. Its elongated silhouette suggests a serpentine creature with sharp "
f"protrusions and an ominous aura. The creature's eyes, a pair of glowing orbs, appear "
f"eerily human-like yet alien. The tunnel is riddled with intricate details that "
f"enhance the eerie atmosphere: dust particles floating lazily in the feeble light, "
f"ancient and crumbling stone, water droplets sparkling on the damp walls, and a "
f"hauntingly beautiful, bioluminescent fungi growing in the shadows. The dimly lit "
f"environment is highlighted by strategically placed light sources that create "
f"dramatic chiaroscuro effects, casting an unsettling and atmospheric glow on the "
f"scene. Digital Art Illustration with Intricate Details (Dark, Atmospheric, "
f"Suspenseful)"},
{"role": "user",
"content": "Now compile the description and style into a single paragraph summary"},
{"role": "assistant",
"content": f"Digital art illustration featuring a dark, atmospheric, and suspenseful scene within "
f"a dimly lit, cavernous tunnel. The subject of the image is a mysterious creature, "
f"depicted only as a menacing shadow on the walls, with elongated silhouette and sharp "
f"protrusions. The creature's eyes, a pair of glowing orbs, appear eerily human-like "
f"yet alien. The tunnel is adorned with intricate details, such as dust particles, "
f"ancient crumbling stone, water droplets, and a hauntingly beautiful bioluminescent "
f"fungi growing in the shadows. Dramatic chiaroscuro effects are created through "
f"strategically placed light sources, casting an unsettling and atmospheric glow on "
f"the scene."},
{"role": "user",
"content": f"create a detailed summary without the title or style"},
{"role": "assistant",
"content": f"A dimly lit, cavernous tunnel is the setting for this chilling digital illustration. "
f"A mysterious creature lurks in the shadows, its elongated silhouette suggestive of a "
f"serpentine creature with sharp protrusions and an ominous aura. The creature's eyes, "
f"a pair of glowing orbs, appear eerily human-like yet alien. The tunnel is riddled "
f"with intricate details that enhance the eerie atmosphere: dust particles floating "
f"lazily in the feeble light, ancient and crumbling stone, water droplets sparkling on "
f"the damp walls, and a hauntingly beautiful, bioluminescent fungi growing in the "
f"shadows. Dramatic chiaroscuro effects are created through strategically placed light "
f"sources, casting an unsettling and atmospheric glow on the scene."},
{"role": "user",
"content": f"Generate a prompt from \"magical garden, sitting on a blue_bench, Disney Princess in "
f"pink_dress, blonde_hair, portrait, Pixar, Disney style, photorealistic, high "
f"details, detailed skin, pores, freckles\""},
{"role": "assistant",
"content": f"In a captivating, Pixar-inspired, Disney-style, photorealistic portrait, a magical "
f"garden unfolds as a breathtaking backdrop. The subject of the image is a Disney "
f"Princess, dressed elegantly in a pink gown with flowing fabric, sitting serenely on "
f"a blue bench. The princess boasts long, blonde hair and a radiant smile. The garden "
f"is meticulously detailed, with vibrant, enchanted flora and fauna, as well as "
f"whimsical details like sparkling fairy lights and a picturesque waterfall. The "
f"princess is highlighted against the lush, detailed foliage, with a focus on the "
f"high-definition details of her porcelain skin, visible freckles, and the intricacies "
f"of her elegant gown. The image is rendered in the captivating, photorealistic style "
f"that exemplifies both the Disney and Pixar brands, capturing the princess's timeless "
f"beauty and the magic of her enchanting surroundings."},
{"role": "user",
"content": req},
]
else:
messages = [
{"role": "system",
"content": f"You are a helpful assistant. Try your best to give the best response possible to "
f"the user."},
{"role": "user",
"content": f"Create a detailed visually descriptive caption of this description, which will be "
f"used as a prompt for a text to image AI system (caption only, no instructions like "
f"\"create an image\").Remove any mention of digital artwork or artwork style. Give "
f"detailed visual descriptions of the character(s), including ethnicity, skin tone, "
f"expression etc. Imagine using keywords for a still for someone who has aphantasia. "
f"Describe the image style, e.g. any photographic or art styles / techniques utilized. "
f"Make sure to fully describe all aspects of the cinematography, with abundant "
f"technical details and visual descriptions. If there is more than one image, combine "
f"the elements and characters from all of the images creatively into a single "
f"cohesive composition with a single background, inventing an interaction between the "
f"characters. Be creative in combining the characters into a single cohesive scene. "
f"Focus on two primary characters (or one) and describe an interesting interaction "
f"between them, such as a hug, a kiss, a fight, giving an object, an emotional "
f"reaction / interaction. If there is more than one background in the images, pick the "
f"most appropriate one. Your output is only the caption itself, no comments or extra "
f"formatting. The caption is in a single long paragraph. If you feel the images are "
f"inappropriate, invent a new scene / characters inspired by these. Additionally, "
f"incorporate a specific movie director's visual style and describe the lighting setup "
f"in detail, including the type, color, and placement of light sources to create the "
f"desired mood and atmosphere. Always frame the scene, including details about the "
f"film grain, color grading, and any artifacts or characteristics specific. "
f"Compress the output to be concise while retaining key visual details. MAX OUTPUT "
f"SIZE no more than 250 characters."
f"\nDescription : {text}"},
]
llm_result = model_to_use.create_chat_completion(messages, **generate_kwargs)
return (llm_result['choices'][0]['message']['content'].strip(),)
else:
return ("NOT A GGUF MODEL",)
class Searge_Output_Node:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"text": (anytype, {}),
}
}
CATEGORY = "Searge-LLM"
FUNCTION = "main"
RETURN_TYPES = ()
OUTPUT_NODE = True
def main(self, text):
return {"ui": {"text": (text,)}}
class Searge_AdvOptionsNode:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"temperature": ("FLOAT", {"default": 1.0, "min": 0.1, "step": 0.1}),
"top_p": ("FLOAT", {"default": 0.9, "min": 0.1, "step": 0.1}),
"top_k": ("INT", {"default": 50, "min": 0}),
"repetition_penalty": ("FLOAT", {"default": 1.2, "min": 0.1, "step": 0.1}),
}
}
CATEGORY = "Searge-LLM"
FUNCTION = "main"
RETURN_TYPES = ("SRGADVOPTIONSCONFIG",)
RETURN_NAMES = ("adv_options_config",)
def main(self, temperature=1.0, top_p=0.9, top_k=50, repetition_penalty=1.2):
options_config = {
"temperature": temperature,
"top_p": top_p,
"top_k": top_k,
"repeat_penalty": repetition_penalty,
}
return (options_config,)
NODE_CLASS_MAPPINGS = {
"Searge_LLM_Node": Searge_LLM_Node,
"Searge_Output_Node": Searge_Output_Node,
"Searge_AdvOptionsNode": Searge_AdvOptionsNode,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"Searge_LLM_Node": "Searge LLM Node",
"Searge_Output_Node": "Searge Output Node",
"Searge_AdvOptionsNode": "Searge Advanced Options Node",
}
__init__.py
+1 -1
View File
@@ -1 +1 @@
from .LLM_Node import *
from .Searge_LLM_Node import *
history/.gitignore
-4
View File
@@ -1,4 +0,0 @@
# Ignore everything in this directory
*
# Except this file
!.gitignore
install.sh
-4
View File
@@ -1,4 +0,0 @@
#!/bin/sh
pip install -r requirements.txt
pip install -i https://pypi.org/simple/ bitsandbytes
pyproject.toml
-15
View File
@@ -1,15 +0,0 @@
[project]
name = "comfyui_llm_node"
description = "The LLM_Node enhances ComfyUI by integrating advanced language model capabilities, enabling a wide range of NLP tasks such as text generation, content summarization, question answering, and more. This flexibility is powered by various transformer model architectures from the transformers library, allowing for the deployment of models like T5, GPT-2, and others based on your project's needs."
version = "1.0.0"
license = { file = "LICENSE" }
dependencies = ["transformers>=4.0.0", "llama-cpp-python", "torch>=1.7.1", "accelerate"]
[project.urls]
Repository = "https://github.com/Big-Idea-Technology/ComfyUI_LLM_Node"
# Used by Comfy Registry https://comfyregistry.org
[tool.comfy]
PublisherId = "nazgut"
DisplayName = "ComfyUI_LLM_Node"
Icon = ""
requirements.txt
+13 -2
View File
@@ -1,4 +1,15 @@
transformers>=4.0.0
llama-cpp-python
torch>=1.7.1
accelerate
accelerate
# llama-cpp-python (CPU only, AVX2)
https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.89+cpuavx2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.89+cpuavx2-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"
https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.89+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.89+cpuavx2-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10"
# llama-cpp-python (CUDA, no tensor cores)
https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.89+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.89+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10"
https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.89+cu121-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.89+cu121-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"
web/assets/js/output.js
+2 -2
View File
@@ -2,9 +2,9 @@ import {app} from "../../scripts/app.js";
import {createTextWidget} from "./utils.js"
app.registerExtension({
name: "LLM_Node.Output_Node",
name: "Searge_LLM_Node.Searge_Output_Node",
async beforeRegisterNodeDef(nodeType, nodeData, app) {
if (nodeData.name === "Output_Node") {
if (nodeData.name === "Searge_Output_Node") {
const onNodeCreated = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function () {
workflow/searge-llm-flux-v1.json
+581
View File
@@ -0,0 +1,581 @@
{
"last_node_id": 28,
"last_link_id": 41,
"nodes": [
{
"id": 3,
"type": "KSampler",
"pos": [
850,
190
],
"size": {
"0": 315,
"1": 262
},
"flags": {
"pinned": true
},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 28
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 40
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 41
},
{
"name": "latent_image",
"type": "LATENT",
"link": 2
}
],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
7
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "KSampler"
},
"widgets_values": [
4815162342,
"fixed",
20,
1,
"ddim",
"ddim_uniform",
1
],
"color": "#233",
"bgcolor": "#355"
},
{
"id": 5,
"type": "EmptyLatentImage",
"pos": [
480,
680
],
"size": {
"0": 320,
"1": 110
},
"flags": {
"pinned": true
},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
2
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "EmptyLatentImage"
},
"widgets_values": [
1216,
832,
1
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 8,
"type": "VAEDecode",
"pos": [
1210,
190
],
"size": {
"0": 210,
"1": 46
},
"flags": {
"pinned": true
},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "samples",
"type": "LATENT",
"link": 7
},
{
"name": "vae",
"type": "VAE",
"link": 12
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
9
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "VAEDecode"
},
"color": "#223",
"bgcolor": "#335"
},
{
"id": 9,
"type": "SaveImage",
"pos": [
1210,
500
],
"size": {
"0": 450,
"1": 360
},
"flags": {
"pinned": true
},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 9
}
],
"properties": {},
"widgets_values": [
"Flux/Image"
],
"color": "#222",
"bgcolor": "#000"
},
{
"id": 10,
"type": "VAELoader",
"pos": [
850,
510
],
"size": {
"0": 315,
"1": 58
},
"flags": {
"pinned": true
},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "VAE",
"type": "VAE",
"links": [
12
],
"slot_index": 0,
"shape": 3
}
],
"properties": {
"Node name for S&R": "VAELoader"
},
"widgets_values": [
"flux-vae.safetensors"
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 19,
"type": "ModelSamplingFlux",
"pos": [
480,
40
],
"size": {
"0": 320,
"1": 130
},
"flags": {
"pinned": true
},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 36
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
28
],
"slot_index": 0,
"shape": 3
}
],
"properties": {
"Node name for S&R": "ModelSamplingFlux"
},
"widgets_values": [
1.15,
0.5,
1216,
832
],
"color": "#2a363b",
"bgcolor": "#3f5159"
},
{
"id": 22,
"type": "Searge_Output_Node",
"pos": [
370,
360
],
"size": {
"0": 430,
"1": 270
},
"flags": {
"pinned": true
},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "text",
"type": "*",
"link": 33
}
],
"properties": {
"Node name for S&R": "Searge_Output_Node"
},
"widgets_values": [
""
]
},
{
"id": 25,
"type": "Searge_LLM_Node",
"pos": [
10,
360
],
"size": {
"0": 340,
"1": 510
},
"flags": {
"pinned": true
},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "adv_options_config",
"type": "SRGADVOPTIONSCONFIG",
"link": null
}
],
"outputs": [
{
"name": "string",
"type": "STRING",
"links": [
33,
38,
39
],
"slot_index": 0,
"shape": 3
}
],
"properties": {
"Node name for S&R": "Searge_LLM_Node"
},
"widgets_values": [
"A psychedelic animal frozen in a frosted jar on a wooden table, sci-fi, wide angle",
1337,
"Mistral-7B-Instruct-v0.3.Q4_K_M.gguf",
4096,
true,
"Generate a prompt from \"{prompt}\""
],
"color": "#232",
"bgcolor": "#353"
},
{
"id": 26,
"type": "DualCLIPLoader",
"pos": [
10,
210
],
"size": {
"0": 340,
"1": 110
},
"flags": {
"pinned": true
},
"order": 2,
"mode": 0,
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
37
],
"slot_index": 0,
"shape": 3
}
],
"properties": {
"Node name for S&R": "DualCLIPLoader"
},
"widgets_values": [
"t5xxl_fp8_e4m3fn.safetensors",
"clip_l.safetensors",
"flux"
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 27,
"type": "UNETLoader",
"pos": [
10,
40
],
"size": {
"0": 315,
"1": 82
},
"flags": {
"pinned": true
},
"order": 3,
"mode": 0,
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
36
],
"slot_index": 0,
"shape": 3
}
],
"properties": {
"Node name for S&R": "UNETLoader"
},
"widgets_values": [
"flux1-dev-fp8.safetensors",
"fp8_e4m3fn"
],
"color": "#2a363b",
"bgcolor": "#3f5159"
},
{
"id": 28,
"type": "CLIPTextEncodeFlux",
"pos": [
370,
210
],
"size": [
430,
110
],
"flags": {
"pinned": true
},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 37
},
{
"name": "clip_l",
"type": "STRING",
"link": 38,
"widget": {
"name": "clip_l"
}
},
{
"name": "t5xxl",
"type": "STRING",
"link": 39,
"widget": {
"name": "t5xxl"
}
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
40,
41
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncodeFlux"
},
"widgets_values": [
"",
"",
3.5
],
"color": "#432",
"bgcolor": "#653"
}
],
"links": [
[
2,
5,
0,
3,
3,
"LATENT"
],
[
7,
3,
0,
8,
0,
"LATENT"
],
[
9,
8,
0,
9,
0,
"IMAGE"
],
[
12,
10,
0,
8,
1,
"VAE"
],
[
28,
19,
0,
3,
0,
"MODEL"
],
[
33,
25,
0,
22,
0,
"*"
],
[
36,
27,
0,
19,
0,
"MODEL"
],
[
37,
26,
0,
28,
0,
"CLIP"
],
[
38,
25,
0,
28,
1,
"STRING"
],
[
39,
25,
0,
28,
2,
"STRING"
],
[
40,
28,
0,
3,
1,
"CONDITIONING"
],
[
41,
28,
0,
3,
2,
"CONDITIONING"
]
],
"groups": [],
"config": {},
"extra": {},
"version": 0.4
}