Initial commit

.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

.gitignore

@@ -0,0 +1,181 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor  
+#  Cursor is an AI-powered code editor.`.cursorignore` specifies files/directories to 
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 claptrap0
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LLM_Hub.py

@@ -0,0 +1,446 @@
+import os
+import torch
+import random
+import importlib
+import folder_paths
+
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig, BitsAndBytesConfig, StoppingCriteria, StoppingCriteriaList, set_seed
+
+GLOBAL_MODELS_DIR = os.path.join(folder_paths.models_dir, "text_encoders", "LLMs")
+
+WEB_DIRECTORY = "./web/js"
+
+try:
+    Llama = importlib.import_module("llama_cpp_cuda").Llama
+except ImportError:
+    Llama = importlib.import_module("llama_cpp").Llama
+
+
+try:
+    from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer
+except ImportError:
+    print("Transformers library not found. Please install it to use safetensors models: pip install transformers torch sentencepiece")
+    AutoModelForCausalLM = None
+    AutoTokenizer = None
+    pipeline = None
+
+
+_loaded_model = None
+_loaded_tokenizer = None
+_current_model_id = None
+_model_is_on_gpu = False
+
+def free_gpu_memory():
+    global _loaded_model, _loaded_tokenizer, _current_model_id, _model_is_on_gpu
+    if _loaded_model is not None:
+        del _loaded_model
+        _loaded_model = None
+    if _loaded_tokenizer is not None:
+        del _loaded_tokenizer
+        _loaded_tokenizer = None
+    _current_model_id = None
+    _model_is_on_gpu = False
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+    print("GPU memory freed.")
+
+
+class AnyType(str):
+
+    def __ne__(self, __value: object) -> bool:
+        return False
+
+
+anytype = AnyType("*")
+
+
+class LLM_Hub:
+    @classmethod
+    def INPUT_TYPES(cls):
+        model_options = []
+        if os.path.isdir(GLOBAL_MODELS_DIR):
+            listed_items = set()
+            for root, dirs, files in os.walk(GLOBAL_MODELS_DIR):
+
+                for file in files:
+                    if file.endswith('.gguf'):
+                        relative_path_to_file = os.path.relpath(os.path.join(root, file), GLOBAL_MODELS_DIR)
+                        if os.path.dirname(relative_path_to_file) == ".":
+                            display_name = os.path.splitext(relative_file_path)[0]
+                        else:
+                            clean_name = os.path.relpath(root, GLOBAL_MODELS_DIR) 
+                        if clean_name not in listed_items:
+                            model_options.append(clean_name)
+                            listed_items.add(clean_name)
+
+                if root == GLOBAL_MODELS_DIR:
+                    for d in dirs:
+                        current_dir_path = os.path.join(root, d)
+
+                        if any(f.endswith(('.json', '.safetensors', '.bin')) for f in os.listdir(current_dir_path)):
+                            clean_name = os.path.relpath(current_dir_path, GLOBAL_MODELS_DIR)
+                            if clean_name not in listed_items:
+                                model_options.append(clean_name)
+                                listed_items.add(clean_name)
+
+        if not model_options:
+            model_options = ["No models found in text_encoders/LLMs/"] 
+
+        return {
+            "required": {
+                "text": ("STRING", {"multiline": True, "dynamicPrompts": True, "default": ""}),
+                "seed": ("INT", {"default": 1234567890, "min": 0, "max": 0xffffffffffffffff}),
+                "model": (model_options, {"tooltip": "Select an LLM. Store your models in text_encoders/LLMs/model_name for clarity."}),
+                "max_tokens": ("INT", {"default": 512, "min": 1, "max": 8192, "tooltip": "The maximum number of tokens to generate. Higher values mean longer responses while also increasing generation time and VRAM usage."}),
+                "context_window": ("INT", {"default": 2048, "min": 256, "max": 8192, "step": 128, "tooltip": "The total context window size (input + generated response). Lower values reduce VRAM."}),
+                "apply_system_prompt": ("BOOLEAN", {"default": False, "tooltip": "When enabled your system prompt in 'system_prompt' will be used. When disabled there is a basic fallback system prompt."}),
+				"load_in_8bit": ("BOOLEAN", {"default": False, "label_on": "8bit", "label_off": "Full Precision", "tooltip": "When enabled to (8bit), HuggingFace models will load in 8-bit to reduce VRAM usage. GGUF models are not affected, as they have their own quantization."}),
+                "stay_on_gpu": ("BOOLEAN", {"default": False, "tooltip": "If enabled, the model will remain on the GPU until you restart ComfyUI or you disable this option. If disabled, the model will offload from the GPU after each run."}), 
+                "system_prompt": ("STRING", {"multiline": True, "default": "You are a prompt engineer for a text-to-image AI system based on \"{prompt}\" (caption only, no instructions like \"create an image\")."}),
+            },
+            "optional": {
+                "settings": ("SETTINGS_FOR_LLM", {"tooltip": "Control settings for the LLM."}),
+            }
+        }
+
+    CATEGORY = "LLM_Hub"
+    FUNCTION = "main"
+    RETURN_TYPES = ("STRING", "STRING",)
+    RETURN_NAMES = ("generated", "original",)
+
+
+    def main(self, text, seed, model, max_tokens, context_window, apply_system_prompt, load_in_8bit, stay_on_gpu, system_prompt, settings=None):
+        model_full_path = os.path.join(GLOBAL_MODELS_DIR, model)
+        if os.path.exists(model_full_path + ".gguf"):
+            model_full_path += ".gguf"
+        elif os.path.exists(model_full_path + ".safetensors"):
+            model_full_path += ".safetensors"
+
+
+        generate_kwargs = {
+        'max_tokens': max_tokens, 
+        'temperature': 0.8,
+        'top_p': 0.8, 
+        'top_k': 10,
+        'repeat_penalty': 1.3, 
+        "stop": ["\n\n", "\n<|", "###", "<|im_end|>", "<|endoftext|>", "<|language_code|>en-US</languagetype>=eng", "Here is the caption:",]
+        }
+
+        if settings:
+            for option in ['temperature', 'top_p', 'top_k', 'repeat_penalty']:
+                if option in settings:
+                    generate_kwargs[option] = settings[option]
+
+		
+        if apply_system_prompt:
+            messages_for_llm = [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": f"Description : {text}"},
+            ]
+        else:
+            messages_for_llm = [
+                {"role": "system", "content": f"You are a prompt engineer for a text-to-image AI system (caption only, no instructions like \"create an image\"). Do not include any meta-commentary or length indicators (e.g., 'approximately X words/characters'). Your output must be a single, detailed, yet concise caption, **aiming for approximately 400 to 500 characters in total**. Prioritize rich visual descriptions. Respond EXCLUSIVELY in English. Your task is to create detailed, visually descriptive captions based on user descriptions. Your output must be only the caption itself, no instructions or extra formatting. Give detailed visual descriptions of characters (ethnicity, skin tone, expression, hair and eye color etc.). Imagine using keywords for someone with aphantasia. Describe the image style, photographic or art techniques utilized. Fully describe all aspects of the cinematography, with abundant technical details and visual descriptions. Incorporate a specific visual style. Describe the lighting setup in detail (type, color, placement). Always frame the scene, including details about color grading, camera placement."},
+                {"role": "user", "content": f"Description : {text}"},
+            ]
+		
+        global _loaded_model, _loaded_tokenizer, _current_model_id, _model_is_on_gpu
+        actual_model_path = None
+        model_type = None 
+        potential_path = os.path.join(GLOBAL_MODELS_DIR, model)
+
+
+        if os.path.isfile(potential_path + '.gguf'):
+            actual_model_path = potential_path + '.gguf'
+            model_type = 'gguf'
+        elif os.path.isdir(potential_path):
+            gguf_files_in_dir = [f for f in os.listdir(potential_path) if f.endswith('.gguf')]
+            if gguf_files_in_dir:
+                actual_model_path = os.path.join(potential_path, gguf_files_in_dir[0])
+                model_type = 'gguf'
+            elif any(f.endswith(('.safetensors', '.bin', '.json')) for f in os.listdir(potential_path)):
+                actual_model_path = potential_path
+                model_type = 'hf_dir'
+        
+        if not actual_model_path or not model_type:
+            return (f"Error: {model} not found or unsupported format. Please check the model file name and directory structure under {GLOBAL_MODELS_DIR}.", text)
+
+
+        current_model_key = f"{actual_model_path}_offload:{stay_on_gpu}"
+
+        should_reload = False
+        if _loaded_model is None or _current_model_id != current_model_key:
+            should_reload = True
+            if _loaded_model is not None:
+                print("Model or GPU setting changed, freeing existing model.")
+                free_gpu_memory()
+        elif _model_is_on_gpu and stay_on_gpu:
+
+             pass
+        elif not _model_is_on_gpu and not stay_on_gpu:
+
+             pass
+
+
+        if not _loaded_model or _current_model_id != current_model_key:
+            print(f"Loading {model}...")
+
+            try:
+                if model_type == 'gguf':
+                    if not Llama:
+                        return ("Error: llama_cpp or llama_cpp_cuda not imported. Cannot load GGUF model.", text)
+                    
+                    n_gpu_layers_val = 0 
+                    if torch.cuda.is_available():
+                        n_gpu_layers_val = -1
+                    else:
+                        print("⚠️ Warning: No GPU available. GGUF model will run on CPU.")
+
+                    model_to_use = Llama(
+                        model_path=actual_model_path,
+                        n_gpu_layers=n_gpu_layers_val,
+                        seed=int(seed % (2**32 - 1)),
+                        verbose=False,
+                        n_ctx=context_window,
+                    )
+                    _loaded_model = model_to_use
+                    _model_is_on_gpu = (n_gpu_layers_val != 0)
+                    _current_model_id = current_model_key
+
+                elif model_type == 'hf_dir':
+                    if not (AutoModelForCausalLM and AutoTokenizer and pipeline):
+                        return ("Error: Transformers library not found. Cannot load safetensors model. Please install it: pip install transformers torch sentencepiece", text)
+                    
+                    hf_model_dir = actual_model_path
+                    tokenizer = AutoTokenizer.from_pretrained(hf_model_dir)
+                    device_map_setting = "auto" if torch.cuda.is_available() else "cpu"
+                    
+                    if torch.cuda.is_available() and load_in_8bit:
+                        quantization_status = "8bit"
+                        print(f"Loading {model} on GPU with {quantization_status}")
+                        quant_config = BitsAndBytesConfig(load_in_8bit=True)
+                        model_hf = AutoModelForCausalLM.from_pretrained(
+                            hf_model_dir,
+                            torch_dtype="auto",
+                            device_map=device_map_setting,
+                            quantization_config=quant_config
+                        )
+                        _model_is_on_gpu = True
+                    elif torch.cuda.is_available() and not load_in_8bit:
+                        model_hf = AutoModelForCausalLM.from_pretrained(
+                            hf_model_dir,
+                            torch_dtype="auto",
+                            device_map=device_map_setting
+                        )
+                        _model_is_on_gpu = True
+                        actual_dtype_str = str(model_hf.dtype).replace('torch.', '')
+                        print(f"Loading {model} on GPU in {actual_dtype_str} precision")
+                    else:
+                        print(f"Loading {model} on CPU (no GPU available)")
+                        model_hf = AutoModelForCausalLM.from_pretrained(hf_model_dir, torch_dtype=torch.float32, device_map="cpu")
+                        _model_is_on_gpu = False
+
+                    _loaded_model = model_hf
+                    _loaded_tokenizer = tokenizer
+                    _current_model_id = current_model_key
+                   
+                device_str = "GPU" if _model_is_on_gpu else "CPU"
+                precision_display_part = "" 
+
+                if model_type == 'hf_dir':
+                    if _model_is_on_gpu:
+                        if load_in_8bit:
+                            precision_display_part = "8bit"
+                        else:
+                            precision_display_part = str(model_hf.dtype).replace('torch.', '')
+                    else:
+                        precision_display_part = str(model_hf.dtype).replace('torch.', '')
+
+                    if precision_display_part:
+                        print("\n---------------------------------")
+                        print(f"Loaded successfully on {device_str} in {precision_display_part}")
+                        print("\n---------------------------------")
+                    else:
+                        print("\n---------------------------------")
+                        print(f"Loaded successfully on {device_str}")
+                        print("\n---------------------------------")
+                else:
+                    print("\n---------------------------------")
+                    print(f"Loaded successfully on {device_str}")
+                    print("\n---------------------------------")
+
+
+            except Exception as e:
+                free_gpu_memory()
+                return (f"Error loading {model} from {actual_model_path}: {e}", text)
+
+            
+        llm_result_content = ""
+        try:
+            capped_seed = seed % (2**32 - 1)
+            set_seed(capped_seed)
+
+            if model_type == 'gguf':
+                llm_result = _loaded_model.create_chat_completion(messages_for_llm, **generate_kwargs)
+                llm_result_content = llm_result['choices'][0]['message']['content'].strip()
+            
+            elif model_type == 'hf_dir': 
+
+                if _model_is_on_gpu: 
+                    generator = pipeline(
+                        "text-generation",
+                        model=_loaded_model,
+                        tokenizer=_loaded_tokenizer,
+                    )
+                else:
+                    generator = pipeline(
+                        "text-generation",
+                        model=_loaded_model,
+                        tokenizer=_loaded_tokenizer,
+                    )
+
+
+                if hasattr(_loaded_tokenizer, 'apply_chat_template') and _loaded_tokenizer.chat_template:
+                    chat_prompt = _loaded_tokenizer.apply_chat_template(messages_for_llm, tokenize=False, add_generation_prompt=True)
+
+                else:
+                    chat_prompt = ""
+                    for message in messages_for_llm:
+                        if message["role"] == "system":
+                            chat_prompt += f"<|system|>{message['content']}\n"
+                        elif message["role"] == "user":
+                            chat_prompt += f"<|user|>{message['content']}\n"
+                        elif message["role"] == "assistant":
+                            chat_prompt += f"<|assistant|>{message['content']}\n"
+                    chat_prompt += "<|assistant|>"
+
+    
+                stop_list = generate_kwargs.get('stop', [])
+                if stop_list:
+                    class StopGenerationCriteria(StoppingCriteria):
+                        def __init__(self, stops=[], encounters=1):
+                            super().__init__()
+                            global _loaded_tokenizer
+                            self.stops = [torch.tensor(_loaded_tokenizer.encode(stop_token, add_special_tokens=False), dtype=torch.long) for stop_token in stops]
+                            self.encounters = encounters
+
+                        def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor):
+                            for stop in self.stops:
+                                if len(input_ids[0]) >= len(stop) and torch.all(stop.to(input_ids.device) == input_ids[0][-len(stop):]).item():
+                                    self.encounters -= 1
+                                    if self.encounters == 0:
+                                        return True
+                            return False
+                    
+                    stop_criteria_list = StoppingCriteriaList([StopGenerationCriteria(stops=stop_list)])
+                else:
+                    stop_criteria_list = None
+                
+                
+                hf_generate_kwargs = {
+                    'max_new_tokens': generate_kwargs.get('max_tokens', 512),
+                    'temperature': generate_kwargs.get('temperature', 0.8),
+                    'top_p': generate_kwargs.get('top_p', 0.8),
+                    'top_k': generate_kwargs.get('top_k', 10),
+                    'repetition_penalty': generate_kwargs.get('repeat_penalty', 1.3),
+                    'do_sample': True if generate_kwargs.get('temperature', 1.0) > 0 else False,
+                    'stopping_criteria': stop_criteria_list,
+                    'pad_token_id': _loaded_tokenizer.eos_token_id,
+                    'eos_token_id': _loaded_tokenizer.eos_token_id,
+                    
+                }
+                
+                result = generator(chat_prompt, **hf_generate_kwargs)
+                generated_text = result[0]['generated_text']
+                
+                if generated_text.startswith(chat_prompt):
+                    llm_result_content = generated_text[len(chat_prompt):].strip()
+                else:
+                    llm_result_content = generated_text.strip()
+    
+            
+        except Exception as e:
+            free_gpu_memory()
+            return (f"Error during inference with '{model}': {e}", text)
+
+
+        if not stay_on_gpu and _model_is_on_gpu:
+            print(f"Offloading {model} from GPU after run as stay on GPU is disabled.")
+            free_gpu_memory()
+        elif stay_on_gpu and _model_is_on_gpu:
+            print(f"{model} remains on GPU.")
+        elif not _model_is_on_gpu:
+            print(f"{model} remains on CPU.")
+            
+
+        return (llm_result_content, text)
+
+class Generated_Output:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "text": (anytype, {}),
+            },
+            "hidden": {
+                "unique_id": "UNIQUE_ID", "extra_pnginfo": "EXTRA_PNGINFO",
+            },
+        }
+
+    CATEGORY = "LLM_Hub"
+    FUNCTION = "main"
+    RETURN_TYPES = ()
+    RETURN_NAMES = ()
+    OUTPUT_NODE = True
+    
+    def main(self, text, unique_id=None, extra_pnginfo=None):
+        if unique_id is not None and extra_pnginfo is not None and len(extra_pnginfo) > 0:
+            workflow = None
+            if "workflow" in extra_pnginfo:
+                workflow = extra_pnginfo["workflow"]
+            node = None
+            if workflow and "nodes" in workflow:
+                node = next((x for x in workflow["nodes"] if str(x["id"]) == unique_id), None)
+            if node:
+                node["widgets_values"] = [str(text)]
+        return {"ui": {"text": (str(text),)}}
+
+
+class LLM_Settings:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "temperature": ("FLOAT", {"default": 0.8, "min": 0.1, "step": 0.01, "tooltip": "How creative or crazy the text is. High means wild text, low means boring text."}),
+                "top_p": ("FLOAT", {"default": 0.8, "min": 0.1, "step": 0.01, "tooltip": "Picks words that are good enough. High means more choices, low means very few choices."}),
+                "top_k": ("INT", {"default": 10, "min": 0, "tooltip": "Picks from the best X words. High X means many choices, low X means few choices. X = value"}),
+                "repetition_penalty": ("FLOAT", {"default": 1.3, "min": 0.1, "step": 0.01, "tooltip": "Stops the text from saying the same thing too many times. High means less repeating."}),
+            }
+        }
+
+    CATEGORY = "LLM_Hub"
+    FUNCTION = "main"
+    RETURN_TYPES = ("SETTINGS_FOR_LLM",)
+    RETURN_NAMES = ("settings",)
+
+    def main(self, temperature=0.8, top_p=0.8, top_k=10, repetition_penalty=1.3):
+        options_config = {
+            "temperature": temperature,
+            "top_p": top_p,
+            "top_k": top_k,
+            "repetition_penalty": repetition_penalty,
+        }
+
+        return (options_config,)
+
+
+NODE_CLASS_MAPPINGS = {
+    "LLM_Hub": LLM_Hub,
+    "LLM_Settings": LLM_Settings,
+    "Generated_Output": Generated_Output,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "LLM_Hub": "LLM Hub",
+    "LLM_Settings": "LLM Settings",
+    "Generated_Output": "Generated Output",
+}

README.md

@@ -0,0 +1,103 @@
+## 🧠 LLM Hub
+
+Utilize the power of an LLM into ComfyUI to transform your text-to-image and text-to-video ideas into highly detailed prompts for generation while giving you full control. 
+
+## 🤖 Usage
+
+<img src="img/usage.png" alt="LLM Hub for ComfyUI">
+
+## 📝 Requirements
+
+- Create a directory named **`LLMs`** inside **`ComfyUI/models/text_encoders/`**
+- Create a another new directory for each LLM with the model name inside **`LLMs`**.
+- Place your LLM models in their respective directory.
+- Every .safetensors model needs the .json files and the model should be named **`model.safetensors`**
+
+How your directory structure should look like:
+```
+ComfyUI/
+└── models/
+    └── text_encoders/
+        └── LLMs/
+            └── GGUF_model/
+                └── model.gguf
+            └── safetensors_model/
+                └── model.safetensors
+                └── config.json
+                └── tokenizer.json
+                └── tokenizer_config.json
+                └── generation_config.json (Optional)
+                └── special_tokens_map.json (Optional)
+```
+
+GGUF models don't need to be named "model.gguf".
+
+## 🛠️ Installation
+
+- Run the following commands:
+
+For activating venv:
+```
+cd path/to/your/ComfyUI
+```
+Linux:
+```
+source venv/bin/activate
+```
+
+Windows:
+```
+.venv\Scripts\activate
+```
+
+Go to your custom nodes folder:
+```
+cd custom_nodes
+```
+
+Afterwards clone the repo:
+```
+git clone https://github.com/X/ComfyUI_LLM_Hub
+```
+
+Go inside the node folder:
+```
+cd ComfyUI_LLM_Hub
+```
+
+If you are using a venv:
+```
+pip install -r --verbose requirements.txt
+```
+- You can delete **`"--verbose"`** if you don't want to see the process of the compiling.
+
+For the portable version of ComfyUI:
+```
+C:/path/to/your/ComfyUI_portable/python_embedded/Scripts/pip.exe install -r requirements.txt
+```
+
+## ⚙️ LLM Settings 
+
+It offers a range of configurable parameters allowing for precise control over the text generation process and model behavior.
+
+The values on this node are also the defaults that `LLM Hub` uses when `LLM Settings` isn't connected.
+
+Breif overview of the parameters:
+
+- **`temperature:`** Controls the randomness of the output. Lower values (**e.g., 0.2-0.5**): Make the output more focused, predictable, and deterministic. 
+  Ideal for tasks requiring factual, precise, or consistent responses. Higher values (**e.g., 0.7-1.0**): Increase creativity and randomness, allowing the model to take more risks. Can lead to more imaginative but potentially less coherent results.
+  `Default: 0.8`
+- **`top_p:`** Filters the set of possible next tokens by cumulative probability. The model only considers tokens whose cumulative probability sum up to `top_p`.Lower values (**e.g., 0.5**): Narrows the selection to only the most probable tokens, improving coherence and reducing the chance of irrelevant words.
+  Higher values (**e.g., 0.9**): Allows for a broader range of tokens, increasing randomness. `Default: 0.8`
+- **`top_k:`** Limits the sampling pool to the k most probable next tokens. Lower values (**e.g., 5-20**): Focuses generation on the top choices, making output more predictable.
+  Higher values (**e.g., 50-100**): Expands the options, outputing more creative text. `Default: 10`
+- **`repetition_penalty:`** Discourages the model from repeating words or phrases that have already appeared in the generated text. Values greater than 1 (**e.g., 1.1-1.5**): Penalize repeated tokens, making them less likely to be chosen again. Essential for preventing repetitive loops or boilerplate text.
+  A value of 1: No penalty applied. `Default: 1.3`
+
+## 📚 Resources
+
+- <a href="https://rumn.medium.com/setting-top-k-top-p-and-temperature-in-llms-3da3a8f74832" target="_SEJ" rel="noreferrer">Parameter settings</a>
+- <a href="https://github.com/0xeb/TheBigPromptLibrary" target="_SEJ" rel="noreferrer">System prompts, instructions, etc.</a>
+
+## 📄 License
+Released under the MIT License. Feel free to use and modify it for your projects, commercial or personal.

__init__.py

@@ -0,0 +1 @@
+from .LLM_Hub import *

requirements.txt

@@ -0,0 +1,5 @@
+transformers
+torch
+accelerate
+bitsandbytes
+llama-cpp-python --config-settings=cmake.args="-DGGML_CUDA=on"

web/js/output.js

@@ -0,0 +1,26 @@
+import {app} from "../../scripts/app.js";
+import {createTextWidget} from "./utils.js"
+
+app.registerExtension({
+    name: "LLM_Hub.Generated_Output",
+    async beforeRegisterNodeDef(nodeType, nodeData, app) {
+        if (nodeData.name === "Generated_Output") {
+            const onNodeCreated = nodeType.prototype.onNodeCreated;
+
+            nodeType.prototype.onNodeCreated = function () {
+                const result = onNodeCreated?.apply(this, arguments);
+                const text = createTextWidget(app, this, "text");
+                const nodeWidth = this.size[0];
+                const nodeHeight = this.size[1];
+                this.setSize([nodeWidth, nodeHeight * 3]);
+                return result;
+            };
+
+            const onExecuted = nodeType.prototype.onExecuted;
+            nodeType.prototype.onExecuted = function (message) {
+                onExecuted?.apply(this, arguments);
+                this.widgets.find(obj => obj.name === "text").value = message.text;
+            };
+        }
+    },
+});

web/js/utils.js

@@ -0,0 +1,8 @@
+import {ComfyWidgets} from "../../scripts/widgets.js";
+
+export function createTextWidget(app, node, widgetName, styles = {}) {
+    const widget = ComfyWidgets["STRING"](node, widgetName, ["STRING", {multiline: true}], app).widget;
+    widget.inputEl.readOnly = true;
+    Object.assign(widget.inputEl.style, styles);
+    return widget;
+}