Added new nodes for automatic detailed processing of tags and words. Added description for nodes.

node_list.json

@@ -0,0 +1,65 @@
+{
+    "Convert String To Int [LP]":"Convert String To Int",
+    "Convert String To Float [LP]":"Convert String To Float",
+    "Convert String To Bool [LP]":"Convert String To Bool",
+    "Convert String To Number [LP]":"Convert String To Number",
+    "Convert String To Combo [LP]":"Convert String To Combo",
+    "Convert Int To String [LP]":"Convert Int To String",
+    "Convert Float To String [LP]":"Convert Float To String",
+    "Convert Bool To String [LP]":"Convert Bool To String",
+    "Convert Float To Int [LP]":"Convert Float To Int",
+    "Convert Int To Float [LP]":"Convert Int To Float",
+    "Convert Int To Bool [LP]":"Convert Int To Bool",
+    "Convert Bool To Int [LP]":"Convert Bool To Int",
+    "Image Overlay [LP]":"Image Overlay",
+    "Fast Checker Pattern [LP]":"Fast Checker Pattern",
+    "Image Remove Background [LP]":"Removes background for images with user-defined model",
+    "File Counter [LP]":"Counts the number of files in the specified folder",
+    "Image Loader From Path [LP]":"Loading multiple images from specific folder",
+    "Load Image [LP]":"Load Image by filepath",
+    "Preview Image Bridge [LP]":"Preview Image Bridge is needed for conditions, cycles and other cases when it is necessary to output an image with an output connection. It has an output pin, unlike the usual Preview Image.",
+    "Load LoRA Tag [LP]":"Loads lores by text tags from a text field. An implementation that exactly copies the behavior of Automatic1111 or ForgeWebUI.",
+    "Simple Float Slider [LP]":"A simple slider that has no precision or upper/lower values settings. From 0.00000 to 1.00000",
+    "Simple Float Slider - Tenths Step [LP]":"A simple slider that has no precision or upper/lower values settings. From 0.0 to 1.0",
+    "Simple Float Slider - Hundredths Step [LP]":"A simple slider that has no precision or upper/lower values settings. From 0.00 to 1.00",
+    "Text [LP]":"Input fields for multiline text",
+    "String [LP]":"Input fields for single-line text (string)",
+    "Find Value From File [LP]":"Finding a value by key from a text file",
+    "Show Text [LP]":"Show Text node",
+    "Show Text Bridge [LP]":"Show Text node for conditions and cycles",
+    "LLM Loader [LP]":"Loads a large language model specified by the user. Models are stored in the LLavacheckpoints folder.",
+    "LLM Sampler [LP]":"Generates text using the LLM. Simple settings.",
+    "LLM Advanced [LP]":"Generates text using the LLM. Advanced settings.",
+    "Tag Category [LP]":"Defines categories for tags that are passed as input. By default, a list with categories placed on custom_nodes/ComfyUI-LevelPixel/nodes/tags/tag_category.json",
+    "Tag Category Filter [LP]":"Filters tags to exclude or keep only those that match the specified categories.",
+    "Tag Category Keeper [LP]":"Filters tags to keep only those that match the specified categories.",
+    "Tag Category Remover [LP]":"Filters tags to exclude only those that match the specified categories.",
+    "Tag Switcher [LP]":"Tag Switcher",
+    "Tag Merger [LP]":"Tag Merger",
+    "Tag Replace [LP]":"Tag Replace",
+    "Tag Remover [LP]":"Removes the specified tags from the text",
+    "Resorting Tags [LP]":"Merge and re-sort input arrays of tags using a rating system, producing sorted and combined text (tags). The node is needed to create the most relevant tags by combining several tag variants from different neural models (from LLM, VLM, and others) and selecting the best and most frequently occurring ones. Use with the Text To List node. Input tags from priority_texts have a higher rating and are all used in the output set. Input tags from inclusive_texts have a medium rating and are all used in the output set. Input tags from auxiliary_texts have a lower rating, and tags that are not in priority_texts and inclusive_texts will not be added to the output tag set.",
+    "Remove Duplicate Tags [LP]":"Removes duplicate tags, leaving only one tag in the text.",
+    "Keep Only English Tags [LP]":"Removes all tags with symbols that do not correspond to English characters. The entire tag is removed (from comma to comma), regardless of whether it contains English characters along with other characters. The node is needed to exclude any words and characters from tags except English.",
+    "Remove Banned Tags From Tags [LP]":"Removes from the text those tags that are prohibited for use by the user. By default, a list with names of people and brands protected by copyright is used at custom_nodes/ComfyUI-LevelPixel/nodes/tags/example_banned_tags.txt.",
+    "Remove Banned Tags From Text [LP]":"Removes from the tag list those tags that are prohibited for use by the user. By default, a list with names of people and brands protected by copyright is used at custom_nodes/ComfyUI-LevelPixel/nodes/tags/example_banned_tags.txt.",
+    "Text Choice Parser [LP]":"Randomly (or by the specified number) selects the text that is specified in curly brackets with the separator |. You can also set variables in the text using square brackets and replace them in the second field using the syntax variable=value. Convenient for mass generation and controlled substitution of text in the general prompt.",
+    "CLIP Text Encode Translate [LP]":"Tranlsate text from any languages to english",
+    "Text Translate [LP]":"Tranlsate text from any languages to english",
+    "Text To List [LP]":"Convert many texts to list of texts",
+    "Split Compound Text [LP]":"Split Compound Text",
+    "Keep Only English Words [LP]":"Removes all words with symbols that do not correspond to English symbols. The entire word is removed (from space to space), regardless of whether it contains English symbols along with other symbols. The node is necessary for excluding any words and symbols from the text except English.",
+    "Model Unloader [LP]":"Unloads all neural models that are in the model manager from VRAM. Does not unload models that are loaded into VRAM bypassing the model manager.",
+    "Soft Model Unloader [LP]":"Unloads from VRAM those files and cache that are not currently in use. Does not unload models.",
+    "Hard Model Unloader [LP]":"Unloads all neural models that are in the model manager from VRAM. Does not unload models that are loaded into VRAM bypassing the model manager.",
+    "Override CLIP Device [LP]":"Changes the main computing processor and the RAM used for CLIP. You can specify any video card or processor that is available in the system.",
+    "Override VAE Device [LP]":"Changes the main computing processor and the RAM used for VAE. You can specify any video card or processor that is available in the system.",
+    "Delay [LP]":"Delay",
+    "Autotagger [LP]":"Autotagger based on the latest version of the WD tagging model from SmilingWolf.",
+    "LLava Loader [LP]":"Loads a visual large language model specified by the user. Models are stored in the LLavacheckpoints folder.",
+    "LLava Clip Loader [LP]":"Loads a CLIP model specified by the user for VLM model. Models are stored in the LLavacheckpoints folder.",
+    "LLava Sampler Simple [LP]":"Generates text using the VLM. Simple settings.",
+    "LLava Sampler Advanced [LP]":"Generates text using the VLM. Advanced settings.",
+    "LLava Simple [LP]":"Generates text using the loaded VLM from LLava Loader. Simple settings.",
+    "LLava Advanced [LP]":"Generates text using the loaded VLM from LLava Loader. Advanced settings."
+}

nodes/tags/tags_utils_LP.py

@@ -1,7 +1,10 @@
 import os
 import json
+import string
+import re
 
 tag_category = json.load(open(os.path.join(os.path.dirname(os.path.realpath(__file__)),"tag_category.json")))
+banned_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),"banned_tags.txt")
 
 def category_for_tags(tags):
     tags = [tag.strip() for tag in tags.replace(".", ",").replace("\n", ",").split(",")]
@@ -419,6 +422,223 @@ class TagCategory:
                         break
 
         return (", ".join(result),)
+    
+def parse_tags(text):
+    return [tag.strip() for tag in text.split(",") if tag.strip() != ""]
+
+def update_scores(scores, text, base, bonus_first, bonus_4_10, include_new=True):
+    tags = parse_tags(text)
+    for i, tag in enumerate(tags):
+        if i < 3:
+            points = base + bonus_first
+        elif i < 10:
+            points = base + bonus_4_10
+        else:
+            points = base
+
+        if tag in scores:
+            scores[tag] += points
+        else:
+            if include_new:
+                scores[tag] = points
+
+class ResortingTags:
+    def __init__(self):
+        pass
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "priority_texts": ("LIST", {"forceInput": True}),
+                "inclusive_texts": ("LIST", {"forceInput": True}),
+                "auxiliary_texts": ("LIST", {"forceInput": True}),
+            }
+        }
+
+    RETURN_TYPES = ("STRING", "STRING",)
+    RETURN_NAMES = ("tags TEXT", "tags_with_rating TEXT",)
+
+    FUNCTION = "resorting_tags"
+
+    CATEGORY = "LevelPixel/Tags"
+
+    def resorting_tags(self, priority_texts, inclusive_texts, auxiliary_texts):
+        scores = {}
+        for text in priority_texts:
+            update_scores(scores, text, base=3, bonus_first=2, bonus_4_10=1, include_new=True)
+
+        for text in inclusive_texts:
+            update_scores(scores, text, base=2, bonus_first=2, bonus_4_10=1, include_new=True)
+
+        for text in auxiliary_texts:
+            update_scores(scores, text, base=1, bonus_first=1, bonus_4_10=0, include_new=False)
+
+        sorted_tags = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
+        result = ", ".join(tag for tag, _ in sorted_tags)
+        rating_text = ", ".join(f"{tag}:{points}" for tag, points in sorted_tags)
+
+        print("Result tags:")
+        print(result)
+        print("\nRating result tags:")
+        print(rating_text)
+
+        return (result, rating_text,)
+
+class RemoveDuplicateTags:
+    def __init__(self):
+        pass
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "text": ("STRING", {"multiline": True, "default": "", "forceInput": True})
+            }
+        }
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("tags TEXT",)
+
+    FUNCTION = "remove_duplicate_tags"
+
+    CATEGORY = "LevelPixel/Tags"
+    
+    def remove_duplicate_tags(self, text):
+        tags = text.split(',')
+        seen = set()
+        unique_tags = []
+        
+        for tag in tags:
+            tag_clean = tag.strip()
+            if tag_clean and tag_clean not in seen:
+                seen.add(tag_clean)
+                unique_tags.append(tag_clean)
+        
+        result = ", ".join(unique_tags) + ","
+
+        return (result,)
+
+def is_english_core(core: str) -> bool:
+    for ch in core:
+        if ch.isalpha() and ch not in string.ascii_letters:
+            return False
+    return True
+
+class KeepOnlyEnglishTags:
+    def __init__(self):
+        pass
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "text": ("STRING", {"multiline": True, "default": "", "forceInput": True})
+            }
+        }
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("tags TEXT",)
+
+    FUNCTION = "keep_only_english_tags"
+
+    CATEGORY = "LevelPixel/Tags"
+    
+    def keep_only_english_tags(self, text):
+        tags = [tag.strip() for tag in text.split(',') if tag.strip()]
+        
+        filtered_tags = [tag for tag in tags if is_english_core(tag)]
+
+        result = ', '.join(filtered_tags)
+        if text.strip()[-1] == ',':
+            result += ','
+
+        return (result,)
+    
+class RemoveBannedTagsFromTags:
+    def __init__(self):
+        pass
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "text": ("STRING", {"multiline": True, "default": "", "forceInput": True})
+            }
+        }
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("tags TEXT",)
+
+    FUNCTION = "remove_banned_tags_from_tags"
+
+    CATEGORY = "LevelPixel/Tags"
+    
+    def remove_banned_tags_from_tags(self, text):
+        with open(banned_file, "r", encoding="utf-8") as f:
+            banned_tags = {line.strip().lower() for line in f if line.strip()}
+
+        text_lower = text.lower()
+        tokens = text_lower.split(',')
+        wrapper_chars = '\"\'“”‘’()[]{}<>«»'
+
+        kept_tokens = []
+        for token in tokens:
+            token_for_check = token.strip()
+            while token_for_check and token_for_check[0] in wrapper_chars:
+                token_for_check = token_for_check[1:]
+            while token_for_check and token_for_check[-1] in wrapper_chars:
+                token_for_check = token_for_check[:-1]
+
+            if token_for_check in banned_tags:
+                continue
+            else:
+                kept_tokens.append(token)
+
+        result = ",".join(kept_tokens)
+
+        return (result,)
+    
+class RemoveBannedTagsFromText:
+    def __init__(self):
+        pass
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "text": ("STRING", {"multiline": True, "default": "", "forceInput": True})
+            }
+        }
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("tags TEXT",)
+
+    FUNCTION = "remove_banned_tags_from_text"
+
+    CATEGORY = "LevelPixel/Tags"
+    
+    def remove_banned_tags_from_text(self, text):
+        with open(banned_file, "r", encoding="utf-8") as f:
+            banned_tags = [line.strip() for line in f if line.strip()]
+
+        encl_chars = r'\"\'‘’“”«»\(\)\[\]\{\}<>'
+
+        for tag in banned_tags:
+            pattern = re.compile(
+                r'(?<!\w)[' + re.escape(encl_chars) + r']*' + 
+                re.escape(tag.lower()) +
+                r'[' + re.escape(encl_chars) + r']*(?!\w)',
+                re.IGNORECASE
+            )
+            text = pattern.sub("", text)
+        
+        text = re.sub(r'\s+', ' ', text)
+        text = re.sub(r'\s+([,.;?!])', r'\1', text)
+        result = text.strip()
+
+        return (result,)
+
 
 NODE_CLASS_MAPPINGS = {
     "TagCategory|LP": TagCategory,
@@ -429,6 +649,11 @@ NODE_CLASS_MAPPINGS = {
     "TagMerger|LP": TagMerger,
     "TagReplace|LP": TagReplace,
     "TagRemover|LP": TagRemover,
+    "ResortingTags|LP": ResortingTags,
+    "RemoveDuplicateTags|LP": RemoveDuplicateTags,
+    "KeepOnlyEnglishTags|LP": KeepOnlyEnglishTags,
+    "RemoveBannedTagsFromTags|LP": RemoveBannedTagsFromTags,
+    "RemoveBannedTagsFromText|LP": RemoveBannedTagsFromText,
 }
 
 NODE_DISPLAY_NAME_MAPPINGS = {
@@ -439,5 +664,10 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "TagSwitcher|LP": "Tag Switcher [LP]",
     "TagMerger|LP": "Tag Merger [LP]",
     "TagReplace|LP": "Tag Replace [LP]",
-    "TagRemover|LP": "Tag Remover [LP]"
+    "TagRemover|LP": "Tag Remover [LP]",
+    "ResortingTags|LP": "Resorting Tags [LP]",
+    "RemoveDuplicateTags|LP": "Remove Duplicate Tags [LP]",
+    "KeepOnlyEnglishTags|LP": "Keep Only English Tags [LP]",
+    "RemoveBannedTagsFromTags|LP": "Remove Banned Tags From Tags [LP]",
+    "RemoveBannedTagsFromText|LP": "Remove Banned Tags From Text [LP]",
 }

nodes/text/text_utils_LP.py

@@ -1,6 +1,8 @@
 import re
 import random
 import time
+import wordninja
+import string
 from deep_translator import GoogleTranslator
 from langdetect import detect
 
@@ -94,14 +96,177 @@ class TextTranslate:
                     print(f"Translation error: {e}")
         return (text,)
 
+class TextToList:
+    def __init__(self):
+        pass
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+            },
+            "optional": {
+                "delimiter": ("STRING", {"default": " "}),
+                "text_1": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+                "text_2": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+                "text_3": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+                "text_4": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+                "text_5": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+                "text_6": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+                "text_7": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+                "text_8": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+                "text_9": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+                "text_10": ("STRING", {"multiline": True, "default": "", "forceInput": True}),
+            }
+        }
+
+    RETURN_TYPES = ("STRING", "LIST",)
+    RETURN_NAMES = ("concatenated STRING", "text LIST")
+    OUTPUT_IS_LIST = (False, True, )
+    FUNCTION = "text_to_list"
+
+    CATEGORY = "LevelPixel/Text"
+
+    def text_to_list(self,
+                delimiter="",
+                text_1=None,
+                text_2=None,
+                text_3=None,
+                text_4=None,
+                text_5=None,
+                text_6=None,
+                text_7=None,
+                text_8=None,
+                text_9=None,
+                text_10=None):
+
+        list_str = []
+
+        if text_1 is not None and text_1 != "":
+            list_str.append(text_1)
+        if text_2 is not None and text_2 != "":
+            list_str.append(text_2)
+        if text_3 is not None and text_3 != "":
+            list_str.append(text_3)
+        if text_4 is not None and text_4 != "":
+            list_str.append(text_4)
+        if text_5 is not None and text_5 != "":
+            list_str.append(text_5)
+        if text_6 is not None and text_6 != "":
+            list_str.append(text_6)
+        if text_7 is not None and text_7 != "":
+            list_str.append(text_7)
+        if text_8 is not None and text_8 != "":
+            list_str.append(text_8)
+        if text_9 is not None and text_9 != "":
+            list_str.append(text_9)
+        if text_10 is not None and text_10 != "":
+            list_str.append(text_10)
+
+        return delimiter.join(list_str), [list_str]
+    
+class SplitCompoundText:
+    def __init__(self):
+        pass
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "text": ("STRING", {"multiline": True, "default": "", "forceInput": True})
+            }
+        }
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("text TEXT",)
+
+    FUNCTION = "split_compound_text"
+
+    CATEGORY = "LevelPixel/Text"
+
+    def split_compound_text(self, text):
+        has_trailing_comma = text.rstrip().endswith(',')
+        
+        tokens = text.split(',')
+        result_tokens = []
+        
+        for token in tokens:
+            token = token.strip()
+            if token:
+                splitted_words = wordninja.split(token)
+                result_tokens.append(" ".join(splitted_words))
+        
+        result = ", ".join(result_tokens)
+        if has_trailing_comma:
+            result += ","
+        
+        return (result,)
+
+def is_english_core(core: str) -> bool:
+    for ch in core:
+        if ch.isalpha() and ch not in string.ascii_letters:
+            return False
+    return True
+
+def process_token(token: str) -> str:
+    core = token.strip(string.punctuation)
+    if core == "":
+        return token
+    if is_english_core(core):
+        return token
+    else:
+        return ""
+
+class KeepOnlyEnglishWords:
+    def __init__(self):
+        pass
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "text": ("STRING", {"multiline": True, "default": "", "forceInput": True})
+            }
+        }
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("text TEXT",)
+
+    FUNCTION = "keep_only_english_words"
+
+    CATEGORY = "LevelPixel/Text"
+    
+    def keep_only_english_words(self, text):
+        tokens = text.split()
+        processed_tokens = []
+        for token in tokens:
+            filtered = process_token(token)
+            if filtered:
+                processed_tokens.append(filtered)
+        result = " ".join(processed_tokens)
+
+        result = re.sub(r'\s+([,.?!:;])', r'\1', result)
+
+        text = text.rstrip()
+        if text and text[-1] in ".!?" and (not result or result[-1] not in ".!?"):
+            result += text[-1]
+
+        return (result,)
+
 NODE_CLASS_MAPPINGS = {
     "TextChoiceParser|LP": TextChoiceParser,
     "CLIPTextEncodeTranslate|LP": CLIPTextEncodeTranslate,
     "TextTranslate|LP": TextTranslate,
+    "TextToList|LP": TextToList,
+    "SplitCompoundText|LP": SplitCompoundText,
+    "KeepOnlyEnglishWords|LP": KeepOnlyEnglishWords,
 }
 
 NODE_DISPLAY_NAME_MAPPINGS = {
     "TextChoiceParser|LP": "Text Choice Parser [LP]",
     "CLIPTextEncodeTranslate|LP": "CLIP Text Encode Translate [LP]",
     "TextTranslate|LP": "Text Translate [LP]",
+    "TextToList|LP": "Text To List [LP]",
+    "SplitCompoundText|LP": "Split Compound Text [LP]",
+    "KeepOnlyEnglishWords|LP": "Keep Only English Words [LP]",
 }

nodes/vlm/autotagger_LP.py

@@ -1,5 +1,3 @@
-# https://huggingface.co/spaces/SmilingWolf/wd-v1-4-tags
-
 import comfy.utils
 import asyncio
 import aiohttp

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui_level_pixel"
 description = "Various nodes of the Level Pixel company. Includes convenient advanced nodes for working with images from folders; counting files in a folder; cleaning memory; tag filters. Model Unloader, LLM Unloader (GGUF unloaders), Free memory, Tag Filters, Tag Category Filters, Tag Choice Parser, File counter, Image Loader From Path (with counters), Image Remove Background based on RemBG, Autotagger."
-version = "1.1.2"
+version = "1.1.3"
 license = { file = "LICENSE" }
 dependencies = ["torch>=2.0.1", "torchvision>=0.15.2", "pillow>=9.4.0", "numpy", "matplotlib", "scikit-build-core>=0.10.7", "rembg>=2.0.59", "onnxruntime-gpu>=1.20.0", "onnxruntime>=1.20.0", "googletrans", "langdetect", "httpcore", "deep_translator"]
 
@@ -10,6 +10,5 @@ Repository = "https://github.com/LevelPixel/ComfyUI-LevelPixel"
 
 [tool.comfy]
 PublisherId = "Level Pixel"
-DisplayName = "ComfyUI_LevelPixel"
-Icon = ""
-Models = [{location = "/checkpoints/model.safetensors", model_url = "https://example.com/model.zip"}]
+DisplayName = "ComfyUI Level Pixel"
+Icon = ""

requirements.txt

@@ -10,4 +10,5 @@ onnxruntime>=1.20.0
 googletrans
 langdetect
 httpcore
-deep_translator
+deep_translator
+wordninja