fixed: LBW - Certain text encoders, such as T5, were not being reflected in CLIP.

https://github.com/ltdrdata/ComfyUI-Inspire-Pack/issues/185

README.md

@@ -61,7 +61,9 @@ This repository offers various extension nodes for ComfyUI. Nodes here have diff
     * e.g. `prompts/example`
   * `Load Prompts From File (Inspire)`: It sequentially reads prompts from the specified file. The output it returns is ZIPPED_PROMPT.
     * Specify the file located under `ComfyUI-Inspire-Pack/prompts/`
-    * e.g. `prompts/example/prompt2.txt` 
+    * e.g. `prompts/example/prompt2.txt`
+  * `Load Single Prompt From File (Inspire)`: Loads a single prompt from a file containing multiple prompts by using an index.
+  * The prompts file directory can be specified as `inspire_prompts` in `extra_model_paths.yaml`
   * `Unzip Prompt (Inspire)`: Separate ZIPPED_PROMPT into `positive`, `negative`, and name components. 
     * `positive` and `negative` represent text prompts, while `name` represents the name of the prompt. When loaded from a file using `Load Prompts From File (Inspire)`, the name corresponds to the file name.
   * `Zip Prompt (Inspire)`: Create ZIPPED_PROMPT from positive, negative, and name_opt.

__init__.py

@@ -7,7 +7,7 @@
 
 import importlib
 
-version_code = [1, 4]
+version_code = [1, 6, 1]
 version_str = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '')
 print(f"### Loading: ComfyUI-Inspire-Pack ({version_str})")
 

inspire/conditioning_nodes.py

@@ -8,19 +8,24 @@ from nodes import MAX_RESOLUTION
 class ConcatConditioningsWithMultiplier:
     @classmethod
     def INPUT_TYPES(s):
-        flex_inputs = {}
-
         stack = inspect.stack()
-        if stack[1].function == 'get_input_data':
+        if stack[1].function == 'get_input_info':
             # bypass validation
-            for x in range(0, 100):
-                flex_inputs[f"multiplier{x}"] = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01})
-        else:
-            flex_inputs["multiplier1"] = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01})
+            class AllContainer:
+                def __contains__(self, item):
+                    return True
+
+                def __getitem__(self, key):
+                    return "FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}
+
+            return {
+                "required": {"conditioning1": ("CONDITIONING",), },
+                "optional": AllContainer()
+            }
 
         return {
             "required": {"conditioning1": ("CONDITIONING",), },
-            "optional": flex_inputs
+            "optional": {"multiplier1": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), },
         }
 
     RETURN_TYPES = ("CONDITIONING",)

inspire/inspire_server.py

@@ -332,7 +332,7 @@ def populate_wildcards(json_data):
 
         if 'extra_data' in json_data and 'extra_pnginfo' in json_data['extra_data']:
             extra_pnginfo = json_data['extra_data']['extra_pnginfo']
-            if 'workflow' in extra_pnginfo and 'nodes' in extra_pnginfo['workflow']:
+            if 'workflow' in extra_pnginfo and extra_pnginfo['workflow'] is not None and 'nodes' in extra_pnginfo['workflow']:
                 for node in extra_pnginfo['workflow']['nodes']:
                     key = str(node['id'])
                     if key in updated_widget_values:

inspire/lora_block_weight.py

@@ -484,7 +484,7 @@ class LoraLoaderBlockWeight:
 
             if k in muted_weights:
                 pass
-            elif 'text' in k:
+            elif 'text' in k or 'encoder' in k:
                 new_clip.add_patches({k: weights}, strength_clip * ratio)
             else:
                 new_modelpatcher.add_patches({k: weights}, strength_model * ratio)
@@ -546,7 +546,7 @@ class ApplyLBW:
 
             if k in muted_weights:
                 pass
-            elif 'text' in k:
+            elif 'text' in k or 'encoder' in k:
                 new_clip.add_patches({k: weights}, strength_clip * ratio)
             else:
                 new_modelpatcher.add_patches({k: weights}, strength_model * ratio)
@@ -822,9 +822,13 @@ class LoraBlockInfo:
         output_blocks = []
         output_blocks_map = {}
 
-        text_block_count = set()
-        text_blocks = []
-        text_blocks_map = {}
+        text_block_count1 = set()
+        text_blocks1 = []
+        text_blocks_map1 = {}
+
+        text_block_count2 = set()
+        text_blocks2 = []
+        text_blocks_map2 = {}
 
         double_block_count = set()
         double_blocks = []
@@ -902,12 +906,23 @@ class LoraBlockInfo:
                 k_unet_num = k_unet[len("er.text_model.encoder.layers."):len("er.text_model.encoder.layers.")+2]
                 k_unet_int = parse_unet_num(k_unet_num)
 
-                text_block_count.add(k_unet_int)
-                text_blocks.append(k_unet)
-                if k_unet_int in text_blocks_map:
-                    text_blocks_map[k_unet_int].append(k_unet)
+                text_block_count1.add(k_unet_int)
+                text_blocks1.append(k_unet)
+                if k_unet_int in text_blocks_map1:
+                    text_blocks_map1[k_unet_int].append(k_unet)
                 else:
-                    text_blocks_map[k_unet_int] = [k_unet]
+                    text_blocks_map1[k_unet_int] = [k_unet]
+
+            elif k_unet.startswith("r.encoder.block."):
+                k_unet_num = k_unet[len("r.encoder.block."):len("r.encoder.block.")+2]
+                k_unet_int = parse_unet_num(k_unet_num)
+
+                text_block_count2.add(k_unet_int)
+                text_blocks2.append(k_unet)
+                if k_unet_int in text_blocks_map2:
+                    text_blocks_map2[k_unet_int].append(k_unet)
+                else:
+                    text_blocks_map2[k_unet_int] = [k_unet]
 
             else:
                 others.append(k_unet)
@@ -951,10 +966,14 @@ class LoraBlockInfo:
             for x in single_keys:
                 text += f" SINGLE{x}: {len(single_blocks_map[x])}\n"
 
-        text += f"\n-------[Base blocks] ({len(text_block_count) + len(others)}, Subs={len(text_blocks) + len(others)})-------\n"
-        text_keys = sorted(text_blocks_map.keys())
-        for x in text_keys:
-            text += f" TXT_ENC{x}: {len(text_blocks_map[x])}\n"
+        text += f"\n-------[Base blocks] ({len(text_block_count1) + len(text_block_count2) + len(others)}, Subs={len(text_blocks1) + len(text_blocks2) + len(others)})-------\n"
+        text_keys1 = sorted(text_blocks_map1.keys())
+        for x in text_keys1:
+            text += f" TXT_ENC{x}: {len(text_blocks_map1[x])}\n"
+
+        text_keys2 = sorted(text_blocks_map2.keys())
+        for x in text_keys2:
+            text += f" TXT_ENC{x} [B]: {len(text_blocks_map2[x])}\n"
 
         for x in others:
             text += f" {x}\n"

inspire/model_nodes.py

@@ -19,6 +19,7 @@ model_preset = {
     "SDXL ViT-H":           ("ip-adapter_sdxl_vit-h",           "CLIP-ViT-H-14-laion2B-s32B-b79K", None, False),
     "SDXL Plus ViT-H":      ("ip-adapter-plus_sdxl_vit-h",      "CLIP-ViT-H-14-laion2B-s32B-b79K", None, False),
     "SDXL Plus Face ViT-H": ("ip-adapter-plus-face_sdxl_vit-h", "CLIP-ViT-H-14-laion2B-s32B-b79K", None, False),
+    "Kolors Plus":          ("Kolors-IP-Adapter-Plus",          "clip-vit-large-patch14-336", None, False),
 
     # faceid
     "SD1.5 FaceID":                ("ip-adapter-faceid_sd15",                 "CLIP-ViT-H-14-laion2B-s32B-b79K", "ip-adapter-faceid_sd15_lora", True),
@@ -29,6 +30,7 @@ model_preset = {
     "SDXL FaceID":                 ("ip-adapter-faceid_sdxl",                 "CLIP-ViT-H-14-laion2B-s32B-b79K", "ip-adapter-faceid_sdxl_lora", True),
     "SDXL FaceID Portrait":        ("ip-adapter-faceid-portrait_sdxl",        "CLIP-ViT-H-14-laion2B-s32B-b79K", None, True),
     "SDXL FaceID Portrait unnorm": ("ip-adapter-faceid-portrait_sdxl_unnorm", "CLIP-ViT-H-14-laion2B-s32B-b79K", None, True),
+    "Kolors FaceID Plus":          ("Kolors-IP-Adapter-FaceID-Plus",          "clip-vit-large-patch14-336", None, True),
 
     # composition
     "SD1.5 Plus Composition":      ("ip-adapter_sd15", "CLIP-ViT-H-14-laion2B-s32B-b79K", None, False),
@@ -56,7 +58,6 @@ class IPAdapterModelHelper:
         return {
             "required": {
                 "model": ("MODEL",),
-                "clip": ("CLIP",),
                 "preset": (list(model_preset.keys()),),
                 "lora_strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}),
                 "lora_strength_clip": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}),
@@ -64,6 +65,7 @@ class IPAdapterModelHelper:
                 "cache_mode": (["insightface only", "clip_vision only", "all", "none"], {"default": "insightface only"}),
             },
             "optional": {
+                "clip": ("CLIP",),
                 "insightface_model_name": (['buffalo_l', 'antelopev2'],),
             },
             "hidden": {"unique_id": "UNIQUE_ID"}
@@ -75,14 +77,17 @@ class IPAdapterModelHelper:
 
     CATEGORY = "InspirePack/models"
 
-    def doit(self, model, clip, preset, lora_strength_model, lora_strength_clip, insightface_provider, cache_mode="none", unique_id=None, insightface_model_name='buffalo_l'):
+    def doit(self, model, preset, lora_strength_model, lora_strength_clip, insightface_provider, clip=None, cache_mode="none", unique_id=None, insightface_model_name='buffalo_l'):
         if 'IPAdapter' not in nodes.NODE_CLASS_MAPPINGS:
             utils.try_install_custom_node('https://github.com/cubiq/ComfyUI_IPAdapter_plus',
                                           "To use 'IPAdapterModelHelper' node, 'ComfyUI IPAdapter Plus' extension is required.")
             raise Exception(f"[ERROR] To use IPAdapterModelHelper, you need to install 'ComfyUI IPAdapter Plus'")
 
         is_sdxl_preset = 'SDXL' in preset
-        is_sdxl_model = isinstance(clip.tokenizer, sdxl_clip.SDXLTokenizer)
+        if clip is not None:
+            is_sdxl_model = isinstance(clip.tokenizer, sdxl_clip.SDXLTokenizer)
+        else:
+            is_sdxl_model = False
 
         if is_sdxl_preset != is_sdxl_model:
             server.PromptServer.instance.send_sync("inspire-node-output-label", {"node_id": unique_id, "output_idx": 1, "label": "IPADAPTER (fail)"})

inspire/prompt_support.py

@@ -18,15 +18,17 @@ from server import PromptServer
 from .libs import utils, common
 from .backend_support import CheckpointLoaderSimpleShared
 
+
+model_path = folder_paths.models_dir
+utils.add_folder_path_and_extensions("inspire_prompts", [os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "prompts"))], {'.txt'})
+
+
 prompt_builder_preset = {}
 
 
 resource_path = os.path.join(os.path.dirname(__file__), "..", "resources")
 resource_path = os.path.abspath(resource_path)
 
-prompts_path = os.path.join(os.path.dirname(__file__), "..", "prompts")
-prompts_path = os.path.abspath(prompts_path)
-
 
 try:
     pb_yaml_path = os.path.join(resource_path, 'prompt-builder.yaml')
@@ -44,9 +46,12 @@ except Exception as e:
 class LoadPromptsFromDir:
     @classmethod
     def INPUT_TYPES(cls):
-        global prompts_path
         try:
-            prompt_dirs = [d for d in os.listdir(prompts_path) if os.path.isdir(os.path.join(prompts_path, d))]
+            prompt_dirs = []
+            for x in folder_paths.get_folder_paths('inspire_prompts'):
+                for d in os.listdir(x):
+                    if os.path.isdir(os.path.join(x, d)):
+                        prompt_dirs.append(d)
         except Exception:
             prompt_dirs = []
 
@@ -70,16 +75,24 @@ class LoadPromptsFromDir:
         if not reload:
             return prompt_dir
         else:
-            global prompts_path
-            prompt_dir = os.path.join(prompts_path, prompt_dir)
-            files = [f for f in os.listdir(prompt_dir) if f.endswith(".txt")]
+            candidates = []
+            for d in folder_paths.get_folder_paths('inspire_prompts'):
+                candidates.append(os.path.join(d, prompt_dir))
+
+            prompt_files = []
+            for x in candidates:
+                for root, dirs, files in os.walk(x):
+                    for file in files:
+                        if file.endswith(".txt"):
+                            prompt_files.append(os.path.join(root, file))
+
+            prompt_files.sort()
 
             md5 = hashlib.md5()
-            files.sort()
 
-            for file in files:
-                md5.update(file.encode('utf-8'))
-                with open(os.path.join(prompt_dir, file), 'rb') as f:
+            for file_name in prompt_files:
+                md5.update(file_name.encode('utf-8'))
+                with open(folder_paths.get_full_path('inspire_prompts', file_name), 'rb') as f:
                     while True:
                         chunk = f.read(4096)
                         if not chunk:
@@ -90,16 +103,24 @@ class LoadPromptsFromDir:
 
     @staticmethod
     def doit(prompt_dir, reload=False):
-        global prompts_path
-        prompt_dir = os.path.join(prompts_path, prompt_dir)
-        files = [f for f in os.listdir(prompt_dir) if f.endswith(".txt")]
-        files.sort()
+        candidates = []
+        for d in folder_paths.get_folder_paths('inspire_prompts'):
+            candidates.append(os.path.join(d, prompt_dir))
+
+        prompt_files = []
+        for x in candidates:
+            for root, dirs, files in os.walk(x):
+                for file in files:
+                    if file.endswith(".txt"):
+                        prompt_files.append(os.path.join(root, file))
+
+        prompt_files.sort()
 
         prompts = []
-        for file_name in files:
+        for file_name in prompt_files:
             print(f"file_name: {file_name}")
             try:
-                with open(os.path.join(prompt_dir, file_name), "r", encoding="utf-8") as file:
+                with open(file_name, "r", encoding="utf-8") as file:
                     prompt_data = file.read()
                     prompt_list = re.split(r'\n\s*-+\s*\n', prompt_data)
 
@@ -123,15 +144,16 @@ class LoadPromptsFromDir:
 class LoadPromptsFromFile:
     @classmethod
     def INPUT_TYPES(cls):
-        global prompts_path
+        prompt_files = []
         try:
-            prompt_files = []
-            for root, dirs, files in os.walk(prompts_path):
-                for file in files:
-                    if file.endswith(".txt"):
-                        file_path = os.path.join(root, file)
-                        rel_path = os.path.relpath(file_path, prompts_path)
-                        prompt_files.append(rel_path)
+            prompts_paths = folder_paths.get_folder_paths('inspire_prompts')
+            for prompts_path in prompts_paths:
+                for root, dirs, files in os.walk(prompts_path):
+                    for file in files:
+                        if file.endswith(".txt"):
+                            file_path = os.path.join(root, file)
+                            rel_path = os.path.relpath(file_path, prompts_path)
+                            prompt_files.append(rel_path)
         except Exception:
             prompt_files = []
 
@@ -161,9 +183,18 @@ class LoadPromptsFromFile:
         elif not reload:
             return prompt_file
         else:
-            prompt_path = os.path.join(prompts_path, prompt_file)
+            matched_path = None
+            for x in folder_paths.get_folder_paths('inspire_prompts'):
+                matched_path = os.path.join(x, prompt_file)
+                if not os.path.exists(matched_path):
+                    matched_path = None
+                else:
+                    break
 
-            with open(prompt_path, 'rb') as f:
+            if matched_path is None:
+                return float('NaN')
+
+            with open(matched_path, 'rb') as f:
                 while True:
                     chunk = f.read(4096)
                     if not chunk:
@@ -174,12 +205,21 @@ class LoadPromptsFromFile:
 
     @staticmethod
     def doit(prompt_file, text_data_opt=None, reload=False):
-        prompt_path = os.path.join(prompts_path, prompt_file)
+        matched_path = None
+        for d in folder_paths.get_folder_paths('inspire_prompts'):
+            matched_path = os.path.join(d, prompt_file)
+            if not os.path.exists(matched_path):
+                matched_path = None
+            else:
+                break
+
+        if matched_path:
+            print(f"[WARN] LoadPromptsFromFile: file not found '{prompt_file}'")
 
         prompts = []
         try:
             if not text_data_opt:
-                with open(prompt_path, "r", encoding="utf-8") as file:
+                with open(matched_path, "r", encoding="utf-8") as file:
                     prompt_data = file.read()
             else:
                 prompt_data = text_data_opt
@@ -188,8 +228,8 @@ class LoadPromptsFromFile:
 
             pattern = r"positive:(.*?)(?:\n*|$)negative:(.*)"
 
-            for prompt in prompt_list:
-                matches = re.search(pattern, prompt, re.DOTALL)
+            for p in prompt_list:
+                matches = re.search(pattern, p, re.DOTALL)
 
                 if matches:
                     positive_text = matches.group(1).strip()
@@ -207,15 +247,16 @@ class LoadPromptsFromFile:
 class LoadSinglePromptFromFile:
     @classmethod
     def INPUT_TYPES(cls):
-        global prompts_path
+        prompt_files = []
         try:
-            prompt_files = []
-            for root, dirs, files in os.walk(prompts_path):
-                for file in files:
-                    if file.endswith(".txt"):
-                        file_path = os.path.join(root, file)
-                        rel_path = os.path.relpath(file_path, prompts_path)
-                        prompt_files.append(rel_path)
+            prompts_paths = folder_paths.get_folder_paths('inspire_prompts')
+            for prompts_path in prompts_paths:
+                for root, dirs, files in os.walk(prompts_path):
+                    for file in files:
+                        if file.endswith(".txt"):
+                            file_path = os.path.join(root, file)
+                            rel_path = os.path.relpath(file_path, prompts_path)
+                            prompt_files.append(rel_path)
         except Exception:
             prompt_files = []
 
@@ -235,7 +276,17 @@ class LoadSinglePromptFromFile:
 
     @staticmethod
     def doit(prompt_file, index, text_data_opt=None):
-        prompt_path = os.path.join(prompts_path, prompt_file)
+        prompt_path = None
+        prompts_paths = folder_paths.get_folder_paths('inspire_prompts')
+        for d in prompts_paths:
+            prompt_path = os.path.join(d, prompt_file)
+            if os.path.exists(prompt_path):
+                break
+            else:
+                prompt_path = None
+
+        if prompt_path:
+            print(f"[WARN] LoadSinglePromptFromFile: file not found '{prompt_file}'")
 
         prompts = []
         try:
@@ -623,7 +674,7 @@ class SeedExplorer:
             "optional":
                 {
                     "variation_method": (["linear", "slerp"],),
-                    "model": ("model",),
+                    "model": ("MODEL",),
                 }
         }
 

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-inspire-pack"
 description = "This extension provides various nodes to support Lora Block Weight and the Impact Pack. Provides many easily applicable regional features and applications for Variation Seed."
-version = "1.4"
+version = "1.6.1"
 license = { file = "LICENSE" }
 dependencies = ["matplotlib", "cachetools"]
 

requirements.txt

@@ -2,4 +2,4 @@ matplotlib
 cachetools
 numpy<2
 webcolors
-
+opencv-python