From db43d409f86445152dffd630d338d2ec7e1297a2 Mon Sep 17 00:00:00 2001 From: Antonio Cordero Balcazar Date: Sat, 27 Sep 2025 13:22:19 +0200 Subject: [PATCH] * removed unnecessary processing of variables. * error control when reading wildcard or mapping files, and second chance for files with incorrect encoding. * fix processing inside extranetwork tag. --- README.md | 3 +- docs/SYNTAX.md | 6 ++ grammar.lark | 15 ++++- ppp.py | 10 ++- ppp_enmappings.py | 22 +++++-- ppp_wildcards.py | 95 +++++++++++++++++------------ pyproject.toml | 2 +- scripts/ppp_script.py | 5 +- tests/tests.py | 6 +- tests/wildcards2/text/wildcard3.txt | 3 +- 10 files changed, 109 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 0c2a561..261e3ea 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ The Prompt PostProcessor (PPP), formerly known as "sd-webui-sendtonegative", is * [AUTOMATIC1111 Stable Diffusion WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) * [Forge](https://github.com/lllyasviel/stable-diffusion-webui-forge) * [reForge](https://github.com/Panchovix/stable-diffusion-webui-reForge) -* [SD.Next](https://github.com/vladmandic/automatic) * ...and probably other forks Currently this extension has these functions: @@ -50,7 +49,7 @@ On *A1111* compatible webuis: 3. Click the Install button 4. Restart the webui -On *SD.Next* I recommend you disable the native wildcard processing. +On *SD.Next* I recommend you to disable the native wildcard processing and use the old UI. On *ComfyUI*: diff --git a/docs/SYNTAX.md b/docs/SYNTAX.md index 700a3d7..f2c1dc6 100644 --- a/docs/SYNTAX.md +++ b/docs/SYNTAX.md @@ -83,6 +83,8 @@ These are examples of formats you can use to insert a wildcard: | `__2-3$$ / $$path/wildcard__` | select 2 to 3 choices with separator " / " | | `__path/wildcard(var=value)__` | select 1 choice using the specified variable value in the evaluation. | +Wildcards cannot be used inside an extranetwork tag (because some lora names contain double underscores). If you need to choose from multiple loras put the whole extranetwork tag inside a wildcard, or use choices. + ### Wildcard definitions A wildcard definition can be: @@ -115,6 +117,10 @@ Wildcards can contain just one choice. In json and yaml formats this allows the A choice inside a wildcard can also be a list or a dictionary of one element containing a list. These are considered anonymous wildcards. With a list it will be an anonymous wildcard with no choice options, and with a dictionary the key will be the options for the choice containing the anonymous wildcard and the value the choices of the anonymous wildcard. Anonymous wildcards can help formatting complex choice values that are used in only one place and thus creating a regular wildcard is not necessary. See test.yaml for examples. +Note: the files should have UTF-8 encoding. The extension will also try with windows-1252 if that fails. + +Wildcard definitions are reloaded automatically on each generation if they change. + ### Detection of remaining wildcards This extension should run after any other wildcard extensions, so if you don't use the internal wildcards processing, any remaining wildcards present in the prompt or negative_prompt at this point must be invalid. Usually you might not notice this problem until you check the image metadata, so this option gives you some ways to detect and treat the problem. diff --git a/grammar.lark b/grammar.lark index f83b4ed..db80a03 100644 --- a/grammar.lark +++ b/grammar.lark @@ -31,43 +31,51 @@ promptcomppart: content ?old_content.2: ( attention | scheduled | alternate | extranetworktag )+ //#if ALLOW_NEW_CONTENT ?content.2: ( old_content | new_content | plain | specialchars )* - ?inside_content.2: ( new_content | plain | specialchars )* ?content_choice.2: ( old_content | new_content | plain_choice | specialchars_choice )* ?content_var.2: ( old_content | new_content | plain_var | specialchars_var )* ?content_negtag.2: ( old_content | new_content_negtag | plain | specialchars_negtag )* ?content_alternate.2: ( old_content | new_content | plain_alternate | specialchars_alternate )* + ?content_en.2: (new_content_en | plain | specialchars )* //#if ALLOW_WILDCARDS ALLOW_CHOICES ALLOW_COMMVARS ?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | wildcard | choices )+ ?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | commandext | wildcard | choices )+ + ?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif | choices )+ //#elif ALLOW_WILDCARDS !ALLOW_CHOICES ALLOW_COMMVARS ?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | wildcard )+ ?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | commandext | wildcard )+ + ?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif )+ //#elif !ALLOW_WILDCARDS ALLOW_CHOICES ALLOW_COMMVARS ?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | choices )+ ?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | commandext | choices )+ + ?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif | choices )+ //#elif ALLOW_WILDCARDS ALLOW_CHOICES !ALLOW_COMMVARS ?new_content.3: ( wildcard | choices )+ ?new_content_negtag.3: ( wildcard | choices )+ + ?new_content_en.3: ( choices )+ //#elif ALLOW_WILDCARDS !ALLOW_CHOICES !ALLOW_COMMVARS ?new_content.3: ( wildcard )+ ?new_content_negtag.3: ( wildcard )+ + ?new_content_en.3: /(?!)./ // never matches //#elif !ALLOW_WILDCARDS ALLOW_CHOICES !ALLOW_COMMVARS ?new_content.3: ( choices )+ ?new_content_negtag.3: ( choices )+ + ?new_content_en.3: ( choices )+ //#elif !ALLOW_WILDCARDS !ALLOW_CHOICES ALLOW_COMMVARS ?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext )+ ?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | commandext )+ + ?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif )+ //#else ?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | wildcard | choices )+ ?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | wildcard | commandext | choices )+ + ?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif | choices )+ //#endif //#else ?content.2: ( old_content | plain | specialchars )* - ?inside_content.2: ( plain | specialchars )* ?content_choice.2: ( old_content | plain_choice | specialchars_choice )* ?content_var.2: ( old_content | plain_var | specialchars_var )* ?content_negtag.2: ( old_content | plain | specialchars_negtag )* ?content_alternate.2: ( old_content | plain_alternate | specialchars_alternate )* + ?content_en.2: (plain | specialchars )* //#endif // attention modifiers @@ -79,7 +87,8 @@ alternateoption: content_alternate scheduled: "[" [ content ":" ] content ":" numpar "]" // extra network tags -extranetworktag: "<" /(?!ppp:)\w+:/ inside_content ">" +extranetworktag: "<" /(?!ppp:)\w+:/ encontent ">" +?encontent.3: content_en // command: stn (send to negative) commandstn: "" content_negtag "" diff --git a/ppp.py b/ppp.py index 6eb84f7..370a0c7 100644 --- a/ppp.py +++ b/ppp.py @@ -716,9 +716,13 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in negative_prompt = n_processor.start_visit("negative prompt", n_parsed, True) var_keys = set(self.user_variables.keys()).union(set(self.echoed_variables.keys())) - all_variables.update( - {k: self.echoed_variables.get(k, p_processor.get_final_user_variable(k)) for k in var_keys} - ) + for k in var_keys: + ev = self.echoed_variables.get(k) + if ev is None: + ev = p_processor.get_final_user_variable(k) + all_variables[k] = ev + if self.debug_level == DEBUG_LEVEL.full: + self.logger.debug(self.format_output(f"All variables: {all_variables}")) # Insertions in the negative prompt if self.debug_level == DEBUG_LEVEL.full: diff --git a/ppp_enmappings.py b/ppp_enmappings.py index c24e6d0..ff995ca 100644 --- a/ppp_enmappings.py +++ b/ppp_enmappings.py @@ -43,7 +43,9 @@ class PPPENMapping: self.kind: str = kind self.name: str = name self.variants: list[PPPENMappingVariant] = [ - PPPENMappingVariant(**{**{"condition": None, "name": None, "parameters": None, "triggers": None, "weight": 1.0}, **v}) + PPPENMappingVariant( + **{**{"condition": None, "name": None, "parameters": None, "triggers": None, "weight": 1.0}, **v} + ) for v in variants ] @@ -78,7 +80,9 @@ class PPPExtraNetworkMappings: def __sizeof__(self): return ( - self.extranetwork_mappings.__sizeof__() + self.__enmappings_folders.__sizeof__() + self.__enmappings_files.__sizeof__() + self.extranetwork_mappings.__sizeof__() + + self.__enmappings_folders.__sizeof__() + + self.__enmappings_files.__sizeof__() ) def refresh_extranetwork_mappings( @@ -233,9 +237,17 @@ class PPPExtraNetworkMappings: full_path (str): The path to the file. base (str): The base path for the extra network mappings. """ - with open(full_path, "r", encoding="utf-8") as file: - content = yaml.safe_load(file) - self.__add_extranetwork_mapping(content, full_path) + try: + try: + with open(full_path, "r", encoding="utf-8") as file: + content = yaml.safe_load(file) + except: # pylint: disable=bare-except + self.__logger.warning(f"Could not read file '{full_path}' with utf-8 encoding, trying windows-1252...") + with open(full_path, "r", encoding="windows-1252") as file: + content = yaml.safe_load(file) + self.__add_extranetwork_mapping(content, full_path) + except Exception as e: # pylint: disable=broad-except + self.__logger.error(f"Error reading extra network mappings from file '{full_path}': {e}") def __get_extranetwork_mappings_in_directory(self, directory: str): """ diff --git a/ppp_wildcards.py b/ppp_wildcards.py index 24cdf58..d566425 100644 --- a/ppp_wildcards.py +++ b/ppp_wildcards.py @@ -160,9 +160,9 @@ class PPPWildcards: last_modified_cached = self.__wildcard_files.get(full_path, None) # a time or a hash if debug and last_modified_cached is not None and self.__debug_level != DEBUG_LEVEL.none: if full_path == self.LOCALINPUT_FILENAME: - self.__logger.debug("Removing wildcards from input") + self.__logger.debug("Removing from memory wildcards from input") else: - self.__logger.debug(f"Removing wildcards from file: {full_path}") + self.__logger.debug(f"Removing from memory wildcards from file: {full_path}") if full_path in self.__wildcard_files.keys(): del self.__wildcard_files[full_path] for key in list(self.wildcards.keys()): @@ -177,22 +177,25 @@ class PPPWildcards: base (str): The base path for the wildcards. full_path (str): The path to the file. """ - last_modified = os.path.getmtime(full_path) - last_modified_cached = self.__wildcard_files.get(full_path, None) - if last_modified_cached is not None and last_modified == self.__wildcard_files[full_path]: - return - filename = os.path.basename(full_path) - _, extension = os.path.splitext(filename) - if extension not in (".txt", ".json", ".yaml", ".yml"): - return - self.__remove_wildcards_from_path(full_path, False) - if last_modified_cached is not None and self.__debug_level != DEBUG_LEVEL.none: - self.__logger.debug(f"Updating wildcards from file: {full_path}") - if extension == ".txt": - self.__get_wildcards_in_text_file(full_path, base) - elif extension in (".json", ".yaml", ".yml"): - self.__get_wildcards_in_structured_file(full_path, base) - self.__wildcard_files[full_path] = last_modified + try: + last_modified = os.path.getmtime(full_path) + last_modified_cached = self.__wildcard_files.get(full_path, None) + if last_modified_cached is not None and last_modified == self.__wildcard_files[full_path]: + return + filename = os.path.basename(full_path) + _, extension = os.path.splitext(filename) + if extension not in (".txt", ".json", ".yaml", ".yml"): + return + self.__remove_wildcards_from_path(full_path, False) + if last_modified_cached is not None and self.__debug_level != DEBUG_LEVEL.none: + self.__logger.debug(f"Updating wildcards from file: {full_path}") + if extension == ".txt": + self.__get_wildcards_in_text_file(full_path, base) + elif extension in (".json", ".yaml", ".yml"): + self.__get_wildcards_in_structured_file(full_path, base) + self.__wildcard_files[full_path] = last_modified + except Exception as e: # pylint: disable=broad-except + self.__logger.error(f"Error reading wildcard file '{full_path}': {e}") def __get_wildcards_in_input(self, wildcards_input: str): """ @@ -201,23 +204,26 @@ class PPPWildcards: Args: wildcards_input (str): The input string containing wildcards in json or yaml format. """ - new_h = hash(wildcards_input) - h = self.__wildcard_files.get(self.LOCALINPUT_FILENAME, None) - if h == new_h: - return - self.__remove_wildcards_from_path(self.LOCALINPUT_FILENAME, False) - if h is not None and self.__debug_level != DEBUG_LEVEL.none: - self.__logger.debug("Updating wildcards from input") - wildcards_input = wildcards_input.strip() - if wildcards_input != "": - try: - content = yaml.safe_load(wildcards_input) - except yaml.YAMLError as e: - self.__logger.warning(f"Invalid format for input wildcards: {e}") + try: + new_h = hash(wildcards_input) + h = self.__wildcard_files.get(self.LOCALINPUT_FILENAME, None) + if h == new_h: return - if content is not None: - self.__add_wildcard(content, self.LOCALINPUT_FILENAME, [self.LOCALINPUT_FILENAME]) - self.__wildcard_files[self.LOCALINPUT_FILENAME] = new_h + self.__remove_wildcards_from_path(self.LOCALINPUT_FILENAME, False) + if h is not None and self.__debug_level != DEBUG_LEVEL.none: + self.__logger.debug("Updating wildcards from input") + wildcards_input = wildcards_input.strip() + if wildcards_input != "": + try: + content = yaml.safe_load(wildcards_input) + except yaml.YAMLError as e: + self.__logger.warning(f"Invalid format for input wildcards: {e}") + return + if content is not None: + self.__add_wildcard(content, self.LOCALINPUT_FILENAME, [self.LOCALINPUT_FILENAME]) + self.__wildcard_files[self.LOCALINPUT_FILENAME] = new_h + except Exception as e: # pylint: disable=broad-except + self.__logger.error(f"Error reading wildcards input: {e}") def is_dict_choices_options(self, d: dict) -> bool: """ @@ -230,7 +236,8 @@ class PPPWildcards: bool: Whether the dictionary is a valid choices options dictionary or not. """ return all( - k in ["sampler", "repeating", "optional", "count", "from", "to", "prefix", "suffix", "separator"] for k in d.keys() + k in ["sampler", "repeating", "optional", "count", "from", "to", "prefix", "suffix", "separator"] + for k in d.keys() ) def is_dict_choice_options(self, d: dict) -> bool: @@ -390,8 +397,13 @@ class PPPWildcards: """ external_key: str = os.path.relpath(os.path.splitext(full_path)[0], base) external_key_parts = external_key.split(os.sep) - with open(full_path, "r", encoding="utf-8") as file: - content = yaml.safe_load(file) + try: + with open(full_path, "r", encoding="utf-8") as file: + content = yaml.safe_load(file) + except: # pylint: disable=bare-except + self.__logger.warning(f"Could not read file '{full_path}' with utf-8 encoding, trying windows-1252...") + with open(full_path, "r", encoding="windows-1252") as file: + content = yaml.safe_load(file) self.__add_wildcard(content, full_path, external_key_parts) def __get_wildcards_in_text_file(self, full_path, base): @@ -404,8 +416,13 @@ class PPPWildcards: """ external_key: str = os.path.relpath(os.path.splitext(full_path)[0], base) external_key_parts = external_key.split(os.sep) - with open(full_path, "r", encoding="utf-8") as file: - text_content = map(lambda x: x.strip("\n\r"), file.readlines()) + try: + with open(full_path, "r", encoding="utf-8") as file: + text_content = map(lambda x: x.strip("\n\r"), file.readlines()) + except: # pylint: disable=bare-except + self.__logger.warning(f"Could not read file '{full_path}' with utf-8 encoding, trying windows-1252...") + with open(full_path, "r", encoding="windows-1252") as file: + text_content = map(lambda x: x.strip("\n\r"), file.readlines()) text_content = list(filter(lambda x: x.strip() != "" and not x.strip().startswith("#"), text_content)) text_content = [x.split("#")[0].rstrip() if len(x.split("#")) > 1 else x for x in text_content] self.__add_wildcard(text_content, full_path, external_key_parts) diff --git a/pyproject.toml b/pyproject.toml index d0104f8..a7a803d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sd-webui-prompt-postprocessor" description = "Stable Diffusion WebUI & ComfyUI extension to post-process the prompt, including sending content from the prompt to the negative prompt and wildcards." -version = "2.12.0" +version = "2.12.1" license = { file = "LICENSE.txt" } dependencies = ["lark", "numpy", "pyyaml"] diff --git a/scripts/ppp_script.py b/scripts/ppp_script.py index cecfecd..c0f086b 100644 --- a/scripts/ppp_script.py +++ b/scripts/ppp_script.py @@ -297,7 +297,10 @@ class PromptPostProcessorA1111Script(scripts.Script): ] en_mappings_folders = getattr(opts, "ppp_en_mappingsfolders", "") if en_mappings_folders == "": - en_mappings_folders = os.getenv("EXTRANETWORKMAPPINGS_DIR", PPPExtraNetworkMappings.DEFAULT_ENMAPPINGS_FOLDER) + en_mappings_folders = os.getenv( + "EXTRANETWORKMAPPINGS_DIR", + PPPExtraNetworkMappings.DEFAULT_ENMAPPINGS_FOLDER, + ) enmappings_folders = [ (f if os.path.isabs(f) else os.path.abspath(os.path.join(models_path, f))) for f in en_mappings_folders.split(",") diff --git a/tests/tests.py b/tests/tests.py index bd50b1a..5112916 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -799,12 +799,12 @@ class TestPromptPostProcessor(TestPromptPostProcessorBase): def test_ch_choicesinsidelora(self): # simple choices inside a lora self.process( - PromptPair("", ""), - PromptPair("", ""), + PromptPair("", ""), + PromptPair("", ""), ppp=self.nocupppp, ) - def test_ch_removelorawithchoices(self): # remove lora with choices inside + def test_ch_removelorawithchoices(self): self.process( PromptPair("", ""), PromptPair("", ""), diff --git a/tests/wildcards2/text/wildcard3.txt b/tests/wildcards2/text/wildcard3.txt index 61b6647..67ed0e4 100644 --- a/tests/wildcards2/text/wildcard3.txt +++ b/tests/wildcards2/text/wildcard3.txt @@ -1,2 +1,3 @@ # wildcard3 -__2$$,$$text/wildcard2__ \ No newline at end of file +__2$$,$$text/wildcard2__ +0::áüç \ No newline at end of file