diff --git a/.gitignore b/.gitignore index 6bc46b7..cf3174e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ !.vscode/launch.json tests/tests_local.py +tests/logs diff --git a/docs/SYNTAX.md b/docs/SYNTAX.md index ea04600..309824d 100644 --- a/docs/SYNTAX.md +++ b/docs/SYNTAX.md @@ -66,12 +66,26 @@ The generic format is: `__parameters$$wildcard'filter'(var=value)__` The parameters, the filter, and the setting of a variable are optional. The parameters follow the same format as for the choices. -The wildcard identifier can have a relative path and contain globbing formatting, to read multiple wildcards and merge their choices. Note that if there are no parameters specified, the globbing will use the ones from the first wildcard that matches and have parameters (sorted by keys), so if you don't want that you might want to specify them. Also note that, unlike with *Dynamic Prompts*, the wildcard name has to be specified with its full path (unless you use globbing). You can use variables here, with the `${name:default}`, `` or `defaultecho>` formats, to build a dynamic identifier. +Wildcards cannot be used inside an extranetwork tag (because some lora names contain double underscores). If you need to choose from multiple loras put the whole extranetwork tag inside a wildcard, or use choices. -The filter can be used to filter specific choices from the wildcard. The filtering works before applying the choice conditions (if any). The surrounding quotes can be single or double. The filter is a comma separated list of an integer (positional choice index; zero-based) or choice label. You can also compound them with `+`. That is, the comma separated items act as an OR and the `+` inside them as an AND. Using labels can simplify the definitions of complex wildcards where you want to have direct access to specific choices on occasion (you don't need to create wildcards for each individual choice). There are some additional formats when using filters. You can specify `^wildcard` as a filter to use the filter of a previous wildcard in the chain. You can start the filter (regular or inherited) with `#` and it will not be applied to the current wildcard choices, but the filter will remain in memory to use by other descendant wildcards. You use `#` and `^` when you want to pass a filter to inner wildcards (see the test files). +### Identifier + +* Allowed characters are letters, numbers, underscore (`_`), dash (`-`), dot (`.`), and the path separators (`/` and `\`). It cannot start with an underscore because it would be ambiguous whether it's part of the name or just precedes the wildcard. +* Can have a relative path and contain globbing formatting, to read multiple wildcards and merge their choices. Note that if there are no parameters specified, the globbing will use the ones from the first wildcard that matches and have parameters (sorted by keys), so if you don't want that you might want to specify them. Also note that, unlike with *Dynamic Prompts*, the wildcard name has to be specified with its full path (unless you use globbing). +* You can use variables, with the `${name}`, `${name:default}`, `` or `defaultecho>` formats, to build a dynamic identifier. + +### Filter + +The filter can be used to filter specific choices from the wildcard. The filtering works before applying the choice conditions (if any). The surrounding quotes can be single or double. + +The filter is a comma separated list of an integer (positional choice index, zero-based) or choice label. You can also compound them with `+`. That is, the comma separated items act as an OR and the `+` inside them as an AND. Using labels can simplify the definitions of complex wildcards where you want to have direct access to specific choices on occasion (you don't need to create wildcards for each individual choice). There are some additional formats when using filters. You can specify `^wildcard` as a filter to use the filter of a previous wildcard in the chain. You can start the filter (regular or inherited) with `#` and it will not be applied to the current wildcard choices, but the filter will remain in memory to use by other descendant wildcards. You use `#` and `^` when you want to pass a filter to inner wildcards (see the test files). + +### Variable The variable value only applies during the evaluation of the selected choices and is discarded afterward (the variable keeps its original value if there was one). +### Examples + These are examples of formats you can use to insert a wildcard: | Construct | Result | @@ -87,8 +101,6 @@ These are examples of formats you can use to insert a wildcard: | `__2-3$$ / $$path/wildcard__` | select 2 to 3 choices with separator " / " | | `__path/wildcard(var=value)__` | select 1 choice using the specified variable value in the evaluation. | -Wildcards cannot be used inside an extranetwork tag (because some lora names contain double underscores). If you need to choose from multiple loras put the whole extranetwork tag inside a wildcard, or use choices. - ### Wildcard definitions A wildcard definition can be: diff --git a/grammar.lark b/grammar.lark index dc52537..7ae47a2 100644 --- a/grammar.lark +++ b/grammar.lark @@ -1,12 +1,13 @@ %import common (LETTER, DIGIT, INT, CNAME, SIGNED_NUMBER, NUMBER) -_WHITESPACE: /\s+/ -STRING: /("(?!"").*?(?${]|\\.)+/s // exclude only the starting ones @@ -142,7 +143,7 @@ varvalue: content_var // wildcards wildcard.2: "__" [ choicesoptions_sampler | ( choicesoptions _WHITESPACE? "$$" ) ] wildcard_name [ wc_filter ] [ wildcardvar ] "__" -wildcard_name: ( WC_NAME_PLAIN | variableuse | commandecho )+ +wildcard_name.2: ( WC_NAME_PLAIN_START | variableuse | commandecho ) ( WC_NAME_PLAIN | variableuse | commandecho )* wc_filter: /["']/ ( [ /#/ ] wc_filter_or | ( /#?\^/ wildcard_name ) ) /["']/ wc_filter_or: wc_filter_and ( _WHITESPACE? "," _WHITESPACE? wc_filter_and )* wc_filter_and: INDEX ( _WHITESPACE? "+" _WHITESPACE? INDEX )* diff --git a/ppp.py b/ppp.py index 1795899..655f739 100644 --- a/ppp.py +++ b/ppp.py @@ -1267,7 +1267,7 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in self.result += ":" self.__visit(after) self.__shell.pop() - if self.__ppp.cup_emptyconstructs and self.result == start_result + "[:": + if self.__ppp.cup_emptyconstructs and re.fullmatch(re.escape(start_result) + r"\[:\s*", self.result): self.result = start_result else: self.result += f":{pos_str}]" @@ -1294,7 +1294,7 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in self.__visit(opt) self.__shell.pop() self.result += "]" - if self.__ppp.cup_emptyconstructs and self.result == start_result + "[]": + if self.__ppp.cup_emptyconstructs and re.fullmatch(re.escape(start_result) + r"\[\s*\]", self.result): self.result = start_result # self.__shell.pop() t2 = time.monotonic_ns() @@ -1362,7 +1362,7 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in self.result += starttag self.__visit(current_tree) endtag = f":{weight_str})" - if self.__ppp.cup_emptyconstructs and self.result == start_result + starttag: + if self.__ppp.cup_emptyconstructs and re.fullmatch(re.escape(start_result + starttag) + r"\s*", self.result): self.result = start_result else: self.result += endtag diff --git a/ppp_logging.py b/ppp_logging.py index fa9d241..94c3768 100644 --- a/ppp_logging.py +++ b/ppp_logging.py @@ -53,13 +53,14 @@ class PromptPostProcessorLogFactory: # pylint: disable=too-few-public-methods colored_record.levelname = f"{seq}{levelname:8s}{self.COLORS['RESET']}" return super().format(colored_record) - def __init__(self, app: SUPPORTED_APPS = None): # pylint: disable=unused-argument + def __init__(self, filename = None, app: SUPPORTED_APPS = None): # pylint: disable=unused-argument """ Initializes the PromptPostProcessor class. This method sets up the logger for the PromptPostProcessor class and configures its log level and handlers. Args: + filename (str, optional): The name of the file to log to. Defaults to None. app (SUPPORTED_APPS): The application for which the logger is being created. Returns: @@ -71,6 +72,10 @@ class PromptPostProcessorLogFactory: # pylint: disable=too-few-public-methods handler = logging.StreamHandler(sys.stdout) handler.setFormatter(self.ColoredFormatter("%(asctime)s %(levelname)s %(message)s")) ppplog.addHandler(handler) + if filename is not None: + file_handler = logging.FileHandler(filename, encoding="utf-8") + file_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) + ppplog.addHandler(file_handler) ppplog.setLevel(logging.DEBUG) self.log = PromptPostProcessorLogCustomAdapter(ppplog) diff --git a/ppp_wildcards.py b/ppp_wildcards.py index c5692c0..e3989ef 100644 --- a/ppp_wildcards.py +++ b/ppp_wildcards.py @@ -66,7 +66,10 @@ class PPPWildcards: return self.wildcards.__sizeof__() + self.__wildcards_folders.__sizeof__() + self.__wildcard_files.__sizeof__() def refresh_wildcards( - self, debug_level: DEBUG_LEVEL, wildcards_folders: Optional[list[str]], wildcards_input: str = None + self, + debug_level: DEBUG_LEVEL, + wildcards_folders: Optional[list[str]], + wildcards_input: str = None, ): """ Initialize the wildcards. @@ -79,10 +82,19 @@ class PPPWildcards: for fullpath in list(self.__wildcard_files.keys()): if fullpath != self.LOCALINPUT_FILENAME: path = os.path.dirname(fullpath) - if not os.path.exists(fullpath) or not any( - os.path.commonpath([path, folder]) == folder for folder in self.__wildcards_folders - ): + if not os.path.exists(fullpath): self.__remove_wildcards_from_path(fullpath) + else: + a = False + for folder in self.__wildcards_folders: + try: + if os.path.commonpath([folder, path]) == folder: + a = True + break + except ValueError: + pass + if not a: + self.__remove_wildcards_from_path(fullpath) elif wildcards_input is None: self.__remove_wildcards_from_path(fullpath) if wildcards_folders is not None or wildcards_input is not None: @@ -365,6 +377,8 @@ class PPPWildcards: choices = self.__get_choices(obj, full_path, tmp_key_parts) if choices is None: self.__logger.warning(f"Invalid wildcard '{fullkey}' in file '{full_path}'!") + elif fullkey.startswith("_"): + self.__logger.warning(f"Invalid wildcard name '{fullkey}' in file '{full_path}'! (cannot start with underscore)") else: self.wildcards[fullkey] = PPPWildcard(full_path, fullkey, choices) return @@ -384,6 +398,8 @@ class PPPWildcards: choices = self.__get_choices(content, full_path, key_parts) if choices is None: self.__logger.warning(f"Invalid wildcard '{fullkey}' in file '{full_path}'!") + elif fullkey.startswith("_"): + self.__logger.warning(f"Invalid wildcard name '{fullkey}' in file '{full_path}'! (cannot start with underscore)") else: self.wildcards[fullkey] = PPPWildcard(full_path, fullkey, choices) diff --git a/tests/tests.py b/tests/tests.py index 71257b7..c184b7c 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -2,6 +2,7 @@ import os import logging from typing import NamedTuple, Optional import unittest +import datetime from ppp_enmappings import PPPExtraNetworkMappings # pylint: disable=import-error from ppp_wildcards import PPPWildcards # pylint: disable=import-error @@ -19,11 +20,23 @@ class TestPromptPostProcessorBase(unittest.TestCase): A test case class for testing the PromptPostProcessor class. """ - def setUp(self): + def setUp(self, enable_file_logging=False): """ Set up the test case by initializing the necessary objects and configurations. + + Args: + enable_file_logging (bool): Whether to enable logging to a file. Defaults to True. """ - self.lf = PromptPostProcessorLogFactory() + self.enable_file_logging = enable_file_logging + test_name = self.id().split(".")[-1] # Extract the test method name + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + + if self.enable_file_logging: + log_filename = f"tests/logs/{test_name}_{timestamp}.log" + else: + log_filename = None # Disable file logging + + self.lf = PromptPostProcessorLogFactory(log_filename, app=None) self.ppp_logger = self.lf.log self.ppp_logger.setLevel(logging.DEBUG) self.grammar_content = None @@ -183,6 +196,9 @@ class TestPromptPostProcessorBase(unittest.TestCase): class TestPromptPostProcessor(TestPromptPostProcessorBase): + def setUp(self): # pylint: disable=arguments-differ + super().setUp(enable_file_logging=False) + # Send To Negative tests def test_stn_simple(self): # negtags with different parameters and separations @@ -942,6 +958,13 @@ class TestPromptPostProcessor(TestPromptPostProcessorBase): ), ) + def test_wc_invalid_name(self): + self.process( + PromptPair("the choices are: ___invalid__", ""), + PromptPair("the choices are: ___invalid__", ""), + ppp=self.nocupppp, + ) + def test_wc_wildcard1a_text(self): # simple text wildcard self.process( PromptPair("the choices are: __text/wildcard1__", ""), @@ -1194,9 +1217,12 @@ class TestPromptPostProcessor(TestPromptPostProcessorBase): interrupted=True, ) - def test_wc_dynamicwildcard(self): # wildcard built from variables + def test_wc_dynamicwildcard(self): # wildcard built from variables self.process( - PromptPair("the choices are: ${x={1|2|3}}${w=yaml/wildcard${x}}__yaml/wildcard${x}__ __${w}__ ____", ""), + PromptPair( + "the choices are: ${x={1|2|3}}${w=yaml/wildcard${x}}__yaml/wildcard${x}__ __${w}__ ____", + "", + ), PromptPair("the choices are: choice1-choice3-choice1 choice3- choice2 - choice2 choice3", ""), ppp=self.nocupppp, ) diff --git a/tests/wildcards/_invalid.txt b/tests/wildcards/_invalid.txt new file mode 100644 index 0000000..ed8c6da --- /dev/null +++ b/tests/wildcards/_invalid.txt @@ -0,0 +1,3 @@ +# invalid wildcard name +choice1 +choice2 \ No newline at end of file