* Fix pyproject to make the registry publishing action work again.

* Fixed some merging configuration issues.

.github/instructions/docs.instructions.md

@@ -0,0 +1,42 @@
+---
+description: "Use when writing or editing documentation, README files, or any markdown content."
+applyTo: "**/*.md"
+---
+
+# Documentation Formatting
+
+## Software and WebUI Names
+
+All software names, WebUI names, and application names must be written in italic using `*name*` syntax — never plain text, bold, or code spans.
+
+Examples of names that must be italic:
+- *ComfyUI*, *Stable Diffusion WebUI*, *Forge*, *reForge*, *SD.Next*, *InvokeAI*, *Fooocus*, *AUTOMATIC1111*
+- Any other application, platform, or tool name referenced in the docs
+
+**Correct:**
+> This extension is compatible with *ComfyUI* and *Forge*.
+
+**Incorrect:**
+> This extension is compatible with ComfyUI and Forge.
+> This extension is compatible with **ComfyUI** and `Forge`.
+
+**Exceptions:**
+
+- Names inside code blocks (` ``` ` or `` ` `` ) are exempt — they follow code formatting rules.
+- When a software name is the visible text of a hyperlink (e.g., `[ComfyUI](https://...)`), plain text is acceptable.
+
+## Use of github alert blocks
+
+When adding notes, tips, warnings, or important information in the documentation, use GitHub's alert block syntax for better visibility and formatting. The syntax is as follows:
+```md
+> [!NOTE]
+> This is a note.
+> [!TIP]
+> This is a tip.
+> [!IMPORTANT]
+> This is important information.
+> [!WARNING]
+> This is a warning.
+> [!CAUTION]
+> This is a caution.
+```

.github/instructions/python.instructions.md

@@ -0,0 +1,47 @@
+---
+description: "Use when writing, editing, or reviewing Python code."
+applyTo: "**/*.py"
+---
+# Python code
+
+## Python 3.10 Compatibility
+
+All Python code must be compatible with Python 3.10. Do not use language features or standard-library additions introduced in 3.11 or later.
+
+### Forbidden (3.11+)
+
+| Avoid | Use instead |
+|-------|-------------|
+| `tomllib` (stdlib) | `tomli` (third-party) or conditional `try/except ImportError` |
+| `typing.Self` | `TypeVar("Self", bound="ClassName")` or `from __future__ import annotations` |
+| `typing.Never` | `typing.NoReturn` |
+| `typing.TypeVarTuple`, `typing.Unpack` | `typing_extensions` backports |
+| `StrEnum` | `class MyEnum(str, enum.Enum)` |
+| `except*` / `ExceptionGroup` | Not available; raise/catch normally |
+| `asyncio.TaskGroup`, `asyncio.timeout()` | `asyncio.gather()` / `asyncio.wait_for()` |
+
+### Forbidden (3.12+)
+
+| Avoid | Use instead |
+|-------|-------------|
+| `type X = ...` (type alias statement) | `X: TypeAlias = ...` with `typing.TypeAlias` |
+| `def f[T](...)` / `class C[T]` generic syntax | `TypeVar` + `Generic[T]` |
+| `@typing.override` | Omit or use comment |
+| `itertools.batched()` | Manual chunking or `more-itertools` |
+
+### Safe to use (available in 3.10)
+
+- `match`/`case` structural pattern matching
+- `X | Y` union type syntax in annotations (e.g., `int | None`)
+- `typing.TypeAlias`, `typing.ParamSpec`
+- `list[int]`, `dict[str, int]` — built-in generic aliases
+- `zip(..., strict=True)`
+- `str.removeprefix()` / `str.removesuffix()`
+
+## Paths
+
+Use `pathlib.Path` for filesystem paths instead of `str` paths or `os.path`.
+
+## Comments
+
+Do not use emdashes (—) in comments. Use a single dash (-) or parentheses instead. Also avoid any other typographical punctuation that is not basic ASCII.

.github/instructions/tests.instructions.md

@@ -0,0 +1,129 @@
+---
+description: "Use when writing, editing, or reviewing test files. Enforces unittest patterns used in this project: TestCase inheritance, base class setup, process() helper, and naming conventions."
+applyTo: "tests/**/*.py"
+---
+# Test Conventions
+
+All tests use the standard `unittest` module. Do **not** use pytest-specific features (fixtures, `@pytest.mark`, `conftest.py`, etc.).
+
+## File & Class Naming
+
+- One test file per concern, named `tests_{category}.py`
+- One test class per file inheriting from `TestPromptPostProcessorBase`:
+
+```python
+import unittest
+from tests.base_tests import TestPromptPostProcessorBase
+
+class TestMyFeature(TestPromptPostProcessorBase):
+
+    def setUp(self):
+        super().setUp(enable_file_logging=False)
+```
+
+## Test Method Naming
+
+```
+test_{2-3 letter category prefix}_{short_description}
+```
+
+Examples: `test_cl_simple`, `test_ch_choices`, `test_wc_ignore`
+
+## Running a Test Case via `self.process()`
+
+Use the `process()` helper from the base class — never instantiate `PromptPostProcessor` directly in test methods unless there is a need to pass specific options not covered by the default setup or the "nocup" or "nostrict" options.
+
+```python
+def test_cl_simple(self):
+    """simple cleanup"""
+    self.process(
+        InputTuple(
+            "input prompt",                         # positive prompt
+            ""),                                    # negative prompt
+        OutputTuple(
+            "expected output",                      # expected positive prompt
+            "",                                     # expected negative prompt
+            {}                                      # expected variables (optional)
+        ),
+    )
+
+def test_cl_combinatorial(self):
+    """simple cleanup"""
+    self.process(
+        InputTuple(
+            "input prompt",                         # positive prompt
+            ""),                                    # negative prompt
+        [
+            OutputTuple(
+                "expected output",                  # expected positive prompt
+                "",                                 # expected negative prompt
+                {}                                  # expected variables (optional)
+            ),
+            OutputTuple(
+                "expected output",                  # expected positive prompt
+                "",                                 # expected negative prompt
+                {}                                  # expected variables (optional)
+            ),
+        ],
+        combinatorial=True,
+    )
+
+```
+
+### `process()` Signature (key parameters)
+
+| Parameter | Type | Notes |
+|-----------|------|-------|
+| `input_prompts` | `InputTuple` | Prompts input |
+| `expected_output` | `OutputTuple \| list[OutputTuple]` | Single or multiple valid outputs |
+| `seed` | `int` | Optional, defaults to fixed seed |
+| `ppp` | `PromptPostProcessor \| str \| None` | Supported values `"nocup"`, `"nostrict"` or a specific instance |
+| `interrupted` | `bool` | Expected interrupt flag |
+| `combinatorial` | `bool` | Whether to run a combinatorial generation. If a specific ppp instance is used then it is ignored |
+
+## Assertions
+
+Use `assertEqual` or similar methods with a descriptive message string:
+
+```python
+self.assertEqual(result, expected, "Descriptive failure message")
+self.assertIn(key, container, f"Key '{key}' not found in output")
+```
+
+Do not use bare `assert` statements.
+
+## Default Options & Environment
+
+Override `self.defopts` or `self.def_env_info` to pass non-default options — do not hardcode option dicts from scratch:
+
+```python
+def test_cl_custom(self):
+    """cleanup with custom separator"""
+    self.process(
+        InputTuple("a, , b", ""),
+        OutputTuple("a | b", ""),
+        ppp=PromptPostProcessor(
+            self.ppp_logger,
+            self.def_env_info,
+            replace(
+                self.defopts,
+                keep_choices_order=True,
+                cup_do_cleanup=False,
+                do_combinatorial=True,
+            ),
+            self.grammar_content,
+            self.interrupt,
+            self.wildcards_obj,
+            self.extranetwork_maps_obj,
+        ),
+    )
+```
+
+## Entry Point
+
+Every test file must start with:
+
+```python
+if __name__ == "__main__":
+    raise SystemExit("This script must not be run directly")
+```

.github/workflows/publish.yml

@@ -0,0 +1,26 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - master
+    paths:
+      - "pyproject.toml"
+
+permissions:
+  issues: write
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    if: ${{ github.repository_owner == 'acorderob' }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@v1
+        with:
+          ## Add your own personal access token to your Github Repository secrets and reference it here.
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

.gitignore

@@ -1,5 +1,10 @@
 **/__pycache__
+.venv
+venv
 
 .vscode/**/*
-!.vscode/settings.json
-!.vscode/launch.json
+
+logs
+tests/tests_local.py
+tests/local_wildcards
+tests/logs

.markdownlint.jsonc

@@ -0,0 +1,9 @@
+{
+    "default": true,
+    "MD013": false,
+    "MD024": false,
+    "MD033": false,
+    "MD060": {
+        "style": "any"
+    }
+}

.pylintrc

@@ -59,16 +59,6 @@ single-line-class-stmt=no
 single-line-if-stmt=no
 
 [IMPORTS]
-allow-any-import-level=
-allow-reexport-from-package=no
-allow-wildcard-with-all=no
-deprecated-modules=
-ext-import-graph=
-import-graph=
-int-import-graph=
-known-standard-library=
-known-third-party=enchant
-preferred-modules=
 
 [LOGGING]
 logging-format-style=new
@@ -94,9 +84,7 @@ disable=raw-checker-failed,
         missing-module-docstring,
         missing-class-docstring,
         logging-fstring-interpolation,
-        import-outside-toplevel,
         consider-iterating-dictionary,
-        wrong-import-position,
         unnecessary-lambda,
         consider-using-dict-items,
         dangerous-default-value,

.python-version

@@ -0,0 +1 @@
+3.10.11

.vscode/launch.json

@@ -1,16 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Tests",
-            "type": "python",
-            "request": "launch",
-            "program": "tests/tests.py",
-            "console": "integratedTerminal",
-            "justMyCode": true
-        }
-    ]
-}

.vscode/settings.json

@@ -1,16 +0,0 @@
-{
-    "python.analysis.extraPaths": ["../.."],
-    "python.testing.unittestArgs": [
-        "-v",
-        "-s",
-        "./tests",
-        "-p",
-        "test*.py"
-    ],
-    "python.testing.pytestEnabled": false,
-    "python.testing.unittestEnabled": true,
-    "python.analysis.typeCheckingMode": "basic",
-    "black-formatter.args": [
-        "--line-length=120"
-    ]
-}

README.md

@@ -1,110 +1,95 @@
-# Send to Negative for Stable Diffusion WebUI
+# Prompt PostProcessor for Stable Diffusion WebUI and ComfyUI
 
-Extension for the [AUTOMATIC1111 Stable Diffusion WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) or compatible UIs.
+The Prompt PostProcessor (or PPP, formerly known as "sd-webui-sendtonegative"), is an extension designed to process the prompt in several ways. This extension is compatible with:
 
-## Purpose
+* [ComfyUI](https://github.com/comfyanonymous/ComfyUI)
+* [Forge Neo](https://github.com/Haoming02/sd-webui-forge-classic/tree/neo)
+* [SD.Next](https://github.com/vladmandic/sdnext)
+* [Forge Classic](https://github.com/Haoming02/sd-webui-forge-classic/tree/classic)
+* [Forge](https://github.com/lllyasviel/stable-diffusion-webui-forge)
+* [reForge](https://github.com/Panchovix/stable-diffusion-webui-reForge)
+* [AUTOMATIC1111 Stable Diffusion WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui)
+* ...and probably other forks
 
-This extension allows the tagging of parts of the prompt and moves them to the
-negative prompt. This allows useful tricks when using a wildcard extension
-since you can add negative content from choices made in the positive prompt.
+These are some features:
 
-Note: The extension must be loaded after the installed wildcards extension. Extensions
-load by their folder in alphanumeric order.
+* Process wildcards. Compatible with the Dynamic Prompts extension.
+* Sending parts of the prompt to the negative prompt (the original function of the extension). This allows for useful tricks when using wildcards since you can add negative content from choices made in the positive prompt.
+* Define model variants (pony, illustrious, ...) and have them set as system variables along with the current model, to detect which one is being used.
+* Set and modify user variables.
+* Set content based on conditions (like the system/user variables).
+* Map extranetworks (LoRAs) depending on conditions (like the loaded model/variant). This allows you to add "virtual" LoRAs to the prompt that will be translated to the correct one.
+* Clean up the prompt of unnecessary separators or spaces.
+* Combinatorial mode.
 
-With the ["Dynamic Prompts" extension](https://github.com/adieyal/sd-dynamic-prompts)
-this happens by default due to default folder names for both extensions. But if
-this is not the case, you can just rename the extension folder so the ordering
-works out.
+> [!NOTE]
+> When used in an *A1111* compatible webui, the extension must be loaded after any other extension that modifies the prompt (like another wildcards extension).
+>
+> Usually extensions load by their folder name in alphanumeric order, so if the extensions are not loading in the correct order just rename this extension's folder so the ordering works out.
+>
+> When in doubt, just rename this extension's folder with a "z" in front (for example) so that it is the last one to load, or manually set such folder name when installing it.
 
-With the ["AUTOMATIC1111 Wildcards" extension](https://github.com/AUTOMATIC1111/stable-diffusion-webui-wildcards)
-you will have to rename one of the folders, so that it loads before than "Send to Negative".
+If the extension runs before others, like Dynamic Prompts, and the "Process wildcards" option is enabled, the wildcards will be processed by PPP and those extensions will not get them. If you disable processing the wildcards, and you intend another extension to process them, you should keep the "What to do with remaining wildcards?" option as "ignore".
 
-When in doubt, just rename this extension's folder with a "z" in front (for example) so that it is the last one to load, or manually set such folder name when installing it.
+Notes:
 
-Note: it only recognizes regular A1111 prompt formats. So:
+1. Other than its own commands, it only recognizes regular *A1111* prompt formats. So:
 
-* Attention: \[prompt\] (prompt) (prompt:weight)
-* Alternation: \[prompt1|prompt2|...\]
-* Scheduling: \[prompt1:prompt2:step\]
+    * **Extra networks**: `<kind:model...>`
+    * **Attention**: `[prompt] (prompt) (prompt:weight)`
+    * **Alternation**: `[prompt1|prompt2|...]`
+    * **Scheduling**: `[prompt1:prompt2:step]`
+    * **BREAK**: `prompt1 BREAK prompt2`
+    * **Composable Diffusion**: `prompt1:weight1 AND prompt2:weight2`
 
-In SD.Next that means only the A1111 or Full parsers.
+    *ComfyUI* only supports natively the attention using parentheses, so the ones with the brackets will be converted. The other constructs are not natively supported but some custom nodes implement them. For example:
+        [ComfyUI_smZNodes](https://github.com/shiimizu/ComfyUI_smZNodes)
 
-It does not build AND/BREAK separations into the negative prompt.
+    I also recommend a node to add LoRAs from the prompt, like:
+        [LoRA Tag Loader for ComfyUI](https://github.com/badjeff/comfyui_lora_tag_loader)
+
+    In *SD.Next* that means only the *A1111* or *Full* parsers. It will warn you if you use the *Compel* parser.
+    Does not recognize tokenizer separators like `TE2:` and `TE3:`, so sending to negative prompt from those sections of the prompt will not add them in the corresponding section of the negative prompt.
+
+2. It recognizes wildcards in the `__wildcard__` and {choice|choice} formats (and almost everything that [Dynamic Prompts](https://github.com/adieyal/sd-dynamic-prompts) supports).
+3. It does not create *AND/BREAK* constructs when moving content to the negative prompt.
 
 ## Installation
 
+On *ComfyUI*:
+
+1. Go to Manager > Custom Nodes Manager
+2. Search for "Prompt PostProcessor" and install or click Install via Git URL and enter <https://github.com/acorderob/sd-webui-prompt-postprocessor>
+3. Restart
+
+On *A1111* compatible webuis:
+
 1. Go to Extensions > Install from URL
-2. Paste <https://github.com/acorderob/sd-webui-sendtonegative> in the URL for extension's git repository text field
+2. Paste <https://github.com/acorderob/sd-webui-prompt-postprocessor> in the URL for extension's git repository text field
 3. Click the Install button
 4. Restart the webui
 
-## Usage
-
-The format of the tags is like this:
-
-```text
-<!content!>
-```
-
-And an optional position in the negative prompt can be specified like this:
-
-```text
-<!!position!content!>
-```
-
-Where position can be:
-
-* s: at the start (the default)
-* e: at the end
-* pN: at the position of the insertion point "<!!iN!!>" with N being 0-9
-
-If the insertion point is not found it inserts at the start.
-
-## Example
-
-You have a wildcard for hair colors (\_\_haircolors\_\_) with one being
-strawberry blonde, but you don't want strawberries. So in that option you add a
-tag to add to the negative prompt, like so:
-
-```text
-blonde
-strawberry blonde <!strawberry!>
-brunette
-```
-
-Then, if that option is chosen this extension will process it later and move
-that part to the negative prompt.
+On *SD.Next* I recommend you to disable the native wildcard processing and use the old UI.
 
 ## Configuration
 
-Separator used when adding to the negative prompt: You can specify the separator used when adding to the negative prompt (by default it's ", ").
+See the [configuration documentation](docs/CONFIG.md).
 
-Ignore tags with repeated content: by default it ignores repeated content to avoid repetitions in the negative prompt.
+## Usage
 
-Join attention modifiers (weights) when possible: by default it joins attention modifiers when possible (joins into one, multipliying their values).
+See the [syntax documentation](docs/SYNTAX.md).
 
-Try to clean-up the prompt after processing: by default cleans up the positive prompt after processing, removing extra spaces and separators.
+## Cookbook
 
-## Notes
+See the [cookbook](docs/COOKBOOK.md) for interesting usages.
 
-The content of the negative tags is not processed and is copied as is to the negative prompt. Other modifiers around the tags are processed in the following way.
+## Tools
 
-### Attention modifiers (weights)
+A tool `tools/convert_styles.py` exists to convert A1111 or SD.Next styles into a wildcards file.
 
-They will be translated to the negative prompt. For example:
+## Contributing
 
-* `(red<!square!>:1.5)` will end up as `(square:1.5)` in the negative prompt
-* `(red[<!square!>]:1.5)` will end up as `(square:1.35)` in the negative prompt (weight=1.5*0.9)
-* However `(red<![square]!>:1.5)` will end up as `([square]:1.5)` in the negative prompt. The content of the negative tag is copied as is, and not joined with the surrounding modifier.
-
-### Prompt editing constructs (alternation and scheduling)
-
-Negative tags inside such constructs will copy the construct to the negative prompt, but separating its elements. For example:
-
-* Alternation: `[red<!square!>|blue<!circle!>]` will end up as `[square|], [|circle]` in the negative prompt, instead of `[square|circle]`
-* Scheduling: `[red<!square!>:blue<!circle!>:0.5]` will end up as `[square::0.5], [:circle:0.5]` instead of `[square:circle:0.5]`
-
-This should still work as intended, and the only negative point i see is the unnecessary separators.
+To develop, I suggest doing so with the extension isolated from the UI (you can use a symlink to test it in the UI), and with its own virtual environment (venv or .venv), so the tests work and can be debugged properly.
 
 ## License
 
@@ -112,4 +97,4 @@ MIT
 
 ## Contact
 
-If you have any questions or concerns, please leave an issue, or start a thread in the discussions.
+If you have any questions or concerns, please start a thread in the discussions. For bug reports and feature requests open an issue.

__init__.py

@@ -0,0 +1,43 @@
+"""
+@author: ACB
+@title: Prompt Post Processor
+@nickname: ACB PPP
+@description: Node for processing prompts. Includes the following options: send to negative prompt, set variables, if/elif/else command for conditional content, wildcards and choices.
+"""
+
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).resolve().parent))
+
+from .ppp_comfyui import (
+    PromptPostProcessorComfyUINode,
+    PromptPostProcessorWildcardOptionsComfyUINode,
+    PromptPostProcessorENMappingOptionsComfyUINode,
+    PromptPostProcessorSTNOptionsComfyUINode,
+    PromptPostProcessorCleanupOptionsComfyUINode,
+    PromptPostProcessorSelectVariableComfyUINode,
+    PromptPostProcessorWildcardConcatComfyUINode,
+)
+
+NODE_CLASS_MAPPINGS = {
+    "ACBPromptPostProcessor": PromptPostProcessorComfyUINode,
+    "ACBPPPWildcardOptions": PromptPostProcessorWildcardOptionsComfyUINode,
+    "ACBPPPENMappingOptions": PromptPostProcessorENMappingOptionsComfyUINode,
+    "ACBPPPSendToNegativeOptions": PromptPostProcessorSTNOptionsComfyUINode,
+    "ACBPPPCleanupOptions": PromptPostProcessorCleanupOptionsComfyUINode,
+    "ACBPPPSelectVariable": PromptPostProcessorSelectVariableComfyUINode,
+    "ACBPPPWildcardConcat": PromptPostProcessorWildcardConcatComfyUINode,
+}
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "ACBPromptPostProcessor": "ACB Prompt Post Processor",
+    "ACBPPPWildcardOptions": "ACB PPP Wildcard Options",
+    "ACBPPPENMappingOptions": "ACB PPP ExtraNetwork Mapping Options",
+    "ACBPPPSendToNegativeOptions": "ACB PPP Send-To-Negative Options",
+    "ACBPPPCleanupOptions": "ACB PPP Cleanup Options",
+    "ACBPPPSelectVariable": "ACB PPP Select Variable",
+    "ACBPPPWildcardConcat": "ACB PPP Wildcard Concat",
+}
+WEB_DIRECTORY = "./web"
+
+__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]

dev/compare_models.py

@@ -0,0 +1,314 @@
+"""
+Compares the model classes listed in the host's supported models file against
+the model definitions in ppp_config.yaml.defaults for a given host.
+
+The relative path to the supported models file for each host is read from the
+comments preceding the `models:` key in the defaults file.
+"""
+
+import argparse
+import ast
+import re
+import sys
+from pathlib import Path
+
+import yaml
+
+DEFAULT_CONFIG = Path(__file__).parent.parent / "ppp_config.yaml.defaults"
+SUPPORTED_HOSTS = {"comfyui", "reforge", "forge", "forgeneo", "sdnext"}
+
+
+def parse_host_file_paths(config_text: str) -> dict[str, str]:
+    """Extract the host->relative-path mapping from the comments above `models:`."""
+    paths: dict[str, str] = {}
+    in_block = False
+
+    for line in config_text.splitlines():
+        stripped = line.strip()
+
+        if re.match(r"^#\s*Check supported models for each host in:", stripped):
+            in_block = True
+            continue
+
+        if in_block:
+            # Lines like: "#   host: some/relative/path.py"  or "#   host:"
+            m = re.match(r"^#\s*(\w+)\s*:\s*(.*)", stripped)
+            if m:
+                host, rel_path = m.group(1), m.group(2).strip()
+                if rel_path:
+                    paths[host] = rel_path
+            else:
+                # First non-matching line ends the block
+                if not stripped.startswith("#"):
+                    break
+
+    return paths
+
+
+def extract_pipeline_classes(shared_items_path: Path) -> list[tuple[str, None]]:
+    """Parse shared_items.py and return unique diffusers pipeline class names from the pipelines dict."""
+    source = shared_items_path.read_text(encoding="utf-8")
+    tree = ast.parse(source, filename=str(shared_items_path))
+
+    class_names: set[str] = set()
+
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Assign):
+            continue
+        if not any(isinstance(t, ast.Name) and t.id == "pipelines" for t in node.targets):
+            continue
+        if not isinstance(node.value, ast.Dict):
+            continue
+        for value in node.value.values:
+            # Match: getattr(diffusers, 'ClassName', None)
+            if not isinstance(value, ast.Call):
+                continue
+            if not (isinstance(value.func, ast.Name) and value.func.id == "getattr"):
+                continue
+            if len(value.args) < 2:
+                continue
+            cls_arg = value.args[1]
+            if isinstance(cls_arg, ast.Constant) and isinstance(cls_arg.value, str):
+                class_names.add(cls_arg.value)
+
+    return [(name, None) for name in sorted(class_names)]
+
+
+def extract_model_classes(supported_models_path: Path) -> list[tuple[str, str | None]]:
+    """Parse a supported_models.py and return (class_name, parent_class) pairs in the `models` list(s)."""
+    source = supported_models_path.read_text(encoding="utf-8")
+    tree = ast.parse(source, filename=str(supported_models_path))
+
+    class_parents = _build_class_parents(tree)
+    class_names: list[str] = []
+
+    for node in ast.walk(tree):
+        # models = [ClassA, ClassB, ...]
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                if isinstance(target, ast.Name) and target.id == "models":
+                    class_names.extend(_names_from_list(node.value))
+
+        # models += [ClassA, ...]
+        elif isinstance(node, ast.AugAssign):
+            if isinstance(node.target, ast.Name) and node.target.id == "models":
+                class_names.extend(_names_from_list(node.value))
+
+    sentinels = _find_sentinels(class_parents, set(class_names))
+
+    # Only keep classes that ultimately descend from a sentinel base.
+    # Display parent is None when the immediate parent is a sentinel (class appears as a root).
+    return [
+        (name, None if class_parents.get(name) in sentinels else class_parents.get(name))
+        for name in class_names
+        if _has_base_ancestor(name, class_parents, sentinels)
+    ]
+
+
+def _build_class_parents(tree: ast.AST) -> dict[str, str | None]:
+    """Return a mapping of class name -> raw parent name."""
+    parents: dict[str, str | None] = {}
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.ClassDef):
+            continue
+        parent: str | None = None
+        for base in node.bases:
+            if isinstance(base, ast.Attribute) and isinstance(base.value, ast.Name):
+                parent = f"{base.value.id}.{base.attr}"
+            elif isinstance(base, ast.Name):
+                parent = base.id
+            break  # only the first base matters
+        parents[node.name] = parent
+    return parents
+
+
+def _find_sentinels(parents: dict[str, str | None], model_class_names: set[str]) -> set[str]:
+    """Detect root base class names: external refs (dotted) or local classes with no parent not in the models list."""
+    sentinels: set[str] = set()
+    for parent in parents.values():
+        if parent is None:
+            continue
+        if "." in parent:  # external reference, e.g. supported_models_base.BASE
+            sentinels.add(parent)
+        elif parent in parents and parents[parent] is None and parent not in model_class_names:
+            sentinels.add(parent)  # local class with no parent that is not itself a listed model
+    return sentinels
+
+
+def _has_base_ancestor(name: str, parents: dict[str, str | None], sentinels: set[str]) -> bool:
+    visited: set[str] = set()
+    current = parents.get(name)
+    while current is not None:
+        if current in sentinels:
+            return True
+        if current in visited:
+            return False  # cycle guard
+        visited.add(current)
+        current = parents.get(current)
+    return False
+
+
+def _names_from_list(node: ast.expr) -> list[str]:
+    if not isinstance(node, ast.List):
+        return []
+    return [elt.id for elt in node.elts if isinstance(elt, ast.Name)]
+
+
+def _topo_sort_alpha(classes: list[tuple[str, str | None]]) -> list[tuple[str, str | None]]:
+    """Sort classes so each parent immediately precedes its children, with alphabetical ordering at every level."""
+    class_set = {name for name, _ in classes}
+    parent_of = {name: parent for name, parent in classes}
+
+    children_of: dict[str, list[str]] = {name: [] for name, _ in classes}
+    roots: list[str] = []
+    for name, parent in classes:
+        if parent and parent in class_set:
+            children_of[parent].append(name)
+        else:
+            roots.append(name)
+
+    roots.sort()
+    for children in children_of.values():
+        children.sort()
+
+    result: list[tuple[str, str | None]] = []
+
+    def visit(name: str) -> None:
+        result.append((name, parent_of[name]))
+        for child in children_of[name]:
+            visit(child)
+
+    for root in roots:
+        visit(root)
+
+    return result
+
+
+def build_class_to_model_map(config: dict, host: str) -> dict[str, str]:
+    """Return a mapping of class name -> model key for the given host."""
+    mapping: dict[str, str] = {}
+    models_config = config.get("models", {})
+
+    for model_key, model_data in models_config.items():
+        if not isinstance(model_data, dict):
+            continue
+        detect = model_data.get("detect", {})
+        if not isinstance(detect, dict):
+            continue
+        host_detect = detect.get(host)
+        if not isinstance(host_detect, dict):
+            continue
+        classes = host_detect.get("class", [])
+        if isinstance(classes, list):
+            for cls in classes:
+                mapping[cls] = model_key
+
+    return mapping
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Compare host model classes against ppp_config.yaml.defaults mappings."
+    )
+    parser.add_argument(
+        "host",
+        help="Host kind to compare against (e.g. comfyui, reforge, forge, ...).",
+    )
+    parser.add_argument(
+        "root",
+        metavar="ROOT_FOLDER",
+        type=Path,
+        help="Root folder of the host UI installation.",
+    )
+    parser.add_argument(
+        "--config",
+        metavar="CONFIG_FILE",
+        type=Path,
+        default=DEFAULT_CONFIG,
+        help=f"Path to ppp_config.yaml.defaults (default: {DEFAULT_CONFIG}).",
+    )
+    args = parser.parse_args()
+
+    root: Path = args.root
+    host: str = args.host
+    config_path: Path = args.config
+
+    if host not in SUPPORTED_HOSTS:
+        parser.error(f"Host '{host}' is not supported by this script. Supported hosts: {', '.join(sorted(SUPPORTED_HOSTS))}")
+
+    if not root.is_dir():
+        parser.error(f"ROOT_FOLDER does not exist or is not a directory: {root}")
+
+    if not config_path.exists():
+        parser.error(f"Config file not found: {config_path}")
+
+    config_text = config_path.read_text(encoding="utf-8")
+    host_paths = parse_host_file_paths(config_text)
+
+    known_hosts = list((yaml.safe_load(config_text).get("hosts") or {}).keys())
+    if host not in known_hosts:
+        print(f"Warning: '{host}' is not a recognized host. Known hosts: {', '.join(known_hosts)}")
+
+    rel_path = host_paths.get(host)
+    if not rel_path:
+        print(f"Error: no supported-models file path defined for host '{host}' in {config_path.name}.")
+        sys.exit(1)
+
+    supported_models_path = root / rel_path.replace("\\", "/")
+    if not supported_models_path.exists():
+        print(f"Error: file not found: {supported_models_path}")
+        sys.exit(1)
+
+    config = yaml.safe_load(config_text)
+    if host == "sdnext":
+        model_classes: list[tuple[str, str | None]] = extract_pipeline_classes(supported_models_path)
+    else:
+        model_classes = extract_model_classes(supported_models_path)
+    if not model_classes:
+        print("No model classes found in the models list.")
+        sys.exit(1)
+
+    model_classes = _topo_sort_alpha(model_classes)
+
+    class_to_model = build_class_to_model_map(config, host)
+
+    class_set = {cls for cls, _ in model_classes}
+    parent_of_display = {cls: parent for cls, parent in model_classes}
+    depth_cache: dict[str, int] = {}
+
+    def get_depth(name: str) -> int:
+        if name not in depth_cache:
+            p = parent_of_display.get(name)
+            depth_cache[name] = 0 if (not p or p not in class_set) else 1 + get_depth(p)
+        return depth_cache[name]
+
+    # Build display labels: "ClassName" or "ClassName (Parent)", indented by depth
+    labels = [f"{cls} ({parent})" if parent else cls for cls, parent in model_classes]
+    depths = [get_depth(cls) for cls, _ in model_classes]
+    col_width = max(d * 2 + len(lbl) for d, lbl in zip(depths, labels))
+    missing: list[str] = []
+
+    print(f"Model classes in '{supported_models_path}' vs '{config_path.name}' (host: {host})")
+    print("-" * (col_width + 42))
+
+    for (cls, _parent), label, depth in zip(model_classes, labels, depths):
+        indented = "  " * depth + label
+        model = class_to_model.get(cls)
+        if model:
+            print(f"{indented:<{col_width}}  ->  {model}")
+        else:
+            missing.append((cls, _parent))
+            print(f"{indented:<{col_width}}  ->  WARNING: not mapped")
+
+    print("-" * (col_width + 42))
+    print(f"Total: {len(model_classes)} classes, {len(missing)} unmapped")
+
+    if missing:
+        print("\nUnmapped classes:")
+        for cls, parent in missing:
+            label = f"{cls} ({parent})" if parent else cls
+            print(f"  - {label}")
+
+
+if __name__ == "__main__":
+    main()

docs/CONFIG.md

@@ -0,0 +1,196 @@
+# Prompt PostProcessor configuration
+
+## Configuration file
+
+The extension supports a configuration file `ppp_config.yaml` with some settings that don't usually change.
+
+By default this configuration is read from the file `ppp_config.yaml.defaults` in the extension folder. That file must not be modified. If you want to personalize any settings you should first copy it as `ppp_config.yaml` in the same folder, or in the *ComfyUI* user folder (preferred, but only possible in *ComfyUI*). The options in this new file will take precedence over those in the defaults file.
+
+This file contains some options for how the host applications (WebUIs) should act in certain operations, and also defines the supported models, including how to detect them and the model variants definitions. Host names are fixed values (those supported by the extension). The defaults file contains comments to explain the available options.
+
+The model variants now support regular expressions instead of a list of strings to detect the variant. If you used a non default value in previous versions you should create a configuration file and add them with the new format. As before, the default file defines variants for *Pony* and *Illustrious* models.
+
+## Important
+
+**Beware of the combinatorial mode with no limits**. Even very few choice/wildcard constructs can cause a *combinatorial explosion*!
+
+The console log can help you determine the number of combinations that it is trying to generate. There will be an **"Estimated combinations"** message that shows an estimate. You can try first with a limit of 1, then check this message in the log. But note that it is a lower bound estimate, and there could be more combinations.
+
+## ComfyUI
+
+### ACB Prompt Post Processor node
+
+The main node that processes the prompt.
+
+Inputs:
+
+* **model**: Connect here the MODEL or a string with the model class name used by *ComfyUI*. Needed for the model kind system variables. Optional if you set modelname.
+* **modelname**: Filename of the model (with relative path). Needed for the detection of model variants.
+* **seed**: Set or connect here the seed used. By default it is -1 (random). The actual value used can be extracted from the output variables (`_input_seed`).
+* **pos_prompt**: Connect here the prompt text, or fill it as a widget.
+* **neg_prompt**: Connect here the negative prompt text, or fill it as a widget.
+* **debug_level**: What to write to the console.
+* **on_warnings**: Warn on the console or stop the generation.
+* **strict_mode**: Sets the strict mode in comparison operations.
+* **process_wildcards**: Activates the wildcard processing.
+* **do_cleanup**: Activates the cleanup processing.
+* **cleanup_variables**: Do a cleanup of the output variables (depends on do_cleanup).
+* **do_combinatorial**: Activates combinatorial mode, where the output are all the combinations of choices/wildcards of the prompt.
+* **combinatorial_shuffle**: It shuffles the combinatorial results.
+* **combinatorial_limit**: Limit for the number of generated combinations.
+* **wc_options**: Connection to a Wildcards options node.
+* **stn_options**: Connection to a Send-To-Negative options node.
+* **cup_options**: Connection to a Cleanup options node.
+* **en_options**: Connection to a ExtraNetworkMapping options node.
+* **results_file**: Filename to save processing results. Supports `%datetime%`, `%date%`, `%time%`, and `%host%` tokens. The file extension determines the format: `.yaml`/`.yml`, `.jsonl`, `.csv`, or plain text for any other extension. Relative paths are resolved against the extension's `logs` folder. Leave empty to disable.
+
+The options nodes are optional. If you don't need to change any of the default values then you don't need to use them.
+
+The model and modelname are also optional, but if you don't set them you will not be capable of choosing content based on the model type or variant. Native model loader nodes do not output the filename, but there are custom nodes that do (like those from [ComfyUI Image Saver](https://github.com/alexopus/ComfyUI-Image-Saver)).
+
+Setting only the modelname will try to detect its class from the file contents. If you don't want to set the path twice (and you don't use a loader node that outputs the name), you can set it here and then extract it from the output variables (`_modelfullname`) to send to the loader node.
+
+You can instead set it from the prompt (with variable `_modelfullname`) and load the model afterwards.
+
+Outputs:
+
+* **pos_prompt**: Resulting positive prompt.
+* **neg_prompt**: Resulting negative prompt.
+* **variables**: Resulting output variables.
+
+The outputs are lists, and in combinatorial mode there will be multiple elements that *ComfyUI* will process sequentially.
+
+### ACB PPP Select Variable node
+
+Lets you extract the variables used from the output (or just one of them). You can use this to send only part of the prompt to, for example, a detailer node. For example:
+
+With this prompt:
+
+`__quality__, 1girl, ${head:__eyes__, __hair__, __expression__}, __body__, __clothes__, __background__, __style__`
+
+You can then extract the `head` variable and use it as prompt for the head/face detailer.
+
+Inputs:
+
+* **variables**: the variables dictionary from the main node output.
+* **name**: optional name of a variable.
+
+Output:
+
+* **value**: the resulting content, either all the variables (one per line, in "name: value" format) or just the content of the chosen one.
+
+### ACB PPP Wildcards Concat node
+
+This node lets you select up to 10 wildcards that will be concatenated with a chosen separator. You can't specify wildcard folders in the node, so use the **extra_model_paths.yaml** options (see node below) to set them.
+
+Inputs:
+
+* **previous_prompt**: An optional text that will be prepended to the wildcards. Lets you chain multiple nodes or other string nodes.
+* **filter**: a string to filter the identifiers of the wildcards. It matches the start of the identifiers.
+* **separator**: a separator string.
+* **wildcard_<n>**: the wildcards to concatenate.
+
+Output:
+
+* **prompt**: concatenated result.
+
+### ACB PPP Wildcard Options node
+
+Options for wildcard processing, in case you want to change them from the defaults.
+
+* **folders**: You can enter multiple folders separated by commas. You can leave it empty (the default) and add a `ppp_wildcards` or `wildcards` entry in the **extra_model_paths.yaml** file (recommended).
+* **definitions**: Wildcards definitions (in yaml or json format). Direct input added to the ones found in the wildcards folders. Allows wildcards to be included in the workflow.
+* **if_wildcards**: Select what do you want to do with any found wildcards/choices (when process wildcards is off or after the processing).
+  * **Ignore**: do not try to detect wildcards.
+  * **Remove**: detect wildcards and remove them.
+  * **Add visible warning**: detect wildcards and add a warning text to the prompt, that hopefully produces a noticeable generation.
+  * **Stop the generation**: detect wildcards and stop the generation.
+* **choice_separator**: What do you want to use by default to separate multiple choices when the options allow it (by default it's ', ").
+* **keep_choices_order**: If checked, a multiple choice construct will return them in the order they are in the construct.
+
+### ACB PPP Send-To-Negative Options node
+
+Options for sent to negative commands, in case you want to change them from the defaults.
+
+* **separator**: You can specify the separator used when adding to the negative prompt (by default it's ", ").
+* **ignore_repeats**: It ignores repeated content to avoid repetitions in the negative prompt.
+
+### ACB PPP Cleanup Options node
+
+Options for cleanup processing, in case you want to change them from the defaults.
+
+* **extra_spaces**: Removes other unnecessary spaces.
+* **empty_constructs**: Removes attention/scheduling/alternation constructs when they are invalid.
+* **extra_separators**: Removes unnecessary separators. This applies to the configured separator and regular commas.
+* **extra_separators_additional**: Removes unnecessary separators at start or end of lines. This applies to the configured separator and regular commas.
+* **extra_separators_include_eol**: In the previous two options it also removes EOLs attached to the separators.
+* **around_breaks**: Removes consecutive BREAKs and unnecessary commas and space around them.
+* **breaks_with_eol**: Add a newline before BREAKs.
+* **around_ands**: Removes consecutive ANDs and unnecessary commas and space around them.
+* **ands_with_eol**: Add a newline before ANDs.
+* **around_extranetwork_tags**: Removes spaces around extra network tags.
+* **merge_attention**: It merges attention modifiers when possible (merges into one, multiplying their values). Only merges individually nested attention (even through choice/wildcard boundaries).
+* **remove_extranetwork_tags**: Removes all extra network tags.
+
+Please note that *ComfyUI* does not natively support the `BREAK` and `AND` constructs, but the related settings are kept in that UI in case you use a node that supports them and the extension is configured to allow them (see the configuration file below).
+
+### ACB PPP ExtraNetwork Mapping Options node
+
+Options for extranetworks mapping, in case you want to change them from the defaults.
+
+* **folders**: You can enter multiple folders separated by commas. You can leave it empty (the default) and add a `ppp_extranetworkmappings` entry in the **extra_model_paths.yaml** file (recommended).
+* **definitions**: Extranetwork Mappings definitions (in yaml format). Direct input added to the ones found in the extranetwork mappings folders. Allows the mappings to be included in the workflow.
+
+## A1111 (and compatible UIs)
+
+### Panel options
+
+* **Force equal seeds**: Changes the image seeds and variation seeds to be equal to the first of the batch. This allows using the same values for all the images in a batch.
+* **Unlink seed**: Uses the specified seed for the prompt generation instead of the one from the image. This seed is only used for wildcards and choices.
+* **Prompt seed**: The seed to use for the prompt generation. If -1 a random one will be used.
+* **Incremental seed**: When using a batch you can use this to set the rest of the prompt seeds with consecutive values.
+* **Combinatorial mode**: Generate all possible prompt combinations (from choices and wildcards) and cycle through them to fill the batch.
+* **Shuffle combinations**: It shuffles the combinatorial results.
+* **Combinations limit**: Maximum number of combinations to generate (0 = no limit). The actual maximum limit is the number of images (batch size * count).
+
+### General settings
+
+* **Debug level**: What to write to the console. Note: in *SD.Next* debug messages only show if you launch it with the `--debug` argument.
+* **What to do on invalid content warnings**: Warn on the console or stop the generation. This also affects the use of unknown variables, and integer comparisons with undefined or non-numeric variables: in *warn* mode the comparison evaluates to false, in *stop* mode the generation is stopped with an error.
+* **Use strict operators**: Sets strict operations in comparisons.
+* **Apply in img2img**: Check if you want to do the processing in img2img processes.
+* **Add original prompts to metadata**: Adds original prompts to the metadata if they have changed.
+* **Results file**: Filename to save processing results. Supports `%datetime%`, `%date%`, `%time%`, and `%host%` tokens. The file extension determines the format: `.yaml`/`.yml`, `.jsonl`, `.csv`, or plain text for any other extension. Relative paths are resolved against the extension's `logs` folder. Leave empty to disable.
+* **Extranetwork Mappings folders**: You can enter multiple folders separated by commas.
+
+### Wildcard settings
+
+* **Process wildcards**: You can choose to process wildcards and choices with this extension or use a different one.
+* **Wildcards folders**: You can enter multiple folders separated by commas.
+* **What to do with remaining wildcards?**: Select what do you want to do with any found wildcards/choices (when process wildcards is off or after the processing).
+  * **Ignore**: Do not try to detect wildcards.
+  * **Remove**: Detect wildcards and remove them.
+  * **Add visible warning**: Detect wildcards and add a warning text to the prompt, that hopefully produces a noticeable generation.
+  * **Stop the generation**: Detect wildcards and stop the generation.
+* **Default separator used when adding multiple choices**: What do you want to use by default to separate multiple choices when the options allow it (by default it's ", ").
+* **Keep the order of selected choices**: If checked, a multiple choice construct will return them in the order they are in the construct.
+
+### Send to negative prompt settings
+
+* **Separator used when adding to the negative prompt**: You can specify the separator used when adding to the negative prompt (by default it's ", ").
+* **Ignore repeated content**: It ignores repeated content to avoid repetitions in the negative prompt.
+
+### Clean up settings
+
+* **Remove empty constructs**: Removes attention/scheduling/alternation constructs when they are invalid.
+* **Remove extra separators**: Removes unnecessary separators. This applies to the configured separator and regular commas.
+* **Remove additional extra separators**: Removes unnecessary separators at start or end of lines. This applies to the configured separator and regular commas.
+* **The extra separators options also remove EOLs**: In the previous two options it also removes EOLs attached to the separators.
+* **Clean up around BREAKs**: Removes consecutive BREAKs and unnecessary commas and space around them.
+* **Use EOL instead of Space before BREAKs**: Add a newline before BREAKs.
+* **Clean up around ANDs**: Removes consecutive ANDs and unnecessary commas and space around them.
+* **Use EOL instead of Space before ANDs**: Add a newline before ANDs.
+* **Clean up around extra network tags**: Removes spaces around extra network tags.
+* **Merge attention modifiers (weights) when possible**: It merges attention modifiers when possible (merges into one, multiplying their values). Only merges individually nested attention (even through choice/wildcard boundaries).
+* **Remove extra spaces**: Removes other unnecessary spaces.
+* **Remove extra network tags**: Removes all extra network tags.

docs/COOKBOOK.md

@@ -0,0 +1,571 @@
+# Cookbook
+
+This cookbook shows some interesting uses of the features available.
+
+## Wildcard definition
+
+### Grouping
+
+The common use of wildcards is creating multiple of them with simple lists of elements. But this extension supports more interesting ways to group the elements, reducing bloat and improving selection of the wanted elements.
+
+In these examples we assume we want to select colors but we want warm colors to have double weight than cold colors, then have weights for each of them, and we sometimes want to choose only the warm ones, the cold ones, or any of them.
+
+The usual way would be this:
+
+```yaml
+colors_warm:
+    - 3::red
+    - 2::orange
+    - 1::yellow
+colors_cold:
+    - 2::blue
+    - 1::green
+```
+
+And use `__colors_warm__` or `__colors_cold__`, or `{2::__colors_warm__|1::__colors_cold__}` to get all of them.
+
+The `1` weights are not necessary, it just makes the intent clearer.
+
+#### Filters
+
+You can add labels to the elements of a wildcard, allowing you to reduce the availability to only a subset of them. This avoids the need to create multiple wildcards of similar elements.
+
+```yaml
+colors:
+    - "'warm'6::red"
+    - "'warm'4::orange"
+    - "'warm'2::yellow"
+    - "'cold'2::blue"
+    - "'cold'1::green"
+```
+
+And use `__colors'warm'__` or `__colors'cold'__`, or just `__colors__` to get all of them.
+
+Labels can be more complex than this. See the syntax document.
+
+#### Anonymous wildcards
+
+You can also group the elements by using anonymous wildcards, which can also be labeled.
+
+```yaml
+colors:
+    - "'warm'2":  # this needs double quotes to include the single quotes as part of the options
+        - 3::red
+        - 2::orange
+        - 1::yellow
+    - "'cold'1":
+        - 2::blue
+        - 1::green
+```
+
+And use `__colors'warm'__` or `__colors'cold'__`, or just `__colors__` to get all of them.
+
+This produces almost the same effect as before but is easier to read. The only difference is with the weights when you don't filter, because now it first chooses between the warm and the cold options and then the colors inside the chosen one. This allows you to add simpler weights to the two groups and their elements instead of having to calculate the combined weights.
+
+With simple labels this is probably the best method.
+
+#### Wildcard inclusion
+
+Another option, but not as clean, is to include wildcards inside other wildcards.
+
+```yaml
+colors_warm:
+    - 3::red
+    - 2::orange
+    - 1::yellow
+colors_cold:
+    - 2::blue
+    - 1::green
+colors:
+    - "%'warm'2::include colors_warm"
+    - "%'cold'1::include colors_cold"
+```
+
+Then use `__colors_warm__` or `__colors_cold__`, or `__colors'warm'__` or `__colors'cold'__` or `__colors__` to get all of them.
+
+## Prompt building
+
+You can create a complex set of wildcards that build a full prompt, by using the wildcards along with variables with default values. The variables allow you to choose what parts of the prompt you want to change for specific content, or just leave the defaults. Then, inside the UI, you can use styles (with an appropiate A1111 styles node if using *ComfyUI*) to choose which variables to set. This makes it easy to quickly select what do you want to prompt for.
+
+If you use *ComfyUI* and don't have a styles node, you can put them in individual wildcards and use the included `Wildcards Concat` node.
+
+Wildcards:
+
+```yaml
+style:
+    realistic: photograph
+    oil: oil painting
+    sketch: pencil sketch
+
+subject:
+    - "'human'::{man|woman}"
+    - "'orc,fantasy'::orc"
+    - "'goblin,fantasy'::goblin"
+
+clothes:
+    - "'regular'::t-shirt, pants, shoes"
+    - "'regular'::swimsuit"
+    - "'regular'::uniform"
+    - "'regular'::pajamas"
+    - "'armor'::metal armor"
+    - "'armor'::leather armor"
+
+action:
+    - "'passive'::standing"
+    - "'passive'::sitting"
+    - "'passive'::lying down"
+    - "'passive'::crouching"
+    - "'active'::running"
+    - "'active'::dancing"
+    - "'active'::driving"
+
+background:
+    home:
+        - in the kitchen
+        - in the bathroom
+        - in the living room
+    nature:
+        - in a forest
+        - in a swamp
+        - in a desert
+        - in a grotto
+        - in the beach
+    building:
+        - in a cathedral
+        - in an office
+        - in a store
+        - in a castle
+        - in a dungeon
+    any: "{__background/home__|__background/nature__|__background/building__}"
+
+character:
+  - "${style:()}, ${subject:__subject'human'__}, ${clothes:__clothes'regular'__}, ${action:__action'passive'__}, ${background:__background/any__}"
+```
+
+Styles file:
+
+```csv
+name,prompt,negative_prompt
+"Nothing","",""
+"Style: realistic","${style=__style/realistic__}",""
+"Style: oil","${style=__style/oil__}",""
+"Style: sketch","${style=__style/sketch__}",""
+"Subject: human","${subject=__subject'human'__}",""
+"Subject: orc","${subject=__subject'orc'__}",""
+"Subject: goblin","${subject=__subject'goblin'__}",""
+"Subject: fantasy","${subject=__subject'fantasy'__}",""
+"Subject: any","${subject=__subject__}",""
+"Clothes: regular","${clothes=__clothes'regular'__}",""
+"Clothes: armor","${clothes=__clothes'armor'__}",""
+"Clothes: any","${clothes=__clothes__}",""
+"Action: passive","${action=__action'passive'__}",""
+"Action: active","${action=__action'active'__}",""
+"Action: any","${action=__action__}",""
+"Background: home","${background=__background/home__}",""
+"Background: nature","${background=__background/nature__}",""
+"Background: building","${background=__background/building__}",""
+"Background: any","${background=__background/any__}",""
+"Character","__character__",""
+```
+
+You first select the styles that set the variables that you are interested in changing from the default, and you end with the main wildcard that uses them to build the prompt.
+
+> [!TIP]
+> The order of the variables don't usually matter because they are only evaluated when echoed to the prompt. This allows you to use variables inside wildcards that are inside other variables, and only at the end they will be evaluated.
+
+Sample style selections:
+
+Sample 1:
+
+- "Style: oil"
+- "Subject human"
+- "Clothes: regular"
+- "Background: home"
+- "Character"
+
+This would create this prompt:
+
+`${style=__style/oil__}, ${subject=__subject'human'__}, ${clothes=__clothes'regular'__}, ${background=__background/home__}, __character__`
+
+Sample 2:
+
+- "Subject: fantasy"
+- "Clothes: armor"
+- "Action: active"
+- "Background: nature"
+- "Character"
+
+This would create this prompt:
+
+`${subject=__subject'fantasy'__}, ${clothes=__clothes'armor'__}, ${action=__action'active'__}, ${background=__background/nature__}, __character__`
+
+## Using variables for detailer prompts
+
+This only applies to *ComfyUI*, because the variables cannot be used in the ADetailer prompts of the A1111 UIs.
+
+You can put parts of the prompt to be set into variables, and then extract this variables from the output and use them in the prompts of the detailer nodes.
+
+For example, imagine you have this prompt:
+
+`Cyberpunk woman with ${head:long red hair mohawk, green eyes, and neon makeup} riding a futuristic motorcycle`
+
+That will create a variable `head` with the head description and insert it in that position in the prompt. That variable can then be extracted with the "**ACB PPP Select Variable**" node and used as input for the detailer for the head, thus avoiding prompt clutter that does not apply.
+
+## Conditional content based on model
+
+The `_is_*` system variables let you write a single prompt that adapts automatically to the loaded model. This is especially useful for LoRAs that exist in different versions for different model families.
+
+```text
+beautiful woman in a garden
+<ppp:if _is_sd1><lora:woman_detail_sd1:0.8> detailed face<ppp:elif _is_sdxl><lora:woman_detail_xl:0.9> detailed face<ppp:elif _is_flux><lora:woman_detail_flux:1.0> detailed face<ppp:/if>
+```
+
+Only one branch ends up in the final prompt. You can also nest `if` commands inside wildcards to gate certain choices:
+
+```yaml
+poses:
+    - "'standing'::standing upright"
+    - "'sitting'::sitting on a bench"
+    - "'dynamic,sdxl'if _is_sdxl::dynamic action pose"
+    - "'dynamic,flux'if _is_flux::powerful dynamic pose, motion blur"
+```
+
+The last two entries are only available when their respective model is loaded.
+
+## ExtraNetworks mappings
+
+When you have the same LoRA in multiple versions (one per model family), you can create a mapping so the correct one is picked automatically without rewriting your prompt for each model.
+
+**Mapping file** (`enmappings/my_loras.yaml`):
+
+```yaml
+lora:
+  my_character:
+    - condition: "_is_sd1"
+      name: "my_character_sd1"
+      parameters: "0.8"
+      triggers: ["my_char_trigger"]
+    - condition: "_is_sdxl"
+      name: "my_character_xl"
+      parameters: "0.9"
+      triggers: ["my_char_trigger", "detailed"]
+    - condition: "_is_flux"
+      name: "my_character_flux"
+      parameters: "1.0"
+      triggers: ["my_char_trigger"]
+```
+
+**Prompt usage:**
+
+```text
+a portrait of <ppp:ext $lora my_character>extra trigger<ppp:/ext>
+```
+
+The extension picks the mapping whose condition matches the loaded model, builds the correct `<lora:...:...>` tag, prepends the triggers from the mapping, and appends the inline triggers. If no condition matches (e.g. an unsupported model) nothing is added.
+
+You can also specify a weight multiplier directly in the command - if both the command weight and the mapping `parameters` are numbers, they are multiplied:
+
+```text
+<ppp:ext $lora my_character 0.75/>
+```
+
+## Send-to-negative with attention modifiers
+
+When a send-to-negative command sits inside an attention modifier, the weight is carried over to the negative prompt.
+
+```text
+(red apple<ppp:stn>round shape<ppp:/stn>:1.4)
+```
+
+Result in the negative: `(round shape:1.4)` - the surrounding weight is applied.
+
+```text
+(red apple<ppp:stn>[round shape]<ppp:/stn>:1.4)
+(red [apple<ppp:stn>round shape<ppp:/stn>]:1.4)
+```
+
+For both cases, the result in the negative is: `(round shape:1.26)` - the weight is merged, if merge attention is enabled.
+
+## Array variables
+
+Arrays let you collect several values and then use them together, which is useful when you want to pick a random set of attributes and later echo them as a formatted list.
+
+**Example - picking random accessories:**
+
+```text
+${accessories[]=*__accessories__}
+a woman wearing ${accessories[&', ']}
+```
+
+The wildcard `__accessories__` is expanded into an array, then echoed as a comma-separated string.
+
+**Example - building a list of a fixed size from separate wildcards:**
+
+```text
+${colors[]=*__colors__}
+${colors[]+=*__materials__}
+abstract composition of ${colors[&' and ']}
+```
+
+**Initializing from a literal list:**
+
+```text
+${seasons[]=*('spring', 'summer', 'autumn', 'winter')}
+seasonal scene: ${seasons[&', ']}
+```
+
+## Conditional filtering inside wildcards
+
+You can attach an `if` condition directly to a choice inside a wildcard, making the choice invisible unless the condition is true. This lets a single wildcard file serve all model families without duplication.
+
+```yaml
+lora_styles:
+    - "'painterly'::painterly style, oil paint"
+    - "'anime'if _is_sd1 or _is_sdxl::anime style, flat shading"
+    - "'photorealistic'if _is_sdxl or _is_flux::photorealistic, hyper detailed"
+    - "'concept'if _is_flux::concept art, cinematic lighting"
+```
+
+When loaded with SD 1, only `painterly` and `anime` are available. When loaded with Flux, only `painterly`, `photorealistic`, and `concept` are.
+
+You can combine conditions with `and`, `or`, and `not`, and reference any user or system variable:
+
+```yaml
+effects:
+    - "bokeh, shallow depth of field"
+    - "long exposure, motion blur"
+    - "'hdr'if _is_sdxl or _is_flux::HDR, high dynamic range"
+    - "'film'if quality ne 'draft'::film grain, subtle noise"
+```
+
+## `setwcdeffilter` for dynamic routing
+
+`setwcdeffilter` sets a default filter on a wildcard before it is referenced anywhere in the prompt. This avoids having to repeat a filter at every call site when you want to pre-select a subset of choices.
+
+A typical use case is a style variable that narrows what a downstream wildcard will pick:
+
+```text
+<ppp:setwcdeffilter 'clothing' 'armor'/>
+A warrior ${subject} wearing __clothing__
+```
+
+Whatever label filter you pass becomes the default for that wildcard for the rest of the current generation.
+
+You can use globbing to apply the filter to a whole family of wildcards:
+
+```text
+<ppp:setwcdeffilter 'items/*' 'rare'/>
+```
+
+This pre-filters every wildcard under `items/` to only `rare`-labelled choices. To remove a previously set default filter:
+
+```text
+<ppp:setwcdeffilter 'items/*'/>
+```
+
+A practical workflow combining this with variables and styles:
+
+```csv
+name,prompt,negative_prompt
+"Gear: armor","<ppp:setwcdeffilter 'clothing' 'armor'/>",""
+"Gear: casual","<ppp:setwcdeffilter 'clothing' 'casual'/>",""
+"Character","__character__",""
+```
+
+Select a "Gear" style first and then "Character" - the wildcard inside `character` will automatically use the pre-filtered `clothing` choices.
+
+## Globbing wildcards
+
+Instead of referencing a single wildcard file, you can use glob patterns to merge choices from multiple files into one pool. This is handy when you split a large collection into smaller themed files.
+
+File layout:
+
+```text
+wildcards/
+  characters/
+    humans.yaml
+    orcs.yaml
+    elves.yaml
+```
+
+Each file is a regular YAML wildcard. Reference them all at once:
+
+```text
+__characters/*__
+```
+
+This merges all choices from `humans.yaml`, `orcs.yaml`, and `elves.yaml` into a single pool and picks one.
+
+Note that the globbing works on the wildcard key, not the file structure. So if those files contain multiple wildcards, all of them will be merged.
+
+You can apply parameters to the glob call just like a regular wildcard:
+
+```text
+__2$$ / $$characters/*__
+```
+
+Selects 2 choices from the merged pool, separated by " / ".
+
+You can also apply a filter to narrow from within the merged pool:
+
+```text
+__characters/*'fantasy'__
+```
+
+Only choices labelled `fantasy` across all matched files are eligible.
+
+Note: if no parameters are specified in the glob call, the parameters from the first matching file that defines them (sorted by key) are used. To avoid that, specify parameters explicitly in the call.
+
+## Prefix/suffix or container on wildcard parameters
+
+Using the object format for wildcard parameters you can set a prefix and/or suffix, or a container, that wraps the result. This is cleaner than repeating the wrapper in each choice.
+
+In this example, without prefix/suffix, every choice needs to repeat the attention modifier:
+
+```yaml
+qualities:
+    - "(ultra detailed:1.3)"
+    - "(highly detailed:1.3)"
+    - "(intricate details:1.3)"
+```
+
+With object-format parameters and a prefix/suffix:
+
+```yaml
+qualities:
+    - { prefix: "(", suffix: ":1.3)" }  # parameters line
+    - "ultra detailed"
+    - "highly detailed"
+    - "intricate details"
+```
+
+The prefix and suffix are added around the wildcard's result. They can themselves contain constructs.
+
+Another way is with the container property, which does the same in a slightly more flexible way:
+
+```yaml
+qualities:
+    - { container: "(${__choices[]}:1.3)" }  # parameters line
+    - "ultra detailed"
+    - "highly detailed"
+    - "intricate details"
+```
+
+## `ifundefined` / `?=` for safe defaults
+
+When building multi-style workflows you often want a variable to have a sensible default that can be overridden by an earlier style, but only if it hasn't already been set. The `ifundefined` modifier (`?=`) is designed for this.
+
+```text
+${quality?=__qualities__}, ${style?=photograph}, ${subject?=__subject__}
+```
+
+If a "Quality: ultra" style already ran `${quality=ultra detailed, 8k}` earlier in the pipeline, the `?=` here does nothing. If no quality style was chosen, it picks a random value from `__qualities__` as the fallback.
+
+This is more robust than relying on the order of styles. But if you need a specific default for different instances use instead the default value when echoing the variable.
+
+**Styles file example:**
+
+```csv
+name,prompt,negative_prompt
+"Quality: draft","${quality=draft quality}",""
+"Quality: high","${quality=ultra detailed, 8k}",""
+"Style: photo","${style=photograph}",""
+"Style: painting","${style=oil painting}",""
+"Character","${quality?=__qualities__}, ${style?=photograph}, __subject__",""
+```
+
+If the user selects "Quality: high" and "Character", quality is `ultra detailed, 8k`. If they select only "Character", quality falls back to a random value from `__qualities__`.
+
+You can also use `evaluate ifundefined` (`?=!`) to resolve the wildcard immediately rather than lazily:
+
+```text
+${quality?=!__qualities__}
+```
+
+## Informing the model to the node in *ComfyUI*
+
+In older versions, to correctly set the `_is_*` variables it was needed to set the `model` input (to get the class and kind of model) and the `modelname` input (to calculate variants). Now, the class and kind can be inferred from the file, so only the `modelname` is needed.
+
+You can still load the model before the PPP node, and you connect it to the `model` input, but you will still need to fill the `modelname` input to detect variants (and if you don't have a loader node that also outputs the filename it is a repetition).
+
+But a better way is loading the model after the PPP node, setting the filename in `modelname` (which is now a combo) and then extracting `_modelfullname` from the output variables and connecting that to the model loader node filename input. This also let's you use the next tip.
+
+## Choosing the model from the prompt in *ComfyUI*
+
+You can leave the model and modelname inputs disconnected/empty and set the `_modelfullname` variable at the start of the prompt (maybe through a wildcard or a choice construct). Then extract it from the output variables using the `Select Variable` node and use its value to actually load that model.
+
+You can also set and extract user variables for other ksampler inputs, like sampler, scheduler, steps, cfg and latent size.
+
+## Debugging tips
+
+When something isn't generating as expected, the debug setting is your first tool. Enable it in the extension settings; it will log all system variables at generation time, which tells you exactly what values are available for your conditions.
+
+**Finding the right system variable for an unsupported model:**
+
+If you load a model that isn't matched by any `_is_*` variable, check `_modelclass` in the debug log. Use it directly in a condition while you wait for official support:
+
+```text
+<ppp:if _modelclass eq 'StableDiffusionXLPipeline'>content for new model<ppp:/if>
+```
+
+**Inspecting variable values mid-prompt:**
+
+Insert a temporary `echo` to see what a variable resolves to:
+
+```text
+[DEBUG style=${style}] ${style:photograph}, detailed portrait
+```
+
+Remove the bracketed part once you're satisfied.
+
+**Checking which choices are available after filtering:**
+
+If a wildcard seems to produce unexpected results, temporarily use a fixed positional filter to verify specific choices:
+
+```text
+__mywildcard'0'__   # always picks the first choice
+__mywildcard'1'__   # always picks the second choice
+```
+
+This confirms the order and content of choices before relying on label filters.
+
+**Common pitfalls:**
+
+- A `%` choice in a YAML array must be quoted, otherwise YAML treats `%` as invalid syntax.
+- Variables set with `${var=value}` are lazy - they are not evaluated until echoed. Use `${var=!__wildcard__}` (with `!`) for immediate evaluation if the value should only be resolved once (as in, you want the same value to be echoed later multiple times).
+- Wildcards cannot be used inside extranetwork tags (because some LoRA names contain double underscores). Put the entire `<lora:...>` tag inside a wildcard choice instead, or use the `ext` command.
+
+## Recording results to a file
+
+When using wildcards and choices the actual prompt sent to the model can look very different from what you typed. Configuring a results file lets you capture every resolved prompt, along with its metadata, so you always know exactly what was generated.
+
+Common use cases:
+
+- **Reproducibility** - if an image turns out great but you didn't keep the inputs, you can look up the exact resolved prompt and all the inputs from the log.
+- **Dataset building** - when running large batches with combinatorial mode or random wildcards, the file gives you a record of every prompt variant that was generated.
+- **Prompt analysis** - review the log to spot patterns, see which wildcard choices come up most often, or evaluate the output of a new wildcard set before committing to a full run.
+- **Debugging wildcard expansion** - compare the resolved prompts against what you expected to confirm that variables, conditions, and filters are all behaving correctly.
+
+### Setup
+
+Set the `results_file` option (in the extension settings for *A1111*, or the `results_file` input on the main node for *ComfyUI*) to a filename. The extension determines the output format from the file extension:
+
+| Extension        | Format                                             |
+|------------------|----------------------------------------------------|
+| `.yaml` / `.yml` | YAML list of records                               |
+| `.jsonl`         | JSON Lines, one JSON object per line               |
+| `.csv`           | CSV with a header row (semicolon-delimited)        |
+| anything else    | Plain text with labelled sections                  |
+
+Each record contains five sections: `options` (the PPP settings that were active), `inputs` (seed, prompts), `system` (system variables like `_modelclass`), `results` (the final positive and negative prompts), and `variables` (any user variables that were set).
+
+Use `%datetime%`, `%date%`, `%time%`, or `%host%` tokens in the filename to create a new file for each session or application:
+
+```text
+results_%date%.yaml
+```
+
+Relative paths are resolved against the `logs` folder inside the extension directory.
+
+> [!TIP]
+> The `.jsonl` format is the most convenient for programmatic processing. The `.yaml` format is the easiest to read manually.

docs/SYNTAX.md

@@ -0,0 +1,530 @@
+# Prompt PostProcessor syntax
+
+## Commands
+
+The extension uses a format for its commands similar to an extranetwork, but it has a `ppp:` prefix followed by the command, and then a space and any parameters (if any).
+
+`<ppp:command parameters/>`
+
+When a command is associated with any content, it will be between an opening and a closing command:
+
+`<ppp:command parameters>content<ppp:/command>`
+
+For wildcards and choices it uses the formats from the *Dynamic Prompts* extension, but sometimes with some additional options for extra functionality.
+
+## Special characters
+
+Remember that to use any special character (like parentheses, brackets or braces) as-is in the prompt (not as part of a construct), you need to escape them with a backslash. Like:
+
+```text
+velma \(from scooby doo\)
+```
+
+And inside a yaml/json file that backslash must be itself escaped again to be preserved.
+
+```yaml
+characters:
+  - velma \\(from scooby doo\\)
+```
+
+## Choices
+
+The generic format is: `{parameters$$opt1::choice1|opt2::choice2|opt3::choice3}`
+
+Both the construct parameters (up to the `$$`) and the individual choice options (up to the `::`) are optional.
+
+There is also a format where instead of `parameters$$` you just put the sampler, for compatibility with *Dynamic Prompts*.
+
+The construct parameters can be written with the following options (all are optional):
+
+* `~` (random) or `@` (cyclical): sampler (for compatibility with *Dynamic Prompts*). The cyclical sampler cycles through all combinations in order across consecutive `process_prompt` calls, resuming where the previous call left off (as long as the input prompt and negative prompt do not change).
+* `r`: means it allows repetition of the choices.
+* `o`: means it is optional, and no error will be raised if there are no choices to select from.
+* `n` or `n-m` or `n-` or `-m`: number or range of choices to select. Allows zero as the start of a range. Default is 1.
+* `'description'`: optional description (quoted), only valid in wildcard definitions. Used only in the Wildcards Concat node in *ComfyUI*.
+* `$$sep`: separator when multiple choices are selected. Default is set in settings.
+* `$$`: end of the parameters (not optional if any parameters).
+
+Regarding the optional flag, consider this scenario: due to their conditions no choice is available. It will raise an error. If you add the `o` then it will just return an empty string. This is only necessary if all choices have conditions and they could all be false. It is not the same as setting a range starting at 0, because that would be an allowed number of returned choices. If you do this and no choices are available, no error is raised.
+
+The choice options are as follows:
+
+* `%`: indicates that the content of the choice is a command
+* `'identifiers'`: comma separated labels for the choice (optional, quotes can be single or double). Only makes sense inside a wildcard definition. Can be used when specifying the wildcard to select this specific choice. It's case insensitive.
+* `n`: weight of the choice (optional, default 1).
+* `if condition`: filters out the choice if the condition is false (optional; this is an extension to the *Dynamic Prompts* syntax). Same conditions as in the `if` command.
+* `::`: end of choice options (not optional if any options)
+
+Whitespace is allowed between parameters/options.
+
+The only command available is `include wildcard`, which will include the choices of the specified wildcard in place of this choice. This allows composing choices from multiple wildcards. It also works in the choices of a wildcard, but note that in yaml you cannot start an array element with `%` and you will have to put the full choice in quotes, or use the object format.
+
+These are examples of formats you can use to insert a choice construct:
+
+| Construct                                         | Result                                                                                                                                |
+| ---------                                         | ------                                                                                                                                |
+| `{choice1\|5::choice2\|3::choice3}`               | select 1 choice, two of them have weights                                                                                             |
+| `{3$$choice1\|5 if _is_sd1::choice2\|choice3}`    | select 3 choices, one has a weight and a condition                                                                                    |
+| `{2-3$$2::choice1\|choice2\|choice3}`             | select 2 to 3 choices, one of them has a weight                                                                                       |
+| `{r2-3$$choice1\|choice2\|choice3}`               | select 2 to 3 choices allowing repetition                                                                                             |
+| `{2-3$$ / $$choice1\|choice2\|choice3}`           | select 2 to 3 choices with separator ` / `                                                                                            |
+| `{o$$if _is_sd1::choice1\|if _is_sd2::choice2}`   | select 1 choice, both have conditions, if none matches it is allowed because we indicate that it is optional                          |
+| `{choice1\|choice2\|%0.5::include path/wildcard}` | select 1 choice from the two specified and the ones inside the path/wildcard wildcard, which will be weighted with half their weights |
+
+> [!NOTE]
+> The *Dynamic Prompts* format `{2$$__flavours__}` does not work as expected because the wildcard is considered only one possible choice (it will only output one value). You can write it instead as a wildcard with parameters `__2$$flavours__`.
+>
+> Whitespace around the choices is not ignored like in *Dynamic Prompts*, but will be cleaned up if the appropriate cleaning settings are selected.
+
+## Wildcards
+
+The generic format is: `__parameters$$wildcard'filter'(var=value)__`
+
+The parameters, the filter, and the setting of a variable are optional. The parameters follow the same format as for the choices.
+
+> [!WARNING]
+> Wildcards cannot be used inside an extranetwork tag (because some LoRA names contain double underscores). If you need to choose from multiple LoRAs put the whole extranetwork tag inside a wildcard, or use choices.
+
+### Identifier
+
+* Allowed characters are letters, numbers, underscore (`_`), dash (`-`), dot (`.`), and the path separators (`/` and `\`). It cannot start with an underscore because it would be ambiguous whether it's part of the name or just precedes the wildcard.
+* Can have a relative path and contain globbing formatting, to read multiple wildcards and merge their choices. Note that if there are no parameters specified, the globbing will use the ones from the first wildcard that matches and have parameters (sorted by keys), so if you don't want that you might want to specify them. Also note that, unlike with *Dynamic Prompts*, the wildcard name has to be specified with its full path (unless you use globbing).
+* You can use variables, with the `${name}`, `${name:default}`, `<ppp:echo name/>` or `<ppp:echo name>default<ppp:/>echo>` formats, to build a dynamic identifier.
+
+### Filter
+
+The filter can be used to filter specific choices from the wildcard. The filtering works before applying the choice conditions (if any). The surrounding quotes can be single or double.
+
+The filter is a comma separated list of an integer/range (positional choice index, zero-based) or choice label. You can also compound them with `+`. That is, the comma separated items act as an OR and the `+` inside them as an AND. Using labels can simplify the definitions of complex wildcards where you want to have direct access to specific choices on occasion (you don't need to create wildcards for each individual choice). You can use variables for individual labels (`${v}`) or a single variable for the whole filter (`${v}` or `${a[&',']}` or `${a[&'+']}`).
+
+There are some additional formats when using filters.
+
+* You can specify `^wildcard` as a filter to use the filter of a previous wildcard in the chain.
+* You can start the filter (regular or inherited) with `#` and it will not be applied to the current wildcard choices, but the filter will remain in memory to use by other descendant wildcards. You use `#` and `^` when you want to pass a filter to inner wildcards (see the test files).
+
+### Variable
+
+The variable value only applies during the evaluation of the selected choices and is discarded afterward (the variable keeps its original value if there was one).
+
+### Examples
+
+These are examples of formats you can use to insert a wildcard:
+
+| Construct                            | Result                                                                   |
+| ---------                            | ------                                                                   |
+| `__wildcard__`                       | select 1 choice                                                          |
+| `__path/wildcard'0'__`               | select the first choice                                                  |
+| `__path/wildcard'1-2'__`             | select the second or third choice                                        |
+| `__path/wildcard'label'__`           | select the choices with label `label`                                    |
+| `__path/wildcard'0,label1,label2'__` | select the first choice and those with labels `label1` or `label2`       |
+| `__path/wildcard'0,label1+label2'__` | select the first choice and those with both labels `label1` and `label2` |
+| `__3$$path/wildcard__`               | select 3 choices                                                         |
+| `__2-3$$path/wildcard__`             | select 2 to 3 choices                                                    |
+| `__r2-3$$path/wildcard__`            | select 2 to 3 choices allowing repetition                                |
+| `__2-3$$ / $$path/wildcard__`        | select 2 to 3 choices with separator ` / `                               |
+| `__path/wildcard(var=value)__`       | select 1 choice using the specified variable value in the evaluation.    |
+
+### Wildcard definitions
+
+A wildcard definition can be:
+
+* A txt file. The wildcard name will be the relative path of the file, without the extension. Each line will be a choice. Lines starting with `#` or empty are ignored. Doesn't support nesting.
+* An array or scalar value inside a json or yaml file. The wildcard name includes the relative folder path of the file, without the extension, but also the path of the value inside the file (if there is one). If the file contains a dictionary, the filename part is not used for the wildcard name. Supports nesting by having dictionaries inside dictionaries.
+
+The best format is a yaml file with a dictionary of wildcards inside. An editor supporting yaml syntax and linting is recommended (f.e. VSCode).
+
+In a choice, the content after a `#`  is ignored.
+
+If the first choice follows the format of wildcard parameters (*including the final `$$`*), it will be used as default parameters for that wildcard (see examples in the tests folder). Unless the only property used is the wildcard description.
+
+The choices of the wildcard follow the same format as in the choices construct, or the object format of *Dynamic Prompts* (only in structured files).
+
+If using the object format for a choice you can use the following in addition to the standard `weight` and `text`/`content`:
+
+* `if`: the condition (a string)
+* `labels`: list of labels (an array of strings)
+* `command`: indicates the content is a command (a boolean)
+
+```yaml
+{ command: false, labels: ["some_label"], weight: 2, if: "_is_pony", content: "the text" } # "text" property can be used instead of "content"
+```
+
+Wildcard parameters in a json/yaml file can also be in object format, and support some additional properties, that are not included in the string format:
+
+* `prefix`: content to prefix the list of choices
+* `suffix`: content to suffix the list of choices
+* `container`: includes the prefix, choices array variable, and suffix
+
+```yaml
+{ sampler: "~", repeating: false, optional: false, from: 2, to: 3, description: "test wildcard", container: "prefix-${__choices[&'/']}-suffix" }
+{ sampler: "~", repeating: false, optional: false, count: 2, description: "test wildcard", prefix: "prefix-", suffix: "-suffix", separator: "/" }
+{ sampler: "~", repeating: false, optional: false, from: 2, to: 3, description: "test wildcard", prefix: "prefix-", suffix: "-suffix", separator: "/" }
+```
+
+The prefix and suffix are added to the result along with the selected choices and separators. They can contain other constructs, but the separator can't.
+
+The container is a new option that replaces prefix/suffix/separator, and makes use of the recent support for array variables. Its value would be the concatenation of any prefix and/or suffix with the echoing (with the chosen separator) of a temporary `__choices[]` variable that holds the chosen values. This property is preferred over prefix/suffix/separator unless you only need the separator.
+
+It is recommended to use the object format for the wildcard parameters and for choices with complex options.
+
+If your first choice is interpreted as parameters, and you don't need parameters, you can avoid the problem by adding an empty parameters object `{}` as first choice.
+
+Wildcards can contain just one choice. In json and yaml formats this allows the use of a string value for the keys, rather than an array.
+
+A choice inside a wildcard can also be a list or a dictionary of one element containing a list. These are considered anonymous wildcards. With a list it will be an anonymous wildcard with no choice options, and with a dictionary the key will be the options for the choice containing the anonymous wildcard and the value the choices of the anonymous wildcard. Anonymous wildcards can help formatting complex choice values that are used in only one place and thus creating a regular wildcard is not necessary. See test.yaml for examples. Use an anonymous wildcard to group options inside a wildcard, and attach a label to it to be able to choose only from that group.
+
+Remember you can use the include command on choices to compose a wildcard from other wildcards' choices.
+
+> [!IMPORTANT]
+> The files should have UTF-8 encoding. The extension will also try with windows-1252 if that fails.
+
+Wildcard definitions are reloaded automatically on each generation if they change.
+
+### Detection of remaining wildcards
+
+> [!IMPORTANT]
+> This extension should run after any other wildcard extensions, so if you don't use the internal wildcards processing, any remaining wildcards present in the prompt or negative_prompt at this point must be invalid. Usually you might not notice this problem until you check the image metadata, so this option gives you some ways to detect and treat the problem.
+
+## Set Wildcard Default Filter command
+
+This command can be used to set a default filter for a wildcard, before it is used.
+
+The format is:
+
+| Construct                                     | Meaning            |
+| ---------                                     | -------            |
+| `<ppp:setwcdeffilter 'identifier' 'filter'/>` | Sets a filter      |
+| `<ppp:setwcdeffilter 'identifier'/>`          | Removes the filter |
+
+The wildcard identifier supports globbing. The filter does not allow the `^` or `#` flags.
+
+## Variables
+
+The prompt has access to some system variables that contain model information, options, and other things. There is also the possibility of defining user variables.
+
+Variable values `true` and `false` are considered a boolean, and numeric content is an integer or float.
+
+All these variables can be used to output content or behave differently based on their values.
+
+### System variables
+
+Names starting with an underscore are reserved for system variables:
+
+| System variable     | Value                                                                                                                                                                                                                                                                                                                                 |
+| ---------------     | -----                                                                                                                                                                                                                                                                                                                                 |
+| `_model`            | the model identifier (`sd1`, `sd2`, `sdxl`, `sd3`, `flux`, `auraflow`). `_sd` also works but is deprecated.                                                                                                                                                                                                                           |
+| `_modelname`        | the model filename (without path). Do not confuse with the `modelname` input in *ComfyUI* which matches actually to the `_modelfullname` variable. `_sdname` also works but is deprecated.                                                                                                                                            |
+| `_modelfullname`    | the model filename (with path). `_sdfullname` also works but is deprecated. In *ComfyUI* this variable can also be **set** to override the filename used for model detection (see below).                                                                                                                                             |
+| `_modelclass`       | the class used for the model. Note that this is dependent on the webui. In A1111 all SD versions use the same class. Can be used for new models that are not supported yet with the `_is_*` variables. The debug setting will show all system variables when generating in case you need to see which one to use for a certain model. |
+| `_is_kkkk`          | true if the model is of kind *kkkk* (the model identifier, f.e. sdxl; those set in the ppp_config.yaml file)                                                                                                                                                                                                                          |
+| `_is_vvvv`          | true if the model matches the *vvvv* model variant definition (based on its filename). Note that the corresponding variable for the model kind will also be true.                                                                                                                                                                     |
+| `_is_pure_kkkk`     | true if the model is of kind *kkkk* and not a variant.                                                                                                                                                                                                                                                                                |
+| `_is_variant_kkkk`  | true if the model version is any variant of model kind *kkkk* and not the pure version. Note that the corresponding variable for the model kind will also be true.                                                                                                                                                                    |
+| `_is_sd`            | true if the model is any version of SD                                                                                                                                                                                                                                                                                                |
+| `_is_ssd`           | true if the model is SSD (Segmind Stable Diffusion 1B). Note that for an SSD model `_is_sdxl` will also be true.                                                                                                                                                                                                                      |
+| `_is_sdxl_no_ssd`   | true if the model is SDXL and not an SSD model.                                                                                                                                                                                                                                                                                       |
+| `_is_sdxl_no_pony`  | true if the model is SDXL and not a Pony model (the `pony` variant must be defined in settings). Kept to maintain compatibility with previous versions.                                                                                                                                                                               |
+| `_opt_...`          | All the options.                                                                                                                                                                                                                                                                                                                      |
+| `_input_seed`       | The seed used.                                                                                                                                                                                                                                                                                                                        |
+| `_input_pos_prompt` | The original positive prompt.                                                                                                                                                                                                                                                                                                         |
+| `_input_neg_prompt` | The original negative prompt.                                                                                                                                                                                                                                                                                                         |
+
+> [!NOTE]
+> The model path is relative to the checkpoint/difussion_models folder, just as it appears in the load nodes.
+
+## Set command
+
+This command sets the value of a variable that can be checked later.
+
+The format is: `<ppp:set varname [modifiers]>value<ppp:/set>`
+
+These are the available optional modifiers:
+
+* `evaluate`: the value of the variable is evaluated at this moment, instead of when it is used.
+* `add`: the value is "added" to the current value of the variable (depending on their type). When possible, it does not force an immediate evaluation of the old or added values.
+* `ifundefined`: the value will only be set if the variable is undefined.
+
+The `add` and `ifundefined` modifiers are mutually exclusive and cannot be used together.
+
+The *Dynamic Prompts* format also works:
+
+| Construct       | Meaning              |
+| ---------       | -------              |
+| `${var=value}`  | regular evaluation   |
+| `${var=!value}` | immediate evaluation |
+
+If also supports the addition and undefined check as an extension of the *Dynamic Prompts* format:
+
+| Construct        | Meaning                              |
+| ---------        | -------                              |
+| `${var+=value}`  | equivalent to `add`                  |
+| `${var+=!value}` | equivalent to `evaluate add`         |
+| `${var?=value}`  | equivalent to `ifundefined`          |
+| `${var?=!value}` | equivalent to `evaluate ifundefined` |
+
+Set variables are included in the output variables with their last value.
+
+System variables cannot be set, except in some cases (see next point).
+
+### Overriding model information from the prompt (*ComfyUI* only)
+
+In *ComfyUI*, `_modelfullname` can be **set** from the prompt using the `set` command or `${}` syntax. Setting it triggers a re-evaluation of all `_is_*` system variables, so conditions that come later in the prompt will use the new values.
+
+This is useful in workflows where the model information is not passed through the node inputs but is known at prompt time. And the `_modelfullname` variable can be extracted later by using the `Select Variable` node to actually load that model.
+
+If a class is not specified because the `model` input is disconnected but the `modelname` is, it will try to detect the class from the file. This is the preferred way, or just set `_modelfullname` in the prompt.
+
+The model filename is relative to the specific model folder (checkpoints, diffusion_models).
+
+```text
+${_modelfullname=flux/fluxmodel.safetensors}
+```
+
+> [!WARNING]
+> Attempting to set this variable in any other host will trigger a warning or error depending on the **What to do on invalid content warnings?** setting.
+
+## Echo command
+
+This command prints the value of a variable, or the specified default if it doesn't exist. If the variable does not exist and has no default, depending on the **What to do on invalid content warnings?** setting it will assume an empty value, or result in an error.
+
+The format is:
+
+| Construct                              |
+| ---------                              |
+| `<ppp:echo varname/>`                  |
+| `<ppp:echo varname>default<ppp:/echo>` |
+
+The *Dynamic Prompts* format is:
+
+| Construct            |
+| ---------            |
+| `${varname}`         |
+| `${varname:default}` |
+
+An echoed variable that uses the default because it doesn't have a value will be included in the output variables with the last default value used.
+
+## Array variables
+
+There is support for array variables. They use brackets `[]` to differenciate from regular variables.
+
+They can be initialized in several ways:
+
+| Construct                  | Meaning                                                          |
+| ---------                  | -------                                                          |
+| `${var[]=value}`           | initialize and set the first value                               |
+| `${var[]=*()}`             | initialize an empty array                                        |
+| `${var[]=*var2[]}`         | initialize an array from another array                           |
+| `${var[]=*__wildcard__}`   | initialize an array from a wildcard                              |
+| `${var[]=*('value',var2)}` | initialize an array from a list of values (strings or variables) |
+| `${var[]+=}`               | add an empty element to the array                                |
+| `${var[]+=value}`          | add a value to the array                                         |
+| `${var[]+=*var2[]}`        | add elements from another array                                  |
+| `${var[]+=*__wildcard__}`  | add elements from a wildcard                                     |
+
+The star operator `*` has always an inmediate evaluation.
+
+And they can be accesed/echoed with:
+
+* Empty brackets mean the whole array (used when initializing or when echoing the whole array)
+* An integer inside the brackets means an indexed value. A variable identifier (not an indexed array) can be used to get the integer.
+* A hash inside the brackets is used to get the length of the array
+* An ampersand followed by a string inside the brackets (with quotes) is used to get the full array joined with a separator.
+
+| Construct                  | Meaning                                                          |
+| ---------                  | -------                                                          |
+| `${var[]}`                 | echo all elements with a default separator                       |
+| `${var[&' / ']}`           | echo all elements with a specific separator                      |
+| `${var[n]}`                | echo an element from the array                                   |
+| `${var[n]:default}`        | echo an element with a default                                   |
+| `${var[#]}`                | echo the length of the array                                     |
+
+## If command
+
+This command allows you to filter content based on conditions.
+
+The full format is:
+
+`<ppp:if condition1>content one<ppp:elif condition2>content two<ppp:else>other content<ppp:/if>`
+
+Any `elif`s (there can be multiple) and the `else` are optional.
+
+The `conditionN` is a boolean expression, which can use `and`, `or`, `not` and grouping with parentheses, and where the simplest expression can be:
+
+| Construct                           | Meaning                                                                         |
+| ---------                           | -------                                                                         |
+| `operand`                           | check truthyness of the operand, meaning not zero, empty string nor empty array |
+| `operand1 [not] operation operand2` | compare the operands                                                            |
+
+The operands can be a variable (`variable`, `array[]`, `array[index]`), a quoted string, an integer, a boolean, or a parenthesized list of any of them `('string', 1, variable, false, array[], array[2])`. The allowed operations depend on which operands are used.
+
+When an operand is or contains a variable, it is resolved to the variable's current value before the operation.
+
+String comparisons are case insensitive, and substring comparisons (see below) are always considered as strings.
+
+The operation can be preceded by `not` for readability, instead of using it in the front.
+
+The supported operations are: `eq`, `ne`, `gt`, `lt`, `ge`, `le`, `in`, `any_in`, `contains` and `contains_any`. The `any_in` and `contains_any` variants exist because `any` and `contains` consider all elements.
+
+This list shows what they do depending on the kind of operand (R = regular variable, A = array variable).
+
+| Operation      | R1 op R2                 | A1 op A2          | A1 op R2                                       | R1 op A2                                       |
+| ---------      | --------                 | --------          | --------                                       | --------                                       |
+| `eq`           | OK                       | OK (pairwise)     | Error in strict mode, all A1 with R2 otherwise | Error in strict mode, R1 with all A2 otherwise |
+| `ne`           | OK                       | OK (pairwise)     | Error in strict mode, all A1 with R2 otherwise | Error in strict mode, R1 with all A2 otherwise |
+| `gt`           | OK                       | OK (pairwise)     | Error in strict mode, all A1 with R2 otherwise | Error in strict mode, R1 with all A2 otherwise |
+| `lt`           | OK                       | OK (pairwise)     | Error in strict mode, all A1 with R2 otherwise | Error in strict mode, R1 with all A2 otherwise |
+| `ge`           | OK                       | OK (pairwise)     | Error in strict mode, all A1 with R2 otherwise | Error in strict mode, R1 with all A2 otherwise |
+| `le`           | OK                       | OK (pairwise)     | Error in strict mode, all A1 with R2 otherwise | Error in strict mode, R1 with all A2 otherwise |
+| `in`           | OK (substring, R1 in R2) | OK (all A1 in A2) | OK (all substring in A1 in R2)                 | OK (R1 in A2)                                  |
+| `any_in`       | Error                    | OK (any A1 in A2) | OK (any substring in A1 in R2)                 | Error                                          |
+| `contains`     | OK (substring, R2 in R1) | OK (all A2 in A1) | OK (R2 in A1)                                  | OK (all substrings A2 in R1)                   |
+| `contains_any` | Error                    | OK (any A2 in A1) | Error                                          | OK (any substrings A2 in R1)                   |
+
+When a comparison tries to compare undefined variables or the values have different types (f.e. an integer and a string), the behavior depends on the `on_warning` setting: in `warn` mode the comparison evaluates to false, and in `stop` mode an error is raised. In non strict mode a numeric string literal (with no leading zeros) will be considered an integer.
+
+### Example
+
+(multiline to be easier to read)
+
+```text
+<ppp:if _is_sd1><lora:test_sd1> test sd1x
+<ppp:elif _sd_pony><lora:test_pony> test pony
+<ppp:elif _sd_pure_sdxl><lora:test_sdxl> test sdxl
+<ppp:else>unknown model
+<ppp:/if>
+```
+
+Only one of the options will end up in the prompt, depending on the loaded model.
+
+## ExtraNetwork command
+
+This command is a shortcut to add an extranetwork (usually a lora), and its triggers, with conditions. More legible and sometimes shorter than adding regular extranetworks inside if commands.
+
+The full format is:
+
+`<ppp:ext type name [parameters] [if condition]>[triggers]<ppp:/ext>`
+`<ppp:ext type name [parameters] [if condition]/>`
+
+The `type` is the kind of extranetwork, like `lora` or `hypernet`.
+
+The `name` is the extranetwork identifier. If it is not a regular identifier (i.e. starts with a number or contains spaces or symbols) it should be inside quotes.
+
+The `parameters` is optional and its format depends on the extranetwork type. With LoRAs or HyperNets it is usually a single weight number, so if the type is one of those and there are no parameters it will default to `1`. If it is not a number it should go inside quotes.
+
+The `condition` uses the same format as in the `if` command, and it is also optional.
+
+The `triggers` are also optional, and can be any content. If there are no triggers the command ending can be omitted.
+
+If the condition passes (or if there is no condition) the extranetwork tag will be built and added to the result along with any triggers.
+
+### Examples
+
+(multiline to be easier to read)
+
+```text
+<ppp:ext lora test_sd1 if _is_sd1>test sd1x<ppp:/ext>
+<ppp:ext lora test_pony 0.5 if _is_pony>test pony<ppp:/ext>
+<ppp:ext lora test_ilxl if _is_illustrious/>
+<ppp:ext lora 'test sdxl' '1:0.8' if _is_pure_sdxl>test sdxl<ppp:/ext>
+```
+
+Will turn into one of these (or none) depending on the model:
+
+* `<lora:test_sd1:1>test sd1x`
+* `<lora:test_pony:0.5>test pony`
+* `<lora:test_illustrious:1>`
+* `<lora:test sdxl:1:0.8>test sdxl`
+
+### Extranetworks mappings
+
+The extranetwork command supports specifying mappings of extranetworks (like LoRAs), so, for example, a different one can be used depending on the loaded model.
+
+If the type of extranetwork is prefixed with a `$` the command will look for a mapping.
+
+If you have LoRAs that do the same but for different models, create a mapping to group them, configuring there the weight and triggers for each one.
+
+The mappings are configured in yaml files in any of the configured extranetwork mappings folders. The format is like this:
+
+```yaml
+extnettype:
+  mappingname:
+    - condition: "<a supported condition>"
+      name: "<name of the extranetwork>"
+      parameters: "<parameters of the extranetwork>"
+      triggers: [<list of triggers>]
+      weight: 1.0
+    ...
+```
+
+Used like this:
+
+| Construct                                              | Meaning                              |
+| ---------                                              | -------                              |
+| `<ppp:ext $lora mappingname/>`                         | Mapping without additional triggers  |
+| `<ppp:ext $lora mappingname>inline triggers<ppp:/ext>` | Mapping with additional triggers     |
+
+Each mapping can have any number of elements in its list of mappings. There are no mandatory properties for a mapping. The properties mean the following:
+
+* `extnettype`: the kind of extranetwork, for example `lora`.
+* `mappingname`: the name you want to give to the mapping, to be referenced in the command.
+* `condition`: the condition to check for this mapping to be used (usually it should be one of the `_is_*` variables). If the conditions of multiple mappings evaluate to True, one will be chosen randomly. If the condition is missing it is considered True, to be used in the last mapping to catch as an `else` condition, and will be used if no other mapping applies.
+* `name`: name of the real extranetwork. If it is missing no extranetwork tag will be added.
+* `parameters`: parameters for the real extranetwork. If it is missing it is assumed `1` for LoRAs and HyperNets. If both this parameter and the parameter in the ext command are numbers they are multiplied for the result. In other case the parameter of the ext command, if it exists, is used.
+* `triggers`: list of trigger strings. If it is missing, only the inline triggers in the ext command will be added.
+* `weight`: weight for this variant, in case multiple of them apply, to choose one. Default is 1.
+
+See the file in the tests folder as an example.
+
+## Sending content to the negative prompt
+
+The new format for this command is like this:
+
+| Construct                             | Meaning                                                                              |
+| ---------                             | -------                                                                              |
+| `<ppp:stn position>content<ppp:/stn>` | send to negative prompt                                                              |
+| `<ppp:stn iN/>`                       | insertion point to be used in the negative prompt as destination for the pN position |
+
+Where position is optional (defaults to the start) and can be:
+
+* **s**: at the start of the negative prompt
+* **e**: at the end of the negative prompt
+* **pN**: at the position of the insertion point in the negative prompt with N being 0-9. If the insertion point is not found it inserts at the start.
+
+### Example
+
+You have a wildcard for hair colors (`__haircolors__`) with one being strawberry blonde, but you don't want strawberries. So in that option you add a command to add to the negative prompt, like so:
+
+```text
+blonde
+strawberry blonde <ppp:stn>strawberry<ppp:/stn>
+brunette
+```
+
+Then, if that option is chosen this extension will process it later and move that part to the negative prompt.
+
+### Old format
+
+The old format (`<!...!>`) is not supported anymore.
+
+### Notes
+
+Positional insertion commands have less priority that start/end commands, so even if they are at the start or end of the negative prompt, they will end up inside any start/end (and default position) commands.
+
+The content of the negative commands is not processed and is copied as-is to the negative prompt. Other modifiers around the commands are processed in the following way.
+
+### Attention modifiers (weights)
+
+They will be translated to the negative prompt. For example:
+
+* `(red<ppp:stn>square<ppp:/stn>:1.5)` will end up as `(square:1.5)` in the negative prompt
+* `(red[<ppp:stn>square<ppp:/stn>]:1.5)` will end up as `(square:1.35)` in the negative prompt (weight=1.5*0.9) if the merge attention option is enabled or `([square]:1.5)` otherwise.
+* `(red<ppp:stn>[square]<ppp:/stn>:1.5)` will also end up as `(square:1.35)` in the negative prompt if the merge attention option is enabled, because the content of the negative tag is a single attention construct whose weight is merged with the surrounding modifier.
+
+### Prompt editing constructs (alternation and scheduling)
+
+Negative commands inside such constructs will copy the construct to the negative prompt, but separating its elements. For example:
+
+* **Alternation**: `[red<ppp:stn>square<ppp:/stn>|blue<ppp:stn>circle<ppp:/stn>]` will end up as `[square|], [|circle]` in the negative prompt, instead of `[square|circle]`
+* **Scheduling**: `[red<ppp:stn>square<ppp:/stn>:blue<ppp:stn>circle<ppp:/stn>:0.5]` will end up as `[square::0.5], [:circle:0.5]` instead of `[square:circle:0.5]`
+
+This should still work as intended, and the only negative point i see is the unnecessary separators.

grammar.lark

@@ -0,0 +1,211 @@
+%import common (CNAME, INT, SIGNED_NUMBER, NUMBER)
+
+_WHITESPACE:         /\s+/
+STRING:              /("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/i // string with single or double quotes
+BOOLEAN:             /true|false/i
+WC_NAME_PLAIN_START: /(?:(?!__|\$\$|\$\{|<ppp:|[('"])[a-zA-Z0-9\-.\\\/\*\?\[!\]])+/u // starting characters, excluding end and other inside constructs
+WC_NAME_PLAIN:       /(?:(?!__|\$\$|\$\{|<ppp:|[('"])[a-zA-Z0-9\-.\\\/\*\?\[!\]_#])+/u // continuing characters, including globbing characters and anonymous wildcard mark, excluding end and other inside constructs
+INDEX:               INT
+IDENTIFIER:          CNAME
+SIMPLEVALUE:         STRING | SIGNED_NUMBER | BOOLEAN
+VARNAME:             /(?!__)[a-zA-Z_][a-zA-Z0-9_]*/
+
+// plain text and weights
+?plain:                  /((?!__|\bAND\b|\${|\x1d)[^\\()\[\]:<>${]|\\.)+/s // exclude only the starting ones
+?plain_choice:           /((?!__|\bAND\b|\${|\$\$)[^\\()\[\]:<>${|}~@]|\\.)+/s // add the specific internal choice ones
+?plain_alternate:        /((?!__|\bAND\b|\${)[^\\()\[\]:<>${|]|\\.)+/s // add the specific internal alternate ones
+?plain_var:              /((?!__|\bAND\b|\${)[^\\()\[\]:<>${}]|\\.)+/s  // add the specific internal var ones
+?specialchars:           /[_{()\[\]:<>]|\$(?![{$])/ // include only the starting ones
+?specialchars_negtag:    /[_{()\[\]:<>!|}]|\$(?![{$])/ // add the internal negtag ones
+?specialchars_alternate: /[_{()\[\]:<>|]|\$(?![{$])/ // add the internal alternate ones
+?specialchars_choice:    /[_{()\[\]:<>|}]|\$(?![{$])/ // add the internal choice ones
+?specialchars_var:       /[_{()\[\]:<>}]|\$(?![{$])/ // add the internal var ones
+?numpar: _WHITESPACE? SIGNED_NUMBER _WHITESPACE?
+
+start: prompt negative_sep prompt
+?prompt: promptcomp | content
+negative_sep: /\x1d/
+
+// prompt composition with AND
+promptcomp.5: promptcomppart ( [ ":" numpar ] ( /\bAND\b/ promptcomppart [ ":" numpar ] )+ )+
+promptcomppart: content
+
+// simple prompts
+
+?old_content.3: ( attention | scheduled | alternate | extranetworktag )+
+//#if ALLOW_NEW_CONTENT
+    ?content.3: ( old_content | new_content | plain | specialchars )*
+    ?content_choice.3: ( old_content | new_content | plain_choice | specialchars_choice )*
+    ?content_var.3: ( old_content | new_content | plain_var | specialchars_var )*
+    ?content_negtag.3: ( old_content | new_content_negtag | plain | specialchars_negtag )*
+    ?content_alternate.3: ( old_content | new_content | plain_alternate | specialchars_alternate )*
+    ?content_en.3: (new_content_en | plain | specialchars )*
+    //#if ALLOW_WILDCARDS and ALLOW_CHOICES and ALLOW_COMMVARS
+        ?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
+        ?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
+        ?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif | choices )+
+    //#elif ALLOW_WILDCARDS and not ALLOW_CHOICES and ALLOW_COMMVARS
+        ?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard )+
+        ?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard )+
+        ?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif )+
+    //#elif not ALLOW_WILDCARDS and ALLOW_CHOICES and ALLOW_COMMVARS
+        ?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | choices )+
+        ?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext | choices )+
+        ?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif | choices )+
+    //#elif ALLOW_WILDCARDS and ALLOW_CHOICES and not ALLOW_COMMVARS
+        ?new_content.4: ( wildcard | choices )+
+        ?new_content_negtag.4: ( wildcard | choices )+
+        ?new_content_en.4: ( choices )+
+    //#elif ALLOW_WILDCARDS and not ALLOW_CHOICES and not ALLOW_COMMVARS
+        ?new_content.4: ( wildcard )+
+        ?new_content_negtag.4: ( wildcard )+
+        ?new_content_en.4: /(?!)./ // never matches
+    //#elif not ALLOW_WILDCARDS and ALLOW_CHOICES and not ALLOW_COMMVARS
+        ?new_content.4: ( choices )+
+        ?new_content_negtag.4: ( choices )+
+        ?new_content_en.4: ( choices )+
+    //#elif not ALLOW_WILDCARDS and not ALLOW_CHOICES and ALLOW_COMMVARS
+        ?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext )+
+        ?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext )+
+        ?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif )+
+    //#else
+        ?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
+        ?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
+        ?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif | choices )+
+    //#endif
+//#else
+    ?content.3: ( old_content | plain | specialchars )*
+    ?content_choice.3: ( old_content | plain_choice | specialchars_choice )*
+    ?content_var.3: ( old_content | plain_var | specialchars_var )*
+    ?content_negtag.3: ( old_content | plain | specialchars_negtag )*
+    ?content_alternate.3: ( old_content | plain_alternate | specialchars_alternate )*
+    ?content_en.3: (plain | specialchars )*
+//#endif
+
+// attention modifiers
+attention: ( "(" content [ ":" numpar ] ")" ) | ( "[" content "]" )
+
+// prompt scheduling and alternation
+alternate: "[" alternateoption ( "|" alternateoption )+ "]"
+alternateoption: content_alternate
+scheduled: "[" [ content ":" ] content ":" numpar "]"
+
+// extra network tags
+extranetworktag: "<" /(?!ppp:)\w+:/ encontent ">"
+?encontent.4: content_en
+
+//#if ALLOW_COMMVARS or ALLOW_CHOICES or ALLOW_WILDCARDS
+    vardescriptor_get.8: VARNAME [ /\[/ [ SIGNED_NUMBER | IDENTIFIER | separator_descriptor | /#/ ] /\]/ ]
+    separator_descriptor: /&/ STRING
+    vardescriptor_set.9: VARNAME [ /\[/ [ SIGNED_NUMBER | IDENTIFIER ] /\]/ ]
+
+    // conditions
+    ?condition:    operation_or
+    ?operation_or:  operation_and | operation_or _WHITESPACE "or"  _WHITESPACE operation_and
+    ?operation_and: atom_cond    | operation_and _WHITESPACE "and" _WHITESPACE atom_cond
+    ?atom_cond:    operation_not | grouped_condition | basic_condition
+    operation_not: "not" ( ( _WHITESPACE atom_cond ) | ( _WHITESPACE? grouped_condition ) )
+    ?grouped_condition.8: "(" _WHITESPACE? condition _WHITESPACE? ")"
+    ?basic_condition: comparison | truthy_operand
+    ?complexvalue: vardescriptor_get | SIMPLEVALUE
+    truthy_operand: vardescriptor_get
+    comparison: ( complexvalue | listvalue ) _WHITESPACE ( /not/ _WHITESPACE )? /eq|ne|lt|gt|le|ge|contains|in|any_in|contains_any/ _WHITESPACE ( complexvalue | listvalue )
+    listvalue.9: "(" ( _WHITESPACE? complexvalue ( _WHITESPACE? "," _WHITESPACE? complexvalue )* )? _WHITESPACE? ")"
+    varvalue: content_var
+//#endif
+
+//#if ALLOW_COMMVARS
+    // command: stn (send to negative)
+    commandstn: "<ppp:stn" [ _WHITESPACE /s|e|p\d/ ] _WHITESPACE? ">" content_negtag "<ppp:/stn>"
+    commandstni: "<ppp:stn" _WHITESPACE /i\d/ _WHITESPACE? ( "/>" | ">" ) // ">" ending for backward compatibility
+
+    // command: if
+    commandif: commandif_if commandif_elif* commandif_else? "<ppp:/if>"
+    commandif_if: "<ppp:if" _WHITESPACE condition _WHITESPACE? ">" ifvalue
+    commandif_elif: "<ppp:elif" _WHITESPACE condition _WHITESPACE? ">" ifvalue
+    commandif_else: "<ppp:else" _WHITESPACE? ">" ifvalue
+    ifvalue: content
+
+    // command: set
+    commandset: "<ppp:set" _WHITESPACE vardescriptor_set [ commandsetmodifiers ] _WHITESPACE? ">" ( starredvalue | commandsetcontent ) "<ppp:/set>"
+    commandsetmodifiers: (_WHITESPACE /evaluate|ifundefined|add/ )+
+    ?commandsetcontent: content
+
+    // variable set
+    //#if ALLOW_WILDCARDS
+    starredvalue.9: "*" ( wildcard | vardescriptor_get | listvalue )
+    //#else
+    starredvalue.9: "*" ( vardescriptor_get | listvalue )
+    //#endif
+    variableset: "${" _WHITESPACE? vardescriptor_set [ variablesetmodifiers ] _WHITESPACE? "=" [ /!/ ] ( starredvalue | varvalue ) "}"
+    variablesetmodifiers: /[+?!]+/
+
+    // command: echo
+    commandecho: "<ppp:echo" _WHITESPACE vardescriptor_get _WHITESPACE? ( ">" [ commandechodefault "<ppp:/echo>" ] | "/>" )
+    ?commandechodefault: content
+
+    // variable use
+    variableuse: "${" _WHITESPACE? vardescriptor_get _WHITESPACE? [ ":" varvalue ] "}"
+
+    // command: ext
+    commandext: "<ppp:ext" _WHITESPACE commandexttype _WHITESPACE commandextid [ _WHITESPACE? commandextparams ] [ _WHITESPACE? commandextif ] _WHITESPACE? ( ">" [ commandexttriggers "<ppp:/ext>" ] | "/>" )
+    commandexttype: [/\$/] IDENTIFIER
+    ?commandextid.3: STRING | CNAME
+    ?commandextparams: STRING | SIGNED_NUMBER
+    ?commandextif: "if" _WHITESPACE condition
+    ?commandexttriggers.4: content
+//#endif
+
+//#if ALLOW_CHOICES or ALLOW_WILDCARDS
+    // choices parameters (for choices and wildcards)
+    choicesoptions: [ choicesoptions_sampler ] [ _WHITESPACE? choicesoptions_flags ] [ _WHITESPACE? choicesoptions_range ] [ _WHITESPACE? choicesoptions_sep ]
+    choicesoptions_sampler: /[~@]/ // ~ for random, @ for cyclical
+    choicesoptions_flags: /[ro]{1,2}/ // r for repeating, o for optional
+    choicesoptions_range: INT | ( [ INT ] _WHITESPACE? "-" _WHITESPACE? [ INT ] )
+    choicesoptions_sep: "$$" plain
+
+    // choice options
+    choice: [ [ _WHITESPACE? choiceiscmd ] [ _WHITESPACE? choicelabels ] [ _WHITESPACE? choiceweight ] [ _WHITESPACE? choiceif ] _WHITESPACE? "::" ] choicevalue
+    choiceiscmd: /%/ // the option text is a special command
+    choicelabels: ( /"/ IDENTIFIER ( _WHITESPACE? "," _WHITESPACE? IDENTIFIER )* /"/ )
+                | ( /'/ IDENTIFIER ( _WHITESPACE? "," _WHITESPACE? IDENTIFIER )* /'/ )
+    choiceweight: NUMBER
+    choiceif: "if" _WHITESPACE condition
+    choicevalue: content_choice
+//#endif
+
+//#if ALLOW_CHOICES
+    // choices construct
+    choices: "{" [ choicesoptions_sampler | ( choicesoptions _WHITESPACE? "$$" ) ] choice ( "|" choice )* "}"
+//#endif
+
+//#if ALLOW_WILDCARDS
+    // wildcard definition options
+    wcdefoptions: [ choicesoptions_sampler ] [ _WHITESPACE? choicesoptions_flags ] [ _WHITESPACE? choicesoptions_range ] [ _WHITESPACE? wcdescription ] [ _WHITESPACE? choicesoptions_sep ]
+    wcdescription: STRING
+
+    // wildcards construct
+    wildcard: "__" [ choicesoptions_sampler | ( choicesoptions _WHITESPACE? "$$" ) ] wildcard_name [ wc_filter ] [ wildcardvar ] "__"
+    wc_filter_nums: IDENTIFIER | INDEX | (INDEX /-/ INDEX)
+    //#if ALLOW_COMMVARS
+        wildcard_name.2: ( WC_NAME_PLAIN_START | variableuse | commandecho ) ( WC_NAME_PLAIN | variableuse | commandecho )*
+        wc_filter_simple: wc_filter_nums | variableuse
+    //#else
+        wildcard_name.2: ( WC_NAME_PLAIN_START) WC_NAME_PLAIN*
+        wc_filter_simple: wc_filter_nums
+    //#endif
+    wc_filter: ( /"/ ( [ /#/ ] wc_filter_or | ( /#?\^/ wildcard_name ) ) /"/ )
+             | ( /'/ ( [ /#/ ] wc_filter_or | ( /#?\^/ wildcard_name ) ) /'/ )
+    wc_filter_or: wc_filter_and ( _WHITESPACE? "," _WHITESPACE? wc_filter_and )*
+    wc_filter_and: wc_filter_simple ( _WHITESPACE? "+" _WHITESPACE? wc_filter_simple )*
+    wildcardvar: "(" _WHITESPACE? vardescriptor_get _WHITESPACE? "=" varvalue ")"
+
+    //#if ALLOW_COMMVARS
+        // command: setwcdeffilter
+        commandsetwcdeffilter: "<ppp:setwcdeffilter" _WHITESPACE wc_name_str [_WHITESPACE wc_filter_limited ] _WHITESPACE? "/>"
+        wc_name_str: ( /"/ wildcard_name /"/ )
+                   | ( /'/ wildcard_name /'/ )
+        wc_filter_limited: ( /"/ wc_filter_or /"/ )
+                         | ( /'/ wc_filter_or /'/ )
+    //#endif
+//#endif

install.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+requirements_filename = str(Path(__file__).resolve().parent / "requirements.txt")
+
+try:
+    from modules.launch_utils import run_pip  # , requirements_met  # A1111
+    #if not requirements_met(requirements_filename): # fails in all hosts due to ruamel.yaml or not having specified versions
+    run_pip(f'install -r "{requirements_filename}"', "requirements for Prompt Post-Processor")
+except ImportError:
+    import launch
+    launch.run_pip(f'install -r "{requirements_filename}"', "requirements for Prompt Post-Processor")

metadata.ini

@@ -0,0 +1,5 @@
+[Extension]
+Name = sd-webui-prompt-postprocessor
+
+[Scripts]
+After = sd-dynamic-prompts, stable-diffusion-webui-wildcards

ppp.code-workspace

@@ -0,0 +1,68 @@
+{
+	"folders": [
+		{
+			"name": "sd-webui-prompt-postprocessor",
+			"path": "."
+		}
+	],
+	"settings": {
+		"python.analysis.extraPaths": [
+			"../.."
+		],
+		"python.testing.unittestArgs": [
+			"-v",
+			"-s",
+			".",
+			"-p",
+			"test*.py"
+		],
+		"python.testing.pytestEnabled": false,
+		"python.testing.unittestEnabled": true,
+		"python.analysis.typeCheckingMode": "off",
+		"yaml.schemaStore.enable": false,
+		"powershell.cwd": "sd-webui-prompt-postprocessor",
+		"[python]": {
+			"editor.defaultFormatter": "ms-python.black-formatter"
+		},
+		"black-formatter.args": [
+			"--line-length=120"
+		],
+		"python-envs.pythonProjects": [
+			{
+				"path": ".",
+				"envManager": "ms-python.python:venv",
+				"packageManager": "ms-python.python:pip"
+			}
+		]
+	},
+	"launch": {
+		"version": "0.2.0",
+		"configurations": [
+			{
+				"name": "Python Debugger: Attach using Process Id",
+				"type": "debugpy",
+				"request": "attach",
+				"processId": "${command:pickProcess}"
+			},
+			{
+				"name": "Python: Debug Tests (Testing Panel)",
+				"type": "debugpy",
+				"request": "launch",
+				"purpose": [
+					"debug-test"
+				],
+				"console": "integratedTerminal",
+				"justMyCode": false,
+				"cwd": "${workspaceFolder}"
+			},
+			{
+				"name": "Python Debugger (Current File)",
+				"type": "debugpy",
+				"request": "launch",
+				"program": "${file}",
+				"args": "${command:pickArgs}"
+			}
+		],
+		"compounds": []
+	}
+}

ppp_cache.py

@@ -0,0 +1,31 @@
+from collections import OrderedDict
+from logging import Logger
+from typing import Tuple
+
+from ppp_logging import DEBUG_LEVEL
+
+
+class PPPLRUCache:
+
+    ProcessInput = Tuple[int, int, str, str]  # (env_hash, seed, positive_prompt, negative_prompt)
+    ProcessResult = Tuple[str, str]  # (positive_prompt, negative_prompt)
+
+    def __init__(self, capacity: int, logger: Logger = None, debug_level: DEBUG_LEVEL = DEBUG_LEVEL.none):
+        self.cache = OrderedDict()
+        self.capacity = capacity
+        self._logger = logger
+        self._debug_level = debug_level
+
+    def get(self, key: ProcessInput) -> ProcessResult:
+        if key not in self.cache:
+            return None
+        self.cache.move_to_end(key)
+        return self.cache[key]
+
+    def put(self, key: ProcessInput, value: ProcessResult) -> None:
+        self.cache[key] = value
+        self.cache.move_to_end(key)
+        if len(self.cache) > self.capacity:
+            self.cache.popitem(last=False)
+        # if self._logger is not None and self._debug_level != DEBUG_LEVEL.none:
+        #     self._logger.debug(f"Cache size: {self.cache.__sizeof__()}")

ppp_classes.py

@@ -0,0 +1,301 @@
+"""Pydantic models for the PPP configuration file structure (ppp_config.yaml)."""
+
+from dataclasses import dataclass, field
+from logging import Logger
+import re
+from enum import Enum
+from typing import Any, Literal, Optional
+from lark import Lark
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from ppp_logging import DEBUG_LEVEL
+from ppp_wildcards import PPPWildcards
+from ppp_enmappings import PPPExtraNetworkMappings
+from ppp_variables import VariableRepository
+
+
+class SUPPORTED_APPS(Enum):
+    comfyui = "comfyui"
+    a1111 = "a1111"
+    forge = "forge"
+    forgeneo = "forgeneo"
+    reforge = "reforge"
+    sdnext = "sdnext"
+    tests = "tests"  # for testing purposes only, not a real app
+
+
+SUPPORTED_APPS_NAMES = {
+    SUPPORTED_APPS.comfyui: "ComfyUI",
+    SUPPORTED_APPS.sdnext: "SD.Next",
+    SUPPORTED_APPS.forge: "Forge Classic",
+    SUPPORTED_APPS.forgeneo: "Forge Neo",
+    SUPPORTED_APPS.reforge: "reForge",
+    SUPPORTED_APPS.a1111: "A1111 (or compatible)",
+    SUPPORTED_APPS.tests: "Tests",
+}
+
+
+class IFWILDCARDS_CHOICES(Enum):
+    ignore = "ignore"
+    remove = "remove"
+    warn = "warn"
+    stop = "stop"
+
+
+class ONWARNING_CHOICES(Enum):
+    warn = "warn"
+    stop = "stop"
+
+
+# ------------------- Host configuration -------------------
+
+AttentionOption = Literal["ok", "parentheses", "disable", "remove", "error"]
+SchedulingOption = Literal["ok", "before", "after", "first", "remove", "error"]
+AlternationOption = Literal["ok", "first", "remove", "error"]
+AndOption = Literal["ok", "eol", "comma", "remove", "error"]
+BreakOption = Literal["ok", "eol", "comma", "remove", "error"]
+
+
+class HostConfig(BaseModel):
+    """Configuration for a specific host application."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="forbid")
+
+    attention: AttentionOption = "ok"
+    scheduling: SchedulingOption = "ok"
+    alternation: AlternationOption = "ok"
+    and_: AndOption = Field("ok", alias="and")
+    break_: BreakOption = Field("ok", alias="break")
+    seed_bits: int = Field(
+        64, validator=lambda v: v if v in (8, 16, 32, 64) else ValueError("seed_bits must be one of 8, 16, 32, or 64")
+    )
+
+
+# ------------------- Model detection -------------------
+
+
+class ModelDetectConfig(BaseModel):
+    """Detection configuration for a specific host when loading a model."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    class_: Optional[list[str]] = Field(None, alias="class")
+    property: Optional[str] = None
+
+    @model_validator(mode="after")
+    def check_class_or_property(self) -> "ModelDetectConfig":
+        if self.class_ is None and self.property is None:
+            raise ValueError("Either 'class' or 'property' must be specified")
+        return self
+
+
+# ------------------- Variant find_in_filename -------------------
+
+
+class FindInFilenamePattern(BaseModel):
+    """A regex pattern with optional flags used to identify a model variant in the filename."""
+
+    regex: str
+    flags: int = 0
+
+    @field_validator("flags", mode="before")
+    @classmethod
+    def parse_flags(cls, v: object) -> int:
+        if isinstance(v, int):
+            return v
+        if isinstance(v, list):
+            flag_value = 0
+            for flag in v:
+                if not isinstance(flag, str) or not hasattr(re, flag):
+                    raise ValueError(f"Invalid regex flag '{flag}'")
+                flag_value |= getattr(re, flag)
+            return flag_value
+        raise ValueError(f"Expected int or list of flag-name strings, got {type(v).__name__}")
+
+    @model_validator(mode="after")
+    def validate_regex(self) -> "FindInFilenamePattern":
+        try:
+            re.compile(self.regex, self.flags)
+        except re.error as exc:
+            raise ValueError(f"Invalid regex pattern '{self.regex}': {exc}") from exc
+        return self
+
+
+class VariantConfig(BaseModel):
+    """Configuration for a specific model variant."""
+
+    find_in_filename: list[FindInFilenamePattern]
+
+    @field_validator("find_in_filename", mode="before")
+    @classmethod
+    def normalize_find_in_filename(cls, v: object) -> list:
+        """Normalize str / dict / list input to always be a list of FindInFilenamePattern-compatible dicts."""
+        if isinstance(v, str):
+            return [{"regex": v, "flags": re.IGNORECASE}]
+        if isinstance(v, dict):
+            return [v]
+        if isinstance(v, list):
+            normalized = []
+            for item in v:
+                if isinstance(item, str):
+                    normalized.append({"regex": item, "flags": re.IGNORECASE})
+                elif isinstance(item, dict):
+                    normalized.append(item)
+                else:
+                    raise ValueError(f"Expected str or dict in 'find_in_filename' list, got {type(item).__name__}")
+            return normalized
+        raise ValueError(f"Expected str, dict, or list for 'find_in_filename', got {type(v).__name__}")
+
+
+# ------------------- Model configuration -------------------
+
+
+class ModelConfig(BaseModel):
+    """Configuration for a supported base model."""
+
+    detect: Optional[dict[str, Optional[ModelDetectConfig]]] = None
+    variants: Optional[dict[str, VariantConfig]] = None
+
+    @model_validator(mode="after")
+    def check_detect_or_variants(self) -> "ModelConfig":
+        if self.detect is None and self.variants is None:
+            raise ValueError("At least one of 'detect' or 'variants' must be specified")
+        return self
+
+
+# ------------------- Top-level configuration -------------------
+
+
+class PPPConfig(BaseModel):
+    """Top-level PPP configuration structure matching ppp_config.yaml."""
+
+    hosts: Optional[dict[str, Optional[HostConfig]]] = None
+    models: Optional[dict[str, Optional[ModelConfig | None]]] = None
+
+    @model_validator(mode="after")
+    def check_hosts_or_models(self) -> "PPPConfig":
+        if self.hosts is None and self.models is None:
+            raise ValueError("At least one of 'hosts' or 'models' must be specified")
+        return self
+
+
+# ------------------- State object -------------------
+
+
+@dataclass(frozen=True)
+class PPPStateOptions:
+    """Options that can be set for prompt processing."""
+
+    debug_level: DEBUG_LEVEL = DEBUG_LEVEL.minimal
+    on_warning: ONWARNING_CHOICES = ONWARNING_CHOICES.warn
+    process_wildcards: bool = True
+    keep_choices_order: bool = True
+    choice_separator: str = ", "
+    if_wildcards: IFWILDCARDS_CHOICES = IFWILDCARDS_CHOICES.stop
+    stn_ignore_repeats: bool = True
+    stn_separator: str = ", "
+    cup_do_cleanup: bool = True  # whether to do cleanup at all (if False, all other cleanup options are ignored)
+    cup_cleanup_variables: bool = True
+    cup_extra_spaces: bool = True
+    cup_empty_constructs: bool = True
+    cup_extra_separators: bool = True
+    cup_extra_separators2: bool = True
+    cup_extra_separators_include_eol: bool = False
+    cup_breaks: bool = False
+    cup_breaks_eol: bool = False
+    cup_ands: bool = False
+    cup_ands_eol: bool = False
+    cup_extranetwork_tags: bool = False
+    cup_merge_attention: bool = True
+    cup_remove_extranetwork_tags: bool = False
+    strict_operators: bool = True
+    do_combinatorial: bool = False
+    combinatorial_shuffle: bool = False
+    combinatorial_limit: int = 100  # 0 = no limit
+    results_file: str = ""  # empty = disabled; supports %datetime%, %date%, %time%, %host% tokens
+
+    def __post_init__(self):
+        if not self.cup_do_cleanup:
+            object.__setattr__(self, "cup_cleanup_variables", False)
+            object.__setattr__(self, "cup_extra_spaces", False)
+            object.__setattr__(self, "cup_empty_constructs", False)
+            object.__setattr__(self, "cup_extra_separators", False)
+            object.__setattr__(self, "cup_extra_separators2", False)
+            object.__setattr__(self, "cup_extra_separators_include_eol", False)
+            object.__setattr__(self, "cup_breaks", False)
+            object.__setattr__(self, "cup_breaks_eol", False)
+            object.__setattr__(self, "cup_ands", False)
+            object.__setattr__(self, "cup_ands_eol", False)
+            object.__setattr__(self, "cup_extranetwork_tags", False)
+            object.__setattr__(self, "cup_merge_attention", False)
+            object.__setattr__(self, "cup_remove_extranetwork_tags", False)
+
+
+@dataclass
+class PPPStateInputs:
+    """Structured inputs for a single prompt processing call."""
+
+    seed: int = -1
+    pos_prompt: str = ""
+    neg_prompt: str = ""
+    jobinfo: Any = None
+
+
+class CyclicalSamplerState:
+    """Maintains the cycling position for '@' choice samplers across process_prompt calls."""
+
+    def __init__(self):
+        self.current_path: list[int] = []
+        self.last_trace: list[int] = []
+        self.last_prompt_pair: tuple[str, str] | None = None
+
+    def advance(self):
+        """Advance to the next combination, cycling back to the start when all are exhausted."""
+        if not self.last_trace:
+            self.current_path = []
+            return
+        path = list(self.current_path)
+        while len(path) < len(self.last_trace):
+            path.append(0)
+        # Mixed-radix increment: least significant position is last.
+        for i in range(len(path) - 1, -1, -1):
+            path[i] += 1
+            if path[i] < self.last_trace[i]:
+                break
+            path[i] = 0
+        self.current_path = path
+
+    def reset(self):
+        """Reset the cyclical state to the beginning."""
+        self.current_path = []
+        self.last_trace = []
+        self.last_prompt_pair = None
+
+
+@dataclass(frozen=True)
+class PPPState:
+    """State object passed to various PPP components during prompt processing."""
+
+    logger: Logger
+    env_info: dict[str, Any] = field(default_factory=dict)
+    host_config: HostConfig = field(default_factory=HostConfig)
+    options: PPPStateOptions = field(default_factory=PPPStateOptions)
+    inputs: PPPStateInputs = field(default_factory=PPPStateInputs)
+    variables: VariableRepository = field(default_factory=VariableRepository)
+    wildcards_obj: PPPWildcards = field(default_factory=PPPWildcards)
+    extranetwork_mappings_obj: PPPExtraNetworkMappings = field(default_factory=PPPExtraNetworkMappings)
+    parsers: dict[str, Lark] = field(default_factory=dict)
+    cyclical_state: CyclicalSamplerState = field(default_factory=CyclicalSamplerState)
+
+
+class PPPInterrupt(Exception):
+    """
+    Custom exception to handle interruptions in the PromptPostProcessor.
+    This exception can be raised to stop the processing of prompts.
+    """
+
+    def __init__(self, message: str = "Processing interrupted.", pos_prefix: str = "", neg_prefix: str = ""):
+        super().__init__(message)
+        self.message = message
+        self.pos_prefix = pos_prefix
+        self.neg_prefix = neg_prefix

ppp_common.py

@@ -0,0 +1,311 @@
+import ast
+import csv
+from functools import reduce
+import logging
+from pathlib import Path
+import re
+import textwrap
+import time
+import lark
+from ruamel.yaml import YAML as _YAML
+
+from ppp_logging import log
+from ppp_classes import ONWARNING_CHOICES, PPPInterrupt, PPPState
+from ppp_utils import escape_single_quotes, format_output
+
+
+def parse_prompt(
+    state: PPPState,
+    prompt_description: str,
+    prompt: str,
+    parser: lark.Lark,
+    raise_parsing_error: bool = False,
+):
+    """
+    Parses a prompt using the specified parser.
+
+    Args:
+        prompt_description (str): The description of the prompt.
+        prompt (str): The prompt to be parsed.
+        parser (lark.Lark): The parser to be used.
+        raise_parsing_error (bool): Whether to raise a parsing error.
+
+    Returns:
+        Tree: The parsed prompt.
+    """
+    t1 = time.monotonic_ns()
+    parsed_prompt = None
+    try:
+        log(
+            state.logger,
+            state.options.debug_level,
+            logging.DEBUG,
+            f"Parsing {prompt_description}: '{escape_single_quotes(prompt)}'",
+        )
+        parsed_prompt = parser.parse(prompt)
+        # we store the contents so we can use them later even if the meta position is not valid anymore
+        if isinstance(parsed_prompt, lark.Tree):
+            for n in parsed_prompt.iter_subtrees():
+                if isinstance(n, lark.Tree):
+                    if n.meta.empty:
+                        n.meta.content = ""
+                    else:
+                        n.meta.content = prompt[n.meta.start_pos : n.meta.end_pos]
+    except lark.exceptions.UnexpectedInput:
+        if raise_parsing_error:
+            raise
+        log(
+            state.logger,
+            state.options.debug_level,
+            logging.ERROR,
+            f"Parsing failed on prompt!: {escape_single_quotes(prompt)}",
+        )
+    t2 = time.monotonic_ns()
+    log(
+        state.logger,
+        state.options.debug_level,
+        logging.DEBUG,
+        f"Parse {prompt_description} time: {(t2 - t1) / 1_000_000_000:.3f} seconds",
+    )
+    if parsed_prompt:
+        log(
+            state.logger,
+            state.options.debug_level,
+            logging.DEBUG,
+            "Tree:\n"
+            + textwrap.indent(
+                re.sub(r"\n$", "", (parsed_prompt.pretty() if isinstance(parsed_prompt, lark.Tree) else parsed_prompt)),
+                "    ",
+            ),
+            formatted=False,
+        )
+    return parsed_prompt
+
+
+def warn_or_stop(state: PPPState, is_negative: bool, message: str, e: Exception = None):
+    INVALID_CONTENT_STOP = "INVALID CONTENT! {0}\nBREAK "
+    if state.options.on_warning == ONWARNING_CHOICES.stop:
+        raise PPPInterrupt(
+            message,
+            INVALID_CONTENT_STOP.format(message) if not is_negative else "",
+            INVALID_CONTENT_STOP.format(message) if is_negative else "",
+        ) from e
+    log(state.logger, state.options.debug_level, logging.WARNING, format_output(message))
+
+
+def load_grammar() -> str:
+    # Process with lark (debug with https://www.lark-parser.org/ide/)
+    grammar_filename = Path(__file__).resolve().parent / "grammar.lark"
+    with open(grammar_filename, "r", encoding="utf-8") as file:
+        grammar_content = file.read()
+    return grammar_content
+
+
+def preprocess_grammar(grammar_content: str, options: dict[str, bool], logger: logging.Logger, debug_level: int) -> str:
+    """
+    Preprocesses the grammar content to handle conditional compilation directives.
+
+    Args:
+        grammar_content (str): The raw grammar content.
+        options (dict[str,bool]): Options for preprocessing.
+        logger (logging.Logger): The logger object.
+        debug_level (int): The debug level for logging.
+
+    Returns:
+        str: The preprocessed grammar content.
+    """
+    lines = grammar_content.split("\n")
+    result_lines = []
+    skip_current_block = []
+    all_blocks_skipped = []
+
+    def eval_bool_expr(expr: str, constants: dict[str, bool]) -> bool:
+        """
+        Evaluates a boolean expression using known constants.
+        Supports: and, or, not, parentheses, and named constants.
+
+        Args:
+            expr (str): The boolean expression to evaluate.
+            constants (dict[str, bool]): A dictionary of constant values.
+        Returns:
+            bool: The result of the evaluated expression.
+        """
+        tree = ast.parse(expr, mode="eval")
+
+        def _eval(node) -> bool:
+            if isinstance(node, ast.Expression):
+                return _eval(node.body)
+            if isinstance(node, ast.BoolOp):
+                if isinstance(node.op, ast.And):
+                    return all(_eval(v) for v in node.values)
+                if isinstance(node.op, ast.Or):
+                    return any(_eval(v) for v in node.values)
+            if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not):
+                return not _eval(node.operand)
+            if isinstance(node, ast.Name):
+                return bool(constants[node.id])  # raises KeyError for unknown names
+            if isinstance(node, ast.Constant) and isinstance(node.value, bool):
+                return node.value
+            raise ValueError(f"Unsupported construct: {ast.dump(node)}")
+
+        return _eval(tree)
+
+    for line in lines:
+        stripped_line = line.strip()
+        if stripped_line.startswith("//#if"):
+            # Extract condition from the #if directive
+            conditions = stripped_line[5:].strip()
+            # Evaluate the conditions
+            skip_current_block.append(not eval_bool_expr(conditions, options))
+            all_blocks_skipped.append(skip_current_block[-1])
+        elif stripped_line.startswith("//#elif"):
+            if not skip_current_block:
+                log(logger, debug_level, logging.WARNING, "Unmatched //#elif directive found in grammar content.")
+            elif all_blocks_skipped[-1]:
+                # No earlier branch in this if/elif chain was taken, so evaluate this one.
+                # all_blocks_skipped[-1] tracks whether any branch has matched yet;
+                # skip_current_block[-1] tracks whether the current branch should be emitted.
+                conditions = stripped_line[7:].strip()
+                skip_current_block[-1] = not eval_bool_expr(conditions, options)
+                if not skip_current_block[-1]:
+                    all_blocks_skipped[-1] = False  # mark that a branch was taken
+            else:
+                # A previous branch already matched - skip all remaining elif/else branches.
+                skip_current_block[-1] = True
+        elif stripped_line.startswith("//#else"):
+            if not skip_current_block:
+                log(logger, debug_level, logging.WARNING, "Unmatched //#else directive found in grammar content.")
+            elif all_blocks_skipped[-1]:
+                skip_current_block[-1] = False
+            else:
+                skip_current_block[-1] = True
+        elif stripped_line.startswith("//#endif"):
+            if not skip_current_block:
+                log(logger, debug_level, logging.WARNING, "Unmatched //#endif directive found in grammar content.")
+            else:
+                skip_current_block.pop()
+                all_blocks_skipped.pop()
+        elif stripped_line.startswith("//#"):
+            log(
+                logger,
+                debug_level,
+                logging.WARNING,
+                f"Unrecognized directive found in grammar content: {stripped_line}",
+            )
+        elif not any(skip_current_block):
+            # Include the line if we're not skipping any current block
+            result_lines.append(stripped_line)
+    # Check for unclosed blocks at the end
+    if skip_current_block:
+        raise PPPInterrupt(
+            f"Found {len(skip_current_block)} unclosed conditional directive(s) at the end of the grammar file"
+        )
+    return "\n".join(result_lines)
+
+
+def get_model_class_from_filename(filename: str) -> str:
+    try:
+        import folder_paths  # type: ignore
+        import comfy.utils  # type: ignore
+        import comfy.model_detection as model_detection  # type: ignore
+    except ImportError:
+        return ""
+    import json
+
+    if not filename:
+        return ""
+    full_path = (
+        folder_paths.get_full_path("diffusion_models", filename)
+        or folder_paths.get_full_path("checkpoints", filename)
+        or folder_paths.get_full_path("unet", filename)
+    )
+    if not full_path or not full_path.lower().endswith((".safetensors", ".sft")):
+        return ""
+    try:
+        header_bytes = comfy.utils.safetensors_header(full_path)
+        if header_bytes is None:
+            return ""
+        header = json.loads(header_bytes)
+
+        # model_config_from_unet only inspects tensor shapes, not actual data.
+        # A lightweight proxy that exposes .shape and index access lets us avoid
+        # loading the full model into memory.
+        class _ShapeProxy:
+            def __init__(self, shape):
+                self.shape = shape
+
+            def __getitem__(self, i):
+                return self.shape[i]
+
+        mock_sd = {k: _ShapeProxy(v["shape"]) for k, v in header.items() if k != "__metadata__" and "shape" in v}
+
+        prefix = model_detection.unet_prefix_from_state_dict(mock_sd)
+        config = model_detection.model_config_from_unet(mock_sd, prefix)
+        return config.__class__.__name__ if config else ""
+    except Exception:  # pylint: disable=broad-except
+        return ""
+
+
+def sanitize_wc_name(name: str) -> str:
+    # Remove invalid characters
+    return re.sub(r"[^a-zA-Z0-9-_]+", "", re.sub(r"_{2,}", "_", name.replace(" ", "_")))
+
+
+def convert_a1111_styles_to_wildcard(inp: Path, out: Path):
+    """
+    Converts styles from A1111 format to wildcard format in a YAML file.
+    Args:
+        inp (Path): The input CSV file path.
+        out (Path): The output YAML file path.
+    """
+    wildcards = {}
+    with open(inp, "r", encoding="utf-8-sig") as f:
+        for reg in csv.reader(f):
+            name = reg[0].strip()
+            if name.lower() == "name":
+                continue
+            positive = reg[1].strip()
+            negative = reg[2].strip()
+            if negative:
+                positive = f"{positive}<ppp:stn>{negative}<ppp:/stn>"
+            name = sanitize_wc_name(name)
+            wildcards[name] = positive
+    if not wildcards:
+        raise RuntimeError(f"No styles found in {inp} to convert to wildcards.")
+    with open(out, "w", encoding="utf-8-sig") as f:
+        f.write(f"# Original names may contain characters that are replaced in the output.\n# Converted from {inp}\n")
+        _YAML().dump(wildcards, f)
+
+
+def convert_sdnext_styles_to_wildcard(inp: Path, out: Path):
+    """
+    Converts styles from SD.Next format to wildcard format in a YAML file.
+    Args:
+        inp (Path): The input folder path containing style json files or a single json file.
+        out (Path): The output YAML file path.
+    """
+    wildcards = {}
+    files = inp.glob("*.json") if inp.is_dir() else [inp]
+    for file in files:
+        with open(file, "r", encoding="utf-8-sig") as f:
+            data = _YAML(typ='safe').load(f)
+            if not isinstance(data, list):
+                continue
+            wildcards[file] = {}
+            for style in data:
+                name = style.get("name", "").strip()
+                positive = style.get("prompt", "").strip()
+                negative = style.get("negative", "").strip()
+                # extra = style.get("extra", "").strip()
+                if negative:
+                    positive = f"{positive}<ppp:stn>{negative}<ppp:/stn>"
+                name = sanitize_wc_name(name)
+                wildcards[file][name] = positive
+    if not reduce(lambda acc, d: acc or bool(d), wildcards.values(), False):
+        raise RuntimeError(f"No styles found in {inp} to convert to wildcards.")
+    with open(out, "w", encoding="utf-8-sig") as f:
+        f.write("# Original names may contain characters that are replaced in the output.\n")
+        for name, wcs in wildcards.items():
+            f.write(f"# Converted from {name}\n")
+            yaml_writer = _YAML()
+            yaml_writer.dump(wcs, f)

ppp_config.yaml.defaults

@@ -0,0 +1,339 @@
+# Prompt Post-Processor configuration file.
+
+# If you want to change any of these options make first a copy of this file as "ppp_config.yaml" and edit that file.
+# In ComfyUI the file should be in the user folder.
+# This defaults file will be overwritten when updating the extension, but your custom "ppp_config.yaml" should not be affected.
+
+# Options for supported host applications
+hosts:
+    # For each host, define how to handle some operations:
+    #   attention: how to handle attention operators
+    #     - ok: leave as is
+    #     - parentheses: convert to parentheses-based attention
+    #     - disable: remove attention operators
+    #     - remove: remove the whole block, including the text inside
+    #     - error: raise an error
+    #   scheduling: how to handle scheduling operators
+    #     - ok: leave as is
+    #     - before: replace with the first scheduling text
+    #     - after: replace with the second scheduling text
+    #     - first: replace with the first scheduling text if present, otherwise with the second scheduling text
+    #     - remove: remove the whole block, including the text inside
+    #     - error: raise an error
+    #   alternation: how to handle alternation operators
+    #     - ok: leave as is
+    #     - first: replace with the first alternation text
+    #     - remove: remove the whole block, including the text inside
+    #     - error: raise an error
+    #   and: how to handle "AND" operators
+    #     - ok: leave as is
+    #     - eol: replace with end-of-line (newline)
+    #     - comma: replace with comma
+    #     - remove: remove the operator
+    #     - error: raise an error
+    #   break: how to handle "BREAK" operators
+    #     - ok: leave as is
+    #     - eol: replace with end-of-line (newline)
+    #     - comma: replace with comma
+    #     - remove: remove the operator
+    #     - error: raise an error
+    #   seed_bits: number of bits to use for the seed; if not specified, it defaults to 64 bits
+    # Options in the user file override all these options by host (individual options are not inherited and if missing will use defaults in code).
+    a1111:
+        attention: ok
+        scheduling: ok
+        alternation: ok
+        and: ok
+        break: ok
+        seed_bits: 32
+    forge:
+        attention: ok
+        scheduling: ok
+        alternation: ok
+        and: ok
+        break: ok
+        seed_bits: 32
+    forgeneo:
+        attention: ok
+        scheduling: ok
+        alternation: ok
+        and: ok
+        break: ok
+        seed_bits: 32
+    reforge:
+        attention: ok
+        scheduling: ok
+        alternation: ok
+        and: ok
+        break: ok
+        seed_bits: 32
+    sdnext:
+        attention: ok
+        scheduling: ok
+        alternation: ok
+        and: ok
+        break: ok
+        seed_bits: 32
+    comfyui:
+        # ComfyUI only supports parentheses-based attention
+        attention: parentheses
+        # by default ComfyUI does not support scheduling, alternation, AND or BREAK operators, but you may be able to use them with a custom text encoder node
+        scheduling: error
+        alternation: error
+        and: comma
+        break: comma
+        seed_bits: 64
+
+# Supported base models, variants, and options
+# Check supported models for each host in:
+#   a1111:
+#   forge: repositories\huggingface_guess\huggingface_guess\model_list.py
+#   forgeneo: modules_forge\packages\huggingface_guess\model_list.py
+#   reforge: ldm_patched\modules\supported_models.py
+#   sdnext: modules\shared_items.py
+#   comfyui: ComfyUI\comfy\supported_models.py
+models:
+    # We define supported models and how we detect them in each host application (by class or by a known boolean property, or null for not supported).
+    # We can also define here variants and some options.
+    # For each model, detect options in the user file replace these ones by host, and variants are fully replaced (so if you want to add to the default ones you need to copy them to your file).
+    sd1: # Stable Diffusion 1
+        detect:
+            a1111: { property: "is_sd1" }
+            forge: { property: "is_sd1", class: ["SD15", "SD15_instructpix2pix"] }
+            forgeneo: { property: "is_sd1", class: ["SD15"] }
+            reforge: { property: "is_sd1" }
+            sdnext: { class: ["LatentDiffusion", "StableDiffusionPipeline", "StableDiffusionInpaintPipeline", "StableDiffusionInstructPix2PixPipeline", "StableDiffusionUpscalePipeline"] } # LatentDiffusion is for the original backend, StableDiffusionPipeline is for the diffusers backend; cannot differentiate SD1 and SD2, we set both to True
+            comfyui: { class: ["SD15", "SD15_instructpix2pix"] }
+    sd2: # Stable Diffusion 2
+        detect:
+            a1111: { property: "is_sd2" }
+            forge: { property: "is_sd2", class: ["SD20", "SD21UnclipL", "SD21UnclipH"] }
+            forgeneo: null
+            reforge: { property: "is_sd2" }
+            sdnext: { class: ["LatentDiffusion", "StableDiffusionPipeline", "StableDiffusionInpaintPipeline", "StableDiffusionInstructPix2PixPipeline", "StableDiffusionUpscalePipeline"] } # cannot differentiate SD1 and SD2, we set both to True; LatentDiffusion is for the original backend, StableDiffusionPipeline is for the diffusers backend
+            comfyui: { class: ["SD20", "SD21UnclipL", "SD21UnclipH", "LotusD"] }
+    ssd: # Segmind Stable Diffusion 1B
+        detect:
+            a1111: { property: "is_ssd" }
+            forge: { class: ["SSD1B"] }
+            forgeneo: null
+            reforge: { property: "is_ssd" }
+            sdnext: null
+            comfyui: { class: ["SSD1B"]}
+    sdxl: # Stable Diffusion XL
+        detect:
+            a1111: { property: "is_sdxl" }
+            forge: { property: "is_sdxl", class: ["SDXL", "SDXLRefiner", "SDXL_instructpix2pix", "Segmind_Vega", "KOALA_700M", "KOALA_1B"] }
+            forgeneo: { property: "is_sdxl", class: ["SDXL", "SDXLRefiner"] }
+            reforge: { property: "is_sdxl" }
+            sdnext: { class: ["StableDiffusionXLPipeline", "StableDiffusionXLImg2ImgPipeline", "StableDiffusionXLInpaintPipeline", "StableDiffusionXLInstructPix2PixPipeline"] }
+            comfyui: { class: ["SDXL", "SDXLRefiner", "SDXL_instructpix2pix", "Segmind_Vega", "KOALA_700M", "KOALA_1B"] }
+        variants:
+            # At this level goes the name of the defined variants
+            #   find_in_filename: regex patterns as a str or regex/flags dict or list of them to identify the variant in the model filename.
+            #                     If specified as a string, it is considered a regex with the IGNORECASE flag;
+            #                     If specified as a dict, it should have "regex" and optional "flags" keys;
+            #                     If specified as a list, it should be a list of dicts or strings
+            pony: { find_in_filename: "(?:pony|pny|pdxl)" }
+            illustrious: { find_in_filename: { regex: "(?:illustrious|illust|ilxl)", flags: ["IGNORECASE"] } }
+            noobai: { find_in_filename: "(?:noob|nai)" }
+    sd3: # Stable Diffusion 3
+        detect:
+            a1111: { property: "is_sd3" }
+            forge: { property: "is_sd3", class: ["SD3"] }
+            forgeneo: null
+            reforge: { property: "is_sd3" }
+            sdnext: { class: ["StableDiffusion3Pipeline"] }
+            comfyui: { class: ["SD3"] }
+    flux: # Flux 1
+        detect:
+            a1111: null
+            forge: { class: ["Flux", "FluxSchnell"] }
+            forgeneo: { class: ["Flux", "FluxSchnell"] }
+            reforge: { class: ["Flux", "FluxInpaint", "FluxSchnell"] }
+            sdnext: { class: ["FluxPipeline"] }
+            comfyui: { class: ["Flux", "FluxInpaint", "FluxSchnell"] }
+    auraflow: # AuraFlow
+        detect:
+            a1111: null
+            forge: { class: ["AuraFlow"] }
+            forgeneo: null
+            reforge: { class: ["AuraFlow"] }
+            sdnext: { class: ["AuraFlowPipeline"] }
+            comfyui: { class: ["AuraFlow"] }
+    pixart: # PixArt
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: { class: ["PixArtAlpha", "PixArtSigma"] }
+            sdnext: { class: ["PixArtAlphaPipeline", "PixArtSigmaPipeline"] }
+            comfyui: { class: ["PixArtAlpha", "PixArtSigma"] }
+    lumina2: # Lumina2
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: { class: ["Lumina2"] }
+            reforge: { class: ["Lumina2"] }
+            sdnext: { class: ["Lumina2Pipeline"] }
+            comfyui: { class: ["Lumina2"] }
+    zimage: # ZImage
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: { class: ["ZImage"] }
+            reforge: { class: ["ZImage"] }
+            sdnext: { class: ["ZImagePipeline"] }
+            comfyui: { class: ["ZImage"] }
+    ltxv: # LTXV
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: null
+            sdnext: null
+            comfyui: { class: ["LTXV", "LTXAV"] }
+    cosmos: # Cosmos
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: { class: ["CosmosT2V", "CosmosI2V"] }
+            sdnext: { class: ["Cosmos2TextToImagePipeline"] }
+            comfyui: { class: ["CosmosT2V", "CosmosI2V"] }
+    cosmospredict2: # CosmosPredict2
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: { class: ["CosmosT2IPredict2", "CosmosI2VPredict2"] }
+            sdnext: null
+            comfyui: { class: ["CosmosT2IPredict2", "CosmosI2VPredict2"] }
+    genmomochi: # GenmoMochi
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: { class: ["GenmoMochi"] }
+            sdnext: null
+            comfyui: { class: ["GenmoMochi"] }
+    hunyuan: # Hunyuan
+        detect:
+            a1111: null
+            forge: { class: ["HunyuanDiT", "HunyuanDiT1"] }
+            forgeneo: null
+            reforge: { class: ["HunyuanDiT", "HunyuanDiT1"] }
+            sdnext: { class: ["HunyuanDiTPipeline", "HunyuanImagePipeline"] }
+            comfyui: { class: ["HunyuanDiT", "HunyuanDiT1"] }
+    hunyuanvideo: # HunyuanVideo
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: { class: ["HunyuanVideo", "HunyuanVideoI2V", "HunyuanVideoSkyreelsI2V", "HunyuanImage21", "HunyuanImage21Refiner", "HunyuanVideo15", "HunyuanVideo15_SR_Distilled"] }
+            sdnext: null
+            comfyui: { class: ["HunyuanVideo", "HunyuanVideoI2V", "HunyuanVideoSkyreelsI2V", "HunyuanImage21", "HunyuanImage21Refiner", "HunyuanVideo15", "HunyuanVideo15_SR_Distilled"] }
+    hunyuan3d: # Hunyuan3D
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: { class: ["Hunyuan3Dv2", "Hunyuan3Dv2_1", "Hunyuan3Dv2mini"] }
+            sdnext: null
+            comfyui: { class: ["Hunyuan3Dv2", "Hunyuan3Dv2_1", "Hunyuan3Dv2mini"] }
+    wan: # Wan21 & Wan22
+        detect:
+            a1111: null
+            forge: { class: ["WAN21_T2V", "WAN21_I2V"] }
+            forgeneo: { class: ["WAN21_T2V", "WAN21_I2V"] }
+            reforge: { class: ["WAN21_T2V", "WAN21_I2V", "WAN21_FunControl2V", "WAN21_Camera", "WAN22_Camera", "WAN21_Vace", "WAN21_HuMo", "WAN22_S2V", "WAN22_Animate", "WAN22_T2V"] }
+            sdnext: { class: ["WanPipeline"] }
+            comfyui: { class: ["WAN21_T2V", "WAN21_I2V", "WAN21_FunControl2V", "WAN21_Camera", "WAN22_Camera", "WAN21_Vace", "WAN21_HuMo", "WAN22_S2V", "WAN22_Animate", "WAN22_T2V", "WAN21_FlowRVS", "WAN21_SCAIL", "WAN22_WanDancer"] }
+    hidream: # HiDream
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: { class: ["HiDream"] }
+            sdnext: { class: ["HiDreamImagePipeline"] }
+            comfyui: { class: ["HiDream", "HiDreamO1"] }
+    qwenimage: # QwenImage & QwenImageEdit
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: { class: ["QwenImage"] }
+            reforge: null
+            sdnext: { class: ["QwenImagePipeline"] }
+            comfyui: { class: ["QwenImage"] }
+    chroma: # Chroma
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: { class: ["Chroma"] }
+            reforge: { class: ["Chroma", "ChromaRadiance"] }
+            sdnext: { class: ["ChromaPipeline"] }
+            comfyui: { class: ["Chroma", "ChromaRadiance"] }
+    omnigen2: # Omnigen 2
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: { class: ["Omnigen2"] }
+            sdnext: { class: ["OmniGen2Pipeline"] }
+            comfyui: { class: ["Omnigen2"] }
+    flux2: # Flux 2
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: { class: ["Flux2K4B", "Flux2K9B"] }
+            reforge: null
+            sdnext: { class: ["Flux2Pipeline", "Flux2KleinPipeline"] }
+            comfyui: { class: ["Flux2", "Flux2K4B", "Flux2K9B"] }
+    kandinsky5: # Kandinsky 5
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: null
+            sdnext: { class: ["Kandinsky5T2IPipeline"] }
+            comfyui: { class: ["Kandinsky5", "Kandinsky5Image"] }
+    anima: # Anima
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: { class: ["Anima"] }
+            reforge: null
+            sdnext: { class: ["AnimaText2ImagePipeline"] }
+            comfyui: { class: ["Anima"] }
+    longcatimage: # LongCatImage
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: null
+            sdnext: { class: ["LongCatImagePipeline"] }
+            comfyui: { class: ["LongCatImage"] }
+    ernieimage: # ErnieImage
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: { class: ["ErnieImage"] }
+            reforge: null
+            sdnext: { class: ["ErnieImagePipeline"] }
+            comfyui: { class: ["ErnieImage"] }
+    mugen: # Mugen
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: { class: ["Mugen"] }
+            reforge: null
+            sdnext: null
+            comfyui: { class: ["Mugen"] }
+    cogvideox: # CogVideoX
+        detect:
+            a1111: null
+            forge: null
+            forgeneo: null
+            reforge: null
+            sdnext: null
+            comfyui: { class: ["CogVideoX_T2V", "CogVideoX_I2V", "CogVideoX_Inpaint"] }

ppp_enmappings.py

@@ -0,0 +1,341 @@
+from pathlib import Path
+from typing import Optional
+import logging
+from ruamel.yaml import YAML as _YAML
+from ruamel.yaml.error import YAMLError as _YAMLError
+
+from ppp_logging import DEBUG_LEVEL, log
+from ppp_utils import deep_freeze, escape_single_quotes
+
+
+class PPPENMappingVariant:
+    """
+    A class to represent a variant of an extra network mapping.
+
+    Attributes:
+        condition (str): The condition for the variant.
+        name (str): The name of the variant.
+        parameters (float|str): The parameters for the variant.
+        triggers (list[str]): The triggers for the variant.
+        weight (float): The weight for the variant when multiple variants apply.
+    """
+
+    def __init__(self, condition: str, name: str, parameters: float | str, triggers: list[str], weight: float):
+        self.condition: str = condition
+        self.name: str = name
+        self.parameters: float | str = parameters
+        self.triggers: list[str] = triggers
+        self.weight: float = weight
+
+
+class PPPENMapping:
+    """
+    A extra network mapping object.
+
+    Attributes:
+        kind (str): The kind of the extra network.
+        name (str): The name of the extra network mapping.
+        file (Path | None): The path to the file where the extranetwork mapping is defined, or None if from inline input.
+        variants (list[PPPENMappingVariant]): The processed variants of the extranetwork mapping.
+    """
+
+    def __init__(self, fullpath: Path | None, kind: str, name: str, variants: list[dict]):
+        self.file: Path | None = fullpath
+        self.kind: str = kind
+        self.name: str = name
+        self.variants: list[PPPENMappingVariant] = [
+            PPPENMappingVariant(
+                **{**{"condition": None, "name": None, "parameters": None, "triggers": None, "weight": 1.0}, **v}
+            )
+            for v in variants
+        ]
+
+    def __hash__(self) -> int:
+        t = (self.kind, self.name, deep_freeze(self.variants))
+        return hash(t)
+
+    def __sizeof__(self):
+        return (
+            self.kind.__sizeof__()
+            + self.name.__sizeof__()
+            + (self.file.__sizeof__() if self.file is not None else 0)
+            + self.variants.__sizeof__()
+        )
+
+
+class PPPExtraNetworkMappings:
+    """
+    A class to manage extra network mappings.
+
+    Attributes:
+        extranetwork_maps (dict[str, PPPENMapping]): The extra network mappings.
+    """
+
+    DEFAULT_ENMAPPINGS_FOLDER = "extranetworkmappings"
+
+    def __init__(self, logger=None):
+        self.__logger: logging.Logger = logger
+        self.__debug_level = DEBUG_LEVEL.none
+        self.__enmappings_folders: list[Path] = []
+        self.__enmappings_files: dict[Path, float] = {}
+        self.__local_enmappings_input_hash: int | None = None
+        self.extranetwork_mappings: dict[str, PPPENMapping] = {}
+        self.cached_mappings = {}
+
+    def __hash__(self) -> int:
+        return hash(deep_freeze(self.extranetwork_mappings))
+
+    def __sizeof__(self):
+        return (
+            self.extranetwork_mappings.__sizeof__()
+            + self.__enmappings_folders.__sizeof__()
+            + self.__enmappings_files.__sizeof__()
+            + self.cached_mappings.__sizeof__()
+        )
+
+    def get_mapping(self, key: str) -> PPPENMapping | None:
+        """
+        Get an extra network mapping by key.
+
+        Args:
+            key (str): The key of the extra network mapping in the format "kind:name".
+        Returns:
+            PPPENMapping | None: The extra network mapping if found, or None if not found
+        """
+        for k, v in self.extranetwork_mappings.items():
+            if k.lower() == key.lower():
+                return v
+        return None
+
+    def refresh_extranetwork_mappings(
+        self,
+        debug_level: DEBUG_LEVEL,
+        enmappings_folders: Optional[list[Path]],
+        enmappings_input: str = None,
+    ):
+        """
+        Initialize the extra network mappings.
+        """
+        self.__debug_level = debug_level
+        self.__enmappings_folders = [Path(f) for f in (enmappings_folders or [])]
+        # log(self.__logger, self.__debug_level, logging.INFO, "Refreshing extra network mappings...")
+        # t1 = time.monotonic_ns()
+        self.cached_mappings = {}
+        for fullpath in list(self.__enmappings_files.keys()):
+            if not fullpath.exists() or not any(
+                fullpath.parent.is_relative_to(folder) for folder in self.__enmappings_folders
+            ):
+                self.__remove_extranetwork_mappings_from_path(fullpath)
+        if enmappings_input is None and self.__local_enmappings_input_hash is not None:
+            self.__remove_extranetwork_mappings_from_input()
+        if enmappings_folders is not None or enmappings_input is not None:
+            if enmappings_folders is not None:
+                for f in self.__enmappings_folders:
+                    self.__get_extranetwork_mappings_in_path(f)
+            if enmappings_input is not None:
+                self.__get_extranetwork_mappings_in_input(enmappings_input)
+        else:
+            self.extranetwork_mappings = {}
+            self.__enmappings_files = {}
+            self.__local_enmappings_input_hash = None
+        # t2 = time.monotonic_ns()
+        # log(self.__logger, self.__debug_level, logging.INFO, f"Extra network mappings refresh time: {(t2 - t1) / 1_000_000_000:.3f} seconds")
+
+    def __remove_extranetwork_mappings_from_path(self, full_path: Path, debug=True):
+        """
+        Clear all extra network mappings from a file.
+
+        Args:
+            full_path (Path): The path to the file.
+            debug (bool): Whether to print debug messages or not.
+        """
+        if debug and full_path in self.__enmappings_files:
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.DEBUG,
+                f"Removing extra network mappings from file: {full_path}",
+            )
+        if full_path in self.__enmappings_files:
+            del self.__enmappings_files[full_path]
+        for key in list(self.extranetwork_mappings.keys()):
+            if self.extranetwork_mappings[key].file == full_path:
+                del self.extranetwork_mappings[key]
+
+    def __remove_extranetwork_mappings_from_input(self, debug=True):
+        """
+        Clear all extra network mappings loaded from inline input.
+
+        Args:
+            debug (bool): Whether to print debug messages or not.
+        """
+        if debug and self.__local_enmappings_input_hash is not None:
+            log(self.__logger, self.__debug_level, logging.DEBUG, "Removing extra network mappings from input")
+        self.__local_enmappings_input_hash = None
+        for key in list(self.extranetwork_mappings.keys()):
+            if self.extranetwork_mappings[key].file is None:
+                del self.extranetwork_mappings[key]
+
+    def __get_extranetwork_mappings_in_file(self, full_path: Path):
+        """
+        Get all extra network mappings in a file.
+
+        Args:
+            full_path (Path): The path to the file.
+        """
+        last_modified = full_path.stat().st_mtime
+        last_modified_cached = self.__enmappings_files.get(full_path, None)
+        if last_modified_cached is not None and last_modified == self.__enmappings_files[full_path]:
+            return
+        extension = full_path.suffix
+        if extension not in (".yaml", ".yml", ".json"):
+            return
+        self.__remove_extranetwork_mappings_from_path(full_path, False)
+        if last_modified_cached is not None:
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.DEBUG,
+                f"Updating extra network mappings from file: {full_path}",
+            )
+        self.__get_extranetwork_mappings_in_structured_file(full_path)
+        self.__enmappings_files[full_path] = last_modified
+
+    def __get_extranetwork_mappings_in_input(self, enmappings_input: str):
+        """
+        Get all extra network mappings in the string.
+
+        Args:
+            enmappings_input (str): The input string containing extra network mappings in yaml format.
+        """
+        new_h = hash(enmappings_input)
+        if new_h == self.__local_enmappings_input_hash:
+            return
+        was_loaded = self.__local_enmappings_input_hash is not None
+        self.__remove_extranetwork_mappings_from_input(False)
+        if was_loaded:
+            log(self.__logger, self.__debug_level, logging.DEBUG, "Updating extra network mappings from input")
+        enmappings_input = enmappings_input.strip()
+        if enmappings_input != "":
+            try:
+                content = _YAML(typ='safe').load(enmappings_input)
+            except _YAMLError as e:
+                log(
+                    self.__logger,
+                    self.__debug_level,
+                    logging.WARNING,
+                    f"Invalid format for input extra network mappings: {e}",
+                )
+                return
+            if content is not None:
+                self.__add_extranetwork_mapping(content, None)
+        self.__local_enmappings_input_hash = new_h
+
+    def __add_extranetwork_mapping(self, content: dict[str, dict[str, list[dict]]], full_path: Path | None):
+        """
+        Add an extra network mapping to the extra network mappings dictionary.
+
+        Args:
+            content (object): The content of the extra network mapping.
+            full_path (Path | None): The path to the file that contains it, or None if from inline input.
+        """
+        file_str = str(full_path) if full_path is not None else "input"
+        if not isinstance(content, dict):
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.WARNING,
+                f"Invalid extra network mapping in file '{escape_single_quotes(file_str)}'!",
+            )
+            return
+        for kind, maps in content.items():
+            if not isinstance(maps, dict):
+                log(
+                    self.__logger,
+                    self.__debug_level,
+                    logging.WARNING,
+                    f"Invalid extra network mapping definition for '{escape_single_quotes(kind)}:*' in file '{escape_single_quotes(file_str)}'!",
+                )
+            else:
+                for name, variants in maps.items():
+                    key = f"{kind}:{name}"
+                    if not isinstance(variants, list):
+                        log(
+                            self.__logger,
+                            self.__debug_level,
+                            logging.WARNING,
+                            f"Invalid extra network mapping definition for '{escape_single_quotes(key)}' in file '{escape_single_quotes(file_str)}'!",
+                        )
+                    elif self.extranetwork_mappings.get(key, None) is not None:
+                        f = (
+                            str(self.extranetwork_mappings[key].file)
+                            if self.extranetwork_mappings[key].file is not None
+                            else "input"
+                        )
+                        log(
+                            self.__logger,
+                            self.__debug_level,
+                            logging.WARNING,
+                            f"Duplicate extra network mapping '{escape_single_quotes(key)}' in file '{escape_single_quotes(file_str)}' and '{escape_single_quotes(f)}'!",
+                        )
+                    elif not isinstance(variants, list) or not all(isinstance(v, dict) for v in variants):
+                        log(
+                            self.__logger,
+                            self.__debug_level,
+                            logging.WARNING,
+                            f"Invalid extra network mapping definition for '{escape_single_quotes(key)}' in file '{escape_single_quotes(file_str)}'!",
+                        )
+                    else:
+                        self.extranetwork_mappings[key] = PPPENMapping(full_path, kind, name, variants)
+
+    def __get_extranetwork_mappings_in_structured_file(self, full_path: Path):
+        """
+        Get all extra network mappings in a structured file.
+
+        Args:
+            full_path (Path): The path to the file.
+        """
+        try:
+            try:
+                with open(full_path, "r", encoding="utf-8") as file:
+                    content = _YAML(typ='safe').load(file)
+            except:  # pylint: disable=bare-except
+                log(
+                    self.__logger,
+                    self.__debug_level,
+                    logging.WARNING,
+                    f"Could not read file '{escape_single_quotes(str(full_path))}' with utf-8 encoding, trying windows-1252...",
+                )
+                with open(full_path, "r", encoding="windows-1252") as file:
+                    content = _YAML(typ='safe').load(file)
+            self.__add_extranetwork_mapping(content, full_path)
+        except Exception as e:  # pylint: disable=broad-except
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.ERROR,
+                f"Error reading extra network mappings from file '{escape_single_quotes(str(full_path))}': {e}",
+            )
+
+    def __get_extranetwork_mappings_in_path(self, path: Path):
+        """
+        Get all extra network mappings in a path.
+
+        Args:
+            path (Path): The path (folder or file).
+        """
+        if not path.exists():
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.WARNING,
+                f"Extra network mappings path '{escape_single_quotes(str(path))}' does not exist!",
+            )
+            return
+        if path.is_file():
+            self.__get_extranetwork_mappings_in_file(path)
+            return
+        for child in path.iterdir():
+            if child.name.startswith("."):
+                continue
+            self.__get_extranetwork_mappings_in_path(child)

ppp_logging.py

@@ -0,0 +1,121 @@
+from enum import Enum
+import logging
+import sys
+import copy
+
+from ppp_utils import format_output
+
+
+class DEBUG_LEVEL(Enum):
+    none = "none"
+    minimal = "minimal"
+    full = "full"
+
+
+class PromptPostProcessorLogFactory:  # pylint: disable=too-few-public-methods
+    """
+    Factory class for creating loggers for the PromptPostProcessor module.
+    """
+
+    class ColoredFormatter(logging.Formatter):
+        """
+        A custom logging formatter that adds color to log records based on their level.
+
+        Attributes:
+            COLORS (dict): A dictionary mapping log levels to ANSI escape codes for colors.
+
+        Methods:
+            format(record): Formats the log record with color based on its level.
+
+        """
+
+        COLORS = {
+            "DEBUG": "\033[0;36m",  # CYAN
+            "INFO": "\033[0;32m",  # GREEN
+            "WARNING": "\033[0;33m",  # YELLOW
+            "ERROR": "\033[0;31m",  # RED
+            "CRITICAL": "\033[0;37;41m",  # WHITE ON RED
+            "RESET": "\033[0m",  # RESET COLOR
+        }
+
+        def format(self, record):
+            """
+            Formats the log record with color based on the log level.
+
+            Args:
+                record (LogRecord): The log record to be formatted.
+
+            Returns:
+                str: The formatted log record.
+            """
+            colored_record = copy.copy(record)
+            levelname = colored_record.levelname
+            seq = self.COLORS.get(levelname, self.COLORS["RESET"])
+            colored_record.levelname = f"{seq}{levelname:8s}{self.COLORS['RESET']}"
+            return super().format(colored_record)
+
+    def __init__(self, filename=None):
+        """
+        Initializes the PromptPostProcessorLogFactory class.
+
+        This method sets up the logger for the PromptPostProcessor class and configures its log level and handlers.
+
+        Args:
+            filename (str, optional): The name of the file to log to. Defaults to None.
+
+        Returns:
+            None
+        """
+        ppplog = logging.getLogger("PromptPostProcessor")
+        ppplog.propagate = False
+        if not ppplog.handlers:
+            handler = logging.StreamHandler(sys.stdout)
+            handler.setFormatter(self.ColoredFormatter("%(asctime)s %(levelname)s %(message)s"))
+            ppplog.addHandler(handler)
+            if filename is not None:
+                file_handler = logging.FileHandler(filename, encoding="utf-8")
+                file_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+                ppplog.addHandler(file_handler)
+        ppplog.setLevel(logging.DEBUG)
+        self.log = PromptPostProcessorLogCustomAdapter(ppplog)
+
+
+class PromptPostProcessorLogCustomAdapter(logging.LoggerAdapter):
+    """
+    Custom logger adapter for the PromptPostProcessor.
+    This adapter adds a prefix to log messages to indicate that they are related to the PromptPostProcessor.
+    """
+
+    def process(self, msg, kwargs):
+        """
+        Process the log message and keyword arguments.
+        Args:
+            msg (str): The log message.
+            kwargs (dict): The keyword arguments.
+        Returns:
+            tuple: A tuple containing the processed log message and keyword arguments.
+        """
+        return f"[PPP] {msg}", kwargs
+
+
+def log(
+    logger: logging.Logger,
+    debug_level: DEBUG_LEVEL,
+    kind: int,
+    message: str,
+    min_level: DEBUG_LEVEL | None = None,
+    formatted: bool = True,
+    exc_info=None,
+):
+    if logger:
+        if min_level is None:
+            if kind == logging.DEBUG:
+                min_level = DEBUG_LEVEL.full
+            elif kind == logging.INFO:
+                min_level = DEBUG_LEVEL.minimal
+            else:
+                min_level = DEBUG_LEVEL.none
+        i_debug_level = list(DEBUG_LEVEL).index(debug_level)
+        i_min_level = list(DEBUG_LEVEL).index(min_level)
+        if i_debug_level >= i_min_level:
+            logger.log(kind, format_output(message) if formatted else message, exc_info=exc_info)

ppp_utils.py

@@ -0,0 +1,97 @@
+import logging
+from pathlib import Path
+from typing import Any
+
+
+def get_version_from_pyproject() -> str:
+    """
+    Reads the version from the pyproject.toml file.
+
+    Returns:
+        str: The version string.
+    """
+    version_str = "0.0.0"
+    try:
+        pyproject_path = Path(__file__).resolve().parent / "pyproject.toml"
+        with open(pyproject_path, "r", encoding="utf-8") as file:
+            for line in file:
+                if line.startswith("version = "):
+                    version_str = line.split("=")[1].strip().strip('"')
+                    break
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logging.getLogger().exception(e)
+    return version_str
+
+
+def deep_freeze(obj):
+    """
+    Deep freeze an object.
+
+    Args:
+        obj (object): The object to freeze.
+
+    Returns:
+        object: The frozen object.
+    """
+    if isinstance(obj, dict):
+        return tuple((k, deep_freeze(v)) for k, v in sorted(obj.items()))
+    if isinstance(obj, list):
+        return tuple(deep_freeze(i) for i in obj)
+    if isinstance(obj, set):
+        return tuple(deep_freeze(i) for i in sorted(obj))
+    return obj
+
+
+def escape_single_quotes(s: str):
+    """
+    Escape single quotes in a string.
+
+    Args:
+        s (str): The string to escape.
+
+    Returns:
+        str: The escaped string.
+    """
+    return s.replace("'", "\\'")
+
+
+def escape_double_quotes(s: str):
+    """
+    Escape double quotes in a string.
+
+    Args:
+        s (str): The string to escape.
+
+    Returns:
+        str: The escaped string.
+    """
+    return s.replace('"', '\\"')
+
+
+def repr_value(s: Any):
+    """
+    Return a string representation of a value, escaping single quotes.
+
+    Args:
+        s (Any): The value to represent.
+    Returns:
+        str: The string representation of the value.
+    """
+    if isinstance(s, str):
+        return f"'{escape_single_quotes(s)}'"
+    if isinstance(s, bool):
+        return "true" if s else "false"
+    return str(s)
+
+
+def format_output(text: str) -> str:
+    """
+    Formats the output text by encoding it using unicode_escape and decoding it using utf-8.
+
+    Args:
+        text (str): The input text to be formatted.
+
+    Returns:
+        str: The formatted output text.
+    """
+    return text.encode("unicode_escape").decode("utf-8")

ppp_variables.py

@@ -0,0 +1,153 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+ScalarValue = str | int | float | bool
+VariableValue = ScalarValue | list | None
+
+
+@dataclass
+class VariableEntry:
+    """Holds all state for a single user variable."""
+
+    value: Any = field(default=None)  # raw unevaluated value or evaluated on set
+    last_echoed_value: Any = field(default=None)  # raw unevaluated value at last echo
+    last_echoed_evaluated_value: ScalarValue | None = field(default=None)  # evaluated value at last echo
+
+
+class VariableRepository:
+    """
+    Unified repository for system and user prompt variables.
+
+    System variables (underscore-prefixed names like ``_model``) are populated
+    once per processing session and are read-only during prompt evaluation.
+
+    User variables are created and mutated by set/echo constructs in the prompt.
+    Each user variable is stored as a :class:`VariableEntry` that tracks the
+    set value and the last value that was echoed into the prompt output.
+    """
+
+    def __init__(self) -> None:
+        self._system: dict[str, VariableValue] = {}
+        self._vars: dict[str, VariableEntry] = {}
+
+    def name_is_system(self, name: str) -> bool:
+        """Return True if *name* is a system variable (i.e. starts with an underscore)."""
+        return name.startswith("_")
+
+    # ---- System variables ----
+
+    def get_system(self, name: str, default: VariableValue = None) -> VariableValue:
+        """Return the value of a system variable, or *default* if absent."""
+        return self._system.get(name, default)
+
+    def set_system(self, name: str, value: VariableValue) -> None:
+        """Set a system variable."""
+        if not self.name_is_system(name):
+            raise ValueError(f"Invalid system variable name '{name}': must start with an underscore")
+        if value is None:
+            self._system.pop(name, None)
+        else:
+            self._system[name] = value
+
+    def update_system(self, mapping: dict[str, VariableValue]) -> None:
+        """Bulk-update system variables from *mapping*."""
+        for name in mapping:
+            if not self.name_is_system(name):
+                raise ValueError(f"Invalid system variable name '{name}': must start with an underscore")
+        self._system.update(mapping)
+
+    def clear_system(self) -> None:
+        """Remove all system variables."""
+        self._system.clear()
+
+    @property
+    def all_system(self) -> dict[str, VariableValue]:
+        """Return a shallow copy of all system variables."""
+        return self._system.copy()
+
+    # ---- User variables ----
+
+    def _entry(self, name: str) -> VariableEntry:
+        """Return (creating if necessary) the :class:`VariableEntry` for *name*."""
+        if name not in self._vars:
+            self._vars[name] = VariableEntry()
+        return self._vars[name]
+
+    def get_user(self, name: str, default: Any = None) -> Any:
+        """Return the value of a user variable, or *default* if absent."""
+        entry = self._vars.get(name)
+        if entry is None or entry.value is None:
+            return default
+        return entry.value
+
+    def set_user(self, name: str, value: Any) -> None:
+        """Set the value of a user variable."""
+        if self.name_is_system(name):
+            raise ValueError(f"Invalid user variable name '{name}': must not start with an underscore")
+        entry = self._entry(name)
+        entry.value = value
+
+    def delete_user(self, name: str) -> None:
+        """
+        Remove the value for a user variable.
+        """
+        entry = self._vars.get(name)
+        if entry is None:
+            return
+        del self._vars[name]
+
+    def clear_user(self) -> None:
+        """
+        Clear the values for all user variables.
+        """
+        self._vars.clear()
+
+    @property
+    def all_user(self) -> set[str]:
+        """Return the set of all user-variable keys (those with any non-None field)."""
+        return set(self._vars)
+
+    def set_echoed_value(self, name: str, value: Any, evaluated_value: ScalarValue) -> None:
+        """Record that *name* was echoed into the prompt with *value*."""
+        if not self.name_is_system(name):
+            entry = self._entry(name)
+            entry.last_echoed_value = value
+            entry.last_echoed_evaluated_value = evaluated_value
+
+    def get_echoed_value(self, name: str, default: ScalarValue | None = None) -> ScalarValue | None:
+        """Return the last echoed value for *name*, or *default* if it has not been echoed."""
+        entry = self._vars.get(name)
+        if entry is None:
+            return default
+        return entry.last_echoed_evaluated_value if entry.last_echoed_evaluated_value is not None else default
+
+    def backup_user(self) -> dict[str, VariableEntry]:
+        """Return a per-entry shallow-copy snapshot of all user variables for rollback."""
+        return {
+            name: VariableEntry(entry.value, entry.last_echoed_value, entry.last_echoed_evaluated_value)
+            for name, entry in self._vars.items()
+        }
+
+    def restore_user(self, backup: dict[str, VariableEntry]) -> None:
+        """Restore user variables from a snapshot made by :meth:`backup_user_and_echoed`."""
+        self._vars.clear()
+        self._vars.update(
+            {
+                name: VariableEntry(entry.value, entry.last_echoed_value, entry.last_echoed_evaluated_value)
+                for name, entry in backup.items()
+            }
+        )
+
+    # ---- Combined queries ----
+
+    def get(self, name: str, default: Any = None) -> Any:
+        """
+        Return the value of a variable, checking system variables first, then user variables.
+
+        Args:
+            name (str): The name of the variable.
+            default (Any): The value to return if the variable is not found.
+        """
+        if name in self._system:
+            return self._system.get(name, default)
+        return self.get_user(name, default)

ppp_wildcards.py

@@ -0,0 +1,567 @@
+import fnmatch
+from pathlib import Path
+from typing import Any, Optional
+import logging
+from ruamel.yaml import YAML as _YAML
+from ruamel.yaml.error import YAMLError as _YAMLError
+
+from ppp_logging import DEBUG_LEVEL, log
+from ppp_utils import deep_freeze, escape_single_quotes
+
+
+class PPPWildcard:
+    """
+    A wildcard object.
+
+    Attributes:
+        key (str): The key of the wildcard.
+        file (Path | None): The path to the file where the wildcard is defined, or None if from inline input.
+        unprocessed_choices (list[str]): The unprocessed choices of the wildcard.
+        options (dict): The options of the wildcard.
+        choices (list[dict]): The processed choices of the wildcard.
+    """
+
+    def __init__(self, fullpath: Path | None, key: str, choices: list[str]):
+        self.key: str = key
+        self.file: Path | None = fullpath
+        self.unprocessed_choices: list[str] = choices
+        self.choices: list[dict] = None
+        self.options: dict = None
+
+    def __hash__(self) -> int:
+        t = (self.key, deep_freeze(self.unprocessed_choices))
+        return hash(t)
+
+    def __sizeof__(self):
+        return (
+            self.key.__sizeof__()
+            + (self.file.__sizeof__() if self.file is not None else 0)
+            + self.unprocessed_choices.__sizeof__()
+            + self.choices.__sizeof__()
+            + self.options.__sizeof__()
+        )
+
+
+class PPPWildcards:
+    """
+    A class to manage wildcards.
+
+    Attributes:
+        wildcards (dict[str, PPPWildcard]): The wildcards.
+    """
+
+    DEFAULT_WILDCARDS_FOLDER = "wildcards"
+
+    def __init__(self, logger=None):
+        self.__logger: logging.Logger = logger
+        self.__debug_level = DEBUG_LEVEL.none
+        self.__wildcards_folders: list[Path] = []
+        self.__wildcard_files: dict[Path, float] = {}
+        self.__local_input_hash: int | None = None
+        self.__wildcard_default_filters: dict[str, list[list[str]]] = {}
+        self.wildcards: dict[str, PPPWildcard] = {}
+
+    def __hash__(self) -> int:
+        return hash(deep_freeze(self.wildcards))
+
+    def __sizeof__(self):
+        return self.wildcards.__sizeof__() + self.__wildcards_folders.__sizeof__() + self.__wildcard_files.__sizeof__()
+
+    def refresh_wildcards(
+        self,
+        debug_level: DEBUG_LEVEL,
+        wildcards_folders: Optional[list[Path]],
+        wildcards_input: str = None,
+    ):
+        """
+        Initialize the wildcards.
+        """
+        self.reset_default_filters()
+        self.__debug_level = debug_level
+        self.__wildcards_folders = [Path(f) for f in (wildcards_folders or [])]
+        # log(self.__logger, self.__debug_level, logging.INFO, "Refreshing wildcards...")
+        # t1 = time.monotonic_ns()
+        for fullpath in list(self.__wildcard_files.keys()):
+            if not fullpath.exists() or not any(
+                fullpath.parent.is_relative_to(folder) for folder in self.__wildcards_folders
+            ):
+                self.__remove_wildcards_from_path(fullpath)
+        if wildcards_input is None and self.__local_input_hash is not None:
+            self.__remove_wildcards_from_input()
+        if wildcards_folders is not None or wildcards_input is not None:
+            if wildcards_folders is not None:
+                for f in self.__wildcards_folders:
+                    self.__get_wildcards_in_path(f if f.is_dir() else f.parent, f)
+            if wildcards_input is not None:
+                self.__get_wildcards_in_input(wildcards_input)
+        else:
+            self.wildcards = {}
+            self.__wildcard_files = {}
+            self.__local_input_hash = None
+        # t2 = time.monotonic_ns()
+        # log(self.__logger, self.__debug_level, logging.INFO, f"Wildcards refresh time: {(t2 - t1) / 1_000_000_000:.3f} seconds")
+
+    def get_wildcards(self, key: str) -> list[PPPWildcard]:
+        """
+        Get all wildcards that match a key.
+
+        Args:
+            key (str): The key to match.
+
+        Returns:
+            list: A list of all wildcards that match the key.
+        """
+        keys = sorted(fnmatch.filter(self.wildcards.keys(), key))
+        return [self.wildcards[k] for k in keys]
+
+    def __get_wc_in_dict(self, dictionary: dict, prefix="") -> list[tuple[str, Any]]:
+        """
+        Get all wildcards in a dictionary, along their object.
+
+        Args:
+            dictionary (dict): The dictionary to check.
+            prefix (str): The prefix for the current key.
+
+        Returns:
+            list: A list of all leaf wildcards in the dictionary.
+        """
+        wc = []
+        for key, obj in dictionary.items():
+            if isinstance(obj, dict):
+                wc.extend(self.__get_wc_in_dict(obj, prefix + str(key) + "/"))
+            else:
+                wc.append((prefix + str(key), obj))
+        return wc
+
+    def __remove_wildcards_from_path(self, full_path: Path, debug=True):
+        """
+        Clear all wildcards from a file.
+
+        Args:
+            full_path (Path): The path to the file.
+            debug (bool): Whether to print debug messages or not.
+        """
+        if debug and full_path in self.__wildcard_files:
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.DEBUG,
+                f"Removing from memory wildcards from file: {full_path}",
+            )
+        if full_path in self.__wildcard_files:
+            del self.__wildcard_files[full_path]
+        for key in list(self.wildcards.keys()):
+            if self.wildcards[key].file == full_path:
+                del self.wildcards[key]
+
+    def __remove_wildcards_from_input(self, debug=True):
+        """
+        Clear all wildcards loaded from inline input.
+
+        Args:
+            debug (bool): Whether to print debug messages or not.
+        """
+        if debug and self.__local_input_hash is not None:
+            log(self.__logger, self.__debug_level, logging.DEBUG, "Removing from memory wildcards from input")
+        self.__local_input_hash = None
+        for key in list(self.wildcards.keys()):
+            if self.wildcards[key].file is None:
+                del self.wildcards[key]
+
+    def __get_wildcards_in_file(self, base: Path, full_path: Path):
+        """
+        Get all wildcards in a file.
+
+        Args:
+            base (Path): The base path for the wildcards.
+            full_path (Path): The path to the file.
+        """
+        try:
+            last_modified = full_path.stat().st_mtime
+            last_modified_cached = self.__wildcard_files.get(full_path, None)
+            if last_modified_cached is not None and last_modified == self.__wildcard_files[full_path]:
+                return
+            extension = full_path.suffix
+            if extension not in (".txt", ".json", ".yaml", ".yml"):
+                return
+            self.__remove_wildcards_from_path(full_path, False)
+            if last_modified_cached is not None:
+                log(self.__logger, self.__debug_level, logging.DEBUG, f"Updating wildcards from file: {full_path}")
+            if extension == ".txt":
+                self.__get_wildcards_in_text_file(full_path, base)
+            elif extension in (".json", ".yaml", ".yml"):
+                self.__get_wildcards_in_structured_file(full_path, base)
+            self.__wildcard_files[full_path] = last_modified
+        except Exception as e:  # pylint: disable=broad-except
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.ERROR,
+                f"Error reading wildcard file '{escape_single_quotes(str(full_path))}': {e}",
+            )
+
+    def __get_wildcards_in_input(self, wildcards_input: str):
+        """
+        Get all wildcards in the string.
+
+        Args:
+            wildcards_input (str): The input string containing wildcards in json or yaml format.
+        """
+        try:
+            new_h = hash(wildcards_input)
+            if new_h == self.__local_input_hash:
+                return
+            was_loaded = self.__local_input_hash is not None
+            self.__remove_wildcards_from_input(False)
+            if was_loaded:
+                log(self.__logger, self.__debug_level, logging.DEBUG, "Updating wildcards from input")
+            wildcards_input = wildcards_input.strip()
+            if wildcards_input != "":
+                try:
+                    content = _YAML(typ='safe').load(wildcards_input)
+                except _YAMLError as e:
+                    log(self.__logger, self.__debug_level, logging.WARNING, f"Invalid format for input wildcards: {e}")
+                    return
+                if content is not None:
+                    self.__add_wildcard(content, None, [""])
+            self.__local_input_hash = new_h
+        except Exception as e:  # pylint: disable=broad-except
+            log(self.__logger, self.__debug_level, logging.ERROR, f"Error reading wildcards input: {e}")
+
+    # NOTE wcdef and choice options should not have properties in common
+
+    def is_dict_wcdef_options(self, d: dict) -> bool:
+        """
+        Check if a dictionary is a valid wildcard definition options dictionary.
+
+        Args:
+            d (dict): The dictionary to check.
+
+        Returns:
+            bool: Whether the dictionary is a valid wildcard definition options dictionary or not.
+        """
+        return all(
+            k
+            in [
+                "sampler",
+                "repeating",
+                "optional",
+                "count",
+                "from",
+                "to",
+                "prefix",
+                "suffix",
+                "container",
+                "description",
+                "separator",
+            ]
+            for k in d.keys()
+        )
+
+    def is_dict_choice_options(self, d: dict) -> bool:
+        """
+        Check if a dictionary is a valid choice options dictionary.
+
+        Args:
+            d (dict): The dictionary to check.
+
+        Returns:
+            bool: Whether the dictionary is a valid choice options dictionary or not.
+        """
+        return all(k in ["command", "labels", "weight", "if", "content", "text"] for k in d.keys())
+
+    def __get_choices(self, obj: object, full_path: Path | None, key_parts: list[str]) -> list:
+        """
+        We process the choices in the object and return them as a list.
+
+        Args:
+            obj (object): the value of a wildcard
+            full_path (Path | None): path to the file where the wildcard is defined, or None if from inline input
+            key_parts (list[str]): parts of the key for the wildcard
+
+        Returns:
+            list: list of choices
+        """
+        if obj is None:
+            return None
+        if isinstance(obj, (str, dict)):
+            return [obj]
+        if isinstance(obj, (int, float, bool)):
+            return [str(obj)]
+        file_str = str(full_path) if full_path is not None else "input"
+        if not isinstance(obj, list) or len(obj) == 0:
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.WARNING,
+                f"Invalid format in wildcard '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(file_str)}'!",
+            )
+            return None
+        choices = []
+        for i, c in enumerate(obj):
+            if isinstance(c, (str, int, float, bool)):
+                choices.append(str(c))
+            elif isinstance(c, list):
+                # we create an anonymous wildcard
+                choices.append(self.__create_anonymous_wildcard(full_path, key_parts, i, c))
+            elif isinstance(c, dict):
+                choices.append(self.__process_dict_choice(c, full_path, key_parts, i))
+            else:
+                log(
+                    self.__logger,
+                    self.__debug_level,
+                    logging.WARNING,
+                    f"Invalid choice {i+1} in wildcard '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(file_str)}'!",
+                )
+        return choices
+
+    def __process_dict_choice(self, c: dict, full_path: Path | None, key_parts: list[str], i: int) -> dict:
+        """
+        Process a dictionary choice.
+
+        Args:
+            c (dict): The dictionary choice.
+            full_path (Path | None): The path to the file, or None if from inline input.
+            key_parts (list[str]): The parts of the key.
+            i (int): The index of the choice.
+
+        Returns:
+            dict: The processed choice.
+        """
+        if self.is_dict_wcdef_options(c):
+            return c
+        elif self.is_dict_choice_options(c):
+            # we assume it is a choice in object format
+            choice = c
+            choice_content = choice.get("content", choice.get("text", None))
+            if choice_content is not None and isinstance(choice_content, list):
+                # we create an anonymous wildcard
+                choice["content"] = self.__create_anonymous_wildcard(full_path, key_parts, i, choice_content)
+                if "text" in choice:
+                    del choice["text"]
+            return choice
+        if len(c) == 1:
+            # we assume it is an anonymous wildcard with options
+            firstkey = list(c.keys())[0]
+            return self.__create_anonymous_wildcard(full_path, key_parts, i, c[firstkey], firstkey)
+        file_str = str(full_path) if full_path is not None else "input"
+        log(
+            self.__logger,
+            self.__debug_level,
+            logging.WARNING,
+            f"Invalid choice {i+1} in wildcard '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(file_str)}'!",
+        )
+        return None
+
+    def __create_anonymous_wildcard(self, full_path: Path | None, key_parts, i, content, options=None):
+        """
+        Create an anonymous wildcard.
+
+        Args:
+            full_path (Path | None): The path to the file that contains it, or None if from inline input.
+            key_parts (list[str]): The parts of the key.
+            i (int): The index of the wildcard.
+            content (object): The content of the wildcard.
+            options (str): The options for the choice where the wildcard is defined.
+
+        Returns:
+            str: The resulting value for the choice.
+        """
+        new_parts = key_parts + [f"#ANON_{i}"]
+        self.__add_wildcard(content, full_path, new_parts)
+        value = f"__{'/'.join(new_parts)}__"
+        if options is not None:
+            value = f"{options}::{value}"
+        return value
+
+    def __add_wildcard(self, content: object, full_path: Path | None, external_key_parts: list[str]):
+        """
+        Add a wildcard to the wildcards dictionary.
+
+        Args:
+            content (object): The content of the wildcard.
+            full_path (Path | None): The path to the file that contains it, or None if from inline input.
+            external_key_parts (list[str]): The parts of the key.
+        """
+        file_str = str(full_path) if full_path is not None else "input"
+
+        def existing_file_str(wc):
+            return str(wc.file) if wc.file is not None else "input"
+
+        key_parts = external_key_parts.copy()
+        if isinstance(content, dict):
+            key_parts.pop()
+            keys = self.__get_wc_in_dict(content)
+            for key, obj in keys:
+                tmp_key_parts = key_parts.copy()
+                tmp_key_parts.extend(key.split("/"))
+                fullkey = "/".join(tmp_key_parts)
+                if self.wildcards.get(fullkey, None) is not None:
+                    log(
+                        self.__logger,
+                        self.__debug_level,
+                        logging.WARNING,
+                        f"Duplicate wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}' and '{escape_single_quotes(existing_file_str(self.wildcards[fullkey]))}'!",
+                    )
+                else:
+                    choices = self.__get_choices(obj, full_path, tmp_key_parts)
+                    if choices is None:
+                        log(
+                            self.__logger,
+                            self.__debug_level,
+                            logging.WARNING,
+                            f"Invalid wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}'!",
+                        )
+                    elif fullkey.startswith("_"):
+                        log(
+                            self.__logger,
+                            self.__debug_level,
+                            logging.WARNING,
+                            f"Invalid wildcard name '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}'! (cannot start with underscore)",
+                        )
+                    else:
+                        self.wildcards[fullkey] = PPPWildcard(full_path, fullkey, choices)
+            return
+        if isinstance(content, str):
+            content = [content]
+        elif isinstance(content, (int, float, bool)):
+            content = [str(content)]
+        if not isinstance(content, list):
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.WARNING,
+                f"Invalid wildcard in file '{escape_single_quotes(file_str)}'!",
+            )
+            return
+        fullkey = "/".join(key_parts)
+        if self.wildcards.get(fullkey, None) is not None:
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.WARNING,
+                f"Duplicate wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}' and '{escape_single_quotes(existing_file_str(self.wildcards[fullkey]))}'!",
+            )
+        else:
+            choices = self.__get_choices(content, full_path, key_parts)
+            if choices is None:
+                log(
+                    self.__logger,
+                    self.__debug_level,
+                    logging.WARNING,
+                    f"Invalid wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}'!",
+                )
+            elif fullkey.startswith("_"):
+                log(
+                    self.__logger,
+                    self.__debug_level,
+                    logging.WARNING,
+                    f"Invalid wildcard name '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}'! (cannot start with underscore)",
+                )
+            else:
+                self.wildcards[fullkey] = PPPWildcard(full_path, fullkey, choices)
+
+    def __get_wildcards_in_structured_file(self, full_path: Path, base: Path):
+        """
+        Get all wildcards in a structured file.
+
+        Args:
+            full_path (Path): The path to the file.
+            base (Path): The base path for the wildcards.
+        """
+        external_key_parts = list(full_path.with_suffix("").relative_to(base).parts)
+        try:
+            with open(full_path, "r", encoding="utf-8") as file:
+                content = _YAML(typ='safe').load(file)
+        except:  # pylint: disable=bare-except
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.WARNING,
+                f"Could not read file '{escape_single_quotes(str(full_path))}' with utf-8 encoding, trying windows-1252...",
+            )
+            with open(full_path, "r", encoding="windows-1252") as file:
+                content = _YAML(typ='safe').load(file)
+        self.__add_wildcard(content, full_path, external_key_parts)
+
+    def __get_wildcards_in_text_file(self, full_path: Path, base: Path):
+        """
+        Get all wildcards in a text file.
+
+        Args:
+            full_path (Path): The path to the file.
+            base (Path): The base path for the wildcards.
+        """
+        external_key_parts = list(full_path.with_suffix("").relative_to(base).parts)
+        try:
+            with open(full_path, "r", encoding="utf-8") as file:
+                text_content = map(lambda x: x.strip("\n\r"), file.readlines())
+        except:  # pylint: disable=bare-except
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.WARNING,
+                f"Could not read file '{escape_single_quotes(str(full_path))}' with utf-8 encoding, trying windows-1252...",
+            )
+            with open(full_path, "r", encoding="windows-1252") as file:
+                text_content = map(lambda x: x.strip("\n\r"), file.readlines())
+        # First pass: drop blank lines and full-line comments.
+        text_content = list(filter(lambda x: x.strip() != "" and not x.strip().startswith("#"), text_content))
+        # Second pass: strip inline comments from lines that passed the first filter.
+        text_content = [x.split("#")[0].rstrip() if len(x.split("#")) > 1 else x for x in text_content]
+        self.__add_wildcard(text_content, full_path, external_key_parts)
+
+    def __get_wildcards_in_path(self, base: Path, path: Path):
+        """
+        Get all wildcards in a path.
+
+        Args:
+            base (Path): The base path for the wildcards.
+            path (Path): The path (folder or file).
+        """
+        if not path.exists():
+            log(
+                self.__logger,
+                self.__debug_level,
+                logging.WARNING,
+                f"Wildcard path '{escape_single_quotes(str(path))}' does not exist!",
+            )
+            return
+        if path.is_file():
+            self.__get_wildcards_in_file(base, path)
+            return
+        for child in path.iterdir():
+            if child.name.startswith("."):
+                continue
+            self.__get_wildcards_in_path(base, child)
+
+    def set_wildcard_default_filter(self, wildcard_key: str, filter_options: Optional[list[list[str]]]):
+        """
+        Set the default filter for a wildcard.
+
+        Args:
+            wildcard_key (str): The key of the wildcard.
+            filter_options (list[list[str]]): The filter options.
+        """
+        if filter_options is None:
+            if wildcard_key in self.__wildcard_default_filters:
+                del self.__wildcard_default_filters[wildcard_key]
+        else:
+            self.__wildcard_default_filters[wildcard_key] = filter_options
+
+    def get_wildcard_default_filter(self, wildcard_key: str) -> Optional[list[list[str]]]:
+        """
+        Get the default filter for a wildcard.
+
+        Args:
+            wildcard_key (str): The key of the wildcard.
+        Returns:
+            Optional[list[list[str]]]: The filter options or None if not set.
+        """
+        return self.__wildcard_default_filters.get(wildcard_key, None)
+
+    def reset_default_filters(self):
+        """
+        Reset all default filters.
+        """
+        self.__wildcard_default_filters = {}

pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "sd-webui-prompt-postprocessor"
+description = "Stable Diffusion WebUI & ComfyUI extension to post-process the prompt. Features include: wildcards, sending content from the prompt to the negative prompt, variables, model detection, extranetwork mapping, cleanup."
+version = "3.2.2"
+license = { file = "LICENSE.txt" }
+dependencies = ["lark==1.*", "numpy==2.*", "ruamel.yaml==0.*", "pydantic==2.*"]
+requires-python = ">=3.10"
+
+[project.urls]
+Repository = "https://github.com/acorderob/sd-webui-prompt-postprocessor"
+#  Used by Comfy Registry https://registry.comfy.org/
+Documentation = "https://github.com/acorderob/sd-webui-prompt-postprocessor/main/README.md"
+"Bug Tracker" = "https://github.com/acorderob/sd-webui-prompt-postprocessor/issues"
+issues = "https://github.com/acorderob/sd-webui-prompt-postprocessor/issues"
+
+[tool.comfy]
+PublisherId = "acorderob"
+DisplayName = "ACB Prompt PostProcessor"
+Icon = "https://raw.githubusercontent.com/acorderob/sd-webui-prompt-postprocessor/main/images/prompt-postprocessor-icon.png"

requirements.txt

@@ -0,0 +1,4 @@
+lark==1.*
+numpy==2.*
+ruamel.yaml==0.*
+pydantic==2.*

scripts/extension.py

@@ -1,79 +0,0 @@
-if __name__ == "__main__":
-    raise SystemExit("This script must be run from a Stable Diffusion WebUI")
-
-import sys
-import os
-
-sys.path.insert(1, os.path.join(sys.path[0], ".."))
-
-
-# pylint: disable=import-error
-
-from modules import scripts, shared, script_callbacks
-from modules.processing import StableDiffusionProcessing
-from modules.shared import opts
-from sendtonegative import SendToNegative
-from stnlogging import SendToNegativeLogFactory
-
-
-class SendToNegativeScript(scripts.Script):
-    def __init__(self):
-        if not hasattr(self, "callbacks_added"):
-            lf = SendToNegativeLogFactory()
-            self.__logstn = lf.log
-            script_callbacks.on_ui_settings(self.__on_ui_settings)
-            self.callbacks_added = True
-
-    def title(self):
-        return SendToNegative.NAME
-
-    def show(self, is_img2img):
-        return scripts.AlwaysVisible
-
-    def process(self, p: StableDiffusionProcessing, *args, **kwargs):
-        stn = SendToNegative(self.__logstn, opts=opts)
-        for i in range(len(p.all_prompts)):  # pylint: disable=consider-using-enumerate
-            p.all_prompts[i], p.all_negative_prompts[i] = stn.process_prompt(
-                p.all_prompts[i], p.all_negative_prompts[i]
-            )
-        # make it compatible with A1111 hires fix
-        if hasattr(p, "all_hr_prompts") and hasattr(p, "all_hr_negative_prompts"):
-            for i in range(len(p.all_hr_prompts)):  # pylint: disable=consider-using-enumerate
-                p.all_hr_prompts[i], p.all_hr_negative_prompts[i] = stn.process_prompt(
-                    p.all_hr_prompts[i], p.all_hr_negative_prompts[i]
-                )
-
-    def __on_ui_settings(self):
-        section = ("send-to-negative", SendToNegative.NAME)
-        shared.opts.add_option(
-            key="stn_separator",
-            info=shared.OptionInfo(
-                SendToNegative.DEFAULT_SEPARATOR,
-                label="Separator used when adding to the negative prompt",
-                section=section,
-            ),
-        )
-        shared.opts.add_option(
-            key="stn_ignorerepeats",
-            info=shared.OptionInfo(
-                True,
-                label="Ignore tags with repeated content",
-                section=section,
-            ),
-        )
-        shared.opts.add_option(
-            key="stn_joinattention",
-            info=shared.OptionInfo(
-                True,
-                label="Join attention modifiers (weights) when possible",
-                section=section,
-            ),
-        )
-        shared.opts.add_option(
-            key="stn_cleanup",
-            info=shared.OptionInfo(
-                True,
-                label="Try to clean-up the prompt after processing (removes extra spaces or the configured separator)",
-                section=section,
-            ),
-        )

scripts/ppp_script.py

@@ -0,0 +1,864 @@
+if __name__ == "__main__":
+    raise SystemExit("This script must be run from a Stable Diffusion WebUI")
+
+import logging
+import sys
+import os
+import time
+from pathlib import Path
+import numpy as np
+
+sys.path.append(str(Path(__file__).parent))  # base path for the extension
+
+from modules import scripts, shared, script_callbacks  # type: ignore
+from modules.processing import StableDiffusionProcessing  # type: ignore
+from modules.shared import opts  # type: ignore
+from modules.paths import models_path  # type: ignore
+import gradio as gr  # type: ignore
+
+from ppp import PromptPostProcessor
+from ppp_classes import IFWILDCARDS_CHOICES, ONWARNING_CHOICES, SUPPORTED_APPS, SUPPORTED_APPS_NAMES, PPPStateOptions
+from ppp_logging import DEBUG_LEVEL, PromptPostProcessorLogFactory, log
+from ppp_cache import PPPLRUCache
+from ppp_wildcards import PPPWildcards
+from ppp_enmappings import PPPExtraNetworkMappings
+from ppp_common import load_grammar
+
+
+class PromptPostProcessorA1111Script(scripts.Script):
+    """
+    This class represents a script for prompt post-processing.
+    It is responsible for processing prompts and applying various settings and cleanup operations.
+
+    Attributes:
+        callbacks_added (bool): Flag indicating whether the script callbacks have been added.
+
+    Methods:
+        __init__(): Initializes the PromptPostProcessorScript object.
+        title(): Returns the title of the script.
+        show(is_img2img): Determines whether the script should be shown based on the input type.
+        process(p, *args, **kwargs): Processes the prompts and applies post-processing operations.
+        ppp_interrupt(): Interrupts the generation.
+        __on_ui_settings(): Callback function for UI settings.
+    """
+
+    instance_count = 0
+
+    @classmethod
+    def increment_instance_count(cls):
+        cls.instance_count += 1
+        return cls.instance_count
+
+    @classmethod
+    def get_instance_count(cls):
+        return cls.instance_count
+
+    def __init__(self):
+        """
+        Initializes the PromptPostProcessor object.
+
+        Parameters:
+            None
+
+        Returns:
+            None
+        """
+        super().__init__()
+        self.instance_index = self.increment_instance_count()
+        self.name = PromptPostProcessor.NAME
+        self.grammar_content = load_grammar()
+        lf = PromptPostProcessorLogFactory()
+        self.ppp_logger = lf.log
+        self.ppp_debug_level = DEBUG_LEVEL.none.value
+        self.lru_cache = None
+        self.wildcards_obj = None
+        self.extranetwork_mappings_obj = None
+        self.ppp_init = False
+        # log(self.ppp_logger, DEBUG_LEVEL.minimal, logging.INFO, f"Initializing {self.name} instance {self.instance_index}")
+
+        try:
+            # Support for SD.Next
+            import installer  # type: ignore
+            if hasattr(installer, "control_extensions"):
+                if self.title() not in installer.control_extensions:
+                    installer.control_extensions.append(self.title())  # We add the extension to the whitelist.
+        except ImportError:
+            # log(self.ppp_logger, DEBUG_LEVEL.minimal, logging.WARNING, "Could not import control_extensions from installer, SD.Next support will not work.")
+            pass
+        except Exception as e:  # pylint: disable=broad-except
+            log(self.ppp_logger, DEBUG_LEVEL.minimal, logging.ERROR, f"Error while adding to the SD.Next extension whitelist: {e}")
+
+    def title(self):
+        """
+        Returns the title of the script.
+
+        Returns:
+            str: The title of the script.
+        """
+        return PromptPostProcessor.NAME
+
+    def show(self, is_img2img):  # pylint: disable=unused-argument
+        """
+        Determines whether the script should be shown based on the kind of processing.
+
+        Args:
+            is_img2img (bool): Flag indicating whether the processing is image-to-image.
+
+        Returns:
+            scripts.Visibility: The visibility setting for the script.
+        """
+        return scripts.AlwaysVisible
+
+    def ui(self, is_img2img):  # pylint: disable=unused-argument
+        with gr.Accordion(PromptPostProcessor.NAME, open=False):
+            force_equal_seeds = gr.Checkbox(
+                label="Force equal seeds",
+                info="Force all image seeds and variation seeds to be equal to the first one, disabling the default autoincrease.",
+                value=False,
+                # show_label=True,
+                elem_id="ppp_force_equal_seeds",
+            )
+            gr.HTML("<br>")
+            gr.Markdown("""
+                Unlink the seed to use the specified one for the prompts instead of the image seed.
+
+                * A seed of -1 and "Incremental seed" checked will use a random seed for the first prompt and consecutive values for the rest. This is the same as when you use -1 for the image seed.
+                * A seed of -1 and "Incremental seed" unchecked will use a random seed for each prompt.
+                * Any other seed value and "Incremental seed" checked will use the specified seed for the first prompt and consecutive values for the rest.
+                * Any other seed value and "Incremental seed" unchecked will use the specified seed for all the prompts.
+
+                Seeds are only used for the wildcards and choice constructs.
+            """)
+            gr.HTML("<br>")
+            with gr.Row(equal_height=True):
+                unlink_seed = gr.Checkbox(
+                    label="Unlink seed",
+                    value=False,
+                    # show_label=True,
+                    elem_id="ppp_unlink_seed",
+                )
+                seed = gr.Number(
+                    label="Prompt seed",
+                    value=-1,
+                    precision=0,
+                    # minimum=-1,
+                    # maximum=2**32 - 1,
+                    # step=1,
+                    # show_label=True,
+                    min_width=100,
+                    elem_id="ppp_seed",
+                )
+                incremental_seed = gr.Checkbox(
+                    label="Incremental seed (only applies to batches)",
+                    value=False,
+                    # show_label=True,
+                    elem_id="ppp_incremental_seed",
+                )
+            gr.HTML("<br>")
+            with gr.Row(equal_height=True):
+                combinatorial = gr.Checkbox(
+                    label="Combinatorial mode",
+                    info="Generate all prompt combinations and cycle through them to fill the batch.",
+                    value=PromptPostProcessor.DEFAULT_DO_COMBINATORIAL,
+                    elem_id="ppp_combinatorial",
+                )
+                combinatorial_shuffle = gr.Checkbox(
+                    label="Shuffle combinations",
+                    info="Shuffle the combinatorial results.",
+                    value=PromptPostProcessor.DEFAULT_COMBINATORIAL_SHUFFLE,
+                    elem_id="ppp_combinatorial_shuffle",
+                )
+                combinatorial_limit = gr.Number(
+                    label="Combinations limit (0 = no limit)",
+                    value=PromptPostProcessor.DEFAULT_COMBINATORIAL_LIMIT,
+                    precision=0,
+                    min_width=120,
+                    elem_id="ppp_combinatorial_limit",
+                )
+        return [
+            force_equal_seeds,
+            unlink_seed,
+            seed,
+            incremental_seed,
+            combinatorial,
+            combinatorial_shuffle,
+            combinatorial_limit,
+        ]
+
+    def process(
+        self,
+        p: StableDiffusionProcessing,
+        input_force_equal_seeds,
+        input_unlink_seed,
+        input_seed,
+        input_incremental_seed,
+        input_combinatorial,
+        input_combinatorial_shuffle,
+        input_combinatorial_limit,
+    ):  # pylint: disable=arguments-differ
+        """
+        Processes the prompts and applies post-processing operations.
+
+        Args:
+            p (StableDiffusionProcessing): The StableDiffusionProcessing object containing the prompts.
+            input_force_equal_seeds (bool): Flag indicating whether to force equal seeds.
+            input_unlink_seed (bool): Flag indicating whether to unlink the seed.
+            input_seed (int): The seed value.
+            input_incremental_seed (bool): Flag indicating whether to use incremental seed.
+            input_combinatorial (bool): Flag indicating whether to use combinatorial mode.
+            input_combinatorial_shuffle (bool): Flag indicating whether to shuffle the combinatorial results.
+            input_combinatorial_limit (int): Maximum number of combinations (0 = no limit).
+
+        Returns:
+            None
+        """
+        app = SUPPORTED_APPS.a1111
+        if hasattr(p.sd_model, "model_config"):
+            app = SUPPORTED_APPS.forge
+            if not hasattr(p.sd_model, "is_sd2"):
+                app = SUPPORTED_APPS.forgeneo
+        elif hasattr(p.sd_model, "forge_objects"):
+            app = SUPPORTED_APPS.reforge
+        elif hasattr(p.sd_model, "is_sdxl") and not hasattr(p.sd_model, "is_ssd"):
+            app = SUPPORTED_APPS.sdnext
+
+        num_seeds = len(getattr(p, "all_seeds", []))
+        options = PPPStateOptions(
+            debug_level=DEBUG_LEVEL(getattr(opts, "ppp_gen_debug_level", PromptPostProcessor.DEFAULT_DEBUG_LEVEL)),
+            on_warning=ONWARNING_CHOICES(getattr(opts, "ppp_gen_onwarning", PromptPostProcessor.DEFAULT_ON_WARNING)),
+            strict_operators=getattr(opts, "ppp_gen_strict_operators", PromptPostProcessor.DEFAULT_STRICT_OPERATORS),
+            process_wildcards=getattr(opts, "ppp_wil_processwildcards", PromptPostProcessor.DEFAULT_PROCESS_WILDCARDS),
+            if_wildcards=IFWILDCARDS_CHOICES(
+                getattr(opts, "ppp_wil_ifwildcards", PromptPostProcessor.DEFAULT_IF_WILDCARDS)
+            ),
+            choice_separator=getattr(opts, "ppp_wil_choice_separator", PromptPostProcessor.DEFAULT_CHOICE_SEPARATOR),
+            keep_choices_order=getattr(
+                opts, "ppp_wil_keep_choices_order", PromptPostProcessor.DEFAULT_KEEP_CHOICES_ORDER
+            ),
+            stn_separator=getattr(opts, "ppp_stn_separator", PromptPostProcessor.DEFAULT_STN_SEPARATOR),
+            stn_ignore_repeats=getattr(opts, "ppp_stn_ignorerepeats", PromptPostProcessor.DEFAULT_STN_IGNORE_REPEATS),
+            cup_do_cleanup=True,
+            cup_cleanup_variables=True,
+            cup_extra_spaces=getattr(opts, "ppp_cup_extraspaces", PromptPostProcessor.DEFAULT_CUP_EXTRA_SPACES),
+            cup_empty_constructs=getattr(
+                opts, "ppp_cup_emptyconstructs", PromptPostProcessor.DEFAULT_CUP_EMPTY_CONSTRUCTS
+            ),
+            cup_extra_separators=getattr(
+                opts, "ppp_cup_extraseparators", PromptPostProcessor.DEFAULT_CUP_EXTRA_SEPARATORS
+            ),
+            cup_extra_separators2=getattr(
+                opts, "ppp_cup_extraseparators2", PromptPostProcessor.DEFAULT_CUP_EXTRA_SEPARATORS2
+            ),
+            cup_extra_separators_include_eol=getattr(
+                opts,
+                "ppp_cup_extraseparators_include_eol",
+                PromptPostProcessor.DEFAULT_CUP_EXTRA_SEPARATORS_INCLUDE_EOL,
+            ),
+            cup_breaks=getattr(opts, "ppp_cup_breaks", PromptPostProcessor.DEFAULT_CUP_BREAKS),
+            cup_breaks_eol=getattr(opts, "ppp_cup_breaks_eol", PromptPostProcessor.DEFAULT_CUP_BREAKS_EOL),
+            cup_ands=getattr(opts, "ppp_cup_ands", PromptPostProcessor.DEFAULT_CUP_ANDS),
+            cup_ands_eol=getattr(opts, "ppp_cup_ands_eol", PromptPostProcessor.DEFAULT_CUP_ANDS_EOL),
+            cup_extranetwork_tags=getattr(
+                opts, "ppp_cup_extranetworktags", PromptPostProcessor.DEFAULT_CUP_EXTRANETWORK_TAGS
+            ),
+            cup_merge_attention=getattr(
+                opts, "ppp_cup_mergeattention", PromptPostProcessor.DEFAULT_CUP_MERGE_ATTENTION
+            ),
+            cup_remove_extranetwork_tags=getattr(
+                opts, "ppp_rem_removeextranetworktags", PromptPostProcessor.DEFAULT_CUP_REMOVE_EXTRANETWORK_TAGS
+            ),
+            do_combinatorial=input_combinatorial,
+            combinatorial_shuffle=input_combinatorial_shuffle,
+            combinatorial_limit=max(num_seeds, int(input_combinatorial_limit)) if input_combinatorial else 0,
+            results_file=getattr(opts, "ppp_gen_resultsfile", PromptPostProcessor.DEFAULT_RESULTS_FILE),
+        )
+        if not self.ppp_init:
+            self.ppp_init = True
+            self.ppp_debug_level = options.debug_level
+            self.lru_cache = PPPLRUCache(1000, logger=self.ppp_logger, debug_level=self.ppp_debug_level)
+            self.wildcards_obj = PPPWildcards(self.ppp_logger)
+            self.extranetwork_mappings_obj = PPPExtraNetworkMappings(self.ppp_logger)
+            log(
+                self.ppp_logger,
+                DEBUG_LEVEL.minimal,
+                logging.INFO,
+                f"{PromptPostProcessor.NAME} {PromptPostProcessor.VERSION} initialized, running on {SUPPORTED_APPS_NAMES[app]}",
+            )
+        t1 = time.monotonic_ns()
+        if getattr(opts, "prompt_attention", "") == "Compel parser":
+            log(self.ppp_logger, self.ppp_debug_level, logging.WARNING, "Compel parser is not supported!")
+        init_images = getattr(p, "init_images", [None]) or [None]
+        is_i2i = bool(init_images[0])
+        do_i2i = getattr(opts, "ppp_gen_doi2i", False)
+        add_prompts = getattr(opts, "ppp_gen_addpromptstometadata", True)
+        if is_i2i and not do_i2i:
+            log(self.ppp_logger, self.ppp_debug_level, logging.INFO, "Not processing the prompt for i2i")
+            return
+
+        p.extra_generation_params.update(
+            {
+                "PPP force equal seeds": input_force_equal_seeds,
+                "PPP unlink seed": input_unlink_seed,
+                "PPP prompt seed": input_seed,
+                "PPP incremental seed": input_incremental_seed,
+                "PPP combinatorial": input_combinatorial,
+            }
+        )
+
+        log(
+            self.ppp_logger,
+            self.ppp_debug_level,
+            logging.INFO,
+            f"Post-processing prompts ({'i2i' if is_i2i else 't2i'})",
+        )
+        env_info = {
+            "app": app.value,
+            "models_path": models_path,
+            "model_filename": getattr(p.sd_model.sd_checkpoint_info, "filename", ""),
+            "model_class": (
+                p.sd_model.model_config.__class__.__name__
+                if app in (SUPPORTED_APPS.forge, SUPPORTED_APPS.forgeneo)
+                else p.sd_model.__class__.__name__
+            ),
+            "property_base": p.sd_model,
+        }
+        wc_wildcards_folders = getattr(opts, "ppp_wil_wildcardsfolders", "")
+        if wc_wildcards_folders == "":
+            wc_wildcards_folders = os.getenv("WILDCARD_DIR", PPPWildcards.DEFAULT_WILDCARDS_FOLDER)
+        wildcards_folders = [
+            (Path(f) if Path(f).is_absolute() else (Path(models_path) / f).resolve())
+            for f in wc_wildcards_folders.split(",")
+            if f.strip() != ""
+        ]
+        en_mappings_folders = getattr(opts, "ppp_en_mappingsfolders", "")
+        if en_mappings_folders == "":
+            en_mappings_folders = os.getenv(
+                "EXTRANETWORKMAPPINGS_DIR",
+                PPPExtraNetworkMappings.DEFAULT_ENMAPPINGS_FOLDER,
+            )
+        enmappings_folders = [
+            (Path(f) if Path(f).is_absolute() else (Path(models_path) / f).resolve())
+            for f in en_mappings_folders.split(",")
+            if f.strip() != ""
+        ]
+        self.wildcards_obj.refresh_wildcards(
+            self.ppp_debug_level, wildcards_folders if options.process_wildcards else None
+        )
+        self.extranetwork_mappings_obj.refresh_extranetwork_mappings(self.ppp_debug_level, enmappings_folders)
+        ppp = PromptPostProcessor(
+            self.ppp_logger,
+            env_info,
+            options,
+            self.grammar_content,
+            self.ppp_interrupt,
+            self.wildcards_obj,
+            self.extranetwork_mappings_obj,
+        )
+        hash_fullenv = hash((ppp.envinfo_hash, ppp.options_hash, self.wildcards_obj, self.extranetwork_mappings_obj))
+
+        if input_force_equal_seeds:
+            log(self.ppp_logger, self.ppp_debug_level, logging.INFO, "Forcing equal seeds")
+            seeds: list[int] = getattr(p, "all_seeds", [])
+            subseeds: list[int] = getattr(p, "all_subseeds", [])
+            p.all_seeds = [seeds[0] for _ in seeds]
+            p.all_subseeds = [subseeds[0] for _ in subseeds]
+
+        calculated_seeds: list[int] = []
+        if input_unlink_seed:
+            log(self.ppp_logger, self.ppp_debug_level, logging.INFO, "Using unlinked seed")
+            if input_incremental_seed:
+                first_seed = np.random.randint(0, 2**32, dtype=np.int64) if input_seed == -1 else input_seed
+                calculated_seeds = [first_seed + i for i in range(num_seeds)]
+            elif input_seed == -1:
+                calculated_seeds = np.random.randint(0, 2**32, size=num_seeds, dtype=np.int64)
+            else:
+                calculated_seeds = [input_seed for _ in range(num_seeds)]
+        else:
+            seeds: list[int] = getattr(p, "all_seeds", [])
+            subseeds: list[int] = getattr(p, "all_subseeds", [])
+            subseed_strength: float = getattr(p, "subseed_strength", 0.0)
+            if subseed_strength > 0:
+                calculated_seeds = [
+                    int(subseed * subseed_strength + seed * (1 - subseed_strength))
+                    for seed, subseed in zip(seeds, subseeds)
+                ]
+                # if len(set(calculated_seeds)) < len(calculated_seeds):
+                #     self.ppp_logger.info("Adjusting seeds because some are equal.")
+                #     calculated_seeds = [seed + i for i, seed in enumerate(calculated_seeds)]
+            else:
+                calculated_seeds = seeds
+
+        # (prompt type, typeindex) -> (new positive prompt, new negative prompt)
+        prompts_list: dict[tuple[str, int], tuple[str, str]] = {}
+        extra_params = {}
+
+        # adds prompts
+        regular_type = "regular"
+        rpr: list[str] = getattr(p, "all_prompts", None)
+        rnr: list[str] = getattr(p, "all_negative_prompts", None)
+        regular_exists = bool(rpr) and bool(rnr)
+        hiresfix_type = "hiresfix"
+        rph: list[str] = getattr(p, "all_hr_prompts", None)
+        rnh: list[str] = getattr(p, "all_hr_negative_prompts", None)
+        hiresfix_exists = bool(rph) and bool(rnh)
+        for i in range(len(calculated_seeds)):
+            if regular_exists:
+                prompts_list[(regular_type, i)] = None
+            if hiresfix_exists:
+                prompts_list[(hiresfix_type, i)] = None
+
+        ppp.process_prompts_group_start()
+        if input_combinatorial:
+            seed_for_comb = calculated_seeds[0] if calculated_seeds else 0
+            regular_copy = (rpr.copy() if rpr else None, rnr.copy() if rnr else None)
+            hiresfix_copy = (rph.copy() if rph else None, rnh.copy() if rnh else None)
+            regular_changes = False
+            hiresfix_changes = False
+            if regular_exists:
+                log(self.ppp_logger, self.ppp_debug_level, logging.INFO, "processing prompts combinatorially (regular)")
+                comb_results = ppp.process_prompt(
+                    rpr[0],
+                    rnr[0],
+                    seed_for_comb,
+                    jobinfo={
+                        "job_timestamp": shared.state.job_timestamp,
+                        "job": shared.state.job,
+                        "detail": "regular prompt combination",
+                    },
+                )
+                num_comb = len(comb_results)
+                for i in range(len(rpr)):  # pylint: disable=consider-using-enumerate
+                    posp, negp, _ = comb_results[i % num_comb]
+                    prompts_list[(regular_type, i)] = (posp, negp)
+                extra_params["PPP combination"] = [str(1 + (i % num_comb)) for i in range(len(rpr))]
+            if hiresfix_exists:
+                hiresfix_equal = regular_exists and rph == rpr and rnh == rnr
+                if hiresfix_equal:
+                    log(
+                        self.ppp_logger,
+                        self.ppp_debug_level,
+                        logging.INFO,
+                        "hiresfix prompts are the same as regular prompts, skipping combinatorial processing for hiresfix",
+                    )
+                    for i in range(len(rph)):  # pylint: disable=consider-using-enumerate
+                        prompts_list[(hiresfix_type, i)] = prompts_list.get((regular_type, i))
+                else:
+                    log(
+                        self.ppp_logger,
+                        self.ppp_debug_level,
+                        logging.INFO,
+                        "processing prompts combinatorially (hiresfix)",
+                    )
+                    comb_results_hr = ppp.process_prompt(
+                        rph[0],
+                        rnh[0],
+                        seed_for_comb,
+                        jobinfo={
+                            "job_timestamp": shared.state.job_timestamp,
+                            "job": shared.state.job,
+                            "detail": "hiresfix prompt combination",
+                        },
+                    )
+                    num_comb_hr = len(comb_results_hr)
+                    for i in range(len(rph)):  # pylint: disable=consider-using-enumerate
+                        posp, negp, _ = comb_results_hr[i % num_comb_hr]
+                        prompts_list[(hiresfix_type, i)] = (posp, negp)
+                    extra_params["PPP HR combination"] = [str(1 + (i % num_comb_hr)) for i in range(len(rph))]
+        else:
+            # processes prompts
+            for prompttype, typeindex in prompts_list.keys():
+                log(
+                    self.ppp_logger,
+                    self.ppp_debug_level,
+                    logging.INFO,
+                    f"processing prompts ({prompttype}[{typeindex+1}])",
+                )
+                key = (
+                    (hash_fullenv, calculated_seeds[typeindex], rpr[typeindex], rnr[typeindex])
+                    if prompttype == regular_type
+                    else (hash_fullenv, calculated_seeds[typeindex], rph[typeindex], rnh[typeindex])
+                )
+                cached = self.lru_cache.get(key)
+                if cached is None:
+                    hsh, seed, prompt, negative_prompt = key
+                    results = ppp.process_prompt(
+                        prompt,
+                        negative_prompt,
+                        seed,
+                        jobinfo={
+                            "job_timestamp": shared.state.job_timestamp,
+                            "job": shared.state.job,
+                            "detail": f"{prompttype} prompt",
+                        },
+                    )
+                    posp, negp, _ = results[0]
+                    cached = (posp, negp)
+                    self.lru_cache.put(key, cached)
+                    # adds also the result so i2i doesn't process it unnecessarily
+                    self.lru_cache.put((hsh, seed, posp, negp), cached)
+                else:
+                    log(self.ppp_logger, self.ppp_debug_level, logging.INFO, "result already in cache")
+                prompts_list[(prompttype, typeindex)] = cached
+        ppp.process_prompts_group_end()
+
+        # updates the prompts
+        regular_copy = (rpr.copy() if rpr else None, rnr.copy() if rnr else None)
+        hiresfix_copy = (rph.copy() if rph else None, rnh.copy() if rnh else None)
+        regular_changes = False
+        hiresfix_changes = False
+        for (prompttype, typeindex), (posp, negp) in prompts_list.items():
+            if prompttype == regular_type:
+                if rpr[typeindex].strip() != posp.strip() or rnr[typeindex].strip() != negp.strip():
+                    regular_changes = True
+                rpr[typeindex] = posp
+                rnr[typeindex] = negp
+            elif prompttype == hiresfix_type:
+                if rph[typeindex].strip() != posp.strip() or rnh[typeindex].strip() != negp.strip():
+                    hiresfix_changes = True
+                rph[typeindex] = posp
+                rnh[typeindex] = negp
+
+        # initialize extra generation parameters
+        if add_prompts:
+            if regular_changes:
+                extra_params["PPP original prompts"] = regular_copy[0]
+                extra_params["PPP original negative prompts"] = regular_copy[1]
+            if hiresfix_changes:
+                extra_params["PPP original HR prompts"] = hiresfix_copy[0]
+                extra_params["PPP original HR negative prompts"] = hiresfix_copy[1]
+
+        # fill extra generation parameters only if not already present
+        for k, v in extra_params.items():
+            if p.extra_generation_params.get(k) is None:
+                p.extra_generation_params[k] = v
+
+        t2 = time.monotonic_ns()
+        log(
+            self.ppp_logger,
+            self.ppp_debug_level,
+            logging.INFO,
+            f"process time: {(t2 - t1) / 1_000_000_000:.3f} seconds",
+        )
+
+    def ppp_interrupt(self):
+        """
+        Interrupts the generation.
+
+        Returns:
+            None
+        """
+        shared.state.interrupted = True
+
+
+def on_ui_settings():
+    """
+    Callback function for UI settings.
+
+    Returns:
+        None
+    """
+
+    section = ("prompt-post-processor", PromptPostProcessor.NAME)
+
+    def import_old_settings(names, default):
+        for name in names:
+            if hasattr(opts, name):
+                return getattr(opts, name)
+        return default
+
+    def import_bool_to_any(name, value_false, value_true, default):
+        if hasattr(opts, name):
+            return value_true if getattr(opts, name) else value_false
+        return default
+
+    def new_html_title(title):
+        info = shared.OptionInfo(
+            title,
+            "",
+            gr.HTML,
+            section=section,
+        )
+        info.do_not_save = True
+        return info
+
+    # general settings
+    shared.opts.add_option(
+        key="ppp_gen_sep",
+        info=new_html_title("<h2>General settings</h2>"),
+    )
+    shared.opts.add_option(
+        key="ppp_gen_debug_level",
+        info=shared.OptionInfo(
+            default=import_bool_to_any(
+                "ppp_gen_debug",
+                DEBUG_LEVEL.minimal.value,
+                DEBUG_LEVEL.full.value,
+                DEBUG_LEVEL.minimal.value,
+            ),
+            label="Debug level",
+            component=gr.Radio,
+            component_args={
+                "choices": (
+                    ("None", DEBUG_LEVEL.none.value),
+                    ("Minimal", DEBUG_LEVEL.minimal.value),
+                    ("Full", DEBUG_LEVEL.full.value),
+                ),
+            },
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_gen_onwarning",
+        info=shared.OptionInfo(
+            default=ONWARNING_CHOICES.warn.value,
+            label="What to do on invalid content warnings",
+            component=gr.Radio,
+            component_args={
+                "choices": (
+                    ("Show warning in console", ONWARNING_CHOICES.warn.value),
+                    ("Stop the generation", ONWARNING_CHOICES.stop.value),
+                )
+            },
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_gen_strict_operators",
+        info=shared.OptionInfo(
+            default=PromptPostProcessor.DEFAULT_STRICT_OPERATORS,
+            label="Use strict operators",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_gen_doi2i",
+        info=shared.OptionInfo(
+            False,
+            label="Apply in img2img",
+            comment_after='<span class="info">(this includes any pass that contains an initial image, like adetailer)</span>',
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_gen_addpromptstometadata",
+        info=shared.OptionInfo(
+            True,
+            label="Add original prompts to metadata (if they change)",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_gen_resultsfile",
+        info=shared.OptionInfo(
+            PromptPostProcessor.DEFAULT_RESULTS_FILE,
+            label="Results file",
+            comment_after=r'<span class="info">(filename to save processing results; supports %datetime%, %date%, %time%, %host% tokens; extension determines format: .yaml, .jsonl, .csv, .txt; empty = disabled)</span>',
+            section=section,
+        ),
+    )
+
+    shared.opts.add_option(
+        key="ppp_en_mappingsfolders",
+        info=shared.OptionInfo(
+            PPPExtraNetworkMappings.DEFAULT_ENMAPPINGS_FOLDER,
+            label="Extranetwork Mappings folders",
+            comment_after='<span class="info">(absolute or relative to the models folder)</span>',
+            section=section,
+        ),
+    )
+
+    # wildcard settings
+    shared.opts.add_option(
+        key="ppp_wil_sep",
+        info=new_html_title("<br><h2>Wildcard settings</h2>"),
+    )
+    shared.opts.add_option(
+        key="ppp_wil_processwildcards",
+        info=shared.OptionInfo(
+            True,
+            label="Process wildcards",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_wil_wildcardsfolders",
+        info=shared.OptionInfo(
+            PPPWildcards.DEFAULT_WILDCARDS_FOLDER,
+            label="Wildcards folders",
+            comment_after='<span class="info">(absolute or relative to the models folder)</span>',
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_wil_ifwildcards",
+        info=shared.OptionInfo(
+            default=import_old_settings(
+                ["ppp_gen_ifwildcards", "ppp_ifwildcards"],
+                IFWILDCARDS_CHOICES.ignore.value,
+            ),
+            label="What to do with remaining/invalid wildcards?",
+            component=gr.Radio,
+            component_args={
+                "choices": (
+                    ("Ignore", IFWILDCARDS_CHOICES.ignore.value),
+                    ("Remove", IFWILDCARDS_CHOICES.remove.value),
+                    ("Add visible warning", IFWILDCARDS_CHOICES.warn.value),
+                    ("Stop the generation", IFWILDCARDS_CHOICES.stop.value),
+                )
+            },
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_wil_choice_separator",
+        info=shared.OptionInfo(
+            PromptPostProcessor.DEFAULT_CHOICE_SEPARATOR,
+            label="Default separator used when adding multiple choices",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_wil_keep_choices_order",
+        info=shared.OptionInfo(
+            False,
+            label="Keep the order of selected choices",
+            section=section,
+        ),
+    )
+
+    # content removal settings
+    shared.opts.add_option(
+        key="ppp_rem_sep",
+        info=new_html_title("<br><h2>Content removal settings</h2>"),
+    )
+    shared.opts.add_option(
+        key="ppp_rem_removeextranetworktags",
+        info=shared.OptionInfo(
+            False,
+            label="Remove extra network tags",
+            section=section,
+        ),
+    )
+
+    # send to negative settings
+    shared.opts.add_option(
+        key="ppp_stn_sep",
+        info=new_html_title("<br><h2>Send to Negative settings</h2>"),
+    )
+    shared.opts.add_option(
+        key="ppp_stn_separator",
+        info=shared.OptionInfo(
+            PromptPostProcessor.DEFAULT_STN_SEPARATOR,
+            label="Separator used when adding to the negative prompt",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_stn_ignorerepeats",
+        info=shared.OptionInfo(
+            True,
+            label="Ignore repeated content",
+            section=section,
+        ),
+    )
+    # clean-up settings
+    shared.opts.add_option(
+        key="ppp_cup_sep",
+        info=new_html_title("<br><h2>Clean-up settings</h2>"),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_emptyconstructs",
+        info=shared.OptionInfo(
+            True,
+            label="Remove empty constructs (attention, alternation, scheduling)",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_extraseparators",
+        info=shared.OptionInfo(
+            True,
+            label="Remove extra separators",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_extraseparators2",
+        info=shared.OptionInfo(
+            True,
+            label="Remove additional extra separators",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_extraseparators_include_eol",
+        info=shared.OptionInfo(
+            False,
+            label="The extra separators options also remove EOLs",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_breaks",
+        info=shared.OptionInfo(
+            True,
+            label="Clean up around BREAKs",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_breaks_eol",
+        info=shared.OptionInfo(
+            False,
+            label="Use EOL instead of Space before BREAKs",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_ands",
+        info=shared.OptionInfo(
+            True,
+            label="Clean up around ANDs",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_ands_eol",
+        info=shared.OptionInfo(
+            False,
+            label="Use EOL instead of Space before ANDs",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_extranetworktags",
+        info=shared.OptionInfo(
+            False,
+            label="Clean up around extra network tags",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_extraspaces",
+        info=shared.OptionInfo(
+            True,
+            label="Remove extra spaces",
+            section=section,
+        ),
+    )
+    shared.opts.add_option(
+        key="ppp_cup_mergeattention",
+        info=shared.OptionInfo(
+            True,
+            label="Merge attention modifiers (weights) when possible",
+            section=section,
+        ),
+    )
+
+    # Remove old settings
+    # for name in ["ppp_gen_ifwildcards", "ppp_ifwildcards", "ppp_gen_debug", "ppp_stn_doi2i", "ppp_cup_doi2i"]:
+    #     if hasattr(opts, name):
+    #         delattr(opts, name)
+
+
+script_callbacks.on_ui_settings(on_ui_settings)

sendtonegative.py

@@ -1,332 +0,0 @@
-from collections import namedtuple
-import re
-import math
-import lark
-
-
-class SendToNegative:  # pylint: disable=too-few-public-methods
-    NAME = "Send to Negative"
-    VERSION = "2.1.4"
-
-    DEFAULT_SEPARATOR = ", "
-
-    def __init__(
-        self,
-        log,
-        separator=None,
-        ignore_repeats=None,
-        join_attention=None,
-        cleanup=None,
-        opts=None,
-    ):
-        """
-        Format for the tag:
-            <!content!>
-
-            <!!x!content!>
-
-        with x being:
-            s - content is added at the start of the negative prompt. This is the default if no parameter exists.
-
-            e - content is added at the end of the negative prompt.
-
-            pN - content is added where the insertion point N is in the negative prompt or at the start if it does not exist. N can be 0 to 9.
-
-            iN - tags the position of insertion point N. Used only in the negative prompt and does not accept content. N can be 0 to 9.
-        """
-        self.__logger = log
-        if opts is not None and getattr(opts, "prompt_attention", "") == "Compel parser":
-            self.__logger.warning("Compel parser is not supported!")
-        self.__ignore_repeats = (
-            ignore_repeats if ignore_repeats is not None else getattr(opts, "stn_ignorerepeats", True)
-        )
-        self.__join_attention = (
-            join_attention
-            if join_attention is not None
-            else getattr(opts, "stn_joinattention", True)
-            if opts is not None
-            else True
-        )
-        self.__cleanup = (
-            cleanup if cleanup is not None else getattr(opts, "stn_cleanup", True) if opts is not None else True
-        )
-        self.__separator = (
-            separator
-            if separator is not None
-            else getattr(opts, "stn_separator", self.DEFAULT_SEPARATOR)
-            if opts is not None
-            else self.DEFAULT_SEPARATOR
-        )
-        self.__insertion_point_tags = [f"<!!i{x}!!>" for x in range(10)]
-        # Process with lark (debug with https://www.lark-parser.org/ide/)
-        self.__schedule_parser = lark.Lark(
-            r"""
-                start: (prompt | /[\][():|<>!]/+)*
-                ?prompt: (emphasized | deemphasized | scheduled | alternate | modeltag | negtag | plain)*
-                ?nonegprompt: (emphasized | deemphasized | scheduled | alternate | modeltag | plain)*
-                emphasized: "(" prompt [":" numpar] ")"
-                deemphasized: "[" prompt "]"
-                scheduled: "[" [prompt ":"] prompt ":" numpar "]"
-                alternate: "[" alternateoption ("|" alternateoption)+ "]"
-                alternateoption: prompt
-                negtag: "<!" [negtagparameters] nonegprompt "!>"
-                negtagparameters: "!" /s|e|[ip]\d/ "!"
-                modeltag: "<" /(?!!)[^>]+/ ">"
-                numpar: WHITESPACE* NUMBER WHITESPACE*
-                WHITESPACE: /\s+/
-                ?plain: /([^\\[\]():|<>!]|\\.)+/s
-                %import common.SIGNED_NUMBER -> NUMBER
-            """,
-            propagate_positions=True,
-        )
-
-    class ReadTree(lark.visitors.Interpreter):
-        def __init__(self, logger, ignorerepeats, joinattention, prompt, add_at):
-            super().__init__()
-            self.__logger = logger
-            self.__ignore_repeats = ignorerepeats
-            self.__join_attention = joinattention
-            self.__prompt = prompt
-            self.AccumulatedShell = namedtuple("AccumulatedShell", ["type", "info1", "info2"])
-            AccumulatedShell = self.AccumulatedShell
-            self.__shell: list[AccumulatedShell] = []
-            self.NegTag = namedtuple("NegTag", ["start", "end", "content", "parameters", "shell"])
-            NegTag = self.NegTag
-            self.__negtags: list[NegTag] = []
-            self.__already_processed = []
-            self.add_at = add_at
-            self.remove = []
-
-        def __get_numpar_value(self, numpar):
-            return float(next(x for x in numpar.children if x.type == "NUMBER").value)
-
-        def scheduled(self, tree):
-            if len(tree.children) > 2:  # before & after
-                before = tree.children[0]
-            else:
-                before = None
-            after = tree.children[-2]
-            numpar = tree.children[-1]
-            pos = self.__get_numpar_value(numpar)
-            if pos >= 1:
-                pos = int(pos)
-            # self.__shell.append(self.AccumulatedShell("sc", tree.meta.start_pos, pos))
-            if before is not None and hasattr(before, "data"):
-                self.__logger.debug(
-                    f"Shell scheduled before at {[before.meta.start_pos,before.meta.end_pos] if hasattr(before,'meta') and not before.meta.empty else '?'} : {pos}"
-                )
-                self.__shell.append(self.AccumulatedShell("scb", pos, None))
-                self.visit(before)
-                self.__shell.pop()
-            if hasattr(after, "data"):
-                self.__logger.debug(
-                    f"Shell scheduled after at {[after.meta.start_pos,after.meta.end_pos] if hasattr(after,'meta') and not after.meta.empty else '?'} : {pos}"
-                )
-                self.__shell.append(self.AccumulatedShell("sca", pos, None))
-                self.visit(after)
-                self.__shell.pop()
-            # self.__shell.pop()
-
-        def alternate(self, tree):
-            # self.__shell.append(self.AccumulatedShell("al", tree.meta.start_pos, len(tree.children)))
-            for i, opt in enumerate(tree.children):
-                self.__logger.debug(
-                    f"Shell alternate at {[opt.meta.start_pos,opt.meta.end_pos] if hasattr(opt,'meta') and not opt.meta.empty else '?'} : {i+1}"
-                )
-                if hasattr(opt, "data"):
-                    self.__shell.append(self.AccumulatedShell("alo", i + 1, len(tree.children)))
-                    self.visit(opt)
-                    self.__shell.pop()
-            # self.__shell.pop()
-
-        def emphasized(self, tree):
-            numpar = tree.children[-1]
-            weight = self.__get_numpar_value(numpar) if numpar is not None else 1.1
-            self.__logger.debug(
-                f"Shell attention at {[tree.meta.start_pos,tree.meta.end_pos] if hasattr(tree,'meta') and not tree.meta.empty else '?'}: {weight}"
-            )
-            self.__shell.append(self.AccumulatedShell("at", weight, None))
-            self.visit_children(tree)
-            self.__shell.pop()
-
-        def deemphasized(self, tree):
-            weight = 0.9
-            self.__logger.debug(
-                f"Shell attention at {[tree.meta.start_pos,tree.meta.end_pos] if hasattr(tree,'meta') and not tree.meta.empty else '?'}: {weight}"
-            )
-            self.__shell.append(self.AccumulatedShell("at", weight, None))
-            self.visit_children(tree)
-            self.__shell.pop()
-
-        def negtag(self, tree):
-            negtagparameters = tree.children[0]
-            parameters = negtagparameters.children[0].value if negtagparameters is not None else ""
-            rest = []
-            for x in tree.children[1::]:
-                rest.append(
-                    self.__prompt[x.meta.start_pos : x.meta.end_pos]
-                    if hasattr(x, "meta") and not x.meta.empty
-                    else x.value
-                )
-            content = "".join(rest)
-            self.__negtags.append(
-                self.NegTag(tree.meta.start_pos, tree.meta.end_pos, content, parameters, self.__shell.copy())
-            )
-            self.__logger.debug(
-                f"Negative tag at {[tree.meta.start_pos,tree.meta.end_pos] if hasattr(tree,'meta') and not tree.meta.empty else '?'}: {parameters}: {content.encode('unicode_escape').decode('utf-8')}"
-            )
-
-        def start(self, tree):
-            self.visit_children(tree)
-            # process the found negtags
-            for nt in self.__negtags:
-                if self.__join_attention:
-                    # join consecutive attention elements
-                    for i in range(len(nt.shell) - 1, 0, -1):
-                        if nt.shell[i].type == "at" and nt.shell[i - 1].type == "at":
-                            nt.shell[i - 1] = self.AccumulatedShell(
-                                "at",
-                                math.floor(100 * nt.shell[i - 1].info1 * nt.shell[i].info1)
-                                / 100,  # we limit to two decimals
-                                None,
-                            )
-                            nt.shell.pop(i)
-                start = ""
-                end = ""
-                for s in nt.shell:
-                    match s.type:
-                        case "at":
-                            if s.info1 == 0.9:
-                                start += "["
-                                end = "]" + end
-                            elif s.info1 == 1.1:
-                                start += "("
-                                end = ")" + end
-                            else:
-                                start += "("
-                                end = f":{s.info1})" + end
-                        # case "sc":
-                        case "scb":
-                            start += "["
-                            end = f"::{s.info1}]" + end
-                        case "sca":
-                            start += "["
-                            end = f":{s.info1}]" + end
-                        # case "al":
-                        case "alo":
-                            start += "[" + ("|" * int(s.info1 - 1))
-                            end = ("|" * int(s.info2 - s.info1)) + "]" + end
-                content = start + nt.content + end
-                position = nt.parameters or "s"
-                if len(content) > 0:
-                    if content not in self.__already_processed:
-                        if self.__ignore_repeats:
-                            self.__already_processed.append(content)
-                        self.__logger.debug(
-                            f"Adding content at position {position}: {content.encode('unicode_escape').decode('utf-8')}"
-                        )
-                        if position == "e":
-                            self.add_at["end"].append(content)
-                        elif position.startswith("p"):
-                            n = int(position[1])
-                            self.add_at["insertion_point"][n].append(content)
-                        else:  # position == "s" or invalid
-                            self.add_at["start"].append(content)
-                    else:
-                        self.__logger.warning(
-                            f"Ignoring repeated content: {content.encode('unicode_escape').decode('utf-8')}"
-                        )
-                # remove from prompt
-                self.remove.append([nt.start, nt.end])
-
-    def process_prompt(self, original_prompt, original_negative_prompt):
-        """
-        Extract from the prompt the tagged parts and add them to the negative prompt
-        """
-        try:
-            prompt = original_prompt
-            negative_prompt = original_negative_prompt
-            self.__logger.debug(f"Input prompt: {prompt.encode('unicode_escape').decode('utf-8')}")
-            self.__logger.debug(f"Input negative_prompt: {negative_prompt.encode('unicode_escape').decode('utf-8')}")
-            prompt, add_at = self.__find_tags(prompt)
-            negative_prompt = self.__add_to_insertion_points(negative_prompt, add_at["insertion_point"])
-            if len(add_at["start"]) > 0:
-                negative_prompt = self.__add_to_start(negative_prompt, add_at["start"])
-            if len(add_at["end"]) > 0:
-                negative_prompt = self.__add_to_end(negative_prompt, add_at["end"])
-            self.__logger.debug(f"Output prompt: {prompt.encode('unicode_escape').decode('utf-8')}")
-            self.__logger.debug(f"Output negative_prompt: {negative_prompt.encode('unicode_escape').decode('utf-8')}")
-            return prompt, negative_prompt
-        except Exception as e:  # pylint: disable=broad-exception-caught
-            self.__logger.exception(e)
-            return original_prompt, original_negative_prompt
-
-    def __find_tags(self, prompt):
-        add_at = {"start": [], "insertion_point": [[] for x in range(10)], "end": []}
-        tree = self.__schedule_parser.parse(prompt)
-        self.__logger.debug(f"Initial tree:\n{tree.pretty()}")
-
-        readtree = self.ReadTree(self.__logger, self.__ignore_repeats, self.__join_attention, prompt, add_at)
-        readtree.visit(tree)
-
-        for r in readtree.remove[::-1]:
-            prompt = prompt[: r[0]] + prompt[r[1] :]
-        if self.__cleanup:
-            prompt = re.sub(r"\((?::[+-]?[\d\.]+)?\)", "", prompt)  # clean up empty attention
-            prompt = re.sub(r"\[\]", "", prompt)  # clean up empty attention
-            prompt = re.sub(r"\[:?:[+-]?[\d\.]+\]", "", prompt)  # clean up empty scheduling
-            prompt = re.sub(r"\[\|+\]", "", prompt)  # clean up empty alternation
-            # clean up whitespace and extra separators
-            prompt = (
-                prompt.replace("  ", " ")
-                .replace(self.__separator + self.__separator, self.__separator)
-                .replace(" " + self.__separator, self.__separator)
-                .removeprefix(self.__separator)
-                .removesuffix(self.__separator)
-                .strip()
-            )
-        add_at = readtree.add_at
-        self.__logger.debug(f"New negative additions: {add_at}")
-
-        return prompt, add_at
-
-    def __add_to_insertion_points(self, negative_prompt, add_at_insertion_point):
-        for n in range(10):
-            ipp = negative_prompt.find(self.__insertion_point_tags[n])
-            if ipp >= 0:
-                ipl = len(self.__insertion_point_tags[n])
-                if negative_prompt[ipp - len(self.__separator) : ipp] == self.__separator:
-                    ipp -= len(self.__separator)  # adjust for existing start separator
-                    ipl += len(self.__separator)
-                add_at_insertion_point[n].insert(0, negative_prompt[:ipp])
-                if negative_prompt[ipp + ipl : ipp + ipl + len(self.__separator)] == self.__separator:
-                    ipl += len(self.__separator)  # adjust for existing end separator
-                endPart = negative_prompt[ipp + ipl :]
-                if len(endPart) > 0:
-                    add_at_insertion_point[n].append(endPart)
-                negative_prompt = self.__separator.join(add_at_insertion_point[n])
-            else:
-                ipp = 0
-                if negative_prompt.startswith(self.__separator):
-                    ipp = len(self.__separator)
-                add_at_insertion_point[n].append(negative_prompt[ipp:])
-                negative_prompt = self.__separator.join(add_at_insertion_point[n])
-        return negative_prompt
-
-    def __add_to_start(self, negative_prompt, add_at_start):
-        if len(negative_prompt) > 0:
-            ipp = 0
-            if negative_prompt.startswith(self.__separator):
-                ipp = len(self.__separator)  # adjust for existing end separator
-            add_at_start.append(negative_prompt[ipp:])
-        negative_prompt = self.__separator.join(add_at_start)
-        return negative_prompt
-
-    def __add_to_end(self, negative_prompt, add_at_end):
-        if len(negative_prompt) > 0:
-            ipl = len(negative_prompt)
-            if negative_prompt.endswith(self.__separator):
-                ipl -= len(self.__separator)  # adjust for existing start separator
-            add_at_end.insert(0, negative_prompt[:ipl])
-        negative_prompt = self.__separator.join(add_at_end)
-        return negative_prompt

stnlogging.py

@@ -1,40 +0,0 @@
-import sys
-import copy
-import logging
-
-
-class SendToNegativeLogFactory:
-    class ColoredFormatter(logging.Formatter):
-        COLORS = {
-            "DEBUG": "\033[0;36m",  # CYAN
-            "INFO": "\033[0;32m",  # GREEN
-            "WARNING": "\033[0;33m",  # YELLOW
-            "ERROR": "\033[0;31m",  # RED
-            "CRITICAL": "\033[0;37;41m",  # WHITE ON RED
-            "RESET": "\033[0m",  # RESET COLOR
-        }
-
-        def format(self, record):
-            colored_record = copy.copy(record)
-            levelname = colored_record.levelname
-            seq = self.COLORS.get(levelname, self.COLORS["RESET"])
-            colored_record.levelname = f"{seq}{levelname}{self.COLORS['RESET']}"
-            return super().format(colored_record)
-
-    def __init__(self):
-        logsd = logging.getLogger("sd")
-        stnlog = logging.getLogger("SendToNegative")
-        stnlog.setLevel(logging.INFO)
-        stnlog.handlers = logsd.handlers
-        if not stnlog.handlers:
-            handler = logging.StreamHandler(sys.stdout)
-            handler.setFormatter(self.ColoredFormatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
-            stnlog.addHandler(handler)
-            self.log = stnlog
-        else:
-            self.log = SendToNegativeLogCustomAdapter(stnlog)
-
-
-class SendToNegativeLogCustomAdapter(logging.LoggerAdapter):
-    def process(self, msg, kwargs):
-        return f"[SendToNegative] {msg}", kwargs

tests/base_tests.py

@@ -0,0 +1,352 @@
+from dataclasses import replace
+import difflib
+import logging
+from pathlib import Path
+from typing import Any, NamedTuple, Optional
+import unittest
+import datetime
+
+from ppp_classes import IFWILDCARDS_CHOICES, ONWARNING_CHOICES, PPPStateOptions
+from ppp_enmappings import PPPExtraNetworkMappings  # type: ignore
+from ppp_wildcards import PPPWildcards  # type: ignore
+from ppp import PromptPostProcessor  # type: ignore
+from ppp_logging import DEBUG_LEVEL, PromptPostProcessorLogFactory  # type: ignore
+from ppp_common import load_grammar  # type: ignore
+
+
+class InputTuple(NamedTuple):
+    prompt: str = ""
+    negative_prompt: str = ""
+
+
+class OutputTuple(NamedTuple):
+    prompt: str = ""
+    negative_prompt: str = ""
+    variables: dict[str, Any] = None
+
+
+class TestPromptPostProcessorBase(unittest.TestCase):
+    """
+    A test case class for testing the PromptPostProcessor class.
+    """
+
+    def setUp(self, enable_file_logging=False):
+        """
+        Set up the test case by initializing the necessary objects and configurations.
+
+        Args:
+            enable_file_logging (bool): Whether to enable logging to a file. Defaults to True.
+        """
+        self.enable_file_logging = enable_file_logging
+        test_name = self.id().split(".")[-1]  # Extract the test method name
+        timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+
+        if self.enable_file_logging:
+            log_filename = f"tests/logs/{test_name}_{timestamp}.log"
+        else:
+            log_filename = None  # Disable file logging
+
+        self.lf = PromptPostProcessorLogFactory(log_filename)
+        self.ppp_logger = self.lf.log
+        self.ppp_logger.setLevel(logging.DEBUG)
+        self.grammar_content = None
+        self.defopts = PPPStateOptions(
+            debug_level=DEBUG_LEVEL.full,
+            on_warning=ONWARNING_CHOICES.stop,
+            strict_operators=True,
+            process_wildcards=True,
+            if_wildcards=IFWILDCARDS_CHOICES.stop,
+            choice_separator=", ",
+            keep_choices_order=False,
+            stn_separator=", ",
+            stn_ignore_repeats=True,
+            cup_do_cleanup=True,
+            cup_cleanup_variables=True,
+            cup_empty_constructs=True,
+            cup_extra_separators=True,
+            cup_extra_separators2=True,
+            cup_extra_separators_include_eol=False,
+            cup_extra_spaces=True,
+            cup_breaks=True,
+            cup_breaks_eol=False,
+            cup_ands=True,
+            cup_ands_eol=False,
+            cup_extranetwork_tags=True,
+            cup_merge_attention=True,
+            cup_remove_extranetwork_tags=False,
+            do_combinatorial=False,
+            combinatorial_shuffle=False,
+            combinatorial_limit=0,
+            results_file=(Path(__file__).parent / "logs" / "output_%date%.txt") if enable_file_logging else "",
+        )
+        self.def_env_info = {
+            "app": "tests",
+            "ppp_config": None,
+            "model_class": "SDXL",
+            "property_base": {"is_sdxl": True},
+            "models_path": "./webui/models",
+            "model_filename": "./webui/models/Stable-diffusion/testmodel.safetensors",
+        }
+        self.interrupted = False
+        self.wildcards_obj = PPPWildcards(self.lf.log)
+        self.extranetwork_maps_obj = PPPExtraNetworkMappings(self.lf.log)
+        tests_folder = Path(__file__).parent
+        self.wildcards_obj.refresh_wildcards(
+            DEBUG_LEVEL.full,
+            [
+                tests_folder / "wildcards",
+                tests_folder / "wildcards2",
+            ],
+            """
+            yaml_input:
+                wildcardI:
+                    - choice1
+                    - choice2
+                    - choice3
+            """,
+        )
+        self.extranetwork_maps_obj.refresh_extranetwork_mappings(
+            DEBUG_LEVEL.full,
+            [tests_folder / "enmappings"],
+            """
+            """,
+        )
+        self.grammar_content = load_grammar()
+
+    def interrupt(self):
+        self.interrupted = True
+
+    def init_ppp(
+        self,
+        ppp: Optional[str | PromptPostProcessor] = None,
+        combinatorial: bool = False,
+        combinatorial_limit: int = 0,
+    ) -> PromptPostProcessor:
+        if isinstance(ppp, str):
+            if ppp == "nocup":
+                the_obj = PromptPostProcessor(
+                    self.ppp_logger,
+                    self.def_env_info,
+                    replace(
+                        self.defopts,
+                        cup_do_cleanup=False,
+                        cup_cleanup_variables=False,
+                        cup_empty_constructs=False,
+                        cup_extra_separators=False,
+                        cup_extra_separators2=False,
+                        cup_extra_separators_include_eol=False,
+                        cup_extra_spaces=False,
+                        cup_breaks=False,
+                        cup_breaks_eol=False,
+                        cup_ands=False,
+                        cup_ands_eol=False,
+                        cup_extranetwork_tags=False,
+                        cup_merge_attention=False,
+                        do_combinatorial=combinatorial,
+                        combinatorial_limit=combinatorial_limit,
+                    ),
+                    self.grammar_content,
+                    self.interrupt,
+                    self.wildcards_obj,
+                    self.extranetwork_maps_obj,
+                )
+            elif ppp == "nostrict":
+                the_obj = PromptPostProcessor(
+                    self.ppp_logger,
+                    self.def_env_info,
+                    replace(
+                        self.defopts,
+                        strict_operators=False,
+                        do_combinatorial=combinatorial,
+                        combinatorial_limit=combinatorial_limit,
+                    ),
+                    self.grammar_content,
+                    self.interrupt,
+                    self.wildcards_obj,
+                    self.extranetwork_maps_obj,
+                )
+        else:
+            the_obj = ppp
+        if not the_obj:
+            the_obj = PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    do_combinatorial=combinatorial,
+                    combinatorial_limit=combinatorial_limit,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            )
+        return the_obj
+
+    def _comp_diff(self, result: str, expected: str) -> list[str]:
+        return list(
+            difflib.ndiff(
+                result.splitlines(True),
+                expected.splitlines(True),
+                linejunk=None,
+                charjunk=None,
+            )
+        )
+
+    def process(
+        self,
+        input_prompts: InputTuple,
+        expected_output: Optional[OutputTuple | list[OutputTuple]] = None,
+        seed: int = 1,
+        ppp: Optional[str | PromptPostProcessor] = None,
+        interrupted: bool = False,
+        combinatorial: bool = False,
+        combinatorial_limit: int = 0,
+        specific_wc_folders: Optional[list[Path]] = None,
+        specific_em_folders: Optional[list[Path]] = None,
+    ):
+        """
+        Process the prompt and compare the results with the expected prompts.
+
+        Args:
+            input_prompts (InputTuple): The input prompts.
+            expected_output (OutputTuple | list[OutputTuple], optional): The expected output. When a list is provided, the test will run once for each expected output, using the same input prompt, but seed will be incremented for each iteration.
+            seed (int, optional): The seed value. Defaults to 1.
+            ppp (Optional[str | PromptPostProcessor], optional): The PromptPostProcessor instance or type. Defaults to None.
+            interrupted (bool, optional): The interrupted flag. Defaults to False.
+            combinatorial (bool, optional): The combinatorial flag. Defaults to False.
+            combinatorial_limit (int, optional): The combinatorial limit. Defaults to 0.
+            specific_wc_folders (Optional[list[Path]], optional): A list of specific wildcard folders to refresh. Defaults to None.
+            specific_em_folders (Optional[list[Path]], optional): A list of specific extranetwork mapping folders to refresh. Defaults to None.
+
+        Returns:
+            None
+        """
+        if specific_wc_folders is not None:
+            self.wildcards_obj.refresh_wildcards(
+                DEBUG_LEVEL.full,
+                specific_wc_folders,
+            )
+        if specific_em_folders is not None:
+            self.extranetwork_maps_obj.refresh_extranetwork_mappings(
+                DEBUG_LEVEL.full,
+                specific_em_folders,
+            )
+        the_obj: PromptPostProcessor = self.init_ppp(ppp, combinatorial, combinatorial_limit)
+        out = (
+            [OutputTuple("", "", None)]
+            if expected_output is None
+            else expected_output if isinstance(expected_output, list) else [expected_output]
+        )
+        if the_obj.state.options.do_combinatorial:
+            # combinatorial
+            errors = []
+            the_obj.process_prompts_group_start()
+            result = the_obj.process_prompt(
+                input_prompts.prompt,
+                input_prompts.negative_prompt,
+                seed,
+            )
+            the_obj.process_prompts_group_end()
+            self.assertTrue(
+                self.interrupted == interrupted,
+                f"Interrupted flag is incorrect: got {self.interrupted} but expected {interrupted}",
+            )
+            if not self.interrupted and expected_output is not None:
+                if len(result) != len(out):
+                    errors.append(f"Incorrect number of combinations: got {len(result)} but expected {len(out)}")
+                for out_prompt, out_negative_prompt, out_variables in out:
+                    found = None
+                    for r_prompt, r_negative_prompt, r_variables in result:
+                        if r_prompt == out_prompt and r_negative_prompt == out_negative_prompt:
+                            found = OutputTuple(r_prompt, r_negative_prompt, r_variables)
+                            break
+                    if not found:
+                        errors.extend(
+                            [
+                                "Combination not found in output",
+                                "Prompt:",
+                                out_prompt,
+                                "Negative Prompt:",
+                                out_negative_prompt,
+                            ]
+                        )
+                    elif out_variables:
+                        missing_vars = {}
+                        incorrect_vars = {}
+                        expected_values = {}
+                        sorted_var_keys = sorted(out_variables.keys())
+                        for var_name in sorted_var_keys:
+                            var_value = out_variables[var_name]
+                            if var_name not in found.variables:
+                                missing_vars[var_name] = var_value
+                            elif found.variables[var_name] != var_value:
+                                incorrect_vars[var_name] = found.variables[var_name]
+                                expected_values[var_name] = var_value
+                        if missing_vars or incorrect_vars:
+                            errors.extend(
+                                [
+                                    "Combination found, but variables do not match",
+                                    "Prompt:",
+                                    out_prompt,
+                                    "Negative Prompt:",
+                                    out_negative_prompt,
+                                ]
+                            )
+                            if missing_vars:
+                                errors.append("Missing variables:")
+                                errors.append(str(missing_vars))
+                            if incorrect_vars:
+                                errors.append("Incorrect variables:")
+                                errors.extend(self._comp_diff(str(incorrect_vars), str(expected_values)))
+            if errors:
+                raise AssertionError("\n".join(errors))
+            return
+
+        # non-combinatorial
+        errors = []
+        the_obj.process_prompts_group_start()
+        for eo in out:
+            result = the_obj.process_prompt(
+                input_prompts.prompt,
+                input_prompts.negative_prompt,
+                seed,
+            )
+            self.assertTrue(
+                self.interrupted == interrupted,
+                f"Interrupted flag is incorrect: got {self.interrupted} but expected {interrupted}",
+            )
+            if not self.interrupted and expected_output is not None:
+                result_prompt, result_negative_prompt, output_variables = result[0] if result else (None, None, None)
+                if result_prompt != eo.prompt or result_negative_prompt != eo.negative_prompt:
+                    errors.append("Incorrect result")
+                    if result_prompt != eo.prompt:
+                        errors.append("Prompt:")
+                        errors.extend(self._comp_diff(result_prompt, eo.prompt))
+                    if result_negative_prompt != eo.negative_prompt:
+                        errors.append("Negative Prompt:")
+                        errors.extend(self._comp_diff(result_negative_prompt, eo.negative_prompt))
+                if eo.variables:
+                    missing_vars = {}
+                    incorrect_vars = {}
+                    expected_values = {}
+                    sorted_var_keys = sorted(eo.variables.keys())
+                    for var_name in sorted_var_keys:
+                        var_value = eo.variables[var_name]
+                        if var_name not in output_variables:
+                            missing_vars[var_name] = var_value
+                        elif output_variables[var_name] != var_value:
+                            incorrect_vars[var_name] = output_variables[var_name]
+                            expected_values[var_name] = var_value
+                    if missing_vars or incorrect_vars:
+                        errors.append("Variables do not match")
+                        if missing_vars:
+                            errors.append("Missing variables:")
+                            errors.append(str(missing_vars))
+                        if incorrect_vars:
+                            errors.append("Incorrect variables:")
+                            errors.extend(self._comp_diff(str(incorrect_vars), str(expected_values)))
+            seed += 1
+        the_obj.process_prompts_group_end()
+        if errors:
+            raise AssertionError("\n".join(errors))

tests/enmappings/mappings.yaml

@@ -0,0 +1,20 @@
+lora:
+  lora1:
+    - condition: _is_pony
+      name: lorapony
+      parameters: 0.8
+      triggers: ["triggerpony1", "triggerpony2"]
+    - condition: _is_illustrious
+      name: loraillustrious
+      parameters: "0.9:0.8"
+      triggers: ["triggerillustrious1", "triggerillustrious2"]
+    - triggers: ["triggergeneric1", "triggergeneric2", "{one|two}"]
+  loraany:
+    - condition: true
+      name: loraany1
+      parameters: 0.8
+      triggers: ["trigger1", "trigger2"]
+    - condition: true
+      name: loraany2
+      parameters: 1.0
+      triggers: ["trigger3", "trigger4"]

tests/tests.py

@@ -1,183 +0,0 @@
-import logging
-import unittest
-import sys
-import os
-
-sys.path.insert(1, os.path.join(sys.path[0], ".."))
-
-from sendtonegative import SendToNegative  # pylint: disable=import-error
-from stnlogging import SendToNegativeLogFactory
-
-
-class TestSendToNegative(unittest.TestCase):
-    def setUp(self):
-        lf = SendToNegativeLogFactory()
-        self.__log = lf.log
-        self.__log.setLevel(logging.DEBUG)
-        self.defstn = SendToNegative(self.__log, separator=", ", ignore_repeats=True, join_attention=True, cleanup=True)
-
-    def process(
-        self,
-        prompt,
-        negative_prompt,
-        expected_prompt,
-        expected_negative_prompt,
-        stn=None,
-    ):
-        the_obj = self.defstn if stn is None else stn
-        result_prompt, result_negative_prompt = the_obj.process_prompt(prompt, negative_prompt)
-        self.assertEqual(result_prompt, expected_prompt, f"Prompt should be '{expected_prompt}'")
-        self.assertEqual(
-            result_negative_prompt,
-            expected_negative_prompt,
-            f"Negative Prompt should be '{expected_negative_prompt}'",
-        )
-
-    def test_tag_default(self):
-        self.process(
-            "flowers<!red!>",
-            "normal quality, worse quality",
-            "flowers",
-            "red, normal quality, worse quality",
-        )
-
-    def test_tag_start(self):
-        self.process(
-            "flowers<!!s!red!>",
-            "normal quality, worse quality",
-            "flowers",
-            "red, normal quality, worse quality",
-        )
-
-    def test_tag_end(self):
-        self.process(
-            "flowers<!!e!red!>",
-            "normal quality, worse quality",
-            "flowers",
-            "normal quality, worse quality, red",
-        )
-
-    def test_tag_insertion_mid_sep(self):
-        self.process(
-            "flowers<!!p0!red!>",
-            "normal quality, <!!i0!!>, worse quality",
-            "flowers",
-            "normal quality, red, worse quality",
-        )
-
-    def test_tag_insertion_mid_no_sep(self):
-        self.process(
-            "flowers<!!p0!red!>",
-            "normal quality<!!i0!!>worse quality",
-            "flowers",
-            "normal quality, red, worse quality",
-        )
-
-    def test_tag_insertion_start_sep(self):
-        self.process(
-            "flowers<!!p0!red!>",
-            "<!!i0!!>, normal quality, worse quality",
-            "flowers",
-            "red, normal quality, worse quality",
-        )
-
-    def test_tag_insertion_start_no_sep(self):
-        self.process(
-            "flowers<!!p0!red!>",
-            "<!!i0!!>normal quality, worse quality",
-            "flowers",
-            "red, normal quality, worse quality",
-        )
-
-    def test_tag_insertion_end_sep(self):
-        self.process(
-            "flowers<!!p0!red!>",
-            "normal quality, worse quality, <!!i0!!>",
-            "flowers",
-            "normal quality, worse quality, red",
-        )
-
-    def test_tag_insertion_end_no_sep(self):
-        self.process(
-            "flowers<!!p0!red!>",
-            "normal quality, worse quality<!!i0!!>",
-            "flowers",
-            "normal quality, worse quality, red",
-        )
-
-    def test_complex(self):
-        self.process(
-            "<!red!> (<!!s!pink!>), flowers <!!e!purple!>, <!!e!blue!>, <!!p0!yellow!> <!!p1!green!>",
-            "normal quality, <!!i0!!>, bad quality<!!i1!!>, worse quality",
-            "flowers",
-            "red, (pink), normal quality, yellow, bad quality, green, worse quality, purple, blue",
-        )
-
-    def test_complex_no_cleanup(self):
-        self.process(
-            "<!red!> (<!!s!pink!>), flowers <!!e!purple!>, <!!e!blue!>, <!!p0!yellow!> <!!p1!green!>",
-            "normal quality, <!!i0!!>, bad quality<!!i1!!>, worse quality",
-            " (), flowers , ,  ",
-            "red, (pink), normal quality, yellow, bad quality, green, worse quality, purple, blue",
-            SendToNegative(self.__log, separator=", ", ignore_repeats=True, join_attention=True, cleanup=False),
-        )
-
-    def test_inside_attention1(self):
-        self.process(
-            "[<!neg1!>] this is a ((test<!!e!neg2!>) (test:2.0): 1.5 )",
-            "normal quality",
-            "this is a ((test) (test:2.0): 1.5 )",
-            "[neg1], normal quality, (neg2:1.65)",
-        )
-
-    def test_inside_attention2(self):
-        self.process(
-            "(red<![square]!>:1.5)",
-            "",
-            "(red:1.5)",
-            "([square]:1.5)",
-        )
-
-    def test_inside_alternation1(self):
-        self.process(
-            "this is a (([complex|simple<!neg1!>|regular] test)(test:2.0):1.5)",
-            "normal quality",
-            "this is a (([complex|simple|regular] test)(test:2.0):1.5)",
-            "([|neg1|]:1.65), normal quality",
-        )
-
-    def test_inside_alternation2(self):
-        self.process(
-            "this is a (([complex<!neg1!>|simple<!neg2!>|regular<!neg3!>] test)(test:2.0):1.5)",
-            "normal quality",
-            "this is a (([complex|simple|regular] test)(test:2.0):1.5)",
-            "([neg1||]:1.65), ([|neg2|]:1.65), ([||neg3]:1.65), normal quality",
-        )
-
-    def test_inside_alternation3(self):
-        self.process(
-            "this is a (([complex<!neg1!>[one|two<!neg12!>||three|four(<!neg14!>)]|simple<!neg2!>|regular<!neg3!>] test)(test:2.0):1.5)",
-            "normal quality",
-            "this is a (([complex[one|two||three|four]|simple|regular] test)(test:2.0):1.5)",
-            "([neg1||]:1.65), ([[|neg12|||]||]:1.65), ([[||||(neg14)]||]:1.65), ([|neg2|]:1.65), ([||neg3]:1.65), normal quality",
-        )
-
-    def test_inside_scheduling(self):
-        self.process(
-            "this is [abc<!neg1!>:def<!!e!neg2!>: 5 ]",
-            "normal quality",
-            "this is [abc:def: 5 ]",
-            "[neg1::5], normal quality, [neg2:5]",
-        )
-
-    def test_complex_features(self):
-        self.process(
-            "[<!neg5!>] this is: a (([complex|simple<!neg6!>|regular] test<!neg1!>)(test:2.0):1.5) \nBREAK with [abc<!neg4!>:def<!!p0!neg2(neg3:1.6)!>:5] <lora:xxx:1>",
-            "normal quality, <!!i0!!>",
-            "this is: a (([complex|simple|regular] test)(test:2.0):1.5) \nBREAK with [abc:def:5] <lora:xxx:1>",
-            "[neg5], ([|neg6|]:1.65), (neg1:1.65), [neg4::5], normal quality, [neg2(neg3:1.6):5]",
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/tests_choices.py

@@ -0,0 +1,169 @@
+from dataclasses import replace
+
+from ppp import PromptPostProcessor  # type: ignore
+from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
+
+if __name__ == "__main__":
+    raise SystemExit("This script must not be run directly")
+
+
+class TestChoices(TestPromptPostProcessorBase):
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp(enable_file_logging=False)
+
+    # Choices tests
+
+    def test_ch_choices(self):  # simple choices with weights
+        self.process(
+            InputTuple("the choices are: {3::choice1|2::choice2|choice3}", ""),
+            OutputTuple("the choices are: choice2", ""),
+            ppp="nocup",
+        )
+
+    def test_ch_cyclical(self):  # cyclical sampler cycles through all choices
+        ppp_instance = self.init_ppp("nocup")
+        self.process(
+            InputTuple("the choices are: {@choice1|choice2|choice3}", ""),
+            [
+                OutputTuple("the choices are: choice1", ""),
+                OutputTuple("the choices are: choice2", ""),
+                OutputTuple("the choices are: choice3", ""),
+                OutputTuple("the choices are: choice1", ""),  # cycles back
+            ],
+            ppp=ppp_instance,
+        )
+
+    def test_ch_cyclical_multiple_constructs(self):  # two independent @ constructs cycle together
+        ppp_instance = self.init_ppp("nocup")
+        self.process(
+            InputTuple("{@a|b} {@c|d}", ""),
+            [
+                OutputTuple("a c", ""),
+                OutputTuple("a d", ""),
+                OutputTuple("b c", ""),
+                OutputTuple("b d", ""),
+                OutputTuple("a c", ""),  # cycles back
+            ],
+            ppp=ppp_instance,
+        )
+
+    def test_ch_cyclical_resets_on_prompt_change(self):  # state resets when the prompt pair changes
+        ppp_instance = self.init_ppp("nocup")
+        # Advance the cycle to position 1 (choice2).
+        self.process(
+            InputTuple("the choices are: {@choice1|choice2|choice3}", ""),
+            [
+                OutputTuple("the choices are: choice1", ""),
+                OutputTuple("the choices are: choice2", ""),
+            ],
+            ppp=ppp_instance,
+        )
+        # A different prompt must restart from position 0 (choice1).
+        self.process(
+            InputTuple("the choices are: {@choice1|choice2|choice3} different", ""),
+            OutputTuple("the choices are: choice1 different", ""),
+            ppp=ppp_instance,
+        )
+
+    def test_ch_cyclical_mixed_samplers(self):  # @ construct cycles while a ~ construct alongside is unaffected
+        ppp_instance = self.init_ppp("nocup")
+        self.process(
+            InputTuple("{@a|b|c} {x|y}", ""),
+            [
+                OutputTuple("a y", ""),
+                OutputTuple("b x", ""),
+                OutputTuple("c x", ""),
+                OutputTuple("a y", ""),  # @ cycles back
+            ],
+            ppp=ppp_instance,
+        )
+
+    def test_ch_choices_withcomments(self):  # choices with comments and multiline
+        self.process(
+            InputTuple(
+                "the choices are: {\n3::choice1 # this is option 1\n|2::choice2\n# this was option 2\n|choice3 # this is option 3\n}",
+                "",
+            ),
+            OutputTuple("the choices are: choice2", ""),
+            ppp="nocup",
+        )
+
+    def test_ch_choices_multiple(self):  # choices with multiple selection
+        self.process(
+            InputTuple("the choices are: {~2$$, $$3::choice1|2:: choice2 |choice3}", ""),
+            OutputTuple("the choices are:  choice2 , choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_ch_choices_if_multiple(self):  # choices with if and multiple selection
+        self.process(
+            InputTuple("the choices are: {2$$, $$3::choice1|2 if _is_sd1::choice2|choice3}", ""),
+            OutputTuple("the choices are: choice1, choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_ch_choices_set_if_multiple(self):  # choices with if user variable and multiple selection
+        self.process(
+            InputTuple("${var=test}the choices are: {2$$, $$3::choice1|2 if not var eq 'test'::choice2|choice3}", ""),
+            OutputTuple("the choices are: choice1, choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_ch_choices_set_if_nested(self):  # nested choices with if user variable and multiple selection
+        self.process(
+            InputTuple(
+                "${var=test}the choices are: {2$$, $$3::choice1${var2=test2} {if var2 eq 'test2'::choice11|choice12}|2 if not var eq 'test'::choice2|choice3}",
+                "",
+            ),
+            OutputTuple("the choices are: choice1 choice11, choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_ch_choicesinsidelora(self):  # simple choices inside a lora
+        self.process(
+            InputTuple("<lora:test1:1><lora:test__other__name:1><lora:test2:{0.2|0.5|0.7|1}>", ""),
+            OutputTuple("<lora:test1:1><lora:test__other__name:1><lora:test2:0.7>", ""),
+            ppp="nocup",
+        )
+
+    def test_ch_removelorawithchoices(self):
+        self.process(
+            InputTuple("<lora:test1:1><lora:test2:{0.2|0.5|0.7|1}>", ""),
+            OutputTuple("", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    cup_remove_extranetwork_tags=True,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_ch_cmd_includewildcard(self):
+        self.process(
+            InputTuple("{ch_one|ch_two|%0.5::include yaml/wildcard1}", ""),
+            OutputTuple("ch_two", ""),
+            ppp="nocup",
+        )
+
+    # Combinatorial
+
+    def test_ch_combinatorial(self):
+        self.process(
+            InputTuple("{choice1|choice2|choice3}, ${v:{option1|option2}}", ""),
+            [
+                OutputTuple("choice1, option1", ""),
+                OutputTuple("choice1, option2", ""),
+                OutputTuple("choice2, option1", ""),
+                OutputTuple("choice2, option2", ""),
+                OutputTuple("choice3, option1", ""),
+                OutputTuple("choice3, option2", "", {"v": "option2"}),
+            ],
+            combinatorial=True,
+        )

tests/tests_cleanup.py

@@ -0,0 +1,199 @@
+import logging
+from dataclasses import replace
+
+from ppp import PromptPostProcessor  # type: ignore
+from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
+
+
+if __name__ == "__main__":
+    raise SystemExit("This script must not be run directly")
+
+
+class TestCleanup(TestPromptPostProcessorBase):
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp(enable_file_logging=False)
+
+    # Cleanup tests
+
+    def test_cl_simple(self):  # simple cleanup
+        self.process(
+            InputTuple("  this is a ((test ), , ,  (), ,   [] ( , test ,:2.0):1.5), (red:1.5)  ", "  normal quality  "),
+            OutputTuple("this is a ((test), (test,:2):1.5), (red:1.5)", "normal quality"),
+        )
+
+    def test_cl_complex(self):  # complex cleanup
+        self.process(
+            InputTuple(
+                "  this is BREAKABLE a ((test)), ,AND AND(() [] <lora:test> ANDERSON (test:2.0):1.5) :o BREAK \n BREAK (red:1.5)  ",
+                "  [:hands, feet, :0.15]normal quality  ",
+            ),
+            OutputTuple(
+                "this is BREAKABLE a (test:1.21) AND(<lora:test> ANDERSON (test:2):1.5) :o BREAK (red:1.5)",
+                "[:hands, feet, :0.15]normal quality",
+            ),
+        )
+
+    def test_cl_removenetworktags(self):  # remove network tags
+        self.process(
+            InputTuple("this is a <lora:test:1> test__yaml/wildcard7__", ""),
+            OutputTuple("this is a test", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    cup_remove_extranetwork_tags=True,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_cl_dontremoveseparatorsoneol(self):  # don't remove separators on eol
+        self.process(
+            InputTuple("this is a test,\nsecond line", ""),
+            OutputTuple("this is a test,\nsecond line", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    cup_extra_separators2=False,
+                    cup_extra_separators_include_eol=False,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_cl_separatorswitheol(self):  # don't remove eols with the separators
+        self.process(
+            InputTuple(
+                """{      (d:0.9) ,, (l:1.1)  | (l:1.1)         (d:0.9),,, }
+                        (l:1.1)  
+                        (d:0.9)""",
+                "",
+            ),
+            OutputTuple(
+                """ (l:1.1)         (d:0.9), 
+                        (l:1.1)  
+                        (d:0.9)""",
+                "",
+            ),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    cup_empty_constructs=False,
+                    cup_extra_separators=True,
+                    cup_extra_separators2=False,
+                    cup_extra_separators_include_eol=False,
+                    cup_extra_spaces=False,
+                    cup_breaks=False,
+                    cup_breaks_eol=False,
+                    cup_ands=False,
+                    cup_ands_eol=False,
+                    cup_extranetwork_tags=False,
+                    cup_merge_attention=False,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_cl_mergeattention(self):  # merge attention
+        self.process(
+            InputTuple(
+                "this is (a test:0.9) of (attention (merging:1.2)) where ((this)) ((is joined:1.2)) and ([this too]:1.3)",
+                "",
+            ),
+            OutputTuple(
+                "this is [a test] of (attention (merging:1.2)) where (this:1.21) (is joined:1.32) and (this too:1.17)",
+                "",
+            ),
+        )
+
+    def test_cl_not_mergeattention(self):  # not merge attention
+        self.process(
+            InputTuple(
+                "this is (a test:0.9) of not (attention (merging:1.2)) where ((this)) ((is not joined:1.2)) and neither is ([this]:1.3)",
+                "",
+            ),
+            OutputTuple(
+                "this is (a test:0.9) of not (attention (merging:1.2)) where ((this)) ((is not joined:1.2)) and neither is ([this]:1.3)",
+                "",
+            ),
+            ppp="nocup",
+        )
+
+    # Result warnings tests
+
+    def test_cl_warn_unmatched_open_paren(self):  # unmatched open parenthesis triggers warning
+        with self.assertLogs("PromptPostProcessor", level=logging.WARNING) as cm:
+            self.process(
+                InputTuple("(unclosed paren", ""),
+                OutputTuple("(unclosed paren", ""),
+            )
+        self.assertTrue(
+            any("Unmatched" in msg for msg in cm.output),
+            "Expected an 'Unmatched' warning for open parenthesis",
+        )
+
+    def test_cl_warn_unmatched_close_paren(self):  # unmatched close parenthesis triggers warning
+        with self.assertLogs("PromptPostProcessor", level=logging.WARNING) as cm:
+            self.process(
+                InputTuple("extra close paren)", ""),
+                OutputTuple("extra close paren)", ""),
+            )
+        self.assertTrue(
+            any("Unmatched" in msg for msg in cm.output),
+            "Expected an 'Unmatched' warning for close parenthesis",
+        )
+
+    def test_cl_warn_mismatched_brackets(self):  # mismatched bracket types trigger warning
+        with self.assertLogs("PromptPostProcessor", level=logging.WARNING) as cm:
+            self.process(
+                InputTuple("(mismatched]", ""),
+                OutputTuple("(mismatched]", ""),
+            )
+        self.assertTrue(
+            any("Mismatched" in msg or "Unmatched" in msg for msg in cm.output),
+            "Expected a 'Mismatched' or 'Unmatched' warning for bracket mismatch",
+        )
+
+    def test_cl_warn_unmatched_open_bracket(self):  # unmatched open bracket triggers warning
+        with self.assertLogs("PromptPostProcessor", level=logging.WARNING) as cm:
+            self.process(
+                InputTuple("unclosed [bracket", ""),
+                OutputTuple("unclosed [bracket", ""),
+            )
+        self.assertTrue(
+            any("Unmatched" in msg for msg in cm.output),
+            "Expected an 'Unmatched' warning for open bracket",
+        )
+
+    def test_cl_warn_unmatched_complex(self):  # unmatched complex case triggers warning
+        with self.assertLogs("PromptPostProcessor", level=logging.WARNING) as cm:
+            self.process(
+                InputTuple("[(unmatched [bracket))", ""),
+                OutputTuple("[(unmatched [bracket))", ""),
+            )
+        self.assertTrue(
+            any("Unmatched" in msg for msg in cm.output),
+            "Expected an 'Unmatched' warning",
+        )
+
+    def test_cl_warn_escaped_unmatched_no_false_warning(self):  # escaped unmatched paren/bracket does not trigger warning
+        with self.assertNoLogs("PromptPostProcessor", level=logging.WARNING):
+            self.process(
+                InputTuple(r"text with \(escaped unmatched\]", ""),
+                OutputTuple(r"text with \(escaped unmatched\]", ""),
+            )

tests/tests_host.py

@@ -0,0 +1,438 @@
+from ppp import PromptPostProcessor  # type: ignore
+from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
+
+if __name__ == "__main__":
+    raise SystemExit("This script must not be run directly")
+
+
+class TestHosts(TestPromptPostProcessorBase):
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp(enable_file_logging=False)
+
+    # Hosts tests
+
+    def test_host_attention_parentheses(self):
+        self.process(
+            InputTuple(
+                "[test1] (test2) (test3:1.5) [(test4)]",
+                "",
+            ),
+            OutputTuple("(test1:0.9) (test2) (test3:1.5) (test4:0.99)", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"attention": "parentheses"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_attention_disable(self):
+        self.process(
+            InputTuple(
+                "[test1] (test2) (test3:1.5)",
+                "",
+            ),
+            OutputTuple("test1 test2 test3", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"attention": "disable"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_attention_remove(self):
+        self.process(
+            InputTuple(
+                "[test1] (test2) (test3:1.5)",
+                "",
+            ),
+            OutputTuple("", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"attention": "remove"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_attention_error(self):
+        self.process(
+            InputTuple(
+                "[test1] (test2) (test3:1.5)",
+                "",
+            ),
+            OutputTuple("", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"attention": "error"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+            interrupted=True,
+        )
+
+    def test_host_scheduling_before(self):
+        self.process(
+            InputTuple(
+                "[test1:test2:0.5]",
+                "",
+            ),
+            OutputTuple("test1", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"scheduling": "before"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_scheduling_after(self):
+        self.process(
+            InputTuple(
+                "[test1:test2:0.5]",
+                "",
+            ),
+            OutputTuple("test2", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"scheduling": "after"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_scheduling_first(self):
+        self.process(
+            InputTuple(
+                "[test1::0.5] [:test2:0.5] [test3:test4:0.5]",
+                "",
+            ),
+            OutputTuple("test1 test3", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"scheduling": "first"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_scheduling_remove(self):
+        self.process(
+            InputTuple(
+                "[test1:test2:0.5]",
+                "",
+            ),
+            OutputTuple("", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"scheduling": "remove"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_scheduling_error(self):
+        self.process(
+            InputTuple(
+                "[test1:test2:0.5]",
+                "",
+            ),
+            OutputTuple("", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"scheduling": "error"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+            interrupted=True,
+        )
+
+    def test_host_alternation_first(self):
+        self.process(
+            InputTuple(
+                "[test1|test2|test3]",
+                "",
+            ),
+            OutputTuple("test1", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"alternation": "first"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_alternation_remove(self):
+        self.process(
+            InputTuple(
+                "[test1|test2|test3]",
+                "",
+            ),
+            OutputTuple("", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"alternation": "remove"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_alternation_error(self):
+        self.process(
+            InputTuple(
+                "[test1|test2|test3]",
+                "",
+            ),
+            OutputTuple("", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"alternation": "error"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+            interrupted=True,
+        )
+
+    def test_host_and_eol(self):
+        self.process(
+            InputTuple(
+                "test1 AND test2:2",
+                "",
+            ),
+            OutputTuple("test1\ntest2", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"and": "eol"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_and_comma(self):
+        self.process(
+            InputTuple(
+                "test1 AND test2:2",
+                "",
+            ),
+            OutputTuple("test1, test2", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"and": "comma"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_and_remove(self):
+        self.process(
+            InputTuple(
+                "test1 AND test2:2",
+                "",
+            ),
+            OutputTuple("test1 test2", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"and": "remove"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_and_error(self):
+        self.process(
+            InputTuple(
+                "test1 AND test2:2",
+                "",
+            ),
+            OutputTuple("", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"and": "error"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+            interrupted=True,
+        )
+
+    def test_host_break_eol(self):
+        self.process(
+            InputTuple(
+                "test1 BREAK test2",
+                "",
+            ),
+            OutputTuple("test1\ntest2", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"break": "eol"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_break_comma(self):
+        self.process(
+            InputTuple(
+                "test1 BREAK test2",
+                "",
+            ),
+            OutputTuple("test1, test2", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"break": "comma"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_break_remove(self):
+        self.process(
+            InputTuple(
+                "test1 BREAK test2",
+                "",
+            ),
+            OutputTuple("test1 test2", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"break": "remove"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_host_break_error(self):
+        self.process(
+            InputTuple(
+                "test1 BREAK test2",
+                "",
+            ),
+            OutputTuple("", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "ppp_config": {"hosts": {"tests": {"break": "error"}}},
+                },
+                self.defopts,
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+            interrupted=True,
+        )

tests/tests_performance.py

@@ -0,0 +1,75 @@
+from .base_tests import InputTuple, TestPromptPostProcessorBase
+
+if __name__ == "__main__":
+    raise SystemExit("This script must not be run directly")
+
+
+class TestPerformance(TestPromptPostProcessorBase):
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp(enable_file_logging=False)
+
+    # Performance tests
+
+    def test_parser_performance_simple_simpleparser(
+        self,
+    ):  # performance test with a large prompt without new constructs
+        large_prompt = ", ".join(
+            ["(this:1.2) is a [test] using a [simple|low complexity] prompt with <lora:test:1>"] * 15
+        )
+        self.process(
+            InputTuple(large_prompt, ""),
+            ppp="nocup",
+        )
+
+    def test_parser_performance_simple_fullparser(
+        self,
+    ):  # performance test with a large prompt without new constructs but using full parser
+        # we trick it to use the full parser by including some characters
+        large_prompt = "{__${x:}}" + ", ".join(
+            ["(this:1.2) is a [test] using a [simple|low complexity] prompt with <lora:test:1>"] * 15
+        )
+        self.process(
+            InputTuple(large_prompt, ""),
+            ppp="nocup",
+        )
+
+    def test_parser_performance_complex_fullparser(
+        self,
+    ):  # performance test with a large prompt with new constructs (full parser)
+        large_prompt = ", ".join(["__yaml/wildcard1__, (__yaml/wildcard2__), __yaml/wildcard3__, {one|two|three}"] * 15)
+        self.process(
+            InputTuple(large_prompt, ""),
+            ppp="nocup",
+        )
+
+    # the following tests are performance tests with only one kind of the old constructs
+    # same number of constructs and approximately the same full length
+
+    def test_parser_performance_simple_attention(self):  # performance test with only attention
+        large_prompt = ", ".join(["(one:1.2) two (three) four [five] six"] * 20)
+        self.process(
+            InputTuple(large_prompt, ""),
+            ppp="nocup",
+        )
+
+    def test_parser_performance_simple_schedules(self):  # performance test with only schedules
+        large_prompt = ", ".join(["[one:1:0.5] two [three:0.8] four [five:5:0.2] six"] * 20)
+        self.process(
+            InputTuple(large_prompt, ""),
+            ppp="nocup",
+        )
+
+    def test_parser_performance_simple_alternation(self):  # performance test with only alternation
+        large_prompt = ", ".join(["[one|1] two [three|3] four [five|5] six"] * 20)
+        self.process(
+            InputTuple(large_prompt, ""),
+            ppp="nocup",
+        )
+
+    def test_parser_performance_simple_extranetwork(self):  # performance test with only extra networks
+        large_prompt = ", ".join(["<lora:one:1> two <lora:three:1> four <lora:five:1> six"] * 20)
+        self.process(
+            InputTuple(large_prompt, ""),
+            ppp="nocup",
+        )

tests/tests_stn.py

@@ -0,0 +1,124 @@
+from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
+
+if __name__ == "__main__":
+    raise SystemExit("This script must not be run directly")
+
+
+class TestSendToNegative(TestPromptPostProcessorBase):
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp(enable_file_logging=False)
+
+    # Send To Negative tests
+
+    def test_stn_simple(self):  # negtags with different parameters and separations
+        self.process(
+            InputTuple(
+                "flowers<ppp:stn>red<ppp:/stn>, <ppp:stn s>green<ppp:/stn>, <ppp:stn e>blue<ppp:/stn><ppp:stn p0>yellow<ppp:/stn>, <ppp:stn p1>purple<ppp:/stn><ppp:stn p2>black<ppp:/stn>",
+                "<ppp:stn i0/>normal quality<ppp:stn i1>, worse quality<ppp:stn i2/>",
+            ),
+            OutputTuple("flowers", "red, green, yellow, normal quality, purple, worse quality, black, blue"),
+        )
+
+    def test_stn_complex(self):  # complex negtags
+        self.process(
+            InputTuple(
+                "<ppp:stn>red<ppp:/stn> ((<ppp:stn s>pink<ppp:/stn>)), flowers <ppp:stn e>purple<ppp:/stn>, <ppp:stn p0>mauve<ppp:/stn><ppp:stn e>blue<ppp:/stn>, <ppp:stn p0>yellow<ppp:/stn> <ppp:stn p1>green<ppp:/stn>",
+                "normal quality, <ppp:stn i0/>, bad quality<ppp:stn i1/>, worse quality",
+            ),
+            OutputTuple(
+                "flowers",
+                "red, (pink:1.21), normal quality, mauve, yellow, bad quality, green, worse quality, purple, blue",
+            ),
+        )
+
+    def test_stn_complex_nocleanup(self):  # complex negtags with no cleanup
+        self.process(
+            InputTuple(
+                "<ppp:stn>red<ppp:/stn> ((<ppp:stn s>pink<ppp:/stn>)), flowers <ppp:stn e>purple<ppp:/stn>, <ppp:stn p0>mauve<ppp:/stn><ppp:stn e>blue<ppp:/stn>, <ppp:stn p0>yellow<ppp:/stn> <ppp:stn p1>green<ppp:/stn>",
+                "normal quality, <ppp:stn i0/>, bad quality<ppp:stn i1/>, worse quality",
+            ),
+            OutputTuple(
+                " (()), flowers , ,  ",
+                "red, ((pink)), normal quality, mauve, yellow, bad quality, green, worse quality, purple, blue",
+            ),
+            ppp="nocup",
+        )
+
+    def test_stn_inside_attention(self):  # negtag inside attention
+        self.process(
+            InputTuple(
+                "[<ppp:stn>neg1<ppp:/stn>] this is a ((test<ppp:stn e>neg2<ppp:/stn>) (test:2.0): 1.5 ) (red<ppp:stn>[square]<ppp:/stn>:1.5)",
+                "normal quality",
+            ),
+            OutputTuple(
+                "this is a ((test) (test:2):1.5) (red:1.5)", "[neg1], (square:1.35), normal quality, (neg2:1.65)"
+            ),
+        )
+
+    def test_stn_inside_alternation(self):  # negtag inside alternation
+        self.process(
+            InputTuple(
+                "this is a (([complex<ppp:stn>neg1<ppp:/stn>|simple<ppp:stn>neg2<ppp:/stn>|regular<ppp:stn>neg3<ppp:/stn>] test)(test:2.0):1.5)",
+                "normal quality",
+            ),
+            OutputTuple(
+                "this is a (([complex|simple|regular] test)(test:2):1.5)",
+                "([neg1||]:1.65), ([|neg2|]:1.65), ([||neg3]:1.65), normal quality",
+            ),
+        )
+
+    def test_stn_inside_alternation_recursive(self):  # negtag inside alternation (recursive alternation)
+        self.process(
+            InputTuple(
+                "this is a (([complex<ppp:stn>neg1<ppp:/stn>[one|two<ppp:stn>neg12<ppp:/stn>||three|four(<ppp:stn>neg14<ppp:/stn>)]|simple<ppp:stn>neg2<ppp:/stn>|regular<ppp:stn>neg3<ppp:/stn>] test)(test:2.0):1.5)",
+                "normal quality",
+            ),
+            OutputTuple(
+                "this is a (([complex[one|two||three|four]|simple|regular] test)(test:2):1.5)",
+                "([neg1||]:1.65), ([[|neg12|||]||]:1.65), ([[||||(neg14)]||]:1.65), ([|neg2|]:1.65), ([||neg3]:1.65), normal quality",
+            ),
+        )
+
+    def test_stn_inside_scheduling(self):  # negtag inside scheduling
+        self.process(
+            InputTuple("this is [abc<ppp:stn>neg1<ppp:/stn>:def<ppp:stn e>neg2<ppp:/stn>: 5 ]", "normal quality"),
+            OutputTuple("this is [abc:def:5]", "[neg1::5], normal quality, [neg2:5]"),
+        )
+
+    def test_stn_complex_features(self):  # complex negtags with AND, BREAK and other features
+        self.process(
+            InputTuple(
+                "[<ppp:stn>neg5<ppp:/stn>] this \\(is\\): a (([complex|simple<ppp:stn>neg6<ppp:/stn>|regular] test<ppp:stn>neg1<ppp:/stn>)(test:2.0):1.5) \nBREAK, BREAK with [abc<ppp:stn>neg4<ppp:/stn>:def<ppp:stn p0>neg2(neg3:1.6)<ppp:/stn>:5]:0.5 AND loratrigger <lora:xxx:1> AND AND hypernettrigger <hypernet:yyy>:0.3",
+                "normal quality, <ppp:stn i0/>",
+            ),
+            OutputTuple(
+                "this \\(is\\): a (([complex|simple|regular] test)(test:2):1.5)\nBREAK with [abc:def:5]:0.5 AND loratrigger <lora:xxx:1> AND hypernettrigger <hypernet:yyy>:0.3",
+                "[neg5], ([|neg6|]:1.65), (neg1:1.65), [neg4::5], normal quality, [neg2(neg3:1.6):5]",
+            ),
+        )
+
+    def test_stn_complex_features_newformat(self):  # complex negtags with AND, BREAK and other features (new format)
+        self.process(
+            InputTuple(
+                "[<ppp:stn>neg5<ppp:/stn>] this \\(is\\): a (([complex|simple<ppp:stn>neg6<ppp:/stn>|regular] test<ppp:stn>neg1<ppp:/stn>)(test:2.0):1.5) \nBREAK, BREAK with [abc<ppp:stn>neg4<ppp:/stn>:def<ppp:stn p0>neg2(neg3:1.6)<ppp:/stn>:5]:0.5 AND loratrigger <lora:xxx:1> AND AND hypernettrigger <hypernet:yyy>:0.3",
+                "normal quality, <ppp:stn i0/>",
+            ),
+            OutputTuple(
+                "this \\(is\\): a (([complex|simple|regular] test)(test:2):1.5)\nBREAK with [abc:def:5]:0.5 AND loratrigger <lora:xxx:1> AND hypernettrigger <hypernet:yyy>:0.3",
+                "[neg5], ([|neg6|]:1.65), (neg1:1.65), [neg4::5], normal quality, [neg2(neg3:1.6):5]",
+            ),
+        )
+
+    def test_stn_inside_alternation_recursive_2(self):  # negtag inside alternation (recursive alternation)
+        self.process(
+            InputTuple(
+                "[pos1<ppp:stn>neg1<ppp:/stn>[pos11|pos12<ppp:stn>neg12<ppp:/stn>||pos14|pos15<ppp:stn>neg15<ppp:/stn>]|pos2<ppp:stn>neg2<ppp:/stn>|pos3<ppp:stn>neg3<ppp:/stn>]",
+                "",
+            ),
+            OutputTuple(
+                "[pos1[pos11|pos12||pos14|pos15]|pos2|pos3]",
+                "[neg1||], [[|neg12|||]||], [[||||neg15]||], [|neg2|], [||neg3]",
+                # "[neg1[|neg12|||neg15]|neg2|neg3]", # expected output if the constructs were unified
+            ),
+        )

tests/tests_variants.py

@@ -0,0 +1,105 @@
+from dataclasses import replace
+
+from ppp import PromptPostProcessor
+from ppp_classes import ONWARNING_CHOICES  # type: ignore
+from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
+
+if __name__ == "__main__":
+    raise SystemExit("This script must not be run directly")
+
+
+class TestModelVariants(TestPromptPostProcessorBase):
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp(enable_file_logging=False)
+
+    # Model variants tests
+
+    def test_variants(self):
+        self.process(
+            InputTuple(
+                "<ppp:if _is_test1>test1<ppp:/if><ppp:if _is_test2>test2<ppp:/if><ppp:if _is_test3>test3<ppp:/if><ppp:if _is_test4>test4<ppp:/if>",
+                "",
+            ),
+            OutputTuple("test1test2", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "model_filename": "./webui/models/Stable-diffusion/testmodel.safetensors",
+                    "ppp_config": {
+                        "models": {
+                            "sd1": {
+                                "detect": {"tests": {"class": ["SD15", "SD15_instructpix2pix"]}},
+                                "variants": {
+                                    "test3": {"find_in_filename": "testmodel"},
+                                    "sdxl": {"find_in_filename": "testmodel"},
+                                },
+                            },
+                            "sdxl": {
+                                "detect": {
+                                    "tests": {
+                                        "class": [
+                                            "SDXL",
+                                            "SDXLRefiner",
+                                            "SDXL_instructpix2pix",
+                                            "Segmind_Vega",
+                                            "KOALA_700M",
+                                            "KOALA_1B",
+                                        ]
+                                    }
+                                },
+                                "variants": {
+                                    "test1": {"find_in_filename": "testmodel"},
+                                    "test2": {"find_in_filename": "testmodel"},
+                                },
+                            },
+                            "something": {
+                                "detect": {"tests": {"class": ["something"]}},
+                                "variants": {
+                                    "test4": {"find_in_filename": "testmodel"},
+                                },
+                            },
+                        }
+                    },
+                },
+                replace(
+                    self.defopts,
+                    on_warning=ONWARNING_CHOICES.warn,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_variants_null_model(self):
+        """null model in config disables detection and its variants"""
+        self.process(
+            InputTuple(
+                "<ppp:if _is_sdxl>SDXL<ppp:else>not SDXL<ppp:/if>, <ppp:if _is_pony>PONY<ppp:else>not PONY<ppp:/if>",
+                "",
+            ),
+            OutputTuple("not SDXL, not PONY", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                {
+                    **self.def_env_info,
+                    "model_filename": "./webui/models/Stable-diffusion/ponymodel.safetensors",
+                    "ppp_config": {
+                        "models": {
+                            "sdxl": None,
+                        }
+                    },
+                },
+                replace(
+                    self.defopts,
+                    on_warning=ONWARNING_CHOICES.warn,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )

tests/tests_wildcards.py

@@ -0,0 +1,631 @@
+from dataclasses import replace
+
+from ppp import PromptPostProcessor
+from ppp_classes import IFWILDCARDS_CHOICES
+from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
+
+if __name__ == "__main__":
+    raise SystemExit("This script must not be run directly")
+
+
+class TestWildcards(TestPromptPostProcessorBase):
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp(enable_file_logging=False)
+
+    # Wildcards tests
+
+    def test_wc_ignore(self):  # wildcards with ignore option
+        self.process(
+            InputTuple("__bad_wildcard__", "{option1|option2}"),
+            OutputTuple("__bad_wildcard__", "{option1|option2}"),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    process_wildcards=False,
+                    if_wildcards=IFWILDCARDS_CHOICES.ignore,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_wc_remove(self):  # wildcards with remove option
+        self.process(
+            InputTuple(
+                "[<ppp:stn>neg5<ppp:/stn>] this is: __bad_wildcard__ a (([complex|simple<ppp:stn>neg6<ppp:/stn>|regular] test<ppp:stn>neg1<ppp:/stn>)(test:2.0):1.5) \nBREAK, BREAK with [abc<ppp:stn>neg4<ppp:/stn>:def<ppp:stn p0>neg2(neg3:1.6)<ppp:/stn>:5] <lora:xxx:1>",
+                "normal quality, <ppp:stn i0/> {option1|option2}",
+            ),
+            OutputTuple(
+                "this is: a (([complex|simple|regular] test)(test:2):1.5)\nBREAK with [abc:def:5]<lora:xxx:1>",
+                "[neg5], ([|neg6|]:1.65), (neg1:1.65), [neg4::5], normal quality, [neg2(neg3:1.6):5]",
+            ),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    process_wildcards=False,
+                    if_wildcards=IFWILDCARDS_CHOICES.remove,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_wc_warn(self):  # wildcards with warn option
+        self.process(
+            InputTuple("__bad_wildcard__", "{option1|option2}"),
+            OutputTuple(PromptPostProcessor.WILDCARD_WARNING + "__bad_wildcard__", "{option1|option2}"),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    process_wildcards=False,
+                    if_wildcards=IFWILDCARDS_CHOICES.warn,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_wc_stop(self):  # wildcards with stop option
+        self.process(
+            InputTuple("__bad_wildcard__", "{option1|option2}"),
+            OutputTuple(
+                PromptPostProcessor.WILDCARD_STOP.format("__bad_wildcard__") + "__bad_wildcard__",
+                "{option1|option2}",
+            ),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    process_wildcards=False,
+                    if_wildcards=IFWILDCARDS_CHOICES.stop,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+            interrupted=True,
+        )
+
+    def test_wcinvar_warn(self):  # wildcards in var with warn option
+        self.process(
+            InputTuple("${v=__bad_wildcard__}${v}", ""),
+            OutputTuple(PromptPostProcessor.WILDCARD_WARNING + "__bad_wildcard__", ""),
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    process_wildcards=False,
+                    if_wildcards=IFWILDCARDS_CHOICES.warn,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_wc_invalid_name(self):
+        self.process(
+            InputTuple("the choices are: ___invalid__", ""),
+            OutputTuple("the choices are: ___invalid__", ""),
+            ppp="nocup",
+            interrupted=True,
+        )
+
+    def test_wc_wildcard1a_text(self):  # simple text wildcard
+        self.process(
+            InputTuple("the choices are: __text/wildcard1__", ""),
+            OutputTuple("the choices are: choice2", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard1a_json(self):  # simple json wildcard
+        self.process(
+            InputTuple("the choices are: __json/wildcard1__", ""),
+            OutputTuple("the choices are: choice2", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard1a_yaml(self):  # simple yaml wildcard
+        self.process(
+            InputTuple("the choices are: __yaml/wildcard1__", ""),
+            OutputTuple("the choices are: choice2", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard1b_text(self):  # simple text wildcard with multiple choices
+        self.process(
+            InputTuple("the choices are: __2-$$text/wildcard1__", ""),
+            OutputTuple("the choices are: choice3, choice1", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard1b_json(self):  # simple json wildcard with multiple choices
+        self.process(
+            InputTuple("the choices are: __2-$$json/wildcard1__", ""),
+            OutputTuple("the choices are: choice3, choice1", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard1b_yaml(self):  # simple yaml wildcard with multiple choices
+        self.process(
+            InputTuple("the choices are: __2-$$yaml/wildcard1__", ""),
+            OutputTuple("the choices are: choice3, choice1", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard2_text(self):  # simple text wildcard with default options
+        self.process(
+            InputTuple("the choices are: __text/wildcard2__", ""),
+            OutputTuple("the choices are: choice3-choice1", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard2_json(self):  # simple json wildcard with default options
+        self.process(
+            InputTuple("the choices are: __json/wildcard2__", ""),
+            OutputTuple("the choices are: choice3-choice1", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard2_yaml(self):  # simple yaml wildcard with default options
+        self.process(
+            InputTuple("the choices are: __yaml/wildcard2__", ""),
+            OutputTuple("the choices are: choice3-choice1", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_test2_yaml(self):  # simple yaml wildcard
+        self.process(
+            InputTuple("the choice is: __testwc/test2__", ""),
+            OutputTuple("the choice is: 2", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_test3_yaml(self):  # simple yaml wildcard
+        self.process(
+            InputTuple("the choice is: __testwc/test3__", ""),
+            OutputTuple("the choice is: one choice", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_test3_yaml_2(self):  # simple yaml wildcard build from variable
+        self.process(
+            InputTuple("${v=3}the choice is: __testwc/test${v}__", ""),
+            OutputTuple("the choice is: one choice", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_index(self):  # wildcard with positional index filter
+        self.process(
+            InputTuple("the choice is: __yaml/wildcard2'2'__", ""),
+            OutputTuple("the choice is: choice3-choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_index_range(self):  # wildcard with positional index range filter
+        self.process(
+            InputTuple("the choice is: __yaml/wildcard2'2-3'__", ""),
+            OutputTuple("the choice is: choice3-choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_label(self):  # wildcard with label filter
+        self.process(
+            InputTuple("the choice is: __yaml/wildcard2'label1'__", ""),
+            OutputTuple("the choice is: choice3-choice1", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_label2(self):  # wildcard with label filter in multiple choices
+        self.process(
+            InputTuple("the choice is: __yaml/wildcard2'label2'__", ""),
+            OutputTuple("the choice is: choice1-choice1", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_label3(self):  # wildcard with multiple label filter
+        self.process(
+            InputTuple("the choice is: __4$$-$$yaml/wildcard2'label1,label2'__", ""),
+            OutputTuple("the choice is: choice1-choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_indexlabel(self):  # wildcard with mixed index and label filter
+        self.process(
+            InputTuple("the choice is: __yaml/wildcard2'2,label2'__", ""),
+            OutputTuple("the choice is: choice3-choice1", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_compound(self):  # wildcard with compound filter
+        self.process(
+            InputTuple("the choice is: __4$$-$$yaml/wildcard2'label1+label3'__", ""),
+            OutputTuple("the choice is: choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_compound_var(self):  # wildcard with compound filter in a variable
+        self.process(
+            InputTuple("${v[]=*('label1','label3')}the choice is: __4$$-$$yaml/wildcard2'${v[&'+']}'__", ""),
+            OutputTuple("the choice is: choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_compound2(self):  # wildcard with inherited compound filter
+        self.process(
+            InputTuple("the choice is: __yaml/wildcard2bis'#label1+label3'__", ""),
+            OutputTuple("the choice is: choice3bis", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_compound3(self):  # wildcard with doubly inherited compound filter
+        self.process(
+            InputTuple("the choice is: __yaml/wildcard2bisbis'#label1+label3'__", ""),
+            OutputTuple("the choice is: choice3bisbis", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_filter_compound4(self):  # wildcard with doubly inherited compound filter with variable
+        self.process(
+            InputTuple("${v=label1}the choice is: __yaml/wildcard2bisbis'#${v}+label3'__", ""),
+            OutputTuple("the choice is: choice3bisbis", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_default_filter(self):  # wildcard with default filter
+        self.process(
+            InputTuple(
+                "<ppp:setwcdeffilter 'yaml/wildcard2' 'label1+label3' />the choice is: __yaml/wildcard2__, <ppp:setwcdeffilter 'yaml/wildcard2' />__yaml/wildcard2__",
+                "",
+            ),
+            OutputTuple("the choice is: choice3-choice3, choice3-choice1- choice2 ", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_default_filter2(self):  # wildcard with default filter with variable
+        self.process(
+            InputTuple(
+                "${v=label1}<ppp:setwcdeffilter 'yaml/wildcard2' '${v}+label3' />the choice is: __yaml/wildcard2__, <ppp:setwcdeffilter 'yaml/wildcard2' />__yaml/wildcard2__",
+                "",
+            ),
+            OutputTuple("the choice is: choice3-choice3, choice3-choice1- choice2 ", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_nested_wildcard_text(self):  # nested text wildcard with repeating multiple choices
+        self.process(
+            InputTuple("the choices are: __r3$$-$$text/wildcard3__", ""),
+            OutputTuple("the choices are: choice3,choice1- choice2 ,choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_nested_wildcard_json(self):  # nested json wildcard with repeating multiple choices
+        self.process(
+            InputTuple("the choices are: __r3$$-$$json/wildcard3__", ""),
+            OutputTuple("the choices are: choice3,choice1- choice2 ,choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_nested_wildcard_yaml(self):  # nested yaml wildcard with repeating multiple choices
+        self.process(
+            InputTuple("the choices are: __r3$$-$$yaml/wildcard3__", ""),
+            OutputTuple("the choices are: choice3,choice1- choice2 ,choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_optional(self):  # empty wildcard with no error
+        self.process(
+            InputTuple("the choices are: __yaml/empty_wildcard__", ""),
+            OutputTuple("the choices are: ", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard4_yaml(self):  # simple yaml wildcard with one option
+        self.process(
+            InputTuple("the choices are: __yaml/wildcard4__", ""),
+            OutputTuple("the choices are: inline text", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard6_yaml(self):  # simple yaml wildcard with object formatted choices
+        self.process(
+            InputTuple("the choices are: __yaml/wildcard6__", ""),
+            OutputTuple("the choices are: choice2", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_choice_wildcard_mix(self):  # choices with wildcard mix
+        self.process(
+            InputTuple("the choices are: {__~2$$yaml/wildcard2__|choice0}", ""),
+            [
+                OutputTuple("the choices are: choice0", ""),
+                OutputTuple("the choices are: choice1, choice3", ""),
+                OutputTuple("the choices are: choice1, choice3", ""),
+            ],
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_globbing(self):  # wildcard with globbing
+        self.process(
+            InputTuple("the choices are: __yaml/*card[12]__, __yaml/wildcard?__", ""),
+            OutputTuple("the choices are: choice3-choice2, <lora:test2:1>- choice2 -choice3", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_globbing2(self):  # wildcard with globbing
+        self.process(
+            InputTuple("the choices are: __*/wildcard1__", ""),
+            OutputTuple("the choices are: choice2", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcardwithvar(self):  # wildcard with inline variable
+        self.process(
+            InputTuple("the choices are: __yaml/wildcard5(var=test)__, __yaml/wildcard5__", ""),
+            OutputTuple("the choices are: inline test, inline default", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcardPS_yaml(self):  # yaml wildcard with object formatted choices and options and prefix and suffix
+        self.process(
+            InputTuple("the choices are: __yaml/wildcardPS__", ""),
+            OutputTuple("the choices are: prefix1-choice2/choice3-suffix", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcardPS2_yaml(self):  # yaml wildcard with object formatted choices and options and prefix and suffix
+        self.process(
+            InputTuple("the choices are: [__yaml/wildcardPS2__]", ""),
+            OutputTuple("the choices are: (prefix2-choice2-suffix:1.35)", ""),
+        )
+
+    def test_wc_wildcardContainer_yaml(self):  # yaml wildcard with object formatted choices and options and container
+        self.process(
+            InputTuple("the choices are: [__yaml/wildcardContainer__]", ""),
+            OutputTuple("the choices are: (prefix1-choice2/choice3-suffix:1.35)", ""),
+        )
+
+    def test_wc_wildcardAt_yaml(self):  # yaml wildcard with attention in choices
+        self.process(
+            InputTuple("the choices are: [__yaml/wildcardAt__]", ""),
+            OutputTuple("the choices are: (choice2:1.35)", ""),
+        )
+
+    def test_wc_merge_attention_bracket(self):  # bracket attention from wildcard merges with outer attention
+        self.process(
+            InputTuple("(__yaml/wildcardAtBracket__:1.5)", ""),
+            OutputTuple("(the content:1.35)", ""),
+        )
+
+    def test_wc_no_merge_attention_alternation(self):  # alternation from wildcard is not merged as attention
+        self.process(
+            InputTuple("(__yaml/wildcardAlt__:1.5)", ""),
+            OutputTuple("([cat|dog]:1.5)", ""),
+        )
+
+    def test_wc_no_merge_attention_scheduling(self):  # scheduling from wildcard is not merged as attention
+        self.process(
+            InputTuple("(__yaml/wildcardSched__:1.5)", ""),
+            OutputTuple("([cat:dog:0.5]:1.5)", ""),
+        )
+
+    def test_wc_anonymouswildcard_yaml(self):  # yaml anonymous wildcard
+        self.process(
+            InputTuple("the choices are: __yaml/anonwildcards__", ""),
+            OutputTuple("the choices are: six", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_wildcard_input(self):  # simple yaml wildcard input
+        self.process(
+            InputTuple("the choices are: __yaml_input/wildcardI__", ""),
+            OutputTuple("the choices are: choice2", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_circular(self):  # wildcard circular reference
+        self.process(
+            InputTuple("the choices are: __yaml/circular1__", ""),
+            OutputTuple("", ""),
+            ppp="nocup",
+            interrupted=True,
+        )
+
+    def test_wc_including(self):  # wildcard including another wildcard
+        self.process(
+            InputTuple("the choices are: __yaml/including__", ""),
+            OutputTuple("the choices are: choice4", ""),
+            ppp="nocup",
+        )
+
+    def test_wc_circular_including(self):  # wildcard including another wildcard in a circular reference
+        self.process(
+            InputTuple("the choices are: __yaml/including1__", ""),
+            OutputTuple("", ""),
+            ppp="nocup",
+            interrupted=True,
+        )
+
+    def test_wc_dynamicwildcard(self):  # wildcard built from variables
+        self.process(
+            InputTuple(
+                "the choices are: ${x={1|2|3}}${w=yaml/wildcard${x}}__yaml/wildcard${x}__ __${w}__ __<ppp:echo w/>__",
+                "",
+            ),
+            OutputTuple("the choices are: choice1-choice3-choice1 choice3- choice2 - choice2  choice3", ""),
+            ppp="nocup",
+        )
+
+    # Combinatorial
+
+    def test_wc_combinatorial_1(self):  # combinatorial wildcard with variable
+        self.process(
+            InputTuple("the choices are: __2$$yaml/wildcard2__, ${v:{option1|option2}}", ""),
+            [  # 12 combinations
+                OutputTuple("the choices are: choice1, choice2, option1", "", {"v": "option1"}),
+                OutputTuple("the choices are: choice1, choice2, option2", "", {"v": "option2"}),
+                OutputTuple("the choices are: choice1, choice3, option1", "", {"v": "option1"}),
+                OutputTuple("the choices are: choice1, choice3, option2", "", {"v": "option2"}),
+                OutputTuple("the choices are: choice2, choice1, option1", "", {"v": "option1"}),
+                OutputTuple("the choices are: choice2, choice1, option2", "", {"v": "option2"}),
+                OutputTuple("the choices are: choice2, choice3, option1", "", {"v": "option1"}),
+                OutputTuple("the choices are: choice2, choice3, option2", "", {"v": "option2"}),
+                OutputTuple("the choices are: choice3, choice1, option1", "", {"v": "option1"}),
+                OutputTuple("the choices are: choice3, choice1, option2", "", {"v": "option2"}),
+                OutputTuple("the choices are: choice3, choice2, option1", "", {"v": "option1"}),
+                OutputTuple("the choices are: choice3, choice2, option2", "", {"v": "option2"}),
+            ],
+            combinatorial=True,
+        )
+
+    def test_wc_combinatorial_2(self):  # combinatorial wildcard
+        self.process(
+            InputTuple("__yaml/wildcard2__", ""),
+            [  # 36 combinations
+                # groups of 3
+                ## same choice repeated 3 times
+                OutputTuple("choice1-choice1-choice1", ""),
+                OutputTuple(" choice2 - choice2 - choice2 ", ""),
+                OutputTuple("choice3-choice3-choice3", ""),
+                ## one choice repeated 2 times in all positions
+                OutputTuple("choice1-choice1- choice2 ", ""),
+                OutputTuple("choice1-choice1-choice3", ""),
+                OutputTuple(" choice2 - choice2 -choice1", ""),
+                OutputTuple(" choice2 - choice2 -choice3", ""),
+                OutputTuple("choice3-choice3-choice1", ""),
+                OutputTuple("choice3-choice3- choice2 ", ""),
+                OutputTuple(" choice2 -choice1-choice1", ""),
+                OutputTuple("choice3-choice1-choice1", ""),
+                OutputTuple("choice1- choice2 - choice2 ", ""),
+                OutputTuple("choice3- choice2 - choice2 ", ""),
+                OutputTuple("choice1-choice3-choice3", ""),
+                OutputTuple(" choice2 -choice3-choice3", ""),
+                OutputTuple("choice1- choice2 -choice1", ""),
+                OutputTuple("choice1-choice3-choice1", ""),
+                OutputTuple(" choice2 -choice1- choice2 ", ""),
+                OutputTuple(" choice2 -choice3- choice2 ", ""),
+                OutputTuple("choice3-choice1-choice3", ""),
+                OutputTuple("choice3- choice2 -choice3", ""),
+                ## choices 1, 2, 3 in all positions
+                OutputTuple("choice1- choice2 -choice3", ""),
+                OutputTuple("choice1-choice3- choice2 ", ""),
+                OutputTuple(" choice2 -choice1-choice3", ""),
+                OutputTuple(" choice2 -choice3-choice1", ""),
+                OutputTuple("choice3-choice1- choice2 ", ""),
+                OutputTuple("choice3- choice2 -choice1", ""),
+                # groups of 2
+                ## same choice repeated 2 times
+                OutputTuple("choice1-choice1", ""),
+                OutputTuple(" choice2 - choice2 ", ""),
+                OutputTuple("choice3-choice3", ""),
+                ## choices 1 and 2 in all positions
+                OutputTuple("choice1- choice2 ", ""),
+                OutputTuple(" choice2 -choice1", ""),
+                ## choices 2 and 3 in all positions
+                OutputTuple(" choice2 -choice3", ""),
+                OutputTuple("choice3- choice2 ", ""),
+                ## choices 1 and 3 in all positions
+                OutputTuple("choice1-choice3", ""),
+                OutputTuple("choice3-choice1", ""),
+            ],
+            ppp="nocup",
+            combinatorial=True,
+        )
+
+    def test_wc_combinatorial_3(self):  # combinatorial wildcard (keep choice order)
+        self.process(
+            InputTuple("__2-3$$-$$yaml/wildcard2__", ""),
+            [  # 4 combinations
+                # groups of 3
+                ## choices 1, 2, 3
+                OutputTuple("choice1- choice2 -choice3", ""),
+                # groups of 2
+                ## choices 1 and 2
+                OutputTuple("choice1- choice2 ", ""),
+                ## choices 2 and 3
+                OutputTuple(" choice2 -choice3", ""),
+                ## choices 1 and 3
+                OutputTuple("choice1-choice3", ""),
+            ],
+            ppp=PromptPostProcessor(
+                self.ppp_logger,
+                self.def_env_info,
+                replace(
+                    self.defopts,
+                    keep_choices_order=True,
+                    cup_do_cleanup=False,
+                    do_combinatorial=True,
+                ),
+                self.grammar_content,
+                self.interrupt,
+                self.wildcards_obj,
+                self.extranetwork_maps_obj,
+            ),
+        )
+
+    def test_wc_combinatorial_4(self):  # combinatorial wildcard (don't keep choice order)
+        self.process(
+            InputTuple("__2-3$$-$$yaml/wildcard2__", ""),
+            [  # 12 combinations
+                # groups of 3
+                ## choices 1, 2, 3 in all positions
+                OutputTuple("choice1- choice2 -choice3", ""),
+                OutputTuple("choice1-choice3- choice2 ", ""),
+                OutputTuple(" choice2 -choice1-choice3", ""),
+                OutputTuple(" choice2 -choice3-choice1", ""),
+                OutputTuple("choice3-choice1- choice2 ", ""),
+                OutputTuple("choice3- choice2 -choice1", ""),
+                # groups of 2
+                ## choices 1 and 2 in all positions
+                OutputTuple("choice1- choice2 ", ""),
+                OutputTuple(" choice2 -choice1", ""),
+                ## choices 2 and 3 in all positions
+                OutputTuple(" choice2 -choice3", ""),
+                OutputTuple("choice3- choice2 ", ""),
+                ## choices 1 and 3 in all positions
+                OutputTuple("choice1-choice3", ""),
+                OutputTuple("choice3-choice1", ""),
+            ],
+            ppp="nocup",
+            combinatorial=True,
+        )
+
+    def test_wc_combinatorial_5(self):  # combinatorial nested wildcards and multiselection enmappings
+        self.process(
+            InputTuple("{__yaml/wildcard1__|__yaml/wildcard3__|<ppp:ext $lora loraany/>}", ""),
+            [  # 11 combinations
+                # first wildcard
+                OutputTuple("choice1", ""),
+                OutputTuple("choice2", ""),
+                OutputTuple("choice3", ""),
+                # second wildcard (nested)
+                OutputTuple("choice1, choice2 ", ""),
+                OutputTuple(" choice2 ,choice1", ""),
+                OutputTuple("choice1,choice3", ""),
+                OutputTuple("choice3,choice1", ""),
+                OutputTuple(" choice2 ,choice3", ""),
+                OutputTuple("choice3, choice2 ", ""),
+                # ppp:ext
+                OutputTuple("<lora:loraany1:0.8> trigger1, trigger2, ", ""),
+                OutputTuple("<lora:loraany2:1> trigger3, trigger4, ", ""),
+            ],
+            ppp="nocup",
+            combinatorial=True,
+        )

tests/wildcards/_invalid.txt

@@ -0,0 +1,3 @@
+# invalid wildcard name
+choice1
+choice2

tests/wildcards/test.json

@@ -0,0 +1,19 @@
+{
+  "json": {
+    "wildcard1": [
+      "choice1",
+      "choice2",
+      "choice3"
+    ],
+    "wildcard2": [
+      "r2-3$$-$$",
+      "4::choice1",
+      "3:: choice2 ",
+      "2::choice3",
+      "5 if _is_sd1::choice4"
+    ],
+    "wildcard3": [
+      "__2$$,$$json/wildcard2__"
+    ]
+  }
+}

tests/wildcards/test.yaml

@@ -0,0 +1,138 @@
+yaml:
+  wildcard1:
+    - "'First wildcard'$$"
+    - choice1
+    - choice2
+    - choice3
+
+  wildcard2:
+    - ~r2-3'Wildcard description'$$-$$
+    - "'label1,label2'4::choice1"
+    - "3:: choice2 "
+    - { labels: ["label1", "label3"], weight: 2, content: choice3 }
+    - 5 if _is_sd1::choice4
+
+  wildcard2bis:
+    - __1$$yaml/wildcard2'^yaml/wildcard2bis'__bis
+
+  wildcard2bisbis:
+    - __1$$yaml/wildcard2bis'#^yaml/wildcard2bisbis'__bis
+
+  wildcard3:
+    - __2$$,$$yaml/wildcard2__
+
+  wildcard4: inline text
+
+  wildcard5: inline ${var:default}
+
+  wildcard6:
+    - { weight: 2, text: choice1 }
+    - { weight: 3, content: choice2 }
+    - { text: choice3 }
+    - { weight: 4, if: _is_ssd, text: choice4 }
+
+  wildcard7:
+    - { description: "Lora wildcard" }
+    - <lora:test1:1>
+    - <lora:test2:1>
+    - <lora:test3:1>
+
+  wildcardAt:
+    - (choice1:0.5)
+    - (choice2:1.5)
+    - (choice3:1)
+
+  wildcardAtBracket:
+    - "[the content]"
+
+  wildcardAlt:
+    - "[cat|dog]"
+
+  wildcardSched:
+    - "[cat:dog:0.5]"
+
+  wildcardPS:
+    - {
+        sampler: "~",
+        repeating: false,
+        optional: false,
+        count: 2,
+        prefix: "prefix{1|2}-",
+        suffix: "-suffix",
+        separator: "/",
+      }
+    - { weight: 3, text: choice1 }
+    - { weight: 2, text: choice2 }
+    - { weight: 1, text: choice3 }
+
+  wildcardPS2:
+    - {
+        prefix: "(prefix{1|2}-",
+        suffix: "-suffix:1.5)",
+      }
+    - { text: choice1 }
+    - { text: choice2 }
+    - { text: choice3 }
+  
+  wildcardContainer:
+    - {
+        count: 2,
+        container: "(prefix{1|2}-${_choices[&'/']}-suffix:1.5)",
+      }
+    - { weight: 3, text: choice1 }
+    - { weight: 2, text: choice2 }
+    - { weight: 1, text: choice3 }
+
+  more_nested:
+    even_more_nested: # this would be __yaml/more_nested/even_more_nested__
+      - one
+      - two
+
+  anonwildcards:
+    - one
+    - two
+    - # choice without options and anonymous wildcard
+      - three
+      - four
+    - 3 if _is_sdxl: # choice with options and anonymous wildcard
+      - five
+      - six
+    - { weight: 1, text: [seven, eight] } # anonymous wildcard used in a choice in object format
+    - # choice without options and anonymous wildcard with parameters
+      - { count: 2, prefix: "#" }
+      - nine
+      - ten
+
+  empty_wildcard:
+    - o$$ # parameters: optional
+    - if false::1
+    - if false::2
+    - if false::3
+    - if _sd in ("test1", "test2")::4
+    - if (false or false)::5
+
+  circular1:
+    - 5::__yaml/circular2__
+    - choice1
+    - choice2
+
+  circular2:
+    - choice3
+    - choice4
+    - 5::__yaml/circular1__
+
+  including:
+    - "%0.5::include yaml/wildcard1"
+    - choice4
+    - choice5
+    - "%0.2::include yaml/wildcard7"
+
+  including1:
+    - choice1
+    - choice2
+    - "%5::include yaml/including2"
+
+  including2:
+    - choice3
+    - choice4
+    - "%5::include yaml/including1"

tests/wildcards/testwc/test2.yaml

@@ -0,0 +1,3 @@
+- one
+- 2
+- three

tests/wildcards/testwc/test3.yaml

@@ -0,0 +1 @@
+one choice

tests/wildcards2/text/wildcard1.txt

@@ -0,0 +1,4 @@
+# wildcard1
+choice1
+choice2
+choice3

tests/wildcards2/text/wildcard2.txt

@@ -0,0 +1,6 @@
+# wildcard2
+r2-3$$-$$
+4::choice1
+3:: choice2 
+2::choice3
+5 if _is_sd1::choice4

tests/wildcards2/text/wildcard3.txt

@@ -0,0 +1,3 @@
+# wildcard3 (with windows-1252 encoding)
+__2$$,$$text/wildcard2__
+0::áüç

tools/convert_styles.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+"""
+Converts style files (A1111 CSV or SD.Next JSON) to a PPP-compatible YAML wildcard file.
+
+Usage:
+    python convert_styles.py [--format {a1111,sdnext}] <input> <output>
+
+Arguments:
+    input   Path to the A1111 styles CSV file, a SD.Next JSON file, or a folder of SD.Next JSON files.
+    output  Path to the output YAML wildcard file.
+
+Options:
+    --format    Force the input format. If omitted, the format is inferred from the input:
+                - a1111  : input is a .csv file
+                - sdnext : input is a .json file or a directory
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+# Allow importing from the parent package when running directly from this folder.
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from ppp_common import convert_a1111_styles_to_wildcard, convert_sdnext_styles_to_wildcard
+
+
+def detect_format(inp: Path) -> str:
+    if inp.is_dir():
+        return "sdnext"
+    if inp.suffix.lower() == ".csv":
+        return "a1111"
+    if inp.suffix.lower() == ".json":
+        return "sdnext"
+    return ""
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert A1111 or SD.Next style files to a PPP YAML wildcard file.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument("input", type=Path, help="Input CSV file (A1111) or JSON file/folder (SD.Next).")
+    parser.add_argument("output", type=Path, help="Output YAML wildcard file.")
+    parser.add_argument(
+        "--format",
+        choices=["a1111", "sdnext"],
+        default=None,
+        help="Force input format. Auto-detected from the input path when not specified.",
+    )
+    args = parser.parse_args()
+
+    inp: Path = args.input
+    out: Path = args.output
+
+    if not inp.exists():
+        print(f"Error: input path does not exist: {inp}", file=sys.stderr)
+        sys.exit(1)
+
+    if out.suffix.lower() not in (".yaml", ".yml"):
+        print(f"Error: output path is not a YAML file: {out}", file=sys.stderr)
+        sys.exit(1)
+
+    fmt = args.format or detect_format(inp)
+    if not fmt:
+        print(
+            f"Error: could not detect format from '{inp}'. Use --format to specify it explicitly.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    out.parent.mkdir(parents=True, exist_ok=True)
+
+    try:
+        if fmt == "a1111":
+            convert_a1111_styles_to_wildcard(inp, out)
+        else:
+            convert_sdnext_styles_to_wildcard(inp, out)
+    except RuntimeError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Converted '{inp}' ({fmt}) -> '{out}'")
+
+
+if __name__ == "__main__":
+    main()

web/docs/ACBPPPCleanupOptions.md

@@ -0,0 +1,25 @@
+# ACB PPP Cleanup Options node
+
+Provides clean up options to the main PPP node.
+
+## Inputs
+
+* **extra_spaces**: Removes other unnecessary spaces.
+* **empty_constructs**: Removes attention/scheduling/alternation constructs when they are invalid.
+* **extra_separators**: Removes unnecessary separators. This applies to the configured separator and regular commas.
+* **extra_separators_additional**: Removes unnecessary separators at start or end of lines. This applies to the configured separator and regular commas.
+* **extra_separators_include_eol**: In the previous two options it also removes EOLs attached to the separators.
+* **around_breaks**: Removes consecutive BREAKs and unnecessary commas and space around them.
+* **breaks_with_eol**: Add a newline before BREAKs.
+* **around_ands**: Removes consecutive ANDs and unnecessary commas and space around them.
+* **ands_with_eol**: Add a newline before ANDs.
+* **around_extranetwork_tags**: Removes spaces around extra network tags.
+* **merge_attention**: It merges attention modifiers when possible (merges into one, multiplying their values). Only merges individually nested modifiers.
+* **remove_extranetwork_tags**: Removes all extra network tags.
+
+> [!NOTE]
+> *ComfyUI* does not natively support the `BREAK` and `AND` constructs, but the related settings are kept in that UI in case you use a node that supports them and the extension is configured to allow them (see the configuration file below).
+
+## Outputs
+
+* **options**: The options to send to the PPP node.

web/docs/ACBPPPENMappingOptions.md

@@ -0,0 +1,12 @@
+# ACB PPP ExtraNetwork Mapping Options node
+
+Provides extranetwork mapping options to the main PPP node.
+
+## Inputs
+
+* **folders**: You can enter multiple folders separated by commas. You can leave it empty (the default) and add a `ppp_extranetworkmappings` entry in the **extra_model_paths.yaml** file (recommended).
+* **definitions**: Extranetwork Mappings definitions (in yaml format). Direct input added to the ones found in the extranetwork mappings folders. Allows the mappings to be included in the workflow.
+
+## Outputs
+
+* **options**: The options to send to the PPP node.

web/docs/ACBPPPSelectVariable.md

@@ -0,0 +1,12 @@
+# ACB PPP Select Variable node
+
+Extracts a specific variable from PPP output variables.
+
+## Inputs
+
+* **variables**: Connect to the variables output from the main node.
+* **name**: Name of the variable to get, or empty to get all of them.
+
+## Outputs
+
+* **value**: The result value.

web/docs/ACBPPPSendToNegativeOptions.md

@@ -0,0 +1,12 @@
+# ACB PPP Send-To-Negative Options node
+
+Provides send-to-negative options to the main PPP node.
+
+## Inputs
+
+* **separator**: You can specify the separator used when adding to the negative prompt (by default it's ", ").
+* **ignore_repeats**: It ignores repeated content to avoid repetitions in the negative prompt.
+
+## Outputs
+
+* **options**: The options to send to the PPP node.

web/docs/ACBPPPWildcardConcat.md

@@ -0,0 +1,14 @@
+# ACB PPP Wildcard Concat node
+
+Concatenates up to 10 selected wildcards.
+
+## Inputs
+
+* **previous_prompt**: Previous prompt to add to.
+* **filter**: Filter text to reduce the available wildcard keys in the wildcard_N selectors.
+* **separator**: Separator string to join the selected wildcards.
+* **wildcard_1** to **wildcard_10**: Wildcard selectors.
+
+## Outputs
+
+* **prompt**: The resulting prompt.

web/docs/ACBPPPWildcardOptions.md

@@ -0,0 +1,19 @@
+# ACB PPP Wildcard Options node
+
+Provides wildcard options to the main PPP node.
+
+## Inputs
+
+* **folders**: You can enter multiple folders separated by commas. You can leave it empty (the default) and add a `ppp_wildcards` or `wildcards` entry in the **extra_model_paths.yaml** file (recommended).
+* **definitions**: Wildcards definitions (in yaml or json format). Direct input added to the ones found in the wildcards folders. Allows wildcards to be included in the workflow.
+* **if_wildcards**: Select what do you want to do with any found wildcards/choices (when process wildcards is off or after the processing).
+  * **Ignore**: do not try to detect wildcards.
+  * **Remove**: detect wildcards and remove them.
+  * **Add visible warning**: detect wildcards and add a warning text to the prompt, that hopefully produces a noticeable generation.
+  * **Stop the generation**: detect wildcards and stop the generation.
+* **choice_separator**: What do you want to use by default to separate multiple choices when the options allow it (by default it's ', ").
+* **keep_choices_order**: If checked, a multiple choice construct will return them in the order they are in the construct.
+
+## Outputs
+
+* **options**: The options to send to the PPP node.

web/docs/ACBPromptPostProcessor.md

@@ -0,0 +1,47 @@
+# ACB Prompt Post Processor node
+
+Main PPP node that processes prompts.
+
+## Inputs
+
+* **model**: Connect here the MODEL or a string with the model class name used by *ComfyUI*. Needed for the model kind system variables. Optional if you set modelname.
+* **modelname**: Filename of the model (with relative path). Needed for the detection of model variants.
+* **seed**: Set or connect here the seed used. By default it is -1 (random). The actual value used can be extracted from the output variables (`_input_seed`).
+* **pos_prompt**: Connect here the prompt text, or fill it as a widget.
+* **neg_prompt**: Connect here the negative prompt text, or fill it as a widget.
+* **debug_level**: What to write to the console.
+* **on_warnings**: Warn on the console or stop the generation.
+* **strict_mode**: Sets the strict mode in comparison operations.
+* **process_wildcards**: Activates the wildcard processing.
+* **do_cleanup**: Activates the cleanup processing.
+* **cleanup_variables**: Do a cleanup of the output variables (depends on do_cleanup).
+* **do_combinatorial**: Activates combinatorial mode, where the output are all the combinations of choices/wildcards of the prompt.
+* **combinatorial_shuffle**: It shuffles the combinatorial results.
+* **combinatorial_limit**: Limit for the number of generated combinations.
+* **wc_options**: Connection to a Wildcards options node.
+* **stn_options**: Connection to a Send-To-Negative options node.
+* **cup_options**: Connection to a Cleanup options node.
+* **en_options**: Connection to a ExtraNetworkMapping options node.
+* **results_file**: Filename to save processing results. Supports `%datetime%`, `%date%`, `%time%`, and `%host%` tokens. The file extension determines the format: `.yaml`/`.yml`, `.jsonl`, `.csv`, or plain text for any other extension. Relative paths are resolved against the extension's `logs` folder. Leave empty to disable.
+
+The options nodes are optional. If you don't need to change any of the default values then you don't need to use them.
+
+The model and modelname are also optional, but if you don't set them you will not be capable of choosing content based on the model type or variant. Native model loader nodes do not output the filename, but there are custom nodes that do (like those from [ComfyUI Image Saver](https://github.com/alexopus/ComfyUI-Image-Saver)).
+
+Setting only the modelname will try to detect its class from the file contents. If you don't want to set the path twice (and you don't use a loader node that outputs the name), you can set it here and then extract it from the output variables (`_modelfullname`) to send to the loader node.
+
+You can instead set it from the prompt (with variable `_modelfullname`) and load the model afterwards.
+
+## Outputs
+
+* **pos_prompt**: Resulting positive prompt.
+* **neg_prompt**: Resulting negative prompt.
+* **variables**: Resulting output variables.
+
+The outputs are lists, and in combinatorial mode there will be multiple elements that *ComfyUI* will process sequentially.
+
+## Notes
+
+You can use the "**ACB PPP Select Variable**" node to choose one and output its value. You can use this to send only part of the prompt to, for example, a detailer node. For example:
+
+With this prompt: `__quality__, 1girl, ${head:!__eyes__, __hair__, __expression__}, __body__, __clothes__, __background__, __style__` then you extract the `head` variable and use `${head}` as prompt for the head/face detailer.

web/ppp_wildcard_concat.js

@@ -0,0 +1,51 @@
+import { app } from "../../scripts/app.js";
+import { api } from "../../scripts/api.js";
+
+app.registerExtension({
+    name: "ACB.PPP.WildcardConcat",
+
+    async nodeCreated(node) {
+        if (node.comfyClass !== "ACBPPPWildcardConcat") return;
+
+        const refreshWildcards = async () => {
+            const filterWidget = node.widgets?.find((w) => w.name === "filter");
+            if (!filterWidget) return;
+
+            try {
+                const resp = await api.fetchApi(
+                    `/acb_ppp/wildcards?filter=${encodeURIComponent(filterWidget.value ?? "")}`
+                );
+                const data = await resp.json();
+                const wildcards = data.wildcards ?? [];
+
+                for (let i = 1; i <= 10; i++) {
+                    const widget = node.widgets?.find((w) => w.name === `wildcard_${i}`);
+                    if (widget) {
+                        const current = widget.value;
+                        widget.options.values = wildcards;
+                        widget.value = wildcards.includes(current)
+                            ? current
+                            : wildcards[0] ?? "(none)";
+                    }
+                }
+
+                app.graph.setDirtyCanvas(true, false);
+            } catch (err) {
+                console.error("[ACB PPP] Failed to refresh wildcard list:", err);
+            }
+        };
+
+        // Button to manually reload wildcards from disk
+        node.addWidget("button", "Refresh 🔄", null, refreshWildcards, { serialize: false });
+
+        // Auto-refresh all wildcard dropdowns when the filter value changes
+        const filterWidget = node.widgets?.find((w) => w.name === "filter");
+        if (filterWidget) {
+            const origCallback = filterWidget.callback;
+            filterWidget.callback = async function (...args) {
+                if (origCallback) origCallback.apply(this, args);
+                await refreshWildcards();
+            };
+        }
+    },
+});