Compare commits
@@ -0,0 +1,42 @@
|
||||
---
|
||||
description: "Use when writing or editing documentation, README files, or any markdown content."
|
||||
applyTo: "**/*.md"
|
||||
---
|
||||
|
||||
# Documentation Formatting
|
||||
|
||||
## Software and WebUI Names
|
||||
|
||||
All software names, WebUI names, and application names must be written in italic using `*name*` syntax — never plain text, bold, or code spans.
|
||||
|
||||
Examples of names that must be italic:
|
||||
- *ComfyUI*, *Stable Diffusion WebUI*, *Forge*, *reForge*, *SD.Next*, *InvokeAI*, *Fooocus*, *AUTOMATIC1111*
|
||||
- Any other application, platform, or tool name referenced in the docs
|
||||
|
||||
**Correct:**
|
||||
> This extension is compatible with *ComfyUI* and *Forge*.
|
||||
|
||||
**Incorrect:**
|
||||
> This extension is compatible with ComfyUI and Forge.
|
||||
> This extension is compatible with **ComfyUI** and `Forge`.
|
||||
|
||||
**Exceptions:**
|
||||
|
||||
- Names inside code blocks (` ``` ` or `` ` `` ) are exempt — they follow code formatting rules.
|
||||
- When a software name is the visible text of a hyperlink (e.g., `[ComfyUI](https://...)`), plain text is acceptable.
|
||||
|
||||
## Use of github alert blocks
|
||||
|
||||
When adding notes, tips, warnings, or important information in the documentation, use GitHub's alert block syntax for better visibility and formatting. The syntax is as follows:
|
||||
```md
|
||||
> [!NOTE]
|
||||
> This is a note.
|
||||
> [!TIP]
|
||||
> This is a tip.
|
||||
> [!IMPORTANT]
|
||||
> This is important information.
|
||||
> [!WARNING]
|
||||
> This is a warning.
|
||||
> [!CAUTION]
|
||||
> This is a caution.
|
||||
```
|
||||
+15
-5
@@ -1,12 +1,14 @@
|
||||
---
|
||||
description: "Use when writing, editing, or reviewing Python code. Enforces Python 3.10 compatibility — avoid syntax and stdlib features introduced in 3.11 or later."
|
||||
description: "Use when writing, editing, or reviewing Python code."
|
||||
applyTo: "**/*.py"
|
||||
---
|
||||
# Python 3.10 Compatibility
|
||||
# Python code
|
||||
|
||||
## Python 3.10 Compatibility
|
||||
|
||||
All Python code must be compatible with Python 3.10. Do not use language features or standard-library additions introduced in 3.11 or later.
|
||||
|
||||
## Forbidden (3.11+)
|
||||
### Forbidden (3.11+)
|
||||
|
||||
| Avoid | Use instead |
|
||||
|-------|-------------|
|
||||
@@ -18,7 +20,7 @@ All Python code must be compatible with Python 3.10. Do not use language feature
|
||||
| `except*` / `ExceptionGroup` | Not available; raise/catch normally |
|
||||
| `asyncio.TaskGroup`, `asyncio.timeout()` | `asyncio.gather()` / `asyncio.wait_for()` |
|
||||
|
||||
## Forbidden (3.12+)
|
||||
### Forbidden (3.12+)
|
||||
|
||||
| Avoid | Use instead |
|
||||
|-------|-------------|
|
||||
@@ -27,7 +29,7 @@ All Python code must be compatible with Python 3.10. Do not use language feature
|
||||
| `@typing.override` | Omit or use comment |
|
||||
| `itertools.batched()` | Manual chunking or `more-itertools` |
|
||||
|
||||
## Safe to use (available in 3.10)
|
||||
### Safe to use (available in 3.10)
|
||||
|
||||
- `match`/`case` structural pattern matching
|
||||
- `X | Y` union type syntax in annotations (e.g., `int | None`)
|
||||
@@ -35,3 +37,11 @@ All Python code must be compatible with Python 3.10. Do not use language feature
|
||||
- `list[int]`, `dict[str, int]` — built-in generic aliases
|
||||
- `zip(..., strict=True)`
|
||||
- `str.removeprefix()` / `str.removesuffix()`
|
||||
|
||||
## Paths
|
||||
|
||||
Use `pathlib.Path` for filesystem paths instead of `str` paths or `os.path`.
|
||||
|
||||
## Comments
|
||||
|
||||
Do not use emdashes (—) in comments. Use a single dash (-) or parentheses instead. Also avoid any other typographical punctuation that is not basic ASCII.
|
||||
@@ -65,7 +65,7 @@ def test_cl_combinatorial(self):
|
||||
{} # expected variables (optional)
|
||||
),
|
||||
],
|
||||
combinatorial=True,
|
||||
ppp=self.init_ppp(None, run_mode=RUN_MODE.combinatorial),
|
||||
)
|
||||
|
||||
```
|
||||
@@ -79,7 +79,10 @@ def test_cl_combinatorial(self):
|
||||
| `seed` | `int` | Optional, defaults to fixed seed |
|
||||
| `ppp` | `PromptPostProcessor \| str \| None` | Supported values `"nocup"`, `"nostrict"` or a specific instance |
|
||||
| `interrupted` | `bool` | Expected interrupt flag |
|
||||
| `combinatorial` | `bool` | Whether to run a combinatorial generation. If a specific ppp instance is used then it is ignored |
|
||||
| `specific_wc_folders` | `list[Path]` | Optional list of specific wildcard folders to use for this test |
|
||||
| `specific_em_folders` | `list[Path]` | Optional list of specific extranetwork folders to use for this test |
|
||||
| `input_vars` | `dict[str, Any]` | Optional dictionary of input variables to set before processing |
|
||||
|
||||
|
||||
## Assertions
|
||||
|
||||
@@ -94,23 +97,35 @@ Do not use bare `assert` statements.
|
||||
|
||||
## Default Options & Environment
|
||||
|
||||
Override `self.defopts` or `self.def_env_info` to pass non-default options — do not hardcode option dicts from scratch:
|
||||
Override `self.defopts` or `self.def_env_info` to pass non-default options.
|
||||
|
||||
```python
|
||||
def test_cl_custom(self):
|
||||
|
||||
def test_cl_custom1(self): # only option changes
|
||||
"""cleanup with custom separator"""
|
||||
self.process(
|
||||
InputTuple("a, , b", ""),
|
||||
OutputTuple("a | b", ""),
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
keep_choices_order=True,
|
||||
cup_do_cleanup=False,
|
||||
run_mode=RUN_MODE.combinatorial,
|
||||
),
|
||||
)
|
||||
|
||||
def test_cl_custom2(self): # only environment changes
|
||||
"""cleanup with custom separator"""
|
||||
self.process(
|
||||
InputTuple("a, , b", ""),
|
||||
OutputTuple("a | b", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
keep_choices_order=True,
|
||||
cup_do_cleanup=False,
|
||||
do_combinatorial=True,
|
||||
self.def_env_info,
|
||||
model_filename="./webui/models/Stable-diffusion/testmodel.safetensors",
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
|
||||
+2
-2
@@ -3,10 +3,10 @@
|
||||
venv
|
||||
|
||||
.vscode/**/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/launch.json
|
||||
|
||||
logs
|
||||
tests/tests_local.py
|
||||
tests/local_wildcards
|
||||
tests/logs
|
||||
tools/*.bat
|
||||
dev/*.bat
|
||||
|
||||
Vendored
-23
@@ -1,23 +0,0 @@
|
||||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Python Debugger: Attach using Process Id",
|
||||
"type": "debugpy",
|
||||
"request": "attach",
|
||||
"processId": "${command:pickProcess}"
|
||||
},
|
||||
{
|
||||
"name": "Python: Debug Tests (Testing Panel)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"purpose": ["debug-test"],
|
||||
"console": "integratedTerminal",
|
||||
"justMyCode": false,
|
||||
"cwd": "${workspaceFolder}"
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
-23
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"python.analysis.extraPaths": ["../.."],
|
||||
"python.testing.unittestArgs": [
|
||||
"-v",
|
||||
"-s",
|
||||
".",
|
||||
"-p",
|
||||
"test*.py"
|
||||
],
|
||||
"python.testing.pytestEnabled": false,
|
||||
"python.testing.unittestEnabled": true,
|
||||
"python.analysis.typeCheckingMode": "off",
|
||||
"black-formatter.args": [
|
||||
"--line-length=120"
|
||||
],
|
||||
"python-envs.pythonProjects": [
|
||||
{
|
||||
"path": ".",
|
||||
"envManager": "ms-python.python:venv",
|
||||
"packageManager": "ms-python.python:pip"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -3,6 +3,9 @@
|
||||
The Prompt PostProcessor (or PPP, formerly known as "sd-webui-sendtonegative"), is an extension designed to process the prompt in several ways. This extension is compatible with:
|
||||
|
||||
* [ComfyUI](https://github.com/comfyanonymous/ComfyUI)
|
||||
* [Forge Neo](https://github.com/Haoming02/sd-webui-forge-classic/tree/neo)
|
||||
* [SD.Next](https://github.com/vladmandic/sdnext)
|
||||
* [Forge Classic](https://github.com/Haoming02/sd-webui-forge-classic/tree/classic)
|
||||
* [Forge](https://github.com/lllyasviel/stable-diffusion-webui-forge)
|
||||
* [reForge](https://github.com/Panchovix/stable-diffusion-webui-reForge)
|
||||
* [AUTOMATIC1111 Stable Diffusion WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui)
|
||||
@@ -12,14 +15,19 @@ These are some features:
|
||||
|
||||
* Process wildcards. Compatible with the Dynamic Prompts extension.
|
||||
* Sending parts of the prompt to the negative prompt (the original function of the extension). This allows for useful tricks when using wildcards since you can add negative content from choices made in the positive prompt.
|
||||
* Set and modify local variables.
|
||||
* Define model variants (pony, illustrious, ...).
|
||||
* Filter content based on the loaded SD model/variant or a variable.
|
||||
* Map extranetworks (LoRAs) depending on conditions (like the loaded model variant). This allows you to add "virtual" LoRAs to the prompt that will be translated to the correct one.
|
||||
* Define model variants (pony, illustrious, ...) and have them set as system variables along with the current model, to detect which one is being used.
|
||||
* Set and modify user variables.
|
||||
* Set content based on conditions (like the system/user variables).
|
||||
* Map extranetworks (LoRAs) depending on conditions (like the loaded model/variant). This allows you to add "virtual" LoRAs to the prompt that will be translated to the correct one.
|
||||
* Clean up the prompt of unnecessary separators or spaces.
|
||||
* Combinatorial mode.
|
||||
|
||||
Note: when used in an *A1111* compatible webui, the extension must be loaded after any other extension that modifies the prompt (like another wildcards extension). Usually extensions load by their folder name in alphanumeric order, so if the extensions are not loading in the correct order just rename this extension's folder so the ordering works out. When in doubt, just rename this extension's folder with a "z" in front (for example) so that it is the last one to load, or manually set such folder name when installing it.
|
||||
> [!NOTE]
|
||||
> When used in an *A1111* compatible webui, the extension must be loaded after any other extension that modifies the prompt (like another wildcards extension).
|
||||
>
|
||||
> Usually extensions load by their folder name in alphanumeric order, so if the extensions are not loading in the correct order just rename this extension's folder so the ordering works out.
|
||||
>
|
||||
> When in doubt, just rename this extension's folder with a "z" in front (for example) so that it is the last one to load, or manually set such folder name when installing it.
|
||||
|
||||
If the extension runs before others, like Dynamic Prompts, and the "Process wildcards" option is enabled, the wildcards will be processed by PPP and those extensions will not get them. If you disable processing the wildcards, and you intend another extension to process them, you should keep the "What to do with remaining wildcards?" option as "ignore".
|
||||
|
||||
@@ -41,7 +49,6 @@ Notes:
|
||||
[LoRA Tag Loader for ComfyUI](https://github.com/badjeff/comfyui_lora_tag_loader)
|
||||
|
||||
In *SD.Next* that means only the *A1111* or *Full* parsers. It will warn you if you use the *Compel* parser.
|
||||
|
||||
Does not recognize tokenizer separators like `TE2:` and `TE3:`, so sending to negative prompt from those sections of the prompt will not add them in the corresponding section of the negative prompt.
|
||||
|
||||
2. It recognizes wildcards in the `__wildcard__` and {choice|choice} formats (and almost everything that [Dynamic Prompts](https://github.com/adieyal/sd-dynamic-prompts) supports).
|
||||
@@ -76,9 +83,20 @@ See the [syntax documentation](docs/SYNTAX.md).
|
||||
|
||||
See the [cookbook](docs/COOKBOOK.md) for interesting usages.
|
||||
|
||||
## Tools
|
||||
|
||||
### User tools
|
||||
|
||||
* `tools/convert_styles.py`: converts A1111 or SD.Next styles into a wildcards file.
|
||||
* `tools/check_loras.py`: checks for valid loras inside wildcards.
|
||||
|
||||
### Dev tools
|
||||
|
||||
* `dev/compare_models.py`: checks for new supported models in hosts.
|
||||
|
||||
## Contributing
|
||||
|
||||
To develop, I suggest doing so with the extension isolated from the UI (you can use a symlink to test it in the UI), and with its own virtual environment (venv or .venv), so the tests work and can be debugged properly.
|
||||
To develop, I suggest doing so with the extension isolated from the host UI (you can use a symlink to test it in the UI), and with its own virtual environment (venv or .venv), so the tests work and can be debugged properly.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+6
-2
@@ -6,12 +6,14 @@
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.append(str(Path(__file__).resolve().parent))
|
||||
|
||||
# pylint: disable=wrong-import-position
|
||||
from .ppp_comfyui import (
|
||||
PromptPostProcessorComfyUINode,
|
||||
PromptPostProcessorRunModeOptionsComfyUINode,
|
||||
PromptPostProcessorWildcardOptionsComfyUINode,
|
||||
PromptPostProcessorENMappingOptionsComfyUINode,
|
||||
PromptPostProcessorSTNOptionsComfyUINode,
|
||||
@@ -22,6 +24,7 @@ from .ppp_comfyui import (
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"ACBPromptPostProcessor": PromptPostProcessorComfyUINode,
|
||||
"ACBPPPRunModeOptions": PromptPostProcessorRunModeOptionsComfyUINode,
|
||||
"ACBPPPWildcardOptions": PromptPostProcessorWildcardOptionsComfyUINode,
|
||||
"ACBPPPENMappingOptions": PromptPostProcessorENMappingOptionsComfyUINode,
|
||||
"ACBPPPSendToNegativeOptions": PromptPostProcessorSTNOptionsComfyUINode,
|
||||
@@ -31,6 +34,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"ACBPromptPostProcessor": "ACB Prompt Post Processor",
|
||||
"ACBPPPRunModeOptions": "ACB PPP Run Mode Options",
|
||||
"ACBPPPWildcardOptions": "ACB PPP Wildcard Options",
|
||||
"ACBPPPENMappingOptions": "ACB PPP ExtraNetwork Mapping Options",
|
||||
"ACBPPPSendToNegativeOptions": "ACB PPP Send-To-Negative Options",
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
"""
|
||||
Compares the model classes listed in the host's supported models file against
|
||||
the model definitions in ppp_config.yaml.defaults for a given host.
|
||||
|
||||
The relative path to the supported models file for each host is read from the
|
||||
comments preceding the `models:` key in the defaults file.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
DEFAULT_CONFIG = Path(__file__).parent.parent / "ppp_config.yaml.defaults"
|
||||
SUPPORTED_HOSTS = {"comfyui", "reforge", "forge", "forgeneo", "sdnext"}
|
||||
|
||||
|
||||
def parse_host_file_paths(config_text: str) -> dict[str, str]:
|
||||
"""Extract the host->relative-path mapping from the comments above `models:`."""
|
||||
paths: dict[str, str] = {}
|
||||
in_block = False
|
||||
|
||||
for line in config_text.splitlines():
|
||||
stripped = line.strip()
|
||||
|
||||
if re.match(r"^#\s*Check supported models for each host in:", stripped):
|
||||
in_block = True
|
||||
continue
|
||||
|
||||
if in_block:
|
||||
# Lines like: "# host: some/relative/path.py" or "# host:"
|
||||
m = re.match(r"^#\s*(\w+)\s*:\s*(.*)", stripped)
|
||||
if m:
|
||||
host, rel_path = m.group(1), m.group(2).strip()
|
||||
if rel_path:
|
||||
paths[host] = rel_path
|
||||
else:
|
||||
# First non-matching line ends the block
|
||||
if not stripped.startswith("#"):
|
||||
break
|
||||
|
||||
return paths
|
||||
|
||||
|
||||
def extract_pipeline_classes(shared_items_path: Path) -> list[tuple[str, None]]:
|
||||
"""Parse shared_items.py and return unique diffusers pipeline class names from the pipelines dict."""
|
||||
source = shared_items_path.read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=str(shared_items_path))
|
||||
|
||||
class_names: set[str] = set()
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Assign):
|
||||
continue
|
||||
if not any(isinstance(t, ast.Name) and t.id == "pipelines" for t in node.targets):
|
||||
continue
|
||||
if not isinstance(node.value, ast.Dict):
|
||||
continue
|
||||
for value in node.value.values:
|
||||
# Match: getattr(diffusers, 'ClassName', None)
|
||||
if not isinstance(value, ast.Call):
|
||||
continue
|
||||
if not (isinstance(value.func, ast.Name) and value.func.id == "getattr"):
|
||||
continue
|
||||
if len(value.args) < 2:
|
||||
continue
|
||||
cls_arg = value.args[1]
|
||||
if isinstance(cls_arg, ast.Constant) and isinstance(cls_arg.value, str):
|
||||
class_names.add(cls_arg.value)
|
||||
|
||||
return [(name, None) for name in sorted(class_names)]
|
||||
|
||||
|
||||
def extract_model_classes(supported_models_path: Path) -> list[tuple[str, str | None]]:
|
||||
"""Parse a supported_models.py and return (class_name, parent_class) pairs in the `models` list(s)."""
|
||||
source = supported_models_path.read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=str(supported_models_path))
|
||||
|
||||
class_parents = _build_class_parents(tree)
|
||||
class_names: list[str] = []
|
||||
|
||||
for node in ast.walk(tree):
|
||||
# models = [ClassA, ClassB, ...]
|
||||
if isinstance(node, ast.Assign):
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Name) and target.id == "models":
|
||||
class_names.extend(_names_from_list(node.value))
|
||||
|
||||
# models += [ClassA, ...]
|
||||
elif isinstance(node, ast.AugAssign):
|
||||
if isinstance(node.target, ast.Name) and node.target.id == "models":
|
||||
class_names.extend(_names_from_list(node.value))
|
||||
|
||||
sentinels = _find_sentinels(class_parents, set(class_names))
|
||||
|
||||
# Only keep classes that ultimately descend from a sentinel base.
|
||||
# Display parent is None when the immediate parent is a sentinel (class appears as a root).
|
||||
return [
|
||||
(name, None if class_parents.get(name) in sentinels else class_parents.get(name))
|
||||
for name in class_names
|
||||
if _has_base_ancestor(name, class_parents, sentinels)
|
||||
]
|
||||
|
||||
|
||||
def _build_class_parents(tree: ast.AST) -> dict[str, str | None]:
|
||||
"""Return a mapping of class name -> raw parent name."""
|
||||
parents: dict[str, str | None] = {}
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.ClassDef):
|
||||
continue
|
||||
parent: str | None = None
|
||||
for base in node.bases:
|
||||
if isinstance(base, ast.Attribute) and isinstance(base.value, ast.Name):
|
||||
parent = f"{base.value.id}.{base.attr}"
|
||||
elif isinstance(base, ast.Name):
|
||||
parent = base.id
|
||||
break # only the first base matters
|
||||
parents[node.name] = parent
|
||||
return parents
|
||||
|
||||
|
||||
def _find_sentinels(parents: dict[str, str | None], model_class_names: set[str]) -> set[str]:
|
||||
"""Detect root base class names: external refs (dotted) or local classes with no parent not in the models list."""
|
||||
sentinels: set[str] = set()
|
||||
for parent in parents.values():
|
||||
if parent is None:
|
||||
continue
|
||||
if "." in parent: # external reference, e.g. supported_models_base.BASE
|
||||
sentinels.add(parent)
|
||||
elif parent in parents and parents[parent] is None and parent not in model_class_names:
|
||||
sentinels.add(parent) # local class with no parent that is not itself a listed model
|
||||
return sentinels
|
||||
|
||||
|
||||
def _has_base_ancestor(name: str, parents: dict[str, str | None], sentinels: set[str]) -> bool:
|
||||
visited: set[str] = set()
|
||||
current = parents.get(name)
|
||||
while current is not None:
|
||||
if current in sentinels:
|
||||
return True
|
||||
if current in visited:
|
||||
return False # cycle guard
|
||||
visited.add(current)
|
||||
current = parents.get(current)
|
||||
return False
|
||||
|
||||
|
||||
def _names_from_list(node: ast.expr) -> list[str]:
|
||||
if not isinstance(node, ast.List):
|
||||
return []
|
||||
return [elt.id for elt in node.elts if isinstance(elt, ast.Name)]
|
||||
|
||||
|
||||
def _topo_sort_alpha(classes: list[tuple[str, str | None]]) -> list[tuple[str, str | None]]:
|
||||
"""Sort classes so each parent immediately precedes its children, with alphabetical ordering at every level."""
|
||||
class_set = {name for name, _ in classes}
|
||||
parent_of = {name: parent for name, parent in classes}
|
||||
|
||||
children_of: dict[str, list[str]] = {name: [] for name, _ in classes}
|
||||
roots: list[str] = []
|
||||
for name, parent in classes:
|
||||
if parent and parent in class_set:
|
||||
children_of[parent].append(name)
|
||||
else:
|
||||
roots.append(name)
|
||||
|
||||
roots.sort()
|
||||
for children in children_of.values():
|
||||
children.sort()
|
||||
|
||||
result: list[tuple[str, str | None]] = []
|
||||
|
||||
def visit(name: str) -> None:
|
||||
result.append((name, parent_of[name]))
|
||||
for child in children_of[name]:
|
||||
visit(child)
|
||||
|
||||
for root in roots:
|
||||
visit(root)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def build_class_to_model_map(config: dict, host: str) -> dict[str, str]:
|
||||
"""Return a mapping of class name -> model key for the given host."""
|
||||
mapping: dict[str, str] = {}
|
||||
models_config = config.get("models", {})
|
||||
|
||||
for model_key, model_data in models_config.items():
|
||||
if not isinstance(model_data, dict):
|
||||
continue
|
||||
detect = model_data.get("detect", {})
|
||||
if not isinstance(detect, dict):
|
||||
continue
|
||||
host_detect = detect.get(host)
|
||||
if not isinstance(host_detect, dict):
|
||||
continue
|
||||
classes = host_detect.get("class", [])
|
||||
if isinstance(classes, list):
|
||||
for cls in classes:
|
||||
mapping[cls] = model_key
|
||||
|
||||
return mapping
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare host model classes against ppp_config.yaml.defaults mappings."
|
||||
)
|
||||
parser.add_argument(
|
||||
"host",
|
||||
help="Host kind to compare against (e.g. comfyui, reforge, forge, ...).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"root",
|
||||
metavar="ROOT_FOLDER",
|
||||
type=Path,
|
||||
help="Root folder of the host UI installation.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
metavar="CONFIG_FILE",
|
||||
type=Path,
|
||||
default=DEFAULT_CONFIG,
|
||||
help=f"Path to ppp_config.yaml.defaults (default: {DEFAULT_CONFIG}).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
root: Path = args.root
|
||||
host: str = args.host
|
||||
config_path: Path = args.config
|
||||
|
||||
if host not in SUPPORTED_HOSTS:
|
||||
parser.error(f"Host '{host}' is not supported by this script. Supported hosts: {', '.join(sorted(SUPPORTED_HOSTS))}")
|
||||
|
||||
if not root.is_dir():
|
||||
parser.error(f"ROOT_FOLDER does not exist or is not a directory: {root}")
|
||||
|
||||
if not config_path.exists():
|
||||
parser.error(f"Config file not found: {config_path}")
|
||||
|
||||
config_text = config_path.read_text(encoding="utf-8")
|
||||
host_paths = parse_host_file_paths(config_text)
|
||||
|
||||
known_hosts = list((yaml.safe_load(config_text).get("hosts") or {}).keys())
|
||||
if host not in known_hosts:
|
||||
print(f"Warning: '{host}' is not a recognized host. Known hosts: {', '.join(known_hosts)}")
|
||||
|
||||
rel_path = host_paths.get(host)
|
||||
if not rel_path:
|
||||
print(f"Error: no supported-models file path defined for host '{host}' in {config_path.name}.")
|
||||
sys.exit(1)
|
||||
|
||||
supported_models_path = root / rel_path.replace("\\", "/")
|
||||
if not supported_models_path.exists():
|
||||
print(f"Error: file not found: {supported_models_path}")
|
||||
sys.exit(1)
|
||||
|
||||
config = yaml.safe_load(config_text)
|
||||
if host == "sdnext":
|
||||
model_classes: list[tuple[str, str | None]] = extract_pipeline_classes(supported_models_path)
|
||||
else:
|
||||
model_classes = extract_model_classes(supported_models_path)
|
||||
if not model_classes:
|
||||
print("No model classes found in the models list.")
|
||||
sys.exit(1)
|
||||
|
||||
model_classes = _topo_sort_alpha(model_classes)
|
||||
|
||||
class_to_model = build_class_to_model_map(config, host)
|
||||
|
||||
class_set = {cls for cls, _ in model_classes}
|
||||
parent_of_display = {cls: parent for cls, parent in model_classes}
|
||||
depth_cache: dict[str, int] = {}
|
||||
|
||||
def get_depth(name: str) -> int:
|
||||
if name not in depth_cache:
|
||||
p = parent_of_display.get(name)
|
||||
depth_cache[name] = 0 if (not p or p not in class_set) else 1 + get_depth(p)
|
||||
return depth_cache[name]
|
||||
|
||||
# Build display labels: "ClassName" or "ClassName (Parent)", indented by depth
|
||||
labels = [f"{cls} ({parent})" if parent else cls for cls, parent in model_classes]
|
||||
depths = [get_depth(cls) for cls, _ in model_classes]
|
||||
col_width = max(d * 2 + len(lbl) for d, lbl in zip(depths, labels))
|
||||
missing: list[str] = []
|
||||
|
||||
print(f"Model classes in '{supported_models_path}' vs '{config_path.name}' (host: {host})")
|
||||
print("-" * (col_width + 42))
|
||||
|
||||
for (cls, _parent), label, depth in zip(model_classes, labels, depths):
|
||||
indented = " " * depth + label
|
||||
model = class_to_model.get(cls)
|
||||
if model:
|
||||
print(f"{indented:<{col_width}} -> {model}")
|
||||
else:
|
||||
missing.append((cls, _parent))
|
||||
print(f"{indented:<{col_width}} -> WARNING: not mapped")
|
||||
|
||||
print("-" * (col_width + 42))
|
||||
print(f"Total: {len(model_classes)} classes, {len(missing)} unmapped")
|
||||
|
||||
if missing:
|
||||
print("\nUnmapped classes:")
|
||||
for cls, parent in missing:
|
||||
label = f"{cls} ({parent})" if parent else cls
|
||||
print(f" - {label}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+69
-18
@@ -4,15 +4,15 @@
|
||||
|
||||
The extension supports a configuration file `ppp_config.yaml` with some settings that don't usually change.
|
||||
|
||||
By default this configuration is read from the file `ppp_config.yaml.defaults` in the extension folder. That file must not be modified. If you want to personalize any settings you should first copy it as `ppp_config.yaml` in the same folder, or in the ComfyUI user folder (preferred, but only possible in ComfyUI). The options in this new file will take precedence over those in the defaults file.
|
||||
By default this configuration is read from the file `ppp_config.yaml.defaults` in the extension folder. That file must not be modified. If you want to personalize any settings you should first copy it as `ppp_config.yaml` in the same folder, or in the *ComfyUI* user folder (preferred, but only possible in *ComfyUI*). The options in this new file will take precedence over those in the defaults file.
|
||||
|
||||
This file contains some options for how the host applications (WebUIs) should act in certain operations, and also define the supported models, including how to detect them and the model variants definitions. Host names are fixed values (those supported by the extension). The defaults file contains comments to explain the available options.
|
||||
This file contains some options for how the host applications (WebUIs) should act in certain operations, and also defines the supported models, including how to detect them and the model variants definitions. Host names are fixed values (those supported by the extension). The defaults file contains comments to explain the available options.
|
||||
|
||||
The model variants now support regular expressions instead of a list of strings to detect the variant. If you used a non default value in previous versions you should create a configuration file and add them with the new format. As before, the default file defines variants for *Pony* and *Illustrious* models.
|
||||
|
||||
## Important
|
||||
|
||||
**Beware of the combinatorial mode with no limits**. Even very few choice/wildcard constructs can cause a *combinatorial explosion*!
|
||||
**Beware of the combinatorial mode with no limits**. Even very few choice/wildcard constructs can cause a *combinatorial explosion*! In ComfyUI there is no other limit, but in the other hosts this is also limited by the batch count/size.
|
||||
|
||||
The console log can help you determine the number of combinations that it is trying to generate. There will be an **"Estimated combinations"** message that shows an estimate. You can try first with a limit of 1, then check this message in the log. But note that it is a lower bound estimate, and there could be more combinations.
|
||||
|
||||
@@ -24,9 +24,9 @@ The main node that processes the prompt.
|
||||
|
||||
Inputs:
|
||||
|
||||
* **model**: Connect here the MODEL or a string with the model class name used by *ComfyUI*. Needed for the model kind system variables.
|
||||
* **modelname**: Name of the model. Needed for the detection of model variants.
|
||||
* **seed**: Connect here the seed used. By default it is -1 (random).
|
||||
* **model**: Connect here the MODEL or a string with the model class name used by *ComfyUI*. Needed for the model kind system variables. Optional if you set modelname.
|
||||
* **modelname**: Filename of the model (with relative path). Needed for the detection of model variants.
|
||||
* **seed**: Set or connect here the seed used. By default it is -1 (random). The actual value used can be extracted from the output variables (`_input_seed`).
|
||||
* **pos_prompt**: Connect here the prompt text, or fill it as a widget.
|
||||
* **neg_prompt**: Connect here the negative prompt text, or fill it as a widget.
|
||||
* **debug_level**: What to write to the console.
|
||||
@@ -35,29 +35,51 @@ Inputs:
|
||||
* **process_wildcards**: Activates the wildcard processing.
|
||||
* **do_cleanup**: Activates the cleanup processing.
|
||||
* **cleanup_variables**: Do a cleanup of the output variables (depends on do_cleanup).
|
||||
* **do_combinatorial**: Activates combinatorial mode, where the output are all the combinations of choices/wildcards of the prompt.
|
||||
* **combinatorial_shuffle**: It shuffles the combinatorial results.
|
||||
* **combinatorial_limit**: Limit for the number of generated combinations.
|
||||
* **results_file**: Filename to save processing results. Supports `%datetime%`, `%date%`, `%time%`, and `%host%` tokens. The file extension determines the format: `.yaml`/`.yml`, `.jsonl`, `.csv`, or plain text for any other extension. Relative paths are resolved against the extension's `logs` folder. Leave empty to disable.
|
||||
* **run_mode**: Sets how the process works. `single` or `multiple` for regular one or more results, or `combinatorial` for combinatorial mode, where the output are all the combinations of choices/wildcards of the prompt.
|
||||
* **wc_options**: Connection to a Wildcards options node.
|
||||
* **stn_options**: Connection to a Send-To-Negative options node.
|
||||
* **cup_options**: Connection to a Cleanup options node.
|
||||
* **en_options**: Connection to a ExtraNetworkMapping options node.
|
||||
* **rm_options**: Connection to a Run Mode options node.
|
||||
|
||||
The options nodes are optional. If you don't need to change any of the default values then you don't need to use them.
|
||||
|
||||
The model and modelname are also optional, but if you don't set them you will not be capable of choosing content based on the model type or variant. Native model loader nodes do not output the filename, but there are custom nodes that do (like those from [ComfyUI Image Saver](https://github.com/alexopus/ComfyUI-Image-Saver)).
|
||||
|
||||
Setting only the modelname will try to detect its class from the file contents. If you don't want to set the path twice (and you don't use a loader node that outputs the name), you can set it here and then extract it from the output variables (`_modelfullname`) to send to the loader node.
|
||||
|
||||
You can instead set it from the prompt (with variable `_modelfullname`) and load the model afterwards.
|
||||
|
||||
Outputs:
|
||||
|
||||
* **pos_prompt**: the resulting positive prompt
|
||||
* **neg_prompt**: the resulting negative prompt
|
||||
* **variables**: the dictionary of variables set or echoed.
|
||||
* **pos_prompt**: Resulting positive prompt.
|
||||
* **neg_prompt**: Resulting negative prompt.
|
||||
* **variables**: Resulting output variables.
|
||||
|
||||
The outputs are lists, and in combinatorial mode there will be multiple elements that ComfyUI will process sequentially.
|
||||
The outputs are lists, and in combinatorial/multiple modes there will be multiple elements that *ComfyUI* will process sequentially.
|
||||
|
||||
The run_mode can be explained like this:
|
||||
|
||||
* **single**: only one result is returned.
|
||||
* **multiple**: multiple results are returned, the count is in results_limit.
|
||||
* **combinatorial**: all the combinations are returned, up to results_limit.
|
||||
|
||||
In `single` and `multiple` modes the default choice sampler is the one set in `default_sampler`. In `combinatorial` mode the default sampling is equivalent to `cyclical`. In all modes specified samplers are respected. The value of random samplers in `combinatorial` mode depends on `comb_random_fixed`.
|
||||
|
||||
`Multiple` mode with a default of `cyclical` sampler is very similar to `combinatorial`. The only difference is that `comb_ramdom_fixed` does not apply and random samplers are thus not cached.
|
||||
|
||||
Single mode with a default of cyclical sampler can be used as similar to combinatorial but in separated *ComfyUI* runs instead of one.
|
||||
|
||||
### ACB PPP Select Variable node
|
||||
|
||||
Lets you extract the variables used from the output (or just one of them). You can use this to send only part of the prompt to, for example, a detailer node. For example:
|
||||
|
||||
With this prompt: `__quality__, 1girl, ${head:__eyes__, __hair__, __expression__}, __body__, __clothes__, __background__, __style__` then you extract the `head` variable and use it as prompt for the head/face detailer.
|
||||
With this prompt:
|
||||
|
||||
`__quality__, 1girl, ${head:__eyes__, __hair__, __expression__}, __body__, __clothes__, __background__, __style__`
|
||||
|
||||
You can then extract the `head` variable and use it as prompt for the head/face detailer.
|
||||
|
||||
Inputs:
|
||||
|
||||
@@ -70,7 +92,7 @@ Output:
|
||||
|
||||
### ACB PPP Wildcards Concat node
|
||||
|
||||
This node lets you select up to 10 wildcards that will be concatenated with a chosen separator. You can't specify wildcard folders in the node, so use the other available options to set them.
|
||||
This node lets you select up to 10 wildcards that will be concatenated with a chosen separator. You can't specify wildcard folders in the node, so use the **extra_model_paths.yaml** options (see node below) to set them.
|
||||
|
||||
Inputs:
|
||||
|
||||
@@ -83,6 +105,20 @@ Output:
|
||||
|
||||
* **prompt**: concatenated result.
|
||||
|
||||
### ACB PPP Run Mode Options node
|
||||
|
||||
Options for the run mode, in case you want to change them from the defaults.
|
||||
|
||||
* **results_limit**: Limit for the number of generated results (except in `single` mode). Important for combinatorial mode.
|
||||
* **results_shuffle**: It shuffles the results.
|
||||
* **comb_random_fixed**: If True all specified random samplers will have a fixed value across the combinations.
|
||||
* **default_sampler**: The default choice sampler when not specified (in non combinatorial mode). Also applies to extranetwork mapping selection.
|
||||
* **next_seed**: Choose what to do with the seed in the following prompts in `multiple` or `combinatorial` mode. Value can be:
|
||||
* `randomize`: Next prompts will have a random seed (default).
|
||||
* `input`: Next prompts will have the same seed as the input (or first prompt).
|
||||
* `increment`: The seeds in next prompts will increment.
|
||||
* `decrement`: The seeds in next prompts will decrement.
|
||||
|
||||
### ACB PPP Wildcard Options node
|
||||
|
||||
Options for wildcard processing, in case you want to change them from the defaults.
|
||||
@@ -138,9 +174,23 @@ Options for extranetworks mapping, in case you want to change them from the defa
|
||||
* **Unlink seed**: Uses the specified seed for the prompt generation instead of the one from the image. This seed is only used for wildcards and choices.
|
||||
* **Prompt seed**: The seed to use for the prompt generation. If -1 a random one will be used.
|
||||
* **Incremental seed**: When using a batch you can use this to set the rest of the prompt seeds with consecutive values.
|
||||
* **Combinatorial mode**: Generate all possible prompt combinations (from choices and wildcards) and cycle through them to fill the batch.
|
||||
* **Shuffle combinations**: It shuffles the combinatorial results.
|
||||
* **Combinations limit**: Maximum number of combinations to generate (0 = no limit). The actual maximum limit is the number of images (batch size * count).
|
||||
* **Run mode**: Sets how the process works. `single` or `multiple` for regular one or more results, or `combinatorial` for combinatorial mode, where the output are all the combinations of choices/wildcards of the prompt.
|
||||
* **Results limit**: Maximum number of combinations to generate (0 = no limit). The actual maximum limit is the number of images (batch size * count).
|
||||
* **Shuffle results**: It shuffles the results.
|
||||
* **Fix random sampler across combinations**: If checked all specified random samplers will have a fixed value across the combinations.
|
||||
* **Default sampler**: The default choice sampler when not specified.
|
||||
|
||||
The `Run mode` can be explained like this:
|
||||
|
||||
* **single**: only one result is returned (for each image).
|
||||
* **multiple**: multiple results are returned, the count is in `Results limit` (but limited by the number of images). The results are assigned to the images. If there are more images than results, the results wrap.
|
||||
* **combinatorial**: all the combinations are returned, up to `Results limit` (but limited by the number of images). The results are assigned to the images. If there are more images than results, the results wrap.
|
||||
|
||||
In single and multiple modes the default choice sampler is the one set in `Default sampler`. In combinatorial mode the default sampling is equivalent to cyclical. In all modes specified samplers are respected. The value of random samplers in combinatorial mode depends on `Fix random sampler across combinations`.
|
||||
|
||||
Multiple mode with a default of cyclical sampler is very similar to combinatorial. The only difference is that `Fix random sampler across combinations` does not apply and random samplers are thus not cached. There is no reason to use this mode with a random default sampler, since it would be the same as in single mode.
|
||||
|
||||
Single mode with a default of cyclical sampler can be used as similar to combinatorial but in separated runs instead of one.
|
||||
|
||||
### General settings
|
||||
|
||||
@@ -149,6 +199,7 @@ Options for extranetworks mapping, in case you want to change them from the defa
|
||||
* **Use strict operators**: Sets strict operations in comparisons.
|
||||
* **Apply in img2img**: Check if you want to do the processing in img2img processes.
|
||||
* **Add original prompts to metadata**: Adds original prompts to the metadata if they have changed.
|
||||
* **Results file**: Filename to save processing results. Supports `%datetime%`, `%date%`, `%time%`, and `%host%` tokens. The file extension determines the format: `.yaml`/`.yml`, `.jsonl`, `.csv`, or plain text for any other extension. Relative paths are resolved against the extension's `logs` folder. Leave empty to disable.
|
||||
* **Extranetwork Mappings folders**: You can enter multiple folders separated by commas.
|
||||
|
||||
### Wildcard settings
|
||||
|
||||
+103
-3
@@ -85,7 +85,9 @@ Then use `__colors_warm__` or `__colors_cold__`, or `__colors'warm'__` or `__col
|
||||
|
||||
## Prompt building
|
||||
|
||||
You can create a complex set of wildcards that build a full prompt, by using the wildcards along with variables with default values. The variables allow you to choose what parts of the prompt you want to change for specific content, or just leave the defaults. Then, inside the UI, you can use styles (with an appropiate styles node if using ComfyUI) to choose which variables to set. This makes it easy to quickly select what do you want to prompt for.
|
||||
You can create a complex set of wildcards that build a full prompt, by using the wildcards along with variables with default values. The variables allow you to choose what parts of the prompt you want to change for specific content, or just leave the defaults. Then, inside the UI, you can use styles (with an appropiate A1111 styles node if using *ComfyUI*) to choose which variables to set. This makes it easy to quickly select what do you want to prompt for.
|
||||
|
||||
If you use *ComfyUI* and don't have a styles node, you can put them in individual wildcards and use the included `Wildcards Concat` node.
|
||||
|
||||
Wildcards:
|
||||
|
||||
@@ -166,7 +168,10 @@ name,prompt,negative_prompt
|
||||
"Character","__character__",""
|
||||
```
|
||||
|
||||
You first select the styles that set the variables that you are interested in changing from the default, and you end with the main wildcard that uses them to build the prompt. Note that the order of the variables don't usually matter because they are only evaluated when echoed to the prompt. This allows you to use variables inside wildcards that are inside other variables, and only at the end they will be evaluated.
|
||||
You first select the styles that set the variables that you are interested in changing from the default, and you end with the main wildcard that uses them to build the prompt.
|
||||
|
||||
> [!TIP]
|
||||
> The order of the variables don't usually matter because they are only evaluated when echoed to the prompt. This allows you to use variables inside wildcards that are inside other variables, and only at the end they will be evaluated.
|
||||
|
||||
Sample style selections:
|
||||
|
||||
@@ -196,7 +201,7 @@ This would create this prompt:
|
||||
|
||||
## Using variables for detailer prompts
|
||||
|
||||
This only applies to ComfyUI, because the variables cannot be used in the ADetailer prompts of the A1111 UIs.
|
||||
This only applies to *ComfyUI*, because the variables cannot be used in the ADetailer prompts of the A1111 UIs.
|
||||
|
||||
You can put parts of the prompt to be set into variables, and then extract this variables from the output and use them in the prompts of the detailer nodes.
|
||||
|
||||
@@ -477,6 +482,66 @@ You can also use `evaluate ifundefined` (`?=!`) to resolve the wildcard immediat
|
||||
${quality?=!__qualities__}
|
||||
```
|
||||
|
||||
## Informing the model to the node in *ComfyUI*
|
||||
|
||||
In older versions, to correctly set the `_is_*` variables it was needed to set the `model` input (to get the class and kind of model) and the `modelname` input (to calculate variants). Now, the class and kind can be inferred from the file, so only the `modelname` is needed.
|
||||
|
||||
You can still load the model before the PPP node, and you connect it to the `model` input, but you will still need to fill the `modelname` input to detect variants (and if you don't have a loader node that also outputs the filename it is a repetition).
|
||||
|
||||
But a better way is loading the model after the PPP node, setting the filename in `modelname` (which is now a combo) and then extracting `_modelfullname` from the output variables and connecting that to the model loader node filename input. This also let's you use the next tip.
|
||||
|
||||
## Choosing the model from the prompt in *ComfyUI*
|
||||
|
||||
You can leave the model and modelname inputs disconnected/empty and set the `_modelfullname` variable at the start of the prompt (maybe through a wildcard or a choice construct). Then extract it from the output variables using the `Select Variable` node and use its value to actually load that model.
|
||||
|
||||
You can also set and extract user variables for other ksampler inputs, like sampler, scheduler, steps, cfg and latent size.
|
||||
|
||||
## Seed behavior
|
||||
|
||||
The seed determines which choices are picked for wildcards/choices with `~` (random) samplers and other random selections.
|
||||
|
||||
This seed can be different than the one used for the image.
|
||||
|
||||
Each host handles it differently.
|
||||
|
||||
### A1111 and derivatives
|
||||
|
||||
By default, PPP uses the same seed for image and prompt, which A1111 auto-increments across the batch. Each image therefore gets independently seeded wildcard expansions.
|
||||
|
||||
The seeds are pre-calculated before the prompt postprocess begins.
|
||||
|
||||
The extension provides options to change this:
|
||||
|
||||
- **Force equal seeds**: sets every seed in the batch to the first one before processing. All images get the same expansion.
|
||||
- **Unlink seed**: separates the prompt seed from the image seed. The table below shows how it behaves depending on the seed value and the *Incremental seed* toggle:
|
||||
|
||||
| Seed value | Incremental | Prompt seed per image |
|
||||
|------------|-------------|------------------------------------------|
|
||||
| -1 | Yes | Random base seed, then base+1, base+2, … |
|
||||
| -1 | No | Independent random seed per image |
|
||||
| N | Yes | N, N+1, N+2, … |
|
||||
| N | No | N for every image |
|
||||
|
||||
If a subseed strength is set, the effective seed is `subseed × strength + seed × (1 − strength)` per image.
|
||||
|
||||
In **multiple** or **combinatorial** run mode, results are generated using their corresponding seed in the batch, both for prompts and images. The `next_seed` option is set to `input` with these hosts (and a list of them is provided) because we need the image seeds pre-calculated.
|
||||
|
||||
### ComfyUI
|
||||
|
||||
The seed is an explicit node input (default: -1 for random, which gets an actual value as soon as possible and is considered the starting seed). In multiple result modes (`multiple` and `combinatorial`) the `next_seed` option is used to calculate the rest of the seeds.
|
||||
|
||||
Each result seed is added as the output variable `_output_seed`, so it can be extracted and used as the image seed.
|
||||
|
||||
## When the cyclical sampler resets
|
||||
|
||||
The `@` (cyclical) sampler tracks its position so each call advances through combinations in order.
|
||||
|
||||
The state is retained across executions within the same session. Each execution with the same prompts advances the position. The cycle only resets when either the positive or negative prompt text changes.
|
||||
|
||||
## Keeping references up to date
|
||||
|
||||
After updating your loras (deleting old ones, updating to new versions) run the `tools\check_loras.py` script to check if there are broken references in your wildcards or extranetwork mappings.
|
||||
|
||||
## Debugging tips
|
||||
|
||||
When something isn't generating as expected, the debug setting is your first tool. Enable it in the extension settings; it will log all system variables at generation time, which tells you exactly what values are available for your conditions.
|
||||
@@ -515,3 +580,38 @@ This confirms the order and content of choices before relying on label filters.
|
||||
- A `%` choice in a YAML array must be quoted, otherwise YAML treats `%` as invalid syntax.
|
||||
- Variables set with `${var=value}` are lazy - they are not evaluated until echoed. Use `${var=!__wildcard__}` (with `!`) for immediate evaluation if the value should only be resolved once (as in, you want the same value to be echoed later multiple times).
|
||||
- Wildcards cannot be used inside extranetwork tags (because some LoRA names contain double underscores). Put the entire `<lora:...>` tag inside a wildcard choice instead, or use the `ext` command.
|
||||
|
||||
## Recording results to a file
|
||||
|
||||
When using wildcards and choices the actual prompt sent to the model can look very different from what you typed. Configuring a results file lets you capture every resolved prompt, along with its metadata, so you always know exactly what was generated.
|
||||
|
||||
Common use cases:
|
||||
|
||||
- **Reproducibility** - if an image turns out great but you didn't keep the inputs, you can look up the exact resolved prompt and all the inputs from the log.
|
||||
- **Dataset building** - when running large batches with combinatorial mode or random wildcards, the file gives you a record of every prompt variant that was generated.
|
||||
- **Prompt analysis** - review the log to spot patterns, see which wildcard choices come up most often, or evaluate the output of a new wildcard set before committing to a full run.
|
||||
- **Debugging wildcard expansion** - compare the resolved prompts against what you expected to confirm that variables, conditions, and filters are all behaving correctly.
|
||||
|
||||
### Setup
|
||||
|
||||
Set the `results_file` option (in the extension settings for *A1111*, or the `results_file` input on the main node for *ComfyUI*) to a filename. The extension determines the output format from the file extension:
|
||||
|
||||
| Extension | Format |
|
||||
|------------------|---------------------------------------------|
|
||||
| `.yaml` / `.yml` | YAML list of records |
|
||||
| `.jsonl` | JSON Lines, one JSON object per line |
|
||||
| `.csv` | CSV with a header row (semicolon-delimited) |
|
||||
| anything else | Plain text with labelled sections |
|
||||
|
||||
Each record contains five sections: `options` (the PPP settings that were active), `inputs` (seed, prompts), `system` (system variables like `_modelclass`), `results` (the final positive and negative prompts), and `variables` (any user variables that were set).
|
||||
|
||||
Use `%datetime%`, `%date%`, `%time%`, or `%host%` tokens in the filename to create a new file for each session or application:
|
||||
|
||||
```text
|
||||
results_%date%.yaml
|
||||
```
|
||||
|
||||
Relative paths are resolved against the `logs` folder inside the extension directory.
|
||||
|
||||
> [!TIP]
|
||||
> The `.jsonl` format is the most convenient for programmatic processing. The `.yaml` format is the easiest to read manually.
|
||||
|
||||
+128
-82
@@ -1,8 +1,12 @@
|
||||
# Prompt PostProcessor syntax
|
||||
|
||||
## Basic usage
|
||||
|
||||
The extension works by modifying the prompt and negative prompt, expanding, replacing and cleaning content, before the image is generated.
|
||||
|
||||
## Commands
|
||||
|
||||
The extension uses a format for its commands similar to an extranetwork, but it has a "ppp:" prefix followed by the command, and then a space and any parameters (if any).
|
||||
The extension uses a format for its commands similar to an extranetwork, but it has a `ppp:` prefix followed by the command, and then a space and any parameters (if any).
|
||||
|
||||
`<ppp:command parameters/>`
|
||||
|
||||
@@ -31,50 +35,51 @@ characters:
|
||||
|
||||
The generic format is: `{parameters$$opt1::choice1|opt2::choice2|opt3::choice3}`
|
||||
|
||||
Both the construct parameters (up to the `$$`) and the individual choice options (up to the '::') are optional.
|
||||
Both the construct parameters (up to the `$$`) and the individual choice options (up to the `::`) are optional.
|
||||
|
||||
There is also a format where instead of `parameters$$` you just put the sampler, for compatibility with *Dynamic Prompts*.
|
||||
|
||||
The construct parameters can be written with the following options (all are optional):
|
||||
|
||||
* "**~**" (random) or "**@**" (cyclical): sampler (for compatibility with *Dynamic Prompts*). The cyclical sampler cycles through all combinations in order across consecutive `process_prompt` calls, resuming where the previous call left off (as long as the input prompt and negative prompt do not change).
|
||||
* "**r**": means it allows repetition of the choices.
|
||||
* "**o**": means it is "optional", and no error will be raised if there are no choices to select from.
|
||||
* "**n**" or "**n-m**" or "**n-**" or "**-m**": number or range of choices to select. Allows zero as the start of a range. Default is 1.
|
||||
* "**'description'**": optional description, only valid in wildcard definitions. Used only in the Wildcards Concat node in ComfyUI.
|
||||
* "**$$sep**": separator when multiple choices are selected. Default is set in settings.
|
||||
* "**$$**": end of the parameters (not optional if any parameters).
|
||||
* `~` (random) or `@` (cyclical): sampler (for compatibility with *Dynamic Prompts*). The cyclical sampler cycles through all combinations in order across consecutive `process_prompt` calls, resuming where the previous call left off (as long as the input prompt and negative prompt do not change). In combinatorial mode you can use the random sampler to stop a specific choice/wildcard from being expanded into all its combinations.
|
||||
* `r`: means it allows repetition of the choices.
|
||||
* `o`: means it is optional, and no error will be raised if there are no choices to select from.
|
||||
* `n` or `n-m` or `n-` or `-m`: number or range of choices to select. Allows zero as the start of a range. Default is 1.
|
||||
* `'description'`: optional description (quoted), only valid in wildcard definitions. Used only in the Wildcards Concat node in *ComfyUI*.
|
||||
* `$$sep`: separator when multiple choices are selected. Default is set in settings.
|
||||
* `$$`: end of the parameters (not optional if any parameters).
|
||||
|
||||
Regarding the "optional" flag, consider this scenario: due to their conditions no choice is available. It will raise an error. If you add the `o` then it will just return an empty string. This is only necessary if all choices have conditions and they could all be false. It is not the same as setting a range starting at 0, because that would be an allowed number of returned choices. If you do this and no choices are available, no error is raised.
|
||||
Regarding the optional flag, consider this scenario: due to their conditions no choice is available. It will raise an error. If you add the `o` then it will just return an empty string. This is only necessary if all choices have conditions and they could all be false. It is not the same as setting a range starting at 0, because that would be an allowed number of returned choices. If you do this and no choices are available, no error is raised.
|
||||
|
||||
The choice options are as follows:
|
||||
|
||||
* "**%**": indicates that the content of the choice is a command
|
||||
* "**'identifiers'**": comma separated labels for the choice (optional, quotes can be single or double). Only makes sense inside a wildcard definition. Can be used when specifying the wildcard to select this specific choice. It's case insensitive.
|
||||
* "**n**": weight of the choice (optional, default 1).
|
||||
* "**if condition**": filters out the choice if the condition is false (optional; this is an extension to the *Dynamic Prompts* syntax). Same conditions as in the `if` command.
|
||||
* "**::**": end of choice options (not optional if any options)
|
||||
* `%`: indicates that the content of the choice is a command
|
||||
* `'identifiers'`: comma separated labels for the choice (optional, quotes can be single or double). Only makes sense inside a wildcard definition. Can be used when specifying the wildcard to select this specific choice. It's case insensitive.
|
||||
* `n`: weight of the choice (optional, default 1).
|
||||
* `if condition`: filters out the choice if the condition is false (optional; this is an extension to the *Dynamic Prompts* syntax). Same conditions as in the `if` command.
|
||||
* `else`: flag to indicate this is the choice to use if no other choice is available after conditions. It won't be considered if other choices are available. Also optional.
|
||||
* `::`: end of choice options (not optional if any options)
|
||||
|
||||
Whitespace is allowed between parameters/options.
|
||||
|
||||
The only command available is `include wildcard`, which will include the choices of the specified wildcard in place of this choice. This allows composing choices from multiple wildcards. It also works in the choices of a wildcard, but note that in yaml you cannot start an array element with "%" and you will have to put the full choice in quotes, or use the object format.
|
||||
The only command available is `include wildcard`, which will include the choices of the specified wildcard in place of this choice. This allows composing choices from multiple wildcards. It also works in the choices of a wildcard, but note that in yaml you cannot start an array element with `%` and you will have to put the full choice in quotes, or use the object format.
|
||||
|
||||
These are examples of formats you can use to insert a choice construct:
|
||||
|
||||
| Construct | Result |
|
||||
| --------- | ------ |
|
||||
|---------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `{choice1\|5::choice2\|3::choice3}` | select 1 choice, two of them have weights |
|
||||
| `{3$$choice1\|5 if _is_sd1::choice2\|choice3}` | select 3 choices, one has a weight and a condition |
|
||||
| `{2-3$$2::choice1\|choice2\|choice3}` | select 2 to 3 choices, one of them has a weight |
|
||||
| `{r2-3$$choice1\|choice2\|choice3}` | select 2 to 3 choices allowing repetition |
|
||||
| `{2-3$$ / $$choice1\|choice2\|choice3}` | select 2 to 3 choices with separator " / " |
|
||||
| `{2-3$$ / $$choice1\|choice2\|choice3}` | select 2 to 3 choices with separator ` / ` |
|
||||
| `{o$$if _is_sd1::choice1\|if _is_sd2::choice2}` | select 1 choice, both have conditions, if none matches it is allowed because we indicate that it is optional |
|
||||
| `{choice1\|choice2\|%0.5::include path/wildcard}` | select 1 choice from the two specified and the ones inside the path/wildcard wildcard, which will be weighted with half their weights |
|
||||
|
||||
Notes:
|
||||
|
||||
* The *Dynamic Prompts* format `{2$$__flavours__}` does not work as expected because the wildcard is considered only one possible choice (it will only output one value). You can write it instead as a wildcard with parameters `__2$$flavours__`.
|
||||
* Whitespace around the choices is not ignored like in *Dynamic Prompts*, but will be cleaned up if the appropriate cleaning settings are selected.
|
||||
> [!NOTE]
|
||||
> The *Dynamic Prompts* format `{2$$__flavours__}` does not work as expected because the wildcard is considered only one possible choice (it will only output one value). You can write it instead as a wildcard with parameters `__2$$flavours__`.
|
||||
>
|
||||
> Whitespace around the choices is not ignored like in *Dynamic Prompts*, but will be cleaned up if the appropriate cleaning settings are selected.
|
||||
|
||||
## Wildcards
|
||||
|
||||
@@ -82,7 +87,8 @@ The generic format is: `__parameters$$wildcard'filter'(var=value)__`
|
||||
|
||||
The parameters, the filter, and the setting of a variable are optional. The parameters follow the same format as for the choices.
|
||||
|
||||
Wildcards cannot be used inside an extranetwork tag (because some LoRA names contain double underscores). If you need to choose from multiple LoRAs put the whole extranetwork tag inside a wildcard, or use choices.
|
||||
> [!WARNING]
|
||||
> Wildcards cannot be used inside an extranetwork tag (because some LoRA names contain double underscores). If you need to choose from multiple LoRAs put the whole extranetwork tag inside a wildcard, or use choices.
|
||||
|
||||
### Identifier
|
||||
|
||||
@@ -110,17 +116,17 @@ The variable value only applies during the evaluation of the selected choices an
|
||||
These are examples of formats you can use to insert a wildcard:
|
||||
|
||||
| Construct | Result |
|
||||
| --------- | ------ |
|
||||
|--------------------------------------|--------------------------------------------------------------------------|
|
||||
| `__wildcard__` | select 1 choice |
|
||||
| `__path/wildcard'0'__` | select the first choice |
|
||||
| `__path/wildcard'1-2'__` | select the second or third choice |
|
||||
| `__path/wildcard'label'__` | select the choices with label "label" |
|
||||
| `__path/wildcard'0,label1,label2'__` | select the first choice and those with labels "label1" or "label2" |
|
||||
| `__path/wildcard'0,label1+label2'__` | select the first choice and those with both labels "label1" and "label2" |
|
||||
| `__path/wildcard'label'__` | select the choices with label `label` |
|
||||
| `__path/wildcard'0,label1,label2'__` | select the first choice and those with labels `label1` or `label2` |
|
||||
| `__path/wildcard'0,label1+label2'__` | select the first choice and those with both labels `label1` and `label2` |
|
||||
| `__3$$path/wildcard__` | select 3 choices |
|
||||
| `__2-3$$path/wildcard__` | select 2 to 3 choices |
|
||||
| `__r2-3$$path/wildcard__` | select 2 to 3 choices allowing repetition |
|
||||
| `__2-3$$ / $$path/wildcard__` | select 2 to 3 choices with separator " / " |
|
||||
| `__2-3$$ / $$path/wildcard__` | select 2 to 3 choices with separator ` / ` |
|
||||
| `__path/wildcard(var=value)__` | select 1 choice using the specified variable value in the evaluation. |
|
||||
|
||||
### Wildcard definitions
|
||||
@@ -141,6 +147,7 @@ The choices of the wildcard follow the same format as in the choices construct,
|
||||
If using the object format for a choice you can use the following in addition to the standard `weight` and `text`/`content`:
|
||||
|
||||
* `if`: the condition (a string)
|
||||
* `else`: flag (boolean) to indicate this is the choice to use if no other choice is available after conditions. It won't be considered if other choices are available.
|
||||
* `labels`: list of labels (an array of strings)
|
||||
* `command`: indicates the content is a command (a boolean)
|
||||
|
||||
@@ -174,13 +181,15 @@ A choice inside a wildcard can also be a list or a dictionary of one element con
|
||||
|
||||
Remember you can use the include command on choices to compose a wildcard from other wildcards' choices.
|
||||
|
||||
Note: the files should have UTF-8 encoding. The extension will also try with windows-1252 if that fails.
|
||||
> [!IMPORTANT]
|
||||
> The files should have UTF-8 encoding. The extension will also try with windows-1252 if that fails.
|
||||
|
||||
Wildcard definitions are reloaded automatically on each generation if they change.
|
||||
|
||||
### Detection of remaining wildcards
|
||||
|
||||
This extension should run after any other wildcard extensions, so if you don't use the internal wildcards processing, any remaining wildcards present in the prompt or negative_prompt at this point must be invalid. Usually you might not notice this problem until you check the image metadata, so this option gives you some ways to detect and treat the problem.
|
||||
> [!IMPORTANT]
|
||||
> This extension should run after any other wildcard extensions, so if you don't use the internal wildcards processing, any remaining wildcards present in the prompt or negative_prompt at this point must be invalid. Usually you might not notice this problem until you check the image metadata, so this option gives you some ways to detect and treat the problem.
|
||||
|
||||
## Set Wildcard Default Filter command
|
||||
|
||||
@@ -189,12 +198,52 @@ This command can be used to set a default filter for a wildcard, before it is us
|
||||
The format is:
|
||||
|
||||
| Construct | Meaning |
|
||||
| --------- | ------- |
|
||||
|-----------------------------------------------|--------------------|
|
||||
| `<ppp:setwcdeffilter 'identifier' 'filter'/>` | Sets a filter |
|
||||
| `<ppp:setwcdeffilter 'identifier'/>` | Removes the filter |
|
||||
|
||||
The wildcard identifier supports globbing. The filter does not allow the `^` or `#` flags.
|
||||
|
||||
## Variables
|
||||
|
||||
The prompt has access to some system variables that contain model information, options, and other things. There is also the possibility of defining user variables.
|
||||
|
||||
Variable values `true` and `false` are considered a boolean, and numeric content is an integer or float.
|
||||
|
||||
All these variables can be used to output content or behave differently based on their values.
|
||||
|
||||
### System variables
|
||||
|
||||
Names starting with an underscore are reserved for system variables:
|
||||
|
||||
| System variable | Value |
|
||||
|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `_model` | The model identifier (`sd1`, `sd2`, `sdxl`, `sd3`, `flux`, `auraflow`). `_sd` also works but is deprecated. |
|
||||
| `_modelname` | The model filename (without path). Do not confuse with the `modelname` input in *ComfyUI* which matches actually to the `_modelfullname` variable. `_sdname` also works but is deprecated. |
|
||||
| `_modelfullname` | The model filename (with path). `_sdfullname` also works but is deprecated. In *ComfyUI* this variable can also be **set** to override the filename used for model detection (see below). |
|
||||
| `_modelclass` | The class used for the model. Note that this is dependent on the webui. In A1111 all SD versions use the same class. Can be used for new models that are not supported yet with the `_is_*` variables. The debug setting will show all system variables when generating in case you need to see which one to use for a certain model. |
|
||||
| `_is_kkkk` | True if the model is of kind *kkkk* (the model identifier, f.e. sdxl; those set in the ppp_config.yaml file) |
|
||||
| `_is_vvvv` | True if the model matches the *vvvv* model variant definition (based on its filename). Note that the corresponding variable for the model kind will also be true. |
|
||||
| `_is_pure_kkkk` | True if the model is of kind *kkkk* and not a variant. |
|
||||
| `_is_variant_kkkk` | True if the model version is any variant of model kind *kkkk* and not the pure version. Note that the corresponding variable for the model kind will also be true. |
|
||||
| `_is_sd` | True if the model is any version of SD |
|
||||
| `_is_ssd` | True if the model is SSD (Segmind Stable Diffusion 1B). Note that for an SSD model `_is_sdxl` will also be true. |
|
||||
| `_is_sdxl_no_ssd` | True if the model is SDXL and not an SSD model. |
|
||||
| `_is_sdxl_no_pony` | True if the model is SDXL and not a Pony model (the `pony` variant must be defined in settings). Kept to maintain compatibility with previous versions. |
|
||||
| `_opt_...` | All the options. |
|
||||
| `_input_seed` | The starting seed used. |
|
||||
| `_input_pos_prompt` | The original positive prompt. |
|
||||
| `_input_neg_prompt` | The original negative prompt. |
|
||||
| `_input_prev_pos_prompt` | The positive prompt result of the previous phase. Available only in the hires fix phase in A1111 compatible hosts and with no combinatorial generation. |
|
||||
| `_input_prev_neg_prompt` | The negative prompt result of the previous phase. Available only in the hires fix phase in A1111 compatible hosts and with no combinatorial generation. |
|
||||
| `_output_seed` | The seed used for the specific result. This variable changes for each result depending on the `next_seed` option. You can feed it to the ksampler. |
|
||||
|
||||
> [!NOTE]
|
||||
> The model path is relative to the checkpoint/difussion_models folder, just as it appears in the load nodes.
|
||||
|
||||
> [!NOTE]
|
||||
> You can use the `_input_prev_pos_prompt` and `_input_prev_neg_prompt` only in the hires fix prompt boxes in A1111 compatible hosts, and only without the combinatorial option.
|
||||
|
||||
## Set command
|
||||
|
||||
This command sets the value of a variable that can be checked later.
|
||||
@@ -204,7 +253,7 @@ The format is: `<ppp:set varname [modifiers]>value<ppp:/set>`
|
||||
These are the available optional modifiers:
|
||||
|
||||
* `evaluate`: the value of the variable is evaluated at this moment, instead of when it is used.
|
||||
* `add`: the value is added to the current value of the variable. It does not force an immediate evaluation of the old nor the added value.
|
||||
* `add`: the value is "added" to the current value of the variable (depending on their type). When possible, it does not force an immediate evaluation of the old or added values.
|
||||
* `ifundefined`: the value will only be set if the variable is undefined.
|
||||
|
||||
The `add` and `ifundefined` modifiers are mutually exclusive and cannot be used together.
|
||||
@@ -212,21 +261,40 @@ The `add` and `ifundefined` modifiers are mutually exclusive and cannot be used
|
||||
The *Dynamic Prompts* format also works:
|
||||
|
||||
| Construct | Meaning |
|
||||
| --------- | ------- |
|
||||
|-----------------|----------------------|
|
||||
| `${var=value}` | regular evaluation |
|
||||
| `${var=!value}` | immediate evaluation |
|
||||
|
||||
If also supports the addition and undefined check as an extension of the *Dynamic Prompts* format:
|
||||
|
||||
| Construct | Meaning |
|
||||
| --------- | ------- |
|
||||
| `${var+=value}` | equivalent to "add" |
|
||||
| `${var+=!value}` | equivalent to "evaluate add" |
|
||||
| `${var?=value}` | equivalent to "ifundefined" |
|
||||
| `${var?=!value}` | equivalent to "evaluate ifundefined" |
|
||||
|------------------|--------------------------------------|
|
||||
| `${var+=value}` | equivalent to `add` |
|
||||
| `${var+=!value}` | equivalent to `evaluate add` |
|
||||
| `${var?=value}` | equivalent to `ifundefined` |
|
||||
| `${var?=!value}` | equivalent to `evaluate ifundefined` |
|
||||
|
||||
Set variables are included in the output variables with their last value.
|
||||
|
||||
System variables cannot be set, except in some cases (see next point).
|
||||
|
||||
### Overriding model information from the prompt (*ComfyUI* only)
|
||||
|
||||
In *ComfyUI*, `_modelfullname` can be **set** from the prompt using the `set` command or `${}` syntax. Setting it triggers a re-evaluation of all `_is_*` system variables, so conditions that come later in the prompt will use the new values.
|
||||
|
||||
This is useful in workflows where the model information is not passed through the node inputs but is known at prompt time. And the `_modelfullname` variable can be extracted later by using the `Select Variable` node to actually load that model.
|
||||
|
||||
If a class is not specified because the `model` input is disconnected but the `modelname` is, it will try to detect the class from the file. This is the preferred way, or just set `_modelfullname` in the prompt.
|
||||
|
||||
The model filename is relative to the specific model folder (checkpoints, diffusion_models).
|
||||
|
||||
```text
|
||||
${_modelfullname=flux/fluxmodel.safetensors}
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> Attempting to set this variable in any other host will trigger a warning or error depending on the **What to do on invalid content warnings?** setting.
|
||||
|
||||
## Echo command
|
||||
|
||||
This command prints the value of a variable, or the specified default if it doesn't exist. If the variable does not exist and has no default, depending on the **What to do on invalid content warnings?** setting it will assume an empty value, or result in an error.
|
||||
@@ -234,14 +302,14 @@ This command prints the value of a variable, or the specified default if it does
|
||||
The format is:
|
||||
|
||||
| Construct |
|
||||
| --------- |
|
||||
|----------------------------------------|
|
||||
| `<ppp:echo varname/>` |
|
||||
| `<ppp:echo varname>default<ppp:/echo>` |
|
||||
|
||||
The *Dynamic Prompts* format is:
|
||||
|
||||
| Construct |
|
||||
| --------- |
|
||||
|----------------------|
|
||||
| `${varname}` |
|
||||
| `${varname:default}` |
|
||||
|
||||
@@ -254,7 +322,7 @@ There is support for array variables. They use brackets `[]` to differenciate fr
|
||||
They can be initialized in several ways:
|
||||
|
||||
| Construct | Meaning |
|
||||
| --------- | ------- |
|
||||
|----------------------------|------------------------------------------------------------------|
|
||||
| `${var[]=value}` | initialize and set the first value |
|
||||
| `${var[]=*()}` | initialize an empty array |
|
||||
| `${var[]=*var2[]}` | initialize an array from another array |
|
||||
@@ -265,22 +333,22 @@ They can be initialized in several ways:
|
||||
| `${var[]+=*var2[]}` | add elements from another array |
|
||||
| `${var[]+=*__wildcard__}` | add elements from a wildcard |
|
||||
|
||||
The star operator `*` is always with inmediate evaluation.
|
||||
The star operator `*` has always an inmediate evaluation.
|
||||
|
||||
And can be accesed/echoed with:
|
||||
And they can be accesed/echoed with:
|
||||
|
||||
* Empty brackets mean the whole array (used when initializing or when echoing the whole array)
|
||||
* An integer inside the brackets means an indexed value. A variable identifier (not an indexed array) can be used to get the integer.
|
||||
* A hash inside the brackets is used to get the length of the array
|
||||
* An ampersand followed by a string inside the brackets (with quotes) is used to get the full array joined with a separator.
|
||||
|
||||
| Construct | Meaning |
|
||||
| --------- | ------- |
|
||||
| `${var[]}` | echo all elements with a default separator |
|
||||
| `${var[&' / ']}` | echo all elements with a specific separator |
|
||||
| `${var[n]}` | echo an element from the array |
|
||||
| `${var[n]:default}` | echo an element with a default |
|
||||
| `${var[#]}` | echo the length of the array |
|
||||
| Construct | Meaning |
|
||||
|---------------------|---------------------------------------------|
|
||||
| `${var[]}` | echo all elements with a default separator |
|
||||
| `${var[&' / ']}` | echo all elements with a specific separator |
|
||||
| `${var[n]}` | echo an element from the array |
|
||||
| `${var[n]:default}` | echo an element with a default |
|
||||
| `${var[#]}` | echo the length of the array |
|
||||
|
||||
## If command
|
||||
|
||||
@@ -295,7 +363,7 @@ Any `elif`s (there can be multiple) and the `else` are optional.
|
||||
The `conditionN` is a boolean expression, which can use `and`, `or`, `not` and grouping with parentheses, and where the simplest expression can be:
|
||||
|
||||
| Construct | Meaning |
|
||||
| --------- | ------- |
|
||||
|-------------------------------------|---------------------------------------------------------------------------------|
|
||||
| `operand` | check truthyness of the operand, meaning not zero, empty string nor empty array |
|
||||
| `operand1 [not] operation operand2` | compare the operands |
|
||||
|
||||
@@ -303,7 +371,7 @@ The operands can be a variable (`variable`, `array[]`, `array[index]`), a quoted
|
||||
|
||||
When an operand is or contains a variable, it is resolved to the variable's current value before the operation.
|
||||
|
||||
Variable values `true` and `false` are considered a boolean, and an all digits value is an integer. Except in substring operations indicated below.
|
||||
String comparisons are case insensitive, and substring comparisons (see below) are always considered as strings.
|
||||
|
||||
The operation can be preceded by `not` for readability, instead of using it in the front.
|
||||
|
||||
@@ -312,7 +380,7 @@ The supported operations are: `eq`, `ne`, `gt`, `lt`, `ge`, `le`, `in`, `any_in`
|
||||
This list shows what they do depending on the kind of operand (R = regular variable, A = array variable).
|
||||
|
||||
| Operation | R1 op R2 | A1 op A2 | A1 op R2 | R1 op A2 |
|
||||
| --------- | -------- | -------- | -------- | -------- |
|
||||
|----------------|--------------------------|-------------------|------------------------------------------------|------------------------------------------------|
|
||||
| `eq` | OK | OK (pairwise) | Error in strict mode, all A1 with R2 otherwise | Error in strict mode, R1 with all A2 otherwise |
|
||||
| `ne` | OK | OK (pairwise) | Error in strict mode, all A1 with R2 otherwise | Error in strict mode, R1 with all A2 otherwise |
|
||||
| `gt` | OK | OK (pairwise) | Error in strict mode, all A1 with R2 otherwise | Error in strict mode, R1 with all A2 otherwise |
|
||||
@@ -326,28 +394,6 @@ This list shows what they do depending on the kind of operand (R = regular varia
|
||||
|
||||
When a comparison tries to compare undefined variables or the values have different types (f.e. an integer and a string), the behavior depends on the `on_warning` setting: in `warn` mode the comparison evaluates to false, and in `stop` mode an error is raised. In non strict mode a numeric string literal (with no leading zeros) will be considered an integer.
|
||||
|
||||
The variable can be one set with the `set` command (user variables) or you can use system variables like these (names starting with an underscore are reserved for system variables):
|
||||
|
||||
| System variable | Value |
|
||||
| --------------- | ----- |
|
||||
| `_model` | the loaded model identifier (`"sd1"`, `"sd2"`, `"sdxl"`, `"sd3"`, `"flux"`, `"auraflow"`). `_sd` also works but is deprecated. |
|
||||
| `_modelname` | the loaded model filename (without path). `_sdname` also works but is deprecated. |
|
||||
| `_modelfullname` | the loaded model filename (with path). `_sdfullname` also works but is deprecated. |
|
||||
| `_modelclass` | the class used for the model. Note that this is dependent on the webui. In A1111 all SD versions use the same class. Can be used for new models that are not supported yet with the `_is_*` variables. The debug setting will show all system variables when generating in case you need to see which one to use for a certain model. |
|
||||
| `_is_sd` | true if the loaded model version is any version of SD |
|
||||
| `_is_sd1` | true if the loaded model version is SD 1.x |
|
||||
| `_is_sd2` | true if the loaded model version is SD 2.x |
|
||||
| `_is_sdxl` | true if the loaded model version is SDXL (includes Pony models) |
|
||||
| `_is_sd3` | true if the loaded model version is SD 3.x |
|
||||
| `_is_flux` | true if the loaded model is Flux |
|
||||
| `_is_auraflow` | true if the loaded model is AuraFlow |
|
||||
| `_is_ssd` | true if the loaded model version is SSD (Segmind Stable Diffusion 1B). Note that for an SSD model `_is_sdxl` will also be true. |
|
||||
| `_is_sdxl_no_ssd` | true if the loaded model version is SDXL and not an SSD model. |
|
||||
| `_is_sdxl_no_pony` | true if the loaded model version is SDXL and not a Pony model (the "pony" variant must be defined in settings). Kept to maintain compatibility with previous versions. |
|
||||
| `_is_vvvv` | true if the loaded model matches the *vvvv* model variant definition (based on its filename). Note that the corresponding variable for the model kind will also be true. |
|
||||
| `_is_pure_kkkk` | true if the loaded model is of kind *kkkk* (f.e. sdxl) and not a variant. |
|
||||
| `_is_variant_kkkk` | true if the loaded model version is any variant of model kind *kkkk* and not the pure version. Note that the corresponding variable for the model kind will also be true. |
|
||||
|
||||
### Example
|
||||
|
||||
(multiline to be easier to read)
|
||||
@@ -424,18 +470,18 @@ extnettype:
|
||||
|
||||
Used like this:
|
||||
|
||||
| Construct | Meaning |
|
||||
| --------- | ------- |
|
||||
| `<ppp:ext $lora mappingname/>` | Mapping without additional triggers |
|
||||
| `<ppp:ext $lora mappingname>inline triggers<ppp:/ext>` | Mapping with additional triggers |
|
||||
| Construct | Meaning |
|
||||
|--------------------------------------------------------|-------------------------------------|
|
||||
| `<ppp:ext $lora mappingname/>` | Mapping without additional triggers |
|
||||
| `<ppp:ext $lora mappingname>inline triggers<ppp:/ext>` | Mapping with additional triggers |
|
||||
|
||||
Each mapping can have any number of elements in its list of mappings. There are no mandatory properties for a mapping. The properties mean the following:
|
||||
|
||||
* `extnettype`: the kind of extranetwork, for example `lora`.
|
||||
* `mappingname`: the name you want to give to the mapping, to be referenced in the command.
|
||||
* `condition`: the condition to check for this mapping to be used (usually it should be one of the `_is_*` variables). If the conditions of multiple mappings evaluate to True, one will be chosen randomly. If the condition is missing it is considered True, to be used in the last mapping to catch as an "else" condition, and will be used if no other mapping applies.
|
||||
* `condition`: the condition to check for this mapping to be used (usually it should be one of the `_is_*` variables). If the conditions of multiple mappings evaluate to True, one will be chosen randomly. If the condition is missing it is considered True, to be used in the last mapping to catch as an `else` condition, and will be used if no other mapping applies.
|
||||
* `name`: name of the real extranetwork. If it is missing no extranetwork tag will be added.
|
||||
* `parameters`: parameters for the real extranetwork. If it is missing it is assumed "1" for LoRAs and HyperNets. If both this parameter and the parameter in the ext command are numbers they are multiplied for the result. In other case the parameter of the ext command, if it exists, is used.
|
||||
* `parameters`: parameters for the real extranetwork. If it is missing it is assumed `1` for LoRAs and HyperNets. If both this parameter and the parameter in the ext command are numbers they are multiplied for the result. In other case the parameter of the ext command, if it exists, is used.
|
||||
* `triggers`: list of trigger strings. If it is missing, only the inline triggers in the ext command will be added.
|
||||
* `weight`: weight for this variant, in case multiple of them apply, to choose one. Default is 1.
|
||||
|
||||
@@ -446,7 +492,7 @@ See the file in the tests folder as an example.
|
||||
The new format for this command is like this:
|
||||
|
||||
| Construct | Meaning |
|
||||
| --------- | ------- |
|
||||
|---------------------------------------|--------------------------------------------------------------------------------------|
|
||||
| `<ppp:stn position>content<ppp:/stn>` | send to negative prompt |
|
||||
| `<ppp:stn iN/>` | insertion point to be used in the negative prompt as destination for the pN position |
|
||||
|
||||
|
||||
+72
-69
@@ -1,12 +1,11 @@
|
||||
%import common (LETTER, DIGIT, INT, CNAME, SIGNED_NUMBER, NUMBER)
|
||||
%import common (CNAME, INT, SIGNED_NUMBER, NUMBER)
|
||||
|
||||
_WHITESPACE: /\s+/
|
||||
STRING: /("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/i // string with single or double quotes
|
||||
BOOLEAN: /true|false/i
|
||||
WC_NAME_PLAIN_START: /(?:(?!__|\$\$|\$\{|<ppp:|[('"])[a-zA-Z0-9\-.\\\/])+/u // starting characters, excluding end and other inside constructs
|
||||
WC_NAME_PLAIN: /(?:(?!__|\$\$|\$\{|<ppp:|[('"])[a-zA-Z0-9\-.\\\/_#\*\?\[!\]])+/u // continuing characters, including globbing characters and anonymous wildcard mark, excluding end and other inside constructs
|
||||
WC_NAME_PLAIN_START: /(?:(?!__|\$\$|\$\{|<ppp:|[('"])[a-zA-Z0-9\-.\\\/\*\?\[!\]])+/u // starting characters, excluding end and other inside constructs
|
||||
WC_NAME_PLAIN: /(?:(?!__|\$\$|\$\{|<ppp:|[('"])[a-zA-Z0-9\-.\\\/\*\?\[!\]_#])+/u // continuing characters, including globbing characters and anonymous wildcard mark, excluding end and other inside constructs
|
||||
INDEX: INT
|
||||
FILTER: INDEX | IDENTIFIER
|
||||
IDENTIFIER: CNAME
|
||||
SIMPLEVALUE: STRING | SIGNED_NUMBER | BOOLEAN
|
||||
VARNAME: /(?!__)[a-zA-Z_][a-zA-Z0-9_]*/
|
||||
@@ -28,59 +27,59 @@ start: prompt negative_sep prompt
|
||||
negative_sep: /\x1d/
|
||||
|
||||
// prompt composition with AND
|
||||
promptcomp.4: promptcomppart ( [ ":" numpar ] ( /\bAND\b/ promptcomppart [ ":" numpar ] )+ )+
|
||||
promptcomp.5: promptcomppart ( [ ":" numpar ] ( /\bAND\b/ promptcomppart [ ":" numpar ] )+ )+
|
||||
promptcomppart: content
|
||||
|
||||
// simple prompts
|
||||
|
||||
?old_content.2: ( attention | scheduled | alternate | extranetworktag )+
|
||||
?old_content.3: ( attention | scheduled | alternate | extranetworktag )+
|
||||
//#if ALLOW_NEW_CONTENT
|
||||
?content.2: ( old_content | new_content | plain | specialchars )*
|
||||
?content_choice.2: ( old_content | new_content | plain_choice | specialchars_choice )*
|
||||
?content_var.2: ( old_content | new_content | plain_var | specialchars_var )*
|
||||
?content_negtag.2: ( old_content | new_content_negtag | plain | specialchars_negtag )*
|
||||
?content_alternate.2: ( old_content | new_content | plain_alternate | specialchars_alternate )*
|
||||
?content_en.2: (new_content_en | plain | specialchars )*
|
||||
?content.3: ( old_content | new_content | plain | specialchars )*
|
||||
?content_choice.3: ( old_content | new_content | plain_choice | specialchars_choice )*
|
||||
?content_var.3: ( old_content | new_content | plain_var | specialchars_var )*
|
||||
?content_negtag.3: ( old_content | new_content_negtag | plain | specialchars_negtag )*
|
||||
?content_alternate.3: ( old_content | new_content | plain_alternate | specialchars_alternate )*
|
||||
?content_en.3: (new_content_en | plain | specialchars )*
|
||||
//#if ALLOW_WILDCARDS and ALLOW_CHOICES and ALLOW_COMMVARS
|
||||
?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
|
||||
?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
|
||||
?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif | choices )+
|
||||
?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
|
||||
?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
|
||||
?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif | choices )+
|
||||
//#elif ALLOW_WILDCARDS and not ALLOW_CHOICES and ALLOW_COMMVARS
|
||||
?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard )+
|
||||
?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard )+
|
||||
?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif )+
|
||||
?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard )+
|
||||
?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard )+
|
||||
?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif )+
|
||||
//#elif not ALLOW_WILDCARDS and ALLOW_CHOICES and ALLOW_COMMVARS
|
||||
?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | choices )+
|
||||
?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | commandext | choices )+
|
||||
?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif | choices )+
|
||||
?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | choices )+
|
||||
?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext | choices )+
|
||||
?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif | choices )+
|
||||
//#elif ALLOW_WILDCARDS and ALLOW_CHOICES and not ALLOW_COMMVARS
|
||||
?new_content.3: ( wildcard | choices )+
|
||||
?new_content_negtag.3: ( wildcard | choices )+
|
||||
?new_content_en.3: ( choices )+
|
||||
?new_content.4: ( wildcard | choices )+
|
||||
?new_content_negtag.4: ( wildcard | choices )+
|
||||
?new_content_en.4: ( choices )+
|
||||
//#elif ALLOW_WILDCARDS and not ALLOW_CHOICES and not ALLOW_COMMVARS
|
||||
?new_content.3: ( wildcard )+
|
||||
?new_content_negtag.3: ( wildcard )+
|
||||
?new_content_en.3: /(?!)./ // never matches
|
||||
?new_content.4: ( wildcard )+
|
||||
?new_content_negtag.4: ( wildcard )+
|
||||
?new_content_en.4: /(?!)./ // never matches
|
||||
//#elif not ALLOW_WILDCARDS and ALLOW_CHOICES and not ALLOW_COMMVARS
|
||||
?new_content.3: ( choices )+
|
||||
?new_content_negtag.3: ( choices )+
|
||||
?new_content_en.3: ( choices )+
|
||||
?new_content.4: ( choices )+
|
||||
?new_content_negtag.4: ( choices )+
|
||||
?new_content_en.4: ( choices )+
|
||||
//#elif not ALLOW_WILDCARDS and not ALLOW_CHOICES and ALLOW_COMMVARS
|
||||
?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext )+
|
||||
?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | commandext )+
|
||||
?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif )+
|
||||
?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext )+
|
||||
?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext )+
|
||||
?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif )+
|
||||
//#else
|
||||
?new_content.3: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
|
||||
?new_content_negtag.3: ( variableset | variableuse | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
|
||||
?new_content_en.3: ( variableset | variableuse | commandset | commandecho | commandif | choices )+
|
||||
?new_content.4: ( variableset | variableuse | commandstn | commandstni | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
|
||||
?new_content_negtag.4: ( variableset | variableuse | commandset | commandecho | commandif | commandext | commandsetwcdeffilter | wildcard | choices )+
|
||||
?new_content_en.4: ( variableset | variableuse | commandset | commandecho | commandif | choices )+
|
||||
//#endif
|
||||
//#else
|
||||
?content.2: ( old_content | plain | specialchars )*
|
||||
?content_choice.2: ( old_content | plain_choice | specialchars_choice )*
|
||||
?content_var.2: ( old_content | plain_var | specialchars_var )*
|
||||
?content_negtag.2: ( old_content | plain | specialchars_negtag )*
|
||||
?content_alternate.2: ( old_content | plain_alternate | specialchars_alternate )*
|
||||
?content_en.2: (plain | specialchars )*
|
||||
?content.3: ( old_content | plain | specialchars )*
|
||||
?content_choice.3: ( old_content | plain_choice | specialchars_choice )*
|
||||
?content_var.3: ( old_content | plain_var | specialchars_var )*
|
||||
?content_negtag.3: ( old_content | plain | specialchars_negtag )*
|
||||
?content_alternate.3: ( old_content | plain_alternate | specialchars_alternate )*
|
||||
?content_en.3: (plain | specialchars )*
|
||||
//#endif
|
||||
|
||||
// attention modifiers
|
||||
@@ -93,7 +92,7 @@ scheduled: "[" [ content ":" ] content ":" numpar "]"
|
||||
|
||||
// extra network tags
|
||||
extranetworktag: "<" /(?!ppp:)\w+:/ encontent ">"
|
||||
?encontent.3: content_en
|
||||
?encontent.4: content_en
|
||||
|
||||
//#if ALLOW_COMMVARS or ALLOW_CHOICES or ALLOW_WILDCARDS
|
||||
vardescriptor_get.8: VARNAME [ /\[/ [ SIGNED_NUMBER | IDENTIFIER | separator_descriptor | /#/ ] /\]/ ]
|
||||
@@ -101,36 +100,36 @@ extranetworktag: "<" /(?!ppp:)\w+:/ encontent ">"
|
||||
vardescriptor_set.9: VARNAME [ /\[/ [ SIGNED_NUMBER | IDENTIFIER ] /\]/ ]
|
||||
|
||||
// conditions
|
||||
?condition: grouped_condition | ungrouped_condition
|
||||
?ungrouped_condition: operation | basic_condition
|
||||
?condition: operation_or
|
||||
?operation_or: operation_and | operation_or _WHITESPACE "or" _WHITESPACE operation_and
|
||||
?operation_and: atom_cond | operation_and _WHITESPACE "and" _WHITESPACE atom_cond
|
||||
?atom_cond: operation_not | grouped_condition | basic_condition
|
||||
operation_not: "not" ( ( _WHITESPACE atom_cond ) | ( _WHITESPACE? grouped_condition ) )
|
||||
?grouped_condition.8: "(" _WHITESPACE? condition _WHITESPACE? ")"
|
||||
?basic_condition: comparison | truthy_operand
|
||||
?operation: operation_not | operation_and | operation_or
|
||||
operation_and: condition _WHITESPACE "and" _WHITESPACE condition
|
||||
operation_or: condition _WHITESPACE "or" _WHITESPACE condition
|
||||
operation_not: "not" ( ( _WHITESPACE ungrouped_condition ) | ( _WHITESPACE? grouped_condition ) )
|
||||
?complexvalue: vardescriptor_get | SIMPLEVALUE
|
||||
truthy_operand: vardescriptor_get
|
||||
comparison: ( complexvalue | listvalue ) _WHITESPACE ( /not/ _WHITESPACE )? /eq|ne|lt|gt|le|ge|contains|in|any_in|contains_any/ _WHITESPACE ( complexvalue | listvalue )
|
||||
listvalue.9: "(" ( _WHITESPACE? complexvalue ( _WHITESPACE? "," _WHITESPACE? complexvalue )* )? _WHITESPACE? ")"
|
||||
varvalue: content_var
|
||||
//#endif
|
||||
|
||||
//#if ALLOW_COMMVARS
|
||||
// command: stn (send to negative)
|
||||
commandstn: "<ppp:stn" [ _WHITESPACE /s|e|p\d/ ] _WHITESPACE? ">" content_negtag "<ppp:/stn>"
|
||||
commandstni: "<ppp:stn" _WHITESPACE /i\d/ _WHITESPACE? ( "/>" | ">" )
|
||||
commandstni: "<ppp:stn" _WHITESPACE /i\d/ _WHITESPACE? ( "/>" | ">" ) // ">" ending for backward compatibility
|
||||
|
||||
// command: if
|
||||
commandif.2: commandif_if commandif_elif* commandif_else? "<ppp:/if>"
|
||||
commandif: commandif_if commandif_elif* commandif_else? "<ppp:/if>"
|
||||
commandif_if: "<ppp:if" _WHITESPACE condition _WHITESPACE? ">" ifvalue
|
||||
commandif_elif: "<ppp:elif" _WHITESPACE condition _WHITESPACE? ">" ifvalue
|
||||
commandif_else: "<ppp:else" _WHITESPACE? ">" ifvalue
|
||||
ifvalue.3: content
|
||||
ifvalue: content
|
||||
|
||||
// command: set
|
||||
commandset: "<ppp:set" _WHITESPACE vardescriptor_set [ commandsetmodifiers ] _WHITESPACE? ">" ( starredvalue | commandsetcontent ) "<ppp:/set>"
|
||||
commandsetmodifiers: (_WHITESPACE /evaluate|ifundefined|add/ )+
|
||||
?commandsetcontent.3: content
|
||||
?commandsetcontent: content
|
||||
|
||||
// variable set
|
||||
//#if ALLOW_WILDCARDS
|
||||
@@ -138,25 +137,24 @@ extranetworktag: "<" /(?!ppp:)\w+:/ encontent ">"
|
||||
//#else
|
||||
starredvalue.9: "*" ( vardescriptor_get | listvalue )
|
||||
//#endif
|
||||
variableset.2: "${" _WHITESPACE? vardescriptor_set [ variablesetmodifiers ] _WHITESPACE? "=" [ /!/ ] ( starredvalue | varvalue ) "}"
|
||||
variableset: "${" _WHITESPACE? vardescriptor_set [ variablesetmodifiers ] _WHITESPACE? "=" [ /!/ ] ( starredvalue | varvalue ) "}"
|
||||
variablesetmodifiers: /[+?!]+/
|
||||
|
||||
// command: echo
|
||||
commandecho: "<ppp:echo" _WHITESPACE vardescriptor_get _WHITESPACE? ( ">" [ commandechodefault "<ppp:/echo>" ] | "/>" )
|
||||
?commandechodefault.3: content
|
||||
?commandechodefault: content
|
||||
|
||||
// variable use
|
||||
variableuse.2: "${" _WHITESPACE? vardescriptor_get _WHITESPACE? [ ":" varvalue ] "}"
|
||||
variableuse: "${" _WHITESPACE? vardescriptor_get _WHITESPACE? [ ":" varvalue ] "}"
|
||||
|
||||
// command: ext
|
||||
commandext: "<ppp:ext" _WHITESPACE commandexttype _WHITESPACE commandextid [ _WHITESPACE? commandextparams ] [ _WHITESPACE? commandextif ] _WHITESPACE? ( ">" [ commandexttriggers "<ppp:/ext>" ] | "/>" )
|
||||
commandexttype: [/\$/] IDENTIFIER
|
||||
?commandextid.2: STRING | CNAME
|
||||
?commandextparams.2: STRING | SIGNED_NUMBER
|
||||
?commandextid.3: STRING | CNAME
|
||||
?commandextparams: STRING | SIGNED_NUMBER
|
||||
?commandextif: "if" _WHITESPACE condition
|
||||
?commandexttriggers.3: content
|
||||
?commandexttriggers.4: content
|
||||
//#endif
|
||||
varvalue: content_var
|
||||
|
||||
//#if ALLOW_CHOICES or ALLOW_WILDCARDS
|
||||
// choices parameters (for choices and wildcards)
|
||||
@@ -167,26 +165,28 @@ varvalue: content_var
|
||||
choicesoptions_sep: "$$" plain
|
||||
|
||||
// choice options
|
||||
choice: [ [ _WHITESPACE? choiceiscmd ] [ _WHITESPACE? choicelabels ] [ _WHITESPACE? choiceweight ] [ _WHITESPACE? choiceif ] _WHITESPACE? "::" ] choicevalue
|
||||
choice: [ [ _WHITESPACE? choiceiscmd ] [ _WHITESPACE? choicelabels ] [ _WHITESPACE? choiceweight ] [ _WHITESPACE? ( choiceif | choiceelse ) ] _WHITESPACE? "::" ] choicevalue
|
||||
choiceiscmd: /%/ // the option text is a special command
|
||||
choicelabels: /["']/ IDENTIFIER ( _WHITESPACE? "," _WHITESPACE? IDENTIFIER )* /["']/
|
||||
choicelabels: ( /"/ IDENTIFIER ( _WHITESPACE? "," _WHITESPACE? IDENTIFIER )* /"/ )
|
||||
| ( /'/ IDENTIFIER ( _WHITESPACE? "," _WHITESPACE? IDENTIFIER )* /'/ )
|
||||
choiceweight: NUMBER
|
||||
choiceif: "if" _WHITESPACE condition
|
||||
choiceelse: "else"
|
||||
choicevalue: content_choice
|
||||
//#endif
|
||||
|
||||
//#if ALLOW_CHOICES
|
||||
// choices construct
|
||||
choices.2: "{" [ choicesoptions_sampler | ( choicesoptions _WHITESPACE? "$$" ) ] choice ( "|" choice )* "}"
|
||||
choices: "{" [ choicesoptions_sampler | ( choicesoptions "$$" ) ] choice ( "|" choice )* "}"
|
||||
//#endif
|
||||
|
||||
//#if ALLOW_WILDCARDS
|
||||
// wildcard definition options
|
||||
wcdefoptions: [ choicesoptions_sampler ] [ _WHITESPACE? choicesoptions_flags ] [ _WHITESPACE? choicesoptions_range ] [ _WHITESPACE? wcdescription ] [ _WHITESPACE? choicesoptions_sep ]
|
||||
wcdefoptions: [ choicesoptions_sampler ] [ _WHITESPACE? choicesoptions_flags ] [ _WHITESPACE? choicesoptions_range ] [ _WHITESPACE? wcdescription ] [ _WHITESPACE? choicesoptions_sep ] "$$"
|
||||
wcdescription: STRING
|
||||
|
||||
// wildcards construct
|
||||
wildcard.2: "__" [ choicesoptions_sampler | ( choicesoptions _WHITESPACE? "$$" ) ] wildcard_name [ wc_filter ] [ wildcardvar ] "__"
|
||||
wildcard: "__" [ choicesoptions_sampler | ( choicesoptions "$$" ) ] wildcard_name [ wc_filter ] [ wildcardvar ] "__"
|
||||
wc_filter_nums: IDENTIFIER | INDEX | (INDEX /-/ INDEX)
|
||||
//#if ALLOW_COMMVARS
|
||||
wildcard_name.2: ( WC_NAME_PLAIN_START | variableuse | commandecho ) ( WC_NAME_PLAIN | variableuse | commandecho )*
|
||||
@@ -195,15 +195,18 @@ varvalue: content_var
|
||||
wildcard_name.2: ( WC_NAME_PLAIN_START) WC_NAME_PLAIN*
|
||||
wc_filter_simple: wc_filter_nums
|
||||
//#endif
|
||||
wc_filter: /["']/ ( [ /#/ ] wc_filter_or | ( /#?\^/ wildcard_name ) ) /["']/
|
||||
wc_filter: ( /"/ ( [ /#/ ] wc_filter_or | ( /#?\^/ wildcard_name ) ) /"/ )
|
||||
| ( /'/ ( [ /#/ ] wc_filter_or | ( /#?\^/ wildcard_name ) ) /'/ )
|
||||
wc_filter_or: wc_filter_and ( _WHITESPACE? "," _WHITESPACE? wc_filter_and )*
|
||||
wc_filter_and: wc_filter_simple ( _WHITESPACE? "+" _WHITESPACE? wc_filter_simple )*
|
||||
wildcardvar.7: "(" _WHITESPACE? vardescriptor_get _WHITESPACE? "=" varvalue ")"
|
||||
wildcardvar: "(" _WHITESPACE? vardescriptor_get _WHITESPACE? "=" varvalue ")"
|
||||
|
||||
//#if ALLOW_COMMVARS
|
||||
// command: setwcdeffilter
|
||||
commandsetwcdeffilter: "<ppp:setwcdeffilter" _WHITESPACE wc_name_str [_WHITESPACE wc_filter_limited ] _WHITESPACE? "/>"
|
||||
wc_name_str: /["']/ wildcard_name /["']/
|
||||
wc_filter_limited: /["']/ wc_filter_or /["']/
|
||||
wc_name_str: ( /"/ wildcard_name /"/ )
|
||||
| ( /'/ wildcard_name /'/ )
|
||||
wc_filter_limited: ( /"/ wc_filter_or /"/ )
|
||||
| ( /'/ wc_filter_or /'/ )
|
||||
//#endif
|
||||
//#endif
|
||||
|
||||
+10
-7
@@ -1,13 +1,16 @@
|
||||
import os
|
||||
"""
|
||||
Install dependencies for Prompt Post-Processor extension. For A1111 hosts.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
requirements_filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.txt")
|
||||
requirements_filename = str(Path(__file__).resolve().parent / "requirements.txt")
|
||||
|
||||
try:
|
||||
from modules.launch_utils import requirements_met, run_pip # A1111
|
||||
|
||||
if not requirements_met(requirements_filename):
|
||||
run_pip(f'install -r "{requirements_filename}"', "requirements for Prompt Post-Processor")
|
||||
from modules.launch_utils import run_pip # , requirements_met # A1111
|
||||
#if not requirements_met(requirements_filename): # fails in all hosts due to ruamel.yaml or not having specified versions
|
||||
run_pip(f'install -r "{requirements_filename}"', "requirements for Prompt Post-Processor")
|
||||
except ImportError:
|
||||
import launch
|
||||
|
||||
launch.run_pip(f'install -r "{requirements_filename}"', "requirements for Prompt Post-Processor")
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
+51
-5
@@ -6,17 +6,63 @@
|
||||
}
|
||||
],
|
||||
"settings": {
|
||||
"python.analysis.extraPaths": [
|
||||
"../.."
|
||||
],
|
||||
"python.testing.unittestArgs": [
|
||||
"-v",
|
||||
"-s",
|
||||
".",
|
||||
"-p",
|
||||
"test*.py"
|
||||
],
|
||||
"python.testing.pytestEnabled": false,
|
||||
"python.testing.unittestEnabled": true,
|
||||
"python.analysis.typeCheckingMode": "off",
|
||||
"yaml.schemaStore.enable": false,
|
||||
"powershell.cwd": "sd-webui-prompt-postprocessor",
|
||||
"[python]": {
|
||||
"editor.defaultFormatter": "ms-python.black-formatter"
|
||||
},
|
||||
"yaml.schemaStore.enable": false,
|
||||
"cSpell.enabledFiletypes": [
|
||||
"lark"
|
||||
"black-formatter.args": [
|
||||
"--line-length=120"
|
||||
],
|
||||
"powershell.cwd": "sd-webui-prompt-postprocessor"
|
||||
"python-envs.pythonProjects": [
|
||||
{
|
||||
"path": ".",
|
||||
"envManager": "ms-python.python:venv",
|
||||
"packageManager": "ms-python.python:pip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"launch": {
|
||||
"version": "0.2.0",
|
||||
"configurations": []
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Python Debugger: Attach using Process Id",
|
||||
"type": "debugpy",
|
||||
"request": "attach",
|
||||
"processId": "${command:pickProcess}"
|
||||
},
|
||||
{
|
||||
"name": "Python: Debug Tests (Testing Panel)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"purpose": [
|
||||
"debug-test"
|
||||
],
|
||||
"console": "integratedTerminal",
|
||||
"justMyCode": false,
|
||||
"cwd": "${workspaceFolder}"
|
||||
},
|
||||
{
|
||||
"name": "Python Debugger (Current File)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${file}",
|
||||
"args": "${command:pickArgs}"
|
||||
}
|
||||
],
|
||||
"compounds": []
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,18 @@
|
||||
import csv
|
||||
import dataclasses
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from io import StringIO
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import textwrap
|
||||
import time
|
||||
from typing import Any, Callable, Optional
|
||||
import lark
|
||||
import numpy as np
|
||||
import yaml
|
||||
from ruamel.yaml import YAML as _YAML
|
||||
|
||||
from pydantic import ValidationError
|
||||
from ppp_classes import (
|
||||
@@ -15,6 +20,9 @@ from ppp_classes import (
|
||||
HostConfig,
|
||||
ModelConfig,
|
||||
ModelDetectConfig,
|
||||
PPPEnvInfo,
|
||||
PPPException,
|
||||
RUN_MODE,
|
||||
VariantConfig,
|
||||
PPPConfig,
|
||||
IFWILDCARDS_CHOICES,
|
||||
@@ -22,12 +30,21 @@ from ppp_classes import (
|
||||
PPPInterrupt,
|
||||
PPPState,
|
||||
PPPStateOptions,
|
||||
PPPStateInputs,
|
||||
)
|
||||
from ppp_variables import VariableRepository
|
||||
from ppp_variables import VariableRepository, VariableEntry, VariableValue
|
||||
from ppp_logging import DEBUG_LEVEL, log
|
||||
from ppp_tree import TreeProcessor
|
||||
from ppp_utils import escape_single_quotes
|
||||
from ppp_common import load_grammar, parse_prompt, preprocess_grammar, warn_or_stop
|
||||
from ppp_utils import escape_single_quotes, get_version_from_pyproject
|
||||
from ppp_common import (
|
||||
WARN_STOP_WHERE,
|
||||
clamp_host_bits,
|
||||
get_model_class_from_filename,
|
||||
load_grammar,
|
||||
parse_prompt,
|
||||
preprocess_grammar,
|
||||
warn_or_stop,
|
||||
)
|
||||
from ppp_wildcards import PPPWildcards
|
||||
from ppp_enmappings import PPPExtraNetworkMappings
|
||||
|
||||
@@ -37,26 +54,6 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
The PromptPostProcessor class is responsible for processing and manipulating prompt strings.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_version_from_pyproject() -> str:
|
||||
"""
|
||||
Reads the version from the pyproject.toml file.
|
||||
|
||||
Returns:
|
||||
str: The version string.
|
||||
"""
|
||||
version_str = "0.0.0"
|
||||
try:
|
||||
pyproject_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pyproject.toml")
|
||||
with open(pyproject_path, "r", encoding="utf-8") as file:
|
||||
for line in file:
|
||||
if line.startswith("version = "):
|
||||
version_str = line.split("=")[1].strip().strip('"')
|
||||
break
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logging.getLogger().exception(e)
|
||||
return version_str
|
||||
|
||||
NAME = "Prompt Post-Processor"
|
||||
VERSION = get_version_from_pyproject()
|
||||
|
||||
@@ -84,9 +81,14 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
DEFAULT_CUP_MERGE_ATTENTION = defopt["cup_merge_attention"]
|
||||
DEFAULT_CUP_REMOVE_EXTRANETWORK_TAGS = defopt["cup_remove_extranetwork_tags"]
|
||||
DEFAULT_STRICT_OPERATORS = defopt["strict_operators"]
|
||||
DEFAULT_DO_COMBINATORIAL = defopt["do_combinatorial"]
|
||||
DEFAULT_COMBINATORIAL_SHUFFLE = defopt["combinatorial_shuffle"]
|
||||
DEFAULT_COMBINATORIAL_LIMIT = defopt["combinatorial_limit"]
|
||||
DEFAULT_RUN_MODE = defopt["run_mode"].value
|
||||
DEFAULT_RESULTS_SHUFFLE = defopt["results_shuffle"]
|
||||
DEFAULT_RESULTS_LIMIT = defopt["results_limit"]
|
||||
DEFAULT_COMB_RANDOM_FIXED = defopt["comb_random_fixed"]
|
||||
DEFAULT_DEFAULT_SAMPLER = defopt["default_sampler"].value
|
||||
DEFAULT_RESULTS_FILE = defopt["results_file"]
|
||||
DEFAULT_NEXT_SEED = defopt["next_seed"].value
|
||||
|
||||
WILDCARD_WARNING = '(WARNING TEXT "INVALID WILDCARD" IN BRIGHT RED:1.5)\nBREAK '
|
||||
WILDCARD_STOP = "INVALID WILDCARD! {0}\nBREAK "
|
||||
UNPROCESSED_STOP = "UNPROCESSED CONSTRUCTS!\nBREAK "
|
||||
@@ -94,7 +96,7 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
def __init__(
|
||||
self,
|
||||
logger: logging.Logger,
|
||||
env_info: dict[str, Any],
|
||||
env_info: PPPEnvInfo,
|
||||
options: PPPStateOptions,
|
||||
grammar_content: Optional[str] = None,
|
||||
interrupt: Optional[Callable] = None,
|
||||
@@ -107,7 +109,7 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
Args:
|
||||
logger: The logger object.
|
||||
interrupt: The interrupt function.
|
||||
env_info: A dictionary with information for the environment and loaded model.
|
||||
env_info: Environment and model information.
|
||||
options: The options object for configuring PPP behavior.
|
||||
grammar_content: Optional. The grammar content to be used for parsing.
|
||||
wildcards_obj: Optional. The wildcards object to be used for processing wildcards.
|
||||
@@ -136,8 +138,10 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
|
||||
self.state = PPPState(
|
||||
logger=self.logger,
|
||||
env_info=env_info,
|
||||
host_config=host_config,
|
||||
options=options,
|
||||
inputs=PPPStateInputs(),
|
||||
variables=VariableRepository(),
|
||||
wildcards_obj=wildcards_obj,
|
||||
extranetwork_mappings_obj=extranetwork_mappings_obj,
|
||||
@@ -279,17 +283,17 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
)
|
||||
self.__init_sysvars()
|
||||
|
||||
def log(self, kind, message: str, min_level: DEBUG_LEVEL | None = None, exc_info: bool = False):
|
||||
def log(self, kind, message: str, min_level: DEBUG_LEVEL | None = None, exc_info=None):
|
||||
log(self.logger, self.debug_level, kind, message, min_level, exc_info=exc_info)
|
||||
|
||||
def __load_config_and_detect(self, env_info: dict[str, Any]) -> HostConfig:
|
||||
def __load_config_and_detect(self, env_info: PPPEnvInfo) -> HostConfig:
|
||||
"""Loads config files, performs model detection, and returns the resolved host config."""
|
||||
self.env_info = env_info
|
||||
|
||||
default_config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "ppp_config.yaml.defaults")
|
||||
main_folder = Path(__file__).resolve().parent
|
||||
default_config_file = str(main_folder / "ppp_config.yaml.defaults")
|
||||
_yaml_rt = _YAML()
|
||||
try:
|
||||
with open(default_config_file, "r", encoding="utf-8") as f:
|
||||
default_raw: dict[str, Any] = yaml.safe_load(f)
|
||||
default_raw: dict[str, Any] = _yaml_rt.load(f)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
self.config = {}
|
||||
raise PPPInterrupt(
|
||||
@@ -302,27 +306,67 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
raise PPPInterrupt(errmsg)
|
||||
self.log(logging.WARNING, errmsg)
|
||||
|
||||
user_config_file = self.env_info.get("ppp_config", "")
|
||||
app = env_info.app
|
||||
user_config_file = env_info.ppp_config or ""
|
||||
if isinstance(user_config_file, dict):
|
||||
user_cfg, _ = self.__parse_configuration(user_config_file, "forced configuration")
|
||||
else:
|
||||
user_raw: dict[str, Any] = {}
|
||||
if user_config_file == "":
|
||||
if self.env_info.get("app", "") == SUPPORTED_APPS.comfyui.value:
|
||||
if app == SUPPORTED_APPS.comfyui:
|
||||
try:
|
||||
import folder_paths # type: ignore
|
||||
import folder_paths # type: ignore # pylint: disable=import-outside-toplevel,import-error
|
||||
|
||||
user_dir = folder_paths.get_user_directory()
|
||||
if user_dir and os.path.isdir(user_dir):
|
||||
user_config_file = os.path.join(user_dir, "default", "ppp_config.yaml")
|
||||
if user_dir and Path(user_dir).is_dir():
|
||||
user_config_file = str(Path(user_dir) / "default" / "ppp_config.yaml")
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
self.log(logging.WARNING, "Failed to get user directory for PPP config.")
|
||||
if not user_config_file or not os.path.exists(user_config_file):
|
||||
user_config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "ppp_config.yaml")
|
||||
if user_config_file and os.path.exists(user_config_file):
|
||||
if not user_config_file or not Path(user_config_file).exists():
|
||||
user_config_file = str(main_folder / "ppp_config.yaml")
|
||||
if user_config_file and Path(user_config_file).exists():
|
||||
user_raw: dict[str, Any] = {}
|
||||
with open(user_config_file, "r", encoding="utf-8") as f:
|
||||
user_raw = yaml.safe_load(f)
|
||||
user_raw = _yaml_rt.load(f)
|
||||
user_cfg, _ = self.__parse_configuration(user_raw, "user configuration")
|
||||
default_hosts = set((self.config.hosts or {}).keys())
|
||||
user_hosts = set((user_cfg.hosts or {}).keys())
|
||||
missing_hosts = default_hosts - user_hosts
|
||||
default_models = set((self.config.models or {}).keys())
|
||||
user_models = set((user_cfg.models or {}).keys())
|
||||
missing_models = default_models - user_models
|
||||
if user_raw is not None and (missing_hosts or missing_models):
|
||||
raw_default_hosts = (default_raw or {}).get("hosts") or {}
|
||||
raw_default_models = (default_raw or {}).get("models") or {}
|
||||
if missing_hosts:
|
||||
self.log(
|
||||
logging.INFO,
|
||||
f"Adding missing host(s) from default to user configuration: {', '.join(sorted(missing_hosts))}",
|
||||
)
|
||||
if not user_raw.get("hosts"):
|
||||
user_raw["hosts"] = {}
|
||||
for host in sorted(missing_hosts):
|
||||
if host in raw_default_hosts:
|
||||
user_raw["hosts"][host] = raw_default_hosts[host]
|
||||
if missing_models:
|
||||
self.log(
|
||||
logging.INFO,
|
||||
f"Adding missing model(s) from default to user configuration: {', '.join(sorted(missing_models))}",
|
||||
)
|
||||
if not user_raw.get("models"):
|
||||
user_raw["models"] = {}
|
||||
for model in sorted(missing_models):
|
||||
if model in raw_default_models:
|
||||
user_raw["models"][model] = raw_default_models[model]
|
||||
try:
|
||||
with open(user_config_file, "w", encoding="utf-8") as f:
|
||||
_yaml_rt.dump(user_raw, f)
|
||||
user_cfg, _ = self.__parse_configuration(user_raw, "user configuration")
|
||||
self.log(
|
||||
logging.INFO,
|
||||
f"Saved updated user configuration to '{escape_single_quotes(user_config_file)}'.",
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
self.log(logging.WARNING, f"Failed to save updated user configuration: {exc}")
|
||||
else:
|
||||
user_cfg = None
|
||||
if user_cfg is not None:
|
||||
@@ -332,7 +376,7 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
self.known_models: list[str] = list(self.models_config.keys())
|
||||
|
||||
# Patch for tests (copy comfyui)
|
||||
if self.env_info.get("app", "") == "tests":
|
||||
if app == SUPPORTED_APPS.tests:
|
||||
if self.config.hosts is None:
|
||||
self.config.hosts = {}
|
||||
self.config.hosts.setdefault("tests", HostConfig())
|
||||
@@ -343,30 +387,14 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
model.detect = {}
|
||||
model.detect.setdefault("tests", model.detect.get("comfyui", None))
|
||||
|
||||
host_config: HostConfig | None = (self.config.hosts or {}).get(self.env_info.get("app", ""))
|
||||
host_config: HostConfig | None = (self.config.hosts or {}).get(app.value)
|
||||
if host_config is None:
|
||||
raise PPPInterrupt(
|
||||
f"No host configuration found for app '{escape_single_quotes(self.env_info.get('app', ''))}'. Please check your configuration."
|
||||
f"No host configuration found for app '{escape_single_quotes(app.value)}'. Please check your configuration."
|
||||
)
|
||||
|
||||
# Update env_info with model detection
|
||||
prop_base = self.env_info.get("property_base", None)
|
||||
model_class = self.env_info.get("model_class", "")
|
||||
app = self.env_info.get("app", "")
|
||||
for m in self.known_models:
|
||||
self.env_info["is_" + m] = False
|
||||
model_obj = self.models_config.get(m)
|
||||
model_detect = (model_obj.detect if model_obj else None) or {}
|
||||
model_detect_for_app: ModelDetectConfig | None = model_detect.get(app)
|
||||
if model_detect_for_app is not None:
|
||||
cls_list = model_detect_for_app.class_ or []
|
||||
if model_class in cls_list:
|
||||
self.env_info["is_" + m] = True
|
||||
elif model_detect_for_app.property is not None and prop_base is not None:
|
||||
prop = model_detect_for_app.property
|
||||
attr = getattr(prop_base, prop, None)
|
||||
if isinstance(attr, bool) and attr:
|
||||
self.env_info["is_" + m] = True
|
||||
self.__run_model_detection(env_info)
|
||||
self.variants_definitions: dict[str, tuple[str, list[FindInFilenamePattern]]] = {}
|
||||
for m in self.known_models:
|
||||
model_obj = self.models_config.get(m)
|
||||
@@ -378,24 +406,73 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
logging.WARNING,
|
||||
f"Variant name '{escape_single_quotes(v)}' in model '{escape_single_quotes(m)}' conflicts with a known model name. Discarding variant.",
|
||||
)
|
||||
self.log(logging.DEBUG, f"Host configuration: {host_config}", min_level=DEBUG_LEVEL.minimal)
|
||||
self.log(
|
||||
logging.DEBUG,
|
||||
f"Host configuration ({escape_single_quotes(app.value)}): {host_config}",
|
||||
min_level=DEBUG_LEVEL.minimal,
|
||||
)
|
||||
|
||||
return host_config
|
||||
|
||||
def __run_model_detection(self, env_info: PPPEnvInfo) -> None:
|
||||
"""Updates the is_* model detection flags in env_info based on model_class."""
|
||||
prop_base = env_info.property_base
|
||||
model_class = env_info.model_class
|
||||
model_name = env_info.model_filename
|
||||
app = env_info.app
|
||||
if not model_class and model_name and app == SUPPORTED_APPS.comfyui:
|
||||
try:
|
||||
model_class = get_model_class_from_filename(Path(model_name))
|
||||
except PPPException as e:
|
||||
self.log(
|
||||
logging.WARNING,
|
||||
f"Could not detect model class from filename '{model_name}': {e}",
|
||||
min_level=DEBUG_LEVEL.minimal,
|
||||
)
|
||||
if model_class:
|
||||
env_info.model_class = model_class
|
||||
self.log(
|
||||
logging.DEBUG,
|
||||
f"Detected model class '{model_class}' from filename '{model_name}'",
|
||||
min_level=DEBUG_LEVEL.minimal,
|
||||
)
|
||||
for m in self.known_models:
|
||||
env_info.is_flags[m] = False
|
||||
model_obj = self.models_config.get(m)
|
||||
model_detect = (model_obj.detect if model_obj else None) or {}
|
||||
model_detect_for_app: ModelDetectConfig | None = model_detect.get(app.value)
|
||||
if model_detect_for_app is not None:
|
||||
cls_list = model_detect_for_app.class_ or []
|
||||
if model_class and model_class in cls_list:
|
||||
env_info.is_flags[m] = True
|
||||
elif model_detect_for_app.property is not None and prop_base is not None:
|
||||
prop = model_detect_for_app.property
|
||||
attr = getattr(prop_base, prop, None)
|
||||
if isinstance(attr, bool) and attr:
|
||||
env_info.is_flags[m] = True
|
||||
|
||||
def __on_model_info_update(self) -> None:
|
||||
"""Called when _modelfullname or _modelclass are set via a prompt command."""
|
||||
self.__run_model_detection(self.state.env_info)
|
||||
self.__init_sysvars()
|
||||
self.log(logging.DEBUG, f"Updated system variables: {self.state.variables.all_system}")
|
||||
|
||||
def update(
|
||||
self,
|
||||
env_info: dict[str, Any],
|
||||
env_info: PPPEnvInfo,
|
||||
options: PPPStateOptions,
|
||||
wildcards_obj: PPPWildcards,
|
||||
extranetwork_mappings_obj: PPPExtraNetworkMappings,
|
||||
) -> None:
|
||||
"""Updates env_info, options, wildcards and enmappings while preserving the cyclical state and compiled parsers."""
|
||||
"""Updates env_info, options, wildcards and enmappings while preserving the inputs, cyclical state and compiled parsers."""
|
||||
self.debug_level = options.debug_level
|
||||
host_config = self.__load_config_and_detect(env_info)
|
||||
self.state = PPPState(
|
||||
logger=self.logger,
|
||||
env_info=env_info,
|
||||
host_config=host_config,
|
||||
options=options,
|
||||
inputs=self.state.inputs,
|
||||
variables=VariableRepository(),
|
||||
wildcards_obj=wildcards_obj,
|
||||
extranetwork_mappings_obj=extranetwork_mappings_obj,
|
||||
@@ -423,14 +500,16 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
for model_key, user_model in user_config.models.items():
|
||||
cfg_model = self.config.models.get(model_key)
|
||||
if user_model is None:
|
||||
# User wants to disable this model: remove it
|
||||
if cfg_model is not None:
|
||||
self.config.models.pop(model_key, None)
|
||||
# User wants to disable this model: keep the key as None so _is_* variables
|
||||
# are still set to False (rather than being undefined)
|
||||
self.config.models[model_key] = None
|
||||
elif cfg_model is None:
|
||||
# New model from user config: add with whatever was specified
|
||||
self.config.models[model_key] = user_model
|
||||
else:
|
||||
# Merge detect per-host
|
||||
# model_fields_set contains only fields the user explicitly supplied,
|
||||
# so checking it prevents a missing "detect" key from erasing default detection rules.
|
||||
if "detect" in user_model.model_fields_set and user_model.detect is not None:
|
||||
if cfg_model.detect is None:
|
||||
cfg_model.detect = {}
|
||||
@@ -486,6 +565,9 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
parsed_models: dict[str, ModelConfig | None] = {}
|
||||
raw_models: dict[str, dict | None] = cfg.get("models") or {}
|
||||
for model_key, model_value in raw_models.items():
|
||||
if model_value is None:
|
||||
parsed_models[model_key] = None
|
||||
continue
|
||||
if not isinstance(model_value, dict) or not any(k in model_value for k in ("detect", "variants")):
|
||||
self.logger.warning(
|
||||
f"{where.capitalize()}: Invalid format for model '{escape_single_quotes(model_key)}'. Discarding model."
|
||||
@@ -556,15 +638,24 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
result,
|
||||
)
|
||||
|
||||
def envinfo_hash(self) -> str:
|
||||
"""
|
||||
Generates a hash string based on the environment information.
|
||||
|
||||
Returns:
|
||||
str: A hash string representing the environment information.
|
||||
"""
|
||||
return hash(tuple(sorted(self.env_info.items())))
|
||||
@property
|
||||
def envinfo_hash(self) -> int:
|
||||
"""Returns a hash of the environment information for cache-invalidation purposes."""
|
||||
ei = self.state.env_info
|
||||
ppp_config_key = ei.ppp_config if isinstance(ei.ppp_config, (str, type(None))) else id(ei.ppp_config)
|
||||
return hash(
|
||||
(
|
||||
ei.app,
|
||||
ppp_config_key,
|
||||
ei.model_class,
|
||||
ei.model_filename,
|
||||
ei.models_path,
|
||||
id(ei.property_base),
|
||||
tuple(sorted(ei.is_flags.items())),
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def options_hash(self) -> str:
|
||||
"""
|
||||
Generates a hash string based on the options.
|
||||
@@ -583,31 +674,39 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
Initializes the system variables.
|
||||
"""
|
||||
vs = self.state.variables
|
||||
|
||||
# We keep any existing input variables
|
||||
input_vars = {k: v for k, v in vs.all_system.items() if k.startswith("_input_")}
|
||||
vs.clear_system()
|
||||
|
||||
# Option related variables
|
||||
for opt_name in self.defopt.keys():
|
||||
opt_value = getattr(self.state.options, opt_name)
|
||||
var_name = "_opt_" + opt_name
|
||||
if isinstance(opt_value, (bool, str, int)):
|
||||
if isinstance(opt_value, (bool, str, int, float)):
|
||||
vs.set_system(var_name, opt_value)
|
||||
elif isinstance(opt_value, Enum):
|
||||
vs.set_system(var_name, str(opt_value).split(".", 1)[-1])
|
||||
|
||||
# Model related variables
|
||||
sdchecks = {x: self.env_info.get("is_" + x, False) for x in self.known_models}
|
||||
sdchecks = {x: self.state.env_info.is_flags.get(x, False) for x in self.known_models}
|
||||
# Adding "" as a sentinel that is always True lets next() return "" when no model matches,
|
||||
# giving a well-defined empty-string fallback without a separate None check.
|
||||
sdchecks.update({"": True})
|
||||
model_name_val = next((k for k, v in sdchecks.items() if v), "")
|
||||
vs.set_system("_model", model_name_val)
|
||||
vs.set_system("_sd", model_name_val) # deprecated
|
||||
model_filename = self.env_info.get("model_filename", "")
|
||||
model_filename = self.state.env_info.model_filename
|
||||
vs.set_system("_sdfullname", model_filename) # deprecated
|
||||
vs.set_system("_modelfullname", model_filename)
|
||||
vs.set_system("_sdname", os.path.basename(model_filename)) # deprecated
|
||||
vs.set_system("_modelname", os.path.basename(model_filename))
|
||||
vs.set_system("_modelclass", self.env_info.get("model_class", ""))
|
||||
vs.set_system("_sdname", Path(model_filename).name) # deprecated
|
||||
vs.set_system("_modelname", Path(model_filename).name)
|
||||
vs.set_system("_modelclass", self.state.env_info.model_class)
|
||||
is_models = {}
|
||||
for model_name, model_type_and_substrings in self.variants_definitions.items():
|
||||
# A variant is only active when its parent model type is currently loaded
|
||||
# (or when the variant has no parent restriction, indicated by an empty string).
|
||||
# If the parent is not active, short-circuit to False before testing filename patterns.
|
||||
if not (model_type_and_substrings[0] == "" or sdchecks.get(model_type_and_substrings[0], False)):
|
||||
is_models[model_name] = False
|
||||
else:
|
||||
@@ -625,15 +724,25 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
for x in sdchecks.keys():
|
||||
if x != "":
|
||||
vs.set_system("_is_" + x, sdchecks[x])
|
||||
# _is_pure_X: model X is active but no named variant matched the filename
|
||||
# _is_variant_X: model X is active AND at least one named variant matched
|
||||
vs.set_system("_is_pure_" + x, sdchecks[x] and not any(is_models.values()))
|
||||
vs.set_system("_is_variant_" + x, sdchecks[x] and any(is_models.values()))
|
||||
# special cases
|
||||
vs.set_system("_is_sd", sdchecks["sd1"] or sdchecks["sd2"] or sdchecks["sdxl"] or sdchecks["sd3"])
|
||||
is_ssd = self.env_info.get("is_ssd", False)
|
||||
vs.set_system(
|
||||
"_is_sd",
|
||||
sdchecks.get("sd1", False)
|
||||
or sdchecks.get("sd2", False)
|
||||
or sdchecks.get("sdxl", False)
|
||||
or sdchecks.get("sd3", False),
|
||||
)
|
||||
is_ssd = self.state.env_info.is_flags.get("ssd", False)
|
||||
vs.set_system("_is_ssd", is_ssd)
|
||||
vs.set_system("_is_sdxl_no_ssd", sdchecks["sdxl"] and not is_ssd)
|
||||
vs.set_system("_is_sdxl_no_ssd", sdchecks.get("sdxl", False) and not is_ssd)
|
||||
# backcompatibility (but the modern one to use would be _is_pure_sdxl)
|
||||
vs.set_system("_is_sdxl_no_pony", sdchecks["sdxl"] and not vs.get_system("_is_pony", False))
|
||||
vs.set_system("_is_sdxl_no_pony", sdchecks.get("sdxl", False) and not vs.get_system("_is_pony", False))
|
||||
|
||||
vs.update_system(input_vars)
|
||||
|
||||
def init_wildcards_options(self):
|
||||
"""Initializes the wildcard options."""
|
||||
@@ -686,7 +795,11 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
self.log(logging.DEBUG, f"BREAK construct {break_replacements[break_processing][0]}")
|
||||
elif break_processing == "error":
|
||||
if re.search(r"\bBREAK\b", text):
|
||||
warn_or_stop(self.state, where == -1, "BREAK constructs are not allowed!")
|
||||
warn_or_stop(
|
||||
self.state,
|
||||
WARN_STOP_WHERE.negative if where == -1 else WARN_STOP_WHERE.positive,
|
||||
"BREAK constructs are not allowed!",
|
||||
)
|
||||
|
||||
if self.state.options.cup_ands:
|
||||
# collapse ANDs with space after
|
||||
@@ -703,6 +816,8 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
text = re.sub(r"(\s*\bAND)+\Z", "", text)
|
||||
|
||||
escapedSeparator = re.escape(self.state.options.stn_separator)
|
||||
# When EOL inclusion is on, use \s* so newlines are treated as whitespace around separators.
|
||||
# Otherwise, restrict to horizontal whitespace only to preserve intentional line breaks.
|
||||
optwhitespace = r"\s*" if self.state.options.cup_extra_separators_include_eol else r"[ \t\v\f]*"
|
||||
optwhitespace_separator = optwhitespace + escapedSeparator + optwhitespace
|
||||
optwhitespace_comma = optwhitespace + "," + optwhitespace
|
||||
@@ -821,10 +936,11 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
|
||||
def __postprocess_result(
|
||||
self,
|
||||
result: tuple[str, list[tuple[str, bool]], tuple[dict[str, str | None], dict[str, str | None]]],
|
||||
result: tuple[str, list[tuple[str, bool]], dict[str, VariableEntry]],
|
||||
) -> tuple[str, str, dict[str, str | None]]:
|
||||
variables = {}
|
||||
unified_prompt, rem_wildcards, (_, echoed_variables_snapshot) = result
|
||||
all_variables = self.state.variables.all_system
|
||||
unechoed_variables: list[str] = []
|
||||
unified_prompt, rem_wildcards, variables_snapshot = result
|
||||
|
||||
# Split the unified prompt back into prompt and negative prompt
|
||||
split_parts = unified_prompt.split("\x1d", 1)
|
||||
@@ -838,20 +954,36 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
self.log(logging.INFO, f"Result prompt: {prompt}")
|
||||
self.log(logging.INFO, f"Result negative_prompt: {negative_prompt}")
|
||||
try:
|
||||
# Get and clean variables
|
||||
var_keys = sorted(echoed_variables_snapshot.keys())
|
||||
# Get and clean variables - prefer the explicitly echoed value; fall back to the evaluated value.
|
||||
var_keys = sorted(variables_snapshot.keys())
|
||||
for k in var_keys:
|
||||
ev = echoed_variables_snapshot.get(k)
|
||||
variables[k] = self.__cleanup(ev, 0) if self.state.options.cup_cleanup_variables else ev
|
||||
entry = variables_snapshot[k]
|
||||
if (
|
||||
entry.last_echoed_evaluated_value is None
|
||||
and entry.value is not None
|
||||
and not k.startswith("_output_")
|
||||
):
|
||||
unechoed_variables.append(k)
|
||||
ev = entry.last_echoed_evaluated_value if entry.last_echoed_evaluated_value is not None else entry.value
|
||||
if ev is not None:
|
||||
if isinstance(ev, str) and self.state.options.cup_cleanup_variables:
|
||||
ev = self.__cleanup(ev, 0)
|
||||
all_variables[k] = ev
|
||||
|
||||
self.log(logging.DEBUG, f"Result variables: {variables}")
|
||||
self.log(logging.INFO, f"Result variables: {all_variables}")
|
||||
if unechoed_variables:
|
||||
unechoed_values = {k: v for k, v in all_variables.items() if k in unechoed_variables}
|
||||
self.log(
|
||||
logging.INFO,
|
||||
f"Variables that were never echoed: {unechoed_values}",
|
||||
)
|
||||
|
||||
# Result checks
|
||||
warnings = []
|
||||
|
||||
# Check for special character sequences that should not be in the result
|
||||
compound_prompt = prompt + "\n" + negative_prompt
|
||||
found_sequences = re.findall(r"::|\$\$|\$\{|[{}]", compound_prompt)
|
||||
found_sequences = re.findall(r"::|\$\$|\$\{|[{}]|__", compound_prompt)
|
||||
if found_sequences:
|
||||
s = ", ".join(map(lambda x: '"' + x + '"', set(found_sequences)))
|
||||
warnings.append(f"Probably invalid character sequences: {s}.")
|
||||
@@ -878,8 +1010,7 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
if warnings:
|
||||
self.log(
|
||||
logging.WARNING,
|
||||
"Found some weird things in the result. Something might be wrong!\n"
|
||||
+ "\n".join(f" - {w}" for w in warnings),
|
||||
"Found some weird things in the result. Something might be wrong!: " + ", ".join(warnings),
|
||||
)
|
||||
|
||||
# Check for wildcards not processed
|
||||
@@ -919,29 +1050,72 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
self.log(logging.ERROR, "Interrupting!")
|
||||
self.interrupt()
|
||||
|
||||
v = self.state.variables.get_all_system()
|
||||
v.update(variables)
|
||||
return prompt, negative_prompt, v
|
||||
return prompt, negative_prompt, all_variables
|
||||
|
||||
def __processprompts(
|
||||
self, rng: np.random.Generator, prompt: str, negative_prompt: str
|
||||
) -> list[tuple[str, str, dict[str, str | None]]]:
|
||||
self,
|
||||
prompt: str,
|
||||
negative_prompt: str,
|
||||
starting_seed: int | list[int],
|
||||
jobinfo: Any = None,
|
||||
input_vars: dict[str, Any] | None = None,
|
||||
) -> list[tuple[str, str, dict[str, Any]]]:
|
||||
"""
|
||||
Process the prompt and negative prompt.
|
||||
|
||||
Args:
|
||||
rng (numpy.random.Generator): The random number generator.
|
||||
prompt (str): The prompt.
|
||||
negative_prompt (str): The negative prompt.
|
||||
starting_seed (int | list[int]): The starting seed for the random number generator.
|
||||
jobinfo (Any): Additional job information to be stored in the input state.
|
||||
input_vars (dict[str, Any] | None): Additional input variables to be set as system variables with the "_input_" prefix.
|
||||
|
||||
Returns:
|
||||
list: A list of tuples, each containing the processed prompt, negative prompt, and all variables.
|
||||
list[tuple[str, str, dict[str, Any]]]: A list of tuples, each containing the processed prompt, negative prompt, and all variables.
|
||||
"""
|
||||
self.state.variables.clear_user()
|
||||
self.state.variables.clear_echoed()
|
||||
|
||||
# We update the input state
|
||||
self.state.inputs.seed = (
|
||||
clamp_host_bits(self.state.host_config.seed_bits, starting_seed)
|
||||
if not isinstance(starting_seed, list)
|
||||
else [clamp_host_bits(self.state.host_config.seed_bits, s) for s in starting_seed]
|
||||
)
|
||||
self.state.inputs.pos_prompt = prompt
|
||||
self.state.inputs.neg_prompt = negative_prompt
|
||||
self.state.inputs.jobinfo = jobinfo
|
||||
|
||||
# Input related system variables
|
||||
if input_vars:
|
||||
for k, v in input_vars.items():
|
||||
self.state.variables.set_system("_input_" + k, v)
|
||||
for input_name in self.state.inputs.__dict__.keys():
|
||||
input_value = getattr(self.state.inputs, input_name)
|
||||
var_name = "_input_" + input_name
|
||||
if input_value is None:
|
||||
self.state.variables.set_system(var_name, None)
|
||||
elif isinstance(input_value, VariableValue):
|
||||
self.state.variables.set_system(var_name, input_value)
|
||||
elif isinstance(input_value, dict):
|
||||
for k, v in input_value.items():
|
||||
self.state.variables.set_system(f"{var_name}_{k}", str(v))
|
||||
elif isinstance(input_value, Enum):
|
||||
self.state.variables.set_system(var_name, str(input_value).split(".", 1)[-1])
|
||||
else:
|
||||
self.log(
|
||||
logging.WARNING,
|
||||
f"Input '{input_name}' has an unsupported type {type(input_value).__name__} for a system variable and will be skipped.",
|
||||
)
|
||||
|
||||
filtered_sysvars_inputs = {k: v for k, v in self.state.variables.all_system.items() if k.startswith("_input_")}
|
||||
self.log(logging.INFO, f"Inputs: {filtered_sysvars_inputs}")
|
||||
|
||||
rng = np.random.default_rng(
|
||||
self.state.inputs.seed if not isinstance(self.state.inputs.seed, list) else self.state.inputs.seed[0]
|
||||
)
|
||||
|
||||
# Parse both prompts
|
||||
processor = TreeProcessor(self.state, rng)
|
||||
processor = TreeProcessor(self.state, rng, on_model_info_update=self.__on_model_info_update)
|
||||
# We use the ASCII Group Separator character between prompt and negative prompt since it's unlikely to appear in prompts
|
||||
unified_prompt = prompt + "\x1d" + negative_prompt
|
||||
prompt_parser, parser_description = self.__get_best_parser(unified_prompt)
|
||||
@@ -969,56 +1143,185 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
t2 = time.monotonic_ns()
|
||||
self.log(logging.INFO, f"Visit time: {(t2 - t1) / 1_000_000_000:.3f} seconds")
|
||||
|
||||
final_results = []
|
||||
final_results: list[tuple[str, str, dict[str, Any]]] = []
|
||||
for i, r in enumerate(results):
|
||||
if self.state.options.do_combinatorial:
|
||||
if self.state.options.run_mode == RUN_MODE.combinatorial:
|
||||
self.log(logging.INFO, f"Combination {i + 1}:")
|
||||
elif self.state.options.run_mode == RUN_MODE.multiple:
|
||||
self.log(logging.INFO, f"Result {i + 1}:")
|
||||
final_results.append(self.__postprocess_result(r))
|
||||
if self.state.options.do_combinatorial:
|
||||
if self.state.options.run_mode == RUN_MODE.combinatorial:
|
||||
self.log(logging.INFO, f"Total combinations: {len(final_results)}")
|
||||
if self.state.options.combinatorial_shuffle:
|
||||
rng.shuffle(final_results)
|
||||
self.log(logging.INFO, "Combinations shuffled")
|
||||
if self.state.options.results_shuffle:
|
||||
rng.shuffle(final_results)
|
||||
self.log(logging.INFO, "Results shuffled")
|
||||
return final_results
|
||||
|
||||
def process_prompts_group_start(self):
|
||||
"""Start of a prompt processing group."""
|
||||
self.log(logging.INFO, f"System variables: {self.state.variables.get_all_system()}")
|
||||
self.log(logging.INFO, f"Combinatorial: {self.state.options.do_combinatorial}")
|
||||
filtered_sysvars = {
|
||||
k: v for k, v in self.state.variables.all_system.items() if not k.startswith(("_input_", "_output_"))
|
||||
}
|
||||
self.log(logging.DEBUG, f"System variables: {filtered_sysvars}", DEBUG_LEVEL.minimal)
|
||||
self.log(logging.INFO, f"Run mode: {self.state.options.run_mode.name}")
|
||||
if self.state.options.run_mode == RUN_MODE.combinatorial:
|
||||
if self.state.options.results_limit > 0:
|
||||
self.log(logging.INFO, f"Up to {self.state.options.results_limit} combinations")
|
||||
else:
|
||||
self.log(logging.INFO, "No combinations limit")
|
||||
elif self.state.options.run_mode == RUN_MODE.multiple:
|
||||
if self.state.options.results_limit < 1:
|
||||
self.state.options.results_limit = 1
|
||||
self.log(logging.INFO, f"Returning {self.state.options.results_limit} results")
|
||||
|
||||
def _expand_filename(self) -> Path:
|
||||
"""Expand %...% tokens in a filename template and resolve relative paths against the extension logs folder."""
|
||||
now = datetime.now()
|
||||
substitutions = {
|
||||
r"%datetime%": now.strftime(r"%Y-%m-%d_%H-%M-%S"),
|
||||
r"%date%": now.strftime(r"%Y-%m-%d"),
|
||||
r"%time%": now.strftime(r"%H-%M-%S"),
|
||||
r"%host%": self.state.env_info.app.value,
|
||||
}
|
||||
result = str(self.state.options.results_file)
|
||||
for token, value in substitutions.items():
|
||||
result = result.replace(token, value)
|
||||
path = Path(result)
|
||||
if not path.is_absolute():
|
||||
path = Path(__file__).resolve().parent / "logs" / path
|
||||
return path
|
||||
|
||||
def __save_results(self, results: list[tuple[str, str, dict[str, Any]]]) -> None:
|
||||
"""Append processing results to the configured results file."""
|
||||
if not self.state.options.results_file:
|
||||
return
|
||||
try:
|
||||
filepath = self._expand_filename()
|
||||
self.log(logging.INFO, f"Saving results to file: {filepath}")
|
||||
ext = filepath.suffix.lower()
|
||||
records = []
|
||||
for result_prompt, result_neg_prompt, all_variables in results:
|
||||
records.append(
|
||||
{
|
||||
"options": {
|
||||
k.removeprefix("_opt_"): v for k, v in all_variables.items() if k.startswith("_opt_")
|
||||
},
|
||||
"system_variables": {
|
||||
k: v
|
||||
for k, v in all_variables.items()
|
||||
if k.startswith("_")
|
||||
and not k.startswith("_opt_")
|
||||
and not k.startswith(("_input_", "_output_"))
|
||||
},
|
||||
"inputs": {
|
||||
k.removeprefix("_input_"): v for k, v in all_variables.items() if k.startswith("_input_")
|
||||
},
|
||||
"outputs": {
|
||||
k.removeprefix("_output_"): v for k, v in all_variables.items() if k.startswith("_output_")
|
||||
},
|
||||
"prompt_results": {"prompt": result_prompt, "negative_prompt": result_neg_prompt},
|
||||
"user_variables": {k: v for k, v in all_variables.items() if not k.startswith("_")},
|
||||
}
|
||||
)
|
||||
filepath.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_exists = filepath.exists()
|
||||
if ext in (".yaml", ".yml"):
|
||||
with open(filepath, "a", encoding="utf-8-sig") as f:
|
||||
if not file_exists:
|
||||
f.write("records:\n")
|
||||
_yaml_dump = _YAML()
|
||||
_yaml_dump.default_flow_style = False
|
||||
for record in records:
|
||||
_sio = StringIO()
|
||||
_yaml_dump.dump(record, _sio)
|
||||
y = _sio.getvalue()
|
||||
f.write(f" - {textwrap.indent(y, ' ' * 4).strip()}\n")
|
||||
elif ext == ".jsonl":
|
||||
with open(filepath, "a", encoding="utf-8-sig") as f:
|
||||
for record in records:
|
||||
f.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
|
||||
elif ext == ".csv":
|
||||
# Build ordered column list from the current batch - header written only on new file.
|
||||
# Columns from later calls that weren't in the original header will be silently dropped.
|
||||
# The 'variables' section is serialized as a single JSON string column.
|
||||
all_columns: list[str] = []
|
||||
for record in records:
|
||||
for section, data in record.items():
|
||||
if section == "user_variables":
|
||||
if "user_variables" not in all_columns:
|
||||
all_columns.append("user_variables")
|
||||
else:
|
||||
for k in data:
|
||||
col = f"{section}.{k}"
|
||||
if col not in all_columns:
|
||||
all_columns.append(col)
|
||||
with open(filepath, "a", newline="", encoding="utf-8-sig") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=all_columns, delimiter=";", restval="", extrasaction="ignore")
|
||||
if not file_exists:
|
||||
writer.writeheader()
|
||||
for record in records:
|
||||
row: dict[str, Any] = {}
|
||||
for section, data in record.items():
|
||||
if section == "user_variables":
|
||||
row["user_variables"] = json.dumps(data, ensure_ascii=False, default=str)
|
||||
else:
|
||||
for k, v in data.items():
|
||||
row[f"{section}.{k}"] = (
|
||||
v if isinstance(v, (str, int, float, bool)) or v is None else str(v)
|
||||
)
|
||||
writer.writerow(row)
|
||||
else: # plain text
|
||||
with open(filepath, "a", encoding="utf-8-sig") as f:
|
||||
for record in records:
|
||||
for section, data in record.items():
|
||||
f.write(f"[{section}]\n")
|
||||
for k, v in data.items():
|
||||
f.write(f"{k}: {v}\n")
|
||||
f.write(f"#{'-'*70}\n")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
self.log(logging.WARNING, "Failed to save results to file", exc_info=e)
|
||||
|
||||
def process_prompt(
|
||||
self,
|
||||
original_prompt: str,
|
||||
original_negative_prompt: str,
|
||||
seed: int = 0,
|
||||
):
|
||||
starting_seed: int | list[int] = -1,
|
||||
jobinfo: Any = None,
|
||||
input_vars: dict[str, Any] | None = None,
|
||||
) -> list[tuple[str, str, dict[str, Any]]]:
|
||||
"""
|
||||
Initializes the random number generator and processes the prompt and negative prompt.
|
||||
|
||||
Args:
|
||||
original_prompt (str): The original prompt.
|
||||
original_negative_prompt (str): The original negative prompt.
|
||||
seed (int): The seed.
|
||||
starting_seed (int | list[int]): The starting seed or list of starting seeds.
|
||||
jobinfo (Any): Optional job information, available as `_input_jobinfo`.
|
||||
input_vars (dict[str, Any] | None): Optional dictionary of input variables to set before processing.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing the processed prompt, negative prompt and all the prompt variables.
|
||||
list[tuple[str, str, dict[str, Any]]]: A list of tuples containing the processed prompt, negative prompt and all the prompt variables.
|
||||
"""
|
||||
results: list[tuple[str, str, dict[str, Any]]]
|
||||
try:
|
||||
if seed == -1:
|
||||
seed = np.random.randint(0, 2**32, dtype=np.int64)
|
||||
if isinstance(starting_seed, list):
|
||||
starting_seed = [
|
||||
np.random.randint(0, 1 << (self.state.host_config.seed_bits - 1), dtype=np.int64) if s == -1 else s
|
||||
for s in starting_seed
|
||||
]
|
||||
elif starting_seed == -1:
|
||||
starting_seed = np.random.randint(0, 1 << (self.state.host_config.seed_bits - 1), dtype=np.int64)
|
||||
prompt = original_prompt
|
||||
negative_prompt = original_negative_prompt
|
||||
self.log(logging.INFO, f"Input seed: {seed}")
|
||||
self.log(logging.INFO, f"Input prompt: {prompt}")
|
||||
self.log(logging.INFO, f"Input negative_prompt: {negative_prompt}")
|
||||
t1 = time.monotonic_ns()
|
||||
if self.state.cyclical_state.last_prompt_pair != (original_prompt, original_negative_prompt):
|
||||
self.state.cyclical_state.reset()
|
||||
self.state.cyclical_state.last_prompt_pair = (original_prompt, original_negative_prompt)
|
||||
results = self.__processprompts(np.random.default_rng(seed & 0xFFFFFFFF), prompt, negative_prompt)
|
||||
results = self.__processprompts(prompt, negative_prompt, starting_seed, jobinfo, input_vars or {})
|
||||
t2 = time.monotonic_ns()
|
||||
self.log(logging.INFO, f"Process prompt pair time: {(t2 - t1) / 1_000_000_000:.3f} seconds")
|
||||
# self.log(logging.DEBUG,f"Wildcards memory usage: {self.state.wildcards_obj.__sizeof__()}")
|
||||
self.__save_results(results)
|
||||
return results
|
||||
except PPPInterrupt as e:
|
||||
self.log(logging.ERROR, e.message)
|
||||
@@ -1029,8 +1332,8 @@ class PromptPostProcessor: # pylint: disable=too-few-public-methods,too-many-in
|
||||
self.log(logging.ERROR, "Interrupting!")
|
||||
self.interrupt()
|
||||
return [(prompt, negative_prompt, {})]
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
self.log(logging.ERROR, "Unexpected error", exc_info=True)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
self.log(logging.ERROR, "Unexpected error", exc_info=e)
|
||||
return [(original_prompt, original_negative_prompt, {})]
|
||||
|
||||
def process_prompts_group_end(self):
|
||||
|
||||
+77
-6
@@ -4,7 +4,7 @@ from dataclasses import dataclass, field
|
||||
from logging import Logger
|
||||
import re
|
||||
from enum import Enum
|
||||
from typing import Literal, Optional
|
||||
from typing import Any, Literal, Optional
|
||||
from lark import Lark
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
@@ -18,6 +18,7 @@ class SUPPORTED_APPS(Enum):
|
||||
comfyui = "comfyui"
|
||||
a1111 = "a1111"
|
||||
forge = "forge"
|
||||
forgeneo = "forgeneo"
|
||||
reforge = "reforge"
|
||||
sdnext = "sdnext"
|
||||
tests = "tests" # for testing purposes only, not a real app
|
||||
@@ -26,7 +27,8 @@ class SUPPORTED_APPS(Enum):
|
||||
SUPPORTED_APPS_NAMES = {
|
||||
SUPPORTED_APPS.comfyui: "ComfyUI",
|
||||
SUPPORTED_APPS.sdnext: "SD.Next",
|
||||
SUPPORTED_APPS.forge: "Forge",
|
||||
SUPPORTED_APPS.forge: "Forge Classic",
|
||||
SUPPORTED_APPS.forgeneo: "Forge Neo",
|
||||
SUPPORTED_APPS.reforge: "reForge",
|
||||
SUPPORTED_APPS.a1111: "A1111 (or compatible)",
|
||||
SUPPORTED_APPS.tests: "Tests",
|
||||
@@ -44,6 +46,25 @@ class ONWARNING_CHOICES(Enum):
|
||||
warn = "warn"
|
||||
stop = "stop"
|
||||
|
||||
|
||||
class RUN_MODE(Enum):
|
||||
single = "single"
|
||||
multiple = "multiple"
|
||||
combinatorial = "combinatorial"
|
||||
|
||||
|
||||
class DEFAULT_SAMPLER(Enum):
|
||||
random = "random"
|
||||
cyclical = "cyclical"
|
||||
|
||||
|
||||
class NEXT_SEED(Enum):
|
||||
randomize = "randomize"
|
||||
input = "input"
|
||||
increment = "increment"
|
||||
decrement = "decrement"
|
||||
|
||||
|
||||
# ------------------- Host configuration -------------------
|
||||
|
||||
AttentionOption = Literal["ok", "parentheses", "disable", "remove", "error"]
|
||||
@@ -63,6 +84,9 @@ class HostConfig(BaseModel):
|
||||
alternation: AlternationOption = "ok"
|
||||
and_: AndOption = Field("ok", alias="and")
|
||||
break_: BreakOption = Field("ok", alias="break")
|
||||
seed_bits: int = Field(
|
||||
64, validator=lambda v: v if v in (8, 16, 32, 64) else ValueError("seed_bits must be one of 8, 16, 32, or 64")
|
||||
)
|
||||
|
||||
|
||||
# ------------------- Model detection -------------------
|
||||
@@ -172,8 +196,10 @@ class PPPConfig(BaseModel):
|
||||
raise ValueError("At least one of 'hosts' or 'models' must be specified")
|
||||
return self
|
||||
|
||||
|
||||
# ------------------- State object -------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PPPStateOptions:
|
||||
"""Options that can be set for prompt processing."""
|
||||
@@ -201,9 +227,13 @@ class PPPStateOptions:
|
||||
cup_merge_attention: bool = True
|
||||
cup_remove_extranetwork_tags: bool = False
|
||||
strict_operators: bool = True
|
||||
do_combinatorial: bool = False
|
||||
combinatorial_shuffle: bool = False
|
||||
combinatorial_limit: int = 100 # 0 = no limit
|
||||
results_file: str = "" # empty = disabled; supports %datetime%, %date%, %time%, %host% tokens
|
||||
run_mode: RUN_MODE = RUN_MODE.single
|
||||
results_limit: int = 100 # 0 = no limit
|
||||
results_shuffle: bool = False
|
||||
comb_random_fixed: bool = True # if True, the random sampler will be fixed across all DFS runs
|
||||
default_sampler: DEFAULT_SAMPLER = DEFAULT_SAMPLER.random
|
||||
next_seed: NEXT_SEED = NEXT_SEED.randomize # how to determine the next seed for each prompt
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.cup_do_cleanup:
|
||||
@@ -221,6 +251,17 @@ class PPPStateOptions:
|
||||
object.__setattr__(self, "cup_merge_attention", False)
|
||||
object.__setattr__(self, "cup_remove_extranetwork_tags", False)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PPPStateInputs:
|
||||
"""Structured inputs for a single prompt processing call."""
|
||||
|
||||
seed: int | list[int] = -1
|
||||
pos_prompt: str = ""
|
||||
neg_prompt: str = ""
|
||||
jobinfo: Any = None
|
||||
|
||||
|
||||
class CyclicalSamplerState:
|
||||
"""Maintains the cycling position for '@' choice samplers across process_prompt calls."""
|
||||
|
||||
@@ -252,13 +293,33 @@ class CyclicalSamplerState:
|
||||
self.last_prompt_pair = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PPPEnvInfo:
|
||||
"""Environment and model information passed to PPP at construction time."""
|
||||
|
||||
app: SUPPORTED_APPS = SUPPORTED_APPS.a1111
|
||||
ppp_config: str | dict | None = None
|
||||
model_class: str = ""
|
||||
model_filename: str = ""
|
||||
property_base: Any = None
|
||||
models_path: str = ""
|
||||
_is_flags: dict[str, bool] = field(default_factory=dict, init=False, repr=False)
|
||||
|
||||
@property
|
||||
def is_flags(self) -> dict[str, bool]:
|
||||
"""Boolean model-detection flags keyed by model name (e.g. 'sdxl' -> True)."""
|
||||
return self._is_flags
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PPPState:
|
||||
"""State object passed to various PPP components during prompt processing."""
|
||||
|
||||
logger: Logger
|
||||
env_info: PPPEnvInfo = field(default_factory=PPPEnvInfo)
|
||||
host_config: HostConfig = field(default_factory=HostConfig)
|
||||
options: PPPStateOptions = field(default_factory=PPPStateOptions)
|
||||
inputs: PPPStateInputs = field(default_factory=PPPStateInputs)
|
||||
variables: VariableRepository = field(default_factory=VariableRepository)
|
||||
wildcards_obj: PPPWildcards = field(default_factory=PPPWildcards)
|
||||
extranetwork_mappings_obj: PPPExtraNetworkMappings = field(default_factory=PPPExtraNetworkMappings)
|
||||
@@ -266,7 +327,17 @@ class PPPState:
|
||||
cyclical_state: CyclicalSamplerState = field(default_factory=CyclicalSamplerState)
|
||||
|
||||
|
||||
class PPPInterrupt(Exception):
|
||||
class PPPException(Exception):
|
||||
"""
|
||||
Custom exception to handle exceptions in the PromptPostProcessor.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str = "An error occurred during prompt processing."):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
|
||||
|
||||
class PPPInterrupt(PPPException):
|
||||
"""
|
||||
Custom exception to handle interruptions in the PromptPostProcessor.
|
||||
This exception can be raised to stop the processing of prompts.
|
||||
|
||||
+246
-89
@@ -1,21 +1,36 @@
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit("This script must be run from ComfyUI")
|
||||
|
||||
# pylint: disable=wrong-import-position,wrong-import-order
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import folder_paths # type: ignore
|
||||
import nodes # type: ignore
|
||||
import folder_paths # type: ignore # pylint: disable=import-error
|
||||
import nodes # type: ignore # pylint: disable=import-error
|
||||
|
||||
from ppp import PromptPostProcessor
|
||||
from ppp_classes import IFWILDCARDS_CHOICES, ONWARNING_CHOICES, SUPPORTED_APPS, PPPStateOptions
|
||||
from ppp_classes import (
|
||||
DEFAULT_SAMPLER,
|
||||
IFWILDCARDS_CHOICES,
|
||||
ONWARNING_CHOICES,
|
||||
PPPEnvInfo,
|
||||
SUPPORTED_APPS,
|
||||
PPPException,
|
||||
RUN_MODE,
|
||||
NEXT_SEED,
|
||||
PPPStateOptions,
|
||||
)
|
||||
from ppp_common import get_model_class_from_filename, load_grammar
|
||||
from ppp_logging import DEBUG_LEVEL, PromptPostProcessorLogFactory, log
|
||||
from ppp_utils import escape_single_quotes
|
||||
from ppp_wildcards import PPPWildcards
|
||||
from ppp_enmappings import PPPExtraNetworkMappings
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit("This script must be run from ComfyUI")
|
||||
|
||||
|
||||
def _resolve_wildcards_folders(override: str = "") -> list[str]:
|
||||
def _resolve_wildcards_folders(override: str = "") -> list[Path]:
|
||||
"""Return the resolved list of wildcard folder paths."""
|
||||
folders_str = override
|
||||
if folders_str == "":
|
||||
@@ -31,13 +46,13 @@ def _resolve_wildcards_folders(override: str = "") -> list[str]:
|
||||
if folders_str == "":
|
||||
folders_str = os.getenv("WILDCARD_DIR", PPPWildcards.DEFAULT_WILDCARDS_FOLDER)
|
||||
return [
|
||||
(f if os.path.isabs(f) else os.path.abspath(os.path.join(folder_paths.models_dir, f)))
|
||||
(Path(f) if Path(f).is_absolute() else (Path(folder_paths.models_dir) / f).resolve())
|
||||
for f in folders_str.split(",")
|
||||
if f.strip() != ""
|
||||
]
|
||||
|
||||
|
||||
def _resolve_enmappings_folders(override: str = "") -> list[str]:
|
||||
def _resolve_enmappings_folders(override: str = "") -> list[Path]:
|
||||
"""Return the resolved list of extra-network mapping folder paths."""
|
||||
folders_str = override
|
||||
if folders_str == "":
|
||||
@@ -49,7 +64,7 @@ def _resolve_enmappings_folders(override: str = "") -> list[str]:
|
||||
if folders_str == "":
|
||||
folders_str = os.getenv("EXTRANETWORKMAPPINGS_DIR", PPPExtraNetworkMappings.DEFAULT_ENMAPPINGS_FOLDER)
|
||||
return [
|
||||
(f if os.path.isabs(f) else os.path.abspath(os.path.join(folder_paths.models_dir, f)))
|
||||
(Path(f) if Path(f).is_absolute() else (Path(folder_paths.models_dir) / f).resolve())
|
||||
for f in folders_str.split(",")
|
||||
if f.strip() != ""
|
||||
]
|
||||
@@ -60,14 +75,10 @@ class PromptPostProcessorComfyUINode:
|
||||
Node for processing prompts.
|
||||
"""
|
||||
|
||||
logger = None
|
||||
|
||||
def __init__(self):
|
||||
lf = PromptPostProcessorLogFactory()
|
||||
self.logger = lf.log
|
||||
grammar_filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), "grammar.lark")
|
||||
with open(grammar_filename, "r", encoding="utf-8") as file:
|
||||
self.grammar_content = file.read()
|
||||
self.grammar_content = load_grammar()
|
||||
self.wildcards_obj = PPPWildcards(lf.log)
|
||||
self.extranetwork_mappings_obj = PPPExtraNetworkMappings(lf.log)
|
||||
self.ppp: PromptPostProcessor | None = None
|
||||
@@ -88,6 +99,9 @@ class PromptPostProcessorComfyUINode:
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
models_list = ["(none)"] + sorted(
|
||||
set(folder_paths.get_filename_list("checkpoints") + folder_paths.get_filename_list("diffusion_models"))
|
||||
)
|
||||
return {
|
||||
"required": {
|
||||
"pos_prompt": (
|
||||
@@ -96,6 +110,7 @@ class PromptPostProcessorComfyUINode:
|
||||
"multiline": True,
|
||||
"default": "",
|
||||
"dynamicPrompts": False,
|
||||
"tooltip": "Positive prompt to process",
|
||||
},
|
||||
),
|
||||
"neg_prompt": (
|
||||
@@ -104,6 +119,7 @@ class PromptPostProcessorComfyUINode:
|
||||
"multiline": True,
|
||||
"default": "",
|
||||
"dynamicPrompts": False,
|
||||
"tooltip": "Negative prompt to process",
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -113,14 +129,16 @@ class PromptPostProcessorComfyUINode:
|
||||
{
|
||||
"default": "",
|
||||
"placeholder": "internal model class name",
|
||||
"tooltip": "Model or model class name. Optional if you set modelname.",
|
||||
},
|
||||
),
|
||||
"modelname": (
|
||||
"STRING",
|
||||
"COMBO",
|
||||
{
|
||||
"default": "",
|
||||
"placeholder": "full path of the model",
|
||||
"dynamicPrompts": False,
|
||||
"options": models_list,
|
||||
"default": "(none)",
|
||||
"placeholder": "relative path of the model",
|
||||
"tooltip": "Relative path of the model. Needed to detect variants.",
|
||||
},
|
||||
),
|
||||
"seed": (
|
||||
@@ -132,15 +150,17 @@ class PromptPostProcessorComfyUINode:
|
||||
},
|
||||
),
|
||||
"debug_level": (
|
||||
[e.value for e in DEBUG_LEVEL],
|
||||
"COMBO",
|
||||
{
|
||||
"options": [e.value for e in DEBUG_LEVEL],
|
||||
"default": PromptPostProcessor.DEFAULT_DEBUG_LEVEL,
|
||||
"tooltip": "Debug level",
|
||||
},
|
||||
),
|
||||
"on_warnings": (
|
||||
[e.value for e in ONWARNING_CHOICES],
|
||||
"COMBO",
|
||||
{
|
||||
"options": [e.value for e in ONWARNING_CHOICES],
|
||||
"default": PromptPostProcessor.DEFAULT_ON_WARNING,
|
||||
"tooltip": "How to handle invalid content warnings",
|
||||
},
|
||||
@@ -181,29 +201,20 @@ class PromptPostProcessorComfyUINode:
|
||||
"label_off": "No",
|
||||
},
|
||||
),
|
||||
"do_combinatorial": (
|
||||
"BOOLEAN",
|
||||
"results_file": (
|
||||
"STRING",
|
||||
{
|
||||
"default": PromptPostProcessor.DEFAULT_DO_COMBINATORIAL,
|
||||
"tooltip": "Enable combinatorial mode",
|
||||
"label_on": "Yes",
|
||||
"label_off": "No",
|
||||
"default": PromptPostProcessor.DEFAULT_RESULTS_FILE,
|
||||
"tooltip": r"Filename to save processing results. Supports %datetime%, %date%, %time%, %host% tokens. Empty = disabled.",
|
||||
"dynamicPrompts": False,
|
||||
},
|
||||
),
|
||||
"combinatorial_shuffle": (
|
||||
"BOOLEAN",
|
||||
"run_mode": (
|
||||
"COMBO",
|
||||
{
|
||||
"default": PromptPostProcessor.DEFAULT_COMBINATORIAL_SHUFFLE,
|
||||
"tooltip": "Shuffle the combinatorial results",
|
||||
"label_on": "Yes",
|
||||
"label_off": "No",
|
||||
},
|
||||
),
|
||||
"combinatorial_limit": (
|
||||
"INT",
|
||||
{
|
||||
"default": PromptPostProcessor.DEFAULT_COMBINATORIAL_LIMIT,
|
||||
"tooltip": "Limit for combinatorial mode",
|
||||
"options": [e.value for e in RUN_MODE],
|
||||
"default": PromptPostProcessor.DEFAULT_RUN_MODE,
|
||||
"tooltip": "Run mode",
|
||||
},
|
||||
),
|
||||
"wc_options": (
|
||||
@@ -234,6 +245,13 @@ class PromptPostProcessorComfyUINode:
|
||||
"tooltip": "ExtraNetworks mapping options",
|
||||
},
|
||||
),
|
||||
"rm_options": (
|
||||
"PPP_OPTIONS_RM",
|
||||
{
|
||||
"default": None,
|
||||
"tooltip": "Run mode options",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -265,6 +283,11 @@ class PromptPostProcessorComfyUINode:
|
||||
"neg_prompt",
|
||||
"variables",
|
||||
)
|
||||
OUTPUT_TOOLTIPS = (
|
||||
"Processed positive prompt (list of prompts if combinatorial/multiple mode is enabled)",
|
||||
"Processed negative prompt (list of prompts if combinatorial/multiple mode is enabled)",
|
||||
"Output variables (list of dictionaries if combinatorial/multiple mode is enabled)",
|
||||
)
|
||||
|
||||
FUNCTION = "process"
|
||||
|
||||
@@ -276,61 +299,82 @@ class PromptPostProcessorComfyUINode:
|
||||
|
||||
def process(
|
||||
self,
|
||||
model,
|
||||
modelname,
|
||||
pos_prompt,
|
||||
neg_prompt,
|
||||
seed,
|
||||
debug_level,
|
||||
on_warnings,
|
||||
strict_operators,
|
||||
process_wildcards,
|
||||
do_cleanup,
|
||||
cleanup_variables,
|
||||
do_combinatorial,
|
||||
combinatorial_shuffle,
|
||||
combinatorial_limit,
|
||||
results_file,
|
||||
run_mode,
|
||||
model=None,
|
||||
wc_options=None,
|
||||
stn_options=None,
|
||||
cup_options=None,
|
||||
en_options=None,
|
||||
strict_operators=None,
|
||||
rm_options=None,
|
||||
):
|
||||
modelclass = (
|
||||
model.model.model_config.__class__.__name__ if model is not None and not isinstance(model, str) else model
|
||||
) or ""
|
||||
if modelname == "(none)":
|
||||
modelname = ""
|
||||
if modelclass == "" and modelname != "":
|
||||
try:
|
||||
modelclass = get_model_class_from_filename(Path(modelname))
|
||||
except PPPException as e:
|
||||
log(
|
||||
self.logger,
|
||||
DEBUG_LEVEL.minimal,
|
||||
logging.WARNING,
|
||||
f"Could not detect model class from filename '{modelname}': {e}",
|
||||
)
|
||||
if modelclass:
|
||||
log(
|
||||
self.logger,
|
||||
DEBUG_LEVEL.minimal,
|
||||
logging.DEBUG,
|
||||
f"Detected model class '{modelclass}' from filename '{modelname}'",
|
||||
)
|
||||
if modelclass == "":
|
||||
log(
|
||||
self.logger,
|
||||
DEBUG_LEVEL.minimal,
|
||||
logging.WARNING,
|
||||
"Model class is not provided. System variables might not be properly set.",
|
||||
"Model class was not provided nor detected. System model variables will not be properly set.",
|
||||
)
|
||||
if modelname == "":
|
||||
log(
|
||||
self.logger,
|
||||
DEBUG_LEVEL.minimal,
|
||||
logging.WARNING,
|
||||
"Modelname is not provided. System variables will not be properly set.",
|
||||
"Modelname was not provided. System model and variant variables will not be properly set.",
|
||||
)
|
||||
# model class values in ComfyUI\comfy\supported_models.py
|
||||
env_info = {
|
||||
"app": SUPPORTED_APPS.comfyui.value,
|
||||
"models_path": folder_paths.models_dir,
|
||||
"model_filename": modelname or "", # path is relative to checkpoints folder
|
||||
"model_class": modelclass,
|
||||
"property_base": None,
|
||||
}
|
||||
env_info = PPPEnvInfo(
|
||||
app=SUPPORTED_APPS.comfyui,
|
||||
models_path=folder_paths.models_dir,
|
||||
model_filename=modelname or "", # path is relative to checkpoints folder
|
||||
model_class=modelclass,
|
||||
property_base=None,
|
||||
)
|
||||
wildcards_folders = _resolve_wildcards_folders(wc_options["wc_wildcards_folders"] if wc_options else "")
|
||||
enmappings_folders = _resolve_enmappings_folders(en_options["en_mappings_folders"] if en_options else "")
|
||||
|
||||
options = PPPStateOptions(
|
||||
debug_level=DEBUG_LEVEL(debug_level),
|
||||
on_warning=ONWARNING_CHOICES(on_warnings) if on_warnings else PromptPostProcessor.DEFAULT_ON_WARNING,
|
||||
on_warning=ONWARNING_CHOICES(on_warnings if on_warnings else PromptPostProcessor.DEFAULT_ON_WARNING),
|
||||
strict_operators=(
|
||||
strict_operators if strict_operators is not None else PromptPostProcessor.DEFAULT_STRICT_OPERATORS
|
||||
),
|
||||
process_wildcards=process_wildcards,
|
||||
if_wildcards=(wc_options["wc_if_wildcards"] if wc_options else IFWILDCARDS_CHOICES.stop.value),
|
||||
if_wildcards=IFWILDCARDS_CHOICES(
|
||||
wc_options["wc_if_wildcards"] if wc_options else PromptPostProcessor.DEFAULT_IF_WILDCARDS
|
||||
),
|
||||
choice_separator=(
|
||||
wc_options["wc_choice_separator"] if wc_options else PromptPostProcessor.DEFAULT_CHOICE_SEPARATOR
|
||||
),
|
||||
@@ -381,9 +425,19 @@ class PromptPostProcessorComfyUINode:
|
||||
if cup_options
|
||||
else PromptPostProcessor.DEFAULT_CUP_REMOVE_EXTRANETWORK_TAGS
|
||||
),
|
||||
do_combinatorial=do_combinatorial,
|
||||
combinatorial_shuffle=combinatorial_shuffle,
|
||||
combinatorial_limit=combinatorial_limit,
|
||||
run_mode=RUN_MODE(run_mode if run_mode else PromptPostProcessor.DEFAULT_RUN_MODE),
|
||||
results_file=results_file,
|
||||
results_shuffle=(
|
||||
rm_options["results_shuffle"] if rm_options else PromptPostProcessor.DEFAULT_RESULTS_SHUFFLE
|
||||
),
|
||||
results_limit=rm_options["results_limit"] if rm_options else PromptPostProcessor.DEFAULT_RESULTS_LIMIT,
|
||||
comb_random_fixed=(
|
||||
rm_options["comb_random_fixed"] if rm_options else PromptPostProcessor.DEFAULT_COMB_RANDOM_FIXED
|
||||
),
|
||||
default_sampler=DEFAULT_SAMPLER(
|
||||
rm_options["default_sampler"] if rm_options else PromptPostProcessor.DEFAULT_DEFAULT_SAMPLER
|
||||
),
|
||||
next_seed=NEXT_SEED(rm_options["next_seed"] if rm_options else PromptPostProcessor.DEFAULT_NEXT_SEED),
|
||||
)
|
||||
self.wildcards_obj.refresh_wildcards(
|
||||
options.debug_level,
|
||||
@@ -413,27 +467,101 @@ class PromptPostProcessorComfyUINode:
|
||||
self.extranetwork_mappings_obj,
|
||||
)
|
||||
self.ppp.process_prompts_group_start()
|
||||
results = self.ppp.process_prompt(pos_prompt, neg_prompt, seed if seed is not None else 1)
|
||||
results = self.ppp.process_prompt(
|
||||
pos_prompt,
|
||||
neg_prompt,
|
||||
seed if seed is not None else -1,
|
||||
jobinfo={"job_timestamp": datetime.now().isoformat()},
|
||||
)
|
||||
self.ppp.process_prompts_group_end()
|
||||
|
||||
# with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "logs", "last_prompts_comfyui.txt"), "w", encoding="utf-8") as f:
|
||||
# f.write(f"Seed: {seed if seed is not None else 1}\n")
|
||||
# f.write(f"In Positive: {pos_prompt}\n")
|
||||
# f.write(f"In Negative: {neg_prompt}\n")
|
||||
# f.write("\n")
|
||||
# for i, (posp, negp, var) in enumerate(results):
|
||||
# f.write(f"Index: {i}\n")
|
||||
# f.write(f"Out Positive: {posp}\n")
|
||||
# f.write(f"Out Negative: {negp}\n")
|
||||
# f.write(f"Out Variables: {var}\n")
|
||||
# f.write("\n")
|
||||
|
||||
return tuple(zip(*results)) # unzip the list of tuples into tuple of lists
|
||||
|
||||
def interrupt(self):
|
||||
nodes.interrupt_processing(True)
|
||||
|
||||
|
||||
class PromptPostProcessorRunModeOptionsComfyUINode:
|
||||
"""
|
||||
Node for run mode options.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"optional": {
|
||||
"results_limit": (
|
||||
"INT",
|
||||
{
|
||||
"default": PromptPostProcessor.DEFAULT_RESULTS_LIMIT,
|
||||
"tooltip": "Limit for combinatorial/multiple mode",
|
||||
"min": 0,
|
||||
},
|
||||
),
|
||||
"results_shuffle": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": PromptPostProcessor.DEFAULT_RESULTS_SHUFFLE,
|
||||
"tooltip": "Shuffle the results",
|
||||
"label_on": "Yes",
|
||||
"label_off": "No",
|
||||
},
|
||||
),
|
||||
"comb_random_fixed": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": PromptPostProcessor.DEFAULT_COMB_RANDOM_FIXED,
|
||||
"tooltip": "Fix the value of any specified random samplers across all combinations in combinatorial mode",
|
||||
"label_on": "Yes",
|
||||
"label_off": "No",
|
||||
},
|
||||
),
|
||||
"default_sampler": (
|
||||
"COMBO",
|
||||
{
|
||||
"options": [ds.value for ds in DEFAULT_SAMPLER],
|
||||
"default": PromptPostProcessor.DEFAULT_DEFAULT_SAMPLER,
|
||||
"tooltip": "Default choice sampler",
|
||||
},
|
||||
),
|
||||
"next_seed": (
|
||||
"COMBO",
|
||||
{
|
||||
"options": [e.value for e in NEXT_SEED],
|
||||
"default": PromptPostProcessor.DEFAULT_NEXT_SEED,
|
||||
"tooltip": "Next seed strategy",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("PPP_OPTIONS_RM",)
|
||||
RETURN_NAMES = ("options",)
|
||||
|
||||
FUNCTION = "process"
|
||||
|
||||
CATEGORY = "ACB"
|
||||
|
||||
def process(
|
||||
self,
|
||||
results_limit: int,
|
||||
results_shuffle: bool,
|
||||
comb_random_fixed: bool,
|
||||
default_sampler: str,
|
||||
next_seed: str,
|
||||
):
|
||||
options = {
|
||||
"results_limit": results_limit,
|
||||
"results_shuffle": results_shuffle,
|
||||
"comb_random_fixed": comb_random_fixed,
|
||||
"default_sampler": default_sampler,
|
||||
"next_seed": next_seed,
|
||||
}
|
||||
return (options,)
|
||||
|
||||
|
||||
class PromptPostProcessorWildcardOptionsComfyUINode:
|
||||
"""
|
||||
Node for wildcard options.
|
||||
@@ -465,8 +593,9 @@ class PromptPostProcessorWildcardOptionsComfyUINode:
|
||||
},
|
||||
),
|
||||
"if_wildcards": (
|
||||
[e.value for e in IFWILDCARDS_CHOICES],
|
||||
"COMBO",
|
||||
{
|
||||
"options": [e.value for e in IFWILDCARDS_CHOICES],
|
||||
"default": IFWILDCARDS_CHOICES.stop.value,
|
||||
"tooltip": "How to handle invalid wildcards in the prompt",
|
||||
},
|
||||
@@ -787,7 +916,8 @@ class PromptPostProcessorSelectVariableComfyUINode:
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
lf = PromptPostProcessorLogFactory()
|
||||
self.logger = lf.log
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
@@ -797,6 +927,7 @@ class PromptPostProcessorSelectVariableComfyUINode:
|
||||
"PPP_DICT",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": "Dictionary of variables to select from",
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -808,12 +939,30 @@ class PromptPostProcessorSelectVariableComfyUINode:
|
||||
"multiline": False,
|
||||
"default": "",
|
||||
"dynamicPrompts": False,
|
||||
"tooltip": "Name of the variable to select",
|
||||
},
|
||||
),
|
||||
"if_not_found": (
|
||||
"COMBO",
|
||||
{
|
||||
"options": [e.value for e in ONWARNING_CHOICES],
|
||||
"default": ONWARNING_CHOICES.warn.value,
|
||||
"tooltip": "How to handle the case when the variable name is not found in the input dictionary",
|
||||
},
|
||||
),
|
||||
"default": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": False,
|
||||
"dynamicPrompts": False,
|
||||
"tooltip": "Default value if the variable is not found",
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_TYPES = ("*",)
|
||||
RETURN_NAMES = ("value",)
|
||||
|
||||
FUNCTION = "select"
|
||||
@@ -822,16 +971,23 @@ class PromptPostProcessorSelectVariableComfyUINode:
|
||||
|
||||
def select(
|
||||
self,
|
||||
variables: dict[str, str],
|
||||
variables: dict[str, Any],
|
||||
name: str,
|
||||
if_not_found: str = ONWARNING_CHOICES.warn.value,
|
||||
default: str = "",
|
||||
):
|
||||
value = ""
|
||||
if variables:
|
||||
if name == "":
|
||||
value = "\n".join(f"{k}: {v}" for k, v in variables.items())
|
||||
elif name in variables:
|
||||
value = variables[name]
|
||||
return (value,)
|
||||
return ("\n".join(f"{k}: {v}" for k, v in variables.items()),)
|
||||
if name in variables:
|
||||
return (variables[name],)
|
||||
if if_not_found == ONWARNING_CHOICES.stop.value:
|
||||
raise ValueError(f"Variable '{name}' not found in the input variables")
|
||||
self.logger.warning(
|
||||
f"Variable '{name}' not found in the input variables, using default value '{escape_single_quotes(default)}'"
|
||||
)
|
||||
return (default,)
|
||||
raise ValueError("No variables provided to select from")
|
||||
|
||||
|
||||
class PromptPostProcessorWildcardConcatComfyUINode:
|
||||
@@ -851,13 +1007,13 @@ class PromptPostProcessorWildcardConcatComfyUINode:
|
||||
lf = PromptPostProcessorLogFactory()
|
||||
cls._ppp = PromptPostProcessor(
|
||||
lf.log,
|
||||
{
|
||||
"app": SUPPORTED_APPS.comfyui.value,
|
||||
"models_path": folder_paths.models_dir,
|
||||
"model_filename": "",
|
||||
"model_class": "",
|
||||
"property_base": None,
|
||||
},
|
||||
PPPEnvInfo(
|
||||
app=SUPPORTED_APPS.comfyui,
|
||||
models_path=folder_paths.models_dir,
|
||||
model_filename="",
|
||||
model_class="",
|
||||
property_base=None,
|
||||
),
|
||||
PPPStateOptions(debug_level=DEBUG_LEVEL.minimal),
|
||||
wildcards_obj=PPPWildcards(lf.log),
|
||||
)
|
||||
@@ -975,6 +1131,7 @@ class PromptPostProcessorWildcardConcatComfyUINode:
|
||||
|
||||
|
||||
try:
|
||||
# pylint: disable=import-error
|
||||
from server import PromptServer # type: ignore
|
||||
from aiohttp import web as _aiohttp_web # type: ignore
|
||||
|
||||
|
||||
+187
-9
@@ -1,13 +1,17 @@
|
||||
import ast
|
||||
import csv
|
||||
from enum import Enum
|
||||
from functools import reduce
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import textwrap
|
||||
import time
|
||||
import lark
|
||||
from ruamel.yaml import YAML as _YAML
|
||||
|
||||
from ppp_logging import log
|
||||
from ppp_classes import ONWARNING_CHOICES, PPPInterrupt, PPPState
|
||||
from ppp_classes import ONWARNING_CHOICES, PPPException, PPPInterrupt, PPPState
|
||||
from ppp_utils import escape_single_quotes, format_output
|
||||
|
||||
|
||||
@@ -79,20 +83,26 @@ def parse_prompt(
|
||||
return parsed_prompt
|
||||
|
||||
|
||||
def warn_or_stop(state: PPPState, is_negative: bool, message: str, e: Exception = None):
|
||||
class WARN_STOP_WHERE(Enum):
|
||||
none = 0
|
||||
positive = 1
|
||||
negative = 2
|
||||
|
||||
|
||||
def warn_or_stop(state: PPPState, where: WARN_STOP_WHERE, message: str, e: Exception = None):
|
||||
INVALID_CONTENT_STOP = "INVALID CONTENT! {0}\nBREAK "
|
||||
if state.options.on_warning == ONWARNING_CHOICES.stop:
|
||||
raise PPPInterrupt(
|
||||
message,
|
||||
INVALID_CONTENT_STOP.format(message) if not is_negative else "",
|
||||
INVALID_CONTENT_STOP.format(message) if is_negative else "",
|
||||
INVALID_CONTENT_STOP.format(message) if where == WARN_STOP_WHERE.positive else "",
|
||||
INVALID_CONTENT_STOP.format(message) if where == WARN_STOP_WHERE.negative else "",
|
||||
) from e
|
||||
log(state.logger, state.options.debug_level, logging.WARNING, format_output(message))
|
||||
|
||||
|
||||
def load_grammar() -> str:
|
||||
# Process with lark (debug with https://www.lark-parser.org/ide/)
|
||||
grammar_filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), "grammar.lark")
|
||||
grammar_filename = Path(__file__).resolve().parent / "grammar.lark"
|
||||
with open(grammar_filename, "r", encoding="utf-8") as file:
|
||||
grammar_content = file.read()
|
||||
return grammar_content
|
||||
@@ -159,13 +169,15 @@ def preprocess_grammar(grammar_content: str, options: dict[str, bool], logger: l
|
||||
if not skip_current_block:
|
||||
log(logger, debug_level, logging.WARNING, "Unmatched //#elif directive found in grammar content.")
|
||||
elif all_blocks_skipped[-1]:
|
||||
# Extract condition from the #elif directive
|
||||
# No earlier branch in this if/elif chain was taken, so evaluate this one.
|
||||
# all_blocks_skipped[-1] tracks whether any branch has matched yet;
|
||||
# skip_current_block[-1] tracks whether the current branch should be emitted.
|
||||
conditions = stripped_line[7:].strip()
|
||||
# Evaluate the conditions
|
||||
skip_current_block[-1] = not eval_bool_expr(conditions, options)
|
||||
if not skip_current_block[-1]:
|
||||
all_blocks_skipped[-1] = False
|
||||
all_blocks_skipped[-1] = False # mark that a branch was taken
|
||||
else:
|
||||
# A previous branch already matched - skip all remaining elif/else branches.
|
||||
skip_current_block[-1] = True
|
||||
elif stripped_line.startswith("//#else"):
|
||||
if not skip_current_block:
|
||||
@@ -196,3 +208,169 @@ def preprocess_grammar(grammar_content: str, options: dict[str, bool], logger: l
|
||||
f"Found {len(skip_current_block)} unclosed conditional directive(s) at the end of the grammar file"
|
||||
)
|
||||
return "\n".join(result_lines)
|
||||
|
||||
|
||||
def get_model_config_from_filename(filename: Path) -> object | None:
|
||||
"""
|
||||
Attempts to detect the model class from the given filename by inspecting the file header.
|
||||
Currently only supports ComfyUI models in .safetensors format.
|
||||
The path must be relative to a model folder.
|
||||
"""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
try:
|
||||
import folder_paths # type: ignore
|
||||
import comfy.utils # type: ignore
|
||||
import comfy.model_detection as model_detection # type: ignore
|
||||
except ImportError as e:
|
||||
raise PPPException(f"Error detecting class from '{filename}': {e}") from e
|
||||
import json
|
||||
|
||||
if not filename:
|
||||
return None
|
||||
|
||||
path_keys = ["diffusion_models", "checkpoints", "unet"]
|
||||
full_path: Path | None = None
|
||||
if filename.is_absolute():
|
||||
full_path = filename
|
||||
else:
|
||||
base_folders = []
|
||||
for key in path_keys:
|
||||
base_folders.extend(folder_paths.get_folder_paths(key))
|
||||
for base in base_folders:
|
||||
fname: Path = base / filename
|
||||
if fname.exists():
|
||||
full_path = fname
|
||||
break
|
||||
if not full_path or not full_path.suffix.lower() in (".safetensors", ".sft"):
|
||||
return None
|
||||
try:
|
||||
header_bytes = comfy.utils.safetensors_header(full_path)
|
||||
if header_bytes is None:
|
||||
raise PPPException(f"Error detecting class from '{full_path}': no header")
|
||||
header = json.loads(header_bytes)
|
||||
|
||||
# model_config_from_unet only inspects tensor shapes, not actual data.
|
||||
# A lightweight proxy that exposes .shape and index access lets us avoid
|
||||
# loading the full model into memory.
|
||||
class _ShapeProxy:
|
||||
def __init__(self, shape):
|
||||
self.shape = shape
|
||||
|
||||
def __getitem__(self, i):
|
||||
return self.shape[i]
|
||||
|
||||
mock_sd = {k: _ShapeProxy(v["shape"]) for k, v in header.items() if k != "__metadata__" and "shape" in v}
|
||||
|
||||
prefix = model_detection.unet_prefix_from_state_dict(mock_sd)
|
||||
config = model_detection.model_config_from_unet(mock_sd, prefix, True)
|
||||
if not config:
|
||||
mock_sd, metadata = comfy.utils.convert_old_quants(mock_sd, "", metadata=None)
|
||||
# Allow loading unets from checkpoint files
|
||||
diffusion_model_prefix = model_detection.unet_prefix_from_state_dict(mock_sd)
|
||||
temp_sd = comfy.utils.state_dict_prefix_replace(mock_sd, {diffusion_model_prefix: ""}, filter_keys=True)
|
||||
if len(temp_sd) > 0:
|
||||
mock_sd, metadata = comfy.utils.convert_old_quants(temp_sd, "", metadata=metadata)
|
||||
config = model_detection.model_config_from_unet(mock_sd, "", metadata=metadata)
|
||||
if config is None:
|
||||
mock_sd = model_detection.convert_diffusers_mmdit(mock_sd, "")
|
||||
if mock_sd is not None: # diffusers mmdit
|
||||
config = model_detection.model_config_from_unet(mock_sd, "")
|
||||
else: # diffusers unet
|
||||
config = model_detection.model_config_from_diffusers_unet(mock_sd)
|
||||
|
||||
if not config:
|
||||
raise PPPException(f"Error detecting class from file '{full_path}': no config found")
|
||||
return config
|
||||
except PPPException:
|
||||
raise
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
raise PPPException(f"Error detecting class from file '{full_path}': {e}") from e
|
||||
|
||||
|
||||
def get_model_class_from_filename(filename: Path) -> str:
|
||||
config = get_model_config_from_filename(filename)
|
||||
if not config:
|
||||
return ""
|
||||
c = config.__class__.__name__
|
||||
if not c:
|
||||
raise PPPException(f"Error detecting class from '{filename}': config has no class name {config}")
|
||||
return c
|
||||
|
||||
|
||||
def sanitize_wc_name(name: str) -> str:
|
||||
# Remove invalid characters
|
||||
return re.sub(r"[^a-zA-Z0-9-_]+", "", re.sub(r"_{2,}", "_", name.replace(" ", "_")))
|
||||
|
||||
|
||||
def convert_a1111_styles_to_wildcard(inp: Path, out: Path):
|
||||
"""
|
||||
Converts styles from A1111 format to wildcard format in a YAML file.
|
||||
Args:
|
||||
inp (Path): The input CSV file path.
|
||||
out (Path): The output YAML file path.
|
||||
"""
|
||||
wildcards = {}
|
||||
with open(inp, "r", encoding="utf-8-sig") as f:
|
||||
for reg in csv.reader(f):
|
||||
name = reg[0].strip()
|
||||
if name.lower() == "name":
|
||||
continue
|
||||
positive = reg[1].strip()
|
||||
negative = reg[2].strip()
|
||||
if negative:
|
||||
positive = f"{positive}<ppp:stn>{negative}<ppp:/stn>"
|
||||
name = sanitize_wc_name(name)
|
||||
wildcards[name] = positive
|
||||
if not wildcards:
|
||||
raise RuntimeError(f"No styles found in {inp} to convert to wildcards.")
|
||||
with open(out, "w", encoding="utf-8-sig") as f:
|
||||
f.write(f"# Original names may contain characters that are replaced in the output.\n# Converted from {inp}\n")
|
||||
_YAML().dump(wildcards, f)
|
||||
|
||||
|
||||
def convert_sdnext_styles_to_wildcard(inp: Path, out: Path):
|
||||
"""
|
||||
Converts styles from SD.Next format to wildcard format in a YAML file.
|
||||
Args:
|
||||
inp (Path): The input folder path containing style json files or a single json file.
|
||||
out (Path): The output YAML file path.
|
||||
"""
|
||||
wildcards = {}
|
||||
files = inp.glob("*.json") if inp.is_dir() else [inp]
|
||||
for file in files:
|
||||
with open(file, "r", encoding="utf-8-sig") as f:
|
||||
data = _YAML(typ="safe").load(f)
|
||||
if not isinstance(data, list):
|
||||
continue
|
||||
wildcards[file] = {}
|
||||
for style in data:
|
||||
name = style.get("name", "").strip()
|
||||
positive = style.get("prompt", "").strip()
|
||||
negative = style.get("negative", "").strip()
|
||||
# extra = style.get("extra", "").strip()
|
||||
if negative:
|
||||
positive = f"{positive}<ppp:stn>{negative}<ppp:/stn>"
|
||||
name = sanitize_wc_name(name)
|
||||
wildcards[file][name] = positive
|
||||
if not reduce(lambda acc, d: acc or bool(d), wildcards.values(), False):
|
||||
raise RuntimeError(f"No styles found in {inp} to convert to wildcards.")
|
||||
with open(out, "w", encoding="utf-8-sig") as f:
|
||||
f.write("# Original names may contain characters that are replaced in the output.\n")
|
||||
for name, wcs in wildcards.items():
|
||||
f.write(f"# Converted from {name}\n")
|
||||
yaml_writer = _YAML()
|
||||
yaml_writer.dump(wcs, f)
|
||||
|
||||
|
||||
def clamp_host_bits(bits: int, seed: int) -> int:
|
||||
"""
|
||||
Clamp the seed to the host's configured bit width so the value stays within the range the host expects (and positive).
|
||||
|
||||
Args:
|
||||
bits (int): The host's configured bit width.
|
||||
seed (int): The seed to clamp.
|
||||
|
||||
Returns:
|
||||
int: The clamped seed.
|
||||
"""
|
||||
return int(seed & ((1 << bits ) - 1))
|
||||
|
||||
+175
-40
@@ -37,6 +37,7 @@ hosts:
|
||||
# - comma: replace with comma
|
||||
# - remove: remove the operator
|
||||
# - error: raise an error
|
||||
# seed_bits: number of bits to use for the seed; if not specified, it defaults to 64 bits
|
||||
# Options in the user file override all these options by host (individual options are not inherited and if missing will use defaults in code).
|
||||
a1111:
|
||||
attention: ok
|
||||
@@ -44,24 +45,35 @@ hosts:
|
||||
alternation: ok
|
||||
and: ok
|
||||
break: ok
|
||||
seed_bits: 32
|
||||
forge:
|
||||
attention: ok
|
||||
scheduling: ok
|
||||
alternation: ok
|
||||
and: ok
|
||||
break: ok
|
||||
seed_bits: 32
|
||||
forgeneo:
|
||||
attention: ok
|
||||
scheduling: ok
|
||||
alternation: ok
|
||||
and: ok
|
||||
break: ok
|
||||
seed_bits: 32
|
||||
reforge:
|
||||
attention: ok
|
||||
scheduling: ok
|
||||
alternation: ok
|
||||
and: ok
|
||||
break: ok
|
||||
seed_bits: 32
|
||||
sdnext:
|
||||
attention: ok
|
||||
scheduling: ok
|
||||
alternation: ok
|
||||
and: ok
|
||||
break: ok
|
||||
seed_bits: 32
|
||||
comfyui:
|
||||
# ComfyUI only supports parentheses-based attention
|
||||
attention: parentheses
|
||||
@@ -70,14 +82,16 @@ hosts:
|
||||
alternation: error
|
||||
and: comma
|
||||
break: comma
|
||||
seed_bits: 53 # and not 64, because of frontend JavaScript limits
|
||||
|
||||
# Supported base models, variants, and options
|
||||
# Check supported models for each host in:
|
||||
# A1111:
|
||||
# Forge: repositories\huggingface_guess\huggingface_guess\model_list.py
|
||||
# reForge:
|
||||
# SD.Next:
|
||||
# ComfyUI:
|
||||
# a1111:
|
||||
# forge: repositories\huggingface_guess\huggingface_guess\model_list.py
|
||||
# forgeneo: modules_forge\packages\huggingface_guess\model_list.py
|
||||
# reforge: ldm_patched\modules\supported_models.py
|
||||
# sdnext: modules\shared_items.py
|
||||
# comfyui: ComfyUI\comfy\supported_models.py
|
||||
models:
|
||||
# We define supported models and how we detect them in each host application (by class or by a known boolean property, or null for not supported).
|
||||
# We can also define here variants and some options.
|
||||
@@ -85,30 +99,34 @@ models:
|
||||
sd1: # Stable Diffusion 1
|
||||
detect:
|
||||
a1111: { property: "is_sd1" }
|
||||
forge: { property: "is_sd1" }
|
||||
reforge: { property: "is_sd1" }
|
||||
sdnext: { class: ["LatentDiffusion", "StableDiffusionPipeline"] } # LatentDiffusion is for the original backend, StableDiffusionPipeline is for the diffusers backend; cannot differentiate SD1 and SD2, we set both to True
|
||||
forge: { property: "is_sd1", class: ["SD15", "SD15_instructpix2pix"] }
|
||||
forgeneo: { property: "is_sd1", class: ["SD15"] }
|
||||
reforge: { property: "is_sd1", class: ["SD15", "SD15_instructpix2pix"] }
|
||||
sdnext: { class: ["LatentDiffusion", "StableDiffusionPipeline", "StableDiffusionInpaintPipeline", "StableDiffusionInstructPix2PixPipeline", "StableDiffusionUpscalePipeline"] } # LatentDiffusion is for the original backend, StableDiffusionPipeline is for the diffusers backend; cannot differentiate SD1 and SD2, we set both to True
|
||||
comfyui: { class: ["SD15", "SD15_instructpix2pix"] }
|
||||
sd2: # Stable Diffusion 2
|
||||
detect:
|
||||
a1111: { property: "is_sd2" }
|
||||
forge: { property: "is_sd2" }
|
||||
reforge: { property: "is_sd2" }
|
||||
sdnext: { class: ["LatentDiffusion", "StableDiffusionPipeline"] } # cannot differentiate SD1 and SD2, we set both to True; LatentDiffusion is for the original backend, StableDiffusionPipeline is for the diffusers backend
|
||||
forge: { property: "is_sd2", class: ["SD20", "SD21UnclipL", "SD21UnclipH"] }
|
||||
forgeneo: null
|
||||
reforge: { property: "is_sd2", class: ["SD20", "SD21UnclipL", "SD21UnclipH"] }
|
||||
sdnext: { class: ["LatentDiffusion", "StableDiffusionPipeline", "StableDiffusionInpaintPipeline", "StableDiffusionInstructPix2PixPipeline", "StableDiffusionUpscalePipeline"] } # cannot differentiate SD1 and SD2, we set both to True; LatentDiffusion is for the original backend, StableDiffusionPipeline is for the diffusers backend
|
||||
comfyui: { class: ["SD20", "SD21UnclipL", "SD21UnclipH", "LotusD"] }
|
||||
ssd: # Segmind Stable Diffusion 1B
|
||||
detect:
|
||||
a1111: { property: "is_ssd" }
|
||||
forge: null
|
||||
reforge: { property: "is_ssd" }
|
||||
forge: { class: ["SSD1B"] }
|
||||
forgeneo: null
|
||||
reforge: { property: "is_ssd", class: ["SSD1B"] }
|
||||
sdnext: null
|
||||
comfyui: { class: ["SSD1B"]}
|
||||
sdxl: # Stable Diffusion XL
|
||||
detect:
|
||||
a1111: { property: "is_sdxl" }
|
||||
forge: { property: "is_sdxl" }
|
||||
reforge: { property: "is_sdxl" }
|
||||
sdnext: { class: ["StableDiffusionXLPipeline"] }
|
||||
forge: { property: "is_sdxl", class: ["SDXL", "SDXLRefiner", "SDXL_instructpix2pix", "Segmind_Vega", "KOALA_700M", "KOALA_1B"] }
|
||||
forgeneo: { property: "is_sdxl", class: ["SDXL", "SDXLRefiner"] }
|
||||
reforge: { property: "is_sdxl", class: ["SDXL", "SDXLRefiner", "SDXL_instructpix2pix", "Segmind_Vega", "KOALA_700M", "KOALA_1B"] }
|
||||
sdnext: { class: ["StableDiffusionXLPipeline", "StableDiffusionXLImg2ImgPipeline", "StableDiffusionXLInpaintPipeline", "StableDiffusionXLInstructPix2PixPipeline"] }
|
||||
comfyui: { class: ["SDXL", "SDXLRefiner", "SDXL_instructpix2pix", "Segmind_Vega", "KOALA_700M", "KOALA_1B"] }
|
||||
variants:
|
||||
# At this level goes the name of the defined variants
|
||||
@@ -118,24 +136,28 @@ models:
|
||||
# If specified as a list, it should be a list of dicts or strings
|
||||
pony: { find_in_filename: "(?:pony|pny|pdxl)" }
|
||||
illustrious: { find_in_filename: { regex: "(?:illustrious|illust|ilxl)", flags: ["IGNORECASE"] } }
|
||||
noobai: { find_in_filename: "(?:noob|nai)" }
|
||||
sd3: # Stable Diffusion 3
|
||||
detect:
|
||||
a1111: { property: "is_sd3" }
|
||||
forge: { property: "is_sd3", class: ["SD3"] }
|
||||
reforge: { property: "is_sd3" }
|
||||
forgeneo: null
|
||||
reforge: { property: "is_sd3", class: ["SD3"] }
|
||||
sdnext: { class: ["StableDiffusion3Pipeline"] }
|
||||
comfyui: { class: ["SD3"] }
|
||||
flux: # Flux 1
|
||||
detect:
|
||||
a1111: null
|
||||
forge: { class: ["Flux", "FluxSchnell"] }
|
||||
forgeneo: { class: ["Flux", "FluxSchnell"] }
|
||||
reforge: { class: ["Flux", "FluxInpaint", "FluxSchnell"] }
|
||||
sdnext: { class: ["FluxPipeline"] }
|
||||
comfyui: { class: ["Flux", "FluxInpaint", "FluxSchnell"] }
|
||||
auraflow: # AuraFlow
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forge: { class: ["AuraFlow"] }
|
||||
forgeneo: null
|
||||
reforge: { class: ["AuraFlow"] }
|
||||
sdnext: { class: ["AuraFlowPipeline"] }
|
||||
comfyui: { class: ["AuraFlow"] }
|
||||
@@ -143,34 +165,47 @@ models:
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: { class: ["PixArtAlpha", "PixArtSigma"] }
|
||||
sdnext: { class: ["PixArtAlphaPipeline"] }
|
||||
sdnext: { class: ["PixArtAlphaPipeline", "PixArtSigmaPipeline"] }
|
||||
comfyui: { class: ["PixArtAlpha", "PixArtSigma"] }
|
||||
lumina2: # Lumina2 & ZImage
|
||||
lumina2: # Lumina2
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
reforge: { class: ["Lumina2", "ZImage"] }
|
||||
sdnext: null
|
||||
comfyui: { class: ["Lumina2", "ZImage"] }
|
||||
forgeneo: { class: ["Lumina2"] }
|
||||
reforge: { class: ["Lumina2"] }
|
||||
sdnext: { class: ["Lumina2Pipeline"] }
|
||||
comfyui: { class: ["Lumina2"] }
|
||||
zimage: # ZImage
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: { class: ["ZImage"] }
|
||||
reforge: { class: ["ZImage"] }
|
||||
sdnext: { class: ["ZImagePipeline"] }
|
||||
comfyui: { class: ["ZImage"] }
|
||||
ltxv: # LTXV
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
reforge: null
|
||||
forgeneo: null
|
||||
reforge: { class: ["LTXV"] }
|
||||
sdnext: null
|
||||
comfyui: { class: ["LTXV", "LTXAV"] }
|
||||
cosmos: # Cosmos
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: { class: ["CosmosT2V", "CosmosI2V"] }
|
||||
sdnext: null
|
||||
sdnext: { class: ["Cosmos2TextToImagePipeline"] }
|
||||
comfyui: { class: ["CosmosT2V", "CosmosI2V"] }
|
||||
cosmospredict2: # CosmosPredict2
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: { class: ["CosmosT2IPredict2", "CosmosI2VPredict2"] }
|
||||
sdnext: null
|
||||
comfyui: { class: ["CosmosT2IPredict2", "CosmosI2VPredict2"] }
|
||||
@@ -178,20 +213,23 @@ models:
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: { class: ["GenmoMochi"] }
|
||||
sdnext: null
|
||||
comfyui: { class: ["GenmoMochi"] }
|
||||
hunyuan: # Hunyuan
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forge: { class: ["HunyuanDiT", "HunyuanDiT1"] }
|
||||
forgeneo: null
|
||||
reforge: { class: ["HunyuanDiT", "HunyuanDiT1"] }
|
||||
sdnext: null
|
||||
sdnext: { class: ["HunyuanDiTPipeline", "HunyuanImagePipeline"] }
|
||||
comfyui: { class: ["HunyuanDiT", "HunyuanDiT1"] }
|
||||
hunyuanvideo: # HunyuanVideo
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: { class: ["HunyuanVideo", "HunyuanVideoI2V", "HunyuanVideoSkyreelsI2V", "HunyuanImage21", "HunyuanImage21Refiner", "HunyuanVideo15", "HunyuanVideo15_SR_Distilled"] }
|
||||
sdnext: null
|
||||
comfyui: { class: ["HunyuanVideo", "HunyuanVideoI2V", "HunyuanVideoSkyreelsI2V", "HunyuanImage21", "HunyuanImage21Refiner", "HunyuanVideo15", "HunyuanVideo15_SR_Distilled"] }
|
||||
@@ -199,6 +237,7 @@ models:
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: { class: ["Hunyuan3Dv2", "Hunyuan3Dv2_1", "Hunyuan3Dv2mini"] }
|
||||
sdnext: null
|
||||
comfyui: { class: ["Hunyuan3Dv2", "Hunyuan3Dv2_1", "Hunyuan3Dv2mini"] }
|
||||
@@ -206,55 +245,151 @@ models:
|
||||
detect:
|
||||
a1111: null
|
||||
forge: { class: ["WAN21_T2V", "WAN21_I2V"] }
|
||||
forgeneo: { class: ["WAN21_T2V", "WAN21_I2V"] }
|
||||
reforge: { class: ["WAN21_T2V", "WAN21_I2V", "WAN21_FunControl2V", "WAN21_Camera", "WAN22_Camera", "WAN21_Vace", "WAN21_HuMo", "WAN22_S2V", "WAN22_Animate", "WAN22_T2V"] }
|
||||
sdnext: null
|
||||
comfyui: { class: ["WAN21_T2V", "WAN21_I2V", "WAN21_FunControl2V", "WAN21_Camera", "WAN22_Camera", "WAN21_Vace", "WAN21_HuMo", "WAN22_S2V", "WAN22_Animate", "WAN22_T2V"] }
|
||||
sdnext: { class: ["WanPipeline"] }
|
||||
comfyui: { class: ["WAN21_T2V", "WAN21_I2V", "WAN21_FunControl2V", "WAN21_Camera", "WAN22_Camera", "WAN21_Vace", "WAN21_HuMo", "WAN22_S2V", "WAN22_Animate", "WAN22_T2V", "WAN21_FlowRVS", "WAN21_SCAIL", "WAN22_WanDancer", "WAN21_CausalAR_T2V", "WAN21_SCAIL2", "WAN_Animate2"] }
|
||||
hidream: # HiDream
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: { class: ["HiDream"] }
|
||||
sdnext: null
|
||||
comfyui: { class: ["HiDream"] }
|
||||
sdnext: { class: ["HiDreamImagePipeline"] }
|
||||
comfyui: { class: ["HiDream", "HiDreamO1"] }
|
||||
qwenimage: # QwenImage & QwenImageEdit
|
||||
detect:
|
||||
a1111: null
|
||||
forge: { class: ["QwenImage"] }
|
||||
forge: null
|
||||
forgeneo: { class: ["QwenImage"] }
|
||||
reforge: null
|
||||
sdnext: null
|
||||
sdnext: { class: ["QwenImagePipeline"] }
|
||||
comfyui: { class: ["QwenImage"] }
|
||||
chroma: # Chroma
|
||||
detect:
|
||||
a1111: null
|
||||
forge: { class: ["Chroma"] }
|
||||
forge: null
|
||||
forgeneo: { class: ["Chroma"] }
|
||||
reforge: { class: ["Chroma", "ChromaRadiance"] }
|
||||
sdnext: null
|
||||
sdnext: { class: ["ChromaPipeline"] }
|
||||
comfyui: { class: ["Chroma", "ChromaRadiance"] }
|
||||
omnigen2: # Omnigen 2
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: { class: ["Omnigen2"] }
|
||||
sdnext: null
|
||||
sdnext: { class: ["OmniGen2Pipeline"] }
|
||||
comfyui: { class: ["Omnigen2"] }
|
||||
flux2: # Flux 2
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: { class: ["Flux2K4B", "Flux2K9B"] }
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["Flux2"] }
|
||||
sdnext: { class: ["Flux2Pipeline", "Flux2KleinPipeline"] }
|
||||
comfyui: { class: ["Flux2", "Flux2K4B", "Flux2K9B"] }
|
||||
kandinsky5: # Kandinsky 5
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: null
|
||||
sdnext: null
|
||||
sdnext: { class: ["Kandinsky5T2IPipeline"] }
|
||||
comfyui: { class: ["Kandinsky5", "Kandinsky5Image"] }
|
||||
anima: # Anima
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: { class: ["Anima"] }
|
||||
reforge: null
|
||||
sdnext: { class: ["AnimaText2ImagePipeline"] }
|
||||
comfyui: { class: ["Anima"] }
|
||||
longcatimage: # LongCatImage
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: null
|
||||
sdnext: { class: ["LongCatImagePipeline"] }
|
||||
comfyui: { class: ["LongCatImage"] }
|
||||
ernieimage: # ErnieImage
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: { class: ["ErnieImage"] }
|
||||
reforge: null
|
||||
sdnext: { class: ["ErnieImagePipeline"] }
|
||||
comfyui: { class: ["ErnieImage"] }
|
||||
mugen: # Mugen
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: { class: ["Mugen"] }
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["Anima"] }
|
||||
comfyui: { class: ["Mugen"] }
|
||||
cogvideox: # CogVideoX
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["CogVideoX_T2V", "CogVideoX_I2V", "CogVideoX_Inpaint"] }
|
||||
krea2: # Krea2
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: { class: ["Krea2"] }
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["Krea2"] }
|
||||
ideogram4: # Ideogram4
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["Ideogram4"] }
|
||||
lens: # Lens
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["Lens"] }
|
||||
boogu: # Boogu
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["Boogu"] }
|
||||
mageflow: # MageFlow
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["MageFlow"] }
|
||||
pid: # PiD
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: { class: ["PiD"] }
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["PiD"] }
|
||||
minimaxh3: # MiniMaxH3
|
||||
detect:
|
||||
a1111: null
|
||||
forge: null
|
||||
forgeneo: null
|
||||
reforge: null
|
||||
sdnext: null
|
||||
comfyui: { class: ["MiniMaxH3"] }
|
||||
|
||||
+103
-82
@@ -1,8 +1,8 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
import logging
|
||||
import yaml
|
||||
from ruamel.yaml import YAML as _YAML
|
||||
from ruamel.yaml.error import YAMLError as _YAMLError
|
||||
|
||||
from ppp_logging import DEBUG_LEVEL, log
|
||||
from ppp_utils import deep_freeze, escape_single_quotes
|
||||
@@ -35,12 +35,12 @@ class PPPENMapping:
|
||||
Attributes:
|
||||
kind (str): The kind of the extra network.
|
||||
name (str): The name of the extra network mapping.
|
||||
file (str): The path to the file where the extranetwork mapping is defined.
|
||||
file (Path | None): The path to the file where the extranetwork mapping is defined, or None if from inline input.
|
||||
variants (list[PPPENMappingVariant]): The processed variants of the extranetwork mapping.
|
||||
"""
|
||||
|
||||
def __init__(self, fullpath: str, kind: str, name: str, variants: list[dict]):
|
||||
self.file: str = fullpath
|
||||
def __init__(self, fullpath: Path | None, kind: str, name: str, variants: list[dict]):
|
||||
self.file: Path | None = fullpath
|
||||
self.kind: str = kind
|
||||
self.name: str = name
|
||||
self.variants: list[PPPENMappingVariant] = [
|
||||
@@ -55,7 +55,12 @@ class PPPENMapping:
|
||||
return hash(t)
|
||||
|
||||
def __sizeof__(self):
|
||||
return self.kind.__sizeof__() + self.name.__sizeof__() + self.file.__sizeof__() + self.variants.__sizeof__()
|
||||
return (
|
||||
self.kind.__sizeof__()
|
||||
+ self.name.__sizeof__()
|
||||
+ (self.file.__sizeof__() if self.file is not None else 0)
|
||||
+ self.variants.__sizeof__()
|
||||
)
|
||||
|
||||
|
||||
class PPPExtraNetworkMappings:
|
||||
@@ -67,13 +72,13 @@ class PPPExtraNetworkMappings:
|
||||
"""
|
||||
|
||||
DEFAULT_ENMAPPINGS_FOLDER = "extranetworkmappings"
|
||||
LOCALINPUT_FILENAME = R"//INPUT\\"
|
||||
|
||||
def __init__(self, logger=None):
|
||||
self.__logger: logging.Logger = logger
|
||||
self.__debug_level = DEBUG_LEVEL.none
|
||||
self.__enmappings_folders = []
|
||||
self.__enmappings_files = {}
|
||||
self.__enmappings_folders: list[Path] = []
|
||||
self.__enmappings_files: dict[Path, float] = {}
|
||||
self.__local_enmappings_input_hash: int | None = None
|
||||
self.extranetwork_mappings: dict[str, PPPENMapping] = {}
|
||||
self.cached_mappings = {}
|
||||
|
||||
@@ -88,26 +93,41 @@ class PPPExtraNetworkMappings:
|
||||
+ self.cached_mappings.__sizeof__()
|
||||
)
|
||||
|
||||
def get_mapping(self, key: str) -> PPPENMapping | None:
|
||||
"""
|
||||
Get an extra network mapping by key.
|
||||
|
||||
Args:
|
||||
key (str): The key of the extra network mapping in the format "kind:name".
|
||||
Returns:
|
||||
PPPENMapping | None: The extra network mapping if found, or None if not found
|
||||
"""
|
||||
for k, v in self.extranetwork_mappings.items():
|
||||
if k.lower() == key.lower():
|
||||
return v
|
||||
return None
|
||||
|
||||
def refresh_extranetwork_mappings(
|
||||
self, debug_level: DEBUG_LEVEL, enmappings_folders: Optional[list[str]], enmappings_input: str = None
|
||||
self,
|
||||
debug_level: DEBUG_LEVEL,
|
||||
enmappings_folders: Optional[list[Path]],
|
||||
enmappings_input: str = None,
|
||||
):
|
||||
"""
|
||||
Initialize the extra network mappings.
|
||||
"""
|
||||
self.__debug_level = debug_level
|
||||
self.__enmappings_folders = enmappings_folders or []
|
||||
self.__enmappings_folders = [Path(f) for f in (enmappings_folders or [])]
|
||||
# log(self.__logger, self.__debug_level, logging.INFO, "Refreshing extra network mappings...")
|
||||
# t1 = time.monotonic_ns()
|
||||
self.cached_mappings = {}
|
||||
for fullpath in list(self.__enmappings_files.keys()):
|
||||
if fullpath != self.LOCALINPUT_FILENAME:
|
||||
path = os.path.dirname(fullpath)
|
||||
if not os.path.exists(fullpath) or not any(
|
||||
Path(path).is_relative_to(folder) for folder in self.__enmappings_folders
|
||||
):
|
||||
self.__remove_extranetwork_mappings_from_path(fullpath)
|
||||
elif enmappings_input is None:
|
||||
if not fullpath.exists() or not any(
|
||||
fullpath.parent.is_relative_to(folder) for folder in self.__enmappings_folders
|
||||
):
|
||||
self.__remove_extranetwork_mappings_from_path(fullpath)
|
||||
if enmappings_input is None and self.__local_enmappings_input_hash is not None:
|
||||
self.__remove_extranetwork_mappings_from_input()
|
||||
if enmappings_folders is not None or enmappings_input is not None:
|
||||
if enmappings_folders is not None:
|
||||
for f in self.__enmappings_folders:
|
||||
@@ -117,60 +137,57 @@ class PPPExtraNetworkMappings:
|
||||
else:
|
||||
self.extranetwork_mappings = {}
|
||||
self.__enmappings_files = {}
|
||||
self.__local_enmappings_input_hash = None
|
||||
# t2 = time.monotonic_ns()
|
||||
# log(self.__logger, self.__debug_level, logging.INFO, f"Extra network mappings refresh time: {(t2 - t1) / 1_000_000_000:.3f} seconds")
|
||||
|
||||
# def get_extranetwork_mappings(self, key: str) -> list[PPPENMapping]:
|
||||
# """
|
||||
# Get all extra network mappings that match a key.
|
||||
#
|
||||
# Args:
|
||||
# key (str): The key to match (kind:name).
|
||||
#
|
||||
# Returns:
|
||||
# list: A list of all extra network mappings that match the key.
|
||||
# """
|
||||
# keys = sorted(fnmatch.filter(self.extranetwork_mappings.keys(), key))
|
||||
# return [self.extranetwork_mappings[k] for k in keys]
|
||||
|
||||
def __remove_extranetwork_mappings_from_path(self, full_path: str, debug=True):
|
||||
def __remove_extranetwork_mappings_from_path(self, full_path: Path, debug=True):
|
||||
"""
|
||||
Clear all extra network mappings in a file.
|
||||
Clear all extra network mappings from a file.
|
||||
|
||||
Args:
|
||||
full_path (str): The path to the file.
|
||||
full_path (Path): The path to the file.
|
||||
debug (bool): Whether to print debug messages or not.
|
||||
"""
|
||||
last_modified_cached = self.__enmappings_files.get(full_path, None) # a time or a hash
|
||||
if debug and last_modified_cached is not None:
|
||||
if full_path == self.LOCALINPUT_FILENAME:
|
||||
log(self.__logger, self.__debug_level, logging.DEBUG, "Removing extra network mappings from input")
|
||||
else:
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.DEBUG,
|
||||
f"Removing extra network mappings from file: {full_path}",
|
||||
)
|
||||
if full_path in self.__enmappings_files.keys():
|
||||
if debug and full_path in self.__enmappings_files:
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.DEBUG,
|
||||
f"Removing extra network mappings from file: {full_path}",
|
||||
)
|
||||
if full_path in self.__enmappings_files:
|
||||
del self.__enmappings_files[full_path]
|
||||
for key in list(self.extranetwork_mappings.keys()):
|
||||
if self.extranetwork_mappings[key].file == full_path:
|
||||
del self.extranetwork_mappings[key]
|
||||
|
||||
def __get_extranetwork_mappings_in_file(self, full_path: str):
|
||||
def __remove_extranetwork_mappings_from_input(self, debug=True):
|
||||
"""
|
||||
Clear all extra network mappings loaded from inline input.
|
||||
|
||||
Args:
|
||||
debug (bool): Whether to print debug messages or not.
|
||||
"""
|
||||
if debug and self.__local_enmappings_input_hash is not None:
|
||||
log(self.__logger, self.__debug_level, logging.DEBUG, "Removing extra network mappings from input")
|
||||
self.__local_enmappings_input_hash = None
|
||||
for key in list(self.extranetwork_mappings.keys()):
|
||||
if self.extranetwork_mappings[key].file is None:
|
||||
del self.extranetwork_mappings[key]
|
||||
|
||||
def __get_extranetwork_mappings_in_file(self, full_path: Path):
|
||||
"""
|
||||
Get all extra network mappings in a file.
|
||||
|
||||
Args:
|
||||
full_path (str): The path to the file.
|
||||
full_path (Path): The path to the file.
|
||||
"""
|
||||
last_modified = os.path.getmtime(full_path)
|
||||
last_modified = full_path.stat().st_mtime
|
||||
last_modified_cached = self.__enmappings_files.get(full_path, None)
|
||||
if last_modified_cached is not None and last_modified == self.__enmappings_files[full_path]:
|
||||
return
|
||||
filename = os.path.basename(full_path)
|
||||
_, extension = os.path.splitext(filename)
|
||||
extension = full_path.suffix
|
||||
if extension not in (".yaml", ".yml", ".json"):
|
||||
return
|
||||
self.__remove_extranetwork_mappings_from_path(full_path, False)
|
||||
@@ -192,17 +209,17 @@ class PPPExtraNetworkMappings:
|
||||
enmappings_input (str): The input string containing extra network mappings in yaml format.
|
||||
"""
|
||||
new_h = hash(enmappings_input)
|
||||
h = self.__enmappings_files.get(self.LOCALINPUT_FILENAME, None)
|
||||
if h == new_h:
|
||||
if new_h == self.__local_enmappings_input_hash:
|
||||
return
|
||||
self.__remove_extranetwork_mappings_from_path(self.LOCALINPUT_FILENAME, False)
|
||||
if h is not None:
|
||||
was_loaded = self.__local_enmappings_input_hash is not None
|
||||
self.__remove_extranetwork_mappings_from_input(False)
|
||||
if was_loaded:
|
||||
log(self.__logger, self.__debug_level, logging.DEBUG, "Updating extra network mappings from input")
|
||||
enmappings_input = enmappings_input.strip()
|
||||
if enmappings_input != "":
|
||||
try:
|
||||
content = yaml.safe_load(enmappings_input)
|
||||
except yaml.YAMLError as e:
|
||||
content = _YAML(typ="safe").load(enmappings_input)
|
||||
except _YAMLError as e:
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
@@ -211,23 +228,24 @@ class PPPExtraNetworkMappings:
|
||||
)
|
||||
return
|
||||
if content is not None:
|
||||
self.__add_extranetwork_mapping(content, self.LOCALINPUT_FILENAME)
|
||||
self.__enmappings_files[self.LOCALINPUT_FILENAME] = new_h
|
||||
self.__add_extranetwork_mapping(content, None)
|
||||
self.__local_enmappings_input_hash = new_h
|
||||
|
||||
def __add_extranetwork_mapping(self, content: dict[str, dict[str, list[dict]]], full_path: str):
|
||||
def __add_extranetwork_mapping(self, content: dict[str, dict[str, list[dict]]], full_path: Path | None):
|
||||
"""
|
||||
Add an extra network mapping to the extra network mappings dictionary.
|
||||
|
||||
Args:
|
||||
content (object): The content of the extra network mapping.
|
||||
full_path (str): The path to the file that contains it.
|
||||
full_path (Path | None): The path to the file that contains it, or None if from inline input.
|
||||
"""
|
||||
file_str = str(full_path) if full_path is not None else "input"
|
||||
if not isinstance(content, dict):
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid extra network mapping in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid extra network mapping in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
return
|
||||
for kind, maps in content.items():
|
||||
@@ -236,7 +254,7 @@ class PPPExtraNetworkMappings:
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid extra network mapping definition for '{escape_single_quotes(kind)}:*' in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid extra network mapping definition for '{escape_single_quotes(kind)}:*' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
else:
|
||||
for name, variants in maps.items():
|
||||
@@ -246,75 +264,78 @@ class PPPExtraNetworkMappings:
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid extra network mapping definition for '{escape_single_quotes(key)}' in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid extra network mapping definition for '{escape_single_quotes(key)}' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
elif self.extranetwork_mappings.get(key, None) is not None:
|
||||
f = (
|
||||
str(self.extranetwork_mappings[key].file)
|
||||
if self.extranetwork_mappings[key].file is not None
|
||||
else "input"
|
||||
)
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Duplicate extra network mapping '{escape_single_quotes(key)}' in file '{escape_single_quotes(full_path)}' and '{escape_single_quotes(self.extranetwork_mappings[key].file)}'!",
|
||||
f"Duplicate extra network mapping '{escape_single_quotes(key)}' in file '{escape_single_quotes(file_str)}' and '{escape_single_quotes(f)}'!",
|
||||
)
|
||||
elif not isinstance(variants, list) or not all(isinstance(v, dict) for v in variants):
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid extra network mapping definition for '{escape_single_quotes(key)}' in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid extra network mapping definition for '{escape_single_quotes(key)}' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
else:
|
||||
self.extranetwork_mappings[key] = PPPENMapping(full_path, kind, name, variants)
|
||||
|
||||
def __get_extranetwork_mappings_in_structured_file(self, full_path):
|
||||
def __get_extranetwork_mappings_in_structured_file(self, full_path: Path):
|
||||
"""
|
||||
Get all extra network mappings in a structured file.
|
||||
|
||||
Args:
|
||||
full_path (str): The path to the file.
|
||||
base (str): The base path for the extra network mappings.
|
||||
full_path (Path): The path to the file.
|
||||
"""
|
||||
try:
|
||||
try:
|
||||
with open(full_path, "r", encoding="utf-8") as file:
|
||||
content = yaml.safe_load(file)
|
||||
content = _YAML(typ="safe").load(file)
|
||||
except: # pylint: disable=bare-except
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Could not read file '{escape_single_quotes(full_path)}' with utf-8 encoding, trying windows-1252...",
|
||||
f"Could not read file '{escape_single_quotes(str(full_path))}' with utf-8 encoding, trying windows-1252...",
|
||||
)
|
||||
with open(full_path, "r", encoding="windows-1252") as file:
|
||||
content = yaml.safe_load(file)
|
||||
content = _YAML(typ="safe").load(file)
|
||||
self.__add_extranetwork_mapping(content, full_path)
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.ERROR,
|
||||
f"Error reading extra network mappings from file '{escape_single_quotes(full_path)}': {e}",
|
||||
f"Error reading extra network mappings from file '{escape_single_quotes(str(full_path))}': {e}",
|
||||
)
|
||||
|
||||
def __get_extranetwork_mappings_in_path(self, path: str):
|
||||
def __get_extranetwork_mappings_in_path(self, path: Path):
|
||||
"""
|
||||
Get all extra network mappings in a path.
|
||||
|
||||
Args:
|
||||
path (str): The path (folder or file).
|
||||
path (Path): The path (folder or file).
|
||||
"""
|
||||
if not os.path.exists(path):
|
||||
if not path.exists():
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Extra network mappings path '{escape_single_quotes(path)}' does not exist!",
|
||||
f"Extra network mappings path '{escape_single_quotes(str(path))}' does not exist!",
|
||||
)
|
||||
return
|
||||
if os.path.isfile(path):
|
||||
if path.is_file():
|
||||
self.__get_extranetwork_mappings_in_file(path)
|
||||
return
|
||||
for filename in os.listdir(path):
|
||||
full_path = os.path.abspath(os.path.join(path, filename))
|
||||
if os.path.basename(full_path).startswith("."):
|
||||
for child in path.iterdir():
|
||||
if child.name.startswith("."):
|
||||
continue
|
||||
self.__get_extranetwork_mappings_in_path(full_path)
|
||||
self.__get_extranetwork_mappings_in_path(child)
|
||||
|
||||
+12
-4
@@ -54,15 +54,14 @@ class PromptPostProcessorLogFactory: # pylint: disable=too-few-public-methods
|
||||
colored_record.levelname = f"{seq}{levelname:8s}{self.COLORS['RESET']}"
|
||||
return super().format(colored_record)
|
||||
|
||||
def __init__(self, filename = None):
|
||||
def __init__(self, filename=None):
|
||||
"""
|
||||
Initializes the PromptPostProcessor class.
|
||||
Initializes the PromptPostProcessorLogFactory class.
|
||||
|
||||
This method sets up the logger for the PromptPostProcessor class and configures its log level and handlers.
|
||||
|
||||
Args:
|
||||
filename (str, optional): The name of the file to log to. Defaults to None.
|
||||
app (SUPPORTED_APPS): The application for which the logger is being created.
|
||||
|
||||
Returns:
|
||||
None
|
||||
@@ -98,7 +97,16 @@ class PromptPostProcessorLogCustomAdapter(logging.LoggerAdapter):
|
||||
"""
|
||||
return f"[PPP] {msg}", kwargs
|
||||
|
||||
def log(logger: logging.Logger, debug_level: DEBUG_LEVEL, kind: int, message: str, min_level: DEBUG_LEVEL | None = None, formatted: bool = True, exc_info: bool = False):
|
||||
|
||||
def log(
|
||||
logger: logging.Logger,
|
||||
debug_level: DEBUG_LEVEL,
|
||||
kind: int,
|
||||
message: str,
|
||||
min_level: DEBUG_LEVEL | None = None,
|
||||
formatted: bool = True,
|
||||
exc_info=None,
|
||||
):
|
||||
if logger:
|
||||
if min_level is None:
|
||||
if kind == logging.DEBUG:
|
||||
|
||||
+459
-245
File diff suppressed because it is too large
Load Diff
@@ -1,3 +1,28 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def get_version_from_pyproject() -> str:
|
||||
"""
|
||||
Reads the version from the pyproject.toml file.
|
||||
|
||||
Returns:
|
||||
str: The version string.
|
||||
"""
|
||||
version_str = "0.0.0"
|
||||
try:
|
||||
pyproject_path = Path(__file__).resolve().parent / "pyproject.toml"
|
||||
with open(pyproject_path, "r", encoding="utf-8") as file:
|
||||
for line in file:
|
||||
if line.startswith("version = "):
|
||||
version_str = line.split("=")[1].strip().strip('"')
|
||||
break
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logging.getLogger().exception(e)
|
||||
return version_str
|
||||
|
||||
|
||||
def deep_freeze(obj):
|
||||
"""
|
||||
Deep freeze an object.
|
||||
@@ -43,6 +68,22 @@ def escape_double_quotes(s: str):
|
||||
return s.replace('"', '\\"')
|
||||
|
||||
|
||||
def repr_value(s: Any):
|
||||
"""
|
||||
Return a string representation of a value, escaping single quotes.
|
||||
|
||||
Args:
|
||||
s (Any): The value to represent.
|
||||
Returns:
|
||||
str: The string representation of the value.
|
||||
"""
|
||||
if isinstance(s, str):
|
||||
return f"'{escape_single_quotes(s)}'"
|
||||
if isinstance(s, bool):
|
||||
return "true" if s else "false"
|
||||
return str(s)
|
||||
|
||||
|
||||
def format_output(text: str) -> str:
|
||||
"""
|
||||
Formats the output text by encoding it using unicode_escape and decoding it using utf-8.
|
||||
|
||||
+91
-48
@@ -1,23 +1,34 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
ScalarValue = str | int | float | bool
|
||||
VariableValue = ScalarValue | list | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class VariableEntry:
|
||||
"""Holds all state for a single user variable."""
|
||||
|
||||
value: Any = field(default=None) # raw unevaluated value or evaluated on set
|
||||
last_echoed_value: Any = field(default=None) # raw unevaluated value at last echo
|
||||
last_echoed_evaluated_value: ScalarValue | None = field(default=None) # evaluated value at last echo
|
||||
|
||||
|
||||
class VariableRepository:
|
||||
"""
|
||||
Unified repository for system, user, and echoed prompt variables.
|
||||
Unified repository for system and user prompt variables.
|
||||
|
||||
System variables (underscore-prefixed names like ``_model``) are populated
|
||||
once per processing session and are read-only during prompt evaluation.
|
||||
|
||||
User variables are created and mutated by set/echo constructs in the prompt.
|
||||
|
||||
Echoed variables record which user variables have already been output and
|
||||
with what resolved string value.
|
||||
Each user variable is stored as a :class:`VariableEntry` that tracks the
|
||||
set value and the last value that was echoed into the prompt output.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._system: dict[str, Any] = {}
|
||||
self._user: dict[str, Any] = {}
|
||||
self._echoed: dict[str, str] = {}
|
||||
self._system: dict[str, VariableValue] = {}
|
||||
self._vars: dict[str, VariableEntry] = {}
|
||||
|
||||
def name_is_system(self, name: str) -> bool:
|
||||
"""Return True if *name* is a system variable (i.e. starts with an underscore)."""
|
||||
@@ -25,17 +36,20 @@ class VariableRepository:
|
||||
|
||||
# ---- System variables ----
|
||||
|
||||
def get_system(self, name: str, default: Any = None) -> Any:
|
||||
def get_system(self, name: str, default: VariableValue = None) -> VariableValue:
|
||||
"""Return the value of a system variable, or *default* if absent."""
|
||||
return self._system.get(name, default)
|
||||
|
||||
def set_system(self, name: str, value: Any) -> None:
|
||||
def set_system(self, name: str, value: VariableValue) -> None:
|
||||
"""Set a system variable."""
|
||||
if not self.name_is_system(name):
|
||||
raise ValueError(f"Invalid system variable name '{name}': must start with an underscore")
|
||||
self._system[name] = value
|
||||
if value is None:
|
||||
self._system.pop(name, None)
|
||||
else:
|
||||
self._system[name] = value
|
||||
|
||||
def update_system(self, mapping: dict[str, Any]) -> None:
|
||||
def update_system(self, mapping: dict[str, VariableValue]) -> None:
|
||||
"""Bulk-update system variables from *mapping*."""
|
||||
for name in mapping:
|
||||
if not self.name_is_system(name):
|
||||
@@ -46,43 +60,90 @@ class VariableRepository:
|
||||
"""Remove all system variables."""
|
||||
self._system.clear()
|
||||
|
||||
def get_all_system(self) -> dict[str, Any]:
|
||||
@property
|
||||
def all_system(self) -> dict[str, VariableValue]:
|
||||
"""Return a shallow copy of all system variables."""
|
||||
return self._system.copy()
|
||||
|
||||
# ---- User variables ----
|
||||
|
||||
def _entry(self, name: str) -> VariableEntry:
|
||||
"""Return (creating if necessary) the :class:`VariableEntry` for *name*."""
|
||||
if name not in self._vars:
|
||||
self._vars[name] = VariableEntry()
|
||||
return self._vars[name]
|
||||
|
||||
def get_user(self, name: str, default: Any = None) -> Any:
|
||||
"""Return the value of a user variable, or *default* if absent."""
|
||||
return self._user.get(name, default)
|
||||
entry = self._vars.get(name)
|
||||
if entry is None or entry.value is None:
|
||||
return default
|
||||
return entry.value
|
||||
|
||||
def set_user(self, name: str, value: Any) -> None:
|
||||
"""Set a user variable."""
|
||||
"""Set the value of a user variable."""
|
||||
if self.name_is_system(name):
|
||||
raise ValueError(f"Invalid user variable name '{name}': must not start with an underscore")
|
||||
self._user[name] = value
|
||||
entry = self._entry(name)
|
||||
entry.value = value
|
||||
|
||||
def delete_user(self, name: str) -> None:
|
||||
"""Remove a user variable (no-op if it does not exist)."""
|
||||
self._user.pop(name, None)
|
||||
"""
|
||||
Remove the value for a user variable.
|
||||
"""
|
||||
entry = self._vars.get(name)
|
||||
if entry is None:
|
||||
return
|
||||
del self._vars[name]
|
||||
|
||||
def clear_user(self) -> None:
|
||||
"""Remove all user variables."""
|
||||
self._user.clear()
|
||||
"""
|
||||
Clear the values for all user variables.
|
||||
"""
|
||||
self._vars.clear()
|
||||
|
||||
# ---- Echoed variables ----
|
||||
@property
|
||||
def all_user(self) -> set[str]:
|
||||
"""Return the set of all user-variable keys (those with any non-None field)."""
|
||||
return set(self._vars)
|
||||
|
||||
def get_echoed_value(self, name: str, default: str | None = None) -> str | None:
|
||||
"""Return the echoed string value for *name*, or *default* if not echoed."""
|
||||
return self._echoed.get(name, default)
|
||||
def set_echoed_value(self, name: str, value: Any, evaluated_value: ScalarValue) -> None:
|
||||
"""Record that *name* was echoed into the prompt with *value*."""
|
||||
if not self.name_is_system(name):
|
||||
entry = self._entry(name)
|
||||
entry.last_echoed_value = value
|
||||
entry.last_echoed_evaluated_value = evaluated_value
|
||||
|
||||
def echo(self, name: str, value: str) -> None:
|
||||
"""Record that *name* was echoed with *value*."""
|
||||
self._echoed[name] = value
|
||||
def get_echoed_value(self, name: str, default: ScalarValue | None = None) -> ScalarValue | None:
|
||||
"""Return the last echoed value for *name*, or *default* if it has not been echoed."""
|
||||
entry = self._vars.get(name)
|
||||
if entry is None:
|
||||
return default
|
||||
return entry.last_echoed_evaluated_value if entry.last_echoed_evaluated_value is not None else default
|
||||
|
||||
def clear_echoed(self) -> None:
|
||||
"""Remove all echoed-variable records."""
|
||||
self._echoed.clear()
|
||||
def backup_user(self) -> dict[str, VariableEntry]:
|
||||
"""Return a per-entry shallow-copy snapshot of all user variables for rollback."""
|
||||
return {
|
||||
name: VariableEntry(entry.value, entry.last_echoed_value, entry.last_echoed_evaluated_value)
|
||||
for name, entry in self._vars.items()
|
||||
}
|
||||
|
||||
def restore_user(self, backup: dict[str, VariableEntry]) -> None:
|
||||
"""Restore user variables from a snapshot made by :meth:`backup_user_and_echoed`."""
|
||||
self._vars.clear()
|
||||
self._vars.update(
|
||||
{
|
||||
name: VariableEntry(entry.value, entry.last_echoed_value, entry.last_echoed_evaluated_value)
|
||||
for name, entry in backup.items()
|
||||
}
|
||||
)
|
||||
|
||||
def backup_user_and_output(self) -> dict[str, VariableEntry]:
|
||||
"""Return a per-entry shallow-copy snapshot of all user variables and output variables."""
|
||||
return {
|
||||
name: VariableEntry(entry.value, entry.last_echoed_value, entry.last_echoed_evaluated_value)
|
||||
for name, entry in self._vars.items()
|
||||
} | {name: VariableEntry(entry) for name, entry in self._system.items() if name.startswith("_output_")}
|
||||
|
||||
# ---- Combined queries ----
|
||||
|
||||
@@ -96,22 +157,4 @@ class VariableRepository:
|
||||
"""
|
||||
if name in self._system:
|
||||
return self._system.get(name, default)
|
||||
return self._user.get(name, default)
|
||||
|
||||
def all_user_or_echoed_keys(self) -> set[str]:
|
||||
"""Return the union of user-variable and echoed-variable keys."""
|
||||
return set(self._user.keys()) | set(self._echoed.keys())
|
||||
|
||||
# ---- State backup / restore ----
|
||||
|
||||
def backup_user_and_echoed(self) -> tuple[dict[str, Any], dict[str, str]]:
|
||||
"""Return shallow-copy snapshots of user and echoed variables for rollback."""
|
||||
return self._user.copy(), self._echoed.copy()
|
||||
|
||||
def restore_user_and_echoed(self, backup: tuple[dict[str, Any], dict[str, str]]) -> None:
|
||||
"""Restore user and echoed variables from a snapshot made by :meth:`backup_user_and_echoed`."""
|
||||
user_backup, echoed_backup = backup
|
||||
self._user.clear()
|
||||
self._user.update(user_backup)
|
||||
self._echoed.clear()
|
||||
self._echoed.update(echoed_backup)
|
||||
return self.get_user(name, default)
|
||||
|
||||
+167
-129
@@ -1,9 +1,10 @@
|
||||
import fnmatch
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
import re
|
||||
from typing import Any, Optional
|
||||
import logging
|
||||
import yaml
|
||||
from ruamel.yaml import YAML as _YAML
|
||||
from ruamel.yaml.error import YAMLError as _YAMLError
|
||||
|
||||
from ppp_logging import DEBUG_LEVEL, log
|
||||
from ppp_utils import deep_freeze, escape_single_quotes
|
||||
@@ -15,15 +16,15 @@ class PPPWildcard:
|
||||
|
||||
Attributes:
|
||||
key (str): The key of the wildcard.
|
||||
file (str): The path to the file where the wildcard is defined.
|
||||
file (Path | None): The path to the file where the wildcard is defined, or None if from inline input.
|
||||
unprocessed_choices (list[str]): The unprocessed choices of the wildcard.
|
||||
choices (list[dict]): The processed choices of the wildcard.
|
||||
options (dict): The options of the wildcard.
|
||||
choices (list[dict]): The processed choices of the wildcard.
|
||||
"""
|
||||
|
||||
def __init__(self, fullpath: str, key: str, choices: list[str]):
|
||||
def __init__(self, fullpath: Path | None, key: str, choices: list[str]):
|
||||
self.key: str = key
|
||||
self.file: str = fullpath
|
||||
self.file: Path | None = fullpath
|
||||
self.unprocessed_choices: list[str] = choices
|
||||
self.choices: list[dict] = None
|
||||
self.options: dict = None
|
||||
@@ -35,7 +36,7 @@ class PPPWildcard:
|
||||
def __sizeof__(self):
|
||||
return (
|
||||
self.key.__sizeof__()
|
||||
+ self.file.__sizeof__()
|
||||
+ (self.file.__sizeof__() if self.file is not None else 0)
|
||||
+ self.unprocessed_choices.__sizeof__()
|
||||
+ self.choices.__sizeof__()
|
||||
+ self.options.__sizeof__()
|
||||
@@ -51,13 +52,13 @@ class PPPWildcards:
|
||||
"""
|
||||
|
||||
DEFAULT_WILDCARDS_FOLDER = "wildcards"
|
||||
LOCALINPUT_FILENAME = R"//INPUT\\"
|
||||
|
||||
def __init__(self, logger=None):
|
||||
self.__logger: logging.Logger = logger
|
||||
self.__debug_level = DEBUG_LEVEL.none
|
||||
self.__wildcards_folders = []
|
||||
self.__wildcard_files = {}
|
||||
self.__wildcards_folders: list[Path] = []
|
||||
self.__wildcard_files: dict[Path, float] = {}
|
||||
self.__local_input_hash: int | None = None
|
||||
self.__wildcard_default_filters: dict[str, list[list[str]]] = {}
|
||||
self.wildcards: dict[str, PPPWildcard] = {}
|
||||
|
||||
@@ -70,34 +71,34 @@ class PPPWildcards:
|
||||
def refresh_wildcards(
|
||||
self,
|
||||
debug_level: DEBUG_LEVEL,
|
||||
wildcards_folders: Optional[list[str]],
|
||||
wildcards_folders: Optional[list[Path]],
|
||||
wildcards_input: str = None,
|
||||
):
|
||||
"""
|
||||
Initialize the wildcards.
|
||||
"""
|
||||
self.reset_default_filters()
|
||||
self.__debug_level = debug_level
|
||||
self.__wildcards_folders = wildcards_folders or []
|
||||
self.__wildcards_folders = [Path(f) for f in (wildcards_folders or [])]
|
||||
# log(self.__logger, self.__debug_level, logging.INFO, "Refreshing wildcards...")
|
||||
# t1 = time.monotonic_ns()
|
||||
for fullpath in list(self.__wildcard_files.keys()):
|
||||
if fullpath != self.LOCALINPUT_FILENAME:
|
||||
path = os.path.dirname(fullpath)
|
||||
if not os.path.exists(fullpath) or not any(
|
||||
Path(path).is_relative_to(folder) for folder in self.__wildcards_folders
|
||||
):
|
||||
self.__remove_wildcards_from_path(fullpath)
|
||||
elif wildcards_input is None:
|
||||
if not fullpath.exists() or not any(
|
||||
fullpath.parent.is_relative_to(folder) for folder in self.__wildcards_folders
|
||||
):
|
||||
self.__remove_wildcards_from_path(fullpath)
|
||||
if wildcards_input is None and self.__local_input_hash is not None:
|
||||
self.__remove_wildcards_from_input()
|
||||
if wildcards_folders is not None or wildcards_input is not None:
|
||||
if wildcards_folders is not None:
|
||||
for f in self.__wildcards_folders:
|
||||
self.__get_wildcards_in_path(f if os.path.isdir(f) else os.path.dirname(f), f)
|
||||
self.__get_wildcards_in_path(f if f.is_dir() else f.parent, f)
|
||||
if wildcards_input is not None:
|
||||
self.__get_wildcards_in_input(wildcards_input)
|
||||
else:
|
||||
self.wildcards = {}
|
||||
self.__wildcard_files = {}
|
||||
self.__local_input_hash = None
|
||||
# t2 = time.monotonic_ns()
|
||||
# log(self.__logger, self.__debug_level, logging.INFO, f"Wildcards refresh time: {(t2 - t1) / 1_000_000_000:.3f} seconds")
|
||||
|
||||
@@ -114,84 +115,84 @@ class PPPWildcards:
|
||||
keys = sorted(fnmatch.filter(self.wildcards.keys(), key))
|
||||
return [self.wildcards[k] for k in keys]
|
||||
|
||||
def __get_keys_in_dict(self, dictionary: dict, prefix="") -> list[str]:
|
||||
def __get_wc_in_dict(self, dictionary: dict, prefix="", file_str: str = "") -> list[tuple[str, Any]]:
|
||||
"""
|
||||
Get all keys in a dictionary.
|
||||
Get all wildcards in a dictionary, along their object.
|
||||
|
||||
Args:
|
||||
dictionary (dict): The dictionary to check.
|
||||
prefix (str): The prefix for the current key.
|
||||
file_str (str): The file string for logging purposes.
|
||||
|
||||
Returns:
|
||||
list: A list of all keys in the dictionary, including nested keys.
|
||||
list: A list of all leaf wildcards in the dictionary.
|
||||
"""
|
||||
keys = []
|
||||
for key in dictionary.keys():
|
||||
if isinstance(dictionary[key], dict):
|
||||
keys.extend(self.__get_keys_in_dict(dictionary[key], prefix + key + "/"))
|
||||
else:
|
||||
keys.append(prefix + str(key))
|
||||
return keys
|
||||
|
||||
def __get_nested(self, dictionary: dict, keys: str) -> object:
|
||||
"""
|
||||
Get a nested value from a dictionary.
|
||||
|
||||
Args:
|
||||
dictionary (dict): The dictionary to check.
|
||||
keys (str): The keys to get the value from.
|
||||
|
||||
Returns:
|
||||
object: The value of the nested keys in the dictionary.
|
||||
"""
|
||||
keys = keys.split("/")
|
||||
current_dict = dictionary
|
||||
for key in keys:
|
||||
current_dict = current_dict.get(key)
|
||||
if current_dict is None:
|
||||
return None
|
||||
return current_dict
|
||||
|
||||
def __remove_wildcards_from_path(self, full_path: str, debug=True):
|
||||
"""
|
||||
Clear all wildcards in a file.
|
||||
|
||||
Args:
|
||||
full_path (str): The path to the file.
|
||||
debug (bool): Whether to print debug messages or not.
|
||||
"""
|
||||
last_modified_cached = self.__wildcard_files.get(full_path, None) # a time or a hash
|
||||
if debug and last_modified_cached is not None:
|
||||
if full_path == self.LOCALINPUT_FILENAME:
|
||||
log(self.__logger, self.__debug_level, logging.DEBUG, "Removing from memory wildcards from input")
|
||||
else:
|
||||
wc = []
|
||||
for key, obj in dictionary.items():
|
||||
strkey = str(key)
|
||||
if not self.__check_key_validity(strkey):
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.DEBUG,
|
||||
f"Removing from memory wildcards from file: {full_path}",
|
||||
logging.WARNING,
|
||||
f"Invalid wildcard name part '{escape_single_quotes(prefix + strkey)}' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
if full_path in self.__wildcard_files.keys():
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
wc.extend(self.__get_wc_in_dict(obj, prefix + strkey + "/", file_str))
|
||||
else:
|
||||
wc.append((prefix + strkey, obj))
|
||||
return wc
|
||||
|
||||
def __remove_wildcards_from_path(self, full_path: Path, debug=True):
|
||||
"""
|
||||
Clear all wildcards from a file.
|
||||
|
||||
Args:
|
||||
full_path (Path): The path to the file.
|
||||
debug (bool): Whether to print debug messages or not.
|
||||
"""
|
||||
if debug and full_path in self.__wildcard_files:
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.DEBUG,
|
||||
f"Removing from memory wildcards from file: {full_path}",
|
||||
)
|
||||
if full_path in self.__wildcard_files:
|
||||
del self.__wildcard_files[full_path]
|
||||
for key in list(self.wildcards.keys()):
|
||||
if self.wildcards[key].file == full_path:
|
||||
del self.wildcards[key]
|
||||
|
||||
def __get_wildcards_in_file(self, base, full_path: str):
|
||||
def __remove_wildcards_from_input(self, debug=True):
|
||||
"""
|
||||
Clear all wildcards loaded from inline input.
|
||||
|
||||
Args:
|
||||
debug (bool): Whether to print debug messages or not.
|
||||
"""
|
||||
if debug and self.__local_input_hash is not None:
|
||||
log(self.__logger, self.__debug_level, logging.DEBUG, "Removing from memory wildcards from input")
|
||||
self.__local_input_hash = None
|
||||
for key in list(self.wildcards.keys()):
|
||||
if self.wildcards[key].file is None:
|
||||
del self.wildcards[key]
|
||||
|
||||
def __get_wildcards_in_file(self, base: Path, full_path: Path):
|
||||
"""
|
||||
Get all wildcards in a file.
|
||||
|
||||
Args:
|
||||
base (str): The base path for the wildcards.
|
||||
full_path (str): The path to the file.
|
||||
base (Path): The base path for the wildcards.
|
||||
full_path (Path): The path to the file.
|
||||
"""
|
||||
try:
|
||||
last_modified = os.path.getmtime(full_path)
|
||||
last_modified = full_path.stat().st_mtime
|
||||
last_modified_cached = self.__wildcard_files.get(full_path, None)
|
||||
if last_modified_cached is not None and last_modified == self.__wildcard_files[full_path]:
|
||||
return
|
||||
filename = os.path.basename(full_path)
|
||||
_, extension = os.path.splitext(filename)
|
||||
extension = full_path.suffix
|
||||
if extension not in (".txt", ".json", ".yaml", ".yml"):
|
||||
return
|
||||
self.__remove_wildcards_from_path(full_path, False)
|
||||
@@ -207,7 +208,7 @@ class PPPWildcards:
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.ERROR,
|
||||
f"Error reading wildcard file '{escape_single_quotes(full_path)}': {e}",
|
||||
f"Error reading wildcard file '{escape_single_quotes(str(full_path))}': {e}",
|
||||
)
|
||||
|
||||
def __get_wildcards_in_input(self, wildcards_input: str):
|
||||
@@ -219,22 +220,22 @@ class PPPWildcards:
|
||||
"""
|
||||
try:
|
||||
new_h = hash(wildcards_input)
|
||||
h = self.__wildcard_files.get(self.LOCALINPUT_FILENAME, None)
|
||||
if h == new_h:
|
||||
if new_h == self.__local_input_hash:
|
||||
return
|
||||
self.__remove_wildcards_from_path(self.LOCALINPUT_FILENAME, False)
|
||||
if h is not None:
|
||||
was_loaded = self.__local_input_hash is not None
|
||||
self.__remove_wildcards_from_input(False)
|
||||
if was_loaded:
|
||||
log(self.__logger, self.__debug_level, logging.DEBUG, "Updating wildcards from input")
|
||||
wildcards_input = wildcards_input.strip()
|
||||
if wildcards_input != "":
|
||||
try:
|
||||
content = yaml.safe_load(wildcards_input)
|
||||
except yaml.YAMLError as e:
|
||||
content = _YAML(typ="safe").load(wildcards_input)
|
||||
except _YAMLError as e:
|
||||
log(self.__logger, self.__debug_level, logging.WARNING, f"Invalid format for input wildcards: {e}")
|
||||
return
|
||||
if content is not None:
|
||||
self.__add_wildcard(content, self.LOCALINPUT_FILENAME, [self.LOCALINPUT_FILENAME])
|
||||
self.__wildcard_files[self.LOCALINPUT_FILENAME] = new_h
|
||||
self.__add_wildcard(content, None, [""])
|
||||
self.__local_input_hash = new_h
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
log(self.__logger, self.__debug_level, logging.ERROR, f"Error reading wildcards input: {e}")
|
||||
|
||||
@@ -278,15 +279,15 @@ class PPPWildcards:
|
||||
Returns:
|
||||
bool: Whether the dictionary is a valid choice options dictionary or not.
|
||||
"""
|
||||
return all(k in ["command", "labels", "weight", "if", "content", "text"] for k in d.keys())
|
||||
return all(k in ["command", "labels", "weight", "if", "else", "content", "text"] for k in d.keys())
|
||||
|
||||
def __get_choices(self, obj: object, full_path: str, key_parts: list[str]) -> list:
|
||||
def __get_choices(self, obj: object, full_path: Path | None, key_parts: list[str]) -> list:
|
||||
"""
|
||||
We process the choices in the object and return them as a list.
|
||||
|
||||
Args:
|
||||
obj (object): the value of a wildcard
|
||||
full_path (str): path to the file where the wildcard is defined
|
||||
full_path (Path | None): path to the file where the wildcard is defined, or None if from inline input
|
||||
key_parts (list[str]): parts of the key for the wildcard
|
||||
|
||||
Returns:
|
||||
@@ -298,12 +299,13 @@ class PPPWildcards:
|
||||
return [obj]
|
||||
if isinstance(obj, (int, float, bool)):
|
||||
return [str(obj)]
|
||||
file_str = str(full_path) if full_path is not None else "input"
|
||||
if not isinstance(obj, list) or len(obj) == 0:
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid format in wildcard '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid format in wildcard '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
return None
|
||||
choices = []
|
||||
@@ -320,17 +322,17 @@ class PPPWildcards:
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid choice {i+1} in wildcard '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid choice {i+1} in wildcard '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
return choices
|
||||
|
||||
def __process_dict_choice(self, c: dict, full_path: str, key_parts: list[str], i: int) -> dict:
|
||||
def __process_dict_choice(self, c: dict, full_path: Path | None, key_parts: list[str], i: int) -> dict:
|
||||
"""
|
||||
Process a dictionary choice.
|
||||
|
||||
Args:
|
||||
c (dict): The dictionary choice.
|
||||
full_path (str): The path to the file.
|
||||
full_path (Path | None): The path to the file, or None if from inline input.
|
||||
key_parts (list[str]): The parts of the key.
|
||||
i (int): The index of the choice.
|
||||
|
||||
@@ -353,20 +355,21 @@ class PPPWildcards:
|
||||
# we assume it is an anonymous wildcard with options
|
||||
firstkey = list(c.keys())[0]
|
||||
return self.__create_anonymous_wildcard(full_path, key_parts, i, c[firstkey], firstkey)
|
||||
file_str = str(full_path) if full_path is not None else "input"
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid choice {i+1} in wildcard '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid choice {i+1} in wildcard '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
return None
|
||||
|
||||
def __create_anonymous_wildcard(self, full_path, key_parts, i, content, options=None):
|
||||
def __create_anonymous_wildcard(self, full_path: Path | None, key_parts, i, content, options=None):
|
||||
"""
|
||||
Create an anonymous wildcard.
|
||||
|
||||
Args:
|
||||
full_path (str): The path to the file that contains it.
|
||||
full_path (Path | None): The path to the file that contains it, or None if from inline input.
|
||||
key_parts (list[str]): The parts of the key.
|
||||
i (int): The index of the wildcard.
|
||||
content (object): The content of the wildcard.
|
||||
@@ -382,20 +385,51 @@ class PPPWildcards:
|
||||
value = f"{options}::{value}"
|
||||
return value
|
||||
|
||||
def __add_wildcard(self, content: object, full_path: str, external_key_parts: list[str]):
|
||||
def __check_key_validity(self, key: str) -> bool:
|
||||
"""
|
||||
Check if a key is valid.
|
||||
|
||||
Args:
|
||||
key (str): The key to check.
|
||||
|
||||
Returns:
|
||||
bool: Whether the key is valid or not.
|
||||
"""
|
||||
match = re.match(r"^[a-zA-Z0-9\-_#]+$", key)
|
||||
if match is None:
|
||||
return False
|
||||
return True
|
||||
|
||||
def __add_wildcard(self, content: object, full_path: Path | None, external_key_parts: list[str]):
|
||||
"""
|
||||
Add a wildcard to the wildcards dictionary.
|
||||
|
||||
Args:
|
||||
content (object): The content of the wildcard.
|
||||
full_path (str): The path to the file that contains it.
|
||||
full_path (Path | None): The path to the file that contains it, or None if from inline input.
|
||||
external_key_parts (list[str]): The parts of the key.
|
||||
"""
|
||||
file_str = str(full_path) if full_path is not None else "input"
|
||||
|
||||
def existing_file_str(wc):
|
||||
return str(wc.file) if wc.file is not None else "input"
|
||||
|
||||
key_parts = external_key_parts.copy()
|
||||
if key_parts:
|
||||
for part in key_parts:
|
||||
strkey = str(part)
|
||||
if strkey and not self.__check_key_validity(strkey):
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid wildcard name start '{escape_single_quotes('/'.join(key_parts))}' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
return
|
||||
if isinstance(content, dict):
|
||||
key_parts.pop()
|
||||
keys = self.__get_keys_in_dict(content)
|
||||
for key in keys:
|
||||
key_parts.pop() # we don't want the name of the filename to be included, since the dict keys will be used instead
|
||||
keys = self.__get_wc_in_dict(content, "", file_str)
|
||||
for key, obj in keys:
|
||||
tmp_key_parts = key_parts.copy()
|
||||
tmp_key_parts.extend(key.split("/"))
|
||||
fullkey = "/".join(tmp_key_parts)
|
||||
@@ -404,24 +438,23 @@ class PPPWildcards:
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Duplicate wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(full_path)}' and '{escape_single_quotes(self.wildcards[fullkey].file)}'!",
|
||||
f"Duplicate wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}' and '{escape_single_quotes(existing_file_str(self.wildcards[fullkey]))}'!",
|
||||
)
|
||||
else:
|
||||
obj = self.__get_nested(content, key)
|
||||
choices = self.__get_choices(obj, full_path, tmp_key_parts)
|
||||
if choices is None:
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
elif fullkey.startswith("_"):
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid wildcard name '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(full_path)}'! (cannot start with underscore)",
|
||||
f"Invalid wildcard name '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}'! (cannot start with underscore)",
|
||||
)
|
||||
else:
|
||||
self.wildcards[fullkey] = PPPWildcard(full_path, fullkey, choices)
|
||||
@@ -435,7 +468,7 @@ class PPPWildcards:
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid wildcard in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid wildcard in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
return
|
||||
fullkey = "/".join(key_parts)
|
||||
@@ -444,7 +477,7 @@ class PPPWildcards:
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Duplicate wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(full_path)}' and '{escape_single_quotes(self.wildcards[fullkey].file)}'!",
|
||||
f"Duplicate wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}' and '{escape_single_quotes(existing_file_str(self.wildcards[fullkey]))}'!",
|
||||
)
|
||||
else:
|
||||
choices = self.__get_choices(content, full_path, key_parts)
|
||||
@@ -453,52 +486,50 @@ class PPPWildcards:
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(full_path)}'!",
|
||||
f"Invalid wildcard '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}'!",
|
||||
)
|
||||
elif fullkey.startswith("_"):
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Invalid wildcard name '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(full_path)}'! (cannot start with underscore)",
|
||||
f"Invalid wildcard name '{escape_single_quotes(fullkey)}' in file '{escape_single_quotes(file_str)}'! (cannot start with underscore)",
|
||||
)
|
||||
else:
|
||||
self.wildcards[fullkey] = PPPWildcard(full_path, fullkey, choices)
|
||||
|
||||
def __get_wildcards_in_structured_file(self, full_path, base):
|
||||
def __get_wildcards_in_structured_file(self, full_path: Path, base: Path):
|
||||
"""
|
||||
Get all wildcards in a structured file.
|
||||
|
||||
Args:
|
||||
full_path (str): The path to the file.
|
||||
base (str): The base path for the wildcards.
|
||||
full_path (Path): The path to the file.
|
||||
base (Path): The base path for the wildcards.
|
||||
"""
|
||||
external_key: str = os.path.relpath(os.path.splitext(full_path)[0], base)
|
||||
external_key_parts = external_key.split(os.sep)
|
||||
external_key_parts = list(full_path.with_suffix("").relative_to(base).parts)
|
||||
try:
|
||||
with open(full_path, "r", encoding="utf-8") as file:
|
||||
content = yaml.safe_load(file)
|
||||
content = _YAML(typ="safe").load(file)
|
||||
except: # pylint: disable=bare-except
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Could not read file '{escape_single_quotes(full_path)}' with utf-8 encoding, trying windows-1252...",
|
||||
f"Could not read file '{escape_single_quotes(str(full_path))}' with utf-8 encoding, trying windows-1252...",
|
||||
)
|
||||
with open(full_path, "r", encoding="windows-1252") as file:
|
||||
content = yaml.safe_load(file)
|
||||
content = _YAML(typ="safe").load(file)
|
||||
self.__add_wildcard(content, full_path, external_key_parts)
|
||||
|
||||
def __get_wildcards_in_text_file(self, full_path, base):
|
||||
def __get_wildcards_in_text_file(self, full_path: Path, base: Path):
|
||||
"""
|
||||
Get all wildcards in a text file.
|
||||
|
||||
Args:
|
||||
full_path (str): The path to the file.
|
||||
base (str): The base path for the wildcards.
|
||||
full_path (Path): The path to the file.
|
||||
base (Path): The base path for the wildcards.
|
||||
"""
|
||||
external_key: str = os.path.relpath(os.path.splitext(full_path)[0], base)
|
||||
external_key_parts = external_key.split(os.sep)
|
||||
external_key_parts = list(full_path.with_suffix("").relative_to(base).parts)
|
||||
try:
|
||||
with open(full_path, "r", encoding="utf-8") as file:
|
||||
text_content = map(lambda x: x.strip("\n\r"), file.readlines())
|
||||
@@ -507,38 +538,39 @@ class PPPWildcards:
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Could not read file '{escape_single_quotes(full_path)}' with utf-8 encoding, trying windows-1252...",
|
||||
f"Could not read file '{escape_single_quotes(str(full_path))}' with utf-8 encoding, trying windows-1252...",
|
||||
)
|
||||
with open(full_path, "r", encoding="windows-1252") as file:
|
||||
text_content = map(lambda x: x.strip("\n\r"), file.readlines())
|
||||
# First pass: drop blank lines and full-line comments.
|
||||
text_content = list(filter(lambda x: x.strip() != "" and not x.strip().startswith("#"), text_content))
|
||||
# Second pass: strip inline comments from lines that passed the first filter.
|
||||
text_content = [x.split("#")[0].rstrip() if len(x.split("#")) > 1 else x for x in text_content]
|
||||
self.__add_wildcard(text_content, full_path, external_key_parts)
|
||||
|
||||
def __get_wildcards_in_path(self, base: str, path: str):
|
||||
def __get_wildcards_in_path(self, base: Path, path: Path):
|
||||
"""
|
||||
Get all wildcards in a path.
|
||||
|
||||
Args:
|
||||
base (str): The base path for the wildcards.
|
||||
path (str): The path (folder or file).
|
||||
base (Path): The base path for the wildcards.
|
||||
path (Path): The path (folder or file).
|
||||
"""
|
||||
if not os.path.exists(path):
|
||||
if not path.exists():
|
||||
log(
|
||||
self.__logger,
|
||||
self.__debug_level,
|
||||
logging.WARNING,
|
||||
f"Wildcard path '{escape_single_quotes(path)}' does not exist!",
|
||||
f"Wildcard path '{escape_single_quotes(str(path))}' does not exist!",
|
||||
)
|
||||
return
|
||||
if os.path.isfile(path):
|
||||
if path.is_file():
|
||||
self.__get_wildcards_in_file(base, path)
|
||||
return
|
||||
for filename in os.listdir(path):
|
||||
full_path = os.path.abspath(os.path.join(path, filename))
|
||||
if os.path.basename(full_path).startswith("."):
|
||||
for child in path.iterdir():
|
||||
if child.name.startswith("."):
|
||||
continue
|
||||
self.__get_wildcards_in_path(base, full_path)
|
||||
self.__get_wildcards_in_path(base, child)
|
||||
|
||||
def set_wildcard_default_filter(self, wildcard_key: str, filter_options: Optional[list[list[str]]]):
|
||||
"""
|
||||
@@ -564,3 +596,9 @@ class PPPWildcards:
|
||||
Optional[list[list[str]]]: The filter options or None if not set.
|
||||
"""
|
||||
return self.__wildcard_default_filters.get(wildcard_key, None)
|
||||
|
||||
def reset_default_filters(self):
|
||||
"""
|
||||
Reset all default filters.
|
||||
"""
|
||||
self.__wildcard_default_filters = {}
|
||||
|
||||
+6
-5
@@ -1,18 +1,19 @@
|
||||
[project]
|
||||
name = "sd-webui-prompt-postprocessor"
|
||||
description = "Stable Diffusion WebUI & ComfyUI extension to post-process the prompt, including sending content from the prompt to the negative prompt and wildcards."
|
||||
version = "3.1.1"
|
||||
description = "Stable Diffusion WebUI & ComfyUI extension to post-process the prompt. Features include: wildcards, sending content from the prompt to the negative prompt, variables, model detection, extranetwork mapping, cleanup."
|
||||
version = "3.3.0"
|
||||
license = { file = "LICENSE.txt" }
|
||||
dependencies = ["lark", "numpy", "pyyaml", "pydantic"]
|
||||
dependencies = ["lark==1.*", "numpy==2.*", "ruamel.yaml==0.*", "pydantic==2.*"]
|
||||
requires-python = ">=3.10"
|
||||
|
||||
[project.urls]
|
||||
Repository = "https://github.com/acorderob/sd-webui-prompt-postprocessor"
|
||||
# Used by Comfy Registry https://comfyregistry.org
|
||||
# Used by Comfy Registry https://registry.comfy.org/
|
||||
Documentation = "https://github.com/acorderob/sd-webui-prompt-postprocessor/main/README.md"
|
||||
"Bug Tracker" = "https://github.com/acorderob/sd-webui-prompt-postprocessor/issues"
|
||||
issues = "https://github.com/acorderob/sd-webui-prompt-postprocessor/issues"
|
||||
|
||||
[tool.comfy]
|
||||
PublisherId = "acorderob"
|
||||
#DisplayName = "sd-webui-prompt-postprocessor"
|
||||
DisplayName = "ACB Prompt PostProcessor"
|
||||
Icon = "https://raw.githubusercontent.com/acorderob/sd-webui-prompt-postprocessor/main/images/prompt-postprocessor-icon.png"
|
||||
|
||||
+4
-4
@@ -1,4 +1,4 @@
|
||||
lark
|
||||
numpy
|
||||
pyyaml
|
||||
pydantic
|
||||
lark==1.*
|
||||
numpy==2.*
|
||||
ruamel.yaml==0.*
|
||||
pydantic==2.*
|
||||
+278
-147
@@ -1,6 +1,7 @@
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit("This script must be run from a Stable Diffusion WebUI")
|
||||
|
||||
# pylint: disable=wrong-import-position,wrong-import-order
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
@@ -10,17 +11,29 @@ import numpy as np
|
||||
|
||||
sys.path.append(str(Path(__file__).parent)) # base path for the extension
|
||||
|
||||
from modules import scripts, shared, script_callbacks # type: ignore
|
||||
from modules.processing import StableDiffusionProcessing # type: ignore
|
||||
from modules.shared import opts # type: ignore
|
||||
from modules.paths import models_path # type: ignore
|
||||
import gradio as gr # type: ignore
|
||||
from modules import scripts, shared, script_callbacks # type: ignore # pylint: disable=import-error
|
||||
from modules.processing import StableDiffusionProcessing # type: ignore # pylint: disable=import-error
|
||||
from modules.shared import opts # type: ignore # pylint: disable=import-error
|
||||
from modules.paths import models_path # type: ignore # pylint: disable=import-error
|
||||
import gradio as gr # type: ignore # pylint: disable=import-error
|
||||
|
||||
from ppp import PromptPostProcessor
|
||||
from ppp_classes import IFWILDCARDS_CHOICES, ONWARNING_CHOICES, SUPPORTED_APPS, SUPPORTED_APPS_NAMES, PPPStateOptions
|
||||
from ppp_classes import (
|
||||
DEFAULT_SAMPLER,
|
||||
IFWILDCARDS_CHOICES,
|
||||
ONWARNING_CHOICES,
|
||||
PPPEnvInfo,
|
||||
SUPPORTED_APPS,
|
||||
SUPPORTED_APPS_NAMES,
|
||||
RUN_MODE,
|
||||
NEXT_SEED,
|
||||
PPPStateOptions,
|
||||
)
|
||||
from ppp_logging import DEBUG_LEVEL, PromptPostProcessorLogFactory, log
|
||||
from ppp_cache import PPPLRUCache
|
||||
from ppp_wildcards import PPPWildcards
|
||||
from ppp_enmappings import PPPExtraNetworkMappings
|
||||
from ppp_common import load_grammar
|
||||
|
||||
|
||||
class PromptPostProcessorA1111Script(scripts.Script):
|
||||
@@ -61,16 +74,37 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
super().__init__()
|
||||
self.instance_index = self.increment_instance_count()
|
||||
self.name = PromptPostProcessor.NAME
|
||||
grammar_filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../grammar.lark")
|
||||
with open(grammar_filename, "r", encoding="utf-8") as file:
|
||||
self.grammar_content = file.read()
|
||||
self.ppp_logger = None
|
||||
self.grammar_content = load_grammar()
|
||||
lf = PromptPostProcessorLogFactory()
|
||||
self.ppp_logger = lf.log
|
||||
self.ppp_debug_level = DEBUG_LEVEL.none.value
|
||||
self.lru_cache = None
|
||||
self.wildcards_obj = None
|
||||
self.extranetwork_mappings_obj = None
|
||||
self.ppp_init = False
|
||||
self.ppp = None
|
||||
# log(self.ppp_logger, DEBUG_LEVEL.minimal, logging.INFO, f"Initializing {self.name} instance {self.instance_index}")
|
||||
|
||||
try:
|
||||
# Support for SD.Next
|
||||
import installer # type: ignore # pylint: disable=import-outside-toplevel
|
||||
|
||||
if hasattr(installer, "control_extensions"):
|
||||
if self.title() not in installer.control_extensions:
|
||||
installer.control_extensions.append(self.title()) # We add the extension to the whitelist.
|
||||
except ImportError:
|
||||
# log(self.ppp_logger, DEBUG_LEVEL.minimal, logging.WARNING, "Could not import control_extensions from installer, SD.Next support will not work.")
|
||||
pass
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
log(
|
||||
self.ppp_logger,
|
||||
DEBUG_LEVEL.minimal,
|
||||
logging.ERROR,
|
||||
f"Error while adding to the SD.Next extension whitelist: {e}",
|
||||
)
|
||||
|
||||
def title(self):
|
||||
"""
|
||||
@@ -97,7 +131,7 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
with gr.Accordion(PromptPostProcessor.NAME, open=False):
|
||||
force_equal_seeds = gr.Checkbox(
|
||||
label="Force equal seeds",
|
||||
info="Force all image seeds and variation seeds to be equal to the first one, disabling the default autoincrease.",
|
||||
info="Force all image seeds and variation seeds to be equal to the first one, disabling the default autoincrement.",
|
||||
value=False,
|
||||
# show_label=True,
|
||||
elem_id="ppp_force_equal_seeds",
|
||||
@@ -110,8 +144,6 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
* A seed of -1 and "Incremental seed" unchecked will use a random seed for each prompt.
|
||||
* Any other seed value and "Incremental seed" checked will use the specified seed for the first prompt and consecutive values for the rest.
|
||||
* Any other seed value and "Incremental seed" unchecked will use the specified seed for all the prompts.
|
||||
|
||||
Seeds are only used for the wildcards and choice constructs.
|
||||
""")
|
||||
gr.HTML("<br>")
|
||||
with gr.Row(equal_height=True):
|
||||
@@ -139,34 +171,51 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
elem_id="ppp_incremental_seed",
|
||||
)
|
||||
gr.HTML("<br>")
|
||||
run_mode = gr.Radio(
|
||||
label="Run mode",
|
||||
choices=[rm.value for rm in RUN_MODE],
|
||||
value=PromptPostProcessor.DEFAULT_RUN_MODE,
|
||||
info="Select the run mode for prompt processing. 'Single' produces one result, 'Multiple' produces many results to fill the batch, and 'Combinatorial' generates all prompt combinations and cycles through them to fill the batch.",
|
||||
elem_id="ppp_run_mode",
|
||||
)
|
||||
with gr.Row(equal_height=True):
|
||||
combinatorial = gr.Checkbox(
|
||||
label="Combinatorial mode",
|
||||
info="Generate all prompt combinations and cycle through them to fill the batch.",
|
||||
value=PromptPostProcessor.DEFAULT_DO_COMBINATORIAL,
|
||||
elem_id="ppp_combinatorial",
|
||||
)
|
||||
combinatorial_shuffle = gr.Checkbox(
|
||||
label="Shuffle combinations",
|
||||
info="Shuffle the combinatorial results.",
|
||||
value=PromptPostProcessor.DEFAULT_COMBINATORIAL_SHUFFLE,
|
||||
elem_id="ppp_combinatorial_shuffle",
|
||||
)
|
||||
combinatorial_limit = gr.Number(
|
||||
label="Combinations limit (0 = no limit)",
|
||||
value=PromptPostProcessor.DEFAULT_COMBINATORIAL_LIMIT,
|
||||
results_limit = gr.Number(
|
||||
label="Results limit (0 = no limit)",
|
||||
value=PromptPostProcessor.DEFAULT_RESULTS_LIMIT,
|
||||
precision=0,
|
||||
min_width=120,
|
||||
elem_id="ppp_combinatorial_limit",
|
||||
elem_id="ppp_results_limit",
|
||||
)
|
||||
default_sampler = gr.Radio(
|
||||
label="Default sampler",
|
||||
choices=[ds.value for ds in DEFAULT_SAMPLER],
|
||||
value=PromptPostProcessor.DEFAULT_DEFAULT_SAMPLER,
|
||||
info="Select the default sampler.",
|
||||
elem_id="ppp_default_sampler",
|
||||
)
|
||||
with gr.Row(equal_height=True):
|
||||
results_shuffle = gr.Checkbox(
|
||||
label="Shuffle results",
|
||||
info="Shuffle the results.",
|
||||
value=PromptPostProcessor.DEFAULT_RESULTS_SHUFFLE,
|
||||
elem_id="ppp_results_shuffle",
|
||||
)
|
||||
comb_random_fixed = gr.Checkbox(
|
||||
label="Fix random sampler across combinations",
|
||||
info="Fix the value of any specified random samplers across all combinations in combinatorial mode.",
|
||||
value=PromptPostProcessor.DEFAULT_COMB_RANDOM_FIXED,
|
||||
elem_id="ppp_comb_random_fixed",
|
||||
)
|
||||
return [
|
||||
force_equal_seeds,
|
||||
unlink_seed,
|
||||
seed,
|
||||
incremental_seed,
|
||||
combinatorial,
|
||||
combinatorial_shuffle,
|
||||
combinatorial_limit,
|
||||
run_mode,
|
||||
results_limit,
|
||||
results_shuffle,
|
||||
comb_random_fixed,
|
||||
default_sampler,
|
||||
]
|
||||
|
||||
def process(
|
||||
@@ -176,9 +225,11 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
input_unlink_seed,
|
||||
input_seed,
|
||||
input_incremental_seed,
|
||||
input_combinatorial,
|
||||
input_combinatorial_shuffle,
|
||||
input_combinatorial_limit,
|
||||
input_run_mode,
|
||||
input_results_limit,
|
||||
input_results_shuffle,
|
||||
input_comb_random_fixed,
|
||||
input_default_sampler,
|
||||
): # pylint: disable=arguments-differ
|
||||
"""
|
||||
Processes the prompts and applies post-processing operations.
|
||||
@@ -189,26 +240,25 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
input_unlink_seed (bool): Flag indicating whether to unlink the seed.
|
||||
input_seed (int): The seed value.
|
||||
input_incremental_seed (bool): Flag indicating whether to use incremental seed.
|
||||
input_combinatorial (bool): Flag indicating whether to use combinatorial mode.
|
||||
input_combinatorial_shuffle (bool): Flag indicating whether to shuffle the combinatorial results.
|
||||
input_combinatorial_limit (int): Maximum number of combinations (0 = no limit).
|
||||
input_run_mode (str): The run mode for prompt processing.
|
||||
input_results_limit (int): Maximum number of results (0 = no limit).
|
||||
input_results_shuffle (bool): Flag indicating whether to shuffle the results.
|
||||
input_comb_random_fixed (bool): Flag indicating whether to fix the random sampler across all combinations.
|
||||
input_default_sampler (str): The default sampler for prompt processing.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
app = (
|
||||
SUPPORTED_APPS.forge
|
||||
if hasattr(p.sd_model, "model_config")
|
||||
else (
|
||||
SUPPORTED_APPS.reforge
|
||||
if hasattr(p.sd_model, "forge_objects")
|
||||
else (
|
||||
SUPPORTED_APPS.sdnext
|
||||
if hasattr(p.sd_model, "is_sdxl") and not hasattr(p.sd_model, "is_ssd")
|
||||
else SUPPORTED_APPS.a1111
|
||||
)
|
||||
)
|
||||
)
|
||||
app = SUPPORTED_APPS.a1111
|
||||
if hasattr(p.sd_model, "model_config"):
|
||||
app = SUPPORTED_APPS.forge
|
||||
if not hasattr(p.sd_model, "is_sd2"):
|
||||
app = SUPPORTED_APPS.forgeneo
|
||||
elif hasattr(p.sd_model, "forge_objects"):
|
||||
app = SUPPORTED_APPS.reforge
|
||||
elif hasattr(p.sd_model, "is_sdxl") and not hasattr(p.sd_model, "is_ssd"):
|
||||
app = SUPPORTED_APPS.sdnext
|
||||
|
||||
num_seeds = len(getattr(p, "all_seeds", []))
|
||||
options = PPPStateOptions(
|
||||
debug_level=DEBUG_LEVEL(getattr(opts, "ppp_gen_debug_level", PromptPostProcessor.DEFAULT_DEBUG_LEVEL)),
|
||||
@@ -254,13 +304,18 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
cup_remove_extranetwork_tags=getattr(
|
||||
opts, "ppp_rem_removeextranetworktags", PromptPostProcessor.DEFAULT_CUP_REMOVE_EXTRANETWORK_TAGS
|
||||
),
|
||||
do_combinatorial=input_combinatorial,
|
||||
combinatorial_shuffle=input_combinatorial_shuffle,
|
||||
combinatorial_limit=max(num_seeds, int(input_combinatorial_limit)) if input_combinatorial else 0,
|
||||
results_file=getattr(opts, "ppp_gen_resultsfile", PromptPostProcessor.DEFAULT_RESULTS_FILE),
|
||||
run_mode=RUN_MODE(input_run_mode if input_run_mode else PromptPostProcessor.DEFAULT_RUN_MODE),
|
||||
results_limit=min(num_seeds, int(input_results_limit)),
|
||||
results_shuffle=input_results_shuffle,
|
||||
comb_random_fixed=input_comb_random_fixed,
|
||||
default_sampler=DEFAULT_SAMPLER(
|
||||
input_default_sampler if input_default_sampler else PromptPostProcessor.DEFAULT_DEFAULT_SAMPLER
|
||||
),
|
||||
next_seed=NEXT_SEED.input, # we use the calculated seeds
|
||||
)
|
||||
if self.ppp_logger is None:
|
||||
lf = PromptPostProcessorLogFactory()
|
||||
self.ppp_logger = lf.log
|
||||
if not self.ppp_init:
|
||||
self.ppp_init = True
|
||||
self.ppp_debug_level = options.debug_level
|
||||
self.lru_cache = PPPLRUCache(1000, logger=self.ppp_logger, debug_level=self.ppp_debug_level)
|
||||
self.wildcards_obj = PPPWildcards(self.ppp_logger)
|
||||
@@ -288,9 +343,16 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
"PPP unlink seed": input_unlink_seed,
|
||||
"PPP prompt seed": input_seed,
|
||||
"PPP incremental seed": input_incremental_seed,
|
||||
"PPP combinatorial": input_combinatorial,
|
||||
"PPP run mode": input_run_mode,
|
||||
"PPP default sampler": input_default_sampler,
|
||||
}
|
||||
)
|
||||
if input_run_mode == RUN_MODE.combinatorial.value:
|
||||
p.extra_generation_params.update(
|
||||
{
|
||||
"PPP combinatorial random fixed": input_comb_random_fixed,
|
||||
}
|
||||
)
|
||||
|
||||
log(
|
||||
self.ppp_logger,
|
||||
@@ -298,20 +360,22 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
logging.INFO,
|
||||
f"Post-processing prompts ({'i2i' if is_i2i else 't2i'})",
|
||||
)
|
||||
env_info = {
|
||||
"app": app.value,
|
||||
"models_path": models_path,
|
||||
"model_filename": getattr(p.sd_model.sd_checkpoint_info, "filename", ""),
|
||||
"model_class": p.sd_model.__class__.__name__,
|
||||
"property_base": p.sd_model,
|
||||
}
|
||||
if app == SUPPORTED_APPS.forge:
|
||||
env_info["model_class"] = p.sd_model.model_config.__class__.__name__
|
||||
env_info = PPPEnvInfo(
|
||||
app=app,
|
||||
models_path=models_path,
|
||||
model_filename=getattr(p.sd_model.sd_checkpoint_info, "filename", ""),
|
||||
model_class=(
|
||||
p.sd_model.model_config.__class__.__name__
|
||||
if app in (SUPPORTED_APPS.forge, SUPPORTED_APPS.forgeneo)
|
||||
else p.sd_model.__class__.__name__
|
||||
),
|
||||
property_base=p.sd_model,
|
||||
)
|
||||
wc_wildcards_folders = getattr(opts, "ppp_wil_wildcardsfolders", "")
|
||||
if wc_wildcards_folders == "":
|
||||
wc_wildcards_folders = os.getenv("WILDCARD_DIR", PPPWildcards.DEFAULT_WILDCARDS_FOLDER)
|
||||
wildcards_folders = [
|
||||
(f if os.path.isabs(f) else os.path.abspath(os.path.join(models_path, f)))
|
||||
(Path(f) if Path(f).is_absolute() else (Path(models_path) / f).resolve())
|
||||
for f in wc_wildcards_folders.split(",")
|
||||
if f.strip() != ""
|
||||
]
|
||||
@@ -322,7 +386,7 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
PPPExtraNetworkMappings.DEFAULT_ENMAPPINGS_FOLDER,
|
||||
)
|
||||
enmappings_folders = [
|
||||
(f if os.path.isabs(f) else os.path.abspath(os.path.join(models_path, f)))
|
||||
(Path(f) if Path(f).is_absolute() else (Path(models_path) / f).resolve())
|
||||
for f in en_mappings_folders.split(",")
|
||||
if f.strip() != ""
|
||||
]
|
||||
@@ -330,17 +394,25 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
self.ppp_debug_level, wildcards_folders if options.process_wildcards else None
|
||||
)
|
||||
self.extranetwork_mappings_obj.refresh_extranetwork_mappings(self.ppp_debug_level, enmappings_folders)
|
||||
ppp = PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
env_info,
|
||||
options,
|
||||
self.grammar_content,
|
||||
self.ppp_interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_mappings_obj,
|
||||
)
|
||||
if self.ppp is None:
|
||||
self.ppp = PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
env_info,
|
||||
options,
|
||||
self.grammar_content,
|
||||
self.ppp_interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_mappings_obj,
|
||||
)
|
||||
else:
|
||||
self.ppp.update(
|
||||
env_info,
|
||||
options,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_mappings_obj,
|
||||
)
|
||||
hash_fullenv = hash(
|
||||
(ppp.envinfo_hash(), ppp.options_hash(), self.wildcards_obj, self.extranetwork_mappings_obj)
|
||||
(self.ppp.envinfo_hash, self.ppp.options_hash, self.wildcards_obj, self.extranetwork_mappings_obj)
|
||||
)
|
||||
|
||||
if input_force_equal_seeds:
|
||||
@@ -354,10 +426,10 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
if input_unlink_seed:
|
||||
log(self.ppp_logger, self.ppp_debug_level, logging.INFO, "Using unlinked seed")
|
||||
if input_incremental_seed:
|
||||
first_seed = np.random.randint(0, 2**32, dtype=np.int64) if input_seed == -1 else input_seed
|
||||
first_seed = np.random.randint(0, 1 << self.ppp.state.host_config.seed_bits, dtype=np.int64) if input_seed == -1 else input_seed
|
||||
calculated_seeds = [first_seed + i for i in range(num_seeds)]
|
||||
elif input_seed == -1:
|
||||
calculated_seeds = np.random.randint(0, 2**32, size=num_seeds, dtype=np.int64)
|
||||
calculated_seeds = np.random.randint(0, 1 << self.ppp.state.host_config.seed_bits, size=num_seeds, dtype=np.int64)
|
||||
else:
|
||||
calculated_seeds = [input_seed for _ in range(num_seeds)]
|
||||
else:
|
||||
@@ -375,40 +447,56 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
else:
|
||||
calculated_seeds = seeds
|
||||
|
||||
# (prompt type, typeindex) -> (new positive prompt, new negative prompt)
|
||||
prompts_list: dict[tuple[str, int], tuple[str, str]] = {}
|
||||
# [index][indextype] -> (new positive prompt, new negative prompt)
|
||||
prompts_list: list[list[tuple[str, str]]] = []
|
||||
extra_params = {}
|
||||
|
||||
# adds prompts
|
||||
regular_type = "regular"
|
||||
rpr: list[str] = getattr(p, "all_prompts", None)
|
||||
rnr: list[str] = getattr(p, "all_negative_prompts", None)
|
||||
regular_exists = rpr is not None and rnr is not None
|
||||
regular_exists = bool(rpr) and bool(rnr)
|
||||
hiresfix_type = "hiresfix"
|
||||
rph: list[str] = getattr(p, "all_hr_prompts", None)
|
||||
rnh: list[str] = getattr(p, "all_hr_negative_prompts", None)
|
||||
hiresfix_exists = rph is not None and rnh is not None
|
||||
hiresfix_exists = bool(rph) and bool(rnh)
|
||||
for i in range(len(calculated_seeds)):
|
||||
if regular_exists:
|
||||
prompts_list[(regular_type, i)] = None
|
||||
prompts_list.append([])
|
||||
prompts_list[i].append(None)
|
||||
if hiresfix_exists:
|
||||
prompts_list[(hiresfix_type, i)] = None
|
||||
prompts_list[i].append(None)
|
||||
|
||||
ppp.process_prompts_group_start()
|
||||
if input_combinatorial:
|
||||
seed_for_comb = calculated_seeds[0] if calculated_seeds else 0
|
||||
self.ppp.process_prompts_group_start()
|
||||
if input_run_mode in (RUN_MODE.multiple.value, RUN_MODE.combinatorial.value):
|
||||
regular_copy = (rpr.copy() if rpr else None, rnr.copy() if rnr else None)
|
||||
hiresfix_copy = (rph.copy() if rph else None, rnh.copy() if rnh else None)
|
||||
regular_changes = False
|
||||
hiresfix_changes = False
|
||||
if regular_exists:
|
||||
log(self.ppp_logger, self.ppp_debug_level, logging.INFO, "processing prompts combinatorially (regular)")
|
||||
comb_results = ppp.process_prompt(rpr[0], rnr[0], seed_for_comb)
|
||||
if input_run_mode == RUN_MODE.combinatorial.value:
|
||||
msg = "processing prompts combinatorially (regular)"
|
||||
else:
|
||||
msg = "processing prompts for multiple results (regular)"
|
||||
log(self.ppp_logger, self.ppp_debug_level, logging.INFO, msg)
|
||||
comb_results = self.ppp.process_prompt(
|
||||
rpr[0],
|
||||
rnr[0],
|
||||
calculated_seeds,
|
||||
jobinfo={
|
||||
"job_timestamp": shared.state.job_timestamp,
|
||||
"job": shared.state.job,
|
||||
"detail": "regular prompt combination",
|
||||
},
|
||||
)
|
||||
num_comb = len(comb_results)
|
||||
for i in range(len(rpr)): # pylint: disable=consider-using-enumerate
|
||||
posp, negp, _ = comb_results[i % num_comb]
|
||||
prompts_list[(regular_type, i)] = (posp, negp)
|
||||
extra_params["PPP combination"] = [str(1 + (i % num_comb)) for i in range(len(rpr))]
|
||||
prompts_list[i][0] = (posp, negp)
|
||||
if input_run_mode == RUN_MODE.combinatorial.value:
|
||||
field_name = "PPP combination"
|
||||
else:
|
||||
field_name = "PPP result"
|
||||
extra_params[field_name] = [str(1 + (i % num_comb)) for i in range(len(rpr))]
|
||||
if hiresfix_exists:
|
||||
hiresfix_equal = regular_exists and rph == rpr and rnh == rnr
|
||||
if hiresfix_equal:
|
||||
@@ -416,80 +504,114 @@ class PromptPostProcessorA1111Script(scripts.Script):
|
||||
self.ppp_logger,
|
||||
self.ppp_debug_level,
|
||||
logging.INFO,
|
||||
"hiresfix prompts are the same as regular prompts, skipping combinatorial processing for hiresfix",
|
||||
"hiresfix prompts are the same as regular prompts, skipping processing for hiresfix",
|
||||
)
|
||||
for i in range(len(rph)): # pylint: disable=consider-using-enumerate
|
||||
prompts_list[(hiresfix_type, i)] = prompts_list.get((regular_type, i))
|
||||
prompts_list[i][1] = prompts_list[i][0]
|
||||
else:
|
||||
if input_run_mode == RUN_MODE.combinatorial.value:
|
||||
msg = "processing prompts combinatorially (hiresfix)"
|
||||
else:
|
||||
msg = "processing prompts for multiple results (hiresfix)"
|
||||
log(
|
||||
self.ppp_logger,
|
||||
self.ppp_debug_level,
|
||||
logging.INFO,
|
||||
"processing prompts combinatorially (hiresfix)",
|
||||
msg,
|
||||
)
|
||||
comb_results_hr = self.ppp.process_prompt(
|
||||
rph[0],
|
||||
rnh[0],
|
||||
calculated_seeds,
|
||||
jobinfo={
|
||||
"job_timestamp": shared.state.job_timestamp,
|
||||
"job": shared.state.job,
|
||||
"detail": "hiresfix prompt combination",
|
||||
},
|
||||
)
|
||||
comb_results_hr = ppp.process_prompt(rph[0], rnh[0], seed_for_comb)
|
||||
num_comb_hr = len(comb_results_hr)
|
||||
for i in range(len(rph)): # pylint: disable=consider-using-enumerate
|
||||
posp, negp, _ = comb_results_hr[i % num_comb_hr]
|
||||
prompts_list[(hiresfix_type, i)] = (posp, negp)
|
||||
extra_params["PPP HR combination"] = [str(1 + (i % num_comb_hr)) for i in range(len(rph))]
|
||||
prompts_list[i][1] = (posp, negp)
|
||||
if input_run_mode == RUN_MODE.combinatorial.value:
|
||||
field_name = "PPP HR combination"
|
||||
else:
|
||||
field_name = "PPP HR result"
|
||||
extra_params[field_name] = [str(1 + (i % num_comb_hr)) for i in range(len(rph))]
|
||||
else:
|
||||
# processes prompts
|
||||
for prompttype, typeindex in prompts_list.keys():
|
||||
log(
|
||||
self.ppp_logger,
|
||||
self.ppp_debug_level,
|
||||
logging.INFO,
|
||||
f"processing prompts ({prompttype}[{typeindex+1}])",
|
||||
)
|
||||
key = (
|
||||
(hash_fullenv, calculated_seeds[typeindex], rpr[typeindex], rnr[typeindex])
|
||||
if prompttype == regular_type
|
||||
else (hash_fullenv, calculated_seeds[typeindex], rph[typeindex], rnh[typeindex])
|
||||
)
|
||||
cached = self.lru_cache.get(key)
|
||||
if cached is None:
|
||||
hsh, seed, prompt, negative_prompt = key
|
||||
results = ppp.process_prompt(prompt, negative_prompt, seed)
|
||||
posp, negp, _ = results[0]
|
||||
cached = (posp, negp)
|
||||
self.lru_cache.put(key, cached)
|
||||
# adds also the result so i2i doesn't process it unnecessarily
|
||||
self.lru_cache.put((hsh, seed, posp, negp), cached)
|
||||
else:
|
||||
log(self.ppp_logger, self.ppp_debug_level, logging.INFO, "result already in cache")
|
||||
prompts_list[(prompttype, typeindex)] = cached
|
||||
ppp.process_prompts_group_end()
|
||||
|
||||
# with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "logs", f"last_prompts_{app.value}.txt"), "w", encoding="utf-8") as f:
|
||||
# for (prompttype, typeindex), (posp, negp) in prompts_list.items():
|
||||
# f.write(f"Key: {prompttype}[{typeindex}]\n")
|
||||
# f.write(f"Seed: {calculated_seeds[typeindex]}\n")
|
||||
# f.write(f"In Positive: {rpr[typeindex] if prompttype == regular_type else rph[typeindex]}\n")
|
||||
# f.write(f"In Negative: {rnr[typeindex] if prompttype == regular_type else rnh[typeindex]}\n")
|
||||
# f.write(f"Out Positive: {posp}\n")
|
||||
# f.write(f"Out Negative: {negp}\n")
|
||||
# f.write("\n")
|
||||
for index, grouplist in enumerate(prompts_list):
|
||||
for typeindex in range(len(grouplist)):
|
||||
typeprompt = [regular_type, hiresfix_type][typeindex]
|
||||
log(
|
||||
self.ppp_logger,
|
||||
self.ppp_debug_level,
|
||||
logging.INFO,
|
||||
f"processing prompts ({typeprompt}[{index+1}])",
|
||||
)
|
||||
key = (
|
||||
(hash_fullenv, calculated_seeds[index], rpr[index], rnr[index])
|
||||
if typeindex == 0
|
||||
else (hash_fullenv, calculated_seeds[index], rph[index], rnh[index])
|
||||
)
|
||||
cached = self.lru_cache.get(key)
|
||||
if cached is None:
|
||||
hsh, seed, prompt, negative_prompt = key
|
||||
if typeindex > 0:
|
||||
prev_prompts = prompts_list[index][typeindex - 1]
|
||||
input_vars = {
|
||||
"prev_pos_prompt": prev_prompts[0],
|
||||
"prev_neg_prompt": prev_prompts[1],
|
||||
}
|
||||
else:
|
||||
input_vars = None
|
||||
results = self.ppp.process_prompt(
|
||||
prompt,
|
||||
negative_prompt,
|
||||
seed,
|
||||
jobinfo={
|
||||
"job_timestamp": shared.state.job_timestamp,
|
||||
"job": shared.state.job,
|
||||
"detail": f"{typeprompt} prompt",
|
||||
},
|
||||
input_vars=input_vars,
|
||||
)
|
||||
posp, negp, _ = results[0]
|
||||
cached = (posp, negp)
|
||||
self.lru_cache.put(key, cached)
|
||||
# adds also the result so i2i doesn't process it unnecessarily
|
||||
self.lru_cache.put((hsh, seed, posp, negp), cached)
|
||||
else:
|
||||
log(self.ppp_logger, self.ppp_debug_level, logging.INFO, "result already in cache")
|
||||
prompts_list[index][typeindex] = cached
|
||||
self.ppp.process_prompts_group_end()
|
||||
|
||||
# updates the prompts
|
||||
regular_copy = (rpr.copy() if rpr else None, rnr.copy() if rnr else None)
|
||||
hiresfix_copy = (rph.copy() if rph else None, rnh.copy() if rnh else None)
|
||||
regular_changes = False
|
||||
hiresfix_changes = False
|
||||
for (prompttype, typeindex), (posp, negp) in prompts_list.items():
|
||||
if prompttype == regular_type:
|
||||
if rpr[typeindex].strip() != posp.strip() or rnr[typeindex].strip() != negp.strip():
|
||||
regular_changes = True
|
||||
rpr[typeindex] = posp
|
||||
rnr[typeindex] = negp
|
||||
elif prompttype == hiresfix_type:
|
||||
if rph[typeindex].strip() != posp.strip() or rnh[typeindex].strip() != negp.strip():
|
||||
hiresfix_changes = True
|
||||
rph[typeindex] = posp
|
||||
rnh[typeindex] = negp
|
||||
for index, grouplist in enumerate(prompts_list):
|
||||
for typeindex, groupprompts in enumerate(grouplist):
|
||||
if groupprompts is None:
|
||||
continue
|
||||
posp, negp = groupprompts
|
||||
if typeindex == 0:
|
||||
if rpr[index].strip() != posp.strip() or rnr[index].strip() != negp.strip():
|
||||
regular_changes = True
|
||||
rpr[index] = posp
|
||||
rnr[index] = negp
|
||||
elif typeindex == 1:
|
||||
if rph[index].strip() != posp.strip() or rnh[index].strip() != negp.strip():
|
||||
hiresfix_changes = True
|
||||
rph[index] = posp
|
||||
rnh[index] = negp
|
||||
|
||||
# initialize extra generation parameters
|
||||
if add_prompts:
|
||||
if hiresfix_exists:
|
||||
extra_params["PPP Hires prompt"] = rph
|
||||
extra_params["PPP Hires negative prompt"] = rnh
|
||||
if regular_changes:
|
||||
extra_params["PPP original prompts"] = regular_copy[0]
|
||||
extra_params["PPP original negative prompts"] = regular_copy[1]
|
||||
@@ -617,6 +739,15 @@ def on_ui_settings():
|
||||
section=section,
|
||||
),
|
||||
)
|
||||
shared.opts.add_option(
|
||||
key="ppp_gen_resultsfile",
|
||||
info=shared.OptionInfo(
|
||||
PromptPostProcessor.DEFAULT_RESULTS_FILE,
|
||||
label="Results file",
|
||||
comment_after=r'<span class="info">(filename to save processing results; supports %datetime%, %date%, %time%, %host% tokens; extension determines format: .yaml, .jsonl, .csv, .txt; empty = disabled)</span>',
|
||||
section=section,
|
||||
),
|
||||
)
|
||||
|
||||
shared.opts.add_option(
|
||||
key="ppp_en_mappingsfolders",
|
||||
|
||||
+53
-42
@@ -1,16 +1,26 @@
|
||||
from dataclasses import replace
|
||||
from dataclasses import replace, make_dataclass
|
||||
import difflib
|
||||
import os
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, NamedTuple, Optional
|
||||
import unittest
|
||||
import datetime
|
||||
|
||||
from ppp_classes import IFWILDCARDS_CHOICES, ONWARNING_CHOICES, PPPStateOptions
|
||||
from ppp_classes import (
|
||||
DEFAULT_SAMPLER,
|
||||
IFWILDCARDS_CHOICES,
|
||||
NEXT_SEED,
|
||||
ONWARNING_CHOICES,
|
||||
PPPEnvInfo,
|
||||
RUN_MODE,
|
||||
SUPPORTED_APPS,
|
||||
PPPStateOptions,
|
||||
)
|
||||
from ppp_enmappings import PPPExtraNetworkMappings # type: ignore
|
||||
from ppp_wildcards import PPPWildcards # type: ignore
|
||||
from ppp import PromptPostProcessor # type: ignore
|
||||
from ppp_logging import DEBUG_LEVEL, PromptPostProcessorLogFactory # type: ignore
|
||||
from ppp_common import load_grammar # type: ignore
|
||||
|
||||
|
||||
class InputTuple(NamedTuple):
|
||||
@@ -73,26 +83,31 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
cup_extranetwork_tags=True,
|
||||
cup_merge_attention=True,
|
||||
cup_remove_extranetwork_tags=False,
|
||||
do_combinatorial=False,
|
||||
combinatorial_limit=0,
|
||||
combinatorial_shuffle=False,
|
||||
results_file=(Path(__file__).parent / "logs" / "output_%date%.txt") if enable_file_logging else "",
|
||||
run_mode=RUN_MODE.single,
|
||||
results_limit=0,
|
||||
results_shuffle=False,
|
||||
comb_random_fixed=True,
|
||||
default_sampler=DEFAULT_SAMPLER.random,
|
||||
next_seed=NEXT_SEED.randomize,
|
||||
)
|
||||
self.def_env_info = PPPEnvInfo(
|
||||
app=SUPPORTED_APPS.tests,
|
||||
ppp_config=None,
|
||||
model_class="SDXL",
|
||||
property_base=make_dataclass("PropertyBase", [("is_sdxl", bool)])(is_sdxl=True),
|
||||
models_path="./webui/models",
|
||||
model_filename="./webui/models/Stable-diffusion/testmodel.safetensors",
|
||||
)
|
||||
self.def_env_info = {
|
||||
"app": "tests",
|
||||
"ppp_config": None,
|
||||
"model_class": "SDXL",
|
||||
"property_base": {"is_sdxl": True},
|
||||
"models_path": "./webui/models",
|
||||
"model_filename": "./webui/models/Stable-diffusion/testmodel.safetensors",
|
||||
}
|
||||
self.interrupted = False
|
||||
self.wildcards_obj = PPPWildcards(self.lf.log)
|
||||
self.extranetwork_maps_obj = PPPExtraNetworkMappings(self.lf.log)
|
||||
tests_folder = Path(__file__).parent
|
||||
self.wildcards_obj.refresh_wildcards(
|
||||
DEBUG_LEVEL.full,
|
||||
[
|
||||
os.path.abspath(os.path.join(os.path.dirname(__file__), "wildcards")),
|
||||
os.path.abspath(os.path.join(os.path.dirname(__file__), "wildcards2")),
|
||||
tests_folder / "wildcards",
|
||||
tests_folder / "wildcards2",
|
||||
],
|
||||
"""
|
||||
yaml_input:
|
||||
@@ -104,13 +119,11 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
)
|
||||
self.extranetwork_maps_obj.refresh_extranetwork_mappings(
|
||||
DEBUG_LEVEL.full,
|
||||
[os.path.abspath(os.path.join(os.path.dirname(__file__), "enmappings"))],
|
||||
[tests_folder / "enmappings"],
|
||||
"""
|
||||
""",
|
||||
)
|
||||
grammar_filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../grammar.lark")
|
||||
with open(grammar_filename, "r", encoding="utf-8") as file:
|
||||
self.grammar_content = file.read()
|
||||
self.grammar_content = load_grammar()
|
||||
|
||||
def interrupt(self):
|
||||
self.interrupted = True
|
||||
@@ -118,8 +131,7 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
def init_ppp(
|
||||
self,
|
||||
ppp: Optional[str | PromptPostProcessor] = None,
|
||||
combinatorial: bool = False,
|
||||
combinatorial_limit: int = 0,
|
||||
**kwargs,
|
||||
) -> PromptPostProcessor:
|
||||
if isinstance(ppp, str):
|
||||
if ppp == "nocup":
|
||||
@@ -141,8 +153,7 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
cup_ands_eol=False,
|
||||
cup_extranetwork_tags=False,
|
||||
cup_merge_attention=False,
|
||||
do_combinatorial=combinatorial,
|
||||
combinatorial_limit=combinatorial_limit,
|
||||
**kwargs,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -156,8 +167,7 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
replace(
|
||||
self.defopts,
|
||||
strict_operators=False,
|
||||
do_combinatorial=combinatorial,
|
||||
combinatorial_limit=combinatorial_limit,
|
||||
**kwargs,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -172,8 +182,7 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
do_combinatorial=combinatorial,
|
||||
combinatorial_limit=combinatorial_limit,
|
||||
**kwargs,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -199,10 +208,9 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
seed: int = 1,
|
||||
ppp: Optional[str | PromptPostProcessor] = None,
|
||||
interrupted: bool = False,
|
||||
combinatorial: bool = False,
|
||||
combinatorial_limit: int = 0,
|
||||
specific_wc_folders: Optional[list[str]] = None,
|
||||
specific_em_folders: Optional[list[str]] = None,
|
||||
specific_wc_folders: Optional[list[Path]] = None,
|
||||
specific_em_folders: Optional[list[Path]] = None,
|
||||
input_vars: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Process the prompt and compare the results with the expected prompts.
|
||||
@@ -213,10 +221,9 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
seed (int, optional): The seed value. Defaults to 1.
|
||||
ppp (Optional[str | PromptPostProcessor], optional): The PromptPostProcessor instance or type. Defaults to None.
|
||||
interrupted (bool, optional): The interrupted flag. Defaults to False.
|
||||
combinatorial (bool, optional): The combinatorial flag. Defaults to False.
|
||||
combinatorial_limit (int, optional): The combinatorial limit. Defaults to 0.
|
||||
specific_wc_folders (Optional[list[str]], optional): A list of specific wildcard folders to refresh. Defaults to None.
|
||||
specific_em_folders (Optional[list[str]], optional): A list of specific extranetwork mapping folders to refresh. Defaults to None.
|
||||
specific_wc_folders (Optional[list[Path]], optional): A list of specific wildcard folders to refresh. Defaults to None.
|
||||
specific_em_folders (Optional[list[Path]], optional): A list of specific extranetwork mapping folders to refresh. Defaults to None.
|
||||
input_vars (Optional[dict[str, Any]], optional): A dictionary of input variables. Defaults to None.
|
||||
|
||||
Returns:
|
||||
None
|
||||
@@ -231,13 +238,13 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
DEBUG_LEVEL.full,
|
||||
specific_em_folders,
|
||||
)
|
||||
the_obj: PromptPostProcessor = self.init_ppp(ppp, combinatorial, combinatorial_limit)
|
||||
the_obj: PromptPostProcessor = ppp if isinstance(ppp, PromptPostProcessor) else self.init_ppp(ppp)
|
||||
out = (
|
||||
[OutputTuple("", "", None)]
|
||||
if expected_output is None
|
||||
else expected_output if isinstance(expected_output, list) else [expected_output]
|
||||
)
|
||||
if the_obj.state.options.do_combinatorial:
|
||||
if the_obj.state.options.run_mode in (RUN_MODE.multiple, RUN_MODE.combinatorial):
|
||||
# combinatorial
|
||||
errors = []
|
||||
the_obj.process_prompts_group_start()
|
||||
@@ -245,6 +252,8 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
input_prompts.prompt,
|
||||
input_prompts.negative_prompt,
|
||||
seed,
|
||||
jobinfo={"test_case": self.id()},
|
||||
input_vars=input_vars,
|
||||
)
|
||||
the_obj.process_prompts_group_end()
|
||||
self.assertTrue(
|
||||
@@ -253,7 +262,7 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
)
|
||||
if not self.interrupted and expected_output is not None:
|
||||
if len(result) != len(out):
|
||||
errors.append(f"Incorrect number of combinations: got {len(result)} but expected {len(out)}")
|
||||
errors.append(f"Incorrect number of results: got {len(result)} but expected {len(out)}")
|
||||
for out_prompt, out_negative_prompt, out_variables in out:
|
||||
found = None
|
||||
for r_prompt, r_negative_prompt, r_variables in result:
|
||||
@@ -263,7 +272,7 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
if not found:
|
||||
errors.extend(
|
||||
[
|
||||
"Combination not found in output",
|
||||
"Result not found in output",
|
||||
"Prompt:",
|
||||
out_prompt,
|
||||
"Negative Prompt:",
|
||||
@@ -285,7 +294,7 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
if missing_vars or incorrect_vars:
|
||||
errors.extend(
|
||||
[
|
||||
"Combination found, but variables do not match",
|
||||
"Result found, but variables do not match",
|
||||
"Prompt:",
|
||||
out_prompt,
|
||||
"Negative Prompt:",
|
||||
@@ -310,6 +319,8 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
input_prompts.prompt,
|
||||
input_prompts.negative_prompt,
|
||||
seed,
|
||||
jobinfo={"test_case": self.id()},
|
||||
input_vars=input_vars,
|
||||
)
|
||||
self.assertTrue(
|
||||
self.interrupted == interrupted,
|
||||
@@ -338,7 +349,7 @@ class TestPromptPostProcessorBase(unittest.TestCase):
|
||||
incorrect_vars[var_name] = output_variables[var_name]
|
||||
expected_values[var_name] = var_value
|
||||
if missing_vars or incorrect_vars:
|
||||
errors.append("Result correct, but variables do not match")
|
||||
errors.append("Variables do not match")
|
||||
if missing_vars:
|
||||
errors.append("Missing variables:")
|
||||
errors.append(str(missing_vars))
|
||||
|
||||
+135
-29
@@ -1,6 +1,4 @@
|
||||
from dataclasses import replace
|
||||
|
||||
from ppp import PromptPostProcessor # type: ignore
|
||||
from ppp_classes import DEFAULT_SAMPLER, NEXT_SEED, RUN_MODE # type: ignore
|
||||
from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
|
||||
|
||||
if __name__ == "__main__":
|
||||
@@ -22,7 +20,6 @@ class TestChoices(TestPromptPostProcessorBase):
|
||||
)
|
||||
|
||||
def test_ch_cyclical(self): # cyclical sampler cycles through all choices
|
||||
ppp_instance = self.init_ppp("nocup")
|
||||
self.process(
|
||||
InputTuple("the choices are: {@choice1|choice2|choice3}", ""),
|
||||
[
|
||||
@@ -31,11 +28,10 @@ class TestChoices(TestPromptPostProcessorBase):
|
||||
OutputTuple("the choices are: choice3", ""),
|
||||
OutputTuple("the choices are: choice1", ""), # cycles back
|
||||
],
|
||||
ppp=ppp_instance,
|
||||
ppp="nocup",
|
||||
)
|
||||
|
||||
def test_ch_cyclical_multiple_constructs(self): # two independent @ constructs cycle together
|
||||
ppp_instance = self.init_ppp("nocup")
|
||||
self.process(
|
||||
InputTuple("{@a|b} {@c|d}", ""),
|
||||
[
|
||||
@@ -45,7 +41,7 @@ class TestChoices(TestPromptPostProcessorBase):
|
||||
OutputTuple("b d", ""),
|
||||
OutputTuple("a c", ""), # cycles back
|
||||
],
|
||||
ppp=ppp_instance,
|
||||
ppp="nocup",
|
||||
)
|
||||
|
||||
def test_ch_cyclical_resets_on_prompt_change(self): # state resets when the prompt pair changes
|
||||
@@ -67,7 +63,6 @@ class TestChoices(TestPromptPostProcessorBase):
|
||||
)
|
||||
|
||||
def test_ch_cyclical_mixed_samplers(self): # @ construct cycles while a ~ construct alongside is unaffected
|
||||
ppp_instance = self.init_ppp("nocup")
|
||||
self.process(
|
||||
InputTuple("{@a|b|c} {x|y}", ""),
|
||||
[
|
||||
@@ -76,7 +71,7 @@ class TestChoices(TestPromptPostProcessorBase):
|
||||
OutputTuple("c x", ""),
|
||||
OutputTuple("a y", ""), # @ cycles back
|
||||
],
|
||||
ppp=ppp_instance,
|
||||
ppp="nocup",
|
||||
)
|
||||
|
||||
def test_ch_choices_withcomments(self): # choices with comments and multiline
|
||||
@@ -103,6 +98,13 @@ class TestChoices(TestPromptPostProcessorBase):
|
||||
ppp="nocup",
|
||||
)
|
||||
|
||||
def test_ch_choices_if_default(self): # choices with if and a default
|
||||
self.process(
|
||||
InputTuple("the choice is: {if false::choice1|if _is_sd1::choice2|else::choice3}", ""),
|
||||
OutputTuple("the choice is: choice3", ""),
|
||||
ppp="nocup",
|
||||
)
|
||||
|
||||
def test_ch_choices_set_if_multiple(self): # choices with if user variable and multiple selection
|
||||
self.process(
|
||||
InputTuple("${var=test}the choices are: {2$$, $$3::choice1|2 if not var eq 'test'::choice2|choice3}", ""),
|
||||
@@ -131,18 +133,7 @@ class TestChoices(TestPromptPostProcessorBase):
|
||||
self.process(
|
||||
InputTuple("<lora:test1:1><lora:test2:{0.2|0.5|0.7|1}>", ""),
|
||||
OutputTuple("", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
cup_remove_extranetwork_tags=True,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
),
|
||||
ppp=self.init_ppp(None, cup_remove_extranetwork_tags=True),
|
||||
)
|
||||
|
||||
def test_ch_cmd_includewildcard(self):
|
||||
@@ -156,14 +147,129 @@ class TestChoices(TestPromptPostProcessorBase):
|
||||
|
||||
def test_ch_combinatorial(self):
|
||||
self.process(
|
||||
InputTuple("{choice1|choice2|choice3}, ${v:{option1|option2}}", ""),
|
||||
InputTuple("{choice1|choice2|choice3}, ${v:{option1|option2}}, {~a|b}", ""),
|
||||
[
|
||||
OutputTuple("choice1, option1", ""),
|
||||
OutputTuple("choice1, option2", ""),
|
||||
OutputTuple("choice2, option1", ""),
|
||||
OutputTuple("choice2, option2", ""),
|
||||
OutputTuple("choice3, option1", ""),
|
||||
OutputTuple("choice3, option2", "", {"v": "option2"}),
|
||||
OutputTuple("choice1, option1, a", ""),
|
||||
OutputTuple("choice1, option2, b", ""),
|
||||
OutputTuple("choice2, option1, a", ""),
|
||||
OutputTuple("choice2, option2, b", ""),
|
||||
OutputTuple("choice3, option1, b", ""),
|
||||
OutputTuple("choice3, option2, a", "", {"v": "option2"}),
|
||||
],
|
||||
combinatorial=True,
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
run_mode=RUN_MODE.combinatorial,
|
||||
comb_random_fixed=False, # allow different random choices across combinations
|
||||
),
|
||||
)
|
||||
|
||||
def test_ch_comb_random_consistent(self): # ~ sampler picks one value shared across all combinations
|
||||
ppp_instance = self.init_ppp("nocup", run_mode=RUN_MODE.combinatorial)
|
||||
ppp_instance.process_prompts_group_start()
|
||||
result = ppp_instance.process_prompt("{~a|b|c} {x|y}", "", starting_seed=1)
|
||||
ppp_instance.process_prompts_group_end()
|
||||
self.assertEqual(len(result), 2, "Expected exactly 2 combinations ({x|y} expands to 2)")
|
||||
rnd_choices = {r_prompt.split()[0] for r_prompt, _, _ in result}
|
||||
self.assertEqual(
|
||||
len(rnd_choices),
|
||||
1,
|
||||
f"The ~ sampler must yield the same value across all combinations, got: {rnd_choices}",
|
||||
)
|
||||
|
||||
# Multiple
|
||||
|
||||
def test_ch_multiple(self):
|
||||
self.process(
|
||||
InputTuple("{choice1|choice2|choice3}, ${v:{option1|option2}}, {~a|b}", ""),
|
||||
[
|
||||
OutputTuple("choice2, option2, a", ""),
|
||||
OutputTuple("choice1, option1, b", ""),
|
||||
OutputTuple("choice1, option2, b", ""),
|
||||
OutputTuple("choice3, option1, b", ""),
|
||||
OutputTuple("choice1, option1, a", ""),
|
||||
OutputTuple("choice2, option1, a", "", {"v": "option1"}),
|
||||
],
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
run_mode=RUN_MODE.multiple,
|
||||
results_limit=6,
|
||||
),
|
||||
)
|
||||
|
||||
# Default sampler
|
||||
|
||||
def test_ch_default_sampler_cyclical_single(self):
|
||||
self.process(
|
||||
InputTuple("{choice1|choice2|choice3}", ""),
|
||||
[
|
||||
OutputTuple("choice1", ""),
|
||||
OutputTuple("choice2", ""),
|
||||
OutputTuple("choice3", ""),
|
||||
OutputTuple("choice1", ""),
|
||||
],
|
||||
ppp=self.init_ppp("nocup", default_sampler=DEFAULT_SAMPLER.cyclical),
|
||||
)
|
||||
|
||||
def test_ch_default_sampler_cyclical_multiple(self):
|
||||
self.process(
|
||||
InputTuple("{choice1|choice2|choice3}", ""),
|
||||
[
|
||||
OutputTuple("choice1", ""),
|
||||
OutputTuple("choice2", ""),
|
||||
OutputTuple("choice3", ""),
|
||||
OutputTuple("choice1", ""),
|
||||
],
|
||||
ppp=self.init_ppp(
|
||||
"nocup",
|
||||
default_sampler=DEFAULT_SAMPLER.cyclical,
|
||||
run_mode=RUN_MODE.multiple,
|
||||
results_limit=4,
|
||||
),
|
||||
)
|
||||
|
||||
# next_seed / _output_seed
|
||||
|
||||
def test_ch_next_seed_input(self): # input mode keeps the same seed for every result
|
||||
self.process(
|
||||
InputTuple("{@a|b|c}", ""),
|
||||
[
|
||||
OutputTuple("a", "", {"_output_seed": 1}),
|
||||
OutputTuple("b", "", {"_output_seed": 1}),
|
||||
OutputTuple("c", "", {"_output_seed": 1}),
|
||||
],
|
||||
seed=1,
|
||||
ppp=self.init_ppp("nocup", run_mode=RUN_MODE.multiple, results_limit=3, next_seed=NEXT_SEED.input),
|
||||
)
|
||||
|
||||
def test_ch_next_seed_increment(self): # increment mode increases the seed by 1 for each result
|
||||
self.process(
|
||||
InputTuple("{@a|b|c}", ""),
|
||||
[
|
||||
OutputTuple("a", "", {"_output_seed": 1}),
|
||||
OutputTuple("b", "", {"_output_seed": 2}),
|
||||
OutputTuple("c", "", {"_output_seed": 3}),
|
||||
],
|
||||
seed=1,
|
||||
ppp=self.init_ppp("nocup", run_mode=RUN_MODE.multiple, results_limit=3, next_seed=NEXT_SEED.increment),
|
||||
)
|
||||
|
||||
def test_ch_next_seed_decrement(self): # decrement mode decreases the seed by 1 for each result
|
||||
self.process(
|
||||
InputTuple("{@a|b|c}", ""),
|
||||
[
|
||||
OutputTuple("a", "", {"_output_seed": 3}),
|
||||
OutputTuple("b", "", {"_output_seed": 2}),
|
||||
OutputTuple("c", "", {"_output_seed": 1}),
|
||||
],
|
||||
seed=3,
|
||||
ppp=self.init_ppp("nocup", run_mode=RUN_MODE.multiple, results_limit=3, next_seed=NEXT_SEED.decrement),
|
||||
)
|
||||
|
||||
def test_ch_next_seed_randomize(self): # randomize mode produces a distinct seed for every result
|
||||
ppp_instance = self.init_ppp("nocup", run_mode=RUN_MODE.multiple, results_limit=3, next_seed=NEXT_SEED.randomize)
|
||||
ppp_instance.process_prompts_group_start()
|
||||
results = ppp_instance.process_prompt("{@a|b|c}", "", starting_seed=1)
|
||||
ppp_instance.process_prompts_group_end()
|
||||
seeds = [r_vars.get("_output_seed") for _, _, r_vars in results]
|
||||
self.assertEqual(len(seeds), 3, f"Expected 3 results, got {len(seeds)}")
|
||||
self.assertEqual(len(set(seeds)), 3, f"Expected 3 distinct seeds, got: {seeds}")
|
||||
|
||||
+21
-49
@@ -1,10 +1,7 @@
|
||||
import logging
|
||||
from dataclasses import replace
|
||||
|
||||
from ppp import PromptPostProcessor # type: ignore
|
||||
from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit("This script must not be run directly")
|
||||
|
||||
@@ -38,36 +35,17 @@ class TestCleanup(TestPromptPostProcessorBase):
|
||||
self.process(
|
||||
InputTuple("this is a <lora:test:1> test__yaml/wildcard7__", ""),
|
||||
OutputTuple("this is a test", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
cup_remove_extranetwork_tags=True,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
),
|
||||
ppp=self.init_ppp(None, cup_remove_extranetwork_tags=True),
|
||||
)
|
||||
|
||||
def test_cl_dontremoveseparatorsoneol(self): # don't remove separators on eol
|
||||
self.process(
|
||||
InputTuple("this is a test,\nsecond line", ""),
|
||||
OutputTuple("this is a test,\nsecond line", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
cup_extra_separators2=False,
|
||||
cup_extra_separators_include_eol=False,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
cup_extra_separators2=False,
|
||||
cup_extra_separators_include_eol=False,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -85,27 +63,19 @@ class TestCleanup(TestPromptPostProcessorBase):
|
||||
(d:0.9)""",
|
||||
"",
|
||||
),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
cup_empty_constructs=False,
|
||||
cup_extra_separators=True,
|
||||
cup_extra_separators2=False,
|
||||
cup_extra_separators_include_eol=False,
|
||||
cup_extra_spaces=False,
|
||||
cup_breaks=False,
|
||||
cup_breaks_eol=False,
|
||||
cup_ands=False,
|
||||
cup_ands_eol=False,
|
||||
cup_extranetwork_tags=False,
|
||||
cup_merge_attention=False,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
cup_empty_constructs=False,
|
||||
cup_extra_separators=True,
|
||||
cup_extra_separators2=False,
|
||||
cup_extra_separators_include_eol=False,
|
||||
cup_extra_spaces=False,
|
||||
cup_breaks=False,
|
||||
cup_breaks_eol=False,
|
||||
cup_ands=False,
|
||||
cup_ands_eol=False,
|
||||
cup_extranetwork_tags=False,
|
||||
cup_merge_attention=False,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -191,7 +161,9 @@ class TestCleanup(TestPromptPostProcessorBase):
|
||||
"Expected an 'Unmatched' warning",
|
||||
)
|
||||
|
||||
def test_cl_warn_escaped_unmatched_no_false_warning(self): # escaped unmatched paren/bracket does not trigger warning
|
||||
def test_cl_warn_escaped_unmatched_no_false_warning(
|
||||
self,
|
||||
): # escaped unmatched paren/bracket does not trigger warning
|
||||
with self.assertNoLogs("PromptPostProcessor", level=logging.WARNING):
|
||||
self.process(
|
||||
InputTuple(r"text with \(escaped unmatched\]", ""),
|
||||
|
||||
+81
-80
@@ -1,3 +1,4 @@
|
||||
from dataclasses import replace
|
||||
from ppp import PromptPostProcessor # type: ignore
|
||||
from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
|
||||
|
||||
@@ -21,10 +22,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("(test1:0.9) (test2) (test3:1.5) (test4:0.99)", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"attention": "parentheses"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"attention": "parentheses"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -42,10 +43,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1 test2 test3", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"attention": "disable"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"attention": "disable"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -63,10 +64,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"attention": "remove"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"attention": "remove"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -84,10 +85,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"attention": "error"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"attention": "error"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -106,10 +107,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"scheduling": "before"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"scheduling": "before"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -127,10 +128,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"scheduling": "after"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"scheduling": "after"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -148,10 +149,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1 test3", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"scheduling": "first"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"scheduling": "first"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -169,10 +170,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"scheduling": "remove"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"scheduling": "remove"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -190,10 +191,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"scheduling": "error"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"scheduling": "error"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -212,10 +213,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"alternation": "first"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"alternation": "first"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -233,10 +234,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"alternation": "remove"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"alternation": "remove"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -254,10 +255,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"alternation": "error"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"alternation": "error"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -276,10 +277,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1\ntest2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"and": "eol"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"and": "eol"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -297,10 +298,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1, test2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"and": "comma"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"and": "comma"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -318,10 +319,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1 test2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"and": "remove"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"and": "remove"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -339,10 +340,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"and": "error"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"and": "error"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -361,10 +362,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1\ntest2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"break": "eol"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"break": "eol"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -382,10 +383,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1, test2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"break": "comma"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"break": "comma"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -403,10 +404,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1 test2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"break": "remove"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"break": "remove"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -424,10 +425,10 @@ class TestHosts(TestPromptPostProcessorBase):
|
||||
OutputTuple("", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"ppp_config": {"hosts": {"tests": {"break": "error"}}},
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
ppp_config={"hosts": {"tests": {"break": "error"}}},
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
|
||||
+91
-77
@@ -1,5 +1,4 @@
|
||||
from dataclasses import replace
|
||||
|
||||
from ppp import PromptPostProcessor # type: ignore
|
||||
from ppp_classes import ONWARNING_CHOICES # type: ignore
|
||||
from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
|
||||
@@ -21,7 +20,21 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
"${v1=}<ppp:set v2><ppp:/set>${v3:}",
|
||||
"",
|
||||
),
|
||||
OutputTuple("", "",{"v1": "", "v2": "", "v3": ""}),
|
||||
OutputTuple("", "", {"v1": "", "v2": "", "v3": ""}),
|
||||
)
|
||||
|
||||
# Typed and output variables test
|
||||
def test_typed_output_variables(self):
|
||||
self.process(
|
||||
InputTuple(
|
||||
"${str=value}${int=42}${float=3.14}${array[]=*('a','b','c')}${bool=true}${str2:default1},${str2:default2}",
|
||||
"",
|
||||
),
|
||||
OutputTuple(
|
||||
"default1,default2",
|
||||
"",
|
||||
{"str": "value", "int": 42, "float": 3.14, "array[]": "a, b, c", "bool": True, "str2": "default2"},
|
||||
),
|
||||
)
|
||||
|
||||
# Echoed variables tests
|
||||
@@ -43,18 +56,7 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
"",
|
||||
),
|
||||
OutputTuple("", "", {"v1": ""}),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
on_warning=ONWARNING_CHOICES.warn,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
),
|
||||
ppp=self.init_ppp(None, on_warning=ONWARNING_CHOICES.warn),
|
||||
)
|
||||
|
||||
# Variable in extranetworks
|
||||
@@ -99,31 +101,53 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
|
||||
# Array variable tests
|
||||
|
||||
def test_array_variable_1(self): # array variable set with += and test of index value and full array with and without default separator
|
||||
def test_array_variable_1(
|
||||
self,
|
||||
): # array variable set with += and test of index value and full array with and without default separator
|
||||
self.process(
|
||||
InputTuple(
|
||||
"${v1[]=val1}${v1[]+=val2}${v1[]+=val3}${v1[1]:defval},${v1[]:defval2},${v1[&'.']:defval3}",
|
||||
"",
|
||||
),
|
||||
OutputTuple("val2,val1, val2, val3,val1.val2.val3", "", {"v1[]": "val1, val2, val3", "v1[1]": "val2", "v1[&'.']": "val1.val2.val3"}),
|
||||
OutputTuple(
|
||||
"val2,val1, val2, val3,val1.val2.val3",
|
||||
"",
|
||||
{"v1[]": "val1, val2, val3", "v1[1]": "val2", "v1[&'.']": "val1.val2.val3"},
|
||||
),
|
||||
)
|
||||
|
||||
def test_array_variable_2(self): # override of array variable value, test of default value when array variable is empty, test of default value when array variable is not set
|
||||
def test_array_variable_2(
|
||||
self,
|
||||
): # override of array variable value, test of default value when array variable is empty, test of default value when array variable is not set
|
||||
self.process(
|
||||
InputTuple(
|
||||
"${v1[]=val1}${v1[]=val2}${v1[]:defval},${v2[]:defval2},${v2[1]:defval3},${v3[]=}${v3[]:defval4}",
|
||||
"",
|
||||
),
|
||||
OutputTuple("val2,defval2,defval3", "", {"v1[]": "val2", "v2[]": "defval2", "v2[1]": "defval3", "v3[]": ""}),
|
||||
OutputTuple(
|
||||
"val2,defval2,defval3", "", {"v1[]": "val2", "v2[]": "defval2", "v2[1]": "defval3", "v3[]": ""}
|
||||
),
|
||||
)
|
||||
|
||||
def test_array_variable_3(self): # access array index by variable, set array variable to expanded array variable and add expanded array
|
||||
def test_array_variable_3(
|
||||
self,
|
||||
): # access array index by variable, set array variable to expanded array variable and add expanded array
|
||||
self.process(
|
||||
InputTuple(
|
||||
"${v1[]=val1}${v1[]+=val2}${v2=1}${v1[v2]:defval1}${v3[]=${v1[]}}${v3[]+=${v1[]}}, ${v3[&'.']}",
|
||||
"",
|
||||
),
|
||||
OutputTuple("val2, val1, val2.val1, val2", "", {"v1[]": "val1, val2", "v2": "1", "v1[v2]": "val2", "v3[]": "val1, val2, val1, val2", "v3[&'.']": "val1, val2.val1, val2"}),
|
||||
OutputTuple(
|
||||
"val2, val1, val2.val1, val2",
|
||||
"",
|
||||
{
|
||||
"v1[]": "val1, val2",
|
||||
"v2": 1,
|
||||
"v1[v2]": "val2",
|
||||
"v3[]": "val1, val2, val1, val2",
|
||||
"v3[&'.']": "val1, val2.val1, val2",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
def test_array_variable_4(self): # test list in array
|
||||
@@ -177,7 +201,7 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
"${v1[]=val1}${v1[]+=val2}${v1[]+=val3}${v1[#]:defval}, <ppp:if v1[#] eq 3>OK<ppp:else>not OK<ppp:/if>",
|
||||
"",
|
||||
),
|
||||
OutputTuple("3, OK", "", {"v1[]": "val1, val2, val3", "v1[#]": "3"}),
|
||||
OutputTuple("3, OK", "", {"v1[]": "val1, val2, val3", "v1[#]": 3}),
|
||||
)
|
||||
|
||||
def test_array_variable_10(self): # array variable set with expanded values from wildcards in command format
|
||||
@@ -189,6 +213,15 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
OutputTuple("choice3", "", {"v1[]": "choice2, choice1, choice3, choice1"}),
|
||||
)
|
||||
|
||||
def test_array_variable_11(self): # array variable set and indexed value added
|
||||
self.process(
|
||||
InputTuple(
|
||||
"${v1[]=*(1,2,3)}${v1[0]+=10}${v2[]=*('1','2','3')}${v2[1]+=10}",
|
||||
"",
|
||||
),
|
||||
OutputTuple("", "", {"v1[]": "11, 2, 3", "v2[]": "1, 210, 3"}),
|
||||
)
|
||||
|
||||
# Operator tests
|
||||
|
||||
## R vs R
|
||||
@@ -636,7 +669,6 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
OutputTuple("OK", ""),
|
||||
)
|
||||
|
||||
|
||||
# NaN/undefined variable integer comparison tests
|
||||
|
||||
def test_cmd_if_undefined_var_int_compare_warn(self): # undefined var integer compare with on_warning=warn
|
||||
@@ -646,18 +678,7 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
"",
|
||||
),
|
||||
OutputTuple("not OK", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
on_warning=ONWARNING_CHOICES.warn,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
),
|
||||
ppp=self.init_ppp(None, on_warning=ONWARNING_CHOICES.warn),
|
||||
)
|
||||
|
||||
def test_cmd_if_undefined_var_int_compare_stop(self): # undefined var integer compare with on_warning=stop
|
||||
@@ -677,18 +698,7 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
"",
|
||||
),
|
||||
OutputTuple("not OK", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
on_warning=ONWARNING_CHOICES.warn,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
),
|
||||
ppp=self.init_ppp(None, on_warning=ONWARNING_CHOICES.warn),
|
||||
)
|
||||
|
||||
def test_cmd_if_nonnumeric_var_int_compare_stop(self): # non-numeric var integer compare with on_warning=stop
|
||||
@@ -708,18 +718,22 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
"",
|
||||
),
|
||||
OutputTuple("not OK", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
on_warning=ONWARNING_CHOICES.warn,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
ppp=self.init_ppp(None, on_warning=ONWARNING_CHOICES.warn),
|
||||
)
|
||||
|
||||
# Input variables
|
||||
|
||||
def test_input_variables(self):
|
||||
self.process(
|
||||
InputTuple(
|
||||
"${_input_prev_positive_prompt}, high quality",
|
||||
"",
|
||||
),
|
||||
OutputTuple(
|
||||
"this is a test, high quality",
|
||||
"",
|
||||
),
|
||||
input_vars={"prev_positive_prompt": "this is a test"},
|
||||
)
|
||||
|
||||
# Command tests
|
||||
@@ -757,10 +771,10 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
OutputTuple("this is PONY", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"model_filename": "./webui/models/Stable-diffusion/ponymodel.safetensors",
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
model_filename="./webui/models/Stable-diffusion/ponymodel.safetensors",
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -990,10 +1004,10 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
OutputTuple("<lora:lorapony:0.8>inlinetrigger, triggerpony1, triggerpony2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"model_filename": "./webui/models/Stable-diffusion/ponymodel.safetensors",
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
model_filename="./webui/models/Stable-diffusion/ponymodel.safetensors",
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -1011,10 +1025,10 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
OutputTuple("<lora:lorapony:0.4>inlinetrigger, triggerpony1, triggerpony2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"model_filename": "./webui/models/Stable-diffusion/ponymodel.safetensors",
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
model_filename="./webui/models/Stable-diffusion/ponymodel.safetensors",
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -1032,10 +1046,10 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
OutputTuple("<lora:lorapony:0.6:0.8>inlinetrigger, triggerpony1, triggerpony2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"model_filename": "./webui/models/Stable-diffusion/ponymodel.safetensors",
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
model_filename="./webui/models/Stable-diffusion/ponymodel.safetensors",
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
@@ -1053,10 +1067,10 @@ class TestVarCommands(TestPromptPostProcessorBase):
|
||||
OutputTuple("<lora:loraillustrious:0.9:0.8>inlinetrigger, triggerillustrious1, triggerillustrious2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"model_filename": "./webui/models/Stable-diffusion/ilxlmodel.safetensors",
|
||||
},
|
||||
replace(
|
||||
self.def_env_info,
|
||||
model_filename="./webui/models/Stable-diffusion/ilxlmodel.safetensors",
|
||||
),
|
||||
self.defopts,
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
|
||||
+35
-5
@@ -24,10 +24,10 @@ class TestModelVariants(TestPromptPostProcessorBase):
|
||||
OutputTuple("test1test2", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
{
|
||||
**self.def_env_info,
|
||||
"model_filename": "./webui/models/Stable-diffusion/testmodel.safetensors",
|
||||
"ppp_config": {
|
||||
replace(
|
||||
self.def_env_info,
|
||||
model_filename="./webui/models/Stable-diffusion/testmodel.safetensors",
|
||||
ppp_config={
|
||||
"models": {
|
||||
"sd1": {
|
||||
"detect": {"tests": {"class": ["SD15", "SD15_instructpix2pix"]}},
|
||||
@@ -62,7 +62,37 @@ class TestModelVariants(TestPromptPostProcessorBase):
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
replace(
|
||||
self.defopts,
|
||||
on_warning=ONWARNING_CHOICES.warn,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
),
|
||||
)
|
||||
|
||||
def test_variants_null_model(self):
|
||||
"""null model in config disables detection and its variants"""
|
||||
self.process(
|
||||
InputTuple(
|
||||
"<ppp:if _is_sdxl>SDXL<ppp:else>not SDXL<ppp:/if>, <ppp:if _is_pony>PONY<ppp:else>not PONY<ppp:/if>",
|
||||
"",
|
||||
),
|
||||
OutputTuple("not SDXL, not PONY", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
replace(
|
||||
self.def_env_info,
|
||||
model_filename="./webui/models/Stable-diffusion/ponymodel.safetensors",
|
||||
ppp_config={
|
||||
"models": {
|
||||
"sdxl": None,
|
||||
}
|
||||
},
|
||||
),
|
||||
replace(
|
||||
self.defopts,
|
||||
on_warning=ONWARNING_CHOICES.warn,
|
||||
|
||||
+62
-84
@@ -1,7 +1,6 @@
|
||||
from dataclasses import replace
|
||||
|
||||
from ppp import PromptPostProcessor
|
||||
from ppp_classes import IFWILDCARDS_CHOICES
|
||||
from ppp_classes import IFWILDCARDS_CHOICES, RUN_MODE
|
||||
from ppp_logging import DEBUG_LEVEL
|
||||
from .base_tests import OutputTuple, InputTuple, TestPromptPostProcessorBase
|
||||
|
||||
if __name__ == "__main__":
|
||||
@@ -19,18 +18,10 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
self.process(
|
||||
InputTuple("__bad_wildcard__", "{option1|option2}"),
|
||||
OutputTuple("__bad_wildcard__", "{option1|option2}"),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.ignore,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.ignore,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -44,18 +35,10 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
"this is: a (([complex|simple|regular] test)(test:2):1.5)\nBREAK with [abc:def:5]<lora:xxx:1>",
|
||||
"[neg5], ([|neg6|]:1.65), (neg1:1.65), [neg4::5], normal quality, [neg2(neg3:1.6):5]",
|
||||
),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.remove,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.remove,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -63,18 +46,10 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
self.process(
|
||||
InputTuple("__bad_wildcard__", "{option1|option2}"),
|
||||
OutputTuple(PromptPostProcessor.WILDCARD_WARNING + "__bad_wildcard__", "{option1|option2}"),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.warn,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.warn,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -85,18 +60,10 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
PromptPostProcessor.WILDCARD_STOP.format("__bad_wildcard__") + "__bad_wildcard__",
|
||||
"{option1|option2}",
|
||||
),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.stop,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.stop,
|
||||
),
|
||||
interrupted=True,
|
||||
)
|
||||
@@ -105,18 +72,10 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
self.process(
|
||||
InputTuple("${v=__bad_wildcard__}${v}", ""),
|
||||
OutputTuple(PromptPostProcessor.WILDCARD_WARNING + "__bad_wildcard__", ""),
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.warn,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
process_wildcards=False,
|
||||
if_wildcards=IFWILDCARDS_CHOICES.warn,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -128,6 +87,22 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
interrupted=True,
|
||||
)
|
||||
|
||||
def test_wc_load_invalid_wildcards(self):
|
||||
self.wildcards_obj.refresh_wildcards(
|
||||
DEBUG_LEVEL.full,
|
||||
[],
|
||||
"""
|
||||
inv alid1:
|
||||
- choice1
|
||||
- choice2
|
||||
inv.alid2:
|
||||
- choice1
|
||||
- choice2
|
||||
""",
|
||||
)
|
||||
self.assertFalse(self.wildcards_obj.get_wildcards("inv alid1"), "invalid wildcard name should not be accepted")
|
||||
self.assertFalse(self.wildcards_obj.get_wildcards("inv.alid2"), "invalid wildcard name should not be accepted")
|
||||
|
||||
def test_wc_wildcard1a_text(self): # simple text wildcard
|
||||
self.process(
|
||||
InputTuple("the choices are: __text/wildcard1__", ""),
|
||||
@@ -337,6 +312,13 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
ppp="nocup",
|
||||
)
|
||||
|
||||
def test_wc_wildcard_emptydefault(self): # empty wildcard with default
|
||||
self.process(
|
||||
InputTuple("the choices are: __yaml/empty_default_wildcard__", ""),
|
||||
OutputTuple("the choices are: 6", ""),
|
||||
ppp="nocup",
|
||||
)
|
||||
|
||||
def test_wc_wildcard4_yaml(self): # simple yaml wildcard with one option
|
||||
self.process(
|
||||
InputTuple("the choices are: __yaml/wildcard4__", ""),
|
||||
@@ -364,11 +346,18 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
|
||||
def test_wc_wildcard_globbing(self): # wildcard with globbing
|
||||
self.process(
|
||||
InputTuple("the choices are: __yaml/wildcard[12]__, __yaml/wildcard?__", ""),
|
||||
InputTuple("the choices are: __yaml/*card[12]__, __yaml/wildcard?__", ""),
|
||||
OutputTuple("the choices are: choice3-choice2, <lora:test2:1>- choice2 -choice3", ""),
|
||||
ppp="nocup",
|
||||
)
|
||||
|
||||
def test_wc_wildcard_globbing2(self): # wildcard with globbing
|
||||
self.process(
|
||||
InputTuple("the choices are: __*/wildcard1__", ""),
|
||||
OutputTuple("the choices are: choice2", ""),
|
||||
ppp="nocup",
|
||||
)
|
||||
|
||||
def test_wc_wildcardwithvar(self): # wildcard with inline variable
|
||||
self.process(
|
||||
InputTuple("the choices are: __yaml/wildcard5(var=test)__, __yaml/wildcard5__", ""),
|
||||
@@ -485,7 +474,7 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
OutputTuple("the choices are: choice3, choice2, option1", "", {"v": "option1"}),
|
||||
OutputTuple("the choices are: choice3, choice2, option2", "", {"v": "option2"}),
|
||||
],
|
||||
combinatorial=True,
|
||||
ppp=self.init_ppp(None, run_mode=RUN_MODE.combinatorial),
|
||||
)
|
||||
|
||||
def test_wc_combinatorial_2(self): # combinatorial wildcard
|
||||
@@ -538,8 +527,7 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
OutputTuple("choice1-choice3", ""),
|
||||
OutputTuple("choice3-choice1", ""),
|
||||
],
|
||||
ppp="nocup",
|
||||
combinatorial=True,
|
||||
ppp=self.init_ppp("nocup", run_mode=RUN_MODE.combinatorial),
|
||||
)
|
||||
|
||||
def test_wc_combinatorial_3(self): # combinatorial wildcard (keep choice order)
|
||||
@@ -557,19 +545,11 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
## choices 1 and 3
|
||||
OutputTuple("choice1-choice3", ""),
|
||||
],
|
||||
ppp=PromptPostProcessor(
|
||||
self.ppp_logger,
|
||||
self.def_env_info,
|
||||
replace(
|
||||
self.defopts,
|
||||
keep_choices_order=True,
|
||||
cup_do_cleanup=False,
|
||||
do_combinatorial=True,
|
||||
),
|
||||
self.grammar_content,
|
||||
self.interrupt,
|
||||
self.wildcards_obj,
|
||||
self.extranetwork_maps_obj,
|
||||
ppp=self.init_ppp(
|
||||
None,
|
||||
keep_choices_order=True,
|
||||
cup_do_cleanup=False,
|
||||
run_mode=RUN_MODE.combinatorial,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -596,8 +576,7 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
OutputTuple("choice1-choice3", ""),
|
||||
OutputTuple("choice3-choice1", ""),
|
||||
],
|
||||
ppp="nocup",
|
||||
combinatorial=True,
|
||||
ppp=self.init_ppp("nocup", run_mode=RUN_MODE.combinatorial),
|
||||
)
|
||||
|
||||
def test_wc_combinatorial_5(self): # combinatorial nested wildcards and multiselection enmappings
|
||||
@@ -619,6 +598,5 @@ class TestWildcards(TestPromptPostProcessorBase):
|
||||
OutputTuple("<lora:loraany1:0.8> trigger1, trigger2, ", ""),
|
||||
OutputTuple("<lora:loraany2:1> trigger3, trigger4, ", ""),
|
||||
],
|
||||
ppp="nocup",
|
||||
combinatorial=True,
|
||||
ppp=self.init_ppp("nocup", run_mode=RUN_MODE.combinatorial),
|
||||
)
|
||||
|
||||
@@ -6,7 +6,7 @@ yaml:
|
||||
- choice3
|
||||
|
||||
wildcard2:
|
||||
- ~r2-3'Wildcard description'$$-$$
|
||||
- r2-3'Wildcard description'$$-$$
|
||||
- "'label1,label2'4::choice1"
|
||||
- "3:: choice2 "
|
||||
- { labels: ["label1", "label3"], weight: 2, content: choice3 }
|
||||
@@ -111,6 +111,14 @@ yaml:
|
||||
- if _sd in ("test1", "test2")::4
|
||||
- if (false or false)::5
|
||||
|
||||
empty_default_wildcard:
|
||||
- if false::1
|
||||
- if false::2
|
||||
- if false::3
|
||||
- if _sd in ("test1", "test2")::4
|
||||
- if (false or false)::5
|
||||
- else::6
|
||||
|
||||
circular1:
|
||||
- 5::__yaml/circular2__
|
||||
- choice1
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Scans wildcard and extranetwork mapping files for LoRA references and checks
|
||||
if those LoRAs exist in the specified folders.
|
||||
|
||||
Recognized LoRA reference formats:
|
||||
<lora:NAME:weight> Standard A1111/ComfyUI inline LoRA tag.
|
||||
<ppp:ext lora NAME ...> PPP explicit LoRA command (quoted or unquoted name).
|
||||
|
||||
LoRA names that contain wildcards or inline choices (e.g. __path__, {a|b}) are
|
||||
reported as dynamic and skipped - they cannot be resolved statically.
|
||||
|
||||
Usage:
|
||||
python check_loras.py -l LORA_FOLDER [LORA_FOLDER ...] [options]
|
||||
|
||||
Options:
|
||||
-w, --wildcards One or more wildcard folder paths to scan.
|
||||
-e, --enmappings One or more enmapping folder paths to scan.
|
||||
-l, --loras One or more folders to search for LoRA files (required).
|
||||
--extensions LoRA file extensions (default: .safetensors .pt .ckpt .bin).
|
||||
--case-sensitive Enable case-sensitive name matching (default: case-insensitive).
|
||||
-v, --verbose Also list LoRAs that were found.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
except ImportError:
|
||||
print("Error: ruamel.yaml is required. Install with: pip install ruamel.yaml", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
WILDCARD_EXTENSIONS = {".yaml", ".yml", ".json", ".txt"}
|
||||
ENMAPPING_EXTENSIONS = {".yaml", ".yml", ".json"}
|
||||
|
||||
# Matches <lora:NAME>, <lora:NAME:w>, <lora:NAME:w1:w2>
|
||||
# The name ends at the first colon or closing >, but may contain spaces.
|
||||
_RE_LORA_STANDARD = re.compile(r"<lora:([^:>\s][^:>]*?)(?::[^>]*)?>")
|
||||
|
||||
# Matches <ppp:ext lora NAME ...> where NAME is single-quoted, double-quoted, or unquoted.
|
||||
# The unquoted form stops before whitespace, >, or / to avoid capturing the rest of the tag.
|
||||
_RE_LORA_PPP_EXT = re.compile(r"<ppp:ext\s+lora\s+(?:'([^']*)'|\"([^\"]*)\"|([\w.\-()][^\s>/'\"]*))")
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str | None:
|
||||
for encoding in ("utf-8", "cp1252"):
|
||||
try:
|
||||
return path.read_text(encoding=encoding)
|
||||
except (UnicodeDecodeError, OSError):
|
||||
continue
|
||||
print(f"Warning: could not read file: {path}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def _is_dynamic(name: str) -> bool:
|
||||
# Names with choice syntax {a|b} or wildcard references __path__ cannot be
|
||||
# statically resolved.
|
||||
return "{" in name or "}" in name or name.count("__") >= 2
|
||||
|
||||
|
||||
_RE_SD_ESCAPE = re.compile(r"\\(.)")
|
||||
|
||||
|
||||
def _unescape_sd(name: str) -> str:
|
||||
# SD prompts escape special characters with a backslash (e.g. \( \) \[ \] \: \\).
|
||||
# The actual filename on disk has no escaping, so strip them before lookup.
|
||||
return _RE_SD_ESCAPE.sub(r"\1", name)
|
||||
|
||||
|
||||
def _extract_lora_names(text: str) -> list[str]:
|
||||
names = []
|
||||
for m in _RE_LORA_STANDARD.finditer(text):
|
||||
names.append(_unescape_sd(m.group(1).strip()))
|
||||
for m in _RE_LORA_PPP_EXT.finditer(text):
|
||||
name = m.group(1) or m.group(2) or m.group(3)
|
||||
if name:
|
||||
names.append(_unescape_sd(name.strip()))
|
||||
return names
|
||||
|
||||
|
||||
def _walk_strings_with_path(value, path: str = "") -> list[tuple[str, str]]:
|
||||
"""Recursively collect (string_value, key_path) pairs from a parsed YAML/JSON structure."""
|
||||
if isinstance(value, str):
|
||||
return [(value, path)]
|
||||
if isinstance(value, list):
|
||||
result = []
|
||||
for item in value:
|
||||
result.extend(_walk_strings_with_path(item, path))
|
||||
return result
|
||||
if isinstance(value, dict):
|
||||
result = []
|
||||
for k, v in value.items():
|
||||
child_path = f"{path} > {k}" if path else str(k)
|
||||
result.extend(_walk_strings_with_path(v, child_path))
|
||||
return result
|
||||
return []
|
||||
|
||||
|
||||
def scan_wildcard_file(path: Path) -> list[tuple[str, str]]:
|
||||
"""Return (lora_name, location) pairs found in a wildcard file."""
|
||||
text = _read_text(path)
|
||||
if text is None:
|
||||
return []
|
||||
|
||||
found = []
|
||||
suffix = path.suffix.lower()
|
||||
|
||||
if suffix == ".txt":
|
||||
for lineno, line in enumerate(text.splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
for name in _extract_lora_names(stripped):
|
||||
found.append((name, f"line {lineno}"))
|
||||
return found
|
||||
|
||||
yaml = YAML(typ="safe")
|
||||
try:
|
||||
data = yaml.load(text)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
print(f"Warning: could not parse {path}: {exc}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
for string_val, key_path in _walk_strings_with_path(data):
|
||||
for name in _extract_lora_names(string_val):
|
||||
found.append((name, key_path))
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def scan_enmapping_file(path: Path) -> list[tuple[str, str]]:
|
||||
"""Return (lora_name, context_snippet) pairs found in an enmapping file."""
|
||||
text = _read_text(path)
|
||||
if text is None:
|
||||
return []
|
||||
|
||||
yaml = YAML(typ="safe")
|
||||
try:
|
||||
data = yaml.load(text)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
print(f"Warning: could not parse {path}: {exc}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
|
||||
found = []
|
||||
lora_section = data.get("lora", {})
|
||||
if not isinstance(lora_section, dict):
|
||||
return []
|
||||
|
||||
for mapping_key, variants in lora_section.items():
|
||||
if not isinstance(variants, list):
|
||||
continue
|
||||
for variant in variants:
|
||||
if not isinstance(variant, dict):
|
||||
continue
|
||||
name = variant.get("name")
|
||||
if name and isinstance(name, str):
|
||||
found.append((name.strip(), f"lora > {mapping_key}"))
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def build_lora_index(lora_folders: list[Path], extensions: set[str], case_sensitive: bool) -> set[str]:
|
||||
index: set[str] = set()
|
||||
for folder in lora_folders:
|
||||
if not folder.is_dir():
|
||||
print(f"Warning: LoRA folder does not exist or is not a directory: {folder}", file=sys.stderr)
|
||||
continue
|
||||
for f in folder.rglob("*"):
|
||||
if f.is_file() and f.suffix.lower() in extensions:
|
||||
stem = f.stem if case_sensitive else f.stem.lower()
|
||||
index.add(stem)
|
||||
return index
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Check that LoRAs referenced in wildcard and enmapping files exist on disk.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
parser.add_argument(
|
||||
"-w", "--wildcards",
|
||||
nargs="*", type=Path, default=[], metavar="FOLDER",
|
||||
help="Wildcard folder(s) to scan.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e", "--enmappings",
|
||||
nargs="*", type=Path, default=[], metavar="FOLDER",
|
||||
help="Enmapping folder(s) to scan.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-l", "--loras",
|
||||
nargs="+", type=Path, required=True, metavar="FOLDER",
|
||||
help="Folder(s) to search for LoRA files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extensions",
|
||||
nargs="+", default=[".safetensors", ".pt", ".ckpt", ".bin"], metavar="EXT",
|
||||
help="LoRA file extensions to recognize (default: .safetensors .pt .ckpt .bin).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--case-sensitive",
|
||||
action="store_true",
|
||||
help="Enable case-sensitive name matching (default: case-insensitive).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v", "--verbose",
|
||||
action="store_true",
|
||||
help="Also list LoRAs that were found.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.wildcards and not args.enmappings:
|
||||
print("Error: specify at least one --wildcards or --enmappings folder.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
extensions = {(ext if ext.startswith(".") else f".{ext}").lower() for ext in args.extensions}
|
||||
|
||||
lora_index = build_lora_index(args.loras, extensions, args.case_sensitive)
|
||||
|
||||
# Keyed by lora name; value is a list of (source_file, context) pairs.
|
||||
references: dict[str, list[tuple[str, str]]] = defaultdict(list)
|
||||
|
||||
for folder in args.wildcards:
|
||||
if not folder.is_dir():
|
||||
print(f"Warning: wildcard folder does not exist: {folder}", file=sys.stderr)
|
||||
continue
|
||||
for path in sorted(folder.rglob("*")):
|
||||
if path.is_file() and path.suffix.lower() in WILDCARD_EXTENSIONS:
|
||||
for name, ctx in scan_wildcard_file(path):
|
||||
references[name].append((str(path), ctx))
|
||||
|
||||
for folder in args.enmappings:
|
||||
if not folder.is_dir():
|
||||
print(f"Warning: enmapping folder does not exist: {folder}", file=sys.stderr)
|
||||
continue
|
||||
for path in sorted(folder.rglob("*")):
|
||||
if path.is_file() and path.suffix.lower() in ENMAPPING_EXTENSIONS:
|
||||
for name, ctx in scan_enmapping_file(path):
|
||||
references[name].append((str(path), ctx))
|
||||
|
||||
found_count = 0
|
||||
missing_count = 0
|
||||
dynamic_count = 0
|
||||
|
||||
for lora_name in sorted(references):
|
||||
if _is_dynamic(lora_name):
|
||||
dynamic_count += 1
|
||||
print(f"SKIPPED (dynamic): {lora_name!r}")
|
||||
for source_file, location in references[lora_name]:
|
||||
print(f" {source_file} [{location}]")
|
||||
continue
|
||||
|
||||
lookup = lora_name if args.case_sensitive else lora_name.lower()
|
||||
if lookup in lora_index:
|
||||
found_count += 1
|
||||
if args.verbose:
|
||||
print(f"OK: {lora_name}")
|
||||
else:
|
||||
missing_count += 1
|
||||
print(f"MISSING: {lora_name}")
|
||||
for source_file, location in references[lora_name]:
|
||||
print(f" {source_file} [{location}]")
|
||||
|
||||
total = found_count + missing_count
|
||||
parts = [f"{total} LoRA reference(s) checked", f"{missing_count} missing"]
|
||||
if dynamic_count:
|
||||
parts.append(f"{dynamic_count} dynamic (skipped)")
|
||||
print(f"\n{', '.join(parts)}.")
|
||||
|
||||
sys.exit(1 if missing_count > 0 else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Converts style files (A1111 CSV or SD.Next JSON) to a PPP-compatible YAML wildcard file.
|
||||
|
||||
Usage:
|
||||
python convert_styles.py [--format {a1111,sdnext}] <input> <output>
|
||||
|
||||
Arguments:
|
||||
input Path to the A1111 styles CSV file, a SD.Next JSON file, or a folder of SD.Next JSON files.
|
||||
output Path to the output YAML wildcard file.
|
||||
|
||||
Options:
|
||||
--format Force the input format. If omitted, the format is inferred from the input:
|
||||
- a1111 : input is a .csv file
|
||||
- sdnext : input is a .json file or a directory
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Allow importing from the parent package when running directly from this folder.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from ppp_common import convert_a1111_styles_to_wildcard, convert_sdnext_styles_to_wildcard
|
||||
|
||||
|
||||
def detect_format(inp: Path) -> str:
|
||||
if inp.is_dir():
|
||||
return "sdnext"
|
||||
if inp.suffix.lower() == ".csv":
|
||||
return "a1111"
|
||||
if inp.suffix.lower() == ".json":
|
||||
return "sdnext"
|
||||
return ""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Convert A1111 or SD.Next style files to a PPP YAML wildcard file.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
parser.add_argument("input", type=Path, help="Input CSV file (A1111) or JSON file/folder (SD.Next).")
|
||||
parser.add_argument("output", type=Path, help="Output YAML wildcard file.")
|
||||
parser.add_argument(
|
||||
"--format",
|
||||
choices=["a1111", "sdnext"],
|
||||
default=None,
|
||||
help="Force input format. Auto-detected from the input path when not specified.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
inp: Path = args.input
|
||||
out: Path = args.output
|
||||
|
||||
if not inp.exists():
|
||||
print(f"Error: input path does not exist: {inp}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if out.suffix.lower() not in (".yaml", ".yml"):
|
||||
print(f"Error: output path is not a YAML file: {out}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
fmt = args.format or detect_format(inp)
|
||||
if not fmt:
|
||||
print(
|
||||
f"Error: could not detect format from '{inp}'. Use --format to specify it explicitly.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
if fmt == "a1111":
|
||||
convert_a1111_styles_to_wildcard(inp, out)
|
||||
else:
|
||||
convert_sdnext_styles_to_wildcard(inp, out)
|
||||
except RuntimeError as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Converted '{inp}' ({fmt}) -> '{out}'")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,5 +1,9 @@
|
||||
# ACB PPP Cleanup Options node
|
||||
|
||||
Provides clean up options to the main PPP node.
|
||||
|
||||
## Inputs
|
||||
|
||||
* **extra_spaces**: Removes other unnecessary spaces.
|
||||
* **empty_constructs**: Removes attention/scheduling/alternation constructs when they are invalid.
|
||||
* **extra_separators**: Removes unnecessary separators. This applies to the configured separator and regular commas.
|
||||
@@ -13,4 +17,9 @@
|
||||
* **merge_attention**: It merges attention modifiers when possible (merges into one, multiplying their values). Only merges individually nested modifiers.
|
||||
* **remove_extranetwork_tags**: Removes all extra network tags.
|
||||
|
||||
Please note that *ComfyUI* does not natively support the `BREAK` and `AND` constructs, but the related settings are kept in that UI in case you use a node that supports them and the extension is configured to allow them (see the configuration file below).
|
||||
> [!NOTE]
|
||||
> *ComfyUI* does not natively support the `BREAK` and `AND` constructs, but the related settings are kept in that UI in case you use a node that supports them and the extension is configured to allow them (see the configuration file below).
|
||||
|
||||
## Outputs
|
||||
|
||||
* **options**: The options to send to the PPP node.
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
# ACB PPP ExtraNetwork Mapping Options node
|
||||
|
||||
Provides extranetwork mapping options to the main PPP node.
|
||||
|
||||
## Inputs
|
||||
|
||||
* **folders**: You can enter multiple folders separated by commas. You can leave it empty (the default) and add a `ppp_extranetworkmappings` entry in the **extra_model_paths.yaml** file (recommended).
|
||||
* **definitions**: Extranetwork Mappings definitions (in yaml format). Direct input added to the ones found in the extranetwork mappings folders. Allows the mappings to be included in the workflow.
|
||||
|
||||
## Outputs
|
||||
|
||||
* **options**: The options to send to the PPP node.
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# ACB PPP Run Mode Options node
|
||||
|
||||
Provides run mode options to the main PPP node.
|
||||
|
||||
## Inputs
|
||||
|
||||
* **results_limit**: Limit for the number of generated results (except in `single` mode). Important for combinatorial mode.
|
||||
* **results_shuffle**: It shuffles the results.
|
||||
* **comb_random_fixed**: If True all specified random samplers will have a fixed value across the combinations.
|
||||
* **default_sampler**: The default choice sampler when not specified (in non combinatorial mode). Also applies to extranetwork mapping selection.
|
||||
* **next_seed**: Choose what to do with the seed in the following prompts in `multiple` or `combinatorial` mode. Value can be: `randomize`, `input`, `increment`, `decrement`.
|
||||
|
||||
## Outputs
|
||||
|
||||
* **options**: The options to send to the PPP node.
|
||||
@@ -1,4 +1,12 @@
|
||||
# ACB PPP Wildcard Options node
|
||||
# ACB PPP Select Variable node
|
||||
|
||||
Extracts a specific variable from PPP output variables.
|
||||
|
||||
## Inputs
|
||||
|
||||
* **variables**: Connect to the variables output from the main node.
|
||||
* **name**: Name of the variable to get, or empty to get all of them.
|
||||
|
||||
## Outputs
|
||||
|
||||
* **value**: The result value.
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
# ACB PPP Send-To-Negative Options node
|
||||
|
||||
Provides send-to-negative options to the main PPP node.
|
||||
|
||||
## Inputs
|
||||
|
||||
* **separator**: You can specify the separator used when adding to the negative prompt (by default it's ", ").
|
||||
* **ignore_repeats**: It ignores repeated content to avoid repetitions in the negative prompt.
|
||||
|
||||
## Outputs
|
||||
|
||||
* **options**: The options to send to the PPP node.
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# ACB PPP Wildcard Concat node
|
||||
|
||||
Concatenates up to 10 selected wildcards.
|
||||
|
||||
## Inputs
|
||||
|
||||
* **previous_prompt**: Previous prompt to add to.
|
||||
* **filter**: Filter text to reduce the available wildcard keys in the wildcard_N selectors.
|
||||
* **separator**: Separator string to join the selected wildcards.
|
||||
* **wildcard_1** to **wildcard_10**: Wildcard selectors.
|
||||
|
||||
## Outputs
|
||||
|
||||
* **prompt**: The resulting prompt.
|
||||
@@ -1,5 +1,9 @@
|
||||
# ACB PPP Wildcard Options node
|
||||
|
||||
Provides wildcard options to the main PPP node.
|
||||
|
||||
## Inputs
|
||||
|
||||
* **folders**: You can enter multiple folders separated by commas. You can leave it empty (the default) and add a `ppp_wildcards` or `wildcards` entry in the **extra_model_paths.yaml** file (recommended).
|
||||
* **definitions**: Wildcards definitions (in yaml or json format). Direct input added to the ones found in the wildcards folders. Allows wildcards to be included in the workflow.
|
||||
* **if_wildcards**: Select what do you want to do with any found wildcards/choices (when process wildcards is off or after the processing).
|
||||
@@ -9,3 +13,7 @@
|
||||
* **Stop the generation**: detect wildcards and stop the generation.
|
||||
* **choice_separator**: What do you want to use by default to separate multiple choices when the options allow it (by default it's ', ").
|
||||
* **keep_choices_order**: If checked, a multiple choice construct will return them in the order they are in the construct.
|
||||
|
||||
## Outputs
|
||||
|
||||
* **options**: The options to send to the PPP node.
|
||||
|
||||
@@ -1,30 +1,45 @@
|
||||
# ACB Prompt Post Processor node
|
||||
|
||||
Main PPP node that processes prompts.
|
||||
|
||||
## Inputs
|
||||
|
||||
* **model**: Connect here the MODEL or a string with the model class name used by *ComfyUI*. Needed for the model kind system variables.
|
||||
* **modelname**: Name of the model. Needed for the detection of model variants.
|
||||
* **seed**: Connect here the seed used. By default it is -1 (random).
|
||||
* **model**: Connect here the MODEL or a string with the model class name used by *ComfyUI*. Needed for the model kind system variables. Optional if you set modelname.
|
||||
* **modelname**: Filename of the model (with relative path). Needed for the detection of model variants.
|
||||
* **seed**: Set or connect here the seed used. By default it is -1 (random). The actual value used can be extracted from the output variables (`_input_seed`).
|
||||
* **pos_prompt**: Connect here the prompt text, or fill it as a widget.
|
||||
* **neg_prompt**: Connect here the negative prompt text, or fill it as a widget.
|
||||
* **debug_level**: What to write to the console.
|
||||
* **on_warnings**: Warn on the console or stop the generation.
|
||||
* **strict_mode**: Sets the strict mode in comparison operations.
|
||||
* **process_wildcards**: Activates the wildcard processing.
|
||||
* **do_cleanup**: Activates the cleanup processing.
|
||||
* **cleanup_variables**: Do a cleanup of the output variables (depends on do_cleanup).
|
||||
* **do_combinatorial**: Activates combinatorial mode, where the output are all the combinations of choices/wildcards of the prompt.
|
||||
* **combinatorial_shuffle**: It shuffles the combinatorial results.
|
||||
* **combinatorial_limit**: Limit for the number of generated combinations.
|
||||
* **results_file**: Filename to save processing results. Supports `%datetime%`, `%date%`, `%time%`, and `%host%` tokens. The file extension determines the format: `.yaml`/`.yml`, `.jsonl`, `.csv`, or plain text for any other extension. Relative paths are resolved against the extension's `logs` folder. Leave empty to disable.
|
||||
* **run_mode**: Sets how the process works. `single` or `multiple` for regular one or more results, or `combinatorial` for combinatorial mode, where the output are all the combinations of choices/wildcards of the prompt.
|
||||
* **wc_options**: Connection to a Wildcards options node.
|
||||
* **stn_options**: Connection to a Send-To-Negative options node.
|
||||
* **cup_options**: Connection to a Cleanup options node.
|
||||
* **en_options**: Connection to a ExtraNetworkMapping options node.
|
||||
* **rm_options**: Connection to a Run Mode options node.
|
||||
|
||||
The options nodes are optional. If you don't need to change any of the default values then you don't need to use them.
|
||||
|
||||
The model and modelname are also optional, but if you don't set them you will not be capable of choosing content based on the model type or variant. Native model loader nodes do not output the filename, but there are custom nodes that do (like those from [ComfyUI Image Saver](https://github.com/alexopus/ComfyUI-Image-Saver)).
|
||||
|
||||
Setting only the modelname will try to detect its class from the file contents. If you don't want to set the path twice (and you don't use a loader node that outputs the name), you can set it here and then extract it from the output variables (`_modelfullname`) to send to the loader node.
|
||||
|
||||
You can instead set it from the prompt (with variable `_modelfullname`) and load the model afterwards.
|
||||
|
||||
## Outputs
|
||||
|
||||
The outputs are the final positive and negative prompt and a variables dictionary.
|
||||
* **pos_prompt**: Resulting positive prompt.
|
||||
* **neg_prompt**: Resulting negative prompt.
|
||||
* **variables**: Resulting output variables.
|
||||
|
||||
The outputs are lists, and in combinatorial mode there will be multiple elements that *ComfyUI* will process sequentially.
|
||||
|
||||
## Notes
|
||||
|
||||
You can use the "**ACB PPP Select Variable**" node to choose one and output its value. You can use this to send only part of the prompt to, for example, a detailer node. For example:
|
||||
|
||||
|
||||
+35
-43
@@ -4,56 +4,48 @@ import { api } from "../../scripts/api.js";
|
||||
app.registerExtension({
|
||||
name: "ACB.PPP.WildcardConcat",
|
||||
|
||||
async beforeRegisterNodeDef(nodeType, nodeData) {
|
||||
if (nodeData.name !== "ACBPPPWildcardConcat") return;
|
||||
async nodeCreated(node) {
|
||||
if (node.comfyClass !== "ACBPPPWildcardConcat") return;
|
||||
|
||||
const onNodeCreated = nodeType.prototype.onNodeCreated;
|
||||
const refreshWildcards = async () => {
|
||||
const filterWidget = node.widgets?.find((w) => w.name === "filter");
|
||||
if (!filterWidget) return;
|
||||
|
||||
nodeType.prototype.onNodeCreated = function () {
|
||||
const result = onNodeCreated?.apply(this, arguments);
|
||||
try {
|
||||
const resp = await api.fetchApi(
|
||||
`/acb_ppp/wildcards?filter=${encodeURIComponent(filterWidget.value ?? "")}`
|
||||
);
|
||||
const data = await resp.json();
|
||||
const wildcards = data.wildcards ?? [];
|
||||
|
||||
const self = this;
|
||||
|
||||
const refreshWildcards = async () => {
|
||||
const filterWidget = self.widgets?.find((w) => w.name === "filter");
|
||||
if (!filterWidget) return;
|
||||
|
||||
try {
|
||||
const resp = await api.fetchApi(
|
||||
`/acb_ppp/wildcards?filter=${encodeURIComponent(filterWidget.value ?? "")}`
|
||||
);
|
||||
const data = await resp.json();
|
||||
const wildcards = data.wildcards ?? [];
|
||||
|
||||
for (let i = 1; i <= 10; i++) {
|
||||
const widget = self.widgets?.find((w) => w.name === `wildcard_${i}`);
|
||||
if (widget) {
|
||||
const current = widget.value;
|
||||
widget.options.values = wildcards;
|
||||
widget.value = wildcards.includes(current)
|
||||
? current
|
||||
: wildcards[0] ?? "(none)";
|
||||
}
|
||||
for (let i = 1; i <= 10; i++) {
|
||||
const widget = node.widgets?.find((w) => w.name === `wildcard_${i}`);
|
||||
if (widget) {
|
||||
const current = widget.value;
|
||||
widget.options.values = wildcards;
|
||||
widget.value = wildcards.includes(current)
|
||||
? current
|
||||
: wildcards[0] ?? "(none)";
|
||||
}
|
||||
|
||||
app.graph.setDirtyCanvas(true, false);
|
||||
} catch (err) {
|
||||
console.error("[ACB PPP] Failed to refresh wildcard list:", err);
|
||||
}
|
||||
};
|
||||
|
||||
// Hook the filter widget so that committing a new value (Enter / blur)
|
||||
// immediately refreshes all 10 wildcard dropdowns without running the workflow.
|
||||
const filterWidget = this.widgets?.find((w) => w.name === "filter");
|
||||
if (filterWidget) {
|
||||
const origCallback = filterWidget.callback;
|
||||
filterWidget.callback = async function (...args) {
|
||||
if (origCallback) origCallback.apply(this, args);
|
||||
await refreshWildcards();
|
||||
};
|
||||
app.graph.setDirtyCanvas(true, false);
|
||||
} catch (err) {
|
||||
console.error("[ACB PPP] Failed to refresh wildcard list:", err);
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
// Button to manually reload wildcards from disk
|
||||
node.addWidget("button", "Refresh 🔄", null, refreshWildcards, { serialize: false });
|
||||
|
||||
// Auto-refresh all wildcard dropdowns when the filter value changes
|
||||
const filterWidget = node.widgets?.find((w) => w.name === "filter");
|
||||
if (filterWidget) {
|
||||
const origCallback = filterWidget.callback;
|
||||
filterWidget.callback = async function (...args) {
|
||||
if (origCallback) origCallback.apply(this, args);
|
||||
await refreshWildcards();
|
||||
};
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 169 KiB After Width: | Height: | Size: 185 KiB |
+1
-988
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user