From 373ea8bda3ea0bea69cd1761c75d992784cbc093 Mon Sep 17 00:00:00 2001 From: Eric Hiss Date: Wed, 28 Oct 2026 20:36:44 -0700 Subject: [PATCH] v2.2.0: Prompt Builder nodes, wildcard review tool, docs refresh - Add Wildcard Prompt Composer and Wildcard Character Composer builder nodes (core/builder, web/prompt_builder.js, builders/, vp wildcard lists) - Add browser-based wildcard review tool (tools/wildcard_review) with manifest-based undo - Add wildcard catalog/curation, surreal compiler/lint/scene, profile token budget - README: document all 18 nodes, bump to v2.2.0, fix stale node/platform lists; CHANGELOG 2.2.0 - Rewrite NODE_COMPARISON for 18 nodes; fix QUICKSTART (dead test_setup.py ref, menu path) and QUICK_REFERENCE node list - gitignore review manifests and CLAUDE.md Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 11 + CHANGELOG.md | 26 + README.md | 259 ++- __init__.py | 17 + alice_prompter_node.py | 12 +- builders/character.yaml | 16 + builders/prompt.yaml | 963 +++++++++ builders/templates/illustration.json | 29 + builders/templates/product_shot.json | 33 + builders/templates/studio_portrait.json | 33 + concept_survival_node.py | 48 +- config.json.example | 12 +- core/builder/__init__.py | 32 + core/builder/compose.py | 272 +++ core/builder/layout.py | 140 ++ core/builder/merge.py | 175 ++ core/builder/more.py | 178 ++ core/builder/sections.py | 59 + core/builder/slots.py | 726 +++++++ core/profiles/rewriter.py | 23 +- core/profiles/token_budget.py | 180 ++ core/profiles/validate.py | 158 ++ core/strategies/surreal/__init__.py | 12 +- core/strategies/surreal/compiler.py | 270 +++ core/strategies/surreal/critic.py | 278 ++- core/strategies/surreal/engine.py | 254 ++- core/strategies/surreal/generators.py | 223 ++- core/strategies/surreal/lint.py | 128 ++ core/strategies/surreal/scene.py | 191 ++ core/strategies/surreal/sources.py | 2 + core/strategies/surreal/vocabulary.py | 220 ++- core/wildcards/__init__.py | 9 +- core/wildcards/catalog.py | 567 ++++++ core/wildcards/curation.py | 349 ++++ core/wildcards/engine.py | 147 +- core/wildcards/library.py | 829 +++++++- core/wildcards/parser.py | 3 + core/wildcards/record.py | 18 +- core/wildcards/resolver.py | 58 +- core/wildcards/syntax_tree.py | 3 +- docs/NODE_COMPARISON.md | 449 ++--- docs/PROMPT_BUILDER_GUIDE.md | 297 +++ docs/QUICKSTART.md | 22 +- docs/QUICK_REFERENCE.md | 27 +- docs/SURREAL_GUIDE.md | 132 +- docs/WILDCARD_GUIDE.md | 147 ++ expansion_engine.py | 19 +- ideogram_magic_node.py | 18 +- image_to_image_node.py | 14 +- image_to_video_node.py | 13 +- platform_rewriter_node.py | 54 +- profile_support.py | 60 + prompt_builder_node.py | 31 + prompt_builder_support.py | 1090 +++++++++++ prompt_expander_node.py | 15 +- prompt_expander_node_advanced.py | 15 +- semantic_variation_node.py | 8 +- surreal_node.py | 67 +- surreal_support.py | 40 +- text_to_image_node.py | 18 +- tools/wildcard_review/review.html | 791 ++++++++ tools/wildcard_review/server.py | 1207 ++++++++++++ tools/wildcard_review/wildcard_review.bat | 6 + web/prompt_builder.js | 2178 +++++++++++++++++++++ wildcard_processor_node.py | 14 +- wildcard_support.py | 99 +- wildcards/README.md | 18 + wildcards/surreal/absences.txt | 2 +- wildcards/surreal/autonomy.txt | 20 + wildcards/surreal/found_strange.yaml | 54 +- wildcards/surreal/motifs.yaml | 1072 ++++++++++ wildcards/surreal/reversals.txt | 20 + wildcards/vp/action.yaml | 146 ++ wildcards/vp/camera.yaml | 50 + wildcards/vp/composition.yaml | 81 + wildcards/vp/format.yaml | 160 ++ wildcards/vp/lighting.yaml | 105 + wildcards/vp/setting.yaml | 97 + wildcards/vp/style.yaml | 223 +++ wildcards/vp/subject.yaml | 386 ++++ wildcards/vp/text.yaml | 65 + 81 files changed, 15705 insertions(+), 588 deletions(-) create mode 100644 builders/character.yaml create mode 100644 builders/prompt.yaml create mode 100644 builders/templates/illustration.json create mode 100644 builders/templates/product_shot.json create mode 100644 builders/templates/studio_portrait.json create mode 100644 core/builder/__init__.py create mode 100644 core/builder/compose.py create mode 100644 core/builder/layout.py create mode 100644 core/builder/merge.py create mode 100644 core/builder/more.py create mode 100644 core/builder/sections.py create mode 100644 core/builder/slots.py create mode 100644 core/profiles/token_budget.py create mode 100644 core/strategies/surreal/compiler.py create mode 100644 core/strategies/surreal/lint.py create mode 100644 core/strategies/surreal/scene.py create mode 100644 core/wildcards/catalog.py create mode 100644 core/wildcards/curation.py create mode 100644 docs/PROMPT_BUILDER_GUIDE.md create mode 100644 prompt_builder_node.py create mode 100644 prompt_builder_support.py create mode 100644 tools/wildcard_review/review.html create mode 100644 tools/wildcard_review/server.py create mode 100644 tools/wildcard_review/wildcard_review.bat create mode 100644 web/prompt_builder.js create mode 100644 wildcards/surreal/autonomy.txt create mode 100644 wildcards/surreal/motifs.yaml create mode 100644 wildcards/surreal/reversals.txt create mode 100644 wildcards/vp/action.yaml create mode 100644 wildcards/vp/camera.yaml create mode 100644 wildcards/vp/composition.yaml create mode 100644 wildcards/vp/format.yaml create mode 100644 wildcards/vp/lighting.yaml create mode 100644 wildcards/vp/setting.yaml create mode 100644 wildcards/vp/style.yaml create mode 100644 wildcards/vp/subject.yaml create mode 100644 wildcards/vp/text.yaml diff --git a/.gitignore b/.gitignore index b791f60..c19d632 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,8 @@ tmp/ # Local notes notes_local/ +# Standing instructions for AI assistants (personal working style, kept local) +CLAUDE.md # Variability engine: phrase vector caches and repulsion memory; cached vendor rewriter prompts cache/ @@ -61,3 +63,12 @@ prompt_database.db # Ideogram's published magic-prompt text: fetched on first use, not redistributed with this package ideogram_prompts/v1_official.txt +# Your own builder definitions (builders_user/ overrides builders/ by id) +builders_user/ +dev/_builder_keys.json +# Wildcard catalog and your curation of the shipped lists +wildcards/_catalog.sqlite* +wildcards/_catalog_user.yaml + +# Wildcard review tool: runtime-generated undo/apply manifests (kept locally, not shipped) +tools/wildcard_review/manifests/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f3ce5d6..c5333ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,32 @@ All notable changes to Eric's Prompt Enhancers for ComfyUI. --- +## [2.2.0] - 2026-10-28 + +### Added - prompt builder nodes + +- **Wildcard Prompt Composer** (`EricPromptBuilder`) and **Wildcard Character Composer** (`EricCharacterBuilder`): + build a prompt section by section with clickable pills instead of typed wildcard syntax. The Composer holds every + section (Format, Subject, Action, Setting, Light & atmosphere, Composition, Camera, Style & finish, plus optional + Text) in the order the chosen platform profile wants; sections can be locked while the seed re-rolls, reordered by + drag, and saved as templates. The Character Composer is the same node limited to the Subject section for a reusable + character. Web panel in `web/prompt_builder.js`; section definitions in `builders/`. +- Documented in `docs/PROMPT_BUILDER_GUIDE.md`. + +### Added - wildcard review tool + +- `tools/wildcard_review`: a local browser tool (`wildcard_review.bat`, served at http://127.0.0.1:8766) for curating + your wildcard library. Per list you can keep or rename it (old names keep working through `_catalog_user.yaml` + aliases), place it on Composer buttons, merge similar lists with a line-by-line preview, drop bad lines (single or + by pattern), and extend a short list with the local LLM. **Apply** performs renames, merges and moves after a + preview; every change writes a manifest under `tools/wildcard_review/manifests/` that + `python tools/wildcard_review/server.py --undo ` reverses. +- Reads the library folder from `LIBRARY_ROOT` at the top of `server.py` (default `A:\Models\wildcards`, overridable + with the `WCR_ROOT` environment variable). LLM features use LM Studio at `http://localhost:1234`. +- Documented in `README.md` (Wildcards → Reviewing and cleaning) and `docs/WILDCARD_GUIDE.md`. + +--- + ## [2.1.0] - 2026-09-21 ### Added - surreal engine (Phase 3) diff --git a/README.md b/README.md index 5f0d88f..5a7b728 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # Eric's Prompt Enhancers for ComfyUI [![License](https://img.shields.io/badge/License-Dual%20(NC%2FCommercial)-blue.svg)](LICENSE) -[![Version](https://img.shields.io/badge/version-2.1.0-green.svg)](CHANGELOG.md) +[![Version](https://img.shields.io/badge/version-2.2.0-green.svg)](CHANGELOG.md) [![ComfyUI](https://img.shields.io/badge/ComfyUI-Compatible-orange.svg)](https://github.com/comfyanonymous/ComfyUI) -A comprehensive suite of **5 AI-powered prompt enhancement nodes** for ComfyUI using local LLMs (LM Studio or Ollama). Transform simple prompts into detailed, platform-optimized descriptions for video and image generation. +A comprehensive suite of **18 nodes** for ComfyUI using local LLMs (LM Studio or Ollama). Transform simple prompts into detailed, platform-optimized descriptions for video and image generation. -![Eric's Prompt Enhancers](https://img.shields.io/badge/Nodes-5-brightgreen) ![Platforms](https://img.shields.io/badge/Image%20Platforms-8-blue) ![Video Support](https://img.shields.io/badge/Video-✓-success) +![Eric's Prompt Enhancers](https://img.shields.io/badge/Nodes-18-brightgreen) ![Platforms](https://img.shields.io/badge/Image%20Platforms-8-blue) ![Video Support](https://img.shields.io/badge/Video-✓-success) --- @@ -26,8 +26,7 @@ Restart ComfyUI. All nodes will appear under: **Add Node → Eric Prompt Enhance ## 🆕 New in 2.0 Version 2.0 keeps every existing node and every saved workflow, and adds three things underneath them. -The rest of this README still describes the 1.x nodes and will be brought up to date in a later 2.0.x release; -the documents linked here are current. +The documents linked here are current. **A real wildcard engine.** The Dynamic Prompts / Impact Pack grammar (`__name__`, `{a|b|c}`, weights, ranges, variables, tag queries such as `__objects[metal AND handheld]__`), read straight from `ComfyUI/models/wildcards`, @@ -45,10 +44,21 @@ Kling 3, Veo 3.1, Wan 3.0 and others. Your own go in `profiles_user/`. The new * writes for any of them, including the structured formats (MiniMax H3 fields, FLUX.2 and Ideogram JSON, edit instructions). See [PLATFORM_NOTES.md](PLATFORM_NOTES.md). -**Surreal as a procedure.** The `surreal` preset and genre now run a generator / critic loop grounded in how +**Prompts stay inside the text-encoder limit.** A text encoder silently drops whatever runs past its +limit (Krea 2: 512 tokens) - from the END, which is where LoRA trigger words usually go. The Platform +Prompt Rewriter now counts the answer the way the model will (the Qwen tokenizer for Qwen-encoder +models, from files already on disk; a conservative estimate otherwise) and aims below the limit with +headroom for triggers. `length_guard` (appended): `retry+trim` asks the LLM once for a tighter version, +then trims; `trim` trims only; `off` reports only. Trimming drops sentences just before the last one, +so the subject (first) and medium / style (last) survive, and verbatim wildcard picks are never cut. +The report shows `length: N/512 tokens`. + +**Surreal as a procedure.** The `surreal` preset and genre now run an ordinary-scene inventory, generator, +critic and structured compiler grounded in how Magritte, Ernst, Dali, Varo, Uelsmann, Atget and Man Ray actually worked: one anomaly stated as plain fact against -a credible world, chosen for affinity rather than distance, with a **Surreal Premise** node that shows every -candidate and the reasoning. See [docs/SURREAL_GUIDE.md](docs/SURREAL_GUIDE.md). +a credible world, chosen for affinity rather than distance. The compiler removes replaced objects before the +platform rewriter and requires visible physical proof of the anomaly. The **Surreal Premise** node shows every +candidate, the structured plan and the reasoning. See [docs/SURREAL_GUIDE.md](docs/SURREAL_GUIDE.md). **The video nodes have a `target_model`.** The three video expanders were written around Wan 2.2. They now write for any video profile, and the default is **MiniMax H3**. Choose `wan22_video` to get the original @@ -57,15 +67,16 @@ it loads with the default (MiniMax H3) - set it to `wan22_video` once and save. --- -## 🎯 The 5 Nodes +## 🎯 The Nodes -Find all nodes under the **"Eric Prompt Enhancers"** category in ComfyUI. +All **18 nodes** live under the **"Eric Prompt Enhancers"** category in ComfyUI. The five LLM expanders come +first, then platform output, wildcards and variation, the prompt builders, and the prompt database. ### 1. 🎬 Video Prompt Expander Simple video prompt expansion with style presets. -- **4 Detail Levels**: Concise, Moderate, Detailed, Exhaustive +- **5 Expansion Tiers**: auto, basic, enhanced, advanced, cinematic (`auto` detects the right tier) - **6 Style Presets**: Cinematic, Surreal, Action, Stylized, Noir, Random - **Auto-Variation**: Generate up to 3 unique variations - **Best for**: Quick video prompt expansion from short ideas @@ -105,7 +116,7 @@ Vision model analyzes images and adds motion descriptions for video generation. Platform-aware image-to-image prompt generation. -- **5 Platforms**: Flux Redux, SDXL Img2Img, Hunyuan Img2Img, Qwen Edit, Wan Edit +- **6 Platforms**: Flux, SDXL, Wan 2.2, HunyuanImage, Qwen-Image, Qwen-Image-Edit - **Transformation Controls**: Style transfer, detail level, creativity settings - **Vision Analysis**: Understands source image characteristics - **Best for**: Image transformation and editing workflows @@ -123,6 +134,92 @@ Advanced multi-platform image prompt enhancement with extensive creative control - **Special Syntax**: Emphasis `(keyword:1.5)` and Alternation `{a|b|c}` support - **Best for**: Professional image generation with platform-specific optimization +### 6. 🎞️ Alice T2V Prompt Refiner + +Refines rough text and/or reference images into structured Alice text-to-video prompts. + +- **Text-only or image-guided**: optional Qwen3-VL vision analysis of reference images +- **Best for**: Alice / text-to-video prompts from a loose idea + +**— Platform output —** + +### 7. ✍️ Platform Prompt Rewriter + +Rewrites a finished prompt for any **platform profile** (60+ open and hosted models), in that model's exact format, +and keeps it inside the text encoder's token limit so trigger words aren't silently dropped. + +- **60+ profiles**: FLUX.2, Krea 2, Qwen-Image, Ideogram 4, Illustrious, MiniMax H3, Wan 2.2, Veo 3.1, and more + (`profiles/`; your own go in `profiles_user/`) +- **Structured formats**: MiniMax H3 fields, FLUX.2 / Ideogram JSON, edit instructions +- **`length_guard`**: counts tokens the way the model will and trims to fit, never cutting verbatim wildcard picks +- **Outputs**: prompt, negative_prompt, report, system_prompt + +### 8. 🪄 Ideogram 4 Magic Prompt + +Builds an Ideogram 4 "magic prompt" (structured JSON plus a high-level line) with a recommended canvas size. + +- **Outputs**: prompt_json, high_level, width, height, report +- **Best for**: Ideogram 4 generation + +**— Wildcards & variation —** + +### 9. 🎲 Wildcard Processor + +Resolves wildcards, variants, variables and tag queries in a prompt — the Dynamic Prompts / Impact Pack grammar +(`__name__`, `{a|b|c}`, weights, ranges, variables, `__objects[metal AND handheld]__`), read from `models/wildcards`. + +### 10. 🔀 Semantic Variation + +Varies a wildcard prompt by moving each random pick to a semantic neighbour, along named direction axes. + +### 11. ✅ Concept Survival Check + +Measures whether the key concepts (a surreal anomaly, exact LoRA triggers, ...) survived the LLM rewrite. + +- **Outputs**: report, all_survived, missing, carried_by_json + +### 12. 🌀 Surreal Premise + +Turns an ordinary scene into **one anomaly stated as plain physical fact**, with an instruction block that protects +it through the rewrite, a Magritte-style title and counter-terms. Eight generator modes feed one critic. + +- Its `llm_instructions` plug into any enhancer's `wildcard_instructions` +- See [docs/SURREAL_GUIDE.md](docs/SURREAL_GUIDE.md) + +**— Prompt builders (click, don't type) —** + +### 13. 🧩 Wildcard Prompt Composer + +Builds a whole prompt section by section with clickable pills instead of typed wildcard syntax. Holds every section +(Format, Subject, Action, Setting, Light & atmosphere, Composition, Camera, Style & finish, plus optional Text) in +the order the chosen platform wants. Lock a section to hold it while the seed re-rolls, drag to reorder, save as a +template. + +### 14. 👤 Wildcard Character Composer + +The same node limited to the Subject section, for a character you reuse across several prompts or branches. + +- See [docs/PROMPT_BUILDER_GUIDE.md](docs/PROMPT_BUILDER_GUIDE.md) + +**— Prompt database —** + +### 15. 💾 Save Prompt to DB + +Saves a prompt (and optional negative) to the local SQLite database, passing the prompt through unchanged so the node +can sit inline before a sampler. + +### 16. 🔎 Search Prompt DB + +Searches the database by keyword, category, platform, rating, tags or favourites, and returns the top match. + +### 17. 📥 Import Prompts to DB + +Bulk-imports prompts from external files (`.txt`, `.csv`, and more) into the local database. + +### 18. 🎰 Browse / Random Prompt DB + +Fetches a specific prompt by ID, or a random/filtered prompt by seed. + --- ## ✨ Key Features @@ -376,6 +473,98 @@ All nodes return: --- +## 🗂️ Maintaining Your Wildcard and LoRA Libraries + +The Wildcard Composer and the LoRA Suggester are only as good as the lists and LoRA cards behind them. This is +how to add to both, clean them up and keep their indexes current. + +### Wildcards + +**Where they live.** Any wildcard folder ComfyUI knows (`ComfyUI/models/wildcards`, plus extra model paths such as +`A:\Models\wildcards`). A few top-level folders are special: + +| Folder | What it is | +|---|---| +| `_nsfw/` | NSFW lists, used only at the matching NSFW level (`_nsfw/suggestive/`, `_nsfw/nudity/` rate lower) | +| `_prompts/` | whole prompts, one per line: reached only through the Composer's **Prompt** line or `__prompts//...__` | +| `_generated/` | lists the tools write | +| `_dumped/` | lists you dumped in the review; the engine skips it | +| `mine/` | your own lists: merges, lists made with the LLM | + +Your choices (stars, hidden lists, dropped lines, NSFW ratings, renames, button assignments) are kept in +`_catalog_user.yaml` in the wildcard folder. The pack files themselves are never edited. + +**Adding lists.** Copy the files in; nothing else is needed to *use* them. ComfyUI rescans the wildcard folders every +10 seconds, so a new list works in `__name__` references and the Composer at once. The catalog behind Find, More +lists and button ranking catches up by itself in the background each time ComfyUI starts: restart ComfyUI to refresh it. + +**Reviewing and cleaning: the wildcard review tool.** Run `tools\wildcard_review\wildcard_review.bat` and open +http://127.0.0.1:8766. (Close an old review window first: a server still running holds the port, and you would +see the old version.) Per list you can: + +- **Keep / Dump / Same as / Merge** (keys K / D / S / M; **A** takes the AI judge's call). Dump hides a list and + never deletes it. +- **Rename** a list you keep, so it is easy to find in pills and pages. Its old name keeps working, so saved workflows + are safe. +- **Put it on Composer buttons**, any number of them. **+ new button** makes one of your own, under the heading you + pick (Setting › Place, Subject › Clothing ...). +- **Merge** it with similar lists. A preview shows every combined line and which lists have it; untick any to leave + them out. The result is written to `mine/`. +- **Drop bad lines.** Click a line and it is never picked again; this is saved at once. **Drop matching** drops every + line with a word or regex. **Suggest junk** asks the local LLM which lines don't belong, and you confirm. +- **Make new lists** with the local LLM (LM Studio): **+ New list...** from a description and a few samples, or + **Extend...** to grow a short list. You check every line before it is saved to `mine/`. + +**Apply** writes your decisions and does the renames and merges, showing a preview first. **Move dumped files**, +in the same window, moves dumped files to `_dumped/`. Every rename, merge, move and new list writes a manifest; to undo +one, run: + + python tools\wildcard_review\server.py --undo tools\wildcard_review\manifests\.json + +The review tool reads the folder set in `LIBRARY_ROOT` at the top of `tools/wildcard_review/server.py` (default +`A:\Models\wildcards`; the `WCR_ROOT` environment variable overrides it). The LLM features use LM Studio at +`http://localhost:1234`; choose the model in the page's header menu. + +**Getting new lists into the review.** The review shows lists the triage and the AI judge have seen. After adding +packs, restart ComfyUI (so the catalog is current), then from this folder run the commands below. The judge needs +LM Studio and only judges lists it has not seen. + + ..\..\..\python_embeded\python.exe -X utf8 dev\wildcard_triage.py + ..\..\..\python_embeded\python.exe -X utf8 dev\wildcard_judge.py --all + +Note: check how a fresh triage run treats lists you have since renamed, merged or moved to `_dumped/` before +re-running it on a reviewed library. + +See [docs/WILDCARD_GUIDE.md](docs/WILDCARD_GUIDE.md) for the syntax and the folder rules, and +[docs/PROMPT_BUILDER_GUIDE.md](docs/PROMPT_BUILDER_GUIDE.md) for the Composer. + +### LoRAs (the LoRA Catalog) + +The LoRA Suggester reads a catalog of LoRA "cards": what each LoRA does, its style and subject words. They are +built by the companion LoRA Catalog tool, kept in its own folder (`A:\LoRA_Catalog` here). The path to its database +is the `lora_suggest` section of `config.json` (default `L:\Models\loras\_lora_catalog.sqlite`). + +**After adding LoRAs**, run `refresh_new_loras.bat` in the LoRA Catalog folder, with LM Studio running. Every step +skips work already done, so it is safe to run any time. It writes `refresh_new_loras.log`. + +| Step | Command | What it does | +|---|---|---| +| 1 | `run.py index` | reads new or changed LoRA files (headers, sidecars, LoRA Manager data) | +| 2 | `run.py crawl` | fetches Civitai records for LoRAs that have none yet (internet; the run carries on if it fails) | +| 3 | `run.py creators` | looks up who made each LoRA (internet; the run carries on if it fails) | +| 4 | `run.py cards --workers 3` | the local LLM writes a card for every LoRA without one | +| 5 | `run.py review-all` | rebuilds the HTML review pages | +| 6 | `run.py facets` | rebuilds `card_facets`, the table the LoRA Suggester reads | + +`refresh_new_loras.bat some/model-id` writes the new cards with another LM Studio model. To redo cards that already +exist, run step 4 by hand with `--force` (add `--only Krea2/graphic-art` to limit it to one folder), then step 6. +Never run `facets` while a cards run is still writing. + +**Checking and correcting cards: the card editor.** Run `card_editor.bat` in the LoRA Catalog folder and open +http://127.0.0.1:8765. Your edits are kept apart from the generated cards and win over them. Reload the page to see +newly written cards; after editing, run `run.py facets` so the Suggester sees your changes. The pick-list words live +in `vocab.yaml` in the same folder. + ## 📚 Documentation ### Quick References @@ -386,8 +575,10 @@ All nodes return: ### Node Guides - [TXT2IMG_GUIDE.md](docs/TXT2IMG_GUIDE.md) - Text-to-Image node complete guide - [IMG2IMG_GUIDE.md](docs/IMG2IMG_GUIDE.md) - Image-to-Image workflows +- [PROMPT_BUILDER_GUIDE.md](docs/PROMPT_BUILDER_GUIDE.md) - Wildcard Prompt / Character Composer panel +- [SURREAL_GUIDE.md](docs/SURREAL_GUIDE.md) - Surreal Premise node and the surreal engine +- [WILDCARD_GUIDE.md](docs/WILDCARD_GUIDE.md) - Wildcard syntax, folders and the review tool - [NODE_COMPARISON.md](docs/NODE_COMPARISON.md) - Which node to use when -- [ADVANCED_NODE_REDESIGN.md](docs/ADVANCED_NODE_REDESIGN.md) - Advanced node v1.8 redesign details ### Setup Guides - [LM_STUDIO_SETUP.md](docs/LM_STUDIO_SETUP.md) - LM Studio configuration @@ -395,14 +586,46 @@ All nodes return: - [CONFIGURATION.md](docs/CONFIGURATION.md) - Advanced configuration options ### Platform & Technical +- [PLATFORM_NOTES.md](PLATFORM_NOTES.md) - Platform profiles and structured formats - [WAN_GUIDE_REFERENCE.md](docs/WAN_GUIDE_REFERENCE.md) - Wan 2.2 video prompt guide -- [WILDCARD_GUIDE.md](docs/WILDCARD_GUIDE.md) - Wildcard syntax examples ### Updates & Fixes - [CHANGELOG.md](CHANGELOG.md) - Complete version history -- [UPDATE_V17_ENHANCED_CONTROLS.md](docs/UPDATE_V17_ENHANCED_CONTROLS.md) - v1.7 features -- [BUGFIX_ADVANCED_NODE.md](docs/BUGFIX_ADVANCED_NODE.md) - v1.8 bug fixes -- [BUGFIX_V161.md](docs/BUGFIX_V161.md) - v1.6.1 syntax fixes + +--- + +## 🆕 What's New in v2.2.0 + +### Wildcard Prompt Composer — build a prompt section by section, with buttons + +Two new builder nodes write a prompt one section at a time, with clickable pills instead of typed wildcard syntax: + +- **Wildcard Prompt Composer** holds every section in one node — Format, Subject, Action, Setting, Light & + atmosphere, Composition, Camera and Style & finish (plus optional Text) — shown in the order the chosen platform + wants (Krea 2, FLUX.2, ...). Each `[Pill]` is a wildcard; the words around them are yours. Lock a section to hold + it while the seed re-rolls, drag sections to reorder, and save the result as a template. +- **Wildcard Character Composer** is the same node limited to the Subject section, for a character you reuse across + several prompts or branches. + +See [docs/PROMPT_BUILDER_GUIDE.md](docs/PROMPT_BUILDER_GUIDE.md) for the full panel reference. + +### Wildcard review tool — curate your library in the browser + +A local browser tool for cleaning up and organising your wildcard lists. Run +`tools\wildcard_review\wildcard_review.bat` and open http://127.0.0.1:8766. Working one list at a time, you can: + +- **Keep or rename** a list; the old name keeps working through aliases, so saved workflows and templates don't break. +- **Put it on Composer buttons** (any number), or make your own button under the heading you pick. +- **Merge** similar lists, with a preview of every combined line so you can untick ones to leave out. +- **Drop bad lines** — one click removes a line from future picks; "Drop matching" clears a pattern across lists. +- **Extend** a short list with the local LLM, checking every line before it's saved. + +**Apply** writes your decisions (renames, merges, moves) after showing a preview, and **every change writes a manifest +you can undo**. The tool reads the folder set in `LIBRARY_ROOT` at the top of `tools/wildcard_review/server.py` +(default `A:\Models\wildcards`); LLM features use LM Studio at `http://localhost:1234`. + +See the **Wildcards → Reviewing and cleaning** section below and +[docs/WILDCARD_GUIDE.md](docs/WILDCARD_GUIDE.md) for full details. --- @@ -431,8 +654,6 @@ All nodes return: **Bonus:** All video nodes now support emphasis `(keyword:1.5)` and alternation `{a|b|c}` syntax! -See [docs/ADVANCED_NODE_REDESIGN.md](docs/ADVANCED_NODE_REDESIGN.md) for complete details. - --- ## 💡 Tips & Best Practices diff --git a/__init__.py b/__init__.py index dbd82f6..4322d17 100644 --- a/__init__.py +++ b/__init__.py @@ -87,9 +87,20 @@ from .semantic_variation_node import SemanticVariation from .concept_survival_node import ConceptSurvivalCheck from .surreal_node import SurrealPremise +# LoRA Suggester: moved to the Eric_LoRA_Catalog node pack (2026-10-11). + # Platform profiles (core/profiles + profiles/*.yaml) from .platform_rewriter_node import PlatformPromptRewriter +# Prompt builders (core/builder + builders/*.yaml + web/prompt_builder.js) +from .prompt_builder_node import CharacterBuilder, PromptBuilder +from .prompt_builder_support import register_routes as _register_builder_routes + +try: + _register_builder_routes() +except Exception as _error: # the nodes still work without the panel's routes + print(f"[Wildcard Composer] routes not registered: {_error}") + # Node class mappings for ComfyUI NODE_CLASS_MAPPINGS = { "EricVideoPromptExpander": AIVideoPromptExpander, @@ -112,6 +123,9 @@ NODE_CLASS_MAPPINGS = { "EricSurrealPremise": SurrealPremise, # Platform profiles "EricPlatformPromptRewriter": PlatformPromptRewriter, + # Prompt builders + "EricPromptBuilder": PromptBuilder, + "EricCharacterBuilder": CharacterBuilder, } # Display names in ComfyUI @@ -136,6 +150,9 @@ NODE_DISPLAY_NAME_MAPPINGS = { "EricSurrealPremise": "Surreal Premise", # Platform profiles "EricPlatformPromptRewriter": "Platform Prompt Rewriter", + # Prompt builders + "EricPromptBuilder": "Wildcard Prompt Composer", + "EricCharacterBuilder": "Wildcard Character Composer", } WEB_DIRECTORY = "./web" diff --git a/alice_prompter_node.py b/alice_prompter_node.py index e02413f..8b12b79 100644 --- a/alice_prompter_node.py +++ b/alice_prompter_node.py @@ -33,6 +33,10 @@ from .wildcard_support import ( normalise_seed, report_lines, resolve_wildcards, + filter_notes_of, + lead_with_filter_notes, + screen_output, + wildcard_nsfw_input, wildcard_optional_inputs, ) @@ -441,6 +445,7 @@ class AlicePromptRefiner: }), # New inputs are only ever APPENDED below this line, so saved workflows keep their widget order. **wildcard_optional_inputs(), + **wildcard_nsfw_input(), }, } @@ -471,6 +476,8 @@ class AlicePromptRefiner: wildcard_seed: int = -1, wildcard_llm_block: bool = True, wildcard_instructions: str = "", + nsfw: str = "off", + violence_filter: str = "off", ) -> Tuple[str, str, str]: image_description = "" @@ -481,7 +488,8 @@ class AlicePromptRefiner: # ---------------------------------------------------------- wildcard_resolution = resolve_wildcards( text_prompt or "", seed=normalise_seed(wildcard_seed), legacy=True, - upstream_instructions=wildcard_instructions, + upstream_instructions=wildcard_instructions, nsfw=nsfw, + violence_filter=violence_filter, ) text_prompt = wildcard_resolution.text @@ -619,6 +627,7 @@ class AlicePromptRefiner: # Verbatim wildcard picks (__=name__) must survive the rewrite word for word refined_prompt, _restored = enforce_verbatim(refined_prompt, wildcard_resolution) + refined_prompt, block_note = screen_output(refined_prompt, "Alice Prompter output") word_count = len(refined_prompt.split()) print( @@ -633,6 +642,7 @@ class AlicePromptRefiner: wildcard_report = report_lines(wildcard_resolution) if wildcard_report: status = f"{status}\n{wildcard_report}" + status = lead_with_filter_notes(status, filter_notes_of(wildcard_resolution, block_note)) return (refined_prompt, image_description, status) except Exception as exc: diff --git a/builders/character.yaml b/builders/character.yaml new file mode 100644 index 0000000..2e961b4 --- /dev/null +++ b/builders/character.yaml @@ -0,0 +1,16 @@ +# Wildcard Character Composer: the Subject section only. +# +# It extends prompt.yaml, so it uses that file's subject (and action) slot buttons; change them there, +# or copy this file to builders_user/character.yaml to give the Wildcard Character Composer its own. +# See docs/PROMPT_BUILDER_GUIDE.md. + +id: character +label: Wildcard Character Composer +description: One character's look, written as a phrase you control, for the Subject section. +extends: prompt + +parts: + - section: subject + label: Subject + slots: [subject, action] + starter: "a {age} {occupation} with {hair_color} hair and {eyes} eyes, {expression}" diff --git a/builders/prompt.yaml b/builders/prompt.yaml new file mode 100644 index 0000000..9d872dd --- /dev/null +++ b/builders/prompt.yaml @@ -0,0 +1,963 @@ +# Wildcard Prompt Composer: every section of an image or video prompt in one node. +# +# parts: one composer per prompt section, shown in the platform's order (or the node's own). +# Sections and their order: dev/PROMPT_TAXONOMY_SPEC.md. +# slots: which sections' buttons the part shows ([] = its own) +# multi: true lets "+ Subject" add Subject 2, 3 ... +# hint: what belongs there; also what the LLM is asked to write when "LLM fills" is on +# optional: true = shown only when added with "+ Section" +# slots: grouped by the section whose palette shows them. +# match: regexes, FULL match against a list's leaf name (last path segment, lower case, +# spaces/hyphens as _, pack prefixes like cops_wc_ / cyber_ / bk_ removed). +# Earlier patterns rank first, then lists of short entries, then bigger lists. +# avoid: regexes searched anywhere in the key; a hit drops the list. +# lists: keys pinned first (the package's own curated lists are pinned here). +# kinds: subject types the button serves (person, animal, creature, object, vehicle, food); +# subject buttons without kinds serve Person when in a person group, else every type. +# group: palette heading. +# +# Copy to builders_user/prompt.yaml to change it; that copy replaces this one. +# character.yaml extends this file, so slot changes here reach the Wildcard Character Composer too. +# See docs/PROMPT_BUILDER_GUIDE.md. + +id: prompt +label: Wildcard Prompt Composer +description: A whole prompt, section by section, in the platform's order. + +parts: + - section: format + label: Format + hint: 'what the picture is: a photograph, an oil painting, a poster, a 3D render' + starter: "" + - section: subject + label: Subject + multi: true + hint: 'who or what the picture is about, and how they look' + starter: "a {age} {occupation} with {hair_color} hair and {eyes} eyes, {expression}" + - section: action + label: Action + hint: 'what the subject is doing, pose and gesture' + starter: "{pose}" + - section: setting + label: Setting + hint: 'where it happens, the place, era, season, weather and time of day' + starter: "in {location}, {time_of_day}" + - section: lighting + label: 'Light & atmosphere' + hint: 'the light - source, direction, quality and colour - and the air: fog, dust, rain, mood' + starter: "{light_setup}" + - section: composition + label: Composition + hint: 'shot size, framing, layout, depth layers and perspective' + starter: "" + - section: camera + label: Camera + hint: 'camera angle, lens, aperture and depth of field, camera and film stock' + starter: "" + - section: style + label: 'Style & finish' + hint: 'art style, movement or artist, colour palette and grade, texture and finish' + starter: "" + - section: text + label: Text + optional: true + hint: 'words shown in the picture, in quotes, with their lettering and placement' + starter: "" + - section: prompt + label: Prompt + optional: true + hint: 'a whole prompt from your prompt files (_prompts): Browse prompts picks a file or a whole theme folder' + starter: "" + +slots: + - section: format + slots: + - id: medium + label: Art medium + lists: ['art/mediums'] + match: + - 'mediums?|art_mediums|photo_medium' + - 'medium_.*' + avoid: ['hair', 'breast', 'animals', 'machines', 'distance'] + + - id: image_type + label: Image type + tooltip: 'What the picture is: a photograph, an oil painting, a 3D render, a film still ...' + lists: ['vp/format/image_type'] + match: + - 'image_types?|types?_of_images?|picture_types?|imagetypes?' + - 'image_(format|kind)s?' + + - id: photo_genre + label: Photo genre + tooltip: 'street, fashion editorial, documentary, macro, aerial, product, food photography ...' + lists: ['vp/format/photo_genre'] + match: + - 'photo(graphy)?_(genres?|types?|styles?)|photographic_styles?|photography_genres?' + + - id: illustration_type + label: Illustration type + lists: ['vp/format/illustration_type'] + match: + - 'illustration_(types?|styles?)|illustrations?|types?_of_illustrations?' + + - id: design_format + label: Design format + tooltip: 'poster, album cover, book cover, sticker, label, packaging ...' + lists: ['vp/format/design_format'] + match: + - 'design_formats?|posters?|poster_(types?|styles?)|covers?|album_covers?|book_covers?|stickers?|packaging' + avoid: ['cover_up', 'covered'] + + - section: subject + slots: + - id: age + label: Age + group: Person + lists: ['vp/subject/age'] + match: + - 'ages?' + - 'age_(groups?|ranges?)' + - '(fe)?male_age(_sfw)?' + avoid: ['object_age', 'age_of_', 'body_traits', 'jewelry', 'steampunk'] + + - id: person + label: Person + group: Person + tooltip: 'Who the subject is (woman, old man, knight...).' + match: + - 'person|people|persons|subjects?' + - '(fe)?male_(characters?|persons?|subjects?)|gender' + - 'characters?' + avoid: ['two_person', 'multi_character', 'hugging'] + + - id: group + label: Group / count + group: Person + kinds: ['person'] + tooltip: 'a couple, three friends, a crowd, a family of four ...' + lists: ['vp/subject/group'] + match: + - 'groups?|group_(sizes?|types?)|crowds?|couples?|number_of_people|people_count' + avoid: ['blood_group', 'age_group', 'ethnic_group', 'group_interactions'] + + - id: name + label: Name + group: Person + tooltip: First names. Woman / Man picks lists of female or male names; set the pill to Exact to keep the name as written. + match: + - 'names?|first_?names?|given_names?|names?(fe)?male|(fe)?male_?names?' + - '.*_(first_?)?names' + - 'surnames?(_\w+)?|last_?names?' + folders: ['(^|/)names?$', 'peoplename', 'firstnames?'] + avoid: ['note names', 'colou?rname', 'company', 'product', 'postname', 'code_name', 'placesin', 'artist', 'brand'] + + - id: occupation + label: Occupation + group: Person + match: + - '(character_)?occupations?' + - 'professions?|jobs' + - 'profession_(fantasy|scifi|cyberpunk|magic|fictional)' + avoid: ['image'] + + - id: nationality + label: Nationality + group: Person + match: + - '(all_|populair_)?nationalit(y|ies)' + - 'nationalities_short' + + - id: ethnicity + label: Ethnicity + group: Person + match: + - 'ethnicit(y|ies)|ethnics' + - 'races?' + - 'beautiful_(mixed_)?races_with_description.*' + avoid: ['demon', 'racetrack', 'racer', 'clothes', 'detail', 'nodetail', 'hairstyle', 'armor'] + + - id: species + label: Fantasy race + group: Person + kinds: ['person', 'creature'] + match: + - '(fantasy|exotic|dark)_races(_male)?' + - 'races_and_species|elder_scrolls_races|fantasy_creatures?' + + - id: body + label: Body type + group: Person + match: + - 'body_types?(_(fe)?male)?|body_shapes?|bodyshapes?|body_fit' + - 'build(_(fe)?male)?|builds|physique' + - '.*buildabody_body_shape' + + - id: skin + label: Skin + group: 'Face & hair' + match: + - 'skin_colou?rs?(_human|_normal)?|skin_?tones|skin' + - 'skin_(features|texture.*|freckles|condition)' + avoid: ['abnormal', 'fur', 'deepone', 'refraction', 'succubus', 'vampir', 'tight'] + + - id: freckles + label: 'Freckles & marks' + group: 'Face & hair' + lists: ['vp/subject/freckles'] + match: + - 'freckles?|moles?|birthmarks?|beauty_marks?|scars?|skin_marks?' + + - id: face + label: Face + group: 'Face & hair' + match: + - 'face_?shapes?|.*visage_faceshape' + - 'face(_features?)?|facial_features' + avoid: ['makeup', 'surface', 'interface', 'emot', 'smile', 'smug', 'drawing', 'accessor', 'teeth', 'neck', 'ear.?shapes?', 'eyelash', 'eye.?shapes?', 'mouth', 'apparence/face', 'lashes', '2000wildcards/face$'] + + - id: hair_style + label: Hair style + group: 'Face & hair' + match: + - 'hair_?styles?(_(fe)?male)?|haircuts?(_(fe)?male)?|hair_cut' + - 'hair(_(fe)?male)?' + - 'hair_(length|texture)|hairlength' + avoid: ['accessor', 'facial_hair', 'body_hair', 'no_hair', 'color', 'ethnicity'] + + - id: hair_color + label: Hair color + group: 'Face & hair' + match: + - 'hair_?colou?rs?' + - 'hair_?colou?rs?_.*|.*haircolor_natural.*' + + - id: hair_texture + label: Hair texture + group: 'Face & hair' + lists: ['vp/subject/hair_texture'] + match: ['hair_textures?|curls?|curl_types?|hair_curl'] + + - id: eyes + label: Eyes + group: 'Face & hair' + match: + - 'eyes?_?colou?rs?|eyes_iris_colors' + - 'eyes|eyes_shape|eye_types' + avoid: ['makeup', 'liner', 'lash', 'eyebrow', '(^|[/_])brows?($|[/_])', 'wear', 'closed', 'gaze'] + + - id: expression + label: Expression + group: 'Face & hair' + match: + - '(facial_)?expressions?|expression_basic' + - 'face_emotions|facial_expressions_and_mood|emotions?' + avoid: ['expressionism', 'withouttheir', 'withtheir', 'lora', 'etoanima', 'etoexpression', 'psycholog'] + + - id: gaze + label: Gaze + group: 'Face & hair' + tooltip: 'Where the subject looks: over the shoulder, eyes closed, past the camera ...' + lists: ['vp/subject/gaze'] + match: + - 'gazes?|eye_contact|looking(_at|_direction)?|eye_direction|gaze_direction' + + - id: makeup + label: Makeup + group: 'Face & hair' + match: ['makeup(_styles)?|makeup_styles_creative|styles_makeup_styles'] + avoid: ['cosplay', 'goth'] + + - id: outfit + label: Outfit + group: Clothing + match: + - 'outfits?|clothing|clothes|costumes?' + - '(wo)?men_outfits|(fe)?male_outfits|.*_outfits?' + - 'wardrobes?|attires?|garments?' + avoid: ['stripclub', 'bustier', 'miniskirt', 'image', 'fully_clothed', 'noclothing', 'dpachartester'] + + - id: fashion_style + label: Fashion style + group: Clothing + tooltip: 'punk, preppy, cottagecore, streetwear, avant-garde ...' + lists: ['vp/subject/fashion_style'] + match: + - 'fashion_styles?|fashion|subcultures?|clothing_styles?|style_of_dress' + + - id: top + label: Top + group: Clothing + match: + - 'tops?|top_casual|clothing_upper_body|clothing_(fe)?male_top|attire_shirts_and_topwear' + - '(girl|guy)?tops?\d?(_detail)?' + avoid: ['rooftop', 'stop', 'hair_styles_top', 'dystopia'] + + - id: bottom + label: Bottom + group: Clothing + match: + - 'bottoms?|bottomwear|clothing_(fe)?male_bottom|attire_pants_and_bottomwear' + - 'pants|skirts?|.*_bottoms|creator_(pants|skirts)' + avoid: ['bikini', 'swimsuit', 'cutout', 'enhancer', 'body_features', 'sexy'] + + - id: outerwear + label: Outerwear + group: Clothing + match: ['outerwear|coats?|jackets?|.*_jackets?|.*_coats?'] + avoid: ['coat_of_arms', 'coating', 'paint_coat'] + + - id: footwear + label: Footwear + group: Clothing + match: + - 'footwear|shoes' + - 'creator_shoes|.*_footwear|modernfootwear|attire_shoes_and_footwear' + + - id: headwear + label: Headwear + group: Accessories + match: + - 'hats?|headwear|headgear' + - 'headwear_.*|.*_hats' + + - id: accessories + label: Accessories + group: Accessories + match: + - 'accessor(y|ies)|jewelry|clothing_accessories|jewelry_and_accessories' + - '.*_accessories' + avoid: ['access_', 'accessible', 'hair_', 'eyes_', 'properties'] + + - id: hair_length + label: Hair length + group: 'Face & hair' + match: ['hair_?length|hairlength|hair_(long|short|medium)'] + + - id: facial_hair + label: Facial hair + group: 'Face & hair' + match: ['facial_hair|beards?|mustaches?|moustaches?'] + + - id: lips + label: Lips + group: 'Face & hair' + match: ['lips?|lipstick|lip_colou?rs?'] + + - id: nose + label: Nose + group: 'Face & hair' + match: ['noses?|nose_shapes?'] + + - id: tattoo + label: Tattoo + group: 'Face & hair' + match: ['tattoos?|.*_tattoos?'] + + - id: dress + label: Dress + group: Clothing + match: + - 'dress(es)?|gowns?' + - '.*_dress(es)?|.*_gowns?' + avoid: ['dress_code', 'undress'] + + - id: uniform + label: Uniform + group: Clothing + match: + - 'uniforms?' + - '.*_uniforms?' + avoid: ['school'] + + - id: armor + label: Armor + group: Clothing + match: + - 'armou?rs?' + - '.*_armou?rs?' + + - id: swimwear + label: Swimwear + group: Clothing + match: ['swim(wear|suits?)|bikinis?|beachwear'] + + - id: legwear + label: Legwear + group: Clothing + match: ['legwear|socks|stockings|tights|hosiery'] + + - id: fabric + label: Fabric + group: Clothing + match: + - 'fabrics?|textiles?' + - 'clothing_materials?|usualclothesmaterialbasic' + avoid: ['building', 'surface', 'render'] + + - id: pattern + label: Pattern + group: Clothing + match: ['patterns?|prints|fabric_patterns?'] + avoid: ['background', 'bg/'] + + - id: eyewear + label: Eyewear + group: Accessories + match: ['eyewear|glasses|sunglasses|goggles'] + + - id: jewelry + label: Jewelry + group: Accessories + match: ['jewel(le)?ry|necklaces?|earrings?|bracelets?'] + avoid: ['properties', 'jeweler'] + + - id: neckwear + label: Neckwear + group: Accessories + match: ['neckwear|scarf|scarves|neckties?|chokers?'] + + - id: bag + label: Bag + group: Accessories + match: ['bags?|handbags?|purses?|backpacks?'] + + - id: gloves + label: Gloves + group: Accessories + match: ['gloves?|handwear|.*_gloves?'] + + - id: piercing + label: Piercing + group: Accessories + match: ['piercings?|.*_piercings?'] + + - id: mask + label: Mask + group: Accessories + match: ['masks?|facewear|face_masks?|.*_masks?'] + avoid: ['face_mask_(sheet|skin)', 'masking'] + + - id: underwear + label: Underwear + group: Clothing + match: + - 'underwear|lingerie|bras?|panties|underwear_(top|bottom)|.*_lingerie|.*_underwear' + + - id: eyebrows + label: Eyebrows + group: 'Face & hair' + match: ['eyebrows?|brows?|eyebrow_shapes?'] + + - id: wings + label: Wings + group: Person + kinds: ['person', 'creature'] + match: ['wings?|.*_wings?'] + avoid: ['building', 'hospital', 'chicken'] + + - id: creature + label: Creature + group: Person + kinds: ['creature'] + tooltip: 'Monsters, dragons, mythical and fantasy creatures as the subject.' + match: + - 'creatures?|monsters?|beasts?|cryptids?' + - 'mythical_creatures?|fantasy_creatures?|.*_creatures?|.*_monsters?|dragons?|dinosaurs?' + avoid: ['creature_feature', 'crea_'] + + - id: animal + label: Animal + group: Animal + kinds: ['animal'] + match: + - 'animals?|animal_(types?|species)|wildlife|.*_animals?' + - 'birds?|breeds?|dog_breeds?|cat_breeds?|horses?|fish|sea_life|insects?|reptiles?' + avoid: ['animal_interaction', 'animal_print', 'animal_ears'] + + - id: animal_coat + label: 'Coat & markings' + group: Animal + kinds: ['animal'] + lists: ['vp/subject/animal_coat'] + match: + - 'coats?_(colou?rs?|patterns?)|fur_(colou?rs?|patterns?)|markings|animal_markings' + + - id: object + label: Object + group: Object + kinds: ['object'] + match: + - 'objects?|items?|products?|gadgets?|tools?|.*_objects?|.*_items?|random_objects?' + avoid: ['held_item', 'items_held', 'rpg'] + + - id: product + label: Product + group: Object + kinds: ['object'] + tooltip: 'a wristwatch, a perfume bottle, wireless headphones, a ceramic mug ... (for product shots)' + lists: ['vp/subject/product'] + match: + - 'product_(categor(y|ies)|types?|names?)|consumer_products?|merchandise|commercial_products?' + + - id: object_condition + label: Condition + group: Object + kinds: ['object', 'vehicle'] + tooltip: 'brand new, weathered, rusted, hand-painted, broken ...' + lists: ['vp/subject/object_condition'] + match: + - 'conditions?|object_conditions?|wear_and_tear|states?_of_(repair|wear)' + + - id: subject_vehicle + label: Vehicle + group: Vehicle + kinds: ['vehicle'] + match: + - 'vehicles?|cars?|car_(types?|models?)|aircraft|airplanes?|ships?|boats?|motorcycles?|trains?|.*_vehicles?|spaceships?' + avoid: ['car_interior', 'carpet'] + + - id: dish + label: Dish + group: Food + kinds: ['food'] + match: + - 'dishes|foods?|meals?|cuisines?|desserts?|.*_dishes|.*_foods?|foods?_.*|pastr(y|ies)|cakes?' + avoid: ['food_play', 'foodporn'] + + - id: drink + label: Drink + group: Food + kinds: ['food'] + match: + - 'drinks?|beverages?|cocktails?|.*_drinks?|drinks?_.*|coffee|teas?|wines?' + + - id: plating + label: Plating + group: Food + kinds: ['food'] + tooltip: 'on a slate board, in a bento box, family style ...' + lists: ['vp/subject/plating'] + match: ['platings?|food_presentation|servings?|tableware|dishware'] + + - section: action + slots: + - id: pose + label: Pose + tooltip: Krea 2 wants each subject grouped with its own actions; FLUX.2 lists action right after the subject. + match: + - 'poses?|poses_general|pose_action|action_pose' + - 'posture_poses|postures?_and_poses|standing_poses|sitting_poses' + avoid: ['stripclub', 'sexy', 'profile_pose', 'rear_pose'] + + - id: gesture + label: Gesture + lists: ['vp/action/gesture', 'gestures'] + match: + - 'gestures?|hand_gestures|pose_hand_gestures' + - 'gestures_.*' + avoid: ['fellatio'] + + - id: activity + label: Activity + lists: ['action'] + match: + - 'actions?|activities|all_activities|human_activities' + - 'actions_.*|.*_actions' + avoid: ['sexy', 'stripclub', 'fire_', 'food_', 'hair_', 'headwear_', 'neck', 'plant_', 'sleeves', 'tree_', 'legwear', 'multi_character'] + + - id: interaction + label: Interaction + lists: ['vp/action/interaction'] + match: + - 'interactions?|group_interactions|item_interaction|animal_interaction' + - 'multi_character_action_.*' + avoid: ['intimate'] + + - id: held_item + label: Holding + lists: ['vp/action/holding'] + match: + - 'holding|held_items?|items_held' + - 'props?|hand_props?' + + - id: weapon + label: Weapon + match: + - 'weapons?|swords?|guns?|blades?' + - '.*_weapons?' + + - id: companion + label: Animal / pet + match: + - 'pets?|companions?|animals?' + - '.*_pets?|.*_animals?' + avoid: ['animal_interaction', 'animals_types'] + + - section: setting + slots: + - id: location + label: Location + group: Place + lists: ['location'] + match: + - 'locations?|places|places_sceneries|scenery|scenes?' + - 'location_.*|.*_locations|random_?places' + - 'settings?|environments?' + avoid: ['background', 'piercing', 'posture', 'knee', 'leg_location', 'stripclub', 'lora', 'eto', 'breast', 'color', 'pattern', '0\d_'] + + - id: background + label: Background + group: Place + tooltip: 'Backgrounds and backdrops (every list with "background" in its name or folder).' + search: background + match: ['backgrounds?|backdrops?|.*_backgrounds?|background_.*'] + avoid: ['background_(character|people|person)s?$'] + + - id: studio_backdrop + label: Studio backdrop + group: Place + tooltip: 'seamless grey paper, painted canvas backdrop, white cyclorama ...' + lists: ['vp/setting/studio_backdrop'] + match: + - 'studio_backdrops?|backdrops?|studio_backgrounds?|photo_backdrops?' + + - id: food + label: 'Food & drink' + group: Things + match: + - 'foods?|dishes|drinks?|beverages?|fruits?|desserts?|meals?|cuisines?|.*_foods?|.*_drinks?' + - 'foods?_.*|drinks?_.*' + avoid: ['food_play', 'foodporn'] + + - id: plants + label: 'Plants & flowers' + group: Things + match: + - 'plants?|flowers?|trees?|flora|.*_flowers?|.*_plants?|.*_trees?' + avoid: ['power_plant', 'planter'] + + - id: interior + label: Interior + group: Place + match: + - 'interiors?|interior_design|interior_setting|rooms' + - 'living_room|bedroom|kitchen|library_interior|random_room' + avoid: ['breast', 'train_interior_yawning'] + + - id: landscape + label: Landscape + group: Place + match: + - 'landscapes?|random_landscape|landscape_basic' + - '.*_landscapes?' + avoid: ['composition', 'aesthetic_styles', 'abstract'] + + - id: architecture + label: Architecture + group: Place + match: + - 'architecture2?|architectural|architecture_list|buildings?' + - '.*_architecture|.*_buildings|architecturebuildings' + avoid: ['burning', 'alcove', 'scale', 'telephoto', 'detail'] + + - id: landmark + label: Landmark + group: Place + tooltip: 'Famous buildings, monuments and places around the world.' + match: + - 'landmarks?|famous_(buildings|places|landmarks|locations|monuments)|location_famous' + - 'monuments?|wonders|world_wonders|wonders_of_.*|tourist_attractions?|attractions' + - 'places_?in_?(asia|america|europe|africa|the_world|\w+)' + + - id: city + label: City / street + group: Place + match: + - 'city|city_streets?|random_city' + - '.*_city|cityscape|futuristic_cityscape' + + - id: era + label: Era + group: Time + lists: ['vp/setting/era'] + match: ['era|eras|decades?|time_period|timeperiods|time_era'] + avoid: ['camera'] + + - id: season + label: Season + group: Time + match: ['seasons?|scene_season_sfw'] + + - id: time_of_day + label: Time of day + group: Time + lists: ['time'] + match: ['time_of_day|timeofday|daytime|time'] + avoid: ['travel', 'signature', 'magic', 'warp', 'distortion', 'portrait', 'part_timer'] + + - id: weather + label: Weather + group: Time + lists: ['weather'] + match: + - 'weather|scene_weather_sfw|weatherseason|sky_condition' + - '.*_weather' + avoid: ['weathering', 'flowers', 'kyubi'] + + - id: room + label: Room + group: Place + match: + - 'rooms?|random_room' + - '(living|dining|bed|bath|class|ball|game|hotel|meeting|throne|locker)_?room' + avoid: ['mushroom', 'stripclub', 'storage'] + + - id: furniture + label: Furniture + group: Things + match: ['furniture|furnishings?'] + + - id: vehicle + label: Vehicle + group: Things + match: + - 'vehicles?|cars?' + - '.*_vehicles?' + + - id: culture + label: Culture + group: Place + match: ['cultures?|cultural'] + + - id: props + label: 'Props & set dressing' + group: Things + lists: ['vp/setting/props'] + match: + - 'set_dressing|decor|decorations?|scene_props|props_(scene|set)|set_props|.*_decor' + avoid: ['decorative_(border|frame)'] + + - section: lighting + slots: + - id: light_setup + label: Light setup + lists: ['light/setups'] + match: + - 'lighting|lightings|light|studiolight|cinematiclight' + - 'lighting_(indoor|outdoor|mood|elements)|variouslights' + avoid: ['lightning', 'stripclub', 'body_light', 'flight', 'strangelight', 'interaction', 'body.?type', 'species', 'profession', 'magical/types'] + + - id: light_quality + label: Light quality + lists: ['light/quality'] + match: ['light_quality|lighting_quality|lighting_mood'] + + - id: light_direction + label: Light direction + tooltip: 'backlit, rim light, side light, top light ...' + lists: ['vp/lighting/light_direction'] + match: + - 'light_directions?|lighting_directions?|backlight(ing)?|rim_light(ing)?|direction_of_light' + + - id: studio_setup + label: Studio setup + tooltip: 'Rembrandt, butterfly, clamshell, split lighting ...' + lists: ['vp/lighting/studio_setup'] + match: + - 'studio_(lighting|setups?|light_setups?)|lighting_(setups?|patterns?|techniques?)|portrait_lighting' + + - id: colour_temperature + label: Colour temperature + lists: ['vp/lighting/colour_temperature'] + match: + - 'colou?r_temperatures?|light_temperatures?|kelvin|white_balance' + + - id: golden + label: Time light + match: ['golden_hour|lighitingtime|.*timelighting.*|weather&lights'] + + - id: atmosphere + label: 'Atmosphere & effects' + tooltip: 'morning fog, dust motes, rain streaks, smoke, heat haze ...' + lists: ['vp/lighting/atmosphere'] + match: + - 'atmospheres?|atmospheric_effects?|atmosphere_(effects?|elements?)|fog|mist|haze|particles?|weather_effects?' + - 'effects?|light_effects?|visual_effects?' + avoid: ['sound', 'audio', 'sfx', 'effect_on', 'special_effects_makeup'] + + - id: mood + label: Mood + match: ['moods?|atmosphere_moods?'] + avoid: ['lighting_mood'] + + - section: composition + slots: + - id: shot + label: Shot size + lists: ['camera/shots'] + match: + - 'shots?|shot_type|shotstyle(_simple)?|camera_distance|framing|body_framing' + avoid: ['pantyshot', 'artistic'] + + - id: composition + label: Framing + lists: ['vp/composition/framing'] + match: + - 'compositions?|photo_composition|composition_techniques|framing_techniques' + - 'frame_composition|rule_of_thirds|layouts?' + + - id: layout + label: 'Layout & space' + tooltip: 'negative space, leading lines, subject small in the frame ...' + lists: ['vp/composition/layout'] + match: + - 'layout_space|negative_space|leading_lines|visual_(balance|flow)|spatial_layout' + + - id: depth_layers + label: Depth layers + lists: ['vp/composition/depth_layers'] + match: + - 'depth_layers?|foreground(_elements?)?|layering|depth_(cues|effects?)' + + - id: perspective + label: Perspective + lists: ['vp/composition/perspective'] + match: ['perspectives?|views?|pov|viewpoints?|camera_perspectives?'] + avoid: ['review', 'preview', 'interview', 'overview'] + + - section: camera + slots: + - id: angle + label: Angle + match: ['camera_angles?|angles?|view_angle'] + avoid: ['strength'] + + - id: lens + label: Lens + lists: ['camera/lenses'] + match: ['lens|lenses|photo_lens|focal_length'] + + - id: camera_body + label: Camera + match: ['cameras?|camera_models|cameratype'] + + - id: film_stock + label: Film stock + lists: ['camera/film_stocks'] + match: ['film_?types?|film_stocks?|film'] + + - id: aperture + label: 'Aperture & focus' + tooltip: 'shallow depth of field, f/1.4 bokeh, deep focus ...' + lists: ['vp/camera/aperture'] + match: + - 'apertures?|depth_of_field|dof|bokeh|focus|f_stops?|focus_(types?|effects?)' + + - id: shutter + label: 'Shutter & motion blur' + tooltip: 'long exposure light trails, motion blur, frozen action ...' + lists: ['vp/camera/shutter'] + match: + - 'shutter(_speeds?)?|long_exposures?|motion_blur|exposure_(types?|effects?)' + + - section: style + slots: + - id: movement + label: Art movement + group: Look + lists: ['art/movements'] + match: ['art_?movements?|styles_art_movements|movement'] + + - id: art_style + label: Art style + group: Look + match: + - 'art_?styles?|artstyles|random_artstyle|general_art_style' + - 'styles_(digital|game)_art_styles|random_(depiction|image|animation)_styles' + avoid: ['hair'] + + - id: color_grade + label: Colour grade + group: Colour + lists: ['vp/style/colour_grade'] + match: ['colou?r_grading|filters_colourgrading|colou?r_grades?'] + + - id: colour_palette + label: Colour palette + group: Colour + tooltip: 'teal and orange, muted earth tones, monochrome red, pastel ...' + lists: ['vp/style/colour_palette'] + match: + - 'colou?r_palettes?|palettes?|colou?r_schemes?|colou?r_combinations?|accent_palette|.*_palettes?' + + - id: photographer + label: Photographer + group: Look + lists: ['vp/style/photographer'] + match: ['photographers?|artist_photographer'] + + - id: artist + label: Artist + group: Look + match: + - 'artists?|painters?' + - 'artist_(fineart|fantasy|scifi|anime|cartoon|botanical|black_white|horror)' + avoid: ['nudity', 'photographer', 'profession'] + + - id: color + label: Colour + group: Colour + match: ['colou?rs?|colou?r_names|colors_basic'] + avoid: ['hair', 'eye', 'skin', 'danbooru'] + + - id: material + label: Material / texture + group: Finish + match: + - 'materials?|textures?|surfaces?|.*_materials?|.*_textures?' + - 'metals?|woods?|stones?|minerals?|gemstones?' + avoid: ['clothing', 'fabric', 'hair', 'skin', 'building_material'] + + - id: texture_finish + label: 'Texture & finish' + group: Finish + tooltip: 'film grain, matte, halftone dots, glossy ...' + lists: ['vp/style/texture_finish'] + match: + - 'finish(es)?|surface_finish|grain|film_grain|image_texture|print_texture|textures?_(effects?|overlays?)' + + - id: theme + label: Theme + group: Look + match: ['themes?|.*_themes?|motifs?|concepts?'] + avoid: ['theme_color'] + + - id: era_aesthetic + label: Era aesthetic + group: Look + tooltip: '1980s VHS, Y2K, Victorian daguerreotype ...' + lists: ['vp/style/era_aesthetic'] + match: + - 'era_aesthetics?|decade_aesthetics?|retro_styles?|vintage_styles?|aesthetics?_by_era' + + - id: rendering + label: Render / quality + group: Finish + match: + - 'rendering|renders?|render_(styles?|engines?)|quality(_tags)?|detail_level|details?_level' + avoid: ['lighting_quality', 'light_quality'] + + - section: text + slots: + - id: typography + label: Typography + tooltip: 'bold condensed sans serif, hand-lettered brush script ...' + lists: ['vp/text/typography'] + match: + - 'typography|fonts?|font_(styles?|types?)|typefaces?|lettering(_styles?)?' + + - id: text_placement + label: Placement + lists: ['vp/text/text_placement'] + match: ['text_(placement|positions?)|placement'] diff --git a/builders/templates/illustration.json b/builders/templates/illustration.json new file mode 100644 index 0000000..530d4bd --- /dev/null +++ b/builders/templates/illustration.json @@ -0,0 +1,29 @@ +{ + "v": 1, + "name": "Illustration", + "builder": "prompt", + "platform": "", + "saved": "", + "composer": { + "v": 1, + "parts": { + "format": [{"t": "text", "v": "an"}, {"t": "tok", "id": 1}, {"t": "text", "v": ""}], + "subject": [{"t": "text", "v": "a"}, {"t": "tok", "id": 2}, {"t": "text", "v": "person"}], + "action": [{"t": "text", "v": ""}, {"t": "tok", "id": 3}, {"t": "text", "v": ""}], + "setting": [{"t": "text", "v": "against"}, {"t": "tok", "id": 4}, {"t": "text", "v": ""}], + "style": [{"t": "text", "v": ""}, {"t": "tok", "id": 5}, {"t": "text", "v": ","}, {"t": "tok", "id": 6}, {"t": "text", "v": ""}] + }, + "tokens": { + "1": {"slot": "", "key": "vp/format/illustration_type", "label": "Illustration type", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "2": {"slot": "", "key": "vp/subject/age", "label": "Age", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "3": {"slot": "", "key": "vp/action/gesture", "label": "Gesture", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "4": {"slot": "", "key": "vp/setting/studio_backdrop", "label": "Studio backdrop", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "5": {"slot": "", "key": "vp/style/colour_palette", "label": "Colour palette", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "6": {"slot": "", "key": "vp/style/texture_finish", "label": "Texture & finish", "mode": "random", "picks": [], "count": 1, "keep": "meaning"} + }, + "next": 7, + "locks": {}, + "fill": {}, + "kinds": {"subject": "person"} + } +} diff --git a/builders/templates/product_shot.json b/builders/templates/product_shot.json new file mode 100644 index 0000000..b441b1b --- /dev/null +++ b/builders/templates/product_shot.json @@ -0,0 +1,33 @@ +{ + "v": 1, + "name": "Product Shot", + "builder": "prompt", + "platform": "", + "saved": "", + "composer": { + "v": 1, + "parts": { + "format": [{"t": "text", "v": "a"}, {"t": "tok", "id": 1}, {"t": "text", "v": ""}], + "subject": [{"t": "text", "v": "a"}, {"t": "tok", "id": 2}, {"t": "text", "v": ""}, {"t": "tok", "id": 3}, {"t": "text", "v": ""}], + "setting": [{"t": "text", "v": "on"}, {"t": "tok", "id": 4}, {"t": "text", "v": ""}], + "lighting": [{"t": "text", "v": ""}, {"t": "tok", "id": 5}, {"t": "text", "v": ""}], + "composition": [{"t": "text", "v": ""}, {"t": "tok", "id": 6}, {"t": "text", "v": ""}], + "camera": [{"t": "text", "v": ""}, {"t": "tok", "id": 7}, {"t": "text", "v": ""}], + "style": [{"t": "text", "v": ""}, {"t": "tok", "id": 8}, {"t": "text", "v": ""}] + }, + "tokens": { + "1": {"slot": "", "key": "vp/format/photo_genre", "label": "Photo genre", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "2": {"slot": "", "key": "vp/subject/object_condition", "label": "Condition", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "3": {"slot": "", "key": "vp/subject/product", "label": "Product", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "4": {"slot": "", "key": "vp/setting/studio_backdrop", "label": "Studio backdrop", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "5": {"slot": "", "key": "vp/lighting/studio_setup", "label": "Studio lighting", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "6": {"slot": "", "key": "vp/composition/framing", "label": "Framing", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "7": {"slot": "", "key": "vp/camera/aperture", "label": "Aperture", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "8": {"slot": "", "key": "vp/style/colour_grade", "label": "Colour grade", "mode": "random", "picks": [], "count": 1, "keep": "meaning"} + }, + "next": 9, + "locks": {}, + "fill": {}, + "kinds": {"subject": "object"} + } +} diff --git a/builders/templates/studio_portrait.json b/builders/templates/studio_portrait.json new file mode 100644 index 0000000..ca55ac9 --- /dev/null +++ b/builders/templates/studio_portrait.json @@ -0,0 +1,33 @@ +{ + "v": 1, + "name": "Studio Portrait", + "builder": "prompt", + "platform": "", + "saved": "", + "composer": { + "v": 1, + "parts": { + "format": [{"t": "text", "v": "a"}, {"t": "tok", "id": 1}, {"t": "text", "v": ""}], + "subject": [{"t": "text", "v": "a"}, {"t": "tok", "id": 2}, {"t": "text", "v": "person,"}, {"t": "tok", "id": 3}, {"t": "text", "v": ""}], + "action": [{"t": "text", "v": ""}, {"t": "tok", "id": 4}, {"t": "text", "v": ""}], + "setting": [{"t": "text", "v": "against"}, {"t": "tok", "id": 5}, {"t": "text", "v": ""}], + "lighting": [{"t": "text", "v": ""}, {"t": "tok", "id": 6}, {"t": "text", "v": ""}], + "camera": [{"t": "text", "v": ""}, {"t": "tok", "id": 7}, {"t": "text", "v": ""}], + "style": [{"t": "text", "v": ""}, {"t": "tok", "id": 8}, {"t": "text", "v": ""}] + }, + "tokens": { + "1": {"slot": "", "key": "vp/format/image_type", "label": "Image type", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "2": {"slot": "", "key": "vp/subject/age", "label": "Age", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "3": {"slot": "", "key": "vp/subject/gaze", "label": "Gaze", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "4": {"slot": "", "key": "vp/action/gesture", "label": "Gesture", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "5": {"slot": "", "key": "vp/setting/studio_backdrop", "label": "Studio backdrop", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "6": {"slot": "", "key": "vp/lighting/studio_setup", "label": "Studio lighting", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "7": {"slot": "", "key": "vp/camera/aperture", "label": "Aperture", "mode": "random", "picks": [], "count": 1, "keep": "meaning"}, + "8": {"slot": "", "key": "vp/style/colour_grade", "label": "Colour grade", "mode": "random", "picks": [], "count": 1, "keep": "meaning"} + }, + "next": 9, + "locks": {}, + "fill": {}, + "kinds": {"subject": "person"} + } +} diff --git a/concept_survival_node.py b/concept_survival_node.py index 05835ec..61bb0a7 100644 --- a/concept_survival_node.py +++ b/concept_survival_node.py @@ -42,7 +42,8 @@ JUDGE_SYSTEM = ( "decide whether the prompt actually states that idea, in any wording. A related topic does not count: a pigeon " "is not a barn owl, light from a window is not light from below, rain outside is not rain indoors. " "Answer with JSON only, no other text, in this shape: " - '{"results": [{"concept": "", "present": true, "phrase": ""}]} ' + '{"results": [{"n": 1, "concept": "", "present": true, "phrase": ""}]} ' + "One result per concept, in the same order, with its number in n. " "Use present=false and an empty phrase when the idea is missing." ) @@ -60,14 +61,51 @@ def judge_with_llm(prompt, concepts, backend, endpoint, model): return None, response.get("error", "LLM call failed") try: parsed = extract_json_object(clean_llm_output(response.get("response", ""), keep_json=True)) - verdicts = {} - for item in parsed.get("results", []): - verdicts[str(item.get("concept", "")).strip().lower()] = (bool(item.get("present")), str(item.get("phrase", "") or "")) - return verdicts, "" + return _match_verdicts(parsed.get("results", []), concepts), "" except Exception as error: return None, f"the judge's answer was not usable JSON ({error})" +def _words(text): + import re + return set(re.findall(r"[a-z]{3,}", (text or "").lower())) + + +def _match_verdicts(results, concepts): + """{concept.lower(): (present, phrase)}. Matched by the concept's number, then by its exact text, then by word + overlap, then by position. Keying on the exact text alone lost every verdict whenever the judge echoed a + concept slightly differently (live 2026-10-01, two-sentence surreal concepts: six of nine runs reported + '[none]' for anomalies that were plainly in the prompt).""" + verdicts, unmatched = {}, [] + lowered = [concept.strip().lower() for concept in concepts] + for position, item in enumerate(results if isinstance(results, list) else []): + if not isinstance(item, dict): + continue + value = (bool(item.get("present")), str(item.get("phrase", "") or "")) + key = None + try: + number = int(item.get("n")) + if 1 <= number <= len(concepts): + key = lowered[number - 1] + except (TypeError, ValueError): + pass + text = str(item.get("concept", "")).strip().lower() + if key is None and text in lowered: + key = text + if key is None and text: + best = max(lowered, key=lambda concept: len(_words(concept) & _words(text)), default=None) + if best is not None and len(_words(best) & _words(text)) >= max(2, len(_words(best)) // 3): + key = best + if key is None: + unmatched.append((position, value)) + continue + verdicts.setdefault(key, value) + for position, value in unmatched: # last resort: same position as the concept list + if position < len(lowered) and lowered[position] not in verdicts: + verdicts[lowered[position]] = value + return verdicts + + class ConceptSurvivalCheck: CATEGORY = "Eric Prompt Enhancers" FUNCTION = "check" diff --git a/config.json.example b/config.json.example index 91f342e..193ea98 100644 --- a/config.json.example +++ b/config.json.example @@ -32,6 +32,16 @@ } }, "wildcards": { - "path": "models/wildcards" + "path": "models/wildcards", + "pool_mode": "scoped", + "strip_lora": true, + "catalog": true, + "content_filter": true + }, + "lora_suggest": { + "catalog": "L:/Models/loras/_lora_catalog.sqlite", + "vocab": "A:/LoRA_Catalog/vocab.yaml", + "block_real_person_nsfw": true, + "platforms": {} } } diff --git a/core/builder/__init__.py b/core/builder/__init__.py new file mode 100644 index 0000000..cd91df4 --- /dev/null +++ b/core/builder/__init__.py @@ -0,0 +1,32 @@ +""" +Prompt builders: slot-button composers that write wildcard templates section by section. + + slots.py builder definitions (builders/*.yaml) and which lists back each slot + compose.py composer state -> wildcard template; platform section order + sections.py PROMPT_SECTIONS, the bundle builders pass along + +No ComfyUI dependency. See docs/PROMPT_BUILDER_GUIDE.md. + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from .compose import (DEFAULT_ORDER, MODES, OPTIONAL_SECTIONS, SECTION_LABELS, SECTIONS, ComposerState, Token, base_section, escape_literal, + join_pieces, join_sections, order_sections, section_label, section_order, section_seed_offset) +from .layout import LAYOUT_FILE, apply_layout, layout_hide, layout_move, read_layout, write_layout +from .merge import MINE_FOLDER, fold_key, merge_entries, merged_key, merged_path, pick_root, slot_words, write_merged +from .more import more_lists, path_words, section_of +from .sections import PromptSections +from .slots import (MY_BUTTONS_FILE, PERSON_GROUPS, SUBJECT_KINDS, kinds_of, BuilderDef, ListInfo, PartDef, SlotDef, SlotMatcher, builder_from_dict, + button_id, clean_button, key_has_terms, leaf_name, load_builders, read_buttons, write_buttons) + +__all__ = [ + "MODES", "SECTIONS", "DEFAULT_ORDER", "OPTIONAL_SECTIONS", "SECTION_LABELS", "ComposerState", "Token", "escape_literal", "join_pieces", + "join_sections", "section_order", "order_sections", "section_label", "section_seed_offset", "base_section", + "PromptSections", + "BuilderDef", "PartDef", "SlotDef", "ListInfo", "SlotMatcher", "builder_from_dict", "leaf_name", "load_builders", + "MINE_FOLDER", "fold_key", "merge_entries", "merged_key", "merged_path", "pick_root", "slot_words", "write_merged", + "LAYOUT_FILE", "apply_layout", "layout_hide", "layout_move", "read_layout", "write_layout", + "more_lists", "path_words", "section_of", + "PERSON_GROUPS", "SUBJECT_KINDS", "kinds_of", + "MY_BUTTONS_FILE", "button_id", "clean_button", "key_has_terms", "read_buttons", "write_buttons", +] diff --git a/core/builder/compose.py b/core/builder/compose.py new file mode 100644 index 0000000..533e2cf --- /dev/null +++ b/core/builder/compose.py @@ -0,0 +1,272 @@ +""" +Composer state -> wildcard template text, and prompt sections in platform order. + +The builder widget stores its state as JSON in the node's hidden `composer` widget: + + { + "v": 1, + "parts": { + "subject": [ {"t": "text", "v": "a"}, {"t": "tok", "id": 1}, {"t": "text", "v": "in typical uniform"} ] + }, + "tokens": { + "1": {"slot": "age", "key": "cyberrealistic_v10/cyberdelia/miscellaneous/cyber_age", + "mode": "subset", "picks": ["young adult", "elderly"], "count": 1, "keep": "exact"} + } + } + +mode random any entry of the list __key__ + fixed always picks[0] (may be your own text) {1$$value} + subset random among picks {a|b|c} + exclude random, never one of picks __key!{a|b}__ +count how many distinct picks, joined with " and " {2$$ and $$...} +keep how the LLM rewriter treats the pick (binding markers of the wildcard engine): + meaning (default, wording may change), exact (=, kept word for word), loose (?, may be dropped) + +Text between pills is used as typed, and may itself hold wildcard syntax ({red|blue}, __name__). Pieces are joined with single spaces and a space before +, . ; : ! ? is removed, so "a", pill, ", with" reads "a young adult, with". + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence, Tuple + +MODES = ("random", "fixed", "subset", "exclude") +KEEP_MARKERS = {"meaning": "", "concept": "", "exact": "=", "verbatim": "=", "loose": "?"} +MAX_COUNT = 8 +JOINER = " and " + +# Canonical prompt sections. This tuple is the SEED order (section_seed_offset): new sections are +# only ever appended, so saved workflows keep re-rolling the same picks. See dev/PROMPT_TAXONOMY_SPEC.md. +SECTIONS = ("subject", "action", "setting", "lighting", "style", "camera", "format", "composition", "text", "prompt") +# The order sections are built and joined in when neither the node nor the platform says otherwise. +DEFAULT_ORDER = ("format", "prompt", "subject", "action", "setting", "lighting", "composition", "camera", "style", "text") +# Sections a node shows only when added ("+ Section"). +OPTIONAL_SECTIONS = ("text", "prompt") +SECTION_LABELS = { + "format": "Format", "subject": "Subject", "action": "Action", "setting": "Setting", + "lighting": "Light & atmosphere", "composition": "Composition", "camera": "Camera", + "style": "Style & finish", "text": "Text", "prompt": "Prompt", +} +# Words in a profile's `structure` line that name each section. +_SECTION_WORDS = { + "prompt": r"full prompt|base prompt|prompt file", + "format": r"format|image type|type of image|medium|intended use|use first|photograph of|poster|product shot", + "subject": r"subjects?|who|character|entity", + "action": r"actions?|pose|poses|motion|doing", + "setting": r"setting|background|context|environment|location|scene|place", + "lighting": r"light|lighting|atmosphere", + "composition": r"composition|compositional|framing|layout|placement|viewpoint|shot size", + "camera": r"camera|technical|lens|focal length|aperture|shot", + "style": r"style|aesthetic|art ?style|finish|stylisation|stylization", + "text": r"text|lettering|typography|wording", +} + +_ESCAPE = re.compile(r"([{}|$\[\]#\\])") +_SPACE_BEFORE_PUNCT = re.compile(r"\s+([,.;:!?])") +_MULTI_SPACE = re.compile(r"[ \t]{2,}") + + +def escape_literal(text: str) -> str: + """Make typed text or a picked entry safe inside wildcard syntax.""" + escaped = _ESCAPE.sub(r"\\\1", text or "") + return escaped.replace("__", "\\_\\_") + + +def join_pieces(pieces: Sequence[str]) -> str: + joined = " ".join(piece.strip() for piece in pieces if piece and piece.strip()) + joined = _SPACE_BEFORE_PUNCT.sub(r"\1", joined) + return _MULTI_SPACE.sub(" ", joined).strip() + + +@dataclass +class Token: + id: str + slot: str = "" + key: str = "" + mode: str = "random" + picks: List[str] = field(default_factory=list) + count: int = 1 + keep: str = "meaning" + + @classmethod + def from_dict(cls, token_id: str, data: Dict[str, Any]) -> "Token": + mode = str(data.get("mode", "random")) + try: + count = int(data.get("count", 1)) + except (TypeError, ValueError): + count = 1 + return cls( + id=str(token_id), + slot=str(data.get("slot", "") or ""), + key=str(data.get("key", "") or "").strip().strip("_").replace("\\", "/"), + mode=mode if mode in MODES else "random", + picks=[str(pick) for pick in (data.get("picks") or []) if str(pick).strip()], + count=max(1, min(MAX_COUNT, count)), + keep=str(data.get("keep", "meaning") or "meaning"), + ) + + @property + def marker(self) -> str: + return KEEP_MARKERS.get(self.keep, "") + + def syntax(self) -> str: + """Wildcard-engine text for this pill. '' when it cannot produce anything.""" + marker = self.marker + many = self.count > 1 + count_prefix = f"{self.count}$${JOINER}$$" if many else "" + + if self.mode == "fixed" and self.picks: + return "{" + marker + "1$$" + escape_literal(self.picks[0]) + "}" + if self.mode == "subset" and self.picks: + options = "|".join(escape_literal(pick) for pick in self.picks) + if not many and len(self.picks) == 1: + return "{" + marker + "1$$" + options + "}" + return "{" + marker + count_prefix + options + "}" + if not self.key: + return "" + if self.mode == "exclude" and self.picks: + # Exclusions cannot contain | { } or a newline (grammar limit); those picks are skipped. + usable = [pick for pick in self.picks if not re.search(r"[|{}\n]", pick)] + body = "__" + self.key + ("!{" + "|".join(usable) + "}" if usable else "") + "__" + else: + body = "__" + self.key + "__" + if many: + return "{" + marker + count_prefix + body + "}" + return "__" + marker + body[2:] if marker else body + + +@dataclass +class ComposerState: + parts: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict) + tokens: Dict[str, Token] = field(default_factory=dict) + locks: Dict[str, int] = field(default_factory=dict) # section -> the seed it is held at + fill: Dict[str, bool] = field(default_factory=dict) # section -> an empty section is written by the LLM + order: List[str] = field(default_factory=list) # the node's own section order ([] = the platform's) + kinds: Dict[str, str] = field(default_factory=dict) # subject section -> person | animal | ... | scene + errors: List[str] = field(default_factory=list) + + @classmethod + def from_json(cls, raw: str) -> "ComposerState": + state = cls() + if not (raw or "").strip(): + return state + try: + data = json.loads(raw) + except ValueError as error: + # Not JSON: treat the whole thing as a plain subject template (hand-written or pasted). + state.parts["subject"] = [{"t": "raw", "v": raw}] + state.errors.append(f"composer is not JSON ({error}); used as plain text") + return state + if not isinstance(data, dict): + state.errors.append("composer JSON is not an object; ignored") + return state + for token_id, token in (data.get("tokens") or {}).items(): + if isinstance(token, dict): + state.tokens[str(token_id)] = Token.from_dict(str(token_id), token) + for section, segments in (data.get("parts") or {}).items(): + if isinstance(segments, list): + state.parts[str(section)] = [segment for segment in segments if isinstance(segment, dict)] + for section, seed in (data.get("locks") or {}).items(): + try: + state.locks[str(section)] = int(seed) + except (TypeError, ValueError): + state.errors.append(f"{section}: lock seed {seed!r} is not a number; ignored") + state.fill = {str(section): True for section, on in (data.get("fill") or {}).items() if on} + order = data.get("order") or [] + if isinstance(order, list): + state.order = [str(section) for section in order if str(section) in SECTIONS] + kinds = data.get("kinds") or {} + if isinstance(kinds, dict): + state.kinds = {str(section): str(kind) for section, kind in kinds.items() if kind} + return state + + def scene_only(self, section: str) -> bool: + """A subject section set to 'Scene only': deliberately no subject.""" + return base_section(section) == "subject" and self.kinds.get(section) == "scene" + + def templates(self) -> Dict[str, str]: + """section -> wildcard template text.""" + out: Dict[str, str] = {} + for section, segments in self.parts.items(): + pieces: List[str] = [] + for segment in segments: + kind = segment.get("t") + if kind in ("text", "raw"): + # Typed text keeps wildcard syntax working: {red|blue} or __name__ typed by hand still resolve. + pieces.append(str(segment.get("v", ""))) + elif kind == "tok": + token = self.tokens.get(str(segment.get("id"))) + if token is None: + self.errors.append(f"{section}: pill {segment.get('id')} has no settings; skipped") + continue + syntax = token.syntax() + if not syntax: + self.errors.append(f"{section}: {token.slot or 'pill'} has no list chosen; skipped") + continue + pieces.append(syntax) + text = join_pieces(pieces) + if text: + out[section] = text + return out + + +# ----------------------------------------------------------------------------- platform order +def base_section(section: str) -> str: + """'subject_2' -> 'subject'.""" + return re.sub(r"_\d+$", "", section or "") + + +def section_label(section: str) -> str: + match = re.match(r"(.+)_(\d+)$", section or "") + if match: + return f"{SECTION_LABELS.get(match.group(1), match.group(1).title())} {match.group(2)}" + return SECTION_LABELS.get(section, (section or "").title()) + + +def section_seed_offset(section: str) -> int: + """Each section rolls from seed + its offset, so re-rolling or locking one leaves the others alone.""" + base = base_section(section) + index = SECTIONS.index(base) if base in SECTIONS else len(SECTIONS) + (sum(map(ord, base)) % 97) + extra = re.search(r"_(\d+)$", section or "") + return index + (100 * (int(extra.group(1)) - 1) if extra else 0) + + +def section_order(structure: str) -> List[str]: + """ + Canonical sections in the order a profile's `structure` line mentions them. Sections the + line does not mention follow in DEFAULT_ORDER. "" -> DEFAULT_ORDER. + """ + lowered = (structure or "").lower() + positions: List[Tuple[int, int, str]] = [] + for default_index, section in enumerate(DEFAULT_ORDER): + match = re.search(r"\b(?:" + _SECTION_WORDS[section] + r")\b", lowered) + position = match.start() if match else 10_000 + default_index + positions.append((position, default_index, section)) + return [section for _pos, _index, section in sorted(positions)] + + +def order_sections(sections, order: Sequence[str]) -> List[str]: + """ + The given section names in platform order. Subject 2, 3 ... follow their base section + (Krea 2 keeps each subject with its own attributes); unknown sections go last. + """ + names = list(sections) + + def key(name: str): + base = base_section(name) + position = list(order).index(base) if base in order else len(order) + number = int(name.rsplit("_", 1)[1]) if name != base else 1 + return (position, number, name) + + return sorted(names, key=key) + + +def join_sections(sections: Dict[str, str], order: Sequence[str]) -> str: + """Section texts in platform order (unknown sections last), comma-joined.""" + ordered = [section for section in order_sections(sections, order) if sections[section].strip()] + return join_pieces([sections[section].strip().rstrip(",") + "," for section in ordered]).rstrip(",").strip() diff --git a/core/builder/layout.py b/core/builder/layout.py new file mode 100644 index 0000000..17284e3 --- /dev/null +++ b/core/builder/layout.py @@ -0,0 +1,140 @@ +""" +Your arrangement of the slot buttons: builders_user/button_layout.yaml, written by the panel's +"Arrange" mode and applied to every builder after the shipped YAML and my_buttons.yaml. + + moved: # a button shown in another section and/or under another group heading + shot: {section: composition, group: ""} + order: # button order inside a section (ids not listed keep their place, after these) + setting: [location, background, landmark] + hidden: [nose, lips] + +The shipped builders/*.yaml is never changed; "Reset layout" deletes this file. + +No ComfyUI dependency. + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import os +from typing import Dict, List, Optional + +try: + import yaml +except ImportError: # pragma: no cover - PyYAML ships with ComfyUI + yaml = None + +LAYOUT_FILE = "button_layout.yaml" + + +def empty_layout() -> Dict: + return {"moved": {}, "order": {}, "hidden": []} + + +def read_layout(path: str) -> Dict: + """The layout file, cleaned. Missing or broken -> an empty layout.""" + layout = empty_layout() + if yaml is None or not path or not os.path.isfile(path): + return layout + try: + with open(path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + except Exception as error: + print(f"[Wildcard Composer] skipped {path}: {error}") + return layout + if not isinstance(data, dict): + return layout + for slot_id, move in (data.get("moved") or {}).items(): + if isinstance(move, dict): + entry = {} + if move.get("section"): + entry["section"] = str(move["section"]).strip() + if "group" in move: + entry["group"] = str(move.get("group") or "").strip() + if entry: + layout["moved"][str(slot_id)] = entry + for section, ids in (data.get("order") or {}).items(): + if isinstance(ids, list): + layout["order"][str(section)] = [str(slot_id) for slot_id in ids if slot_id] + layout["hidden"] = sorted({str(slot_id) for slot_id in (data.get("hidden") or []) if slot_id}) + return layout + + +def write_layout(path: str, layout: Dict) -> None: + if yaml is None: + raise RuntimeError("PyYAML is not installed") + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + header = ("# Your arrangement of the Wildcard Composer buttons (the panel's Arrange mode writes it).\n" + "# Delete this file, or use Reset layout, to get the shipped layout back.\n") + clean = {"moved": layout.get("moved") or {}, "order": layout.get("order") or {}, + "hidden": sorted(set(layout.get("hidden") or []))} + temp = path + ".tmp" + with open(temp, "w", encoding="utf-8") as handle: + handle.write(header + yaml.safe_dump(clean, sort_keys=True, allow_unicode=True, width=120)) + os.replace(temp, path) + + +def apply_layout(builders: Dict, layout: Dict) -> None: + """Move, regroup, hide and reorder the slots of every builder (in place).""" + moved = layout.get("moved") or {} + hidden = set(layout.get("hidden") or []) + order = layout.get("order") or {} + for builder in builders.values(): + for slot in builder.slots: + move = moved.get(slot.id) + if move: + slot.section = move.get("section", slot.section) + if "group" in move: + slot.group = move["group"] + slot.hidden = slot.id in hidden + positions = {section: {slot_id: index for index, slot_id in enumerate(ids)} for section, ids in order.items()} + original = {slot.id: index for index, slot in enumerate(builder.slots)} + + def key(slot): + listed = positions.get(slot.section, {}) + if slot.id in listed: + return (0, listed[slot.id], 0) + return (1, 0, original[slot.id]) + + builder.slots.sort(key=key) + + +def layout_move(layout: Dict, slot_id: str, section: Optional[str] = None, group: Optional[str] = None, + before: Optional[str] = None, section_ids: Optional[List[str]] = None) -> Dict: + """ + Record a move: to `section` (and `group`), placed before the button `before` (None = at the end). + section_ids: the ids currently shown in the target section, in order (from the panel), so the + new order can be written in full. + """ + layout = {"moved": dict(layout.get("moved") or {}), "order": dict(layout.get("order") or {}), + "hidden": list(layout.get("hidden") or [])} + entry = dict(layout["moved"].get(slot_id, {})) + if section: + entry["section"] = section + if group is not None: + entry["group"] = group + if entry: + layout["moved"][slot_id] = entry + if section_ids is not None and (section or entry.get("section")): + target = section or entry["section"] + ids = [i for i in section_ids if i != slot_id] + at = ids.index(before) if before in ids else len(ids) + ids.insert(at, slot_id) + layout["order"][target] = ids + for other, other_ids in list(layout["order"].items()): + if other != target and slot_id in other_ids: + layout["order"][other] = [i for i in other_ids if i != slot_id] + return layout + + +def layout_hide(layout: Dict, slot_id: str, hidden: bool) -> Dict: + layout = {"moved": dict(layout.get("moved") or {}), "order": dict(layout.get("order") or {}), + "hidden": list(layout.get("hidden") or [])} + ids = set(layout["hidden"]) + if hidden: + ids.add(slot_id) + else: + ids.discard(slot_id) + layout["hidden"] = sorted(ids) + return layout diff --git a/core/builder/merge.py b/core/builder/merge.py new file mode 100644 index 0000000..4dca4fd --- /dev/null +++ b/core/builder/merge.py @@ -0,0 +1,175 @@ +""" +Merging the lists behind a slot button into one list of your own. + +Most downloaded packs share one core vocabulary ("black" hair is in 13 of 26 hair colour lists) +and add their own tail. merge_entries() folds near-identical entries across the chosen lists +("Black hair", "black", "black-haired" -> one row) and counts how many lists have each one, so +the shared core sorts first and the one-off tail can be pruned by eye. write_merged() saves the +kept entries as /mine//.txt; lists under mine// are pinned first on +that slot's button (see slots.SlotMatcher). + +No ComfyUI dependency. + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import os +import re +import time +from collections import Counter +from dataclasses import dataclass, field +from typing import Dict, Iterable, List, Optional, Sequence, Set + +MINE_FOLDER = "mine" + +_WEIGHTED = re.compile(r"\(([^():]+):\s*[\d.]+\)") # (auburn hair:1.2) -> auburn hair +_EDGE_PUNCT = re.compile(r"^[\s,.;:!]+|[\s,.;:!]+$") +_FOLD_PUNCT = re.compile(r"[^\w\s']+") +_SPACES = re.compile(r"\s+") +_WORD = re.compile(r"[a-z]+") +_NAME = re.compile(r"[^a-z0-9_\-]+") + + +def slot_words(*texts: str) -> Set[str]: + """Words that only restate the slot ("hair", "eyes", "colour") and are ignored when folding.""" + words: Set[str] = set() + for text in texts: + for word in _WORD.findall((text or "").lower()): + if len(word) < 3: + continue + stem = word[:-1] if word.endswith("s") and len(word) > 3 else word + words.update({word, stem, stem + "s", stem + "ed", stem + "d"}) + if stem in ("color", "colour"): + words.update({"color", "colour", "colors", "colours", "colored", "coloured"}) + return words + + +def clean_text(text: str) -> str: + """How an entry is shown and saved: weights, underscores and stray edge punctuation removed.""" + text = _WEIGHTED.sub(r"\1", text or "") + if "__" not in text: # keep wildcard references intact + text = re.sub(r"(?<=[A-Za-z])_(?=[A-Za-z])", " ", text) + return _SPACES.sub(" ", _EDGE_PUNCT.sub("", text)).strip() + + +def fold_key(text: str, strip_words: Set[str] = frozenset()) -> str: + """ + The identity of an entry for merging: lower case, punctuation and hyphens as spaces, slot words + at either end dropped ('black-haired' -> 'black' for Hair color), a final plural s dropped. + """ + if "__" in text or "{" in text: # wildcard syntax: only exact repeats fold + return text.strip().lower() + folded = _FOLD_PUNCT.sub(" ", clean_text(text).lower().replace("-", " ")) + words = _SPACES.sub(" ", folded).strip().split() + while len(words) > 1 and words[-1] in strip_words: + words.pop() + while len(words) > 1 and words[0] in strip_words: + words.pop(0) + if words and len(words[-1]) > 3 and words[-1].endswith("s") and not words[-1].endswith("ss"): + words[-1] = words[-1][:-1] + return " ".join(words) + + +@dataclass +class MergedRow: + text: str # the most common spelling (shortest, then fewest capitals) + fold: str + sources: List[int] = field(default_factory=list) # indexes into the source key list + variants: List[str] = field(default_factory=list) # other spellings folded into this row + + @property + def lists(self) -> int: + return len(self.sources) + + @property + def words(self) -> int: + return len(self.text.split()) + + def to_dict(self) -> Dict: + return {"text": self.text, "lists": self.lists, "sources": self.sources, "variants": self.variants[:6], + "words": self.words} + + +def merge_entries(lists: Sequence[Iterable[str]], strip_words: Set[str] = frozenset()) -> List[MergedRow]: + """ + One row per folded entry across the lists, most-shared first (then the order first seen). + lists: the entry texts of each source list, in source order. + """ + spellings: Dict[str, Counter] = {} + sources: Dict[str, Set[int]] = {} + order: Dict[str, int] = {} + for index, texts in enumerate(lists): + for raw in texts: + text = clean_text(raw) + if not text: + continue + key = fold_key(text, strip_words) + if not key: + continue + spellings.setdefault(key, Counter())[text] += 1 + sources.setdefault(key, set()).add(index) + order.setdefault(key, len(order)) + rows = [] + for key, counter in spellings.items(): + ranked = sorted(counter.items(), key=lambda item: (-item[1], len(item[0]), sum(c.isupper() for c in item[0]), item[0])) + rows.append(MergedRow(text=ranked[0][0], fold=key, sources=sorted(sources[key]), + variants=[text for text, _count in ranked[1:]])) + rows.sort(key=lambda row: (-row.lists, order[row.fold])) + return rows + + +def merged_name(name: str) -> str: + """'My Hair colours!' -> 'my_hair_colours' ('merged' when nothing is left).""" + return _NAME.sub("_", (name or "").strip().lower()).strip("_-")[:60] or "merged" + + +def merged_path(root: str, slot_id: str, name: str) -> str: + return os.path.join(root, MINE_FOLDER, merged_name(slot_id), merged_name(name) + ".txt") + + +def merged_key(slot_id: str, name: str) -> str: + return f"{MINE_FOLDER}/{merged_name(slot_id)}/{merged_name(name)}" + + +def write_merged(path: str, entries: Sequence[str], sources: Sequence[str], slot_label: str = "", + overwrite: bool = False) -> int: + """Write the kept entries (one per line) with a header naming the source lists. Returns the count.""" + if os.path.exists(path) and not overwrite: + raise FileExistsError(path) + kept: List[str] = [] + seen: Set[str] = set() + for entry in entries: + text = clean_text(str(entry)).replace("\n", " ") + if text and text.casefold() not in seen: + seen.add(text.casefold()) + kept.append(text) + if not kept: + raise ValueError("nothing to save: every entry is unticked") + header = [f"# Merged by the Wildcard Composer on {time.strftime('%Y-%m-%d')}" + + (f" for the {slot_label} button" if slot_label else "") + f": {len(kept)} entries.", + f"# From {len(sources)} lists (your own selection; check their licences before sharing):"] + header += [f"# {key}" for key in sources] + os.makedirs(os.path.dirname(path), exist_ok=True) + temp = path + ".tmp" + with open(temp, "w", encoding="utf-8") as handle: + handle.write("\n".join(header + kept) + "\n") + os.replace(temp, path) + return len(kept) + + +def pick_root(source_roots: Sequence[Optional[str]], search_paths: Sequence[str], avoid: str = "") -> Optional[str]: + """ + Where a merged list goes: the wildcard folder most of its sources come from, never the + package's own shipped folder (`avoid`) while another folder exists. + """ + def allowed(root: str) -> bool: + return not avoid or not os.path.normcase(os.path.abspath(root)).startswith(os.path.normcase(os.path.abspath(avoid))) + counted = Counter(root for root in source_roots if root and allowed(root)) + if counted: + return counted.most_common(1)[0][0] + for root in search_paths: + if allowed(root): + return root + return search_paths[0] if search_paths else None diff --git a/core/builder/more.py b/core/builder/more.py new file mode 100644 index 0000000..b9fd620 --- /dev/null +++ b/core/builder/more.py @@ -0,0 +1,178 @@ +""" +"More lists": the usable lists no slot button offers, sorted into prompt sections and grouped by +the words in their paths, so every list is reachable from its section without a button of its own. + +A list is usable when the catalog read it as words, phrases or tags (not whole prompts or +templates) with 3+ entries. Its section comes from the catalog's facet, or failing that from +words in its path; its group is the most common path word among the section's lists that it has. + +No ComfyUI dependency. + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import re +from collections import Counter +from dataclasses import dataclass, field +from typing import Callable, Dict, Iterable, List, Optional, Sequence + +USABLE_KINDS = ("atoms", "phrases", "tags") +MIN_GROUP = 3 # smaller groups go to "Other" +MAX_LISTS_PER_GROUP = 300 + +# The catalog's facet (catalog.facet_for) -> prompt section. +FACET_SECTION = { + "wardrobe": "subject", "appearance": "subject", "subject": "subject", "expression": "subject", + "creature": "subject", + "pose_action": "action", + "architecture": "setting", "setting": "setting", "nature": "setting", "era_culture": "setting", + "object_material": "setting", + "lighting": "lighting", + "style_artist": "style", "color_mood": "style", "concept": "style", + "camera": "camera", +} + +# For lists the catalog gave no facet: the first path word that names a section. +_WORD_SECTION = [ + ("subject", "hair face eye eyes body skin cloth clothing clothes outfit outfits dress wear attire garment makeup " + "character characters person people girl woman man costume fashion accessory accessories jewelry " + "species race creature creatures monster"), + ("action", "pose poses action actions gesture activity activities interaction"), + ("setting", "place places location locations room rooms environment environments landscape building buildings " + "city world nature forest weather season food plant plants flower vehicle vehicles era scene scenes " + "interior architecture background backgrounds object objects item items"), + ("lighting", "light lights lighting atmosphere atmo mood"), + ("style", "style styles art artist artists painter render rendering color colour colors texture " + "material materials theme themes aesthetic aesthetics palette palettes"), + ("format", "medium mediums format formats illustration illustrations poster posters genre genres"), + ("composition", "composition compositions shot shots perspective perspectives view views framing layout"), + ("camera", "camera cameras angle angles lens lenses aperture bokeh focus exposure photo photography film"), + ("text", "font fonts typography lettering typeface typefaces"), +] +WORD_SECTION: Dict[str, str] = {word: section for section, words in _WORD_SECTION for word in words.split()} + +_NOISE = set("""the and of for with a an wildcards wildcard wc random misc other others all general custom main base +list lists txt yaml sfw nsfw new old extra full simple basic prompt prompts tag tags type types common standard +regular default collection pack module part parts level detail details sdxl flux pony illustrious v1 v2 v3 v4 +female male women woman men man girl girls boy unisex fem adj adjective adjectives word words verb verbs noun nouns +themed any sfw nsfw cops cha dcw nodetail core""".split()) # cops_: a pack family's file prefix +UNSORTED = "_unsorted" # sections value asking for the lists with no clear section +_WORDS = re.compile(r"[a-z]+") +_CAMEL = re.compile(r"(?<=[a-z])(?=[A-Z])") + + +def path_tiers(key: str) -> List[List[str]]: + """ + Meaningful words of a key below its pack folder, by distance from the leaf: [leaf words, + parent folder words, words further up]. Plural s dropped; words that are part of the pack's + own name ('sdvn' in sdvnwildcards_v10/wc-sdvn/...) left out, so groups name content, not packs. + """ + parts = key.split("/") + pack = _CAMEL.sub(" ", parts[0]).lower() if len(parts) > 1 else "" + parts = parts[1:] if len(parts) > 1 else parts + seen: set = set() + tiers: List[List[str]] = [[], [], []] + for depth, part in enumerate(reversed(parts)): + tier = tiers[min(depth, 2)] + for word in _WORDS.findall(_CAMEL.sub(" ", part).lower()): + if len(word) < 3 or word in _NOISE or re.fullmatch(r"v\d+", word) or (pack and word in pack): + continue + if len(word) > 4 and word.endswith("s") and not word.endswith("ss"): + word = word[:-1] + if word not in seen: + seen.add(word) + tier.append(word) + return tiers + + +def path_words(key: str) -> List[str]: + """Meaningful words of a key below its pack folder, leaf words first (see path_tiers).""" + return [word for tier in path_tiers(key) for word in tier] + + +def _word_section(words: Iterable[str]) -> Optional[str]: + for word in words: + if word in WORD_SECTION: + return WORD_SECTION[word] + if word + "s" in WORD_SECTION: + return WORD_SECTION[word + "s"] + return None + + +def section_of(key: str, facet: str) -> Optional[str]: + """The list's own name or folder first ('.../places/...' is Setting), then the catalog's facet, then any path word.""" + tiers = path_tiers(key) + return (_word_section(tiers[0] + tiers[1]) or FACET_SECTION.get(facet) + or _word_section(tiers[2])) + + +@dataclass +class MoreGroup: + word: str + label: str + keys: List[str] = field(default_factory=list) + + +def group_lists(keys: Sequence[str]) -> List[MoreGroup]: + """ + Group keys by a shared path word: the most common of the leaf's own words that 3+ lists share, + else of the parent folder's, else of any; lists with none go to 'Other'. + """ + tiers_of = {key: path_tiers(key) for key in keys} + counts = Counter(word for tiers in tiers_of.values() for tier in tiers for word in tier) + groups: Dict[str, MoreGroup] = {} + for key in keys: + best = "" + for tier in tiers_of[key]: + shared = [word for word in tier if counts[word] >= MIN_GROUP] + if shared: + best = max(shared, key=lambda w: (counts[w], -tier.index(w))) + break + group = groups.get(best) + if group is None: + group = groups[best] = MoreGroup(word=best, label=best.replace("_", " ").title() if best else "Other") + group.keys.append(key) + ordered = sorted((g for g in groups.values() if g.word), key=lambda g: (-len(g.keys), g.word)) + if "" in groups: + ordered.append(groups[""]) + return ordered + + +def more_lists(unassigned: Iterable[str], sections: Sequence[str], row_of: Callable[[str], object]) -> Dict: + """ + The unassigned keys for these sections, grouped. row_of(key) -> the catalog row (with kind, + entries, facet, signature, quality) or None. Lists with identical content keep their best copy. + """ + wanted = set(sections) + want_unsorted = UNSORTED in wanted + chosen: Dict[str, object] = {} + by_signature: Dict[str, str] = {} + skipped = unsorted = 0 + for key in unassigned: + row = row_of(key) + if row is None or getattr(row, "kind", "") not in USABLE_KINDS or getattr(row, "entries", 0) < 3: + skipped += 1 + continue + section = section_of(key, getattr(row, "facet", "") or "") + if section is None: + unsorted += 1 + if not want_unsorted: + continue + elif section not in wanted: + continue + signature = getattr(row, "signature", "") or key + twin = by_signature.get(signature) + if twin is not None: + if getattr(row, "quality", 0) > getattr(chosen[twin], "quality", 0): + del chosen[twin] + chosen[key] = row + by_signature[signature] = key + continue + by_signature[signature] = key + chosen[key] = row + groups = group_lists(sorted(chosen)) + for group in groups: + group.keys.sort(key=lambda k: (-getattr(chosen[k], "quality", 0), k)) + return {"groups": groups, "rows": chosen, "total": len(chosen), "skipped": skipped, "unsorted": unsorted} diff --git a/core/builder/sections.py b/core/builder/sections.py new file mode 100644 index 0000000..91eb2ad --- /dev/null +++ b/core/builder/sections.py @@ -0,0 +1,59 @@ +""" +PROMPT_SECTIONS: what builder nodes pass to each other. + +A Wildcard Character Composer fills `subject`, a Scene Builder `setting` and `lighting`, and so on. Each +builder takes the bundle from the one before it, adds its own sections and passes it on, so the +last builder's `text` output holds the whole prompt in the platform's order. + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from .compose import join_pieces + + +@dataclass +class PromptSections: + sections: Dict[str, str] = field(default_factory=dict) # section -> resolved text + templates: Dict[str, str] = field(default_factory=dict) # section -> wildcard template + llm_instructions: List[str] = field(default_factory=list) # one block per builder that resolved wildcards + sources: List[str] = field(default_factory=list) # builder labels, upstream first + nsfw: Optional[int] = None # the highest nsfw level (0..3) of the builders that made it; None = not said + + def merged_with(self, section_texts: Dict[str, str], templates: Dict[str, str], + llm_instructions: str, source: str, nsfw: Optional[int] = None) -> "PromptSections": + """ + A new bundle: this one, then the given sections (a section both fill is joined, upstream first). + nsfw: the adding builder's level; the bundle keeps the highest (what content its sections may hold). + """ + levels = [level for level in (self.nsfw, nsfw) if level is not None] + merged = PromptSections(dict(self.sections), dict(self.templates), + list(self.llm_instructions), list(self.sources), max(levels) if levels else None) + for section, text in section_texts.items(): + if not text.strip(): + continue + earlier = merged.sections.get(section, "").strip() + merged.sections[section] = join_pieces([earlier.rstrip(",") + ",", text]) if earlier else text.strip() + for section, template in templates.items(): + earlier = merged.templates.get(section, "").strip() + merged.templates[section] = f"{earlier}, {template}" if earlier else template + if llm_instructions.strip(): + merged.llm_instructions.append(llm_instructions.strip()) + merged.sources.append(source) + return merged + + def instructions_text(self) -> str: + return "\n\n".join(self.llm_instructions) + + @staticmethod + def coerce(value) -> "PromptSections": + """Accept a bundle, a plain dict of section -> text, or nothing.""" + if isinstance(value, PromptSections): + return value + if isinstance(value, dict): + return PromptSections(sections={str(k): str(v) for k, v in value.items() if v}) + return PromptSections() diff --git a/core/builder/slots.py b/core/builder/slots.py new file mode 100644 index 0000000..6808392 --- /dev/null +++ b/core/builder/slots.py @@ -0,0 +1,726 @@ +""" +Builder definitions: which slot buttons a builder node shows, and which wildcard lists back each slot. + +A builder is one YAML file (builders/prompt.yaml and builders/character.yaml ship with the +package; files in builders_user/ with the same id replace them): + + id: prompt + label: Wildcard Prompt Composer + parts: # one composer line per prompt section + - section: subject + label: Subject + multi: true # "+ Subject" adds subject_2, subject_3 ... + hint: who or what the picture is about + starter: "a {age} {occupation} with {hair_color} hair" + slots: [subject, action] # optional: which sections' buttons this part shows + slots: + - section: subject # buttons shown for that section + slots: + - id: hair_color + label: Hair color + match: ['hair_?colou?rs?'] # regexes, full match on a list's leaf name + avoid: ['fantasy'] # regexes, searched in the whole key + lists: ['mypack/best'] # optional: pinned lists, shown first + folders: ['names?'] # optional: regexes searched in the list's folder path; + # every list in a matching folder counts (after `match`) + search: 'famous buildings' # optional: words that must all appear in the key + # (see key_has_terms); what "My buttons" categories use + + id: character + extends: prompt # takes prompt's slots for the sections its parts show + parts: [{section: subject, slots: [subject, action], starter: "..."}] + +In `starter`, {slot_id} places a wildcard pill for that slot; everything else is text. + +Your own buttons live in builders_user/my_buttons.yaml (made by the panel's "+ Button"; shared by +every builder, shown under "My buttons" in their section): + + buttons: + - id: my_black_names + label: Black names + section: subject + lists: [blackamericanfemalefirstnames_v10/black_female_names] + - id: my_famous_buildings + label: Famous buildings + section: setting + search: famous buildings + +Until the wildcard catalog (dev/WILDCARD_CATALOG_SPEC.md, P2) ranks lists by quality, a slot's +lists are found by name: every visible list whose leaf name (the last path segment, lower case, +with pack prefixes such as cops_wc_ or cyber_ removed) fully matches one of the slot's `match` +patterns. Your stars rank first, then earlier patterns, then the catalog's quality score (short, +clean prose entries read best between typed words), then bigger lists. Hidden lists are left out. Lists with identical content (the same +file copied into several packs) are shown once. + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import hashlib +import os +import re +from dataclasses import dataclass, field +from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple + +from ..wildcards.library import NSFW_OFF, nsfw_level, violence_name_tier + +try: + import yaml +except ImportError: # pragma: no cover - PyYAML ships with ComfyUI + yaml = None + +# Pack-specific prefixes seen in real libraries; removed before matching a leaf name. +_LEAF_PREFIX = re.compile(r"^(?:cops_(?:bp_)?(?:male_|female_)?wc_?|cyber_|bk_|rd_|fg3_|vc_|wc_|tr_)") +_SEPARATORS = re.compile(r"[\s\-.]+") + +# Lists never offered by a slot, whatever their name (lists rated above the run's nsfw level are already hidden). +_ALWAYS_AVOID = re.compile(r"lora") +# Also left out of runs at nsfw off by name (lists rated by folder / name / content are already invisible to them). +_SFW_AVOID = re.compile(r"e621|nsfw|panties|no_bottom|nud(e|ity)|topless|lingerie") + +GENDERS = ("female", "male") +_TOKENS = re.compile(r"[a-z]+") +_FEMALE_WORDS = {"female", "females", "women", "womens", "woman", "girl", "girls", "fem", "feminine", "lady", + "ladies"} +_MALE_WORDS = {"male", "males", "men", "mens", "man", "boy", "boys", "guy", "guys", "masculine", "gentleman", + "gentlemen"} + + +def gender_of(key: str) -> str: + """'female', 'male' or '' (neutral or both) from the words in a list's key.""" + spaced = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", key).lower() + tokens = _TOKENS.findall(spaced) + female = any(t in _FEMALE_WORDS or t.startswith(("female", "women", "woman", "girl")) for t in tokens) + male = any(t in _MALE_WORDS or t.startswith(("male", "guy")) or (t.startswith("men") and len(t) <= 4) + for t in tokens) + if female and not male: + return "female" + if male and not female: + return "male" + return "" + +# Keys the wildcard grammar can reference as __key__ (see core/wildcards/parser.py). +_REFERENCEABLE = re.compile(r"[a-z0-9][a-z0-9_\-/. ]*") + +MIN_ENTRIES = 3 +MAX_LISTS_PER_SLOT = 250 +FIND_MAX_CANDIDATES = 400 +MY_BUTTONS_FILE = "my_buttons.yaml" +MY_BUTTONS_GROUP = "My buttons" +_KEY_TOKENS = re.compile(r"[a-z0-9]+") + + +def search_terms(text: str) -> List[str]: + return [term for term in _KEY_TOKENS.findall((text or "").lower()) if term] + + +def key_has_terms(key: str, terms: Sequence[str]) -> bool: + """ + True when every term is in the key (folders included). A term of 4+ letters may sit inside a + word ('names' finds peoplenamemegalist ... via 'name'), a shorter one must start a word; a + trailing plural s is optional. Gender words ('female', 'men') match lists named for that gender, + so 'male' does not find female lists. + """ + if not terms: + return False + spaced = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", key).lower() + tokens = _KEY_TOKENS.findall(spaced) + gender = None + for term in terms: + if term in _FEMALE_WORDS or term in _MALE_WORDS: + wanted = "female" if term in _FEMALE_WORDS else "male" + gender = gender_of(key) if gender is None else gender + if gender != wanted: + return False + continue + stems = {term, term[:-1]} if len(term) > 3 and term.endswith("s") else {term} + if not any((token.startswith(stem) or (len(stem) >= 4 and stem in token)) for token in tokens for stem in stems): + return False + return True + + +@dataclass +class SlotDef: + id: str + label: str + match: List[str] = field(default_factory=list) + avoid: List[str] = field(default_factory=list) + lists: List[str] = field(default_factory=list) + section: str = "subject" # which prompt section's palette shows this slot + tooltip: str = "" + group: str = "" # palette heading (Person, Clothing, ...) + folders: List[str] = field(default_factory=list) # regexes searched in the folder part of a key + search: str = "" # words that must all be in the key (key_has_terms) + custom: bool = False # from my_buttons.yaml + kinds: List[str] = field(default_factory=list) # subject types it serves (person, animal ...); [] = see kinds_of + hidden: bool = False # hidden by your button layout (shown in Arrange mode) + + def __post_init__(self) -> None: + self._match = [re.compile(pattern, re.IGNORECASE) for pattern in self.match] + self._avoid = [re.compile(pattern, re.IGNORECASE) for pattern in self.avoid] + self._folders = [re.compile(pattern, re.IGNORECASE) for pattern in self.folders] + self._terms = search_terms(self.search) + + def _avoided(self, key: str, nsfw) -> bool: + return bool(_ALWAYS_AVOID.search(key) or any(pattern.search(key) for pattern in self._avoid) + or (nsfw_level(nsfw) == NSFW_OFF and _SFW_AVOID.search(key))) + + def leaf_rank(self, key: str, nsfw="explicit") -> Optional[int]: + """Index of the first `match` pattern the key's own name fully matches, or None.""" + if not self._match or self._avoided(key, nsfw): + return None + leaf = leaf_name(key) + for index, pattern in enumerate(self._match): + if pattern.fullmatch(leaf): + return index + return None + + def match_rank(self, key: str, nsfw="off", folder_ok: bool = False) -> Optional[int]: + """ + Rank of the key for this slot, or None: its own name against `match` (0, 1, ...), then + `search` words, then `folders`, then (folder_ok: the name itself says nothing any slot knows, + like 'basemodel/eye-color/brown') its folder names against `match`. + """ + if self._avoided(key, nsfw): + return None + leaf = leaf_name(key) + for index, pattern in enumerate(self._match): + if pattern.fullmatch(leaf): + return index + if self._terms and key_has_terms(key, self._terms): + return len(self._match) + folder = key.rsplit("/", 1)[0] if "/" in key else "" + if folder and any(pattern.search(folder) for pattern in self._folders): + return len(self._match) + 1 + if folder_ok and self._match: + for parent in parent_names(key): + if any(pattern.fullmatch(parent) for pattern in self._match): + return len(self._match) + 2 + return None + + +@dataclass +class PartDef: + section: str + label: str + starter: str = "" + slots: List[str] = field(default_factory=list) # sections whose slot buttons this part shows ([] = its own) + multi: bool = False # "+ Subject" may add Subject 2, 3, ... + hint: str = "" # what goes here; also tells the LLM what to write when asked + optional: bool = False # shown only when added with "+ Section" (Text) + + @property + def slot_sections(self) -> List[str]: + return self.slots or [self.section] + + +@dataclass +class BuilderDef: + id: str + label: str + parts: List[PartDef] + slots: List[SlotDef] + description: str = "" + origin: str = "" + + extends: str = "" + + def slot(self, slot_id: str) -> Optional[SlotDef]: + for slot in self.slots: + if slot.id == slot_id: + return slot + return None + + def part(self, section: str) -> Optional[PartDef]: + """The part for a section; Subject 2, 3 ... ('subject_2') use the subject part.""" + base = base_section(section) + for part in self.parts: + if part.section == base: + return part + return None + + +def base_section(section: str) -> str: + """'subject_2' -> 'subject'. Extra subjects share the subject section's slots and place in the order.""" + return re.sub(r"_\d+$", "", section or "") + + +@dataclass +class ListInfo: + key: str + count: int + rank: int + pinned: bool = False + also: List[str] = field(default_factory=list) # keys with identical content, folded into this one + words: int = 0 # median words per entry: 1-4 short, 5-8 medium, more = long + quality: float = -1.0 # catalog quality 0..1 (-1 = not indexed yet) + stars: int = 0 # the user's stars (0 = none) + dialect: str = "" # prose | tags | mixed (from the catalog) + display: str = "" + nsfw: int = 0 # the list's rating 0..3 (library.NSFW_LEVELS) + gender: str = "" # female | male | '' (from the key's words) + + @property + def order_score(self) -> float: + """What ranks lists inside one match pattern: the catalog's quality (tag lists read worse + between typed words), or, before the catalog is built, entry length alone.""" + if self.quality >= 0: + return self.quality - (0.15 if self.dialect == "tags" else 0.05 if self.dialect == "mixed" else 0.0) + return {"short": 0.8, "medium": 0.6, "long": 0.3}[self.length_class] + + @property + def length_class(self) -> str: + return "short" if self.words <= 4 else "medium" if self.words <= 8 else "long" + + +# Gender and rating words around a list name ("female_hair_colors", "poses_nsfw") don't change what +# the list is: they are stripped before matching (gender is judged separately, see gender_of). +_QUALIFIER_PREFIX = re.compile(r"^(?:(?:fe)?males?|(?:wo)?mens?|girls?|boys?|fem|n?sfw)_") +_QUALIFIER_SUFFIX = re.compile(r"_(?:(?:fe)?males?|(?:wo)?mens?|girls?|boys?|fem|n?sfw)$") + + +_NUMBER_PREFIX = re.compile(r"^\d+_+(?=[a-z])") # "10-lighting" (The Loom's numbered folders) + + +def leaf_name(key: str) -> str: + leaf = key.rsplit("/", 1)[-1].lower() + leaf = _SEPARATORS.sub("_", leaf).strip("_") + leaf = _NUMBER_PREFIX.sub("", leaf) + leaf = _LEAF_PREFIX.sub("", leaf) + for _ in range(2): + stripped = _QUALIFIER_SUFFIX.sub("", _QUALIFIER_PREFIX.sub("", leaf)) + if not stripped or stripped == leaf: + break + leaf = stripped + return leaf + + +# Folder names that say nothing about the content ("female/", "general/", "v2/"): looked past. +_GENERIC_FOLDER = re.compile(r"(?:(?:fe)?males?|(?:wo)?mens?|girls?|boys?|unisex|n?sfw|general|base|basic|main|misc|" + r"others?|all|common|default|standard|regular|simple|extra|custom|v\d+|types?|\d+|\d*wildcards?(_\w+)?|" + r"collections?|packs?|lists?|misc_\w+)") + + +def parent_names(key: str, limit: int = 1) -> List[str]: + """ + The nearest meaningful folder name(s) of a key, normalised like leaf names, nearest first: + 'basemodel/eye-color/brown' -> ['eye_color']; 'pack/10-lighting/female/soft' -> ['lighting']. + The top folder (the pack) is never used. One level by default: further up, folders tend to + name a whole scene ('locations/.../kneeling/fully_clothed'), not the list. + """ + parts = key.split("/")[1:-1] + found: List[str] = [] + for part in reversed(parts): + name = leaf_name(part) + if not name or _GENERIC_FOLDER.fullmatch(name): + continue + found.append(name) + if len(found) >= limit: + break + return found + + +# ----------------------------------------------------------------------------- loading +def _as_list(value) -> List[str]: + if value is None: + return [] + if isinstance(value, str): + return [value] + return [str(item) for item in value if item is not None] + + +def _slot_from_dict(raw: Dict, default_section: str = "subject") -> Optional[SlotDef]: + if not isinstance(raw, dict) or not raw.get("id"): + return None + return SlotDef( + id=str(raw["id"]).strip(), + label=str(raw.get("label") or raw["id"]).strip(), + match=_as_list(raw.get("match")), + avoid=_as_list(raw.get("avoid")), + lists=[item.strip().replace("\\", "/").lower() for item in _as_list(raw.get("lists"))], + section=str(raw.get("section", "") or default_section).strip(), + tooltip=str(raw.get("tooltip", "") or "").strip(), + group=str(raw.get("group", "") or "").strip(), + folders=_as_list(raw.get("folders")), + search=str(raw.get("search", "") or "").strip(), + kinds=[kind.strip().lower() for kind in _as_list(raw.get("kinds")) if kind.strip()], + ) + + +def builder_from_dict(data: Dict, origin: str = "") -> BuilderDef: + builder_id = str(data.get("id") or os.path.splitext(os.path.basename(origin))[0]).strip() + parts = [] + for part in (data.get("parts") or [{"section": "subject", "label": "Subject"}]): + if not isinstance(part, dict): + continue + section = str(part.get("section", "subject")).strip() or "subject" + parts.append(PartDef( + section=section, + label=str(part.get("label") or section.title()).strip(), + starter=str(part.get("starter", "") or ""), + slots=_as_list(part.get("slots")), + multi=bool(part.get("multi", False)), + hint=str(part.get("hint", "") or "").strip(), + optional=bool(part.get("optional", False)), + )) + slots: List[SlotDef] = [] + for group in data.get("slots") or []: + # Either a flat list of slots, or {section: x, slots: [...]} groups. + if isinstance(group, dict) and isinstance(group.get("slots"), list) and not group.get("id"): + for raw in group["slots"]: + slot = _slot_from_dict(raw, str(group.get("section", "subject"))) + if slot: + slots.append(slot) + else: + slot = _slot_from_dict(group) + if slot: + slots.append(slot) + return BuilderDef(id=builder_id, label=str(data.get("label") or builder_id), parts=parts, slots=slots, + description=str(data.get("description", "") or "").strip(), origin=origin, + extends=str(data.get("extends", "") or "").strip()) + + +def _resolve_extends(builders: Dict[str, BuilderDef]) -> None: + """A builder with `extends: other` gets the other's slots for the sections its parts show (its own slots win).""" + for builder in builders.values(): + base = builders.get(builder.extends) if builder.extends else None + if base is None or base is builder: + continue + wanted = {section for part in builder.parts for section in part.slot_sections} + own = {slot.id for slot in builder.slots} + inherited = [slot for slot in base.slots if slot.section in wanted and slot.id not in own] + builder.slots = inherited + builder.slots + for part in builder.parts: + base_part = base.part(part.section) + if base_part and not part.hint: + part.hint = base_part.hint + + +# ----------------------------------------------------------------------------- my buttons +_BUTTON_ID = re.compile(r"[^a-z0-9_]+") + + +def button_id(label: str, taken: Iterable[str] = ()) -> str: + """'Famous buildings' -> 'my_famous_buildings' (with _2, _3 ... when taken).""" + base = "my_" + (_BUTTON_ID.sub("_", (label or "").lower()).strip("_") or "button") + taken = set(taken) + candidate, n = base, 2 + while candidate in taken: + candidate, n = f"{base}_{n}", n + 1 + return candidate + + +def clean_button(raw: Dict) -> Optional[Dict]: + """A my_buttons.yaml entry with only the fields it may have, or None when it has no label or lists/search.""" + if not isinstance(raw, dict): + return None + label = str(raw.get("label", "") or "").strip() + lists = [item.strip().replace("\\", "/").lower() for item in _as_list(raw.get("lists")) if item.strip()] + search = " ".join(search_terms(str(raw.get("search", "") or ""))) + if not label or not (lists or search): + return None + button = {"id": str(raw.get("id", "") or "").strip(), "label": label, + "section": str(raw.get("section", "") or "subject").strip() or "subject"} + if lists: + button["lists"] = lists + if search: + button["search"] = search + tooltip = str(raw.get("tooltip", "") or "").strip() + if tooltip: + button["tooltip"] = tooltip + return button + + +def read_buttons(path: str) -> List[Dict]: + """The entries of my_buttons.yaml (cleaned, ids filled in). A missing or broken file gives [].""" + if yaml is None or not path or not os.path.isfile(path): + return [] + try: + with open(path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + except Exception as error: + print(f"[Wildcard Composer] skipped {path}: {error}") + return [] + raw_buttons = data.get("buttons") if isinstance(data, dict) else data + buttons: List[Dict] = [] + for raw in raw_buttons or []: + button = clean_button(raw) + if button is None: + continue + if not button["id"] or button["id"] in {b["id"] for b in buttons}: + button["id"] = button_id(button["label"], (b["id"] for b in buttons)) + buttons.append(button) + return buttons + + +def write_buttons(path: str, buttons: List[Dict]) -> None: + """Save my_buttons.yaml atomically (a temp file, then replace).""" + if yaml is None: + raise RuntimeError("PyYAML is not installed") + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + header = ("# Your Wildcard Composer buttons. Made by the panel's \"+ Button\"; safe to edit by hand.\n" + "# lists: exact list keys (shown first). search: words that must all be in a list's key.\n") + text = header + yaml.safe_dump({"buttons": buttons}, sort_keys=False, allow_unicode=True, width=120) + temp = path + ".tmp" + with open(temp, "w", encoding="utf-8") as handle: + handle.write(text) + os.replace(temp, path) + + +def button_slot(button: Dict) -> SlotDef: + return SlotDef(id=button["id"], label=button["label"], lists=list(button.get("lists", [])), + search=button.get("search", ""), section=button.get("section", "subject"), + tooltip=button.get("tooltip", ""), group=MY_BUTTONS_GROUP, custom=True) + + +def apply_buttons(builders: Dict[str, BuilderDef], buttons: List[Dict]) -> None: + """Add your buttons to every builder whose parts show their section (after its own slots).""" + for builder in builders.values(): + shown = {section for part in builder.parts for section in part.slot_sections} + own = {slot.id for slot in builder.slots} + builder.slots = [slot for slot in builder.slots if not slot.custom] + [ + button_slot(button) for button in buttons if button["section"] in shown and button["id"] not in own] + + +# Subject types (the Subject lane's type menu) and which subject buttons each shows. +SUBJECT_KINDS = ("person", "animal", "creature", "object", "vehicle", "food", "scene") +PERSON_GROUPS = ("Person", "Face & hair", "Clothing", "Accessories", "Makeup") + + +def kinds_of(slot: SlotDef) -> List[str]: + """The subject types a subject button serves: its `kinds`, else person for the person groups, else all.""" + if slot.kinds: + return list(slot.kinds) + if slot.section == "subject" and slot.group in PERSON_GROUPS: + return ["person"] + return [] + + +def load_builders(folders: Sequence[str], buttons_path: str = "", layout_path: str = "") -> Dict[str, BuilderDef]: + """Read every *.yaml in the folders; a later folder's builder replaces an earlier one with the same id. + buttons_path (my_buttons.yaml) adds your own buttons to every builder.""" + builders: Dict[str, BuilderDef] = {} + if yaml is None: + return builders + for folder in folders: + if not folder or not os.path.isdir(folder): + continue + for filename in sorted(os.listdir(folder)): + if not filename.lower().endswith((".yaml", ".yml")) or filename.startswith(("_", ".")): + continue + if filename.lower() in (MY_BUTTONS_FILE, "button_layout.yaml"): + continue + path = os.path.join(folder, filename) + try: + with open(path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + builder = builder_from_dict(data, origin=path) + except Exception as error: # a broken user file must not hide the others + print(f"[Wildcard Composer] skipped {path}: {error}") + continue + builders[builder.id] = builder + layout = None + if layout_path: + from .layout import apply_layout, read_layout + layout = read_layout(layout_path) + apply_layout(builders, layout) # before extends, so a button moved into Subject reaches the Character Composer + _resolve_extends(builders) + if buttons_path: + apply_buttons(builders, read_buttons(buttons_path)) + if layout is not None: + apply_layout(builders, layout) # again, for your own buttons and the inherited order + return builders + + +# ----------------------------------------------------------------------------- matching lists +def _signature(texts: Iterable[str]) -> str: + digest = hashlib.sha1() + for text in sorted({text.strip().casefold() for text in texts}): + digest.update(text.encode("utf-8", "replace")) + digest.update(b"\n") + return digest.hexdigest() + + +def slot_has_lists(matcher: "SlotMatcher", builder: "BuilderDef", slot: "SlotDef", nsfw="off") -> int: + """How many lists back a slot (for hiding buttons that would have none).""" + return len(matcher.lists_for(builder, slot, nsfw)) + + +def _violence_hidden(key: str, violence: int) -> bool: + """A list a node with this violence_filter level does not offer: its name is in that tier or a lower one.""" + if not violence: + return False + tier = violence_name_tier(key) + return bool(tier) and tier <= violence + + +class SlotMatcher: + """ + Finds the lists behind each slot in one WildcardLibrary. Results are cached per + (builder, slot, nsfw level) until the library's key set changes. + """ + + def __init__(self, library): + self.library = library + self._cache: Dict[Tuple, Tuple[int, List[ListInfo]]] = {} + self._claimed: Dict[str, Tuple[int, Set[str]]] = {} + self._unassigned: Dict[Tuple, Tuple[Tuple[int, int], List[str]]] = {} + + def lists_for(self, builder: BuilderDef, slot: SlotDef, nsfw="off", gender: str = "", + violence: int = 0) -> List[ListInfo]: + """ + The lists behind a slot, best first (cached until the library or your ratings change). gender 'female' / 'male' leaves out lists named for the + other one and puts lists named for this one first (neutral lists follow). violence: a + violence_filter level (0-3); lists whose name falls in that tier or a lower one are left out. + """ + gender = gender if gender in GENDERS else "" + curation = getattr(self.library, "curation", None) + generation = (self.library.generation, curation.version if curation is not None else 0) + cache_key = (builder.id, slot.id, nsfw_level(nsfw), gender, int(violence)) + cached = self._cache.get(cache_key) + if cached and cached[0] == generation: + return cached[1] + found = self._collect(slot, nsfw, gender, claimed=self.claimed(builder), + violence=int(violence))[:MAX_LISTS_PER_SLOT] + self._cache[cache_key] = (generation, found) + return found + + def clear(self) -> None: + self._cache.clear() + self._claimed.clear() + self._unassigned.clear() + + def claimed(self, builder: BuilderDef) -> Set[str]: + """Keys whose own name some slot of the builder knows. Only the others are matched by folder.""" + generation = self.library.generation + cached = self._claimed.get(builder.id) + if cached and cached[0] == generation: + return cached[1] + slots = [slot for slot in builder.slots if slot._match] + found = {key for key in self.library.visible_keys("explicit") + if any(slot.leaf_rank(key) is not None for slot in slots)} + self._claimed[builder.id] = (generation, found) + return found + + def unassigned(self, builder: BuilderDef, nsfw="off", violence: int = 0) -> List[str]: + """Visible keys no slot button of the builder offers (for the panel's More lists).""" + curation = getattr(self.library, "curation", None) + generation = (self.library.generation, curation.version if curation is not None else 0) + cache_key = (builder.id, nsfw_level(nsfw), int(violence)) + cached = self._unassigned.get(cache_key) + if cached and cached[0] == generation: + return cached[1] + claimed = self.claimed(builder) + found = [] + for key in self.library.visible_keys(nsfw): + if "__" in key or not _REFERENCEABLE.fullmatch(key) or _ALWAYS_AVOID.search(key): + continue + if curation is not None and curation.hidden(key): + continue + if key in claimed or _violence_hidden(key, violence): + continue + if any(key in slot.lists or slot.match_rank(key, "explicit", folder_ok=True) is not None for slot in builder.slots): + continue + found.append(key) + self._unassigned[cache_key] = (generation, found) + return found + + def find(self, query: str, nsfw="off", gender: str = "", also: Sequence[str] = (), + limit: int = 40, violence: int = 0) -> List[ListInfo]: + """ + Lists for the panel's Find box: every list whose key has all the words (key_has_terms), + then `also` (e.g. the catalog's full-text hits on entries), deduplicated and ranked like a slot. + """ + slot = SlotDef(id="_find", label="", search=query) + if not slot._terms: + return [] + return self._collect(slot, nsfw, gender if gender in GENDERS else "", extra=also, + max_candidates=FIND_MAX_CANDIDATES, violence=int(violence))[:limit] + + def _collect(self, slot: SlotDef, nsfw, gender: str, extra: Sequence[str] = (), + max_candidates: int = 0, claimed: Optional[Set[str]] = None, violence: int = 0) -> List[ListInfo]: + curation = getattr(self.library, "curation", None) + visible = self.library.visible_keys(nsfw) + if violence: + visible = [key for key in visible if not _violence_hidden(key, violence)] + visible_set = set(visible) + # A button you pin to a prompt file (prompts/...) offers it; nothing else ever matches one. + pinnable = visible_set | set(getattr(self.library, "prompt_names", lambda _n: [])(nsfw)) + candidates: List[Tuple[int, bool, str]] = [] + for key in slot.lists: + if key in pinnable: + candidates.append((-1, True, key)) + # Your merged lists for this slot (mine//, see merge.py) are pinned too. + mine = f"mine/{slot.id}/" + if slot.id != "_find": + listed = set(slot.lists) + candidates.extend((-1, True, key) for key in visible if key.startswith(mine) and key not in listed) + # Lists you assigned to this button in the wildcard review (_catalog_user.yaml slot:) are pinned next. + if curation is not None and hasattr(curation, "slot_keys"): + taken = {key for _rank, _pin, key in candidates} + candidates.extend((-1, True, key) for key in curation.slot_keys(slot.id) + if key in pinnable and key not in taken) + pinned = {key for _rank, _pin, key in candidates} + for key in visible: + if key in pinned or "__" in key or not _REFERENCEABLE.fullmatch(key): + continue + if curation is not None and curation.hidden(key): + continue + if gender and gender_of(key) not in ("", gender): + continue + rank = slot.match_rank(key, nsfw, folder_ok=claimed is not None and key not in claimed) + if rank is not None: + candidates.append((rank, False, key)) + named = {key for _rank, _pin, key in candidates} + extra_rank = len(slot.match) + 2 + for key in extra: + if key in named or key not in visible_set or "__" in key or not _REFERENCEABLE.fullmatch(key): + continue + if _ALWAYS_AVOID.search(key) or (curation is not None and curation.hidden(key)): + continue + if gender and gender_of(key) not in ("", gender): + continue + named.add(key) + candidates.append((extra_rank, False, key)) + # Whether this search read only its likeliest candidates (the Find box then says "N+ lists"). + self.last_truncated = bool(max_candidates and len(candidates) > max_candidates) + if max_candidates and len(candidates) > max_candidates: + # A broad Find ('hair') can name thousands of lists; read the likeliest ones only. + candidates.sort(key=lambda item: (item[0], len(item[2]))) + candidates = candidates[:max_candidates] + + found: List[ListInfo] = [] + by_signature: Dict[str, ListInfo] = {} + for rank, is_pinned, key in candidates: + entries = self.library.entries_of(key, nsfw=nsfw) + if len(entries) < MIN_ENTRIES and not is_pinned: + continue + signature = _signature(entry.text for entry in entries) + twin = by_signature.get(signature) + if twin is not None: + # Same content in another pack: keep the shorter (usually top-level) key. + if len(key) < len(twin.key) and not twin.pinned: + twin.also.append(twin.key) + twin.key = key + else: + twin.also.append(key) + twin.rank = min(twin.rank, rank) + continue + lengths = sorted(len(entry.text.split()) for entry in entries) + info = ListInfo(key=key, count=len(entries), rank=rank, pinned=is_pinned, + words=lengths[len(lengths) // 2] if lengths else 0, + nsfw=self.library.list_rating(key), gender=gender_of(key)) + if curation is not None: + info.stars = curation.stars(key) + row = curation.catalog.row(key) if curation.catalog is not None else None + if row is not None: + info.quality, info.dialect, info.display = row.quality, row.dialect, row.display + by_signature[signature] = info + found.append(info) + + # Pinned, then your stars, then the match pattern's rank, then quality (short, clean prose + # entries read best between typed words), then size. Long or tag lists stay in the viewer. + found.sort(key=lambda info: (not info.pinned, -info.stars, bool(gender) and info.gender != gender, info.rank, + -round(info.order_score, 1), -min(info.count, 2000), info.key)) + return found diff --git a/core/profiles/rewriter.py b/core/profiles/rewriter.py index 3fc72eb..044d34f 100644 --- a/core/profiles/rewriter.py +++ b/core/profiles/rewriter.py @@ -102,15 +102,22 @@ def run_word_range(profile: Profile, task: str, facts: "RunFacts") -> tuple: return profile.words_for_run(task, facts.duration_seconds, facts.length_words) +def _token_limit_line(profile: Profile) -> str: + """The budget the LLM is told: the encoder limit minus headroom for template tokens and for + LoRA trigger words appended after the rewrite (token_budget.budget_for). 2026-10-08.""" + from .token_budget import budget_for + limit, target = budget_for(profile) + return (f"Hard limit: {target} tokens (about {int(target * 0.72)} English words). The model's text " + f"encoder reads {limit} tokens and silently cuts off the rest; the remaining room is reserved " + f"for LoRA trigger words added after your answer. Put the most important content first.") + + def _length_block(profile: Profile, task: str, facts: Optional["RunFacts"] = None) -> str: low, high = run_word_range(profile, task, facts) if facts is not None else profile.words_for(task) unit = "tags" if profile.caption_style == "tags" else "words" lines = [f"Aim for {low}-{high} {unit}. Shorter than {low} is too thin; longer than {high} wastes the model's attention."] if profile.max_tokens: - lines.append( - f"Hard limit: {profile.max_tokens} tokens (about {int(profile.max_tokens * 0.72)} English words). " - f"Text beyond it is cut off, so the most important content comes first." - ) + lines.append(_token_limit_line(profile)) if profile.max_chars: lines.append(f"Hard limit: {profile.max_chars} characters including spaces.") return "\n".join(lines) @@ -146,8 +153,12 @@ def _text_block(profile: Profile) -> str: if profile.quoted_text == "none": return "This model renders text poorly: avoid asking for readable text; if the user insists, keep it to one short word." quote = '"double quotes"' if profile.quoted_text == "double" else "'single quotes'" + # The first sentence matters: small models read a section about text as an invitation to add some, and put + # signs and plaques into pictures that asked for none ("Next Bus: 12 minutes", seen live 2026-09-21). return ( - f"Text that must appear in the picture goes in {quote}, exactly as the user wrote it - never translated, " + "Put text in the picture ONLY when the user's idea asks for it; otherwise write none - no signs, labels, " + "plaques or captions of your own. " + f"Text the user does ask for goes in {quote}, exactly as the user wrote it - never translated, " f"corrected or paraphrased - with where it sits and what it looks like (surface, font style, colour, size)." ) @@ -345,7 +356,7 @@ def build_official_system(profile: Profile, task: str, facts: RunFacts, official additions.append(f"LENGTH FOR THIS RUN\nAim for {low}-{high} words.") limit_lines = [] if profile.max_tokens: - limit_lines.append(f"Hard limit of the model's text encoder: {profile.max_tokens} tokens (about {int(profile.max_tokens * 0.72)} English words).") + limit_lines.append(_token_limit_line(profile)) if profile.max_chars: limit_lines.append(f"Hard limit: {profile.max_chars} characters.") if limit_lines: diff --git a/core/profiles/token_budget.py b/core/profiles/token_budget.py new file mode 100644 index 0000000..4219199 --- /dev/null +++ b/core/profiles/token_budget.py @@ -0,0 +1,180 @@ +""" +Token budget for platform prompts. + +A text encoder silently drops whatever runs past its limit (Krea 2: 512 tokens incl. a 5-token +template suffix), and LoRA trigger words are usually appended AFTER the rewrite - so a long +answer loses exactly the words that matter most. This module counts tokens as the target +encoder will, decides the budget the rewriter should aim for, and trims an answer to fit at a +sentence / clause boundary while keeping protected terms (verbatim picks, triggers). + +Counting: + * profiles whose encoder is a Qwen model (Krea 2, Qwen-Image, Z-Image, ...) use a Qwen + tokenizer that is ALREADY on disk - config.json "token_budget": {"tokenizer": path}, else the + Hugging Face cache (local_files_only, never a download). Qwen2.5 / Qwen3 share one vocabulary. + * anything else, or no tokenizer found: a conservative estimate (over-counts slightly, so a + trim errs toward keeping the prompt inside the limit). + +Author: Eric Hiss (GitHub: EricRollei) +License: Dual License (Non-Commercial and Commercial Use). See LICENSE. +""" + +from __future__ import annotations + +import json +import math +import os +import re +from typing import Callable, List, Optional, Sequence, Tuple + +_PACKAGE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +_CACHED_QWEN_REPOS = ( + "Qwen/Qwen3-VL-4B-Instruct", "Qwen/Qwen3-VL-8B-Instruct", "Qwen/Qwen3-VL-2B-Instruct", + "Qwen/Qwen2.5-VL-7B-Instruct", "qwen/Qwen2.5-VL-7B-Instruct", +) +_STATE = {"qwen": None, "source": ""} # None = not tried yet, False = unavailable + +# Headroom kept free under the encoder limit: template tokens (Krea 2's suffix is 5) plus room for +# LoRA trigger words merged after the rewrite. 8% of the limit, at least 8 tokens. +RESERVE_FRACTION = 0.08 +RESERVE_MIN = 8 + + +def _config_tokenizer_path() -> str: + try: + with open(os.path.join(_PACKAGE_DIR, "config.json"), "r", encoding="utf-8") as handle: + section = (json.load(handle) or {}).get("token_budget") or {} + return str(section.get("tokenizer") or "") + except Exception: + return "" + + +def _qwen_tokenizer(): + if _STATE["qwen"] is not None: + return _STATE["qwen"] or None + try: + from transformers import AutoTokenizer + except Exception: + _STATE["qwen"] = False + return None + candidates = [] + configured = _config_tokenizer_path() + if configured: + candidates.append((configured, False)) + candidates += [(repo, True) for repo in _CACHED_QWEN_REPOS] + for source, cache_only in candidates: + try: + tok = AutoTokenizer.from_pretrained(source, local_files_only=cache_only) if cache_only \ + else AutoTokenizer.from_pretrained(source) + _STATE["qwen"], _STATE["source"] = tok, source + return tok + except Exception: + continue + _STATE["qwen"] = False + return None + + +def uses_qwen(profile) -> bool: + return "qwen" in (getattr(profile, "encoder", "") or "").lower() + + +def estimate_tokens(text: str) -> int: + """Conservative: the larger of 1.35 tokens per word and 1 token per 3.5 characters.""" + words = len(re.findall(r"\S+", text or "")) + return int(math.ceil(max(words * 1.35, len(text or "") / 3.5))) + + +def counter_for(profile) -> Tuple[Callable[[str], int], str]: + """(count_fn, label). label says how the count was made, for the report.""" + if uses_qwen(profile): + tok = _qwen_tokenizer() + if tok is not None: + return (lambda text: len(tok(text or "", add_special_tokens=False).input_ids), + f"Qwen tokenizer ({os.path.basename(_STATE['source'].rstrip('/\\')) or _STATE['source']})") + return estimate_tokens, "estimate" + + +def budget_for(profile) -> Tuple[int, int]: + """(limit, target). target = limit minus headroom for template tokens and LoRA triggers. + (0, 0) when the profile documents no limit.""" + limit = int(getattr(profile, "max_tokens", 0) or 0) + if limit <= 0: + return 0, 0 + reserve = max(RESERVE_MIN, int(round(limit * RESERVE_FRACTION))) + return limit, max(16, limit - reserve) + + +def _join(body: str, extra: Sequence[str]) -> str: + body = body.rstrip() + if not extra: + return body + tail = ", ".join(extra) + if not body: + return tail + body = body.rstrip(',;:').rstrip() + return f"{body} {tail}" if body[-1:] in '.!?' else f"{body}, {tail}" + + +def fit_to_budget(text: str, budget: int, count: Callable[[str], int], + protect: Sequence[str] = ()) -> Tuple[str, int, List[str]]: + """Trim `text` so count(result) <= budget, cutting the END of the body at the last sentence + end (else clause end, else word) that fits. Every term in `protect` that the trim removed is + re-appended. Returns (text, tokens, dropped_preview). Text already inside the budget comes + back unchanged.""" + n = count(text) + if budget <= 0 or n <= budget: + return text, n, [] + protect = [p for p in (protect or []) if p and p.strip()] + + # 1) Prefer dropping whole sentences from just BEFORE the last one: prose prompts lead with the + # subject and end with medium / style / framing (Krea 2's own structure), so both ends matter. + sentences = [s for s in re.split(r"(?<=[.!?])\s+", text.strip()) if s] + if len(sentences) >= 3: + keep = list(sentences) + removed = [] + while len(keep) >= 3: + removed.insert(0, keep.pop(-2)) + body = " ".join(keep) + low = body.lower() + candidate = _join(body, [p for p in protect if p.lower() not in low]) + m = count(candidate) + if m <= budget: + gone = " ".join(removed) + return candidate, m, [gone[:80] + ("..." if len(gone) > 80 else "")] + + # 2) Fall back to cutting the end at the last sentence / clause / word that fits. + words = list(re.finditer(r"\S+", text)) + if not words: + return text, n, [] + + def build(k): # first k words + any protected terms they lost + body = text[:words[k - 1].end()] if k > 0 else "" + low = body.lower() + lost = [p for p in protect if p.lower() not in low] + return _join(body, lost) + + lo, hi = 0, len(words) # largest k with count(build(k)) <= budget + while lo < hi: + mid = (lo + hi + 1) // 2 + if count(build(mid)) <= budget: + lo = mid + else: + hi = mid - 1 + k = lo + if k <= 0: + out = build(0) + return out, count(out), [text[:80]] + end = words[k - 1].end() + head = text[:end] + cut = end + for pattern in (r"[.!?](?=\s|$)", r"[,;:](?=\s|$)"): + hits = [m.end() for m in re.finditer(pattern, head)] + if hits and hits[-1] >= end * 0.6: + cut = hits[-1] + break + body = text[:cut].rstrip() + if body and body[-1] in ",;:": + body = body[:-1].rstrip() + "." + low = body.lower() + out = _join(body, [p for p in protect if p.lower() not in low]) + dropped = text[cut:].strip() + return out, count(out), [dropped[:80] + ("..." if len(dropped) > 80 else "")] if dropped else [] diff --git a/core/profiles/validate.py b/core/profiles/validate.py index 9788acd..1295bcf 100644 --- a/core/profiles/validate.py +++ b/core/profiles/validate.py @@ -288,6 +288,131 @@ def _validate_tags(text: str, profile: Profile, warnings: List[str]) -> str: return flat +# -- instruction leaks ---------------------------------------------------------------------- +# A long instruction block stops being read as rules and starts being read as text to include. +# Observed in a real run: "the law of the garden holds: only patterned fabric may cross the boundary +# between a body and a wall" and "no symbolic objects, no emotional expression, no text or captions" +# arrived inside a positive Krea2 prompt, the second of those from the very line that forbids writing +# 'no X'. No amount of further instruction fixes this; removing the clauses mechanically does. +# +# The unit of removal is the CLAUSE, not the sentence: these leaks arrive as trailing comma- or +# dash-separated clauses inside an otherwise good sentence, and taking the whole sentence would cost +# real description. +INSTRUCTION_LEAK = re.compile( + r"""(?ix) + (?: the \s+ (?: law | rule ) \s+ (?: of \s+ the \s+ \w+ \s+ )? (?: holds | is | states | applies | governs ) ) + | (?: (?: which | that | it ) \s+ )? (?: obeys | follows | observes ) \s+ (?: this | the ) \s+ (?: law | rule ) + | (?: in \s+ (?: accordance | keeping ) \s+ with \s+ (?: this | the ) \s+ (?: law | rule ) ) + | (?: no \s+ (?: further | additional | other )? \s* symbolic \s+ \w+ ) + | (?: no \s+ (?: text | captions? | titles? | watermarks? | lettering | writing ) \b ) + | (?: no \s+ emotional \s+ (?: expression | melodrama | display ) ) + | (?: no \s+ (?: explanation | symbolism | moral | caption ) ) + | (?: exactly \s+ (?: one | ONE ) \s+ anomal ) + | (?: the \s+ (?: anomaly | premise | rupture | transformation \s+ rule ) \b ) + | (?: stated? \s+ as \s+ plain \s+ physical \s+ fact ) + | (?: (?: the \s+ way \s+ )? a \s+ (?: police \s+ )? report \s+ would ) + | (?: (?: image | diffusion ) \s+ model ) + | (?: (?: positive | negative ) \s+ prompt ) + | (?: register \s* : ) + | (?: THE \s+ SUBJECT ) + | (?: rigorously \s+ ordinary ) + | (?: never \s+ name \s+ the \s+ emotion ) + | (?: pose \s+ the \s+ question \s+ and \s+ refuse ) + """ +) +# A clause that merely excludes something. In a positive prose prompt "no mist" summons mist, exactly as +# it does in a tag list, where _validate_tags has always removed it. +# +# [a-z] was wrong: it matched "no mist" but not "no 4K", "no 3D render" or "no 8M", which is precisely the +# form a repetition collapse takes. Anything that is not whitespace or a separator counts. +NEGATION_CLAUSE = re.compile(r"(?i)^\s*(?:and\s+|but\s+|with\s+)?(?:no|without)\s+[^\s,;]") +_DIGITS = re.compile(r"\d+") +_CLAUSE_SPLIT = re.compile(r"\s*(,|;|\s+[-–—]+\s+)\s*") +_SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+") + + +REPEAT_LIMIT = 3 # the same clause SHAPE more than three times is a model looping, not a description + + +def strip_instruction_leaks(text: str, drop_negations: bool = True) -> Tuple[str, List[str], List[str], int]: + """(cleaned, leaked clauses, negation clauses, repeated clauses dropped). + + drop_negations defaults to True and should stay true for every positive prompt. It was first gated on the + model HAVING a negative prompt, on the reasoning that there was otherwise nowhere to move the exclusion + to - which is backwards. 'no mist' summons mist whether or not the model offers a negative field; with + nowhere to move it, the right answer is still to delete it.""" + leaked: List[str] = [] + negated: List[str] = [] + repeats = 0 + shapes: Dict[str, int] = {} + kept_sentences: List[str] = [] + for sentence in _SENTENCE_SPLIT.split(text): + if not sentence.strip(): + continue + pieces = _CLAUSE_SPLIT.split(sentence) + # pieces alternates clause, separator, clause, ... + clauses = pieces[0::2] + separators = pieces[1::2] + keep_clause, keep_separator = [], [] + for index, clause in enumerate(clauses): + bare = clause.strip() + if not bare: + continue + if INSTRUCTION_LEAK.search(bare): + leaked.append(bare) + continue + if drop_negations and NEGATION_CLAUSE.match(bare): + negated.append(bare) + continue + # Repetition collapse: "no 4K, no 8K, no 16K ..." for twelve thousand characters. Digits are + # normalised away so the whole run reduces to one shape, and only the first few are kept. + shape = _DIGITS.sub("#", bare.lower()) + shapes[shape] = shapes.get(shape, 0) + 1 + if shapes[shape] > REPEAT_LIMIT: + repeats += 1 + continue + keep_clause.append(bare) + keep_separator.append(separators[index] if index < len(separators) else "") + if not keep_clause: + continue + rebuilt = "" + for index, clause in enumerate(keep_clause): + rebuilt += clause + if index < len(keep_clause) - 1: + separator = keep_separator[index].strip() + rebuilt += "; " if separator == ";" else ", " + kept_sentences.append(rebuilt) + cleaned = " ".join(kept_sentences) + # A dropped final clause can leave a dangling comma or a doubled full stop. + cleaned = re.sub(r"\s*,\s*([.!?])", r"\1", cleaned) + cleaned = re.sub(r"\s*,\s*$", "", cleaned.strip()) + cleaned = re.sub(r"([.!?])\1+", r"\1", cleaned) + # Only close a sentence that was one to begin with: a tag list must not gain a full stop. + if cleaned and text.rstrip().endswith((".", "!", "?")) and not cleaned.endswith((".", "!", "?", '"')): + cleaned += "." + return cleaned, leaked, negated, repeats + + +# Where a leak is worth hunting: free-running text. Structured formats (json, named fields, per-shot +# lines) are held in shape by their own validators, and splitting them on commas would damage them. +LEAK_CHECKED_STYLES = ("prose", "tags_prose", "tags", "instruction", "bracket_prose") + +RUNAWAY_FACTOR = 2.5 # past this multiple of the profile's upper word bound the answer is broken, not long + + +def _truncate_words(text: str, limit: int) -> str: + """Cut to `limit` words, preferring the last sentence end, then the last clause end, before that point.""" + words = re.findall(r"\S+", text) + if len(words) <= limit: + return text + head = " ".join(words[:limit]) + for boundary in (r"[.!?](?=[^.!?]*$)", r"[,;](?=[^,;]*$)"): + match = list(re.finditer(boundary, head)) + if match and match[-1].end() > len(head) * 0.5: + return head[:match[-1].end()].rstrip(",; ").rstrip() + ("" if head[match[-1].start()] in ".!?" else ".") + return head.rstrip(",; ") + "." + + def _validate_edit(text: str, profile: Profile, warnings: List[str]) -> str: if not re.search(r"\b(keep|keeping|maintain|maintaining|preserv|unchanged|same|identical|stays?)\b", text, re.I): warnings.append("edit instruction does not say what must stay the same") @@ -329,6 +454,30 @@ def check(text: str, profile: Profile, task: str, word_range: Optional[Tuple[int except Exception as error: # a validator bug must never cost the user their prompt warnings.append(f"validator '{validator_name}' failed ({error}); text passed through") + # instruction leaks: the block was transcribed instead of obeyed. Mechanical, and independent of + # any model choosing to co-operate. + if profile.caption_style in LEAK_CHECKED_STYLES: + try: + repaired, leaked, negated, repeats = strip_instruction_leaks(cleaned) + if repaired and (leaked or negated or repeats): + cleaned = repaired + if leaked: + warnings.append(f"{len(leaked)} clause(s) of the instruction block were copied into the prompt and have been " + "removed: " + " / ".join(clause[:70] for clause in leaked[:3]) + (" ..." if len(leaked) > 3 else "")) + if negated: + destination = "put them in the negative prompt" if profile.negative in ("yes", "inline") else \ + "this model has no negative prompt, so they are simply dropped" + warnings.append(f"{len(negated)} 'no ...' clause(s) removed from the positive prompt (they summon what they " + f"exclude; {destination}): " + " / ".join(clause[:50] for clause in negated[:3]) + + (" ..." if len(negated) > 3 else "")) + if repeats: + warnings.append(f"the model looped: {repeats} repeated clause(s) removed. A prompt that degenerates into a " + "repeating pattern usually means the request was too long or too constrained for it.") + elif (leaked or negated) and not repaired: + warnings.append("the answer was almost entirely instruction text; it has been left alone for you to judge") + except Exception as error: # a guard bug must never cost the user their prompt + warnings.append(f"instruction-leak guard failed ({error}); text passed through") + # prefix: mechanical and safe to repair if profile.prefix and profile.quality_tokens == "required" and not is_json: if not cleaned.lower().startswith(profile.prefix.lower().rstrip(", ")): @@ -351,6 +500,15 @@ def check(text: str, profile: Profile, task: str, word_range: Optional[Tuple[int if not is_json: low, high = word_range if word_range and word_range[1] > 0 else profile.words_for(task) count = word_count(cleaned) + # A runaway answer is not a long prompt, it is a broken one: a model that loops can emit thousands of + # words, and everything past the model's own limit is discarded by the sampler anyway. Cut it at a + # sentence boundary rather than handing the user twelve thousand characters of "no 4K, no 8K". + if high and count > high * RUNAWAY_FACTOR: + trimmed = _truncate_words(cleaned, int(high * RUNAWAY_FACTOR)) + warnings.append(f"runaway answer: {count} words for a {low}-{high} word target, cut to {word_count(trimmed)}. " + "The model almost certainly lost the thread; re-run before using this.") + cleaned = trimmed + count = word_count(cleaned) if count > high * 1.35: warnings.append(f"{count} words; the target for {profile.name} is {low}-{high}") elif count < low * 0.5: diff --git a/core/strategies/surreal/__init__.py b/core/strategies/surreal/__init__.py index 6ed4173..defd413 100644 --- a/core/strategies/surreal/__init__.py +++ b/core/strategies/surreal/__init__.py @@ -6,12 +6,16 @@ See vocabulary.py for the two governing ideas and docs/SURREAL_GUIDE.md for the """ from .critic import Verdict, code_choose, critic_messages, parse_critic, violations +from .compiler import CompilationResult, CompositionPlan, StructuredRenderPlan, build_render_plan, compile_render_plan from .engine import SurrealEngine, SurrealPlan, SurrealResult, SurrealSettings from .generators import MODE_GENERATORS, OPERATION_GENERATORS, SECOND_OPERATIONS, Candidate, Context +from .lint import SUPPRESS_MARKER, lint_prompt, lint_render, suppress_from_instructions from .sources import Item, SurrealSources, head_noun, mentioned +from .scene import SceneEntity, SceneInventory, inventory_for_critic, parse_scene_inventory, scene_messages from .vocabulary import ( - AUTO, BAN_LIST, CHARGE_CHOICES, CHARGES, MODE_CHOICES, MODES, NONE, OPERATION_CHOICES, OPERATIONS, REGISTER_CHOICES, - REGISTERS, STRANGENESS_CHOICES, SUBJECT_TOKEN, + AUTO, BAN_LIST, BANNED_TITLES, CHARGE_CHOICES, CHARGE_LAWS, CHARGES, GENERIC_LAWS, MODE_CHOICES, MODES, NONE, + OPERATION_CHOICES, OPERATIONS, REGISTER_CHOICES, REGISTERS, RUBRIC, STAGING_CHOICES, STRANGENESS_CHOICES, SUBJECT_TOKEN, + TITLE_STRATEGIES, law_seed, ) __all__ = [ @@ -20,4 +24,8 @@ __all__ = [ "MODE_GENERATORS", "OPERATION_GENERATORS", "SECOND_OPERATIONS", "AUTO", "NONE", "MODES", "MODE_CHOICES", "OPERATIONS", "OPERATION_CHOICES", "REGISTERS", "REGISTER_CHOICES", "CHARGES", "CHARGE_CHOICES", "STRANGENESS_CHOICES", "BAN_LIST", "SUBJECT_TOKEN", + "CHARGE_LAWS", "GENERIC_LAWS", "law_seed", "RUBRIC", "TITLE_STRATEGIES", "BANNED_TITLES", + "STAGING_CHOICES", "lint_render", "lint_prompt", "suppress_from_instructions", "SUPPRESS_MARKER", + "SceneEntity", "SceneInventory", "scene_messages", "parse_scene_inventory", "inventory_for_critic", + "CompositionPlan", "StructuredRenderPlan", "CompilationResult", "build_render_plan", "compile_render_plan", ] diff --git a/core/strategies/surreal/compiler.py b/core/strategies/surreal/compiler.py new file mode 100644 index 0000000..2ee8ffc --- /dev/null +++ b/core/strategies/surreal/compiler.py @@ -0,0 +1,270 @@ +""" +Structured surreal render plans and operation-specific prompt compilation. + +The compiler receives a finished artistic decision. It does not choose a mode, invent another object or explain +the symbolism. Its job is to turn the transformed scene into positive, visible image facts while preventing the +ordinary source object from leaking back into the final prompt. + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List, Optional, Sequence, Tuple + +from .lint import lint_render, named_terms +from .scene import SceneInventory + + +@dataclass +class CompositionPlan: + anomaly_scale: str = "dominant" + placement: str = "central" + frame_share: str = "at least one third of the frame" + scale_reference: str = "" + + +@dataclass +class StructuredRenderPlan: + operation: str + anomaly: str + ordinary_scene: str = "" + preserved_scene: str = "" + target: str = "" + replacement: str = "" + preserved_properties: List[str] = field(default_factory=list) + suppress_entities: List[str] = field(default_factory=list) + suppress_properties: List[str] = field(default_factory=list) + visual_evidence: List[str] = field(default_factory=list) + forbidden_interpretations: List[str] = field(default_factory=list) + composition: CompositionPlan = field(default_factory=CompositionPlan) + law: str = "" + edit_instruction: str = "" + + def report_lines(self) -> List[str]: + lines = [f"structured plan: operation={self.operation}"] + if self.target: + lines.append(f"structured plan: target={self.target}") + if self.replacement: + lines.append(f"structured plan: replacement={self.replacement}") + if self.preserved_properties: + lines.append("structured plan: preserve=" + "; ".join(self.preserved_properties)) + if self.suppress_entities: + lines.append("structured plan: removed entities=" + "; ".join(self.suppress_entities)) + if self.suppress_properties: + lines.append("structured plan: replaced properties=" + "; ".join(self.suppress_properties)) + if self.visual_evidence: + lines.append("structured plan: visual proof=" + " | ".join(self.visual_evidence)) + if self.forbidden_interpretations: + lines.append("structured plan: reject=" + " | ".join(self.forbidden_interpretations)) + lines.append( + "structured plan: composition=" + f"{self.composition.anomaly_scale}, {self.composition.placement}, {self.composition.frame_share}" + + (f", scale reference={self.composition.scale_reference}" if self.composition.scale_reference else "") + ) + return lines + + +@dataclass +class CompilationResult: + text: str + plan: StructuredRenderPlan + warnings: List[str] = field(default_factory=list) + ok: bool = True + + +DEFAULT_FORBIDDEN = { + "displacement": ["the replacement merely beside the original", "both the original and replacement visible"], + "material_substitution": ["the new material shown beside the target", "a coating that hides the target's form"], + "scale_violation": ["an ordinary-sized target described only with the word enormous", "everything enlarged together"], + "hybridization": ["two separate objects touching", "a collage with a visible seam"], + "inversion": ["a picture or model of the contained world", "ordinary contents inside the opening"], + "animation_petrification": ["a statue standing beside the living subject", "a surface filter without physical material"], + "multiplication": ["different instances rather than exact copies", "copies without their own shadows"], + "behavior_transfer": ["a literal prop made from the borrowed material", "a small ordinary action instead of a changed physical law"], +} + + +def _unique(items: Sequence[str], limit: int = 8) -> List[str]: + found = [] + for item in items: + text = re.sub(r"\s+", " ", str(item or "")).strip().strip(".") + if text and text.lower() not in {entry.lower() for entry in found}: + found.append(text) + return found[:limit] + + +def _entity_terms(items: Sequence[str]) -> List[str]: + """Full removed descriptions plus their head nouns, because the source idea may use a shorter wording.""" + terms = [] + for item in items: + text = re.sub(r"^(?:a|an|the|some)\s+", "", str(item or "").strip(), flags=re.I) + if not text: + continue + terms.append(text) + first_segment = re.split(r"\s+(?:on|with|in|at|under|over|beside|near|holding)\s+|,", text, maxsplit=1)[0] + words = re.findall(r"[A-Za-z][A-Za-z'-]+", first_segment) + if words: + terms.append(words[-1]) + return _unique(terms, 16) + + +def _sentence(text: str) -> str: + text = re.sub(r"\s+", " ", text or "").strip() + if not text: + return "" + text = text[:1].upper() + text[1:] + return text if text[-1:] in ".!?" else text + "." + + +def _evidence_sentence(plan: StructuredRenderPlan) -> str: + evidence = _unique(plan.visual_evidence, 4) + if not evidence: + return "" + if len(evidence) == 1: + return _sentence(evidence[0]) + return _sentence(", ".join(evidence[:-1]) + ", and " + evidence[-1]) + + +def _clean_context(context: str, suppress: Sequence[str]) -> Tuple[str, List[str]]: + """Drop whole clauses that name removed entities. Never perform a noun-level deletion that leaves broken prose.""" + if not context: + return "", [] + pieces = re.split(r"(?<=[.!?;])\s+|(?<=,)\s+(?=(?:and|with|beside|near|under|over|holding|wearing)\b)", context) + kept, dropped = [], [] + for piece in pieces: + if named_terms(piece, suppress): + dropped.append(piece.strip()) + else: + kept.append(piece.strip()) + cleaned = " ".join(piece for piece in kept if piece) + cleaned = re.sub(r"\s+", " ", cleaned).strip(" ,;") + if cleaned and cleaned[-1:] not in ".!?": + cleaned += "." + return cleaned, dropped + + +def _compile_displacement(plan: StructuredRenderPlan) -> List[str]: + return [_sentence(plan.anomaly), _evidence_sentence(plan), _sentence(plan.preserved_scene)] + + +def _compile_material(plan: StructuredRenderPlan) -> List[str]: + return [_sentence(plan.anomaly), _evidence_sentence(plan), _sentence(plan.preserved_scene)] + + +def _compile_scale(plan: StructuredRenderPlan) -> List[str]: + evidence = list(plan.visual_evidence) + if plan.composition.scale_reference: + evidence.append(f"its scale is visibly measured against {plan.composition.scale_reference}") + return [_sentence(plan.anomaly), _evidence_sentence(_copy_with_evidence(plan, evidence)), _sentence(plan.preserved_scene)] + + +def _compile_behavior(plan: StructuredRenderPlan) -> List[str]: + return [_sentence(plan.anomaly), _evidence_sentence(plan), _sentence(plan.preserved_scene)] + + +def _copy_with_evidence(plan: StructuredRenderPlan, evidence: Sequence[str]) -> StructuredRenderPlan: + return StructuredRenderPlan( + operation=plan.operation, + anomaly=plan.anomaly, + ordinary_scene=plan.ordinary_scene, + preserved_scene=plan.preserved_scene, + target=plan.target, + replacement=plan.replacement, + preserved_properties=list(plan.preserved_properties), + suppress_entities=list(plan.suppress_entities), + suppress_properties=list(plan.suppress_properties), + visual_evidence=list(evidence), + forbidden_interpretations=list(plan.forbidden_interpretations), + composition=plan.composition, + law=plan.law, + edit_instruction=plan.edit_instruction, + ) + + +COMPILERS = { + "displacement": _compile_displacement, + "material_substitution": _compile_material, + "scale_violation": _compile_scale, + "behavior_transfer": _compile_behavior, +} + + +def build_render_plan( + *, + operation: str, + anomaly: str, + idea: str, + inventory: Optional[SceneInventory] = None, + preserved_scene: str = "", + target: str = "", + replacement: str = "", + preserved_properties: Sequence[str] = (), + suppress_entities: Sequence[str] = (), + suppress_properties: Sequence[str] = (), + visual_evidence: Sequence[str] = (), + forbidden_interpretations: Sequence[str] = (), + composition: Optional[CompositionPlan] = None, + law: str = "", + edit_instruction: str = "", +) -> StructuredRenderPlan: + ordinary_scene = inventory.preserved_scene if inventory is not None else idea + context = preserved_scene or ordinary_scene + context, _dropped = _clean_context(context, suppress_entities) + forbidden = list(forbidden_interpretations) + DEFAULT_FORBIDDEN.get(operation, []) + return StructuredRenderPlan( + operation=operation, + anomaly=re.sub(r"\s+", " ", anomaly or "").strip(), + ordinary_scene=ordinary_scene, + preserved_scene=context, + target=target, + replacement=replacement, + preserved_properties=_unique(preserved_properties), + suppress_entities=_entity_terms(suppress_entities), + suppress_properties=_unique(suppress_properties, 12), + visual_evidence=_unique(visual_evidence), + forbidden_interpretations=_unique(forbidden), + composition=composition or CompositionPlan(), + law=law, + edit_instruction=edit_instruction, + ) + + +def compile_render_plan(plan: StructuredRenderPlan) -> CompilationResult: + if not plan.anomaly: + return CompilationResult("", plan, ["structured compiler received no visible anomaly"], ok=False) + compile_operation = COMPILERS.get(plan.operation) + parts = compile_operation(plan) if compile_operation else [ + _sentence(plan.anomaly), + _evidence_sentence(plan), + _sentence(plan.preserved_scene), + ] + text = " ".join(part for part in parts if part) + text = re.sub(r"\s+", " ", text).strip() + warnings = [] + + named = named_terms(text, plan.suppress_entities) + if named and plan.preserved_scene: + plan.preserved_scene, dropped = _clean_context(plan.preserved_scene, named) + if dropped: + warnings.append("structured compiler dropped context naming removed entities: " + " / ".join(dropped[:3])) + parts = compile_operation(plan) if compile_operation else [ + _sentence(plan.anomaly), + _evidence_sentence(plan), + _sentence(plan.preserved_scene), + ] + text = re.sub(r"\s+", " ", " ".join(part for part in parts if part)).strip() + named = named_terms(text, plan.suppress_entities) + if named: + return CompilationResult( + "", + plan, + warnings + ["structured compiler refused a prompt naming removed entities: " + ", ".join(named)], + ok=False, + ) + grammar = lint_render(text) + if grammar: + return CompilationResult("", plan, warnings + grammar, ok=False) + return CompilationResult(text, plan, warnings, ok=True) diff --git a/core/strategies/surreal/critic.py b/core/strategies/surreal/critic.py index bd2369d..5037f7f 100644 --- a/core/strategies/surreal/critic.py +++ b/core/strategies/surreal/critic.py @@ -22,11 +22,24 @@ from dataclasses import dataclass, field from typing import List, Optional, Sequence, Tuple from .generators import SECOND_OPERATIONS, Candidate -from .vocabulary import BAN_LIST, BAN_LIST_FOR_PROMPT, CHARGES, OPERATIONS, SUBJECT_TOKEN +from .lint import lint_render +from .vocabulary import ( + BAN_LIST, BAN_LIST_FOR_PROMPT, BANNED_TITLES, CHARGES, OPERATIONS, RUBRIC, SOFT_BAN_FOR_PROMPT, SUBJECT_TOKEN, + TITLE_STRATEGIES, +) EXPLAINING = re.compile(r"\b(symboli[sz]\w*|represent\w*|metaphor\w*|signif\w*|allegor\w*|evok\w*|as if in a dream|meaning of)\b", re.I) +# A text-to-image model cannot condition on absence. A premise whose anomaly is stated to be unseeable has +# nothing in it to render: "pulling strings from their hands, invisible to the viewer" produces two women in a +# garden. This is not about strangeness, it is about whether there are pixels to put it in. +# "no visible source" is deliberately NOT here: a pool of light whose lamp is out of frame is fully renderable. +UNRENDERABLE = re.compile( + r"\b(invisible|unseen|imperceptible|undetectable|intangible|immaterial" + r"|cann?o?t be seen|can'?t be seen|never be seen|impossible to see|not visible to" + r"|as if never (?:there|been|happened)|leaves no (?:trace|mark|sign))\b", re.I) MAX_PREMISE_WORDS = 70 MAX_AFFINITY_WORDS = 22 +MAX_LAW_WORDS = 45 def violations(text: str, is_procedure: bool = False) -> List[str]: @@ -38,6 +51,10 @@ def violations(text: str, is_procedure: bool = False) -> List[str]: problems.append("ban-list: " + ", ".join(sorted(set(banned)))) if EXPLAINING.search(text or ""): problems.append("explains itself (" + EXPLAINING.search(text).group(0) + "); the image must pose the question and refuse the answer") + unrenderable = UNRENDERABLE.search(text or "") + if unrenderable: + problems.append(f"the anomaly is described as unseeable ('{unrenderable.group(0)}'); an image model has nothing to " + "render and the picture comes back ordinary") if not is_procedure and len(re.findall(r"\S+", text or "")) > MAX_PREMISE_WORDS: problems.append(f"longer than {MAX_PREMISE_WORDS} words; a premise is one or two plain sentences") return problems @@ -49,6 +66,8 @@ def code_choose(candidates: Sequence[Candidate], rng: random.Random, count: int acceptable = [] for candidate in candidates: found = violations(candidate.premise or candidate.procedure, is_procedure=not candidate.premise) + if candidate.premise: + found += lint_render(candidate.rendered()) # what the image model will actually be given if found: notes.append(f"rejected ({'; '.join(found)}): {candidate.line()[:90]}") else: @@ -63,12 +82,16 @@ def code_choose(candidates: Sequence[Candidate], rng: random.Random, count: int and candidate.operation != first.operation and candidate.premise] if seconds and OPERATIONS[first.operation].impossible: chosen.append(rng.choice(seconds)) + notes.append("the code critic cannot check that the second operation obeys the first one's law; the LLM critic can") else: notes.append("count 2 asked for, one operation used: no second operation can read as the same event here") return chosen, notes # ---- LLM critic --------------------------------------------------------------------------------------- +_RUBRIC_LINES = "\n".join(f"- {name}: {text}" for name, text in RUBRIC.items()) +_TITLE_LINES = "\n".join(f"- {name}: {text}" for name, text in TITLE_STRATEGIES.items()) + CRITIC_SYSTEM = f"""You are the critic half of a surrealist working method. A generator has produced candidate premises by procedure and by chance. Your job is the one Magritte, Ernst and Uelsmann did after the accident: read, select ONE, and finish it with full craft. Most candidates are to be thrown away. What makes a premise work: @@ -78,17 +101,42 @@ What makes a premise work: - It is stated as plain physical fact, present tense, the way a police report would state it. No adjectives of strangeness. - It poses a question and refuses the answer: no explanation, no symbolism, no moral. - It has a stake: it serves the emotional charge you are given. An anomaly with no stake reads as a gimmick. +- It can be PHOTOGRAPHED. This premise becomes a prompt for an image model, and an image model cannot render an absence, a negation, a thing stated to be invisible, or a claim about what caused something. It can only render what is present in the frame. So the anomaly must leave a positive mark that a viewer could point at. "Strings that are invisible to the viewer" is not a picture; a taut cord disappearing into a hedge is. "Nothing in the room acknowledges it" is not a picture; the untouched cup and the turned-away face are. Reject any candidate whose anomaly you cannot describe as something seen. +- It is written as the RESULT, never as the change. An image model's text encoder keeps the nouns and loses the grammar: "where the teacup should be there are boots instead" gives it a teacup AND boots, and it draws the teacup. Write only what the camera sees ("a pair of worn work boots stands on the saucer beside her book"), never name what was replaced or removed, never write should / would / instead of / in place of / as if / treats it as, and never say that nobody notices - show people doing ordinary things instead. +- It is unmistakable at a glance. Deadpan is not the same as subtle: the anomaly is large in the frame, central, sharply lit, and touches the ordinary world (a shadow, a reflection, weight, wear). A premise that is strange only to someone who knows the story - a man holding a smooth stone - renders as an ordinary photograph. +- It obeys a LAW. The rupture is not an exception to the world; it is evidence of a rule the world in this picture follows. A law says what may change, what may not, and how the change behaves - "only patterned fabric can cross the boundary between a body and a wall", "whatever the missing person touched has gained weight". A picture whose anomaly obeys no law reads as random; this is the single most common failure. - Never: {BAN_LIST_FOR_PROMPT}. -- Candidates marked "nothing impossible" are a full mode (Atget, Crewdson, Man Ray): do not add an impossibility to them. +- Use sparingly: {SOFT_BAN_FOR_PROMPT}. +- Candidates marked "nothing impossible" are a full mode (Atget, Crewdson, Man Ray): do not add an impossibility to them. Their law governs light, hour, emptiness and process instead. Steps: 1. Name the main subject of the idea in a few words, and list its ordinary attributes: material, scale, habitat, function, what it always comes with. 2. For each candidate decide keep or reject, with a reason of a few words. A candidate that is a PROCEDURE is carried out by you, here, and then judged like the others. 3. Choose the best. Replace "{SUBJECT_TOKEN}" with the real subject. You may sharpen a candidate (a better partner with a truer affinity, a more exact placement) but not change its operation. -4. Write the premise, a restatement of the same fact in different words, a title in Magritte's manner (it names a problem and never describes the picture), and up to five short "normal" counter-terms that an image model would drift back to (for a fur teacup: porcelain teacup, ceramic cup). +4. Write the LAW this picture obeys: one sentence, under {MAX_LAW_WORDS} words, stating what may change and what may not. You are given a suggested law; keep it only if it genuinely governs the premise you chose, and otherwise write your own. If two operations were asked for, the second must be an instance of the same law. +5. Write premise_draft: the premise as one or two plain sentences. +6. Score premise_draft on each rubric dimension from 0 to 4: +{_RUBRIC_LINES} +7. Name the LOWEST-scoring dimension as "weakest", say in a few words the one change that would raise it ("repair"), and then write "premise" as premise_draft with that one repair applied. Change nothing else. If nothing needs repair, repeat premise_draft unchanged and say so. +8. Write "visible": in one clause, what a viewer literally SEES in this picture that they would not see in an ordinary photograph of the same scene. Name a thing, a surface, an edge, a shadow or a light. If the only honest answer is that nothing looks different, go back to step 3 and choose a different candidate. +8b. Write "render": the premise as the image model must receive it - the RESULT only, one or two sentences, the anomaly first and large in the frame, with one line of physical evidence (its shadow, its weight, its texture up close). Never name what was replaced, removed or missing; never should / would / instead / in place of / as if; never that nobody notices. List in "suppress" the things the anomaly replaced or removed, which must not be named anywhere in the prompt (often none: []). +8c. Convert the decision into a structured render plan: + - "target": the entity whose object, material, scale or behaviour changes. + - "replacement": the new object, material or borrowed behaviour, or empty when there is none. + - "preserve": the visible properties and unrelated scene facts that must survive. + - "visual_evidence": two to four POSITIVE things visible in the finished image that prove the anomaly exists. Include contact, shadow, reflection, weight, texture, continuity, an exposed surface or a known-size reference as appropriate. + - "forbidden_interpretations": the two most likely ordinary pictures an image model would retreat to. + - "composition": anomaly_scale ("dominant"), placement, frame_share ("at least one third"), and scale_reference when useful. + - "preserved_scene": the FINAL ordinary context after the transformation. Keep the subject, action, setting, light and viewpoint, but REMOVE every entity named in "suppress" and every ordinary property that the anomaly replaces. This is not the original scene and must not contradict "render". +8d. Write "scene": the same picture as an ORDINARY photograph, with no anomaly at all, in one or two sentences - the subject, the place, the light. And "edit": ONE imperative instruction for an image-editing model that turns that ordinary picture into this one ("Replace the teacup on the saucer with a pair of worn leather work boots at the same size, in the same light. Keep everything else unchanged."). If no single edit can do it (a law of light or time across the whole frame, nothing impossible, a double image), write "edit": "". +9. Write a restatement of the same fact in different words; choose a title strategy and write the title by it; and give up to five short counter-terms - the ordinary things an image model will drift back to INSTEAD of your anomaly, drawn from THIS premise and its subject, never from the examples in these instructions. -Answer with JSON only, no other text: -{{"subject": "...", "attributes": ["...", "..."], "judgements": [{{"n": 1, "keep": false, "reason": "..."}}], "chosen": 2, "operation": "...", "affinity": "one short clause", "premise": "one or two plain sentences", "restatement": "the same fact in other words", "title": "...", "counter_terms": ["...", "..."]}}""" +Title strategies - pick exactly one and name it: +{_TITLE_LINES} +A title never describes what is visible, and is never "Dream", "Untitled", "Reverie" or anything containing the word surreal. + +Answer with JSON only, no other text, and in this key order: +{{"subject": "...", "attributes": ["...", "..."], "judgements": [{{"n": 1, "keep": false, "reason": "..."}}], "chosen": 2, "operation": "...", "affinity": "one short clause", "law": "one sentence: what may change, what may not", "premise_draft": "one or two plain sentences", "scores": {{"reality_anchor": 0, "rupture": 0, "psychic_pressure": 0, "transformation_rule": 0, "formal_unity": 0, "ambiguity": 0, "residual_power": 0, "visibility": 0}}, "weakest": "...", "repair": "a few words", "premise": "premise_draft with that one repair applied", "visible": "one clause: what a viewer sees that is different", "render": "the result only, as the image model reads it", "target": "...", "replacement": "...", "preserve": ["..."], "visual_evidence": ["..."], "forbidden_interpretations": ["..."], "composition": {{"anomaly_scale": "dominant", "placement": "central", "frame_share": "at least one third", "scale_reference": "..."}}, "preserved_scene": "final ordinary context with removed entities omitted", "suppress": [], "suppress_properties": [], "scene": "the ordinary picture, no anomaly", "edit": "one imperative edit, or empty", "restatement": "the same fact in other words", "title_strategy": "...", "title": "...", "counter_terms": ["...", "..."]}}""" @dataclass @@ -97,28 +145,56 @@ class Verdict: chosen_index: int = -1 # index into the candidate list, -1 = the critic wrote its own operation: str = "" affinity: str = "" + law: str = "" # the transformation rule the picture obeys premise: str = "" + visible: str = "" # what a viewer sees that is different: the anomaly as pixels restatement: str = "" title: str = "" + title_strategy: str = "" counter_terms: List[str] = field(default_factory=list) judgements: List[str] = field(default_factory=list) + scores: dict = field(default_factory=dict) # rubric dimension -> 0..4 + weakest: str = "" # the dimension the critic revised for + repair: str = "" # what it changed, in a few words + render: str = "" # the premise as the image model reads it: the result only (lint.py) + suppress: List[str] = field(default_factory=list) # what the anomaly replaced: never named in the prompt + scene: str = "" # the ordinary picture, no anomaly (two-stage, stage 1) + edit: str = "" # one imperative edit that adds the anomaly (two-stage, stage 2) + target: str = "" # entity changed by the selected operation + replacement: str = "" # replacement object, material or borrowed behaviour + preserve: List[str] = field(default_factory=list) + visual_evidence: List[str] = field(default_factory=list) + forbidden_interpretations: List[str] = field(default_factory=list) + composition: dict = field(default_factory=dict) + preserved_scene: str = "" # final context after removed/replaced facts are omitted + suppress_properties: List[str] = field(default_factory=list) + notes: List[str] = field(default_factory=list) # parts of the answer that were dropped, and why -def critic_messages(idea: str, candidates: Sequence[Candidate], count: int, charge: str, strangeness: str, medium: str) -> Tuple[str, str]: +def critic_messages(idea: str, candidates: Sequence[Candidate], count: int, charge: str, strangeness: str, medium: str, + law: str = "", inventory: str = "") -> Tuple[str, str]: lines = [] for number, candidate in enumerate(candidates, 1): kind = "PROCEDURE" if candidate.procedure and not candidate.premise else "premise" impossible = "" if OPERATIONS[candidate.operation].impossible else " (nothing impossible)" bond = f"\n affinity found by the generator: {candidate.affinity}" if candidate.affinity else "" personal = "\n uses the user's own iconography: prefer it when it is as good as the others" if candidate.personal else "" - lines.append(f"{number}. [{candidate.operation}{impossible}] {kind}: {candidate.premise or candidate.procedure}{bond}{personal}") + implied = candidate.law_hint or OPERATIONS[candidate.operation].law + rule = f"\n the law this operation implies: {implied}" if implied else "" + seen = OPERATIONS[candidate.operation].visible + shows = f"\n how this operation becomes visible: {seen}" if seen else "" + lines.append(f"{number}. [{candidate.operation}{impossible}] {kind}: {candidate.premise or candidate.procedure}{bond}{rule}{shows}{personal}") charge_text = CHARGES.get(charge, "") user = ( - f"IDEA\n{idea.strip()}\n\nCANDIDATES\n" + "\n".join(lines) + "\n\n" - f"OPERATIONS WANTED: {count}" + (" (the second must be atmosphere-level - light, shadow, time, absence - so that both read as one event)" if count >= 2 else "") + "\n" + f"IDEA\n{idea.strip()}\n\n" + + (f"ORDINARY SCENE INVENTORY\n{inventory}\n\n" if inventory else "") + + "CANDIDATES\n" + "\n".join(lines) + "\n\n" + f"OPERATIONS WANTED: {count}" + (" (the second must be atmosphere-level - light, shadow, time, absence - so that both read as one event, " + "and it must be an instance of the same law)" if count >= 2 else "") + "\n" f"STRANGENESS: {strangeness} (subtle = a viewer may need a second look; overt = unmistakable at a glance)\n" f"MEDIUM: {medium}" + (" - an anomaly in TIME is available: reversed, slowed, looped, or sound that belongs to something else" if medium == "video" else "") + "\n" + (f"CHARGE: {charge_text}\n" if charge_text else "") + + (f"SUGGESTED LAW (keep it only if it governs the premise you choose; otherwise write your own): {law}\n" if law else "") ) return CRITIC_SYSTEM, user @@ -169,9 +245,15 @@ def parse_critic(answer: str, candidates: Sequence[Candidate]) -> Tuple[Optional data = _first_json_object(answer) if data is None: return None, ["the critic's answer was not JSON"] + # The critic writes a draft, scores it, then rewrites it with one repair. Either may be missing on a small model. premise = str(data.get("premise") or "").strip() + draft = str(data.get("premise_draft") or "").strip() + if not premise: + premise = draft if not premise: return None, ["the critic returned no premise"] + if draft and violations(premise) and not violations(draft): + premise = draft # the repair introduced a problem the draft did not have problems = violations(premise) if SUBJECT_TOKEN in premise: problems.append(f"left {SUBJECT_TOKEN} unresolved") @@ -198,10 +280,180 @@ def parse_critic(answer: str, candidates: Sequence[Candidate]) -> Tuple[Optional for entry in data.get("judgements") or []: if isinstance(entry, dict): judgements.append(f"{entry.get('n', '?')}: {'keep' if entry.get('keep') else 'reject'} - {str(entry.get('reason', '')).strip()}") + scores, weakest = _scores(data) + visible = str(data.get("visible") or "").strip().strip('"') + # Refusals are checked BEFORE placeholders: "nothing looks different" is the critic telling us the truth, + # not a template it forgot to fill in. + blind = UNRENDERABLE.search(visible) + if visible and (blind or re.match(r"(?i)\s*nothing\b", visible)): + return None, [f"the critic says the anomaly is not visible ('{blind.group(0) if blind else 'nothing'}'); " + "an image model would return an ordinary picture"] + if visible.lower().strip(". ").startswith(("one clause", "what a viewer", "n/a", "none")): + visible = "" restatement = str(data.get("restatement") or "").strip() - if violations(restatement): + if violations(restatement) or lint_render(restatement): restatement = "" - counter_terms = [str(term).strip() for term in (data.get("counter_terms") or []) if str(term).strip()][:5] + counter_terms = _clean_counter_terms(data.get("counter_terms"), premise, str(data.get("subject") or "")) + render, scene, edit, suppress, notes = _clean_staging(data) + if not render and lint_render(premise): + notes.append("the premise is written as a change rather than a result (" + lint_render(premise)[0] + "); the " + "rewriter is told to write the result only") + preserved_scene = re.sub(r"\s+", " ", str(data.get("preserved_scene") or "")).strip().strip('"') + if len(preserved_scene.split()) > MAX_RENDER_WORDS or SUBJECT_TOKEN in preserved_scene: + notes.append("the critic's preserved scene was not used (too long, or left THE SUBJECT unresolved)") + preserved_scene = "" + target = re.sub(r"\s+", " ", str(data.get("target") or "")).strip().strip('"') + replacement = re.sub(r"\s+", " ", str(data.get("replacement") or "")).strip().strip('"') + if len(target.split()) > 12: + target = "" + if len(replacement.split()) > 16: + replacement = "" return Verdict(subject=str(data.get("subject") or "").strip(), chosen_index=chosen, operation=operation, affinity=affinity, - premise=premise, restatement=restatement, title=str(data.get("title") or "").strip().strip('"'), - counter_terms=counter_terms, judgements=judgements), [] + law=_clean_law(data.get("law")), premise=premise, visible=visible, restatement=restatement, + title=_clean_title(data.get("title")), title_strategy=_clean_strategy(data.get("title_strategy")), + counter_terms=counter_terms, judgements=judgements, + scores=scores, weakest=weakest, repair=str(data.get("repair") or "").strip(), + render=render, suppress=suppress, scene=scene, edit=edit, + target=target, replacement=replacement, + preserve=_clean_plan_list(data.get("preserve")), + visual_evidence=_clean_plan_list(data.get("visual_evidence")), + forbidden_interpretations=_clean_plan_list(data.get("forbidden_interpretations")), + composition=_clean_composition(data.get("composition")), + preserved_scene=preserved_scene, + suppress_properties=_clean_plan_list(data.get("suppress_properties")), + notes=notes), [] + + +PLACEHOLDER_STARTS = ("the result only", "the ordinary picture", "one imperative", "one or two", "n/a", "none", "empty") +MAX_RENDER_WORDS = 80 +MAX_EDIT_WORDS = 60 + + +def _clean_staging(data: dict) -> Tuple[str, str, str, List[str], List[str]]: + """(render, scene, edit, suppress, notes). Each part is dropped, with a note, rather than guessed at.""" + notes: List[str] = [] + + def text(key: str) -> str: + value = str(data.get(key) or "").strip().strip('"') + return "" if value.lower().startswith(PLACEHOLDER_STARTS) else value + + render, scene, edit = text("render"), text("scene"), text("edit") + if render: + problems = violations(render) + lint_render(render) + if SUBJECT_TOKEN in render: + problems.append(f"left {SUBJECT_TOKEN} unresolved") + if len(render.split()) > MAX_RENDER_WORDS: + problems.append(f"longer than {MAX_RENDER_WORDS} words") + if problems: + notes.append("the critic's render was not used: " + "; ".join(problems)) + render = "" + if scene and (len(scene.split()) > MAX_RENDER_WORDS or SUBJECT_TOKEN in scene): + notes.append("the critic's ordinary-scene prompt was not used (too long, or left THE SUBJECT unresolved)") + scene = "" + if edit and (len(edit.split()) > MAX_EDIT_WORDS or SUBJECT_TOKEN in edit or violations(edit)): + notes.append("the critic's edit instruction was not used (too long, unresolved, or on the ban-list)") + edit = "" + suppress: List[str] = [] + raw = data.get("suppress") + if isinstance(raw, str): + raw = [part for part in re.split(r"[;,]", raw)] + for term in raw or []: + term = str(term).strip().strip(".") + # Live 2026-10-01 the critic also listed operation names and concepts ("taxonomy_error", "mineralogical + # classification"): only plain object names can be drawn by mistake, so only those are kept. + if (not term or len(term.split()) > 5 or term.lower() in ("none", "n/a") or term in suppress or "_" in term + or term.lower().replace(" ", "_") in OPERATIONS or re.search(r"(?i)\b(classification|operation|law|premise)\b", term)): + continue + suppress.append(term) + return render, scene, edit, suppress[:4], notes + + +def _clean_plan_list(value, limit: int = 8) -> List[str]: + if not isinstance(value, list): + return [] + kept = [] + for entry in value: + text = re.sub(r"\s+", " ", str(entry or "")).strip().strip(".") + if text and len(text.split()) <= 24 and text.lower() not in {item.lower() for item in kept}: + kept.append(text) + return kept[:limit] + + +def _clean_composition(value) -> dict: + if not isinstance(value, dict): + return {} + cleaned = {} + limits = {"anomaly_scale": 6, "placement": 12, "frame_share": 10, "scale_reference": 12} + for key, word_limit in limits.items(): + text = re.sub(r"\s+", " ", str(value.get(key) or "")).strip().strip(".") + if text and len(text.split()) <= word_limit: + cleaned[key] = text + return cleaned + + +def _clean_counter_terms(value, premise: str, subject: str) -> List[str]: + """The ordinary things the image model will substitute for the anomaly - drawn from THIS scene. + + Weaker models copy the worked example out of the instructions instead: a garden premise came back with + "porcelain teacup, ceramic cup, porcelain teacup, ceramic cup, porcelain teacup". A term earns its place + only if it shares a content word with the premise or the subject, and duplicates are dropped.""" + words = set(re.findall(r"[a-z]{3,}", (premise + " " + subject).lower())) + kept, seen = [], set() + for term in (value or []): + text = str(term).strip().strip(".,") + key = text.lower() + if not text or key in seen: + continue + # Related if any word of the term touches any word of the premise, either way round: "cup of tea" + # belongs to a premise about a teacup, "porcelain teacup" does not belong to one about a garden. + parts = re.findall(r"[a-z]{3,}", key) + if words and parts and not any(part in word or word in part for part in parts for word in words): + continue # belongs to some other scene than this one + seen.add(key) + kept.append(text) + return kept[:5] + + +def _clean_law(value) -> str: + """A law is one plain sentence. Reject a placeholder echo, an over-long one, or one that explains itself.""" + law = str(value or "").strip().strip('"') + if not law: + return "" + if law.lower().strip(". ").startswith(("one sentence", "what may change", "the law", "n/a", "none")): + return "" + if len(law.split()) > MAX_LAW_WORDS or violations(law): + return "" + return law + + +def _clean_title(value) -> str: + title = str(value or "").strip().strip('"').strip() + if not title: + return "" + stripped = re.sub(r"^(the|a|an)\s+", "", title.lower()).strip(" .!?") + if stripped in BANNED_TITLES or any(word in stripped for word in ("surreal", "untitled")): + return "" + return title + + +def _clean_strategy(value) -> str: + name = str(value or "").strip().lower().replace(" ", "_").replace("-", "_") + return name if name in TITLE_STRATEGIES else "" + + +def _scores(data: dict) -> Tuple[dict, str]: + """(scores, weakest). Unknown keys and unparseable values are dropped rather than guessed at.""" + found = {} + raw = data.get("scores") + if isinstance(raw, dict): + for name, value in raw.items(): + key = str(name).strip().lower().replace(" ", "_") + if key in RUBRIC: + try: + found[key] = max(0, min(4, int(float(value)))) + except (TypeError, ValueError): + continue + weakest = str(data.get("weakest") or "").strip().lower().replace(" ", "_") + if weakest not in RUBRIC: + weakest = min(found, key=lambda key: found[key]) if found else "" + return found, weakest diff --git a/core/strategies/surreal/engine.py b/core/strategies/surreal/engine.py index c945910..b322727 100644 --- a/core/strategies/surreal/engine.py +++ b/core/strategies/surreal/engine.py @@ -1,5 +1,5 @@ """ -The surreal engine: generator -> critic -> brief. +The surreal engine: ordinary-scene inventory -> generator -> critic -> structured compiler -> brief. engine = SurrealEngine(sources) plan = engine.plan(idea, settings, seed) # candidates + the code critic's choice @@ -20,11 +20,14 @@ import random from dataclasses import dataclass, field from typing import Callable, List, Optional, Tuple +from .compiler import CompositionPlan, StructuredRenderPlan, build_render_plan, compile_render_plan from .critic import Verdict, code_choose, critic_messages from .generators import MODE_GENERATORS, Candidate, Context +from .lint import SUPPRESS_MARKER +from .scene import SceneInventory, inventory_for_critic from .sources import SurrealSources from .vocabulary import ( - AUTO, CHARGES, MODE_WEIGHTS, MODES, NONE, OPERATIONS, REGISTERS, SUBJECT_TOKEN, + AUTO, CHARGES, MODE_WEIGHTS, MODES, NONE, OPERATIONS, REGISTERS, RUBRIC, SUBJECT_TOKEN, law_seed, ) PAINTED_REGISTERS = ("academic oil", "metaphysical", "biomorphic", "collage") @@ -40,6 +43,8 @@ class SurrealSettings: strangeness: str = "balanced" medium: str = "image" # image | video candidates: int = 6 # how many the critic gets to choose from + law: str = "" # the transformation rule, written by hand; blank = drawn from the charge + staging: str = "one_stage" # one_stage | two_stage (two_stage = only operations that can be staged as one edit) @dataclass @@ -51,6 +56,8 @@ class SurrealPlan: candidates: List[Candidate] chosen: List[Candidate] # the code critic's choice; the fallback when there is no LLM verdict charge: str + law: str = "" # the law offered to the critic: the user's, the operation's, or the charge's + inventory: Optional[SceneInventory] = None notes: List[str] = field(default_factory=list) @@ -63,7 +70,14 @@ class SurrealResult: negative_additions: str = "" premises: List[str] = field(default_factory=list) operations: List[str] = field(default_factory=list) + law: str = "" + visible: str = "" ok: bool = True + renders: List[str] = field(default_factory=list) # the premises as the image model reads them (result only) + suppress: List[str] = field(default_factory=list) # what the anomaly replaced: never named in the prompt + scene_prompt: str = "" # two-stage, stage 1: the ordinary picture. Blank when the operation cannot be staged + edit_instruction: str = "" # two-stage, stage 2: one imperative edit that adds the anomaly + render_plan: Optional[StructuredRenderPlan] = None class SurrealEngine: @@ -72,13 +86,15 @@ class SurrealEngine: # ---- generator + code critic --------------------------------------------------------------------- def plan(self, idea: str, settings: SurrealSettings, seed: Optional[int] = None, - distance: Optional[Callable[[str], float]] = None) -> SurrealPlan: + distance: Optional[Callable[[str], float]] = None, + inventory: Optional[SceneInventory] = None) -> SurrealPlan: if seed is None: seed = random.SystemRandom().randrange(0, 0x7FFFFFFF) rng = random.Random(seed) settings.count = 2 if int(settings.count or 1) >= 2 else 1 context = Context(idea=idea or "", rng=rng, sources=self.sources, medium=settings.medium, - strangeness=settings.strangeness, distance=distance) + strangeness=settings.strangeness, distance=distance, + staging=settings.staging if settings.staging in ("one_stage", "two_stage") else "one_stage") notes: List[str] = [] modes = self._modes(settings, rng, notes) @@ -105,14 +121,32 @@ class SurrealEngine: charge = settings.charge if charge == AUTO: charge = rng.choice([name for name in CHARGES if name != NONE]) + + # The law must GOVERN the premise that was chosen, or the instruction to remove whatever disobeys it would + # delete the premise itself. So: the user's own law wins; then a candidate's more specific one; then the law + # of the operation actually chosen. The charge's law is only a last resort, and is offered to the LLM critic + # as a suggestion rather than used as written, because an emotion does not know which anomaly was picked. + law = (settings.law or "").strip() + if not law: + law = next((candidate.law_hint for candidate in chosen if candidate.law_hint), "") + if not law: + law = next((OPERATIONS[candidate.operation].law for candidate in chosen + if candidate.operation in OPERATIONS and OPERATIONS[candidate.operation].law), "") + if not law: + law = law_seed(charge, rng) + notes.append("no operation law applied; the law was drawn for the charge and may not fit the premise") return SurrealPlan(idea=idea or "", settings=settings, seed=seed, modes=modes, candidates=candidates, - chosen=chosen, charge=charge, notes=notes) + chosen=chosen, charge=charge, law=law, inventory=inventory, notes=notes) def _modes(self, settings: SurrealSettings, rng: random.Random, notes: List[str]) -> List[str]: if settings.operation != AUTO: if settings.mode not in (AUTO, "affinity_search"): notes.append(f"operation '{settings.operation}' was chosen, so the affinity search is used instead of mode '{settings.mode}'") return ["affinity_search"] + if settings.mode == AUTO and settings.staging == "two_stage": + # Only object-level operations can be staged as one edit; the procedure mode is kept because the LLM critic + # writes its own edit for whatever it finds. + return ["affinity_search", "problem"] if settings.mode != AUTO and settings.mode in MODES: if settings.mode == "archive_composite" and not (self.sources.has_personal() or self.sources.archive): notes.append("archive composite needs your iconography file or prompt database, and both are empty; using read the noise") @@ -157,7 +191,8 @@ class SurrealEngine: # ---- LLM critic ------------------------------------------------------------------------------------- def critic_messages(self, plan: SurrealPlan) -> Tuple[str, str]: - return critic_messages(plan.idea, plan.candidates, plan.settings.count, plan.charge, plan.settings.strangeness, plan.settings.medium) + return critic_messages(plan.idea, plan.candidates, plan.settings.count, plan.charge, plan.settings.strangeness, + plan.settings.medium, law=plan.law, inventory=inventory_for_critic(plan.inventory)) # ---- brief --------------------------------------------------------------------------------------------- def finish(self, plan: SurrealPlan, verdict: Optional[Verdict] = None, critic_problems: Optional[List[str]] = None) -> SurrealResult: @@ -167,21 +202,46 @@ class SurrealEngine: notes.append("LLM critic not used: " + "; ".join(critic_problems) + " - the code critic's choice stands") procedures: List[str] = [] + suppress: List[str] = [] + edit, scene = "", "" + selected_candidate: Optional[Candidate] = None if verdict is not None: premises = [verdict.premise] + renders = [verdict.render or verdict.premise] + suppress = list(verdict.suppress) operations = [verdict.operation] affinity, restatement, title = verdict.affinity, verdict.restatement, verdict.title + visible = verdict.visible register_hint = plan.candidates[verdict.chosen_index].register_hint if verdict.chosen_index >= 0 else "" + selected_candidate = plan.candidates[verdict.chosen_index] if verdict.chosen_index >= 0 else None + notes.extend(verdict.notes) + operation = OPERATIONS.get(verdict.operation) + if verdict.edit and (operation is None or operation.impossible): + edit = verdict.edit + scene = verdict.scene or plan.idea.strip() else: if not plan.chosen: return SurrealResult(text=plan.idea, instructions="", ok=False, why="No acceptable candidate was produced; the idea is passed through unchanged.\n" + "\n".join(notes)) premises = [candidate.premise for candidate in plan.chosen if candidate.premise] + renders = [candidate.rendered() for candidate in plan.chosen if candidate.premise] + for candidate in plan.chosen: + suppress.extend(term for term in candidate.suppress if term not in suppress) procedures = [candidate.procedure for candidate in plan.chosen if candidate.procedure and not candidate.premise] operations = [candidate.operation for candidate in plan.chosen] affinity = plan.chosen[0].affinity restatement, title = "", "" + visible = "" register_hint = plan.chosen[0].register_hint + selected_candidate = plan.chosen[0] + if plan.chosen[0].edit: + edit = plan.chosen[0].edit + scene = plan.idea.strip() + if edit and len(operations) > 1: + notes.append("two-stage: the edit carries the first operation only; the second is in the one-stage prompt") + if settings.staging == "two_stage" and not edit: + notes.append(f"two-stage asked for, but '{', '.join(operations)}' cannot be applied as one edit; use the " + "one-stage text") register = settings.register if settings.register in REGISTERS else "deadpan photographic" if register_hint and register_hint != register: @@ -190,13 +250,83 @@ class SurrealEngine: notes.append(f"register '{register}' -> '{register_hint}': the chosen operation needs it") register = register_hint impossible = any(OPERATIONS[name].impossible for name in operations if name in OPERATIONS) + # The critic's law beats the one the plan offered it: the critic saw the premise it actually chose. + law = (verdict.law if verdict is not None and verdict.law else "") or plan.law + if settings.law.strip(): + law = settings.law.strip() # a hand-written law is never overridden - text = self._text(plan.idea, premises, unresolved=verdict is None) - instructions = self._instructions(premises, procedures, restatement, register, plan.charge, impossible, settings, unresolved=verdict is None) - why = self._why(plan, verdict, operations, affinity, register, notes) + how_seen = next((OPERATIONS[name].visible for name in operations if name in OPERATIONS and OPERATIONS[name].visible), "") + + render_plan = None + compiled_warnings: List[str] = [] + if renders: + candidate = selected_candidate + composition_data = verdict.composition if verdict is not None else {} + composition = CompositionPlan( + anomaly_scale=str(composition_data.get("anomaly_scale") or "dominant"), + placement=str(composition_data.get("placement") or "central"), + frame_share=str(composition_data.get("frame_share") or "at least one third of the frame"), + scale_reference=str(composition_data.get("scale_reference") or (candidate.scale_reference if candidate else "")), + ) + evidence: List[str] = [] + if verdict is not None: + evidence.extend(verdict.visual_evidence) + if candidate is not None: + evidence.extend(candidate.evidence) + if visible: + evidence.append(visible) + render_plan = build_render_plan( + operation=operations[0] if operations else "", + anomaly=" ".join(renders), + idea=plan.idea, + inventory=plan.inventory, + preserved_scene=verdict.preserved_scene if verdict is not None else "", + target=(verdict.target if verdict is not None else "") or (candidate.target if candidate else ""), + replacement=(verdict.replacement if verdict is not None else "") or (candidate.replacement if candidate else ""), + preserved_properties=(verdict.preserve if verdict is not None else []) or (candidate.preserve if candidate else []), + suppress_entities=suppress, + suppress_properties=verdict.suppress_properties if verdict is not None else [], + visual_evidence=evidence, + forbidden_interpretations=(verdict.forbidden_interpretations if verdict is not None else []) + + (candidate.forbidden if candidate else []), + composition=composition, + law=law, + edit_instruction=edit, + ) + compiled = compile_render_plan(render_plan) + compiled_warnings.extend(compiled.warnings) + if not compiled.ok: + return SurrealResult( + text=plan.idea, + instructions="", + why=("Structured surreal compiler refused the selected premise: " + + "; ".join(compiled.warnings) + + ". The original idea was passed through unchanged."), + ok=False, + premises=premises, + operations=operations, + law=law, + renders=renders, + suppress=suppress, + scene_prompt=scene, + edit_instruction=edit, + render_plan=render_plan, + ) + text = compiled.text + else: + text = self._text(plan.idea, renders, unresolved=verdict is None) + notes.extend(compiled_warnings) + instructions = self._instructions(renders, procedures, restatement, register, plan.charge, impossible, settings, + law=law, visible=visible, how_seen=how_seen, unresolved=verdict is None, + suppress=suppress, render_plan=render_plan) + why = self._why(plan, verdict, operations, affinity, register, law, visible, how_seen, notes, + renders=renders, premises=premises, suppress=suppress, scene=scene, edit=edit, + render_plan=render_plan) negative = ", ".join(verdict.counter_terms) if verdict is not None else "" return SurrealResult(text=text, instructions=instructions, why=why, title=title, negative_additions=negative, - premises=premises, operations=operations) + premises=premises, operations=operations, law=law, visible=visible or how_seen, + renders=renders, suppress=suppress, scene_prompt=scene, edit_instruction=edit, + render_plan=render_plan) @staticmethod def _text(idea: str, premises: List[str], unresolved: bool) -> str: @@ -208,24 +338,67 @@ class SurrealEngine: @staticmethod def _instructions(premises: List[str], procedures: List[str], restatement: str, register: str, charge: str, - impossible: bool, settings: SurrealSettings, unresolved: bool) -> str: + impossible: bool, settings: SurrealSettings, law: str, visible: str, how_seen: str, + unresolved: bool, suppress: Optional[List[str]] = None, + render_plan: Optional[StructuredRenderPlan] = None) -> str: + """`premises` here are the RENDERED forms: what the camera sees, never the substitution (lint.py).""" lines = ["SURREAL PREMISE (a deliberate strategy for this prompt; image models drift toward the plausible, so protect it):"] if premises: lines.append("Concepts - keep the MEANING of each, but express it in the wording that works best for the target model:") lines.extend(f" - {premise}" for premise in premises) + if render_plan is not None: + lines.append("STRUCTURED RENDER PLAN - this is already decided. Verbalize it; do not reinterpret it or restore the ordinary source:") + lines.append(f" operation: {render_plan.operation}") + if render_plan.target: + lines.append(f" target: {render_plan.target}") + if render_plan.replacement: + lines.append(f" replacement/material/behaviour: {render_plan.replacement}") + if render_plan.preserved_properties: + lines.append(" preserve: " + "; ".join(render_plan.preserved_properties)) + if render_plan.visual_evidence: + lines.append(" required visual proof: " + " | ".join(render_plan.visual_evidence)) + if render_plan.forbidden_interpretations: + lines.append(" reject these ordinary readings: " + " | ".join(render_plan.forbidden_interpretations)) + lines.append( + f" composition: {render_plan.composition.anomaly_scale}; {render_plan.composition.placement}; " + f"{render_plan.composition.frame_share}" + ) + if suppress: + lines.append(f" {SUPPRESS_MARKER} {'; '.join(suppress)}") + lines.append(" (Naming it makes the image model draw it. Describe the spot it would occupy by what surrounds it - " + "the surface, the hand, the shelf - and never by its name.)") if procedures: lines.append("Do this first, silently. Its one-sentence result is the premise of this prompt: keep its meaning through every rewording.") lines.extend(f" * {procedure}" for procedure in procedures) + if law: + lines.append("The law - the picture follows this rule consistently, and it is what makes the anomaly read as " + "inevitable rather than random:") + lines.append(f" {law}") + lines.append(" Show the rule in effect; never state it, name it or explain it. Anything in the picture that does " + "not obey it is removed.") + lines.append("What the viewer must SEE - an image model can only render what is present in the frame, never an " + "absence, a negation or a cause:") + if visible: + lines.append(f" {visible}") + if how_seen: + lines.append(f" {how_seen}") lines.append("How to write it:") if unresolved and any(SUBJECT_TOKEN in text for text in premises + procedures): lines.append(f" * {SUBJECT_TOKEN} means the main subject of the user's idea. Name it; never write the words '{SUBJECT_TOKEN}'.") if impossible: - lines.append(" * State the anomaly early, as plain physical fact in the present tense, the way a report would. " - "Then restate it once, later, in different words" + (f" (for example: {restatement})" if restatement else "") + - ", so the image model cannot normalise it away.") + lines.append(" * The anomaly is the FIRST thing in the prompt, stated as plain physical fact in the present tense: what " + "the camera sees, never what it replaced, never 'should be', 'instead of', 'in place of' or 'as if'. " + "Then give it a second visible consequence later on - its shadow, its weight on what holds it, its " + "texture up close" + (f" (the same fact in other words: {restatement})" if restatement else "") + + " - so the image model cannot normalise it away.") + lines.append(" * Make it large in the frame and central, lit like everything else. Deadpan is not the same as small: " + "a viewer must see it in a thumbnail at a glance.") lines.append(" * " + ("Exactly these anomalies and no others." if len(premises) + len(procedures) > 1 else "Exactly ONE anomaly.") + " Everything else is rigorously ordinary: a real place, real light with a stated source and direction, " - "real materials, consistent perspective and shadows. Nobody in the picture reacts to the anomaly.") + "real materials, consistent perspective and shadows. People in the picture carry on with ordinary " + "actions; never write that nobody notices or that it goes unnoticed - that has no pixels.") + lines.append(" * Spend the words on the anomaly: it and its evidence take at least a third of the prompt. Leave out " + "camera-body, lens and film-stock detail unless the register needs it.") else: lines.append(" * NOTHING impossible appears in this picture. Do not add an anomaly. The strangeness is entirely in the hour, " "the emptiness, the stillness, the process, and in a story that is not told.") @@ -234,6 +407,15 @@ class SurrealEngine: if charge_text: lines.append(f" * What it serves: {charge_text}. Carry it through light, distance and posture - never name the emotion.") lines.append(" * Pose the question and refuse the answer: no explanation, no symbolism, no moral, no 'as if', no caption.") + lines.append(" * Add NO further symbolic objects. Every noun you add competes with the premise; a long inventory of " + "dreamlike things is the commonest way to lose it. If a detail is not the anchor, the anomaly, the law " + "or the light, leave it out.") + if impossible: + lines.append(" * Order it this way: the impossible fact, stated flatly as what is seen; then the ordinary place and what " + "the subject is doing, briefly; then light and surface. Nothing else.") + else: + lines.append(" * Order it this way: the ordinary place and what the subject is doing; then the hour, the light and the " + "one detail that carries the strangeness; then camera and surface. Nothing else.") lines.append(" * Leave out the cliches of the genre (melting clocks, floating islands, glow, mist) and the words surreal, " "dreamlike, ethereal, whimsical, otherworldly. Do not write 'no X' to exclude anything: an image model reads " "'no mist' as mist. Simply leave things out.") @@ -246,9 +428,23 @@ class SurrealEngine: return "\n".join(lines) @staticmethod - def _why(plan: SurrealPlan, verdict: Optional[Verdict], operations: List[str], affinity: str, register: str, notes: List[str]) -> str: + def _why(plan: SurrealPlan, verdict: Optional[Verdict], operations: List[str], affinity: str, register: str, + law: str, visible: str, how_seen: str, notes: List[str], renders: Optional[List[str]] = None, + premises: Optional[List[str]] = None, suppress: Optional[List[str]] = None, scene: str = "", + edit: str = "", render_plan: Optional[StructuredRenderPlan] = None) -> str: lines = [f"seed {plan.seed} | modes: {', '.join(plan.modes)} | critic: {'LLM' if verdict is not None else 'code'}" - f" | register: {register} | charge: {plan.charge}"] + f" | register: {register} | charge: {plan.charge} | staging: {plan.settings.staging}"] + if renders and premises and renders != premises: + lines.append("rendered as: " + " | ".join(renders)) + if suppress: + lines.append("never named in the prompt: " + ", ".join(suppress)) + if edit: + lines.append(f"two-stage: scene = {scene[:160]}") + lines.append(f"two-stage: edit = {edit}") + if plan.inventory is not None: + lines.append("ordinary inventory: " + plan.inventory.summary()) + if render_plan is not None: + lines.extend(render_plan.report_lines()) for name in operations: operation = OPERATIONS.get(name) if operation is not None: @@ -257,8 +453,30 @@ class SurrealEngine: lines.append(f"subject: {verdict.subject}") if affinity: lines.append(f"affinity: {affinity}") + if law: + if plan.settings.law.strip(): + source = "yours" + elif verdict is not None and verdict.law: + source = "the critic's" + elif any(law in (candidate.law_hint, OPERATIONS.get(candidate.operation, OPERATIONS["displacement"]).law) + for candidate in plan.chosen): + source = "the operation's" + else: + source = f"drawn for '{plan.charge}'" + lines.append(f"law ({source}): {law}") + if visible: + lines.append(f"visible (the critic's): {visible}") + elif how_seen: + lines.append(f"visible (the operation's): {how_seen}") + if verdict is not None and verdict.scores: + ordered = [f"{name} {verdict.scores[name]}" for name in RUBRIC if name in verdict.scores] + lines.append(f"rubric (0-4): {', '.join(ordered)} total {sum(verdict.scores.values())}/{4 * len(verdict.scores)}") + if verdict.weakest: + lines.append(f"revised for the weakest dimension ({verdict.weakest}): {verdict.repair or 'no repair stated'}") + elif verdict is not None: + lines.append("rubric: the critic returned no scores") if verdict is not None and verdict.title: - lines.append(f"title: {verdict.title}") + lines.append(f"title ({verdict.title_strategy or 'strategy not named'}): {verdict.title}") lines.append("candidates (* = chosen):") chosen_indexes = {verdict.chosen_index} if verdict is not None else {plan.candidates.index(c) for c in plan.chosen if c in plan.candidates} for number, candidate in enumerate(plan.candidates): diff --git a/core/strategies/surreal/generators.py b/core/strategies/surreal/generators.py index 04dd5bf..89f9a4c 100644 --- a/core/strategies/surreal/generators.py +++ b/core/strategies/surreal/generators.py @@ -18,8 +18,13 @@ import random from dataclasses import dataclass, field from typing import Callable, Dict, List, Optional, Sequence -from .sources import Item, SurrealSources, bare, mentioned -from .vocabulary import OPERATION_WEIGHTS, SUBJECT_TOKEN +from .sources import Item, SurrealSources, bare, head_noun, mentioned +from .vocabulary import OPERATION_WEIGHTS, OPERATIONS, SUBJECT_TOKEN + + +def _cap(text: str) -> str: + """Capitalise the first letter only (str.capitalize would lower-case the rest, names included).""" + return text[:1].upper() + text[1:] if text else text @dataclass @@ -33,6 +38,23 @@ class Candidate: score: float = 1.0 personal: bool = False # uses the user's own iconography register_hint: str = "" # a mode may insist on a register (process estrangement is photographic) + law_hint: str = "" # a law MORE specific than the operation's own; usually blank, and then + # OPERATIONS[operation].law is used instead + # What the IMAGE MODEL is given (lint.py). `premise` is the idea in the critic's terms and may describe a substitution + # ("where the teacup should be"); `render` describes only the result the camera sees. Blank = the premise is already + # written that way. + render: str = "" + edit: str = "" # one imperative edit applied to an ORDINARY picture (two-stage); blank = not stageable + suppress: List[str] = field(default_factory=list) # things the anomaly replaced: never named in the prompt + target: str = "" # entity whose object, material, scale or behaviour changes + replacement: str = "" # replacement object/material/behaviour, when the operation has one + preserve: List[str] = field(default_factory=list) # properties that must survive the transformation + evidence: List[str] = field(default_factory=list) # positive visible facts proving the anomaly is present + forbidden: List[str] = field(default_factory=list) # likely ordinary readings the compiler must not permit + scale_reference: str = "" # known-size entity used to make a scale violation measurable + + def rendered(self) -> str: + return self.render or self.premise def line(self) -> str: body = self.premise or self.procedure @@ -48,6 +70,7 @@ class Context: medium: str = "image" # image | video strangeness: str = "balanced" distance: Optional[Callable[[str], float]] = None # 0..1, how far a phrase is from the idea (embedding); None = unknown + staging: str = "one_stage" # two_stage = only operations that can be staged as one edit def pick(self, items: Sequence, count: int = 1) -> List: pool = list(items) @@ -110,6 +133,18 @@ def substitution(context: Context, targets: Sequence[Item]) -> List[Candidate]: "affinity_search", "displacement", premise=(f"Where the {bare(target.text)} should be there is {partner.text} instead, the same size, in the same " f"position, and everything around it treats it as the {bare(target.text)}."), + # The result only. The replaced object is named once, as a location, and listed in `suppress` so the + # rewriter describes its spot by what surrounds it and never names it. + render=(f"{_cap(partner.text)} occupies the central object position beside the main subject, at the replaced " + f"object's scale, in full view and sharply lit, casting its own shadow."), + edit=(f"Replace the {bare(target.text)} with {partner.text}, at the same size and in the same position, matching " + f"the scene's light and shadows. Keep everything else unchanged."), + suppress=[bare(target.text), head_noun(target.text)], + target=bare(target.text), + replacement=partner.text, + preserve=["position", "scale", "surrounding light", "all unrelated scene details"], + evidence=[f"{partner.text} is in full view", f"{partner.text} casts its own shadow on the surrounding scene"], + forbidden=[f"{partner.text} beside the {bare(target.text)}", f"both {partner.text} and the {bare(target.text)} visible"], affinity=clause, ingredients=[target.text, partner.text], personal=partner.personal, score=1.0 + 0.5 * len(target.form & partner.form) + context.far(partner.text) + (1.0 if partner.personal else 0.0), )) @@ -127,6 +162,13 @@ def hybridization(context: Context, targets: Sequence[Item]) -> List[Candidate]: "affinity_search", "hybridization", premise=(f"{target.text.capitalize()} and {partner.text} are a single object: it begins as the {bare(target.text)} " f"and, where their shapes coincide, continues as the {bare(partner.text)}, with no join to be found."), + edit=(f"Turn the {bare(target.text)} into a single object that continues seamlessly as {partner.text} where their " + f"shapes meet, in the same light and position. Keep everything else unchanged."), + target=bare(target.text), + replacement=partner.text, + preserve=["recognisable form of both objects", "position", "light"], + evidence=["both original forms remain recognisable", "the transition is continuous with no visible seam"], + forbidden=["two separate objects touching", "a collage seam between the forms"], affinity=clause, ingredients=[target.text, partner.text], personal=partner.personal or target.personal, score=1.0 + 0.5 * len(target.form & partner.form) + context.far(partner.text), )) @@ -149,10 +191,22 @@ def material_substitution(context: Context, targets: Sequence[Item]) -> List[Can contrast = target is not None and (hard_target == is_soft) subject = f"The {bare(target.text)}" if target is not None else SUBJECT_TOKEN touch = "soft" if is_soft else "hard" + noun = bare(target.text) if target is not None else "main subject" candidates.append(Candidate( "affinity_search", "material_substitution", premise=(f"{subject} is made entirely of {material.text}: every detail of its form is exact and only the " f"substance is wrong."), + # One fused noun phrase plus surface evidence: "made entirely of X, only the substance is wrong" is an + # abstraction; the texture on every surface is what the model can draw. + render=(f"{subject}, made of {material.text}: {material.text} covers every surface of it up close, every edge " + f"and detail of its shape intact, and it sits under its own weight as {material.text} would."), + edit=(f"Make the {noun} out of {material.text}, keeping its exact shape, size and position, with the " + f"{material.text} texture clearly visible. Keep everything else unchanged."), + target=noun, + replacement=material.text, + preserve=["exact shape", "size", "position", "edges and defining details"], + evidence=[f"{material.text} covers every surface", f"the object bears the weight and surface behaviour of {material.text}"], + forbidden=[f"{material.text} placed beside the {noun}", "a surface filter with no physical material behaviour"], affinity=(f"a thing meant to be {'hard' if is_soft else 'soft'} to the touch, made {touch}" if contrast or target is None else "right form, wrong substance"), ingredients=[target.text if target else SUBJECT_TOKEN, material.text], @@ -187,6 +241,8 @@ def inversion(context: Context, targets: Sequence[Item]) -> List[Candidate]: "affinity_search", "inversion", premise=(f"Inside the {bare(container.text)} is {vast}: not a picture of it but the thing itself, at full scale, " f"seen through the opening, its own light falling out onto the surroundings."), + edit=(f"Fill the inside of the {bare(container.text)} with {vast}, seen through its opening at full depth, its " + f"light spilling onto the surroundings. Keep everything else unchanged."), affinity="what holds the small now holds the vast", ingredients=[container.text, vast], personal=container.personal, score=1.5 + (0.5 if container in targets else 0.0), )) @@ -205,10 +261,17 @@ def scale_violation(context: Context, targets: Sequence[Item]) -> List[Candidate small = target is not None and target.scale in ("furniture", "large") direction = "small" if small or (target is None and context.rng.random() < 0.4) else "large" subject = f"The {bare(target.text)}" if target is not None else SUBJECT_TOKEN + noun = bare(target.text) if target is not None else "main subject" for phrase in context.pick(SCALE_PHRASES[direction], 1): candidates.append(Candidate( "affinity_search", "scale_violation", premise=f"{subject} is {phrase}; everything else in the scene is its ordinary size, and the light and perspective are consistent.", + edit=f"Make the {noun} {phrase}, keeping everything else at its normal size and unchanged.", + target=noun, + preserve=["identity", "material", "position", "all other objects at ordinary scale"], + evidence=["the wrongly sized subject touches or occludes its ordinary surroundings", "known-size objects make the scale measurable"], + forbidden=["everything enlarged together", "an ordinary-sized subject described only as enormous"], + scale_reference="the ordinary surroundings", affinity="one thing at the wrong size, everything else correct", ingredients=[target.text if target else SUBJECT_TOKEN, phrase], score=1.2 + (0.5 if target is not None else 0.0), )) @@ -216,18 +279,29 @@ def scale_violation(context: Context, targets: Sequence[Item]) -> List[Candidate # Placements are relative to THE SUBJECT, never to furniture the scene may not have (a bus stop has no chair). +# "in place of whatever it was holding" is gone: it is a substitution sentence (lint.py), and it rendered as a man +# simply holding a stone. PLACEMENTS = ["on the ground at THE SUBJECT's feet", "held in THE SUBJECT's arms the way one holds a child", "exactly where THE SUBJECT is looking", "beside THE SUBJECT at the height of its head, resting on nothing visible", "in the exact centre of the scene, in THE SUBJECT's way", - "in THE SUBJECT's hands, in place of whatever it was holding"] + "held up in both of THE SUBJECT's hands, at chest height, filling the space between them"] def displacement(context: Context, targets: Sequence[Item]) -> List[Candidate]: candidates = [] for thing in context.pick_objects(context.sources.objects(), 3): placement = context.pick(PLACEMENTS)[0] + # "Nobody pays it any attention" is an attitude with no pixels (lint.py); the instruction block already keeps the + # people ordinary. The thing is made large in the frame instead, so it reads at a glance. candidates.append(Candidate( "affinity_search", "displacement", premise=f"{thing.text.capitalize()} is {placement}; it has no business being here, it is perfectly ordinary in itself, and nobody pays it any attention.", + render=f"{_cap(thing.text)} is {placement}, large in the frame, in full view and sharply lit, casting its own shadow.", + edit=(f"Add {thing.text} {placement.replace(SUBJECT_TOKEN, 'the main subject')}, at a true scale and matching the " + f"scene's light and shadows. Keep everything else unchanged."), + replacement=thing.text, + preserve=["true scale", "scene light", "scene shadows", "all existing objects"], + evidence=["the out-of-place object is large in the frame", "it casts its own shadow"], + forbidden=["the object reduced to a background detail", "the object shown as a picture or reflection"], affinity="", ingredients=[thing.text, placement], personal=thing.personal, score=0.8 + context.far(thing.text) + (1.0 if thing.personal else 0.0), )) @@ -235,9 +309,10 @@ def displacement(context: Context, targets: Sequence[Item]) -> List[Candidate]: # ---- operations that come from a list and act on THE SUBJECT --------------------------------------- -def _from_list(context: Context, list_name: str, operation: str, affinity: str, count: int = 2, score: float = 1.2) -> List[Candidate]: +def _from_list(context: Context, list_name: str, operation: str, affinity: str, count: int = 2, score: float = 1.2, + law_hint: str = "") -> List[Candidate]: return [Candidate("affinity_search", operation, premise=item.text[0].upper() + item.text[1:] + ".", affinity=affinity, - ingredients=[item.text], score=score) + ingredients=[item.text], score=score, law_hint=law_hint) for item in context.pick(context.sources.items(list_name), count)] @@ -267,10 +342,14 @@ def multiplication(context: Context, targets: Sequence[Item]) -> List[Candidate] number = context.pick(["three", "five", "seven", "twelve", "several dozen"])[0] target = targets[0] if targets else None subject = f"The {bare(target.text)}" if target is not None else SUBJECT_TOKEN + noun = bare(target.text) if target is not None else "main subject" return [Candidate("affinity_search", "multiplication", premise=(f"{subject} appears {number} times in the scene, identical in every detail including pose and wear, " f"evenly spaced, each casting its own shadow."), - affinity="one thing too many times, identical", ingredients=[number], score=1.1)] + edit=(f"Repeat the {noun} {number} times across the scene, every copy identical in pose and wear, evenly " + f"spaced, each with its own shadow. Keep everything else unchanged."), + affinity="one thing too many times, identical", ingredients=[number], score=1.1, + )] LIVING_ACTS = ["breathes slowly, its sides rising and falling", "has turned to face THE SUBJECT", "sleeps curled on the floor like an animal", @@ -290,6 +369,8 @@ def animation_petrification(context: Context, targets: Sequence[Item]) -> List[C candidates.append(Candidate("affinity_search", "animation_petrification", premise=(f"{SUBJECT_TOKEN} is a statue of {material}, caught mid-gesture with every fold and hair carved; " f"the scene around it is living and ordinary."), + edit=(f"Turn the main subject into a statue of {material} in exactly the same pose, every fold " + f"and hair carved, tool marks visible. Keep everything else unchanged."), affinity="the living becomes object", ingredients=[material], score=1.2)) return candidates @@ -308,6 +389,73 @@ def metaphysical_space(context: Context, targets: Sequence[Item]) -> List[Candid register_hint="metaphysical")] +# ---- the three operations taken from the transformation atlas --------------------------------------- +def autonomy(context: Context, targets: Sequence[Item]) -> List[Candidate]: + """A shadow, reflection or garment acts on its own. Distinct from animation: the THING is not alive, its + double is. The law writes itself, which is why this one is allowed as a second operation.""" + return _from_list(context, "autonomy", "autonomy", "the double acts, the thing does not", score=1.4) + + +def causal_reversal(context: Context, targets: Sequence[Item]) -> List[Candidate]: + """The effect is present and its cause is absent or later. Reads in a still as evidence without an event.""" + return _from_list(context, "reversals", "causal_reversal", "the consequence is here and the cause is not", score=1.4) + + +SPECIMEN_MOUNTS = [ + "pinned in a shallow glazed case on a bed of cotton wool, with a printed label and a catalogue number", + "mounted on a brass stand under a glass dome, with an engraved plate at its base", + "laid on a white tray beside a paper scale bar and a folded index card, lit flat from above", + "bolted to a low plinth in a bare gallery, with a wall label at hip height", + "pressed flat between two sheets of glass in a herbarium folder, annotated in ink", +] +DISCIPLINES = ["botanical", "zoological", "mineralogical", "anatomical", "archaeological", "entomological", "ornithological"] + + +def taxonomy_error(context: Context, targets: Sequence[Item]) -> List[Candidate]: + """The thing is unaltered; only its classification is wrong. Nothing impossible is depicted in the object - + the impossibility is in the institution that catalogued it.""" + candidates = [] + target = targets[0] if targets else None + subject = f"The {bare(target.text)}" if target is not None else SUBJECT_TOKEN + for mount, discipline in zip(context.pick(SPECIMEN_MOUNTS, 2), context.pick(DISCIPLINES, 2)): + candidates.append(Candidate( + "affinity_search", "taxonomy_error", + premise=(f"{subject} is {mount}, catalogued as a {discipline} specimen and given a two-part Latin name. " + f"{subject} itself is entirely unaltered, and the display is meticulous and long established."), + affinity="an institution has classified it, and the classification is from the wrong kingdom", + ingredients=[mount, discipline], personal=bool(target is not None and target.personal), score=1.3, + )) + return candidates + + +def behavior_transfer(context: Context, targets: Sequence[Item]) -> List[Candidate]: + """Keep the subject's identity and substance, but give it the physical behaviour of a distant material. + + The LLM critic resolves the borrowed behaviour because the useful source depends on the actual subject: + ocean/fabric, shadow/paper, stone/wet cloth. The structured compiler then requires evidence of both the + preserved identity and the borrowed behaviour. + """ + target = targets[0] if targets else None + subject = f"the {bare(target.text)}" if target is not None else SUBJECT_TOKEN + noun = bare(target.text) if target is not None else "main subject" + return [Candidate( + "affinity_search", "behavior_transfer", + procedure=( + f"Keep {subject} visibly and materially itself. Choose ONE distant familiar material whose physical behaviour has " + f"a true formal affinity with it - a shared surface, fold, flow, tension, edge or repeated shape. Give {subject} that " + "behaviour without turning it into the borrowed material. State the finished visible result, then give at least two " + "pieces of physical proof: one proving the original identity remains and one proving the borrowed behaviour." + ), + affinity="the identity stays; only a physically legible behaviour crosses between distant materials", + ingredients=[target.text if target else SUBJECT_TOKEN], + target=noun, + preserve=["identity", "material", "connection to the surrounding scene"], + evidence=["surface evidence proves the original material remains", "geometry and contact prove the borrowed behaviour"], + forbidden=["a literal prop made from the borrowed material", "a small ordinary action instead of a changed physical law"], + score=1.5 + (0.5 if target is not None else 0.0), + )] + + OPERATION_GENERATORS: Dict[str, Callable[[Context, Sequence[Item]], List[Candidate]]] = { "displacement": lambda context, targets: substitution(context, targets) or displacement(context, targets), "scale_violation": scale_violation, @@ -321,10 +469,15 @@ OPERATION_GENERATORS: Dict[str, Callable[[Context, Sequence[Item]], List[Candida "literalized_language": literalized_language, "multiplication": multiplication, "temporal": temporal, + "autonomy": autonomy, + "causal_reversal": causal_reversal, + "taxonomy_error": taxonomy_error, + "behavior_transfer": behavior_transfer, } # Atmosphere-level operations: the only ones allowed as a SECOND operation, because they can read as part of # the same event as an object-level one (a stone bird in flight / over a day sky above a night street). -SECOND_OPERATIONS = ("physics_negation", "metaphysical_space", "absence", "temporal") +# Autonomy qualifies: a shadow behaving on its own is an ECHO of an object-level anomaly, not a rival to it. +SECOND_OPERATIONS = ("physics_negation", "metaphysical_space", "absence", "temporal", "autonomy") # ---- the eight modes --------------------------------------------------------------------------------- @@ -334,6 +487,8 @@ def affinity_search(context: Context, operation: str = "auto") -> List[Candidate return OPERATION_GENERATORS[operation](context, targets) weights = OPERATION_WEIGHTS.get(context.strangeness, OPERATION_WEIGHTS["balanced"]) names = [name for name in OPERATION_GENERATORS if weights.get(name, 0) > 0] + if context.staging == "two_stage": + names = [name for name in names if name in OPERATIONS and OPERATIONS[name].two_stage] chosen: List[str] = [] pool = list(names) while pool and len(chosen) < 5: @@ -418,7 +573,7 @@ def working_machine(context: Context, operation: str = "auto") -> List[Candidate return [Candidate( "working_machine", "working_machine", premise=(f"{SUBJECT_TOKEN} operates {mechanism}. It is fed with {feedstock} and it produces {product}. Every linkage between " - f"what goes in and what comes out is visible, and each step looks as if it would work."), + f"what goes in and what comes out is visible, and each step is a plausible mechanical linkage."), affinity="a causal chain that convinces before it is noticed to be impossible", ingredients=[mechanism, feedstock, product], score=1.4, )] @@ -447,19 +602,49 @@ def archive_composite(context: Context, operation: str = "auto") -> List[Candida )] +FOUND_STRANGE_GROUPS = ("emptiness", "stillness", "light", "trace") +# For a still image, light and trace put something in the frame a viewer can point at; stillness is posture, +# which a text-to-image model renders weakly and which reads as an ordinary photograph. Video can carry +# stillness, because holding a pose over time is itself visible. +FOUND_STRANGE_IMAGE_GROUPS = ("light", "trace", "light", "trace", "emptiness", "stillness") + + def found_strange(context: Context, operation: str = "auto") -> List[Candidate]: - groups = context.pick(["emptiness", "stillness", "light", "trace"], 2) - hour = context.pick(context.sources.items("hour"), 1) - parts = [item.text for name in groups for item in context.pick(context.sources.items(name), 1)] - if not hour or not parts: + """ONE hour and ONE condition. The old version joined the hour to two conditions drawn independently, + which averaged four and a half unrelated facts per premise - our own generator making the prompt soup + the method exists to avoid - and paired a lit condition with a daylight hour in a third of draws. The + critic, handed two facts that cannot both be true, would invent the relation joining them. + + Two hours x three groups gives the critic six genuinely different candidates instead of two near-copies.""" + hours = context.sources.items("hour") + if not hours: return [] - facts = [hour[0].text] + parts - return [Candidate( - "found_strange", "found_strange", - premise="Nothing impossible happens. " + " ".join(fact[0].upper() + fact[1:] + "." for fact in facts), - affinity="the strange is found, not made; the story is withheld", ingredients=facts, score=1.4, - register_hint="staged photography", - )] + candidates: List[Candidate] = [] + for hour in context.pick(hours, 2): + lit = "dark" in hour.tags # artificial light only reads at a dark hour + pool = FOUND_STRANGE_IMAGE_GROUPS if context.medium == "image" else FOUND_STRANGE_GROUPS + seen_groups: List[str] = [] + for group in context.pick(pool, 4): + if group in seen_groups: + continue + seen_groups.append(group) + if len(seen_groups) > 3: + break + pool = [item for item in context.sources.items(group) if lit or "needs_dark" not in item.tags] + picked = context.pick(pool, 1) + if not picked: + continue + fact = picked[0].text + candidates.append(Candidate( + "found_strange", "found_strange", + # No closing "everything else is ordinary" sentence: the instruction block already says so, + # and a sentence of instruction inside the premise is a sentence the rewriter can transcribe. + premise=f"Nothing impossible happens. It is {hour.text}. {fact[0].upper() + fact[1:]}.", + affinity="the strange is found, not made; the story is withheld", + ingredients=[hour.text, fact], score=1.4 + (0.2 if group in ("light", "trace") else 0.0), + register_hint="staged photography", + )) + return candidates def process_estrangement(context: Context, operation: str = "auto") -> List[Candidate]: diff --git a/core/strategies/surreal/lint.py b/core/strategies/surreal/lint.py new file mode 100644 index 0000000..1744d3c --- /dev/null +++ b/core/strategies/surreal/lint.py @@ -0,0 +1,128 @@ +""" +The renderability linter: grammar a text-to-image model cannot draw. + +An image model's text encoder keeps nouns and attributes and loses roles, counterfactuals and negation. So a +sentence that describes a SUBSTITUTION ("where the teacup should be there are boots instead") hands the encoder +both objects, and the picture comes back with the teacup. A sentence that claims an ATTITUDE ("nobody notices", +"unnoticed by those around him") has no pixels at all; it only spends tokens. The live run of 2026-09-21 shows +both: a teacup premise rendered with an extra coffee mug and a "ghostly" teacup, a stone premise rendered as a +man holding a stone with "completely unnoticed by those around him" appended. + +Two uses: + lint_render(text) problems with a premise as the image model would read it (critic and generators) + lint_prompt(prompt, suppress) the finished platform prompt: attention claims are removed, the rest is reported + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import re +from typing import List, Sequence, Tuple + +# A substitution or a pretence described instead of shown. "instead" alone is NOT here: "casts its shadow toward +# the light instead of away from it" names a direction, not an object, and renders well enough. +SUBSTITUTION = re.compile( + r"\b(?:should\s+(?:be|stand|sit|rest|lie)" + r"|would\s+(?:normally\s+|usually\s+|ordinarily\s+)?(?:be|stand|sit|rest|lie)" + r"|in\s+place\s+of" + r"|instead\s+of\s+(?:the|a|an|his|her|its|their)\b" + r"|there\s+(?:is|are)\b[^.;]{0,80}?\binstead\b" + r"|as\s+if|as\s+though" + r"|treats?\s+(?:it|them|this|these)\s+as" + r"|takes?\s+(?:it|them)\s+for" + r"|mistaken\s+for)\b", + re.I, +) + +# In a FINISHED prompt, 'as if' / 'as though' is usually atmosphere ("as if time has paused"), not a pretence about +# an object, so the prompt-level check uses this narrower pattern; premises are held to the strict one above. +SUBSTITUTION_IN_PROMPT = re.compile( + r"\b(?:should\s+(?:be|stand|sit|rest|lie)" + r"|in\s+place\s+of" + r"|instead\s+of\s+(?:the|a|an|his|her|its|their)\b" + r"|there\s+(?:is|are)\b[^.;]{0,80}?\binstead\b" + r"|treats?\s+(?:it|them|this|these)\s+as)\b", + re.I, +) + +# An attitude of the people in the picture toward the anomaly. Unrenderable, and the rewriter transcribes it +# (live 2026-10-01: "The woman remains unaware of the unusual sight beside her", "The man remains unfazed"). +ATTENTION = re.compile( + r"(?:\b(?:nobody|no\s+one|no-one|none\s+of\s+them|no\s+passer-?by|no\s+person)\b[^.,;]{0,40}?" + r"\b(?:notices?|noticing|pays?|paying|reacts?|reacting|acknowledges?|remarks?|looks?\s+at|seems?\s+to\s+(?:notice|mind|care))\b)" + r"|\bunnoticed\b|\bunremarked\b|\bwithout\s+anyone\s+(?:noticing|reacting)\b" + r"|\b(?:it|the\s+\w+)\s+(?:goes|go)\s+unnoticed\b" + r"|\b(?:remains?|is|are|seems?|appears?|stays?)\s+(?:completely\s+|entirely\s+|quite\s+)?" + r"(?:unaware|unfazed|oblivious|indifferent|unbothered|unperturbed|unconcerned)\b", + re.I, +) + +_CLAUSE_SPLIT = re.compile(r"(?<=[,;])\s+|(?<=[.!?])\s+") + + +def lint_render(text: str) -> List[str]: + """Why this text, read by an image model, would not render its anomaly. Empty list = fine.""" + problems: List[str] = [] + found = SUBSTITUTION.search(text or "") + if found: + problems.append(f"describes a substitution or a pretence ('{found.group(0)}') instead of the result; the image model " + "draws both objects, or the original one") + found = ATTENTION.search(text or "") + if found: + problems.append(f"claims an attitude ('{found.group(0)}') that has no pixels; show people doing ordinary things instead") + return problems + + +def named_terms(prompt: str, suppress: Sequence[str]) -> List[str]: + """Which of the things that must NOT appear are named in the prompt (whole-word, case-insensitive).""" + lowered = (prompt or "").lower() + found = [] + for term in suppress: + term = (term or "").strip().lower() + if term and re.search(rf"(? Tuple[str, List[str]]: + """(prompt, warnings). Attention claims are removed clause by clause (safe: they carry no picture). Substitution + grammar and suppressed names are only reported, because removing them could remove the anomaly with them.""" + warnings: List[str] = [] + if not (prompt or "").strip(): + return prompt, warnings + pieces = _CLAUSE_SPLIT.split(prompt) + kept, dropped = [], [] + for piece in pieces: + if ATTENTION.search(piece): + dropped.append(piece.strip()) + continue + kept.append(piece) + cleaned = prompt + if dropped and kept: + cleaned = " ".join(kept) + cleaned = re.sub(r"\s*,\s*([.!?])", r"\1", cleaned) + cleaned = re.sub(r"[,;]\s*$", ".", cleaned.strip()) + warnings.append("surreal: removed clause(s) claiming nobody notices (no pixels in that): " + + " / ".join(text[:60] for text in dropped[:3])) + found = SUBSTITUTION_IN_PROMPT.search(cleaned) + if found: + warnings.append(f"surreal: the prompt describes a substitution ('{found.group(0)}'); the image model may draw the " + "original object too. Re-run, or use the two-stage outputs.") + named = named_terms(cleaned, suppress) + if named: + warnings.append("surreal: the prompt names what the anomaly replaced (" + ", ".join(named) + "); the image model will " + "probably draw it. Re-run, or use the two-stage outputs.") + return cleaned, warnings + + +SUPPRESS_MARKER = "Not in the picture - never name it, in any wording:" + + +def suppress_from_instructions(instructions: str) -> List[str]: + """Read the suppress list back out of an instruction block (the rewriter receives only the block).""" + for line in (instructions or "").splitlines(): + if SUPPRESS_MARKER in line: + tail = line.split(SUPPRESS_MARKER, 1)[1] + return [term.strip() for term in tail.split(";") if term.strip()] + return [] diff --git a/core/strategies/surreal/scene.py b/core/strategies/surreal/scene.py new file mode 100644 index 0000000..8e11af5 --- /dev/null +++ b/core/strategies/surreal/scene.py @@ -0,0 +1,191 @@ +""" +The first stage of the surreal pipeline: inventory the ordinary scene before anything is changed. + +The inventory is deliberately descriptive rather than creative. It gives the critic stable entity roles, +ordinary properties, relationships and a compact scene context that can survive a later transformation without +forcing the final prompt to repeat the original user sentence. + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from typing import List, Optional, Tuple + + +@dataclass +class SceneEntity: + id: str = "" + name: str = "" + kind: str = "" + role: str = "" + attributes: List[str] = field(default_factory=list) + ordinary_properties: List[str] = field(default_factory=list) + relationships: List[str] = field(default_factory=list) + affordances: List[str] = field(default_factory=list) + + +@dataclass +class SceneInventory: + primary_subject: str = "" + subject_action: str = "" + setting: str = "" + light: str = "" + viewpoint: str = "" + entities: List[SceneEntity] = field(default_factory=list) + preserved_scene: str = "" + + def summary(self) -> str: + parts = [] + if self.primary_subject: + parts.append(f"subject: {self.primary_subject}") + if self.subject_action: + parts.append(f"action: {self.subject_action}") + if self.setting: + parts.append(f"setting: {self.setting}") + if self.light: + parts.append(f"light: {self.light}") + if self.viewpoint: + parts.append(f"viewpoint: {self.viewpoint}") + if self.entities: + names = [entity.name for entity in self.entities if entity.name] + if names: + parts.append("entities: " + ", ".join(names[:12])) + return " | ".join(parts) + + +SCENE_SYSTEM = """You inventory the ORDINARY scene in a user's image idea before a surreal transformation is applied. + +Do not improve the image, add symbolism, choose a surreal operation or write a polished prompt. Record only what +the user supplied or directly implied. Give each important entity a short stable id. For each entity, distinguish +its visible attributes, ordinary physical properties, relationships and useful affordances. An affordance is a +visible possibility such as "can be held", "provides human scale", "has an opening", "casts a shadow", "forms a +continuous surface" or "can support another object". + +`preserved_scene` is one compact sentence containing the subject, action, setting, light and viewpoint. It is +ordinary and contains every important entity. A later stage will remove or alter individual entities. + +Answer with JSON only: +{"primary_subject":"...","subject_action":"...","setting":"...","light":"...","viewpoint":"...", +"entities":[{"id":"...","name":"...","kind":"...","role":"primary|supporting|environment", +"attributes":["..."],"ordinary_properties":["..."],"relationships":["..."],"affordances":["..."]}], +"preserved_scene":"..."}""" + + +def scene_messages(idea: str) -> Tuple[str, str]: + return SCENE_SYSTEM, "USER IDEA\n" + (idea or "").strip() + + +def _first_json_object(text: str) -> Optional[dict]: + text = re.sub(r".*?", "", text or "", flags=re.S | re.I) + start = text.find("{") + while start >= 0: + depth, in_string, escaped = 0, False, False + for position in range(start, len(text)): + character = text[position] + if in_string: + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == '"': + in_string = False + continue + if character == '"': + in_string = True + elif character == "{": + depth += 1 + elif character == "}": + depth -= 1 + if depth == 0: + try: + value = json.loads(text[start:position + 1]) + return value if isinstance(value, dict) else None + except json.JSONDecodeError: + break + start = text.find("{", start + 1) + return None + + +def _short_text(value, words: int = 40) -> str: + text = re.sub(r"\s+", " ", str(value or "")).strip().strip('"') + return " ".join(text.split()[:words]) + + +def _short_list(value, count: int = 8, words: int = 16) -> List[str]: + if not isinstance(value, list): + return [] + found = [] + for entry in value: + text = _short_text(entry, words) + if text and text not in found: + found.append(text) + return found[:count] + + +def parse_scene_inventory(text: str) -> Tuple[Optional[SceneInventory], List[str]]: + data = _first_json_object(text) + if data is None: + return None, ["the scene inventory returned no JSON object"] + entities: List[SceneEntity] = [] + for index, raw in enumerate(data.get("entities") or []): + if not isinstance(raw, dict): + continue + name = _short_text(raw.get("name"), 12) + if not name: + continue + entity_id = re.sub(r"[^a-z0-9_]+", "_", _short_text(raw.get("id"), 6).lower()).strip("_") + entities.append(SceneEntity( + id=entity_id or f"entity_{index + 1}", + name=name, + kind=_short_text(raw.get("kind"), 8), + role=_short_text(raw.get("role"), 4), + attributes=_short_list(raw.get("attributes")), + ordinary_properties=_short_list(raw.get("ordinary_properties")), + relationships=_short_list(raw.get("relationships")), + affordances=_short_list(raw.get("affordances")), + )) + inventory = SceneInventory( + primary_subject=_short_text(data.get("primary_subject"), 16), + subject_action=_short_text(data.get("subject_action"), 20), + setting=_short_text(data.get("setting"), 24), + light=_short_text(data.get("light"), 20), + viewpoint=_short_text(data.get("viewpoint"), 20), + entities=entities[:16], + preserved_scene=_short_text(data.get("preserved_scene"), 80), + ) + if not inventory.primary_subject: + return None, ["the scene inventory did not identify a primary subject"] + if not inventory.preserved_scene: + return None, ["the scene inventory did not return a preserved ordinary scene"] + return inventory, [] + + +def inventory_for_critic(inventory: Optional[SceneInventory]) -> str: + if inventory is None: + return "" + lines = [ + f"primary subject: {inventory.primary_subject}", + f"subject action: {inventory.subject_action or '[not specified]'}", + f"setting: {inventory.setting or '[not specified]'}", + f"light: {inventory.light or '[not specified]'}", + f"viewpoint: {inventory.viewpoint or '[not specified]'}", + f"ordinary scene: {inventory.preserved_scene}", + "entities:", + ] + for entity in inventory.entities: + details = [] + if entity.attributes: + details.append("attributes=" + "; ".join(entity.attributes)) + if entity.ordinary_properties: + details.append("ordinary properties=" + "; ".join(entity.ordinary_properties)) + if entity.relationships: + details.append("relationships=" + "; ".join(entity.relationships)) + if entity.affordances: + details.append("affordances=" + "; ".join(entity.affordances)) + lines.append(f"- {entity.id}: {entity.name} ({entity.kind or 'entity'}, {entity.role or 'supporting'})" + + (f" | {' | '.join(details)}" if details else "")) + return "\n".join(lines) diff --git a/core/strategies/surreal/sources.py b/core/strategies/surreal/sources.py index 8b574c4..c2c2ae1 100644 --- a/core/strategies/surreal/sources.py +++ b/core/strategies/surreal/sources.py @@ -55,6 +55,8 @@ LISTS = { "idioms": "surreal/idioms", "processes": "surreal/processes", "temporal": "surreal/temporal", + "autonomy": "surreal/autonomy", + "reversals": "surreal/reversals", "hour": "surreal/found_strange/hour", "emptiness": "surreal/found_strange/emptiness", "stillness": "surreal/found_strange/stillness", diff --git a/core/strategies/surreal/vocabulary.py b/core/strategies/surreal/vocabulary.py index 4f7b976..77505f6 100644 --- a/core/strategies/surreal/vocabulary.py +++ b/core/strategies/surreal/vocabulary.py @@ -27,26 +27,94 @@ class Operation: id: str what: str # one line, shown in tooltips and in the `why` output lineage: str + law: str = "" # the transformation rule this KIND of violation obeys: what may change, what may not + visible: str = "" # how the anomaly becomes POSITIVELY visible: what the pixels actually show impossible: bool = True # False = nothing impossible is depicted + two_stage: bool = False # can be staged as an ordinary picture plus ONE instruction edit (see STAGING_CHOICES) OPERATIONS: Dict[str, Operation] = {op.id: op for op in [ - Operation("displacement", "an ordinary object where it has no business being", "Magritte, Lautreamont"), - Operation("scale_violation", "one thing at the wrong size, everything else correct", "Magritte"), - Operation("hybridization", "two things fused at a logical seam, best frozen mid-change", "Magritte, Ernst"), - Operation("material_substitution", "right form, wrong substance", "Oppenheim, Dali"), - Operation("physics_negation", "one law suspended, all others intact", "Magritte, Varo"), - Operation("inversion", "container and contained, inside and outside, swap", "Magritte, Escher"), - Operation("double_image", "one form honestly readable two ways", "Dali, Arcimboldo"), - Operation("animation_petrification", "the inanimate lives, or the living becomes object", "Ernst, Bellmer, Tanning"), - Operation("absence", "the veiled face, the shadow with no caster", "Magritte, de Chirico"), - Operation("metaphysical_space", "empty arcades, long wrong shadows, stopped time", "de Chirico, Sage, Tanguy"), - Operation("literalized_language", "an idiom rendered straight", "Magritte, Carrington"), - Operation("multiplication", "one thing too many times, identical", "Magritte, Kusama"), - Operation("working_machine", "an impossible machine that looks as if it works", "Varo"), - Operation("temporal", "video: one thing wrong in time, everything else in ordinary time", "Cocteau, Deren"), - Operation("found_strange", "nothing impossible: hour, emptiness, stillness, a withheld story", "Atget, Crewdson", impossible=False), - Operation("process_estrangement", "nothing impossible: an ordinary subject, one photographic process", "Man Ray, Lee Miller", impossible=False), + Operation("displacement", "an ordinary object where it has no business being", "Magritte, Lautreamont", + visible=("the out-of-place object is in full view, sharply lit and correctly scaled, close to the centre of interest"), + law="Things may appear where they do not belong, unaltered in themselves and correct in size, light and shadow; " + "nothing is changed in its nature, and nobody present acknowledges that it is out of place.", two_stage=True), + Operation("scale_violation", "one thing at the wrong size, everything else correct", "Magritte", + visible=("the wrongly sized thing shares the frame with something of known size, so the error is measurable by eye"), + law="Exactly one thing departs from true scale, and it behaves as its new size demands; perspective, light and " + "every other object stay exactly correct.", two_stage=True), + Operation("hybridization", "two things fused at a logical seam, best frozen mid-change", "Magritte, Ernst", + visible=("the seam is in frame and legible, and both originals stay recognisable in the single object"), + law="Two things may become one only where their shapes already coincide, and the join is continuous with no seam " + "to be found; nothing else in the picture is fused.", two_stage=True), + Operation("material_substitution", "right form, wrong substance", "Oppenheim, Dali", + visible=("the wrong substance is read from the surface itself: weave, grain, sheen, and the way the thing sits under its own weight"), + law="Substance alone may change; every outline, seam, wear mark and proportion stays exact, and the new substance " + "behaves as itself under light, weight and touch.", two_stage=True), + Operation("physics_negation", "one law suspended, all others intact", "Magritte, Varo", + visible=("the suspended law leaves a positive mark: a gap of air beneath, a shadow in the wrong place, a surface that should have given way and has not"), + law="One physical law is suspended, for one thing, throughout the picture; every other law and every other object " + "behaves exactly as it should, including their shadows."), + Operation("inversion", "container and contained, inside and outside, swap", "Magritte, Escher", + visible=("the opening is in frame, and what is inside it is drawn at its own scale with its own light spilling out onto the surroundings"), + law="Any opening may contain something larger than the thing it is cut into; the opening keeps its own true size, " + "and what is inside keeps its own true scale and its own light.", two_stage=True), + Operation("double_image", "one form honestly readable two ways", "Dali, Arcimboldo", + visible=("one silhouette with both readings fully drawn, from the single viewpoint of the camera"), + law="One form may be honestly two things at once from this single viewpoint, both readings complete and fully " + "rendered; nothing is blended, faded, doubled or reflected to achieve it."), + Operation("animation_petrification", "the inanimate lives, or the living becomes object", "Ernst, Bellmer, Tanning", + visible=("the crossing is on the surface: carved folds and tool marks, or warmth, breath and slackness in a thing that has none"), + law="Life may cross between the made and the living in one direction only and for one thing; whatever crosses keeps " + "its exact form, and everything around it carries on unchanged and unsurprised.", two_stage=True), + Operation("absence", "the veiled face, the shadow with no caster", "Magritte, de Chirico", + visible=("an absence needs an EDGE. Draw the clean boundary of what is missing and the surroundings continuing correctly through it; never rely on a thing merely not being there"), + law="What is missing or covered leaves its exact outline behind and is never explained; everything else is fully " + "present, and nothing in the picture acknowledges the gap."), + Operation("metaphysical_space", "empty arcades, long wrong shadows, stopped time", "de Chirico, Sage, Tanguy", + visible=("the shadows are the subject: long, hard-edged, drawn as dark shapes on the ground and pointed the wrong way"), + law="Light, shadow and time in this place follow a rule of their own and hold it consistently across the whole " + "frame; every solid object is ordinary, correct and exactly where it should be."), + Operation("literalized_language", "an idiom rendered straight", "Magritte, Carrington", + visible=("the figure of speech is built from real material in the frame - actual cord, actual glass, actual weight - and never implied"), + law="One figure of speech is true here as plain physical fact, in full and without irony; nothing else in the scene " + "is figurative, and nobody remarks on it."), + Operation("multiplication", "one thing too many times, identical", "Magritte, Kusama", + visible=("every copy is inside the frame at once, evenly placed, each with its own shadow"), + law="Exactly one kind of thing is subject to repetition; every copy is identical down to its wear and its shadow, " + "and nothing else in the picture repeats at all.", two_stage=True), + Operation("working_machine", "an impossible machine that looks as if it works", "Varo", + visible=("every linkage is turned to the camera: what goes in, the mechanism, and what comes out, all visible in the one shot"), + law="Every step of the mechanism is visible and follows from the one before it, and each linkage would work; only " + "the chain taken as a whole is impossible."), + Operation("temporal", "video: one thing wrong in time, everything else in ordinary time", "Cocteau, Deren", + visible=("the difference in time shows as blur, position or a trail, against things in the same frame that are sharp"), + law="One thing runs on a time of its own from the first frame to the last; every other motion, and all sound, " + "keeps ordinary time, and nothing in the shot reacts to the difference."), + Operation("causal_reversal", "the effect is present and its cause is absent, or arrives after it", "Magritte, Bunuel", + visible=("the consequence is rendered at full strength - the soaked ground, the scorch, the pressed dust - and the place its cause should occupy is shown as ordinary empty space"), + law="Consequences arrive before their causes and are complete; every cause is still to come, and nothing in the " + "scene anticipates, explains or reacts to it."), + Operation("autonomy", "a shadow, reflection or garment acts on its own, against what it belongs to", "Magritte, Svankmajer", + visible=("the double is drawn in a position its owner is not in, and both are in frame together, so the disagreement between them is what the picture shows"), + law="Only what is cast by something - a shadow, a reflection, an imprint, a hanging garment - can act on its own; " + "the things that cast them cannot, and never react."), + Operation("taxonomy_error", "the thing is presented as a specimen, classified under the wrong kingdom", "Ernst, Cornell, Broodthaers", + visible=("the apparatus of classification IS the picture: the case, the pin, the cotton wool, the printed label, the catalogue number, the scale bar"), + law="Nothing here is physically altered; only the system that classifies it is wrong, and it is applied with " + "complete institutional consistency."), + Operation("found_strange", "nothing impossible: hour, emptiness, stillness, a withheld story", "Atget, Crewdson", + visible=("with no anomaly to carry it, the picture needs a visible object or a visible light: a thing left where it should not be, or light where there should be none"), + law="Nothing impossible may appear; the hour, the emptiness, the stillness and the light do all of the work, and " + "the story behind the scene is never completed.", impossible=False), + Operation("process_estrangement", "nothing impossible: an ordinary subject, one photographic process", "Man Ray, Lee Miller", + visible=("the process is visible across the whole frame as a physical photographic effect - reversed tones, a solarised edge, a contact shadow - never as a filter laid over a normal picture"), + law="The subject is entirely ordinary and untouched; the estrangement is in the photographic process alone, and it " + "is applied evenly across the whole frame.", impossible=False), + Operation("behavior_transfer", "one thing keeps its identity but obeys the physical behaviour of another material", "Dali, Magritte", + visible=("the original material remains unmistakable while folds, tension, sag, peeling or another borrowed behaviour " + "changes its geometry and leaves a positive contact, shadow or exposed surface"), + law="Exactly one thing may borrow the physical behaviour of one distant material while keeping its own substance, " + "identity and connection to the scene; everything else obeys ordinary physics.", two_stage=True), ]} @@ -65,7 +133,8 @@ MODES: Dict[str, Mode] = {mode.id: mode for mode in [ Mode("double_reading", "Dali", "two things with the same silhouette, one form that is honestly both"), Mode("working_machine", "Varo", "a mechanism, a feedstock and a product, joined by a causal chain that looks functional"), Mode("archive_composite", "Uelsmann, Maar", "joins fragments from YOUR iconography file and prompt database, never from stock"), - Mode("found_strange", "Atget, Crewdson", "no anomaly at all: hour, emptiness, stillness, a story that is withheld"), + Mode("found_strange", "Atget, Crewdson", "no anomaly at all: ONE hour and ONE condition - emptiness, stillness, " + "light or a trace - and a story that is withheld"), Mode("process_estrangement", "Man Ray, Miller", "an ordinary subject and one photographic process"), ]} @@ -77,13 +146,16 @@ STRANGENESS_CHOICES = ["subtle", "balanced", "overt"] OPERATION_WEIGHTS: Dict[str, Dict[str, float]] = { "subtle": {"absence": 3, "material_substitution": 2, "physics_negation": 2.5, "displacement": 2, "literalized_language": 1, "metaphysical_space": 2, "scale_violation": 0.5, "hybridization": 0.5, "inversion": 1, "animation_petrification": 1, - "multiplication": 0.5, "temporal": 2}, + "multiplication": 0.5, "temporal": 2, "causal_reversal": 2, "autonomy": 2.5, "taxonomy_error": 1, + "behavior_transfer": 1.5}, "balanced": {"absence": 2, "material_substitution": 2.5, "physics_negation": 2, "displacement": 2.5, "literalized_language": 1.5, "metaphysical_space": 1, "scale_violation": 2, "hybridization": 1.5, "inversion": 2, "animation_petrification": 1.5, - "multiplication": 1.5, "temporal": 2}, + "multiplication": 1.5, "temporal": 2, "causal_reversal": 2, "autonomy": 2, "taxonomy_error": 1.5, + "behavior_transfer": 2}, "overt": {"absence": 1, "material_substitution": 2, "physics_negation": 1.5, "displacement": 2, "literalized_language": 2, "metaphysical_space": 0.5, "scale_violation": 3, "hybridization": 2.5, "inversion": 3, "animation_petrification": 2, - "multiplication": 2.5, "temporal": 2}, + "multiplication": 2.5, "temporal": 2, "causal_reversal": 1.5, "autonomy": 1.5, "taxonomy_error": 2, + "behavior_transfer": 3}, } MODE_WEIGHTS: Dict[str, Dict[str, float]] = { "subtle": {"affinity_search": 2, "problem": 2, "read_the_noise": 0.5, "double_reading": 0.5, "working_machine": 0.5, @@ -134,6 +206,107 @@ CHARGES: Dict[str, str] = { } CHARGE_CHOICES: List[str] = [AUTO] + list(CHARGES) +# The transformation rule, or LAW. An operation says what is violated; the law says what GOVERNS the violation - +# what may change, what may not, and how the change behaves. Without it a second anomaly reads as a second idea +# rather than an echo of the first, and the picture reads as random. The law is how an emotion becomes physics: +# grief increases gravity, shame removes faces, desire changes scale. +# +# These are seeds, not final text. The code critic uses one as written; the LLM critic is given one as a starting +# point and writes the law that actually fits the premise it chose. +CHARGE_LAWS: Dict[str, Tuple[str, ...]] = { + "dread": ( + "Whatever is not being looked at has already moved; anything directly in view stays exactly as it is.", + "Every surface a person has recently left is still warm and slightly deformed, and no other surface is.", + "Only what is about to be needed has been set out ready; nothing else in the place has been touched.", + ), + "longing": ( + "Only the thing that cannot be reached carries its own light; every other object is lit by the room.", + "Distance behaves as a material: what is wanted is rendered smaller and more finely detailed than perspective " + "allows, and nothing else departs from perspective.", + "Whatever has been left behind holds its full colour, and everything present holds its colour one degree weaker.", + ), + "absurdity": ( + "Any procedure in the scene is carried out perfectly correctly upon the wrong object; the procedure itself is never wrong.", + "One object may stand in for another provided the exchange is treated as routine and size and position are preserved.", + "The rules of the place are obeyed to the letter and apply to the wrong things.", + ), + "grief": ( + "Whatever the missing person touched has gained weight: those objects rest lower and press into what holds them, " + "and nothing else is affected.", + "Only the objects belonging to the one who is gone still cast shadows; nothing else in the room casts any.", + "The space a body would have filled stays open, and everything in the picture is arranged around it.", + ), + "wonder": ( + "Anything made by repetition may repeat past the point of possibility, provided every repetition is identical " + "and fully rendered.", + "One ordinary material continues without limit in one direction at full detail, and no other material does.", + "Whatever is built to hold something may hold something of any size, and its own dimensions never change.", + ), + "tenderness": ( + "Only what is fragile is exempt from force: it cannot fall, break or be reached by weather; everything else obeys ordinary physics.", + "Any object that has been handled often has grown soft in exactly the places it was held, and nowhere else.", + "Whatever is cared for takes the material of the thing that cares for it, and nothing else changes substance.", + ), + "menace": ( + "Whatever is domestic and orderly may be made of the wrong material; nothing overtly threatening appears at all.", + "Every object has been arranged by someone who is not present, and the arrangement is perfect.", + "Only the things meant to open and close have sealed themselves; everything else is untouched.", + ), + "desire": ( + "Only the wanted thing is in true focus at every distance; the rest of the scene obeys the lens.", + "Proximity governs size: the more a thing is wanted the larger it is rendered, and physical distance no longer affects scale.", + "Whatever stands in for the wanted thing has warmed, softened or opened, and nothing else has.", + ), +} +GENERIC_LAWS: Tuple[str, ...] = ( + "Only one class of thing in this picture can change; the class is fixed, and every member of it changes in the same way.", + "The change crosses one boundary only - body to architecture, cloth to skin, inside to outside - and only where a shape, " + "a pattern or a material is already shared.", + "Whatever is transformed keeps its exact outline; substance alone is permitted to change.", + "The anomaly obeys gravity, light and perspective exactly as an ordinary object of its size would.", +) + + +def law_seed(charge: str, rng) -> str: + """One transformation rule to start from, drawn for the charge. Falls back to the generic pool.""" + pool = list(CHARGE_LAWS.get(charge, ())) or list(GENERIC_LAWS) + return rng.choice(pool) + + +# The rubric the LLM critic scores its own chosen premise against, 0 to 4 per dimension. The total is not the point: +# the LOWEST dimension is the one revision to make, which turns a vague judgement into a specific repair. +RUBRIC: Dict[str, str] = { + "reality_anchor": "0 = no stable entry point; 2 = recognisable but generic; 4 = specific, credible, sensuously present", + "rupture": "0 = no violation, or only an error; 2 = visible but familiar; 4 = decisive, fresh, relational", + "psychic_pressure": "0 = arbitrary novelty; 2 = a mood or theme is present; 4 = feels compelled, consequences radiate", + "transformation_rule": "0 = inconsistent accident; 2 = partly legible; 4 = clear enough to infer, strange enough to surprise", + "formal_unity": "0 = parts compete or detach; 2 = mostly coherent; 4 = every formal choice intensifies the premise", + "ambiguity": "0 = empty or incomprehensible; 2 = two readings; 4 = several precise, incompatible readings", + "residual_power": "0 = exhausted at first glance; 2 = some afterimage; 4 = deepens or troubles on return", + # Added after the 2026-09-21 live run: the critic picked premises that are strange only to someone who knows the + # story (a man holding a river stone). Deadpan is not the same as invisible; Magritte's anomalies are large, + # central and lit like everything else. + "visibility": "0 = strange only if you know the story; 2 = noticed on a second look; 4 = unmistakable in a small thumbnail at a glance", +} + +# How the picture is made. one_stage: one prompt carries the anomaly. two_stage: an ORDINARY picture is generated first +# (the realism prior then works for the premise instead of against it) and ONE instruction edit applies the anomaly - +# edit models are trained on exactly replace / make-it-out-of / enlarge / remove, and keep everything else intact. +# The Surreal Premise node always emits the scene prompt and the edit instruction when the operation allows it; +# two_stage additionally restricts the automatic choice to operations that can be staged that way. +STAGING_CHOICES: List[str] = ["one_stage", "two_stage"] + +# How a title earns its keep. The title is a second image, never a caption for the first. +TITLE_STRATEGIES: Dict[str, str] = { + "sideways_association": "adds a second image or a context; never restates what is visible", + "false_classification": "an archival, zoological, clinical or bureaucratic label, related to the form and not a gag", + "temporal_clue": "implies a before or an after without completing the story or explaining the cause", + "ordinary_phrase": "a flat domestic or social phrase that makes an extraordinary event feel routine", + "poetic_friction": "language set against the image rather than beneath it; never decorative obscurity", +} +TITLE_STRATEGY_CHOICES: List[str] = [AUTO] + list(TITLE_STRATEGIES) +BANNED_TITLES: Tuple[str, ...] = ("untitled", "dream", "the dream", "a dream", "surreal", "surrealism", "reverie", "nightmare") + # What the imitators reach for. Checked in every premise and listed for the rewriter. BAN_LIST: Tuple[str, ...] = ( "melting clock", "melting clocks", "floating island", "floating islands", "eye in the sky", "eyes in the sky", @@ -145,6 +318,13 @@ BAN_LIST_FOR_PROMPT = ( "melting clocks, floating islands, eyes in the sky, staircases to nowhere, galaxy or nebula fills, gratuitous glow and " "mist, and the words surreal, dreamlike, ethereal, whimsical, otherworldly, mind-bending, fantastical, magical, bizarre" ) +# Not hard-rejected in code, because a mask, a chessboard floor and an empty plain are legitimate material (Bellmer, +# de Chirico, Tanguy, Sage) and the shipped motif lists contain them. They are inherited SIGNALS of the genre, so a +# premise may use one only when the premise would still stand with it removed. +SOFT_BAN_FOR_PROMPT = ( + "the postcard versions of masks, chessboards and empty deserts: use one only if it is doing work that nothing else " + "could do here, never as a signal that the picture is surreal" +) # Tags in objects.yaml that describe FORM: a shared form tag is a visual rhyme between two objects. FORM_TAGS = frozenset({"round", "long", "flat", "hollow", "container", "hinged", "wheeled", "sharp", "coiled", "paired", "soft", "rigid"}) diff --git a/core/wildcards/__init__.py b/core/wildcards/__init__.py index c1ec939..b81e238 100644 --- a/core/wildcards/__init__.py +++ b/core/wildcards/__init__.py @@ -10,13 +10,18 @@ See parser.py for the grammar, library.py for file formats, record.py for bindin (c) Eric Hiss. Dual license, see LICENSE. """ -from .engine import WildcardEngine, get_engine, split_paths -from .library import Entry, WildcardLibrary +from .catalog import CatalogSet, analyse, display_name +from .curation import Curation, UserOverlay +from .engine import WildcardEngine, blocked_resolution, get_engine, split_paths +from .library import (NSFW_LEVELS, VIOLENCE_LEVELS, Entry, WildcardLibrary, nsfw_level, prompt_block_reason, + violence_level) from .parser import parse from .record import Resolution, Slot from .resolver import MODE_COMBINATORIAL, MODE_RANDOM, MODES, PickContext, tidy __all__ = [ + "CatalogSet", "analyse", "display_name", "Curation", "UserOverlay", "WildcardEngine", "get_engine", "split_paths", "WildcardLibrary", "Entry", "parse", "Resolution", "Slot", "MODE_RANDOM", "MODE_COMBINATORIAL", "MODES", "PickContext", "tidy", + "VIOLENCE_LEVELS", "violence_level", "prompt_block_reason", "blocked_resolution", "NSFW_LEVELS", "nsfw_level", ] diff --git a/core/wildcards/catalog.py b/core/wildcards/catalog.py new file mode 100644 index 0000000..43d4b3f --- /dev/null +++ b/core/wildcards/catalog.py @@ -0,0 +1,567 @@ +""" +Wildcard catalog: a generated index of every list in a library (spec: dev/WILDCARD_CATALOG_SPEC.md, P2). + +One SQLite file per search folder, `/_catalog.sqlite` (or the package cache when the folder +is not writable or is the package's own wildcards/ folder). It describes lists; it never replaces +them. Delete it and it is rebuilt. + +Per list it stores what the builders and the engine need to rank and gate lists: + kind atoms | phrases | prompts | template | tags + dialect prose | tags | mixed (booru_style_tags vs words) + entries, median words / characters, duplicate share inside the list + nsfw sfw | suggestive | explicit, and where that came from (folder | name | content) + signature hash of the entry set: identical lists in different packs share it (dup_of) + quality 0..1, ordering only, never access (see quality_score) + facet a coarse topic (appearance, wardrobe, setting, lighting, camera, ...) + samples 8 entries spread across the list + display "Hair Colors · Mrweaz" + +Updating is incremental: only files whose (mtime, size) changed are analysed again. The analysis +uses the entries exactly as the library serves them (LoRA tags stripped, content filter applied). + +No ComfyUI dependency. `CatalogSet.update(library)` is safe to run in a background thread: it only +reads the library. + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sqlite3 +import statistics +import threading +import time +from dataclasses import dataclass, field +from typing import Callable, Dict, Iterable, List, Optional, Sequence, Tuple + +SCHEMA_VERSION = "2" # bump when the analysis changes: catalogs rebuild themselves +CATALOG_FILE = "_catalog.sqlite" +SAMPLE_COUNT = 8 +ANALYSE_CAP = 3000 # entries looked at per list for the statistics (all are counted and hashed) + +# -- content rates (same vocabulary as dev/wildcard_audit.py and wildcard_cleanup.py) ---------------- +_EXPLICIT = re.compile( + r"\b(sex|sexual|fuck\w*|cum\w*|cock|dick|penis|pussy|vagina|anal|blowjob|fellatio|cunnilingus|penetrat\w*|" + r"orgasm\w*|masturbat\w*|nipples?|genital\w*|porn\w*|nsfw|hentai|bdsm|bondage|ejaculat\w*|erect\w*|" + r"missionary|doggystyle|cowgirl|spread legs|creampie|dildo|vibrator|threesome|gangbang)\b", re.I) +_SUGGEST = re.compile( + r"\b(nude|naked|topless|lingerie|panties|thong|bra|see-through|sheer|sexy|seductive|cleavage|erotic|" + r"sensual|busty|braless|undress\w*|stripper|boudoir|fetish|latex|g-string)\b", re.I) +EXPLICIT_RATE = 0.08 # explicit words per entry -> explicit +SUGGEST_EXPLICIT_RATE = 0.5 # suggestive words per entry -> explicit +SUGGEST_RATE = 0.15 # suggestive words per entry -> suggestive + +_WILDREF = re.compile(r"__[\w\-/*. ]+__") +_WEIGHTED = re.compile(r"\([^()]+:\s*\d*\.?\d+\)") +_BOORU = re.compile(r"^[a-z0-9]+(?:_[a-z0-9()']+)+$") +_BOILERPLATE = re.compile( + r"\b(masterpiece|best quality|high quality|highres|absurdres|ultra[- ]detailed|8k|4k|uhd|" + r"trending on artstation|award[- ]winning|score_\d)\b", re.I) +_MOJIBAKE = re.compile(r"Ã.|â€| |�") +_CJK = re.compile(r"[぀-ヿ㐀-鿿가-힯]") + +# -- display names ----------------------------------------------------------------------------------- +_NOISE = re.compile(r"(_v\d+[\w.]*$|v\d+fixed$|\s*\(\d+\)$|wildcards?|^wc[-_]|^!+)", re.I) +_LEAF_PREFIX = re.compile(r"^(?:cops_(?:bp_)?(?:male_|female_)?wc[-_]?|cyber_|bk_|rd_|fg3_|vc_|tr_)", re.I) + +# -- facets: keyword rules on the key; first rule that matches wins -------------------------------- +FACETS = ("subject", "appearance", "wardrobe", "pose_action", "expression", "setting", "architecture", + "nature", "creature", "object_material", "lighting", "camera", "style_artist", "color_mood", + "era_culture", "concept", "prompt_template") +_FACET_RULES: List[Tuple[str, re.Pattern]] = [(facet, re.compile(pattern, re.I)) for facet, pattern in ( + ("camera", r"camera|lens|angle|shot|framing|composition|focal|aperture|depth.of.field|bokeh"), + ("lighting", r"light(?!ning)|lighting|shadow|illuminat|golden.hour|glow"), + ("expression", r"expression|emotion|mood(?!.*colou?r)|facial|smile|feeling"), + ("pose_action", r"pose|action|gesture|posture|activit|interaction|movement(?!s?$)|dance"), + ("wardrobe", r"cloth|outfit|wear|dress|shirt|top|bottom|pants|skirt|shoe|boot|hat|jacket|coat|costume|" + r"uniform|lingerie|swimsuit|accessor|jewel|glove|sock|stocking|attire|fashion"), + ("appearance", r"hair|eye|face|skin|body|makeup|nose|lip|beard|freckle|tattoo|age|ethnic|race|" + r"nationalit|physique|build|figure|complexion"), + ("subject", r"person|people|character|occupation|profession|job|role|archetype|subject|name|gender|" + r"girl|boy|woman|man|men|women"), + ("creature", r"animal|creature|monster|beast|bird|fish|insect|dragon|species|pet|dog|cat"), + ("architecture", r"architect|building|interior|room|house|castle|city|street|structure"), + ("nature", r"landscape|nature|forest|mountain|ocean|sea|river|flower|plant|tree|weather|season|sky|biome"), + ("setting", r"location|place|setting|scene|background|environment|world|planet|travel"), + ("style_artist", r"artist|style|medium|movement|painter|photographer|illustrat|render|aesthetic|genre|film"), + ("color_mood", r"colou?r|palette|tone|hue|grading|atmosphere|vibe"), + ("era_culture", r"era|decade|century|period|histor|culture|myth|religio|medieval|victorian|\d0s"), + ("object_material", r"object|item|material|fabric|texture|metal|wood|food|drink|weapon|tool|vehicle|prop"), + ("concept", r"concept|idea|theme|abstract|adjective|adj|noun|verb|word"), +)] + + +def display_name(key: str) -> str: + """'mrweazsfwildcardssfw_v30glamour/hair_colors' -> 'Hair Colors · Mrweazsf Glamour'.""" + parts = [part for part in key.split("/") if part] + if not parts: + return key + + def clean(segment: str) -> str: + segment = _LEAF_PREFIX.sub("", segment) + segment = _NOISE.sub(" ", segment) + segment = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", segment) + segment = re.sub(r"[_\-]+", " ", segment) + return re.sub(r"\s+", " ", segment).strip().title() + + leaf = clean(parts[-1]) or parts[-1] + if len(parts) == 1: + return leaf + pack = clean(parts[0]) or parts[0] + return f"{leaf} · {pack}" + + +def facet_for(key: str) -> Tuple[str, float]: + """(facet, confidence). Rules look at the leaf first, then the whole key.""" + leaf = key.rsplit("/", 1)[-1] + for text, confidence in ((leaf, 0.9), (key, 0.6)): + for facet, pattern in _FACET_RULES: + if pattern.search(text): + return facet, confidence + return "", 0.0 + + +def entry_signature(texts: Iterable[str]) -> str: + """Same set of entries (case-insensitive) -> same signature, whatever the order or file.""" + digest = hashlib.sha1() + for text in sorted({text.strip().casefold() for text in texts if text.strip()}): + digest.update(text.encode("utf-8", "replace")) + digest.update(b"\n") + return digest.hexdigest() + + +def _share(texts: Sequence[str], pattern: re.Pattern) -> float: + return sum(1 for text in texts if pattern.search(text)) / len(texts) if texts else 0.0 + + +def _rate(texts: Sequence[str], pattern: re.Pattern) -> float: + return sum(len(pattern.findall(text)) for text in texts) / len(texts) if texts else 0.0 + + +def _stratified(texts: Sequence[str], count: int = SAMPLE_COUNT) -> List[str]: + if len(texts) <= count: + return list(texts) + step = len(texts) / count + return [texts[int(index * step + step / 2)] for index in range(count)] + + +def _band(value: float, points: Sequence[Tuple[float, float]]) -> float: + """Piecewise-linear lookup: points are (x, score) sorted by x.""" + if value <= points[0][0]: + return points[0][1] + for (x0, y0), (x1, y1) in zip(points, points[1:]): + if value <= x1: + return y0 + (y1 - y0) * (value - x0) / (x1 - x0) + return points[-1][1] + + +def quality_score(stats: Dict) -> float: + """ + 0..1, ordering only: uniqueness, entry length that fits between typed words, and cleanliness + (weight syntax, quality boilerplate, mojibake, other scripts), scaled by list size. + Templates and full prompts are scored by size and cleanliness only. + """ + size = _band(stats["entries"], [(0, 0.0), (3, 0.2), (10, 0.6), (20, 1.0), (5000, 1.0), (50000, 0.7), (200000, 0.4)]) + unique = max(0.0, 1.0 - 2.0 * stats["dupe_share"]) + if stats["kind"] in ("template", "prompts"): + fit = 0.6 + else: + fit = _band(stats["median_words"], [(1, 1.0), (6, 1.0), (10, 0.75), (18, 0.45), (30, 0.2), (60, 0.1)]) + dirt = (0.6 * stats["weights_share"] + 1.0 * stats["boilerplate_share"] + 1.5 * stats["mojibake_share"] + + 0.5 * stats["cjk_share"]) + clean = max(0.0, 1.0 - dirt) + # Size scales the rest: a clean two-item list is still a poor source of variety. + return round((0.2 * unique + 0.4 * fit + 0.4 * clean) * (0.4 + 0.6 * size), 4) + + +def analyse(key: str, texts: Sequence[str], folder_nsfw: bool = False, name_nsfw: bool = False) -> Dict: + """Statistics and derived fields for one list. `texts` = entries as the library serves them.""" + count = len(texts) + sample = list(texts[:ANALYSE_CAP]) + words = [len(text.split()) for text in sample] or [0] + chars = [len(text) for text in sample] or [0] + median_words = int(statistics.median(words)) + median_chars = int(statistics.median(chars)) + stats = { + "entries": count, + "median_words": median_words, + "median_chars": median_chars, + "dupe_share": round(1 - len({t.casefold() for t in sample}) / len(sample), 4) if sample else 0.0, + "refs_share": round(_share(sample, _WILDREF), 4), + "weights_share": round(_share(sample, _WEIGHTED), 4), + "booru_share": round(_share(sample, _BOORU), 4), + "boilerplate_share": round(_share(sample, _BOILERPLATE), 4), + "mojibake_share": round(_share(sample, _MOJIBAKE), 4), + "cjk_share": round(_share(sample, _CJK), 4), + "explicit_rate": round(_rate(sample, _EXPLICIT), 4), + "suggest_rate": round(_rate(sample, _SUGGEST), 4), + } + comma_heavy = _share(sample, re.compile(r"(?:[^,]*,){4}")) + if count == 0: + kind = "empty" + elif stats["refs_share"] >= 0.3: + kind = "template" + elif median_chars >= 140 or (comma_heavy >= 0.5 and median_chars >= 60): + kind = "prompts" + elif stats["booru_share"] >= 0.5: + kind = "tags" + elif median_words >= 7: + kind = "phrases" + else: + kind = "atoms" + stats["kind"] = kind + stats["dialect"] = "tags" if stats["booru_share"] >= 0.5 else "mixed" if stats["booru_share"] >= 0.15 else "prose" + + if folder_nsfw: + nsfw, source = "explicit", "folder" + elif name_nsfw: + nsfw, source = "explicit", "name" + elif stats["explicit_rate"] >= EXPLICIT_RATE or stats["suggest_rate"] >= SUGGEST_EXPLICIT_RATE: + nsfw, source = "explicit", "content" + elif stats["suggest_rate"] >= SUGGEST_RATE: + nsfw, source = "suggestive", "content" + else: + nsfw, source = "sfw", "" + stats["nsfw"], stats["nsfw_source"] = nsfw, source + stats["quality"] = quality_score(stats) + leaf = key.rsplit("/", 1)[-1] + if len(leaf) > 60 or re.search(r"[,<>{}\[\]()]", leaf): + # A YAML "key" that is really a prompt (packs that map prompts to prompts): not a usable list name. + stats["quality"] = round(stats["quality"] * 0.2, 4) + stats["signature"] = entry_signature(texts) + stats["facet"], stats["facet_confidence"] = facet_for(key) + stats["samples"] = _stratified(sample) + stats["display"] = display_name(key) + return stats + + +# ----------------------------------------------------------------------------- rows +@dataclass +class ListRow: + key: str + root: str + file: str + display: str + kind: str + dialect: str + entries: int + median_words: int + nsfw: str + nsfw_source: str + quality: float + facet: str + signature: str + samples: List[str] = field(default_factory=list) + dup_of: str = "" + + def to_dict(self) -> Dict: + return { + "key": self.key, "display": self.display, "kind": self.kind, "dialect": self.dialect, + "entries": self.entries, "median_words": self.median_words, "nsfw": self.nsfw, + "nsfw_source": self.nsfw_source, "quality": self.quality, "facet": self.facet, + "samples": self.samples, "dup_of": self.dup_of, "root": self.root, + } + + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT); +CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, mtime REAL, size INTEGER, status TEXT); +CREATE TABLE IF NOT EXISTS lists ( + key TEXT PRIMARY KEY, file TEXT, pack TEXT, display TEXT, kind TEXT, dialect TEXT, + entries INTEGER, median_words INTEGER, median_chars INTEGER, dupe_share REAL, refs_share REAL, + weights_share REAL, booru_share REAL, boilerplate_share REAL, mojibake_share REAL, cjk_share REAL, + explicit_rate REAL, suggest_rate REAL, nsfw TEXT, nsfw_source TEXT, quality REAL, + facet TEXT, facet_confidence REAL, signature TEXT, samples TEXT +); +CREATE INDEX IF NOT EXISTS lists_file ON lists(file); +CREATE INDEX IF NOT EXISTS lists_signature ON lists(signature); +CREATE VIRTUAL TABLE IF NOT EXISTS lists_fts USING fts5(key, display, facet, samples); +""" +_LIST_COLUMNS = ("key", "file", "pack", "display", "kind", "dialect", "entries", "median_words", "median_chars", + "dupe_share", "refs_share", "weights_share", "booru_share", "boilerplate_share", "mojibake_share", + "cjk_share", "explicit_rate", "suggest_rate", "nsfw", "nsfw_source", "quality", "facet", + "facet_confidence", "signature", "samples") + + +class RootCatalog: + """The catalog of one search folder.""" + + def __init__(self, root: str, db_path: str): + self.root = root + self.db_path = db_path + self._lock = threading.Lock() + os.makedirs(os.path.dirname(db_path) or ".", exist_ok=True) + self._connect() + + def _connect(self) -> None: + # Several processes may share one catalog (ComfyUI, a second ComfyUI, the dev scripts): + # WAL lets readers work during a write, and a writer waits for the other instead of failing. + self.connection = sqlite3.connect(self.db_path, check_same_thread=False, timeout=60) + try: + self.connection.execute("PRAGMA journal_mode=WAL") + except sqlite3.DatabaseError: + pass + try: + self.connection.executescript(_SCHEMA) + version = self.connection.execute("SELECT v FROM meta WHERE k='schema_version'").fetchone() + except sqlite3.DatabaseError: + version = None + if version is None or version[0] != SCHEMA_VERSION: + # Generated data only: start again. Tables are dropped rather than the file deleted, + # because another process may have it open. + try: + for table in ("lists_fts", "lists", "files", "meta"): + self.connection.execute(f"DROP TABLE IF EXISTS {table}") + self.connection.executescript(_SCHEMA) + except sqlite3.DatabaseError: + self.connection.close() + try: + os.remove(self.db_path) + except OSError: + pass + self.connection = sqlite3.connect(self.db_path, check_same_thread=False, timeout=60) + self.connection.executescript(_SCHEMA) + self.connection.execute("INSERT OR REPLACE INTO meta VALUES ('schema_version', ?)", (SCHEMA_VERSION,)) + self.connection.commit() + + def known_files(self) -> Dict[str, Tuple[float, int]]: + with self._lock: + return {row[0]: (row[1], row[2]) for row in self.connection.execute("SELECT path, mtime, size FROM files")} + + def replace_file(self, relative: str, mtime: float, size: int, analysed: Dict[str, Dict]) -> None: + """Drop this file's rows and write the new analysis (key -> stats).""" + with self._lock: + cursor = self.connection.cursor() + old_keys = [row[0] for row in cursor.execute("SELECT key FROM lists WHERE file=?", (relative,))] + for key in old_keys: + cursor.execute("DELETE FROM lists_fts WHERE key=?", (key,)) + cursor.execute("DELETE FROM lists WHERE file=?", (relative,)) + for key, stats in analysed.items(): + values = dict(stats, key=key, file=relative, pack=key.split("/", 1)[0] if "/" in key else "", + samples=json.dumps(stats["samples"], ensure_ascii=False)) + cursor.execute(f"INSERT OR REPLACE INTO lists ({','.join(_LIST_COLUMNS)}) VALUES " + f"({','.join('?' * len(_LIST_COLUMNS))})", [values[c] for c in _LIST_COLUMNS]) + cursor.execute("INSERT INTO lists_fts (key, display, facet, samples) VALUES (?, ?, ?, ?)", + (key, stats["display"], stats["facet"], " | ".join(stats["samples"]))) + cursor.execute("INSERT OR REPLACE INTO files VALUES (?, ?, ?, ?)", (relative, mtime, size, "ok")) + + def forget_files(self, relatives: Iterable[str]) -> None: + with self._lock: + cursor = self.connection.cursor() + for relative in relatives: + for (key,) in list(cursor.execute("SELECT key FROM lists WHERE file=?", (relative,))): + cursor.execute("DELETE FROM lists_fts WHERE key=?", (key,)) + cursor.execute("DELETE FROM lists WHERE file=?", (relative,)) + cursor.execute("DELETE FROM files WHERE path=?", (relative,)) + + def commit(self, built_at: Optional[float] = None) -> None: + with self._lock: + if built_at is not None: + self.connection.execute("INSERT OR REPLACE INTO meta VALUES ('built_at', ?)", (str(built_at),)) + self.connection.commit() + + def rows(self) -> List[ListRow]: + with self._lock: + result = [] + for row in self.connection.execute( + "SELECT key, file, display, kind, dialect, entries, median_words, nsfw, nsfw_source, quality, " + "facet, signature, samples FROM lists"): + result.append(ListRow(key=row[0], root=self.root, file=row[1], display=row[2], kind=row[3], + dialect=row[4], entries=row[5], median_words=row[6], nsfw=row[7], + nsfw_source=row[8], quality=row[9], facet=row[10], signature=row[11], + samples=json.loads(row[12] or "[]"))) + return result + + def search(self, query: str, limit: int = 50, any_word: bool = True) -> List[str]: + """ + FTS keys, best first. Words are prefix-matched ('hair col' finds hair_colors). Lists that + have every word come first, then (any_word) lists that have any of them. + """ + terms = [re.sub(r"[^\w]", "", word) for word in query.split()] + terms = [term for term in terms if term] + if not terms: + return [] + # Packs write decades as 80s, 1980s or eighties; search all of them. + decades = {"1920s": "twenties", "1930s": "thirties", "1940s": "forties", "1950s": "fifties", + "1960s": "sixties", "1970s": "seventies", "1980s": "eighties", "1990s": "nineties"} + expanded = [] + for term in terms: + match = re.fullmatch(r"(?:19)?([2-9]0)s", term.lower()) + if match: + full = "19" + match.group(1) + "s" + expanded.append("(" + " OR ".join(f"{alt}*" for alt in (full, match.group(1) + "s", decades[full])) + ")") + else: + expanded.append(f"{term}*") + found: List[str] = [] + expressions = [" AND ".join(expanded)] + if len(expanded) > 1 and any_word: + expressions.append(" OR ".join(expanded)) + with self._lock: + for expression in expressions: + try: + # The list's name counts far more than a word that happens to appear in its samples. + rows = self.connection.execute( + "SELECT key FROM lists_fts WHERE lists_fts MATCH ? " + "ORDER BY bm25(lists_fts, 8.0, 10.0, 3.0, 1.0) LIMIT ?", (expression, limit)).fetchall() + except sqlite3.OperationalError: + continue + for (key,) in rows: + if key not in found: + found.append(key) + if len(found) >= limit: + break + return found[:limit] + + def close(self) -> None: + with self._lock: + self.connection.close() + + +def catalog_path_for(root: str, fallback_dir: str, writable_root: bool = True) -> str: + if writable_root: + candidate = os.path.join(root, CATALOG_FILE) + try: + with open(candidate, "ab"): + pass + return candidate + except OSError: + pass + digest = hashlib.sha1(os.path.normcase(os.path.abspath(root)).encode("utf-8")).hexdigest()[:16] + return os.path.join(fallback_dir, f"catalog_{digest}.sqlite") + + +class CatalogSet: + """ + Catalogs for every search folder of one WildcardLibrary, plus the merged in-memory view the + engine and the builders read (`row(key)`), with duplicate detection across folders. + + in_root_catalog(root) decides whether a folder gets its catalog inside it (True) or in + `fallback_dir` (the package's own wildcards/ folder uses the cache, so git stays clean). + """ + + def __init__(self, library, fallback_dir: str, in_root_catalog: Callable[[str], bool] = lambda root: True): + self.library = library + self.fallback_dir = fallback_dir + self.in_root_catalog = in_root_catalog + self.catalogs: Dict[str, RootCatalog] = {} + self._rows: Dict[str, ListRow] = {} + self.version = 0 + self.status = {"state": "idle", "done": 0, "total": 0, "seconds": 0.0, "changed_files": 0, "error": ""} + self._update_lock = threading.Lock() + self._open_catalogs() + self._reload_rows() + + # -- setup -------------------------------------------------------------------- + def _open_catalogs(self) -> None: + for root in self.library.search_paths: + if root in self.catalogs: + continue + try: + path = catalog_path_for(root, self.fallback_dir, self.in_root_catalog(root)) + self.catalogs[root] = RootCatalog(root, path) + except (OSError, sqlite3.Error) as error: + self.status["error"] = f"catalog for {root} unavailable: {error}" + + def _reload_rows(self) -> None: + """Merged rows: for a key in several folders the highest-priority folder's row wins.""" + rows: Dict[str, ListRow] = {} + for root in reversed(self.library.search_paths): + catalog = self.catalogs.get(root) + if catalog is None: + continue + for row in catalog.rows(): + rows[row.key] = row + # Identical lists in different packs: the best-scored one is the original, the others point to it. + by_signature: Dict[str, List[ListRow]] = {} + for row in rows.values(): + if row.entries >= 3: + by_signature.setdefault(row.signature, []).append(row) + for group in by_signature.values(): + if len(group) < 2: + continue + group.sort(key=lambda r: (-r.quality, len(r.key), r.key)) + for duplicate in group[1:]: + duplicate.dup_of = group[0].key + self._rows = rows + self.version += 1 + + # -- reading ------------------------------------------------------------------ + def row(self, key: str) -> Optional[ListRow]: + return self._rows.get(key) + + def rows(self) -> Dict[str, ListRow]: + return self._rows + + def search(self, query: str, limit: int = 50, any_word: bool = True) -> List[ListRow]: + found: List[ListRow] = [] + seen = set() + for root in self.library.search_paths: + catalog = self.catalogs.get(root) + if catalog is None: + continue + for key in catalog.search(query, limit, any_word): + row = self._rows.get(key) + if row is not None and key not in seen: + seen.add(key) + found.append(row) + return found[:limit] + + # -- building ----------------------------------------------------------------- + def update(self, progress: Optional[Callable[[Dict], None]] = None) -> Dict: + """Analyse new and changed files; forget removed ones. Returns the status dict.""" + if not self._update_lock.acquire(blocking=False): + return dict(self.status) # another update is running + started = time.time() + try: + self.status.update(state="scanning", done=0, total=0, changed_files=0, error="") + self._open_catalogs() + sources = self.library.catalog_sources() # root -> {relative file: (abs path, [keys], folder_nsfw)} + work: List[Tuple[RootCatalog, str, str, List[str], float, int]] = [] + for root, files in sources.items(): + catalog = self.catalogs.get(root) + if catalog is None: + continue + known = catalog.known_files() + for relative, (path, keys) in files.items(): + if relative.startswith("_prompts/"): + continue # prompt files are browsed by folder, not ranked or searched as lists + try: + stat = os.stat(path) + except OSError: + continue + if known.get(relative) != (stat.st_mtime, stat.st_size): + work.append((catalog, relative, path, keys, stat.st_mtime, stat.st_size)) + removed = set(known) - set(files) + if removed: + catalog.forget_files(removed) + self.status.update(state="analysing", total=len(work)) + for index, (catalog, relative, path, keys, mtime, size) in enumerate(work, 1): + analysed = {} + for key in keys: + texts, folder_nsfw, name_nsfw = self.library.catalog_entries(catalog.root, key) + analysed[key] = analyse(key, texts, folder_nsfw=folder_nsfw, name_nsfw=name_nsfw) + catalog.replace_file(relative, mtime, size, analysed) + self.status["done"] = index + if index % 50 == 0: + catalog.commit() # short write transactions: other processes can write in between + if progress: + progress(dict(self.status)) + for catalog in self.catalogs.values(): + catalog.commit(built_at=time.time()) + self.status.update(changed_files=len(work)) + self._reload_rows() + self.status.update(state="ready") + except Exception as error: # a catalog problem must never break wildcard runs + self.status.update(state="error", error=str(error)) + finally: + self.status["seconds"] = round(time.time() - started, 2) + self._update_lock.release() + return dict(self.status) + + def close(self) -> None: + for catalog in self.catalogs.values(): + catalog.close() diff --git a/core/wildcards/curation.py b/core/wildcards/curation.py new file mode 100644 index 0000000..0dfa9a6 --- /dev/null +++ b/core/wildcards/curation.py @@ -0,0 +1,349 @@ +""" +Curation: the catalog's automatic judgement plus the user's own, in one object the library asks. + +The user's judgement lives in `_catalog_user.yaml` in each search folder, written by the list viewer +(stars, hide, drop) and fine to edit by hand. The indexer never touches it. + + lists: + "mrweazsfwildcardssfw_v30glamour/hair_colors": + stars: 5 # 1..5; ranks lists (slot buttons, ambiguous names) + name: "Hair colours, long" + "cushyswildcard_v6/pose": + hidden: true # never picked by a bare name, glob or slot; __exact/key__ still works + slot: hair_color # offered on this Composer button (set in the wildcard review), whatever its name + "thegenesisenginea_v10/the genesis engine/camera": + nsfw: off # the list's level, overrides folder/name/catalog (off | suggestive | nudity | explicit) + ratings: {lace boyshorts: off} # one entry's level in this list (overrides its words) + "thepromptbuilder_v21ponyillustrious/face_expression": + drop: [Sarcasm, Smug] # these entries are never picked (case-insensitive) + favorites: [] + ratings: # an entry's level in every list (overrides its words) + Naked Mole Rat: off + +What the library asks (see WildcardLibrary.curation): + hidden(key) dropped(key) nsfw_rating(key) -> 0..3 / None entry_rating(key, text) -> 0..3 / None + score(key) -> 0..1 version + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import os +import tempfile +import threading +import time +from typing import Dict, List, Optional, Set + +from .library import NSFW_EXPLICIT, NSFW_LEVELS, nsfw_level, strip_nsfw_prefix + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + +OVERLAY_FILE = "_catalog_user.yaml" +RECHECK_SECONDS = 2.0 # how often hand edits to the overlay files are looked for +_HEADER = ( + "# Your curation of the wildcard lists in this folder: stars, hidden lists, dropped entries.\n" + "# Written by the Wildcard Composer's list viewer; safe to edit by hand. See docs/PROMPT_BUILDER_GUIDE.md.\n" +) + + +class UserOverlay: + """_catalog_user.yaml of one folder. Re-read when the file changes on disk.""" + + def __init__(self, root: str): + self.root = root + self.path = os.path.join(root, OVERLAY_FILE) + self.data: Dict = {"lists": {}, "favorites": [], "ratings": {}} + self._mtime = None + self.error = "" + self.reload_if_changed() + + def reload_if_changed(self) -> bool: + try: + mtime = os.path.getmtime(self.path) + except OSError: + mtime = None + if mtime == self._mtime: + return False + self._mtime = mtime + data: Dict = {} + if mtime is not None and yaml is not None: + try: + with open(self.path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + self.error = "" + except Exception as error: # a hand-edit typo must not break wildcard runs + self.error = f"{self.path}: {error}" + data = {} + lists = data.get("lists") if isinstance(data, dict) else None + ratings = data.get("ratings") if isinstance(data, dict) else None + aliases = data.get("aliases") if isinstance(data, dict) else None + self.data = { + "lists": {str(k).strip().lower(): (v if isinstance(v, dict) else {}) for k, v in (lists or {}).items()}, + "favorites": [str(k).lower() for k in (data.get("favorites") or [])] if isinstance(data, dict) else [], + "ratings": {str(k).strip(): v for k, v in ratings.items() if str(k).strip()} if isinstance(ratings, dict) else {}, + # old name -> new name, written when the wildcard review renames a list + "aliases": {str(k).strip().lower(): str(v).strip().lower() for k, v in aliases.items() if str(k).strip() and str(v).strip()} + if isinstance(aliases, dict) else {}, + } + return True + + def list(self, key: str) -> Dict: + return self.data["lists"].get(key, {}) + + def save(self) -> None: + if yaml is None: + raise RuntimeError("PyYAML is not installed") + lists = {key: value for key, value in sorted(self.data["lists"].items()) if value} + payload = {"lists": lists, "favorites": self.data.get("favorites", [])} + if self.data.get("ratings"): + payload["ratings"] = dict(sorted(self.data["ratings"].items(), key=lambda item: item[0].casefold())) + if self.data.get("aliases"): + payload["aliases"] = dict(sorted(self.data["aliases"].items())) + text = _HEADER + yaml.safe_dump(payload, allow_unicode=True, sort_keys=False, width=120) + directory = os.path.dirname(self.path) + handle, temporary = tempfile.mkstemp(prefix="_catalog_user.", suffix=".tmp", dir=directory) + try: + with os.fdopen(handle, "w", encoding="utf-8") as out: + out.write(text) + os.replace(temporary, self.path) + finally: + if os.path.exists(temporary): + os.remove(temporary) + self._mtime = os.path.getmtime(self.path) + + +class Curation: + """ + catalog: a CatalogSet (or None). overlays are created for every search folder of the library. + `version` changes whenever an answer may change, so the library rebuilds its cached views. + """ + + def __init__(self, library, catalog=None): + self.library = library + self.catalog = catalog + self.overlays: Dict[str, UserOverlay] = {root: UserOverlay(root) for root in library.search_paths} + self._lock = threading.Lock() + self._own_version = 0 + self._dropped_cache: Dict[str, Set[str]] = {} + self._ratings_cache: Dict[str, Dict[str, int]] = {} # key ("" = every list) -> {casefolded text: level} + self._slot_cache: Optional[Dict[str, List[str]]] = None # Composer button id -> keys you assigned to it + self._last_check = 0.0 + + # -- bookkeeping ---------------------------------------------------------------- + @property + def version(self) -> int: + changed = False + now = time.time() + if now - self._last_check >= RECHECK_SECONDS: + self._last_check = now + for overlay in self.overlays.values(): + changed = overlay.reload_if_changed() or changed + if changed: + self._own_version += 1 + self._dropped_cache = {} + self._ratings_cache = {} + self._slot_cache = None + self._alias_cache = None + return self._own_version * 1_000_003 + (self.catalog.version if self.catalog is not None else 0) + + def _entry(self, key: str) -> Dict: + """ + The user's settings for a key: the highest-priority folder that says anything wins. The list viewer saves + under the name a list answers to (pack/x for _nsfw/pack/x, prompts/x for _prompts/x); the library asks with + the real key. Both spellings find the same settings. + """ + self.version # picks up hand edits (checked at most every RECHECK_SECONDS) + spellings = [key] + name = strip_nsfw_prefix(key) + if name != key: + spellings.append(name) + for spelling in spellings: + for root in self.library.search_paths: + overlay = self.overlays.get(root) + if overlay is not None: + found = overlay.list(spelling) + if found: + return found + return {} + + def _row(self, key: str): + return self.catalog.row(key) if self.catalog is not None else None + + # -- what the library asks -------------------------------------------------------- + def hidden(self, key: str) -> bool: + return bool(self._entry(key).get("hidden")) + + def dropped(self, key: str) -> Set[str]: + cached = self._dropped_cache.get(key) + if cached is None: + cached = {str(item).strip().casefold() for item in (self._entry(key).get("drop") or []) if str(item).strip()} + self._dropped_cache[key] = cached + return cached + + def nsfw_rating(self, key: str) -> Optional[int]: + """ + A list's level 0..3 when the user or the catalog says: the user's nsfw: wins; the catalog rates a + list explicit by its content. None = no opinion (the folder / name rules decide). + """ + user = self._entry(key).get("nsfw", "") + if isinstance(user, bool) or str(user or "").strip(): + return nsfw_level(user) + row = self._row(key) + if row is not None and row.nsfw == "explicit" and row.nsfw_source == "content": + return NSFW_EXPLICIT + return None + + def entry_rating(self, key: str, text: str) -> Optional[int]: + """An entry's level when the user set one: in this list's ratings, else the top-level ratings. None = its words decide.""" + folded = text.strip().casefold() + own = self._ratings_for(key) + if folded in own: + return own[folded] + every = self._ratings_for("") + return every.get(folded) + + def _ratings_for(self, key: str) -> Dict[str, int]: + self.version # picks up hand edits + cached = self._ratings_cache.get(key) + if cached is not None: + return cached + if key: + raw = self._entry(key).get("ratings") or {} + merged = {str(t).strip().casefold(): nsfw_level(v) for t, v in raw.items()} if isinstance(raw, dict) else {} + else: + merged = {} + for root in reversed(self.library.search_paths): # the highest-priority folder wins + overlay = self.overlays.get(root) + if overlay is not None: + merged.update({str(t).strip().casefold(): nsfw_level(v) for t, v in overlay.data.get("ratings", {}).items()}) + self._ratings_cache[key] = merged + return merged + + def slot_keys(self, slot_id: str) -> List[str]: + """Keys you assigned to a Composer button (`slot:` in _catalog_user.yaml, highest-priority folder first).""" + self.version # picks up hand edits + if self._slot_cache is None: + by_slot: Dict[str, List[str]] = {} + for root in self.library.search_paths: + overlay = self.overlays.get(root) + if overlay is None: + continue + for key, settings in overlay.data.get("lists", {}).items(): + slots = settings.get("slot") or [] + for slot in ([slots] if isinstance(slots, str) else slots): # one button id, or several + slot = str(slot or "").strip() + if slot and not settings.get("hidden") and key not in by_slot.get(slot, []): + by_slot.setdefault(slot, []).append(key) + self._slot_cache = by_slot + return list(self._slot_cache.get(slot_id, [])) + + def alias_of(self, name: str) -> Optional[str]: + """The new name of a list the wildcard review renamed (aliases: in _catalog_user.yaml), else None.""" + self.version # picks up hand edits + cache = getattr(self, "_alias_cache", None) + if cache is None: + cache = {} + for root in reversed(self.library.search_paths): # the highest-priority folder wins + overlay = self.overlays.get(root) + if overlay is not None: + cache.update(overlay.data.get("aliases") or {}) + self._alias_cache = cache + return cache.get(name) + + def stars(self, key: str) -> int: + try: + return max(0, min(5, int(self._entry(key).get("stars", 0) or 0))) + except (TypeError, ValueError): + return 0 + + def quality(self, key: str) -> float: + row = self._row(key) + return row.quality if row is not None else 0.5 + + def score(self, key: str) -> float: + """0..1 for ordering: the user's stars when given, else the catalog's quality.""" + stars = self.stars(key) + if stars: + return 0.5 + stars / 10.0 # any starred list outranks unstarred ones (quality <= 1 but ~0.9 max) + return self.quality(key) * 0.5 + + # -- what the list viewer changes ---------------------------------------------------- + def root_for_key(self, key: str) -> Optional[str]: + """The folder whose list the engine uses for this key (where its curation belongs).""" + sources = self.library.catalog_sources() + # A name a list answers to (pack/x, prompts/x) is found through its real keys (_nsfw/pack/x, _prompts/x). + wanted = [key] + [real for real in self.library.real_keys(key) if real != key] + for root in self.library.search_paths: + for _relative, (_path, keys) in sources.get(root, {}).items(): + if any(real in keys for real in wanted): + return root + return None + + def update(self, key: str, root: Optional[str] = None, **changes) -> Dict: + """ + Change the user's settings for a key and save that folder's _catalog_user.yaml. + changes: stars (0 clears), hidden (bool), nsfw (a level name, '' clears), name (str), + drop (str, add), undrop (str, remove), rate ({"text": ..., "level": ...}; level '' clears). + """ + with self._lock: + root = root or self.root_for_key(key) + if root is None or root not in self.overlays: + raise KeyError(f"no folder holds '{key}'") + overlay = self.overlays[root] + overlay.reload_if_changed() + entry = dict(overlay.list(key)) + if "stars" in changes: + stars = int(changes["stars"] or 0) + if stars: + entry["stars"] = max(1, min(5, stars)) + else: + entry.pop("stars", None) + if "hidden" in changes: + if changes["hidden"]: + entry["hidden"] = True + else: + entry.pop("hidden", None) + if "nsfw" in changes: + if str(changes["nsfw"] or "").strip(): + entry["nsfw"] = NSFW_LEVELS[nsfw_level(changes["nsfw"])] + else: + entry.pop("nsfw", None) + if changes.get("rate"): + text = str(changes["rate"].get("text", "")).strip() + ratings = {str(t): v for t, v in (entry.get("ratings") or {}).items() if str(t).casefold() != text.casefold()} + if text and str(changes["rate"].get("level", "") or "").strip(): + ratings[text] = NSFW_LEVELS[nsfw_level(changes["rate"]["level"])] + if ratings: + entry["ratings"] = ratings + else: + entry.pop("ratings", None) + if "name" in changes: + if changes["name"]: + entry["name"] = str(changes["name"]) + else: + entry.pop("name", None) + drops: List[str] = list(entry.get("drop") or []) + if changes.get("drop"): + text = str(changes["drop"]).strip() + if text and text.casefold() not in {d.casefold() for d in drops}: + drops.append(text) + if changes.get("undrop"): + text = str(changes["undrop"]).strip().casefold() + drops = [d for d in drops if d.casefold() != text] + if drops: + entry["drop"] = drops + else: + entry.pop("drop", None) + if entry: + overlay.data["lists"][key] = entry + else: + overlay.data["lists"].pop(key, None) + overlay.save() + self._own_version += 1 + self._dropped_cache = {} + self._ratings_cache = {} + return entry diff --git a/core/wildcards/engine.py b/core/wildcards/engine.py index 9e0e095..db385d7 100644 --- a/core/wildcards/engine.py +++ b/core/wildcards/engine.py @@ -17,6 +17,21 @@ the first folder that has it wins (see library.py): 3. "wildcard_paths" in /config.json any other folder of packs 4. /wildcards/ lists that ship with the package +Options from config.json (all optional): + "wildcard_paths": ["D:/more/packs"] extra search folders (3. above) + "wildcards": { + "pool_mode": "scoped", "scoped" (default): a bare __name__ never mixes lists of + different packs. "legacy": the old pool-everything lookup. + "strip_lora": true, remove tags from entries at load time + "content_filter": true, drop entries pairing minor words with sexual words + (a word match at load time; false turns it off) + "catalog": true index the lists (catalog.py) and honour the user's + _catalog_user.yaml (curation.py): stars, hidden lists, + dropped entries, content-detected NSFW + } +Lists and entries rated above a run's nsfw level (library.NSFW_LEVELS) are invisible to process(..., nsfw=...). +See library.py for the folder rules and the content filter. + (c) Eric Hiss. Dual license, see LICENSE. """ @@ -27,7 +42,11 @@ import os import random from typing import Dict, Iterable, List, Optional, Tuple -from .library import WildcardLibrary +import threading + +from .catalog import CatalogSet +from .curation import Curation +from .library import POOL_MODES, POOL_SCOPED, WildcardLibrary, prompt_block_reason from .record import Resolution from .resolver import MODE_RANDOM, PickHook, ProviderCallable, Resolver, tidy @@ -49,6 +68,26 @@ def configured_paths() -> List[str]: return [str(path) for path in paths if path] +def configured_options() -> Dict[str, object]: + """The "wildcards" section of config.json, with defaults: pool_mode, strip_lora, catalog, content_filter.""" + options: Dict[str, object] = {"pool_mode": POOL_SCOPED, "strip_lora": True, "catalog": True, "content_filter": True} + try: + with open(CONFIG_PATH, "r", encoding="utf-8") as handle: + section = json.load(handle).get("wildcards", {}) + except (OSError, ValueError): + return options + if isinstance(section, dict): + if section.get("pool_mode") in POOL_MODES: + options["pool_mode"] = section["pool_mode"] + if isinstance(section.get("strip_lora"), bool): + options["strip_lora"] = section["strip_lora"] + if isinstance(section.get("catalog"), bool): + options["catalog"] = section["catalog"] + if isinstance(section.get("content_filter"), bool): + options["content_filter"] = section["content_filter"] + return options + + def split_paths(raw: str) -> List[str]: """Parse a node's extra_paths text: one folder per line, or separated by ';'.""" parts: List[str] = [] @@ -62,14 +101,60 @@ def split_paths(raw: str) -> List[str]: class WildcardEngine: def __init__(self, extra_paths: Optional[Iterable[str]] = None, include_builtin: bool = True, include_configured: bool = True, host_paths: Optional[Iterable[str]] = None, - merge_roots: bool = False): + merge_roots: bool = False, pool_mode: Optional[str] = None, + strip_lora: Optional[bool] = None, content_filter: Optional[bool] = None): paths: List[str] = list(extra_paths or []) paths.extend(host_paths or []) if include_configured: paths.extend(configured_paths()) if include_builtin: paths.append(BUILTIN_WILDCARD_DIR) - self.library = WildcardLibrary(paths, merge_roots=merge_roots) + options = configured_options() if include_configured else {"pool_mode": POOL_SCOPED, "strip_lora": True, + "content_filter": True} + self.library = WildcardLibrary( + paths, merge_roots=merge_roots, + pool_mode=pool_mode if pool_mode is not None else str(options["pool_mode"]), + strip_lora=strip_lora if strip_lora is not None else bool(options["strip_lora"]), + content_filter=content_filter if content_filter is not None else bool(options.get("content_filter", True)), + ) + self.catalog = None + self.curation = None + self.catalog_enabled = bool(options.get("catalog", True)) + self._catalog_thread: Optional[threading.Thread] = None + + # -- catalog and curation ------------------------------------------------------------ + def enable_curation(self, fallback_dir: str, build_in_background: bool = True) -> None: + """ + Attach the catalog (catalog.py) and the user's curation (curation.py) to the library. + Existing catalog files are read at once; new or changed lists are analysed in a background + thread when build_in_background, else call update_catalog() yourself. Idempotent. + """ + if self.curation is not None or not self.catalog_enabled: + return + builtin = os.path.normcase(os.path.abspath(BUILTIN_WILDCARD_DIR)) + try: + self.catalog = CatalogSet( + self.library, fallback_dir, + in_root_catalog=lambda root: os.path.normcase(os.path.abspath(root)) != builtin, + ) + except Exception as error: # no catalog: curation still works from the overlay files + print(f"[wildcards] catalog unavailable: {error}") + self.catalog = None + self.curation = Curation(self.library, self.catalog) + self.library.curation = self.curation + if build_in_background and self.catalog is not None: + self._catalog_thread = threading.Thread(target=self.update_catalog, name="wildcard-catalog", daemon=True) + self._catalog_thread.start() + + def update_catalog(self) -> Dict: + if self.catalog is None: + return {"state": "off"} + status = self.catalog.update() + if status.get("state") == "ready" and status.get("changed_files"): + print(f"[wildcards] catalog: {status['changed_files']} file(s) indexed in {status['seconds']} s") + elif status.get("state") == "error": + print(f"[wildcards] catalog update failed: {status.get('error')}") + return status def process( self, @@ -83,10 +168,19 @@ class WildcardEngine: variables: Optional[Dict[str, str]] = None, cleanup: bool = True, pick_hook: Optional[PickHook] = None, + nsfw="off", + violence=0, ) -> Resolution: """ Resolve every wildcard construct in `text`. Never raises on malformed prompts or broken wildcard files: problems become warnings and the offending text is kept as is. + nsfw: the run's level (library.NSFW_LEVELS name or 0-3): lists and entries rated above it are left out; + a reference with nothing left at that level is dropped (the report says so). + violence: the node's violence_filter (library.VIOLENCE_LEVELS name or 0-3); entries in that + tier or a lower one are skipped. + With the content filter on, a result that pairs a minor term with an explicit sexual term + (library.prompt_block_reason) is re-rolled with derived seeds when the picks caused it, and + blanked (resolution.blocked says why) when the typed text did or no re-roll passes. """ if not text: return Resolution(text=text or "", seed=seed, mode=mode) @@ -99,15 +193,44 @@ class WildcardEngine: self.library.warnings = [] self.library.refresh() - resolver = Resolver( - self.library, seed=seed, mode=mode, index=index, cycle_index=cycle_index, - legacy=legacy, providers=providers, variables=variables, pick_hook=pick_hook, - ) - resolution = resolver.resolve_text(text) - resolution.fresh_roll = fresh_roll - if cleanup: - resolution.text = tidy(resolution.text) - return resolution + + def run(run_seed: int) -> Resolution: + resolver = Resolver( + self.library, seed=run_seed, mode=mode, index=index, cycle_index=cycle_index, + legacy=legacy, providers=providers, variables=variables, pick_hook=pick_hook, + nsfw=nsfw, violence=violence, + ) + result = resolver.resolve_text(text) + result.fresh_roll = fresh_roll + if cleanup: + result.text = tidy(result.text) + return result + + resolution = run(seed) + if not self.library.content_filter: + return resolution + reason = prompt_block_reason(resolution.text) + if not reason: + return resolution + if not prompt_block_reason(text): # the picks made it, not the typed text: re-roll + for attempt in range(1, CONTENT_REROLLS + 1): + retry = run((seed + attempt * 7919) % 0x7FFFFFFF) + if not prompt_block_reason(retry.text): + retry.warnings.append(f"content filter: seed {seed} picked {reason}; re-rolled to seed {retry.seed}") + return retry + return blocked_resolution(resolution, reason) + + +CONTENT_REROLLS = 6 # derived seeds tried before a prompt the content filter fails is blanked + + +def blocked_resolution(resolution: Resolution, reason: str) -> Resolution: + """The prompt blanked by the content filter: empty text, the reason in .blocked and in the warnings.""" + resolution.text = "" + resolution.blocked = reason + resolution.warnings.append(f"content filter: prompt blocked ({reason}); set \"content_filter\": false " + f"in config.json to turn the filter off") + return resolution _ENGINES: Dict[Tuple[str, ...], WildcardEngine] = {} diff --git a/core/wildcards/library.py b/core/wildcards/library.py index f568c51..d1693e1 100644 --- a/core/wildcards/library.py +++ b/core/wildcards/library.py @@ -13,22 +13,75 @@ File formats - text: a cast-iron skillet tags: [metal, kitchen, handheld, heavy] weight: 2 + A leaf that is a single string counts as a one-item list. Tags are what tag queries (__objects[metal AND handheld]__) search. +Folder layout inside a search folder + /... ordinary packs + _nsfw//... NSFW lists, rated explicit. Indexed, but only visible to a run at that level. + _nsfw/suggestive//..., _nsfw/nudity//... rate a folder lower. + Inside that run they answer to their name without the _nsfw/ prefix. + _generated/... lists written by the tools themselves. Always searched. + _prompts//... whole prompts, one per line. Indexed as their own area: a file answers ONLY to + prompts//.../ (and __prompts//*__ or __prompts/__ to every + file in a folder). Bare names, ordinary globs and the Composer's buttons never see them. + _prompts/_nsfw/... is rated like _nsfw/ and answers to prompts//... + _anything_else/ any other top-level folder starting with "_" is skipped + (_duplicates, _quarantine, _cleanup, ...). + Skipped at any depth: __MACOSX, .git and other VCS folders, macOS "._" fork files. + +NSFW levels (NSFW_LEVELS: off, suggestive, nudity, explicit) + A run has a level (a node's nsfw setting); it uses lists and entries rated at that level or lower. + off is SFW: it leaves out suggestive content too. Ratings are word matches, no model: + list its folder (_nsfw/ = explicit), its name below the pack folder (lingerie/underwear/bikini -> + suggestive, nude/naked/topless -> nudity, nsfw/sex/porn/fetish -> explicit), or the user's + override in _catalog_user.yaml (lists: : nsfw: ). A list above the run's level + does not exist for lookup, globs or tag queries. + entry its words (nsfw_word_tier, with exemptions such as "Garter Snake" or "nude pumps"), or the + user's override in _catalog_user.yaml (lists: : ratings: {entry: level}, or the + top-level ratings: {entry: level} for every list). + +Content filter (on by default) + Entries that pair minor-related words with sexual words are dropped at load time in + every folder: a plain word match on each entry, no model involved. `blocked_entries` + counts them. content_filter=False (config.json "wildcards": {"content_filter": false}) + turns it off. prompt_block_reason() is the same test on a whole finished prompt, with explicit + sexual terms only (nudity words alone do not count there); the engine re-rolls or blanks a + prompt it fails (engine.process). + +Violence filter (per run, off by default) + A node's violence_filter setting (VIOLENCE_LEVELS: off, no gore, no gore or violent acts, no + violence or weapons) makes the resolver skip entries whose words, or whose list's name, fall in + that tier or a lower one. A word match (violence_tier / violence_name_tier), no model. + +LoRA tags + tags inside entries are removed at load time unless strip_lora=False + (config.json "wildcards": {"strip_lora": false}). `stripped_lora_tags` counts them. + Several folders Search folders are ordered, highest priority first. When the same wildcard key exists in more than one folder, the FIRST folder that has it wins and the others are ignored for that key. So a pack in models/wildcards overrides a list that ships with this package, the same way ComfyUI model folders behave. Pass merge_roots=True to pool them instead. -Lookup (all case-insensitive) +Lookup (all case-insensitive), pool_mode "scoped" (default) exact __style/painters__ - glob __colors*__ __artists/**__ (fnmatch over keys) - depth-agnostic __painters__ also finds style/painters when no exact key exists + glob __colors*__ __artists/**__ __**/pose__ (fnmatch; the explicit way to pool) + same pack a reference inside a pack's own list resolves inside that pack first: + randomPortrait/.../template.txt using __wildcards/quality__ finds + randomPortrait/wildcards/quality before anything else + depth-agnostic __painters__ also finds style/painters when no exact key exists. + When lists of that name exist in several packs, ONE pack is used + (the calling pack, else the highest-priority folder, else the shortest + path) and a warning names the alternatives. Lists of different packs + are never mixed by accident. + folder __materials__ stands for everything under materials/ (same one-pack rule) relocated pack __look/styles__ also finds theLookWildcards_v10/styles: packs are written for a folder name ("look") that rarely survives unzipping. Accepted only when the folder that holds "styles" has the expected name INSIDE its own name, and exactly one such folder exists. Otherwise nothing is guessed. + pool_mode "legacy" restores the old behaviour: depth-agnostic and folder matches pool + every pack. Loading is lazy for .txt (read on first use) and eager for YAML/JSON (their keys are only known after reading). Files are re-read when their mtime changes; the folder @@ -42,10 +95,12 @@ from __future__ import annotations import fnmatch import json import os +import functools import re +import threading import time from dataclasses import dataclass, field -from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple +from typing import Dict, FrozenSet, Iterable, List, Optional, Set, Tuple from .boolean_expr import BooleanExpressionError, compile_expression, evaluate @@ -54,12 +109,286 @@ try: except ImportError: # pragma: no cover yaml = None -RESCAN_SECONDS = 2.0 +RESCAN_SECONDS = 10.0 TEXT_EXTENSIONS = (".txt",) STRUCTURED_EXTENSIONS = (".yaml", ".yml", ".json") +POOL_SCOPED = "scoped" +POOL_LEGACY = "legacy" +POOL_MODES = (POOL_SCOPED, POOL_LEGACY) + +NSFW_FOLDER = "_nsfw" +GENERATED_FOLDER = "_generated" +PROMPTS_FOLDER = "_prompts" +_SEARCHED_UNDERSCORE_FOLDERS = {NSFW_FOLDER, GENERATED_FOLDER, PROMPTS_FOLDER} +_SKIPPED_FOLDERS = {"__macosx", ".git", ".svn", ".hg", "__pycache__"} +_NSFW_PREFIX = NSFW_FOLDER + "/" +_PROMPTS_PREFIX = PROMPTS_FOLDER + "/" +PROMPTS_NAME = "prompts" # the name the _prompts/ area answers to: prompts// +_PROMPTS_NAME_PREFIX = PROMPTS_NAME + "/" + _WEIGHT_PREFIX = re.compile(r"^\s*(\d+(?:\.\d+)?)::") _HEX_COLOUR = re.compile(r"#[0-9A-Fa-f]{3,8}\b") +_LORA_TAG = re.compile(r"]*>", re.IGNORECASE) +_LORA_SCARS = re.compile(r"^[\s,]+|[\s,]+$|(?<=,)\s*,") + +# (List names are rated by _NAME_TIERS / nsfw_name_tier below.) +# Content filter: an entry matching BOTH patterns is never loaded. +_MINOR_TERMS = re.compile( + r"\b(loli|lolis|shota|child|children|kid|kids|toddler|infant|underage|preteen|pre-teen|little girl|" + r"little boy|young girl|young boy|schoolgirl|elementary school|middle school|" + r"(?:1[0-7]|[1-9]) ?(?:yo|y/o|years? old)|teen|teens|teenage|teenager)\b", + re.IGNORECASE, +) +_SEXUAL_TERMS = re.compile( + r"\b(sex|sexual|fuck\w*|cum|cumshot|cock|dick|penis|pussy|vagina|anal|blowjob|fellatio|cunnilingus|" + r"penetrat\w*|orgasm\w*|masturbat\w*|nipples?|genital\w*|porn\w*|nsfw|hentai|lewd|nude|naked|topless|" + r"bottomless|panties|erotic|creampie|spread legs|pubic)\b", + re.IGNORECASE, +) + + +# Phrases where a minor word names a thing, not a person ("child's pose", "a children's book"); taken +# out before the minor test so they alone never block an adult entry. +_MINOR_EXEMPT = re.compile( + r"\bchild'?s pose\b|\bwinged infants?\b|\binfant (?:jesus|christ)\b|\bchristian child\b|" + r"\bchildren'?s (?:books?|snacks?|toys?|menus?|sections?|stor(?:y|ies)|shows?|tv|television|drawings?|" + r"illustrations?|songs?|library|hospital|museum|theatre|theater|cartoons?|games?)\b", + re.IGNORECASE, +) + + +def is_blocked_entry(text: str) -> bool: + """True for an entry the content filter removes (minor-related and sexual terms together).""" + if not _SEXUAL_TERMS.search(text): + return False + return bool(_MINOR_TERMS.search(_MINOR_EXEMPT.sub(" ", text))) + + +# The same minor test on a whole finished prompt, where terms from different pills meet. Narrower on the +# sexual side: explicit terms only, so nudity next to a child word (putti, a nude study near "mother and +# child") passes; nude / naked / topless / bottomless / nipples are not in this list. +_EXPLICIT_TERMS = re.compile( + r"\b(sex|sexual|fuck\w*|cum|cumshot|cock|dick|penis|pussy|vagina|anal|blowjob|fellatio|cunnilingus|" + r"penetrat\w*|orgasm\w*|masturbat\w*|genital\w*|porn\w*|nsfw|hentai|lewd|panties|erotic|creampie|" + r"spread legs|pubic)\b", + re.IGNORECASE, +) + + +def prompt_block_reason(text: str) -> str: + """ + '' when a finished prompt is fine; else why the content filter blocks it ('"teen" with "erotic"'). + A word match, no model: a minor term and an explicit sexual term anywhere in the same prompt. + """ + if not text: + return "" + sexual = _EXPLICIT_TERMS.search(text) + if not sexual: + return "" + minor = _MINOR_TERMS.search(_MINOR_EXEMPT.sub(" ", text)) + return f'"{minor.group(0)}" with "{sexual.group(0)}"' if minor else "" + + +# Violence filter: a node's violence_filter setting, one of VIOLENCE_LEVELS. Each level blocks its own +# tier and every tier before it: 1 gore and torture, 2 violent acts, 3 weapons and combat. Entries are +# matched by words (no model); lists whose NAME says the tier are hidden as well. +VIOLENCE_LEVELS = ("off", "no gore", "no gore or violent acts", "no violence or weapons") +_VIOLENCE_GORE = re.compile( + r"\b(gore|gory|guro|gruesome|grisly|dismember\w*|decapitat\w*|beheaded|" + r"severed (?:head|heads|limbs?|arms?|legs?|hands?|fingers?)|mutilat\w*|disembowel\w*|eviscerat\w*|" + r"entrails|innards|intestines|guts spill\w*|flayed|skinned alive|impaled|tortur\w*|corpses?|cadavers?|" + r"dead bod(?:y|ies)|rotting flesh|mangled|butchered|(?:open|gaping|bleeding|bloody|stab|gunshot|bullet) wounds?|" + r"covered in blood|blood[- ]?(?:soaked|stained|splatter\w*|spatter\w*|spray\w*|drenched|smeared|dripping)|" + r"pools? of blood|bloodbath|bloodied|bleeding (?:out|profusely)|exposed (?:bone|organs|brain))\b", + re.IGNORECASE, +) +_VIOLENCE_ACTS = re.compile( + r"\b(murder\w*|kill(?:ing|ed|s)? (?:a|an|the|him|her|them|someone|people|man|woman)|slaughter\w*|massacre\w*|" + r"execut(?:ion|ed|ing)|stabb(?:ing|ed)|shot (?:dead|in the|through)|shooting (?:at|someone|a man|a woman|people)|" + r"gunned down|strangl\w*|beat(?:ing|en) (?:up|to death|someone|him|her)|lynching|lynched|hanged|" + r"assault(?:ing|ed)|mauled|bludgeon\w*|war crimes?|genocide|suicide|self[- ]harm|" + r"drowning (?:someone|him|her)|choking (?:someone|him|her))\b", + re.IGNORECASE, +) +_VIOLENCE_WEAPONS = re.compile( + r"\b(guns?|gunfire|rifles?|pistols?|handguns?|revolvers?|shotguns?|machine ?guns?|submachine guns?|firearms?|" + r"sniper\w*|ammunition|ammo|bullets?|grenades?|bombs?|explosives?|missiles?|rocket launchers?|bazookas?|" + r"artillery|swords?|katanas?|daggers?|knives|knife|blades?|machetes?|axes?|battle ?axes?|spears?|halberds?|" + r"maces?|crossbows?|bows? and arrows?|weapons?|weaponry|armed|combat|battles?|battlefields?|battlegrounds?|" + r"warfare|war|firefights?|gunfights?|shootouts?|sword ?fight\w*)\b", + re.IGNORECASE, +) +_VIOLENCE_NAMES = ( + re.compile(r"(? int: + """A node's violence_filter value (name or number) as 0 (off) .. 3.""" + if isinstance(value, bool): + return 0 + if isinstance(value, int): + return max(0, min(3, value)) + text = str(value or "").strip().lower() + if text in VIOLENCE_LEVELS: + return VIOLENCE_LEVELS.index(text) + return int(text) if text.isdigit() and int(text) <= 3 else 0 + + +@functools.lru_cache(maxsize=262144) +def violence_tier(text: str) -> int: + """0 = no violence words; 1 gore / torture; 2 violent acts; 3 weapons / combat (the most severe wins).""" + if _VIOLENCE_GORE.search(text): + return 1 + if _VIOLENCE_ACTS.search(text): + return 2 + if _VIOLENCE_WEAPONS.search(text): + return 3 + return 0 + + +@functools.lru_cache(maxsize=65536) +def violence_name_tier(key: str) -> int: + """The tier a list's NAME says (the path below the pack, as is_nsfw_name reads it), 0 = none.""" + below = key.split("/", 1)[1] if "/" in key else key + spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", below).lower().replace("_", " ").replace("-", " ") + for tier, pattern in enumerate(_VIOLENCE_NAMES, start=1): + if pattern.search(spaced): + return tier + return 0 + + +def violence_blocked(text: str, key: str, level: int) -> bool: + """True when a node with this violence_filter level must skip the entry (its words or its list's name).""" + if level <= 0: + return False + name_tier = violence_name_tier(key) if key else 0 + if name_tier and name_tier <= level: + return True + tier = violence_tier(text) + return bool(tier) and tier <= level + + +# NSFW levels: what a run may use. Lists and entries are rated 0..3; a run at level N uses those rated N or +# lower. Same levels as the LoRA cards and the LoRA Suggester. off is SFW and leaves out suggestive content too. +NSFW_LEVELS = ("off", "suggestive", "nudity", "explicit") +NSFW_OFF, NSFW_SUGGESTIVE, NSFW_NUDITY, NSFW_EXPLICIT = 0, 1, 2, 3 +_LEVEL_ALIASES = {"sfw": 0, "safe": 0, "none": 0, "false": 0, "no": 0, "nsfw": 3, "true": 3, "yes": 3, "nude": 2} + + +def nsfw_level(value) -> int: + """A node's nsfw value (level name, 0-3, or a yes/no) as 0 (off) .. 3 (explicit).""" + if isinstance(value, bool): + return NSFW_EXPLICIT if value else NSFW_OFF + if isinstance(value, int): + return max(0, min(3, value)) + text = str(value or "").strip().lower() + if text in NSFW_LEVELS: + return NSFW_LEVELS.index(text) + if text in _LEVEL_ALIASES: + return _LEVEL_ALIASES[text] + return int(text) if text.isdigit() and int(text) <= 3 else NSFW_OFF + + +# A list's NAME (the path below the pack folder, camelCase split, _ and - read as spaces). A pack called +# "SFWNSFW_bundle" is judged by its files, not its title. The most severe tier that matches wins. +_NAME_TIERS = ( + (NSFW_EXPLICIT, re.compile( + r"nsfw|xxx|(? int: + """The level a list's name says: the path below the pack folder, or the whole name of a loose file / YAML key.""" + spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", relative_path).lower() + spaced = spaced.replace("_", " ").replace("-", " ").replace("\\", "/") + spaced = _NSFW_EXEMPT.sub(" ", spaced) # "nude makeup", "garter snake": the word means something else + for tier, pattern in _NAME_TIERS: + if pattern.search(spaced): + return tier + return NSFW_OFF + + +def nsfw_folder_tier(key: str) -> int: + """_nsfw//... = explicit; _nsfw/suggestive|nudity|explicit//... = that level; elsewhere 0. + The same below _prompts/ (_prompts/_nsfw/...).""" + if key.startswith(_PROMPTS_PREFIX): + key = key[len(_PROMPTS_PREFIX):] + if not key.startswith(_NSFW_PREFIX): + return NSFW_OFF + first = key[len(_NSFW_PREFIX):].split("/", 1)[0] + return NSFW_LEVELS.index(first) if first in NSFW_LEVELS[1:] else NSFW_EXPLICIT + + +def strip_nsfw_prefix(key: str) -> str: + """ + The name a list answers to. An _nsfw/ list: without _nsfw/ (and without its level folder, if it has one). + A _prompts/ file: prompts/ + the rest (and _prompts/_nsfw/x -> prompts/x). + """ + if key.startswith(_PROMPTS_PREFIX): + return _PROMPTS_NAME_PREFIX + strip_nsfw_prefix(key[len(_PROMPTS_PREFIX):]) + if not key.startswith(_NSFW_PREFIX): + return key + rest = key[len(_NSFW_PREFIX):] + first, _, below = rest.partition("/") + return below if first in NSFW_LEVELS[1:] and below else rest + + +def is_prompt_name(name: str) -> bool: + """A lookup name in the prompt-file area (prompts/...), which answers only to names and globs starting prompts/.""" + return name == PROMPTS_NAME or name.startswith(_PROMPTS_NAME_PREFIX) + + +# An ENTRY's words. A word match, no model; the most severe tier that matches wins. Phrases where a rated word +# names something else are taken out first (_NSFW_EXEMPT, and _NSFW_EXEMPT_NAMES case-sensitively for names). +# These lists are for rating only; the content filter (_SEXUAL_TERMS / _EXPLICIT_TERMS) is separate. +_NSFW_WORDS = ( + (NSFW_EXPLICIT, re.compile( + r"\b(sex|sexual|sex acts?|having sex|fuck\w*|cum|cumshot|cock|cocks|dick|dicks|penis|penises|pussy|vagina|" + r"anal|blowjob|fellatio|cunnilingus|orgasm\w*|masturbat\w*|genital\w*|porn\w*|nsfw|hentai|lewd|creampie|" + r"spread legs|bukkake|ejaculat\w*|erections?|intercourse|penetrated|penetration|xxx)\b", re.IGNORECASE)), + (NSFW_NUDITY, re.compile( + r"\b(nude|nudes|naked|nudity|nudist|topless|bottomless|nipples?|areolae?|areolas|bare[- ]breasted|" + r"bare breasts?|exposed breasts?|breasts exposed|naked breasts?|full[- ]frontal|unclothed|undressed|" + r"skinny[- ]dipping|pubic hair|erotic\w*|in the buff|au naturel)\b", re.IGNORECASE)), + (NSFW_SUGGESTIVE, re.compile( + r"\b(lingerie|negligees?|bikinis?|thongs?|g-strings?|bras?|panties|panty|garters?|garter belts?|" + r"cleavage|sexy|seductive\w*|sultry|provocative\w*|pin-?ups?|boudoir|lap ?dances?|strippers?|striptease|" + r"see-through|wet t-shirt|underwear|skimpy|busty|upskirt|downblouse|bedroom eyes|lustful|lusty|" + r"lust|sensual\w*|scantily(?: clad)?)\b", re.IGNORECASE)), +) +_NSFW_EXEMPT = re.compile( + r"\bgarter snakes?\b|\bbikini atoll\b|\bnaked mole[- ]rats?\b|\bnaked eye\b|\bnaked flames?\b|" + r"\bnaked (?:truth|branches|trees?|bulbs?|light ?bulbs?|lightbulbs?|steel|blades?|ambition|singularity)\b|" + r"\bbottomless (?:pits?|chasms?|abyss|depths?|wells?|void|oceans?|seas?|cups?|mugs?|brunch|mimosas?|holes?|" + r"gorges?|caverns?|lakes?|darkness)\b|" + r"\bnude[- ](?:colou?red|toned?|tones|shades?|pumps?|heels?|flats?|shoes?|sandals?|boots?|lipstick|lips?|" + r"lip gloss|makeup|eyeshadow|nails?|nail polish|polish|palette|beige|mesh|leather|suede|stockings|tights|" + r"pantyhose|blush|pink)\b|\b(?:in |a )?nude (?:colou?r|tones?)\b|" + r"\bchristmas stockings?\b|\bstocking stuffers?\b|\bcum laude\b|\bmagna cum\b|\bsumma cum\b|" + r"\bcock(?:'s)?[- ]?(?:crow\w*|fight\w*|a[- ]doodle\w*)\b|\bcocks? crow\w*\b|\bweather ?cock\b", + re.IGNORECASE, +) +_NSFW_EXEMPT_NAMES = re.compile(r"\b(?:[A-Z]\.|[A-Z][a-z]+) Dick\b|\bDick [A-Z][a-z]+\b|\bMoby[- ]Dick\b") + + +@functools.lru_cache(maxsize=262144) +def nsfw_word_tier(text: str) -> int: + """The level an entry's words say: 0 none, 1 suggestive, 2 nudity, 3 explicit.""" + cleaned = _NSFW_EXEMPT.sub(" ", _NSFW_EXEMPT_NAMES.sub(" ", text)) + for tier, pattern in _NSFW_WORDS: + if pattern.search(cleaned): + return tier + return NSFW_OFF @dataclass(frozen=True) @@ -77,6 +406,7 @@ class _TextFile: key: str root_index: int = 0 mtime: float = -1.0 + nsfw: int = 0 # rating by folder / name, 0..3 (NSFW_LEVELS) entries: List[Entry] = field(default_factory=list) @@ -85,7 +415,37 @@ class _StructuredFile: path: str root_index: int = 0 mtime: float = -1.0 + nsfw: int = 0 # rating of the whole file by folder / name, 0..3 keys: Dict[str, List[Entry]] = field(default_factory=dict) + nsfw_keys: Dict[str, int] = field(default_factory=dict) # key -> rating by its own name (rated keys only) + prefix: List[str] = field(default_factory=list) # under _prompts/: the file's folder, put before its keys + stem: str = "" # under _prompts/: the file name, for a file that is one list + + +@dataclass +class _View: + """Lookup names visible to one kind of run (SFW or NSFW-allowed), rebuilt after each rescan.""" + names: List[str] # sorted lookup names + name_set: Set[str] + real: Dict[str, List[str]] # lookup name -> real keys (an _nsfw/ key answers to its unprefixed name) + by_leaf: Dict[str, List[str]] # last path segment -> lookup names + root_of: Dict[str, int] # lookup name -> best (lowest) root index + rating: Dict[str, int] = field(default_factory=dict) # lookup name -> its list rating, 0..3 + hidden: Set[str] = field(default_factory=set) # hidden by the user: exact names only + visible: Optional["_View"] = None # this view without the hidden names (None = no hidden) + prompts: List[str] = field(default_factory=list) # prompt files (prompts/...): NOT in names / by_leaf + prompt_set: Set[str] = field(default_factory=set) + + +def _visible_only(view: "_View") -> "_View": + """The view without the names the user hid (they keep answering to their exact name).""" + names = [name for name in view.names if name not in view.hidden] + by_leaf: Dict[str, List[str]] = {} + for name in names: + by_leaf.setdefault(name.rsplit("/", 1)[-1], []).append(name) + prompts = [name for name in view.prompts if name not in view.hidden] + return _View(names=names, name_set=set(names), real=view.real, by_leaf=by_leaf, root_of=view.root_of, + rating=view.rating, prompts=prompts, prompt_set=set(prompts)) def read_text_file(path: str) -> str: @@ -122,9 +482,29 @@ def parse_text_line(line: str) -> Optional[Tuple[str, float]]: return stripped, weight +def _synchronized(method): + """Serialise library access: the catalog builds in a background thread while nodes resolve prompts.""" + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + with self._lock: + return method(self, *args, **kwargs) + return wrapper + + +def pack_of(name: str) -> str: + """The pack a lookup name belongs to: its first folder. A loose top-level list has pack ''.""" + return name.split("/", 1)[0] if "/" in name else "" + + class WildcardLibrary: - def __init__(self, search_paths: Iterable[str], merge_roots: bool = False): + def __init__(self, search_paths: Iterable[str], merge_roots: bool = False, + pool_mode: str = POOL_SCOPED, strip_lora: bool = True, + content_filter: bool = True): + self._lock = threading.RLock() self.merge_roots = merge_roots + self.pool_mode = pool_mode if pool_mode in POOL_MODES else POOL_SCOPED + self.strip_lora = bool(strip_lora) + self.content_filter = bool(content_filter) self.search_paths: List[str] = [] for path in search_paths: if path and os.path.isdir(path): @@ -134,10 +514,20 @@ class WildcardLibrary: self._text_files: Dict[str, List[_TextFile]] = {} # key -> files (several folders may share a key) self._structured_files: Dict[str, _StructuredFile] = {} # path -> file self._last_scan = 0.0 + self._generation = 0 # bumps whenever the set of keys may have changed + self._views: Dict[int, Tuple[int, _View]] = {} # nsfw level -> (stamp, view) + self._match_cache: Dict[Tuple, List[str]] = {} + self._match_cache_generation = -1 self.warnings: List[str] = [] # transient, reset by the engine before each run self.file_errors: Dict[str, str] = {} # persistent: a broken file stays reported until it is fixed + self.blocked_entries = 0 # entries removed by the content filter (cumulative) + # Optional curation (catalog + the user's _catalog_user.yaml, see curation.py): hides lists from + # name matching, drops entries, adds content-detected NSFW, and ranks packs for ambiguous names. + self.curation = None + self.stripped_lora_tags = 0 # tags removed from entries (cumulative) # -- scanning --------------------------------------------------------------- + @_synchronized def refresh(self, force: bool = False) -> None: now = time.time() if not force and now - self._last_scan < RESCAN_SECONDS: @@ -148,8 +538,16 @@ class WildcardLibrary: seen_structured: Dict[str, _StructuredFile] = {} for root_index, root in enumerate(self.search_paths): - for folder, _subfolders, filenames in os.walk(root): + for folder, subfolders, filenames in os.walk(root): + at_top = os.path.normcase(folder) == os.path.normcase(root) + subfolders[:] = [ + name for name in subfolders + if name.lower() not in _SKIPPED_FOLDERS + and not (at_top and name.startswith("_") and name.lower() not in _SEARCHED_UNDERSCORE_FOLDERS) + ] for filename in sorted(filenames): + if filename.startswith("._") or (at_top and filename.lower().startswith("_catalog")): + continue # macOS forks; the catalog and the user's curation file path = os.path.join(folder, filename) extension = os.path.splitext(filename)[1].lower() if extension in TEXT_EXTENSIONS: @@ -158,20 +556,44 @@ class WildcardLibrary: existing = self._find_text_file(key, path) text_file = existing or _TextFile(path=path, key=key) text_file.root_index = root_index + text_file.nsfw = self._path_rating(key) seen_text.setdefault(key, []).append(text_file) elif extension in STRUCTURED_EXTENSIONS: structured = self._structured_files.get(path) or _StructuredFile(path=path) structured.root_index = root_index + relative = normalise_key(os.path.splitext(os.path.relpath(path, root))[0]) + structured.nsfw = self._path_rating(relative) + if relative.startswith(_PROMPTS_PREFIX): + # Prompt files keep their folder in their names (prompts//.../). + structured.prefix = relative.split("/")[:-1] + structured.stem = relative.rsplit("/", 1)[-1] self._load_structured_if_changed(structured) seen_structured[path] = structured + old_signature = (set(self._text_files), {p: (s.mtime, s.nsfw) for p, s in self._structured_files.items()}, + {k: tuple((f.path, f.nsfw) for f in v) for k, v in self._text_files.items()}) self._text_files = seen_text self._structured_files = seen_structured + new_signature = (set(self._text_files), {p: (s.mtime, s.nsfw) for p, s in self._structured_files.items()}, + {k: tuple((f.path, f.nsfw) for f in v) for k, v in self._text_files.items()}) + if new_signature != old_signature: + self._generation += 1 # Forget errors for files that no longer exist. for path in list(self.file_errors): if not os.path.exists(path): del self.file_errors[path] + @staticmethod + def _path_rating(key: str) -> int: + """key: a normalised relative path. Under _nsfw/ -> its folder's level. Else rated by the name below the pack. + _prompts/ files are rated the same way, by the path below _prompts/.""" + if key.startswith(_PROMPTS_PREFIX): + key = key[len(_PROMPTS_PREFIX):] + if key.startswith(_NSFW_PREFIX): + return nsfw_folder_tier(key) + below_pack = key.split("/", 1)[1] if "/" in key else key + return nsfw_name_tier(below_pack) + def all_warnings(self) -> List[str]: return list(self.file_errors.values()) + list(self.warnings) @@ -182,6 +604,21 @@ class WildcardLibrary: return None # -- loading ---------------------------------------------------------------- + def _make_entry(self, text: str, weight: float, origin: str, key: str, + tags: FrozenSet[str] = frozenset()) -> Optional[Entry]: + """One entry after the load-time rules: LoRA tags stripped, content filter applied.""" + if self.strip_lora and "<" in text: + stripped, count = _LORA_TAG.subn("", text) + if count: + self.stripped_lora_tags += count + text = re.sub(r"\s{2,}", " ", _LORA_SCARS.sub("", stripped)).strip() + if not text: + return None + if self.content_filter and is_blocked_entry(text): + self.blocked_entries += 1 + return None + return Entry(text=text, weight=weight, tags=tags, origin=origin, key=key) + def _load_text_if_changed(self, text_file: _TextFile) -> None: try: mtime = os.path.getmtime(text_file.path) @@ -195,7 +632,9 @@ class WildcardLibrary: for line in read_text_file(text_file.path).splitlines(): parsed = parse_text_line(line) if parsed: - entries.append(Entry(text=parsed[0], weight=parsed[1], origin=text_file.path, key=text_file.key)) + entry = self._make_entry(parsed[0], parsed[1], text_file.path, text_file.key) + if entry: + entries.append(entry) self.file_errors.pop(text_file.path, None) except OSError as error: self.file_errors[text_file.path] = f"could not read {text_file.path}: {error}" @@ -212,6 +651,7 @@ class WildcardLibrary: return structured.mtime = mtime structured.keys = {} + structured.nsfw_keys = {} try: content = read_text_file(structured.path) if structured.path.lower().endswith(".json"): @@ -228,29 +668,40 @@ class WildcardLibrary: self.file_errors.pop(structured.path, None) if isinstance(data, dict) and "nodes" in data and "links" in data: return # a ComfyUI workflow saved next to the packs, not a wildcard file - self._walk_structured(data, [], structured) + if structured.prefix: + start = list(structured.prefix) if isinstance(data, dict) else list(structured.prefix) + [structured.stem] + self._walk_structured(data, start, structured) + else: + self._walk_structured(data, [], structured) + structured.nsfw_keys = {key: tier for key in structured.keys if (tier := nsfw_name_tier(key))} def _walk_structured(self, data, key_parts: List[str], structured: _StructuredFile) -> None: if isinstance(data, dict): for key, value in data.items(): self._walk_structured(value, key_parts + [str(key)], structured) return - if isinstance(data, list) and key_parts: - key = normalise_key("/".join(key_parts)) - origin = f"{structured.path}::{key}" - entries = structured.keys.setdefault(key, []) - for item in data: - entry = self._entry_from_item(item, origin, key) - if entry: - entries.append(entry) + if not key_parts: + return + if isinstance(data, list): + items = data + elif isinstance(data, (str, int, float)) and not isinstance(data, bool): + items = [data] # a single string under a key counts as a one-item list + else: + return + key = normalise_key("/".join(key_parts)) + origin = f"{structured.path}::{key}" + entries = structured.keys.setdefault(key, []) + for item in items: + entry = self._entry_from_item(item, origin, key) + if entry: + entries.append(entry) - @staticmethod - def _entry_from_item(item, origin: str, key: str) -> Optional[Entry]: + def _entry_from_item(self, item, origin: str, key: str) -> Optional[Entry]: if isinstance(item, str): parsed = parse_text_line(item) - return Entry(text=parsed[0], weight=parsed[1], origin=origin, key=key) if parsed else None - if isinstance(item, (int, float)): - return Entry(text=str(item), origin=origin, key=key) + return self._make_entry(parsed[0], parsed[1], origin, key) if parsed else None + if isinstance(item, (int, float)) and not isinstance(item, bool): + return self._make_entry(str(item), 1.0, origin, key) if isinstance(item, dict) and item.get("text"): raw_tags = item.get("tags") or [] if isinstance(raw_tags, str): @@ -260,38 +711,242 @@ class WildcardLibrary: weight = float(item.get("weight", 1.0)) except (TypeError, ValueError): weight = 1.0 - return Entry(text=str(item["text"]).strip(), weight=weight, tags=tags, origin=origin, key=key) + return self._make_entry(str(item["text"]).strip(), weight, origin, key, tags=tags) return None + # -- views ------------------------------------------------------------------ + def _structured_key_rating(self, structured: _StructuredFile, key: str) -> int: + return max(structured.nsfw, structured.nsfw_keys.get(key, 0)) + + @_synchronized + def _view(self, nsfw=NSFW_OFF) -> _View: + """The lookup names a run at this nsfw level can use (lists rated at that level or lower).""" + level = nsfw_level(nsfw) + self.refresh() + stamp = (self._generation, self._curation_version()) + cached = self._views.get(level) + if cached and cached[0] == stamp: + return cached[1] + + sources: Dict[str, List[Tuple[int, int]]] = {} # real key -> [(root_index, rating)] + for key, text_files in self._text_files.items(): + for text_file in text_files: + sources.setdefault(key, []).append((text_file.root_index, text_file.nsfw)) + for structured in self._structured_files.values(): + for key in structured.keys: + sources.setdefault(key, []).append( + (structured.root_index, self._structured_key_rating(structured, key))) + curation = self.curation + if curation is not None: + # The user's overlay can rate a list either way; the catalog marks lists explicit by content. + for key, key_sources in sources.items(): + verdict = curation.nsfw_rating(key) + if verdict is not None and nsfw_folder_tier(key) == NSFW_OFF: + sources[key] = [(root, verdict) for root, _rating in key_sources] + + real: Dict[str, List[str]] = {} + root_of: Dict[str, int] = {} + rating: Dict[str, int] = {} + for key, key_sources in sources.items(): + allowed = [(root, rated) for root, rated in key_sources if rated <= level] + if not allowed: + continue + name = strip_nsfw_prefix(key) + real.setdefault(name, []).append(key) + best_root = min(root for root, _rated in allowed) + root_of[name] = min(root_of.get(name, best_root), best_root) + lowest = min(rated for _root, rated in allowed) + rating[name] = min(rating.get(name, lowest), lowest) + + everything = sorted(real) + # Prompt files are their own area: kept out of names (bare names, globs, folder names, visible_keys). + names = [name for name in everything if not is_prompt_name(name)] + prompts = [name for name in everything if is_prompt_name(name)] + by_leaf: Dict[str, List[str]] = {} + for name in names: + by_leaf.setdefault(name.rsplit("/", 1)[-1], []).append(name) + hidden = {name for name in everything if curation is not None and curation.hidden(name)} + view = _View(names=names, name_set=set(names), real=real, by_leaf=by_leaf, root_of=root_of, + rating=rating, hidden=hidden, prompts=prompts, prompt_set=set(prompts)) + if hidden: + view.visible = _visible_only(view) + self._views[level] = (stamp, view) + return view + + def _curation_version(self) -> int: + return self.curation.version if self.curation is not None else 0 + # -- lookup ----------------------------------------------------------------- + @_synchronized def all_keys(self) -> List[str]: + """Every real key in the library, NSFW included (for reports and scans).""" self.refresh() keys = set(self._text_files.keys()) for structured in self._structured_files.values(): keys.update(structured.keys.keys()) return sorted(keys) - def has(self, name: str) -> bool: - return bool(self._matching_keys(normalise_key(name))) + @_synchronized + def visible_keys(self, nsfw=NSFW_OFF, include_prompts: bool = False) -> List[str]: + """Lookup names a run at this nsfw level can use. Prompt files (prompts/...) only when include_prompts.""" + view = self._view(nsfw) + return list(view.names) + (list(view.prompts) if include_prompts else []) - def _matching_keys(self, key: str) -> List[str]: - keys = self.all_keys() + @_synchronized + def prompt_names(self, nsfw=NSFW_OFF, include_hidden: bool = False) -> List[str]: + """The prompt files (prompts//.../) a run at this nsfw level can use, hidden ones left out.""" + view = self._view(nsfw) + if include_hidden or view.visible is None: + return list(view.prompts) + return list(view.visible.prompts) + + @_synchronized + def real_keys(self, name: str) -> List[str]: + """The real keys behind a lookup name (an _nsfw/ list answers to its name without the prefix).""" + return list(self._view(NSFW_EXPLICIT).real.get(normalise_key(name), [])) + + @_synchronized + def list_rating(self, name: str) -> int: + """A lookup name's list rating 0..3 (NSFW_LEVELS); 0 for an unknown name.""" + return self._view(NSFW_EXPLICIT).rating.get(normalise_key(name), NSFW_OFF) + + def entry_rating(self, entry: Entry) -> int: + """An entry's rating 0..3: the user's override (_catalog_user.yaml ratings) or its words.""" + if self.curation is not None: + override = self.curation.entry_rating(entry.key, entry.text) + if override is not None: + return override + return nsfw_word_tier(entry.text) + + @_synchronized + def entries_of(self, name: str, nsfw=NSFW_OFF, include_dropped: bool = False) -> List[Entry]: + """ + Entries of one lookup name exactly as visible_keys() lists it: no globbing, no pack + scoping, no folder expansion. For browsers and builders that already know the list they want. + include_dropped: also return the entries the user dropped (a list viewer shows them struck out). + nsfw: entries rated above this level are left out (a list viewer asks with NSFW_EXPLICIT and shows ratings). + """ + view = self._view(nsfw) + entries: List[Entry] = [] + for real_key in view.real.get(normalise_key(name), []): + entries.extend(self._entries_for_key(real_key, nsfw=nsfw, apply_drops=not include_dropped)) + return entries + + @property + def generation(self) -> int: + """Bumps whenever the set of keys may have changed (after refresh()).""" + self.refresh() + return self._generation + + @_synchronized + def has(self, name: str, nsfw=NSFW_OFF, context: str = "") -> bool: + return bool(self._matching_names(normalise_key(name), nsfw, context, warn=False)) + + def _pick_one_pack(self, candidates: List[str], view: _View, name: str, context: str, warn: bool) -> List[str]: + """Scoped mode: keep the candidates of one pack only.""" + by_pack: Dict[str, List[str]] = {} + for candidate in candidates: + by_pack.setdefault(pack_of(candidate), []).append(candidate) + if len(by_pack) == 1: + return candidates + context_pack = pack_of(context) if context else None + if context_pack is not None and context_pack in by_pack: + return by_pack[context_pack] + + curation = self.curation + + def rank(pack: str): + members = by_pack[pack] + score = max((curation.score(m) for m in members), default=0.0) if curation is not None else 0.0 + return (min(view.root_of.get(m, 0) for m in members), -score, min(m.count("/") for m in members), pack) + + packs = sorted(by_pack, key=rank) + chosen = packs[0] + if warn: + others = ", ".join(p or "(top level)" for p in packs[1:6]) + (" ..." if len(packs) > 6 else "") + self._warn_once( + f"'__{name}__' matches lists in {len(packs)} packs; used '{chosen or '(top level)'}' " + f"(also in: {others}). Write __{chosen + '/' if chosen else ''}...{name}__ to choose, " + f"or __**/{name}__ to use all of them." + ) + return by_pack[chosen] + + def _warn_once(self, message: str) -> None: + if message not in self.warnings: + self.warnings.append(message) + + def _matching_names(self, key: str, nsfw, context: str = "", warn: bool = True) -> List[str]: if not key: return [] + level = nsfw_level(nsfw) + view = self._view(level) + stamp = (self._generation, self._curation_version()) + if self._match_cache_generation != stamp: + self._match_cache = {} + self._match_cache_generation = stamp + context_key = normalise_key(context) if context else "" + context_key = strip_nsfw_prefix(context_key) + cache_key = (key, level, pack_of(context_key) if context_key else None) + if not warn and cache_key in self._match_cache: + return self._match_cache[cache_key] + found = self._match_names_uncached(key, view, context_key, warn) + self._match_cache[cache_key] = found + return found + + def _match_names_uncached(self, key: str, view: _View, context: str, warn: bool) -> List[str]: + scoped = self.pool_mode == POOL_SCOPED + if key not in view.name_set and key not in view.prompt_set and self.curation is not None: + # A list the wildcard review renamed still answers to its old name (aliases: in _catalog_user.yaml). + renamed = getattr(self.curation, "alias_of", lambda _name: None)(key) + if renamed and (renamed in view.name_set or renamed in view.prompt_set): + return [renamed] + if is_prompt_name(key): + # The prompt-file area answers only to its own names: exact, a glob, or a folder (= every file in it). + if key in view.prompt_set: + return [key] + pool = (view.visible or view).prompts + if any(char in key for char in "*?"): + return [candidate for candidate in pool if fnmatch.fnmatchcase(candidate, key)] + folder = key.rstrip("/") + "/" + return [candidate for candidate in pool if candidate.startswith(folder)] + if key in view.name_set: + return [key] # an exact name always works, hidden or not + if view.visible is not None: + view = view.visible if any(char in key for char in "*?"): - return [candidate for candidate in keys if fnmatch.fnmatchcase(candidate, key)] - if key in keys: - return [key] + return [candidate for candidate in view.names if fnmatch.fnmatchcase(candidate, key)] + + # Same pack first: a list inside a pack that refers to __x__ or __sub/x__ means its own pack's x. + if scoped and context and "/" in context: + own_pack = pack_of(context) + in_pack = own_pack + "/" + key + if in_pack in view.name_set: + return [in_pack] + suffix = "/" + key + leaf = key.rsplit("/", 1)[-1] + same = [c for c in view.by_leaf.get(leaf, []) if c.startswith(own_pack + "/") and c.endswith(suffix)] + if same: + return same + + # Depth-agnostic: __painters__ finds style/painters. suffix = "/" + key - deeper = [candidate for candidate in keys if candidate.endswith(suffix)] + leaf = key.rsplit("/", 1)[-1] + deeper = [candidate for candidate in view.by_leaf.get(leaf, []) if candidate.endswith(suffix)] if deeper: - return deeper + return self._pick_one_pack(deeper, view, key, context, warn) if scoped else deeper + # A folder name stands for everything inside it: __materials__ == __materials/**__ # (only reached when no list of that exact name exists, so a real list always wins). - inside = [candidate for candidate in keys + inside = [candidate for candidate in view.names if candidate.startswith(key + "/") or ("/" + key + "/") in candidate] if inside: - return inside + if not scoped: + return inside + top_level = [candidate for candidate in inside if candidate.startswith(key + "/")] + if top_level: + return top_level + return self._pick_one_pack(inside, view, key, context, warn) + if "/" not in key: return [] # Relocated pack: the folder part of the reference does not exist under that name. @@ -300,28 +955,82 @@ class WildcardLibrary: if len(expected_folder) < 3: return [] relocated = [ - candidate for candidate in keys - if candidate.endswith("/" + last_segment) - and expected_folder in candidate.rsplit("/", 1)[0].rsplit("/", 1)[-1] + candidate for candidate in view.by_leaf.get(last_segment, []) + if "/" in candidate and expected_folder in candidate.rsplit("/", 1)[0].rsplit("/", 1)[-1] ] return relocated if len(relocated) == 1 else [] - def _entries_for_key(self, key: str) -> List[Entry]: + @_synchronized + def _entries_for_key(self, key: str, nsfw=NSFW_EXPLICIT, apply_drops: bool = True) -> List[Entry]: + """ + Entries of one REAL key at an nsfw level: sources of the key rated above it are left out, and so are + entries rated above it (their words, or the user's override). An empty result means none qualify. + """ + level = nsfw_level(nsfw) by_root: Dict[int, List[Entry]] = {} for text_file in self._text_files.get(key, []): + if text_file.nsfw > level: + continue self._load_text_if_changed(text_file) if text_file.entries: by_root.setdefault(text_file.root_index, []).extend(text_file.entries) for structured in self._structured_files.values(): found = structured.keys.get(key) - if found: + if found and self._structured_key_rating(structured, key) <= level: by_root.setdefault(structured.root_index, []).extend(found) if not by_root: return [] if self.merge_roots: - return [entry for root_index in sorted(by_root) for entry in by_root[root_index]] - return by_root[min(by_root)] # highest-priority folder that has this key wins + entries = [entry for root_index in sorted(by_root) for entry in by_root[root_index]] + else: + entries = by_root[min(by_root)] # highest-priority folder that has this key wins + dropped = self.curation.dropped(key) if (self.curation is not None and apply_drops) else None + if dropped: + kept = [entry for entry in entries if entry.text.strip().casefold() not in dropped] + entries = kept or entries # dropping every entry would silently kill the list; keep it whole + if level < NSFW_EXPLICIT: + entries = [entry for entry in entries if self.entry_rating(entry) <= level] + return entries + # -- catalog support (read-only; see catalog.py) -------------------------------- + @_synchronized + def catalog_sources(self) -> Dict[str, Dict[str, Tuple[str, List[str]]]]: + """root -> {file path relative to root: (absolute path, [real keys in that file])}.""" + self.refresh() + out: Dict[str, Dict[str, Tuple[str, List[str]]]] = {root: {} for root in self.search_paths} + for key, text_files in list(self._text_files.items()): + for text_file in text_files: + root = self.search_paths[text_file.root_index] + relative = os.path.relpath(text_file.path, root).replace("\\", "/") + out[root][relative] = (text_file.path, [key]) + for structured in list(self._structured_files.values()): + root = self.search_paths[structured.root_index] + relative = os.path.relpath(structured.path, root).replace("\\", "/") + out[root][relative] = (structured.path, sorted(structured.keys)) + return out + + @_synchronized + def catalog_entries(self, root: str, key: str) -> Tuple[List[str], bool, bool]: + """ + (entry texts, under _nsfw/, NSFW by name) of one real key in ONE folder, before the + user's dropped entries are removed. + """ + root_index = self.search_paths.index(root) + texts: List[str] = [] + folder_nsfw = key.startswith(_NSFW_PREFIX) + name_nsfw = False # rated explicit by its name (the catalog's "explicit by name") + for text_file in self._text_files.get(key, []): + if text_file.root_index == root_index: + self._load_text_if_changed(text_file) + texts.extend(entry.text for entry in text_file.entries) + name_nsfw = name_nsfw or text_file.nsfw >= NSFW_EXPLICIT + for structured in list(self._structured_files.values()): + if structured.root_index == root_index and key in structured.keys: + texts.extend(entry.text for entry in structured.keys[key]) + name_nsfw = name_nsfw or self._structured_key_rating(structured, key) >= NSFW_EXPLICIT + return texts, folder_nsfw, name_nsfw and not folder_nsfw + + @_synchronized def shadowed_keys(self) -> Dict[str, List[str]]: """Keys that exist in more than one folder: {key: [winning folder, shadowed folder, ...]}.""" self.refresh() @@ -337,14 +1046,22 @@ class WildcardLibrary: for key, indices in sorted(roots_by_key.items()) if len(indices) > 1 } - def lookup(self, name: str) -> List[Entry]: - """All entries for a wildcard name (exact, glob, or depth-agnostic). Empty list when unknown.""" + @_synchronized + def lookup(self, name: str, nsfw=NSFW_OFF, context: str = "") -> List[Entry]: + """ + All entries for a wildcard name (exact, glob, same-pack, depth-agnostic). Empty list when unknown. + context: the key of the list whose entry contains this reference, for same-pack resolution. + nsfw: the run's level; lists and entries rated above it are left out. + """ + view = self._view(nsfw) entries: List[Entry] = [] - for key in self._matching_keys(normalise_key(name)): - entries.extend(self._entries_for_key(key)) + for lookup_name in self._matching_names(normalise_key(name), nsfw, context): + for real_key in view.real.get(lookup_name, []): + entries.extend(self._entries_for_key(real_key, nsfw)) return entries - def query_tags(self, expression: str, scope: str = "") -> List[Entry]: + @_synchronized + def query_tags(self, expression: str, scope: str = "", nsfw=NSFW_OFF, context: str = "") -> List[Entry]: """ Entries whose tags satisfy a boolean expression. `scope` limits the search to a wildcard name / glob; with an empty scope every tagged entry in the library is searched. @@ -357,16 +1074,19 @@ class WildcardLibrary: return [] if scope: - candidates = self.lookup(scope if any(c in scope for c in "*?") else scope) + candidates = self.lookup(scope, nsfw, context) if not candidates: - candidates = self.lookup(scope.rstrip("/") + "/**") + candidates = self.lookup(scope.rstrip("/") + "/**", nsfw, context) else: # Only structured files can carry tags; never force-load large .txt packs for a tag search. + level = nsfw_level(nsfw) self.refresh() candidates = [] for structured in self._structured_files.values(): - for key_entries in structured.keys.values(): - candidates.extend(key_entries) + for key, key_entries in structured.keys.items(): + if self._structured_key_rating(structured, key) <= level: + candidates.extend(entry for entry in key_entries + if level >= NSFW_EXPLICIT or self.entry_rating(entry) <= level) matches: List[Entry] = [] for entry in candidates: @@ -382,13 +1102,20 @@ class WildcardLibrary: matches.append(entry) return matches + @_synchronized def stats(self) -> Dict[str, int]: keys = self.all_keys() tagged = 0 total = 0 + nsfw_keys = 0 + visible = set(self.visible_keys(NSFW_OFF, include_prompts=True)) for key in keys: + if key not in visible: + nsfw_keys += 1 for entry in self._entries_for_key(key): total += 1 if entry.tags: tagged += 1 - return {"folders": len(self.search_paths), "wildcards": len(keys), "entries": total, "tagged_entries": tagged} + return {"folders": len(self.search_paths), "wildcards": len(keys), "entries": total, + "tagged_entries": tagged, "nsfw_wildcards": nsfw_keys, + "blocked_entries": self.blocked_entries, "stripped_lora_tags": self.stripped_lora_tags} diff --git a/core/wildcards/parser.py b/core/wildcards/parser.py index 58daf93..d218b9f 100644 --- a/core/wildcards/parser.py +++ b/core/wildcards/parser.py @@ -7,6 +7,7 @@ Grammar (Dynamic Prompts compatible, so existing wildcard packs work, plus exten __name(season=winter)__ parameterized wildcard __name[metal AND NOT large]__ __[metal AND small]__ tag query (scoped / global) __db:surreal portrait__ __llm:an object that ...__ provider slots + __name!{a|b}__ exclusion: any entry except a and b {a|b|c} {a|{b|c}} variants, nested {3::common|1::rare} weights {2$$a|b|c} {1-3$$ and $$a|b|c} {2$$__flavours__} multi-select, separator @@ -74,6 +75,7 @@ _WILDCARD = re.compile( r"([A-Za-z0-9*][A-Za-z0-9_\-/*. ]*?)?" # name (optional when a tag query follows) r"(?:\(([^()\n]*)\))?" # (key=value, key=value) r"(?:\[([^\[\]\n]+)\])?" # [tag expression] + r"(?:!\{([^{}\n]*)\})?" # !{entry|entry} exclusions r"__" ) _CONDITIONAL_OPEN = re.compile(r"\[if\s+", re.IGNORECASE) @@ -316,6 +318,7 @@ class Parser: tag_expression=tag_expression.strip() if tag_expression else None, sampler=sampler, binding=binding, + exclude=[part.strip() for part in (match.group(4) or "").split("|") if part.strip()], ) # -- ${variable} ------------------------------------------------------------ diff --git a/core/wildcards/record.py b/core/wildcards/record.py index a50be10..baea880 100644 --- a/core/wildcards/record.py +++ b/core/wildcards/record.py @@ -37,6 +37,12 @@ class Slot: option_count: int = 0 +def is_filter_note(text: str) -> bool: + """A warning written by the content filter, the violence filter or the nsfw filter.""" + return text.startswith(("content filter:", "violence filter", "nsfw filter")) or ": content filter:" in text \ + or ": violence filter" in text or ": nsfw filter" in text + + @dataclass class Resolution: text: str = "" @@ -48,6 +54,7 @@ class Resolution: combination_space: int = 1 # product of option counts met on this path (combinatorial mode) fresh_roll: bool = False # True when no seed was given and the engine drew one (it is in `seed`) upstream_instructions: str = "" # an llm_instructions block handed in from a node earlier in the graph + blocked: str = "" # why the content filter blanked this prompt ('' = it did not) # -- views ------------------------------------------------------------------ def top_level_slots(self) -> List[Slot]: @@ -92,14 +99,21 @@ class Resolution: seed_text = f"seed={self.seed}" if self.fresh_roll: seed_text += " (fresh roll - use this number as the seed to get these picks again)" - lines = [f"{len(self.top_level_slots())} slot(s) resolved, mode={self.mode}, {seed_text}"] + # Filter notes lead the report: what the content / violence filter held back or blocked. + lines = [f"FILTER: {note}" for note in self.filter_notes()] + lines.append(f"{len(self.top_level_slots())} slot(s) resolved, mode={self.mode}, {seed_text}") for slot in self.top_level_slots(): name = slot.label or slot.source lines.append(f" [{slot.binding}] {name} -> {slot.value}") for warning in self.warnings: - lines.append(f" warning: {warning}") + if not is_filter_note(warning): + lines.append(f" warning: {warning}") return "\n".join(lines) + def filter_notes(self) -> List[str]: + """The warnings the content and violence filters left (re-rolls, blocked prompts, dropped references).""" + return [warning for warning in self.warnings if is_filter_note(warning)] + def format_for_llm(self) -> str: """ Instruction block for the rewriter's system or user prompt. Empty string when diff --git a/core/wildcards/resolver.py b/core/wildcards/resolver.py index 803549b..71f0775 100644 --- a/core/wildcards/resolver.py +++ b/core/wildcards/resolver.py @@ -24,7 +24,8 @@ import re from typing import Callable, Dict, List, Optional, Tuple, Union from .boolean_expr import BooleanExpressionError, compile_expression, evaluate -from .library import Entry, WildcardLibrary +from .library import (NSFW_EXPLICIT, NSFW_LEVELS, VIOLENCE_LEVELS, Entry, WildcardLibrary, nsfw_level, + violence_blocked, violence_level) from .parser import parse from .record import Resolution, Slot from .syntax_tree import ( @@ -144,8 +145,15 @@ class Resolver: providers: Optional[Dict[str, ProviderCallable]] = None, variables: Optional[Dict[str, str]] = None, pick_hook: Optional[PickHook] = None, + nsfw="off", + violence: int = 0, ): self.library = library + self.nsfw = nsfw_level(nsfw) # 0 off .. 3 explicit (library.NSFW_LEVELS) + self._nsfw_emptied = False # set by _entries_for when the level left nothing + self.violence = violence_level(violence) # 0 off .. 3 (library.VIOLENCE_LEVELS) + self._violence_emptied = False # set by _entries_for when the filter left nothing + self._context_keys: List[str] = [] # keys of the lists whose entries are being resolved (innermost last) self.pick_hook = pick_hook self.rng = random.Random(seed) if seed is not None else random.Random() self.chooser = _Chooser(self.rng, mode, index, cycle_index) @@ -255,7 +263,12 @@ class Resolver: origins: List[str] = [] for entry_index in indices: entry = entries[entry_index] - values.append(self._resolve(self._parse_entry(entry.text), scope, slot.depth + 1, slot.index, in_output).strip()) + # References inside this entry resolve in the entry's own pack first (library same-pack rule). + self._context_keys.append(entry.key) + try: + values.append(self._resolve(self._parse_entry(entry.text), scope, slot.depth + 1, slot.index, in_output).strip()) + finally: + self._context_keys.pop() if entry.origin and entry.origin not in origins: origins.append(entry.origin) slot.origin = "; ".join(origins) @@ -271,7 +284,7 @@ class Resolver: # Legacy {category}: a bare name that matches a wildcard file. if self.legacy and only_body.is_plain_text(): name = only_body.plain_text().strip() - if name and self.library.has(name): + if name and self.library.has(name, nsfw=self.nsfw, context=self._context()): legacy_node = Wildcard(source=node.source, name=name, sampler=node.sampler, binding=node.binding) return self._resolve_wildcard(legacy_node, scope, depth, parent, in_output, kind="legacy") # Not a choice at all: keep the braces, still resolve anything inside them. @@ -317,10 +330,31 @@ class Resolver: return slot.value # -- __wildcard__ ----------------------------------------------------------- + def _context(self) -> str: + return self._context_keys[-1] if self._context_keys else "" + def _entries_for(self, node: Wildcard) -> List[Entry]: if node.tag_expression: - return self.library.query_tags(node.tag_expression, scope=node.name) - return self.library.lookup(node.name) + entries = self.library.query_tags(node.tag_expression, scope=node.name, + nsfw=self.nsfw, context=self._context()) + else: + entries = self.library.lookup(node.name, nsfw=self.nsfw, context=self._context()) + if not entries and self.nsfw < NSFW_EXPLICIT and not node.tag_expression \ + and self.library.lookup(node.name, nsfw=NSFW_EXPLICIT, context=self._context()): + self._nsfw_emptied = True # the list exists; everything in it is rated above this run + if self.violence and entries: + kept = [entry for entry in entries if not violence_blocked(entry.text, entry.key, self.violence)] + if not kept: + self._violence_emptied = True + entries = kept + if node.exclude and entries: + banned = {item.casefold() for item in node.exclude} + kept = [entry for entry in entries if entry.text.strip().casefold() not in banned] + if not kept: + self._warn(f"{node.source} excludes every entry; exclusions ignored") + return entries + return kept + return entries @staticmethod def _scope_with_params(scope: Dict[str, _VariableValue], node: Wildcard) -> Dict[str, _VariableValue]: @@ -333,7 +367,21 @@ class Resolver: def _resolve_wildcard(self, node: Wildcard, scope: Dict[str, _VariableValue], depth: int, parent: Optional[int], in_output: bool, kind: Optional[str] = None) -> str: + self._violence_emptied = False + self._nsfw_emptied = False entries = self._entries_for(node) + if not entries and self._nsfw_emptied: + # The list, or every entry in it, is rated above the run's level ("__lingerie__" at off): dropped + # from the prompt rather than left as text. + self._warn(f"nsfw filter ({NSFW_LEVELS[self.nsfw]}): {node.source} dropped - " + f"its list or every entry in it is rated above {NSFW_LEVELS[self.nsfw]}") + return "" + if not entries and self._violence_emptied: + # Every entry is in the blocked tier: the reference itself names that content ("__weapons__"), so + # it is dropped from the prompt rather than left as text. + self._warn(f"violence filter ({VIOLENCE_LEVELS[self.violence]}): {node.source} dropped - " + f"every entry is in the blocked tier") + return "" if not entries: if node.tag_expression: self._warn(f"no entries match {node.source}; left as text") diff --git a/core/wildcards/syntax_tree.py b/core/wildcards/syntax_tree.py index ceb5113..8616404 100644 --- a/core/wildcards/syntax_tree.py +++ b/core/wildcards/syntax_tree.py @@ -75,12 +75,13 @@ class Variant(Node): @dataclass class Wildcard(Node): - """__name__, __dir/name__, __colors*__, __name(k=v)__, __name[tag AND tag]__, __[tag]__.""" + """__name__, __dir/name__, __colors*__, __name(k=v)__, __name[tag AND tag]__, __[tag]__, __name!{a|b}__.""" name: str = "" # "" when the reference is a pure tag query params: Dict[str, str] = field(default_factory=dict) tag_expression: Optional[str] = None sampler: str = SAMPLER_DEFAULT binding: str = BINDING_CONCEPT + exclude: List[str] = field(default_factory=list) # __name!{a|b}__: never pick these entries @dataclass diff --git a/docs/NODE_COMPARISON.md b/docs/NODE_COMPARISON.md index 5ea67ba..095a13a 100644 --- a/docs/NODE_COMPARISON.md +++ b/docs/NODE_COMPARISON.md @@ -1,343 +1,184 @@ # Node Comparison Guide - Which Node Should I Use? +All 18 nodes live under **Add Node → Eric Prompt Enhancers**. This guide groups them by job and +helps you pick. For full per-node descriptions see README.md → "The Nodes". + ## Quick Decision Tree ``` -Do you have an IMAGE or just TEXT? +What do you want to do? -├─ TEXT → Text-to-Video nodes +├─ Expand a short idea into a full prompt (with an LLM) │ │ -│ ├─ Want SIMPLE presets? -│ │ └─ Use: AI Video Prompt Expander (Standard) +│ ├─ For VIDEO, from TEXT +│ │ ├─ Simple, preset-driven → Video Prompt Expander +│ │ └─ Precise camera/lighting → Video Prompt Expander (Advanced) │ │ -│ └─ Want DETAILED controls? -│ └─ Use: AI Video Prompt Expander (Advanced) +│ ├─ For VIDEO, from an IMAGE → Image-to-Video Prompt Expander +│ │ +│ ├─ For an IMAGE, from an IMAGE → Image-to-Image Prompt Expander +│ │ +│ ├─ For an IMAGE, from TEXT → Text-to-Image Prompt Enhancer +│ │ +│ └─ For Alice text-to-video → Alice T2V Prompt Refiner │ -└─ IMAGE → Image-based nodes - │ - ├─ Want to ANIMATE it (video)? - │ └─ Use: Image-to-Video Prompt Expander - │ - └─ Want to TRANSFORM it (image)? - └─ Use: Image-to-Image Prompt Expander +├─ Take a FINISHED prompt and format it for a specific model +│ ├─ Any of 60+ platform profiles → Platform Prompt Rewriter +│ └─ Ideogram 4 (structured JSON) → Ideogram 4 Magic Prompt +│ +├─ Work with WILDCARDS / add variation +│ ├─ Resolve wildcard syntax → Wildcard Processor +│ ├─ Nudge picks to neighbours → Semantic Variation +│ ├─ Did the key idea survive? → Concept Survival Check +│ └─ One surreal anomaly → Surreal Premise +│ +├─ BUILD a prompt with buttons (no typed syntax) +│ ├─ Whole prompt, section by section → Wildcard Prompt Composer +│ └─ A reusable character → Wildcard Character Composer +│ +└─ Keep a LIBRARY of prompts + ├─ Save one → Save Prompt to DB + ├─ Find one → Search Prompt DB + ├─ Bulk-import many → Import Prompts to DB + └─ Fetch one / random → Browse / Random Prompt DB ``` -## Complete Node Comparison - -| Feature | Standard | Advanced | Image-to-Video | Image-to-Image | -|---------|----------|----------|----------------|----------------| -| **Input** | Text only | Text only | Image + Motion | Image + Changes | -| **Output** | Video prompts | Video prompts | Video prompts | Image prompts | -| **Presets** | ✅ | ✅ | ✅ | ❌ (Platform-based) | -| **Wildcards** | ✅ | ✅ | ✅ | ✅ | -| **Random Mode** | ✅ | ✅ (respects controls) | ✅ | ❌ | -| **Aesthetic Controls** | Basic | Full | Medium | Full (image-focused) | -| **Vision Model** | ❌ | ❌ | ✅ | ✅ | -| **Platform Awareness** | ❌ | ❌ | ❌ | ✅ (6 platforms) | -| **Best For** | Quick video prompts | Precise video control | Animating images | Transforming images | - -## Detailed Comparison - -### 1. AI Video Prompt Expander (Standard) - -**When to use:** -- ✅ Quick video prompt generation -- ✅ Preset-based workflows -- ✅ Don't need fine control -- ✅ Text-to-video generation - -**Example:** -``` -Input: "A {animal} in a {location}" -Preset: cinematic -Tier: enhanced -Output: Detailed video prompt with cinematic style -``` - -**Pros:** -- Simple, fast -- Good presets -- Wildcard support -- Random mode - -**Cons:** -- Less control -- No vision model -- No platform optimization - --- -### 2. AI Video Prompt Expander (Advanced) +## 1. LLM expanders & enhancers -**When to use:** -- ✅ Need precise aesthetic control -- ✅ Specific camera/lighting requirements -- ✅ Professional video workflows -- ✅ Text-to-video with details +These turn a short idea (and optionally a reference image) into a full prompt with a local LLM. -**Example:** -``` -Input: "detective in office" -Shot Size: medium close-up -Camera Angle: low angle -Lighting: edge lighting -Preset: noir -Output: Precise video prompt with all specifications -``` +| Node | Input | Output | Vision | Platform-aware | +|------|-------|--------|--------|----------------| +| **Video Prompt Expander** | Text | Video prompt | ❌ | video `target_model` | +| **Video Prompt Expander (Advanced)** | Text (+ optional image) | Video prompt | ✅ (optional) | video `target_model` | +| **Image-to-Video Prompt Expander** | Image + motion | Video prompt | ✅ | video `target_model` | +| **Image-to-Image Prompt Expander** | Image + changes | Image prompt | ✅ | ✅ 6 platforms | +| **Text-to-Image Prompt Enhancer** | Text (+ 1-2 ref images) | Image prompt | ✅ (optional) | ✅ 8 platforms | +| **Alice T2V Prompt Refiner** | Text and/or images | Alice T2V prompt | ✅ (optional) | Alice | -**Pros:** -- Full control -- All Wan 2.2 options -- Respects dropdowns in random mode -- Wildcard support +### Video Prompt Expander +Simple, fast video prompts. Style presets, five `expansion_tier` levels (auto/basic/enhanced/advanced/ +cinematic), up to 3 variations. **Use when** you want a quick video prompt from a short idea. -**Cons:** -- More complex -- No vision model -- No platform optimization +### Video Prompt Expander (Advanced) +Granular video control: operation modes (`expand_from_idea`, `refine_existing`, `modify_style`, +`add_details`), Concise/Moderate/Detailed/Exhaustive detail, 50+ camera/lighting/visual dropdowns, and an +optional reference image (Qwen3-VL). **Use when** you need specific cinematography or want to refine an +existing prompt. + +### Image-to-Video Prompt Expander +A vision model describes your image, then motion and camera are added. **Use when** you're animating an +existing image. + +### Image-to-Image Prompt Expander +Platform-aware image-to-image prompts (6 platforms: flux, sd_xl, wan22, hunyuan_image, qwen_image, +qwen_image_edit) with vision analysis of the source. **Use when** you're transforming or editing an image. + +### Text-to-Image Prompt Enhancer +Multi-platform image prompts (8 platforms incl. Flux, SDXL, Pony, Illustrious, Chroma, Qwen Image/Edit, Wan +Image), 22 genre styles, optional reference images, emphasis `(keyword:1.5)` and alternation `{a|b|c}`. +**Use when** you're generating a still image from text. + +### Alice T2V Prompt Refiner +Refines rough text and/or reference images into structured Alice text-to-video prompts. **Use when** you're +targeting Alice. --- -### 3. Image-to-Video Prompt Expander +## 2. Platform output -**When to use:** -- ✅ Animating existing images -- ✅ Image-to-video workflows -- ✅ Want AI to describe image -- ✅ Focus on motion/camera +Format an already-written prompt for a specific model. -**Example:** -``` -Image: portrait.jpg -Motion: "slowly turns and smiles" -Camera: push in -Output: Video prompt preserving image + adding motion -``` +### Platform Prompt Rewriter +Rewrites a finished prompt for any of 60+ platform profiles, in that model's exact format (including +structured ones: MiniMax H3 fields, FLUX.2 / Ideogram JSON, edit instructions), and keeps it inside the +text encoder's token limit so trigger words aren't dropped. **Use when** you already have a prompt and want +it tuned for a particular model. Outputs: prompt, negative_prompt, report, system_prompt. -**Pros:** -- Vision model describes image -- Focus on motion -- Camera controls -- Preset support - -**Cons:** -- Needs vision model -- Video-only (not for img2img) -- No platform optimization +### Ideogram 4 Magic Prompt +Builds an Ideogram 4 "magic prompt" (structured JSON + high-level line) with a recommended canvas size. +**Use when** you're generating with Ideogram 4. --- -### 4. Image-to-Image Prompt Expander (NEW!) +## 3. Wildcards & variation -**When to use:** -- ✅ Transforming/editing images -- ✅ Image-to-image workflows -- ✅ Platform-specific needs (Flux, SDXL, etc.) -- ✅ Style transfer, lighting changes +### Wildcard Processor +Resolves wildcard syntax (`__name__`, `{a|b|c}`, weights, ranges, variables, `__objects[metal AND handheld]__`) +read from `models/wildcards`. **Use when** you want wildcard expansion without an LLM. -**Example:** -``` -Image: photo.jpg -Change: "make it oil painting" -Platform: flux -Art Style: oil painting -Output: Flux-optimized prompt for artistic transformation -``` +### Semantic Variation +Moves each random wildcard pick to a semantic neighbour, along named direction axes. **Use when** you want +controlled variety across runs. -**Pros:** -- Platform-aware (6 platforms!) -- Vision model -- Art/style controls -- Optimized per model +### Concept Survival Check +Measures whether the key concepts (a surreal anomaly, exact LoRA triggers, ...) survived the LLM rewrite. +**Use when** you need to confirm something important wasn't paraphrased away. Outputs: report, all_survived, +missing, carried_by_json. -**Cons:** -- Needs vision model -- Image-only (not for video) -- No presets (uses platforms) +### Surreal Premise +Turns an ordinary scene into one anomaly stated as plain physical fact, with an instruction block that +protects it through the rewrite. Its `llm_instructions` plug into any enhancer's `wildcard_instructions`. +**Use when** you want a grounded surreal premise. See SURREAL_GUIDE.md. --- -## Use Case Examples +## 4. Prompt builders (click, don't type) -### Use Case 1: Quick Video from Text +### Wildcard Prompt Composer +Builds a whole prompt section by section with clickable pills instead of typed wildcard syntax, in the +chosen platform's section order. Lock a section to hold it while re-rolling, drag to reorder, save as a +template. **Use when** you want to assemble a prompt visually. See PROMPT_BUILDER_GUIDE.md. -**Goal:** Fast video prompt -**Node:** AI Video Prompt Expander (Standard) -``` -Input: "robot in neon city" -Preset: cinematic -Tier: enhanced -``` - -### Use Case 2: Precise Video Control - -**Goal:** Specific cinematography -**Node:** AI Video Prompt Expander (Advanced) -``` -Input: "spaceship flying" -Shot: wide shot -Camera: tracking shot -Lighting: rim lighting -Lens: wide-angle -``` - -### Use Case 3: Animate Portrait - -**Goal:** Make image move -**Node:** Image-to-Video Prompt Expander -``` -Image: portrait.jpg -Motion: "turns head and smiles" -Camera: dolly in -For: Hunyuan Video, CogVideoX -``` - -### Use Case 4: Flux Style Transfer - -**Goal:** Photo to painting -**Node:** Image-to-Image Prompt Expander -``` -Image: photo.jpg -Change: "impressionist painting" -Platform: flux -Art Style: oil painting -For: FLUX.1-dev -``` - -### Use Case 5: SDXL Enhancement - -**Goal:** Enhance with SDXL -**Node:** Image-to-Image Prompt Expander -``` -Image: low_res.jpg -Change: "enhance quality, add details" -Platform: sd_xl -Detail: highly detailed -For: SDXL -``` - -### Use Case 6: Qwen Quick Edit - -**Goal:** Fast specific edit -**Node:** Image-to-Image Prompt Expander -``` -Image: portrait.jpg -Change: "change background to beach" -Platform: qwen_image_edit -For: Qwen Edit Model -``` - -## Feature Availability - -### Wildcards -✅ All nodes support wildcards! -``` -"A {animal:cat|dog|fox} {action:running|flying}" -``` - -### Random Mode -✅ Standard: Full random -✅ Advanced: Respects dropdowns -✅ Img2Vid: Available -❌ Img2Img: Uses platforms instead - -### Vision Model -❌ Standard: No -❌ Advanced: No -✅ Img2Vid: Yes -✅ Img2Img: Yes - -### Platform Optimization -❌ Standard: No -❌ Advanced: No (Wan 2.2 focus) -❌ Img2Vid: No (video focus) -✅ Img2Img: Yes! (6 platforms) - -## Workflow Recommendations - -### Video Generation Workflow - -**Option 1: Simple** -``` -[Standard Node] → [Video Gen Model] -``` - -**Option 2: Advanced** -``` -[Advanced Node] → [Video Gen Model] -``` - -**Option 3: From Image** -``` -[Load Image] → [Img2Vid Node] → [Video Gen Model] -``` - -### Image Generation Workflow - -**From Image:** -``` -[Load Image] → [Img2Img Node] → [Image Gen Model] -``` - -**Platform-Specific:** -``` -For Flux: [Img2Img] (platform: flux) → [Flux Model] -For SDXL: [Img2Img] (platform: sd_xl) → [SDXL Model] -For Qwen: [Img2Img] (platform: qwen_image) → [Qwen Model] -``` - -## Common Questions - -### Q: Which node for best video prompts? -**A:** -- Quick: Standard -- Precise: Advanced -- From image: Img2Vid - -### Q: Which node for Flux img2img? -**A:** Image-to-Image with platform: flux - -### Q: Can I use vision model with video nodes? -**A:** Only Img2Vid and Img2Img have vision support - -### Q: Which supports wildcards? -**A:** All 4 nodes! - -### Q: Which respects my aesthetic dropdowns in random mode? -**A:** Advanced node (video) - -### Q: Which optimizes for my image model? -**A:** Image-to-Image (6 platforms!) - -## Summary - -**Choose based on:** - -| Your Task | Use This Node | -|-----------|---------------| -| Text → Video prompt (quick) | Standard | -| Text → Video prompt (precise) | Advanced | -| Image → Video prompt | Image-to-Video | -| Image → Image prompt (Flux) | Image-to-Image (flux) | -| Image → Image prompt (SDXL) | Image-to-Image (sd_xl) | -| Image → Image prompt (Qwen) | Image-to-Image (qwen) | -| Image → Quick edit | Image-to-Image (qwen_edit) | - -**All nodes support:** -- ✅ Wildcards -- ✅ LM Studio / Ollama -- ✅ Multiple variations -- ✅ File saving -- ✅ Keyword injection - -**Special features:** -- Vision model: Img2Vid, Img2Img only -- Platform optimization: Img2Img only -- Full aesthetic control: Advanced, Img2Img -- Random mode: Standard, Advanced, Img2Vid +### Wildcard Character Composer +The same node limited to the Subject section, for a character you reuse across prompts or branches. --- -**Pick the right tool for your workflow!** +## 5. Prompt database -All nodes work together - use Standard for quick video prompts, Img2Img for Flux transformations, etc. +| Node | Does | +|------|------| +| **Save Prompt to DB** | Saves a prompt (+ optional negative) to the local SQLite DB; passes the prompt through unchanged | +| **Search Prompt DB** | Finds the top match by keyword, category, platform, rating, tags or favourites | +| **Import Prompts to DB** | Bulk-imports prompts from `.txt` / `.csv` / other files | +| **Browse / Random Prompt DB** | Fetches a prompt by ID, or a random/filtered one by seed | + +**Use when** you want to build and reuse a personal library of prompts. + +--- + +## Feature availability (expanders) + +| | Standard | Advanced | Img2Vid | Img2Img | Txt2Img | +|---|---|---|---|---|---| +| **Presets** | ✅ | ✅ | ✅ | ❌ (platforms) | ❌ (genres) | +| **Wildcards** | ✅ | ✅ | ✅ | ✅ | ✅ | +| **Vision model** | ❌ | optional | ✅ | ✅ | optional | +| **Platform optimization** | video target | video target | video target | ✅ 6 | ✅ 8 | +| **Random mode** | ✅ | ✅ (respects controls) | ✅ | ❌ | ✅ | + +--- + +## Common questions + +**Q: Quick video prompt?** Video Prompt Expander. **Precise?** Advanced. **From an image?** Image-to-Video. + +**Q: I already have a prompt — how do I format it for my model?** Platform Prompt Rewriter (60+ profiles). + +**Q: Which nodes support wildcards?** All the expanders, plus the dedicated Wildcard Processor and the +Composer builders. + +**Q: Which use a vision model?** Image-to-Video and Image-to-Image always; Advanced and Text-to-Image +optionally. + +**Q: How do I build a prompt without typing wildcard syntax?** Wildcard Prompt Composer (whole prompt) or +Wildcard Character Composer (one character). + +**Q: Where do I keep prompts I like?** The four Prompt Database nodes. + +--- + +**Pick the right tool for your workflow — the nodes are designed to chain together.** diff --git a/docs/PROMPT_BUILDER_GUIDE.md b/docs/PROMPT_BUILDER_GUIDE.md new file mode 100644 index 0000000..c2fac3e --- /dev/null +++ b/docs/PROMPT_BUILDER_GUIDE.md @@ -0,0 +1,297 @@ +# Wildcard Composer Guide + +Builder nodes write a prompt section by section, with buttons instead of typed wildcard syntax. + +- **Wildcard Prompt Composer** holds every section in one node: Format, Subject, Action, Setting, + Light & atmosphere, Composition, Camera and Style & finish (plus Text when you add it), shown in + the order the chosen platform wants them. Most of the time this is the only builder node you need. +- **Wildcard Character Composer** is the same node limited to the Subject section (with Pose from Action), for + a character you want to reuse across several Wildcard Prompt Composers or branches. + +A Subject phrase, for example: + +```text +a [Age] [Occupation] in typical uniform, with dark wrinkled skin, a [Face] and [Hair color] hair +lit by glowing fiery red light, with a [Expression] expression +``` + +The words are yours; each `[Pill]` is a wildcard. One roll reads: *"a middle-aged adult professor in +typical uniform, with dark wrinkled skin, a gaunt face and cinnamon brown hair lit by glowing fiery +red light, with a thoughtful expression."* + +## The panel + +**Section strip** (top): the platform's section order, read from the `structure` line of the +platform profile you pick in `platform` (Krea 2: subject with its actions, then setting, light, +medium, framing; FLUX.2: subject, action, style, setting, lighting, technical). Click a section to +work on it; it turns amber. Sections with something in them have a solid outline. `+ Subject` adds +Subject 2 (3, ...), placed right after Subject so each person keeps their own attributes, which is +what Krea 2 asks for. `+ Text` adds the optional Text section (words shown in the picture). + +**Your own section order**: drag a section chip onto another to reorder. The node then builds and +joins its sections in that order (saved with the node and its templates) whatever the platform; +**Platform order** goes back to the platform's. The sections and why: dev/PROMPT_TAXONOMY_SPEC.md. + +**Prompt frame**: every section of the prompt, one line each, in platform order. Every gap between +pills is a text box; type there. The amber gap is the insert point: click a gap (in any section) to +move it there. Typed text may use wildcard syntax too (`{red|blue}`, `__name__`). A word doubled +across a pill (`auburn hair` + typed `hair`) is written once. Each line has two switches: + +- **Lock** keeps that section's picks while the seed moves, so you can hold a character and re-roll + only the setting (the section is held at the seed it last rolled with; click again to release). +- **LLM fills** asks the Platform Prompt Rewriter to write the section when you leave it empty + ("a setting that suits the subject"). It needs `llm_instructions` connected to the rewriter's + `wildcard_instructions`. A section you have written in is used as written. + +**Moving wildcards**: drag a pill to a gap (in its own section or another one) to move it there; +drop it on another pill to put it just before that one. The text on either side of where it was +closes up, as when a pill is removed. Clicking a pill opens its list and makes the gap right after +it the insert point, so the next slot button adds a wildcard just after the one you selected. +To add one at the very start, click the small gap before the first pill. + +**Slot buttons**: the buttons for the active section (Setting shows Location, Interior, Era, +Weather...), under headings for the Subject (Person, Face & hair, Clothing, Accessories). A button +appears only when your library has at least one list for it, so another user's panel shows fewer +or more buttons depending on their packs; hover a button to see how many lists back it. + +**Subject type** (the menu on Subject lines): Person, Animal, Creature, Object, Vehicle or Food +shows that type's buttons (an Animal subject isn't offered Makeup); *Any type* shows them all. +*Scene only* means no subject at all (a landscape, an empty plate for a composite, a storyboard +background): the Subject line is left out of the prompt and the rewriter is told not to add one. + +**Arrange** (next to *+ Button*): drag a button onto another button to put it there, onto a group +heading to move it under that heading, or onto a section chip to move it to that section; *hide* +takes it off the palette (Arrange still shows it, to bring it back). The layout is saved in +`builders_user/button_layout.yaml` and applies to every node; **Reset layout** returns to the +shipped one. The shipped `builders/prompt.yaml` is never changed. + +**Any · Woman · Man** (on Subject lines, for people): which lists the buttons use for that subject. Lists +named for the other gender are left out and lists named for this one come first (lists named for +neither follow). Switching moves the pills that are still on Random to the best list for the new +choice; pills with picks keep their list. Subject 2 has its own switch. Click one to insert a pill at the amber gap. Its list is the best match in your +library (see *Where the lists come from*); change it in the list viewer. + +**One row per pill** of the active section: + +| control | what it does | syntax written | +|---|---|---| +| Random | any entry | `__list__` | +| Fixed | always your first pick, or your own text | `{1$$value}` | +| Subset | random among your picks | `{a\|b\|c}` | +| Exclude | random, never your picks | `__list!{a\|b}__` | +| − n + | that many different entries, joined with "and" | `{2$$ and $$…}` | +| Meaning | the LLM polish may reword it, keeping the meaning | (default) | +| Exact | kept word for word (names, LoRA triggers) | `=` marker | +| Loose | inspiration; the LLM may drop it | `?` marker | + +The row also shows what the pill rolled for the current seed. + +**List viewer** (click a pill or its row). Long lists load 300 entries at a time as you scroll. +Lists rated above off join the source menu at that level, marked with it (`suggestive ·`, `explicit ·`). The +viewer's *NSFW level* menu rates the list, and every rated entry has its own small menu (auto = its words); +entries the node's `nsfw` level leaves out are greyed. Both save to `_catalog_user.yaml`. choose the source list (every list in your library +that matches the slot; lists with identical content are shown once, and each is labelled short, +medium or long by its median entry length), search it, tick entries, then *Fix to first pick*, +*Use picks as subset* or *Exclude picks*. Rate while you look: the stars rate the list (starred lists +come first for that slot everywhere), *Hide list* takes it out of every slot and bare name, and +*drop* on an entry means it is never picked again from that list. All three are saved in +`_catalog_user.yaml` next to the lists, so they apply to every node, not just this one. *Or fix to your own value* sets any text, for example an +exact age such as "7-year-old" or "42-year-old". + +**Preview**: exactly what the node outputs for the seed on the node (the same code runs both). +*Re-roll* moves the seed on by one. + +## Outputs and chaining + +| output | use | +|---|---| +| `sections` | every section filled so far. Connect to the next builder's `sections` input. | +| `text` | all sections as one prompt in the platform's order. Use as is, or connect to a Platform Prompt Rewriter's `idea` for LLM polish. | +| `llm_instructions` | which picks were random and how to keep them. Connect to the rewriter's `wildcard_instructions` so Exact picks survive the rewrite. | +| `template` | the prompt as unresolved wildcard syntax, to save or to let another node roll it. | +| `report` | the picks and any warnings. | + +Usually: **Wildcard Prompt Composer** → **Platform Prompt Rewriter** (`idea` ← `text`, +`wildcard_instructions` ← `llm_instructions`). + +Chaining is for reuse: one **Wildcard Character Composer** feeding the `sections` input of several +Wildcard Prompt Composers gives the same character in different scenes. A section both nodes fill is joined, the +upstream one first. + +### With the LoRA Suggester (Krea2) + +| from | to | +|---|---| +| Composer `text` | Platform Prompt Rewriter `idea` | +| Composer `llm_instructions` | Rewriter `wildcard_instructions` | +| Composer `sections` | LoRA Suggester `sections` | +| Rewriter `prompt` | LoRA Suggester `text` | +| LoRA Suggester `prompt_with_triggers` | the Krea2 text encoder | +| LoRA Suggester `suggestions` | Eric Krea2 Multi-LoRA Stack `suggestions` | + +The Suggester picks LoRAs from the Composer's separate sections (style and format feed the style role, setting the +environment role, and so on), so the rewriter's prose cannot blur that choice; the trigger words are added after the +rewrite, so the LLM never sees them. The Composer's `nsfw` level travels with `sections` and caps the Suggester too. + +Rewriter settings for this: platform **Krea 2**, `official_prompt` **off** (Krea's own prompt expands short prompts; +off uses the profile's rules: keep every subject, add nothing, polish a detailed prompt rather than inflate it, +60-150 words), temperature about 0.5. `extra_instructions` like "name the medium and style in the first sentence" +keeps the style words where Krea2's 512-token limit cannot cut them. The Krea2 Magic Prompt node is a short-prompt +expander: after a Composer it rewrites the picks and pushes the style words to the end. + +For variety across a batch, put wildcards in the Style and Format sections (a medium, a movement, a rendering), not +only in the subject: the Suggester's `variety` only reshuffles LoRAs that score close to each other, while a different +style phrase picks a different family of LoRAs. "LLM fills" on the Style section leaves the Suggester nothing to read. + +**Sliders.** Slider LoRAs (age, waist, shot size, saturation ...) are not scored like other LoRAs: the Suggester reads +the prompt for phrases that ask for a place on a slider's axis ("wasp waist", "a very old fisherman", "extreme +close-up") and sets that slider to a signed weight. Everyday style words ("detailed", "dark", "comic") are kept below +the firing line so they do not dial anything. The phrases live in the LoRA Catalog tool's `axes.yaml` and +`axes_ladders.yaml`; what every file is, and how to add phrases, calibrate a slider with your own weights or add an +axis: **SLIDERS_AND_AXES.md** in the LoRA Catalog folder. Node widgets: `slider_max`, `slider_range` (used when a +slider's card has no believable range), `slider_min_strength` (1 any cue, 2 the default, 3 only strong asks). + +**exclude / pin boxes.** Whole words, one entry per line or comma separated. Plain words match a LoRA's name or file +name; `folder:`, `role:` and `axis:` (exclude only, except `axis:` pins) widen it. A pin uses one +LoRA, the best match, and may carry a weight: `waist slider=-8`, `axis:age=+3`. The node's tooltips have the full +list, and the report says what each entry matched. + +## Where the lists come from + +`builders/prompt.yaml` defines the sections and every slot button; `builders/character.yaml` +extends it and keeps the subject and action buttons. A slot finds its lists by name: a list belongs to a slot when its +leaf name (last path segment, lower case, pack prefixes such as `cops_wc_` removed) fully matches one +of the slot's `match` patterns. Your stars rank first, then earlier patterns, then the catalog's +quality score (booru-tag lists rank below prose), then bigger lists; hidden lists are left out. The +viewer's source menu shows each list's stars, size, length or `tags`, and quality (`q85`). Lists +with `lora` in the key are never offered. Lists rated above the node's `nsfw` level (by folder, content, or +words in the key: see the Wildcard guide, NSFW levels) are not offered; the others are marked with their level. + +A slot can also take `folders` (regexes searched in the folder part of the key: every list in a +matching folder counts, after `match`) and `search` (words that must all be in the key, see below). +The Name button uses `folders` for packs such as a name megalist whose lists are called +`black - female`. + +When a list's own name says nothing a button knows (`basemodel/eye-color/brown`, +`loom/10-lighting/female/soft`), its nearest meaningful folder name is matched instead: generic +folders (`female`, `general`, `v2` ...) are looked past, leading numbers dropped, the pack folder +never used, and only one level up (further up, folders tend to name a whole scene). + +## Prompt line (your prompt files) + +**+ Prompt** adds a Prompt line; drag its chip like any other line, and put a Setting, Format, a second +Subject or anything else around it. Its button, **Browse prompts.**, opens your prompt files (`_prompts`, +see the Wildcard guide) folder by folder: click a file to read its first prompts, **Use** inserts it, +**Use the whole folder** inserts a pill that draws from every file in that folder. The search box finds +files by name in every folder. Click the pill afterwards to see all its prompts, search them, pick, fix, +exclude or drop some, exactly as with any list; **Browse prompts.** in that window swaps the file. The +Prompt line feeds every role of the LoRA Suggester. A whole prompt often names its own place or style, so +an added Setting can contradict it; the Platform Prompt Rewriter after the Composer smooths that out. + +## More lists + +**More lists…**, the last button of every palette, shows the usable lists no button above offers: +word, phrase and tag lists with 3+ entries (whole prompts and templates stay with Find). Each gets +a section from the catalog's reading of its content, or from words in its path, and a group from +the path word it shares with the most other lists (Makeup, Creature, Food ...). Click a group, then +a list to insert it; ☆ saves one list as a button, **☆ Button for all N lists** saves the group. +Lists with identical content are shown once. + +## Find, and your own buttons + +The **Find** box next to the section heading searches every list: first lists whose key has all your +words, then (when the catalog is built) lists whose entries match. Click a result to insert that +list at the amber gap; the pill is labelled with the list's name. The Woman / Man switch of the +section filters Find too. + +Save what you find as a button, under **My buttons** at the top of the section's palette: + +* **☆ Button** on a result (or in a pill's list viewer): a button for that one list. +* **☆ Button for "words"** at the foot of the results, or **+ Button**: a category button. It + offers every list whose key has all the words, folders included, now and as you add packs. + A word of 4+ letters may sit inside a longer one (`names` finds `peoplenamemegalist`), a plural + `s` is optional, and gender words (`female`, `men`, ...) follow the list's gender, so `male` does + not find female lists. The editor shows how many lists match while you type. + +Buttons are saved in `builders_user/my_buttons.yaml` (plain YAML, safe to edit by hand) and are +shared by every builder node whose parts show that section; ✎ on a button edits or deletes it. +A button of yours whose lists are all rated above off is hidden while `nsfw` is off. + +```yaml +buttons: + - id: my_black_names + label: Black names + section: subject + lists: [blackamericanfemalefirstnames_v10/black_female_names] + - id: my_famous_buildings + label: Famous buildings + section: setting + search: famous buildings +``` + +To change a slot, add slots or pin your favourite lists, copy `prompt.yaml` to `builders_user/` +(same `id` replaces the shipped one) and edit it. Slots are grouped by the section whose buttons +show them; the package's own curated lists (light/setups, camera/lenses, art/mediums, ...) are +pinned first: + +```yaml + - section: subject + slots: + - id: hair_color + label: Hair color + match: ['hair_?colou?rs?'] + avoid: ['fantasy'] + lists: ['my_pack/hair_colors_curated'] # pinned first +``` + +A new YAML file with a new `id` defines a new builder; it needs a node class (one line in +`prompt_builder_node.py`). + +## Merging lists + +Most packs share one core vocabulary and add their own tail (in one library, "black" was in 13 of +26 hair colour lists). **Merge lists…** in a pill's list viewer folds every list behind that button +into one: + +* Near-identical entries become one row: case, hyphens, weights and the button's own words are + ignored, so "Black hair", "black" and "black-haired" fold for Hair color. The most common + spelling is shown; hover a row for the lists it came from and the other spellings. +* Rows are sorted by how many lists have them (`×13`); entries in 3+ lists are highlighted. Show + only entries in 2+ / 3+ / 5+ lists, or only short, medium or long ones; what is shown (and ticked) + is what gets saved. +* Click a row to leave it out. **Untick shown** / **Tick shown** work on what the search box finds, + so "untick everything with 'glowing'" is two steps. +* Untick source lists at the top to leave them out of the merge. +* **Save list** writes `mine/ + + + + + +
+ +
loading...
+
+ + +

Apply decisions

+
...
+

+ +
 + + + diff --git a/tools/wildcard_review/server.py b/tools/wildcard_review/server.py new file mode 100644 index 0000000..6818f2e --- /dev/null +++ b/tools/wildcard_review/server.py @@ -0,0 +1,1207 @@ +""" +Wildcard review: a small local web server (this machine only) for paging through the wildcard lists of +A:\\Models\\wildcards and deciding Keep / Dump / Same as / Unsure, with the AI judge's opinion pre-filled +(dev/wildcard_judge.py, table `judgements` of _triage.sqlite). Built like the LoRA card editor. + + python tools/wildcard_review/server.py then open http://127.0.0.1:8766 + (or run tools/wildcard_review/wildcard_review.bat) + python tools/wildcard_review/server.py --undo tools/wildcard_review/manifests/.json + +Decisions are saved as you click, to the `reviews` table of A:\\Models\\wildcards\\_triage.sqlite. Nothing touches +the wildcard files or _catalog_user.yaml until you press Apply (preview first). Apply does, per list: + dump hidden: true (gone from buttons, globs and bare names; __exact/name__ still works) + same as hidden: true, same_as: + keep hidden removed; slot: ; nsfw: (each only when you set it) + rename (when you ticked it): a .txt list is renamed in its folder; a list inside a YAML / JSON file is + written out as /.txt and the old key is hidden (the pack file is not edited). Its + settings, buttons and review follow it, and the old name stays an alias (aliases: in + _catalog_user.yaml), so saved workflows and templates keep working. + unsure nothing +"Move dumped files" (preview first) moves whole dumped .txt files, and YAML / JSON files whose every list you +dumped, to A:\\Models\\wildcards\\_dumped\\ (a folder the engine skips). Dumped keys inside files that +also hold kept lists stay hidden. Renames and moves write a manifest in tools/wildcard_review/manifests (--undo). + +Your own Composer buttons (builders_user/my_buttons.yaml) are offered next to the shipped ones, and "+ new button" +makes one (label + section) with the list on it. A list can sit on several buttons. + +Endpoints (JSON unless noted) + GET / the page (review.html) + GET /api/meta facets, Composer buttons by section, counts, library state + GET /api/queue?order=&facet=&verdict=&status=&source=&q= + GET /api/list?key=&seed= one list: catalog row, judge, your review, a sample of entries, similar lists + GET /api/entries?key=&q=&offset=&limit= + GET /api/page?keys=k1,k2,... rows for the grid view (samples only, fast) + POST /api/review {key, decision, same_as?, button?, name?, nsfw?, note?} (button: ids, comma-joined) + POST /api/review_many {items: [{key, decision}]} + POST /api/button {label, section, key?} -> a new button in my_buttons.yaml (with that list on it) + GET /api/apply_preview what Apply would write and rename + POST /api/apply write _catalog_user.yaml, do the renames + GET /api/dump_preview what "Move dumped files" would move + POST /api/dump_move move them to _dumped + +(c) Eric Hiss. Dual license, see LICENSE. +""" + +from __future__ import annotations + +import json +import os +import random +import re +import shutil +import sqlite3 +import sys +import threading +import time +import webbrowser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List, Optional +from urllib.parse import parse_qs, urlparse + +HERE = os.path.dirname(os.path.abspath(__file__)) +PACKAGE = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, PACKAGE) +from core.wildcards import library as L # noqa: E402 +from core.wildcards.curation import UserOverlay # noqa: E402 +from core.builder.slots import button_id, clean_button, read_buttons, write_buttons # noqa: E402 +from core.builder.layout import layout_move, read_layout, write_layout # noqa: E402 +from core.builder.merge import fold_key, merge_entries, merged_key, merged_name, merged_path, write_merged # noqa: E402 + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + +# WCR_ROOT / WCR_BUTTONS / WCR_MANIFESTS let the checks (dev/verify_wildcard_review.py) run on a temporary library. +LIBRARY_ROOT = os.environ.get("WCR_ROOT", r"A:\Models\wildcards") +TRIAGE = os.path.join(LIBRARY_ROOT, "_triage.sqlite") +BUILDERS = os.path.join(PACKAGE, "builders", "prompt.yaml") +MY_BUTTONS = os.environ.get("WCR_BUTTONS", os.path.join(PACKAGE, "builders_user", "my_buttons.yaml")) +MANIFESTS = os.environ.get("WCR_MANIFESTS", os.path.join(HERE, "manifests")) +LAYOUT = os.environ.get("WCR_LAYOUT", os.path.join(PACKAGE, "builders_user", "button_layout.yaml")) +MERGE_PREVIEW_MAX = 4000 # rows the merge preview sends (the merge itself uses every entry) +DUMPED_FOLDER = "_dumped" +STRUCTURED = (".yaml", ".yml", ".json") +PORT = 8766 +SAMPLE = 36 +DECISIONS = ("keep", "dump", "same", "unsure", "merge") +MY_GROUP = "My buttons" + +REVIEWS = """ +CREATE TABLE IF NOT EXISTS reviews ( + key TEXT PRIMARY KEY, decision TEXT, same_as TEXT, button TEXT, name TEXT, nsfw TEXT, note TEXT, + how TEXT, decided_at TEXT, applied_at TEXT +); +""" +# added 2026-10-06: where Move dumped put the file; the old key; a merge's partners (comma-joined), name, unticked +# entries (JSON list) and the merged list a source went into +EXTRA_COLUMNS = ("moved_to", "renamed_from", "merge_with", "merge_name", "merge_drop", "merged_into") + +# -- the local LLM (LM Studio, OpenAI-compatible; the same server dev/wildcard_judge.py uses) ----------------------------- +LLM_URL = os.environ.get("WCR_LLM", "http://localhost:1234/v1") +LLM_DEFAULT_MODEL = "qwen3-vl-8b-instruct-abliterated-v2.0" +JUNK_CHUNK = 120 # lines per Suggest-junk request +JUNK_MAX = 1200 # lines Suggest junk reads at most +GEN_BATCH = 40 # new lines per generate request +NAMEY = re.compile(r"\b(artists?|painters?|photographers?|illustrators?|designers?|directors?|authors?|writers?|" + r"celebrit\w*|actors?|actress\w*|brands?|cit(y|ies)|towns?|villages?|landmarks?|streets?|" + r"restaurants?|people|persons?|names?|surnames?|characters?)\b", re.I) + + +def llm_models() -> List[str]: + import requests + response = requests.get(LLM_URL + "/models", timeout=6) + response.raise_for_status() + return [m.get("id") for m in response.json().get("data") or [] if m.get("id")] + + +def llm_json(model: str, system: str, user: str, schema: Dict[str, Any], max_tokens: int = 2000, + temperature: float = 0.0) -> Dict[str, Any]: + """One chat call that must answer with JSON matching `schema`.""" + import requests + body = {"model": model or LLM_DEFAULT_MODEL, "temperature": temperature, "max_tokens": max_tokens, + "response_format": {"type": "json_schema", "json_schema": {"name": "answer", "strict": True, "schema": schema}}, + "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}]} + response = requests.post(LLM_URL + "/chat/completions", json=body, timeout=600) + response.raise_for_status() + text = response.json()["choices"][0]["message"]["content"] or "" + text = re.sub(r"(?s).*?", "", text).strip() + try: + return json.loads(text) + except ValueError: + found = re.search(r"(?s)\{.*\}", text) + if found: + return json.loads(found.group(0)) + raise ValueError(f"the model did not answer with JSON: {text[:200]}") + + +JUNK_SYSTEM = """You clean wildcard lists for an image-prompt tool. A wildcard list holds interchangeable values of ONE kind +(hair colours, castles, film stocks ...); the tool picks one at random. You are told what the list is meant to hold and +shown its numbered lines. Most lines fit. Return the numbers of the lines that clearly do NOT belong: a different kind of +thing, garbage or fragments, prompt tags or weights, instructions, URLs, empty filler. Unusual but valid members of the +kind DO belong. When in doubt, leave a line out of your answer. If every line fits, return an empty list.""" + +GEN_SYSTEM = """You write wildcard lists for an image-prompt tool. A wildcard list holds interchangeable values of ONE kind; +the tool picks one at random and puts it into an image prompt, so every item must work on its own in a sentence like +"a photo of ..., ". Write NEW items of exactly the kind described, in the same form and length as the examples: +varied, concrete, visual, no duplicates or near-duplicates of the examples or of each other, no numbering, no +explanations. Only real, well-known names when the list is of real people, places or brands; never invent them.""" + +STYLES = {"words": "a single word or a two-word name", "short": "a short phrase of two to five words", + "descriptive": "a descriptive phrase of six to fifteen words"} + + +# Lists the review covers: the judged candidates, plus the 5-7 entry and NSFW lists judged with --extra. +SOURCE_SQL = """ +case when t.stage1_bucket = 'candidate' then 'candidate' + when t.stage1_bucket = 'nsfw' then 'nsfw' + else 'small' end +""" + + +def _words(text: str) -> set: + return {w for w in re.split(r"[^a-z0-9]+", (text or "").lower()) if len(w) > 2} + + +def slug(name: str) -> str: + """'Bird species (real)' -> 'bird_species_real': the file name a renamed list gets.""" + return re.sub(r"[^a-z0-9]+", "_", (name or "").lower()).strip("_")[:80] + + +def button_ids(value: Any) -> List[str]: + """The review's button field: one id, several comma-joined, or a list.""" + items = value if isinstance(value, list) else str(value or "").split(",") + return [str(item).strip() for item in items if str(item).strip()] + + +def _abs(file: str) -> str: + return os.path.normpath(file if os.path.isabs(file) else os.path.join(LIBRARY_ROOT, file)) + + +def _rel(path: str) -> str: + return os.path.relpath(path, LIBRARY_ROOT).replace("\\", "/") + + +class Review: + def __init__(self, load_library: bool = True): + self.lock = threading.Lock() + self.library: Optional[L.WildcardLibrary] = None + self.library_state = "loading" + with self.connect() as connection: + connection.executescript(REVIEWS) + have = {row[1] for row in connection.execute("pragma table_info(reviews)")} + for column in EXTRA_COLUMNS: + if column not in have: + connection.execute(f"alter table reviews add column {column} TEXT") + self._rows: Optional[List[Dict[str, Any]]] = None + self._rows_at = 0.0 + if load_library: + threading.Thread(target=self._load_library, name="wildcard-review-library", daemon=True).start() + else: + self._load_library() + + # -- setup ------------------------------------------------------------------------------------------------------- + def connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(TRIAGE, timeout=30) + connection.row_factory = sqlite3.Row + return connection + + def _load_library(self) -> None: + try: + started = time.time() + library = L.WildcardLibrary([LIBRARY_ROOT]) + library.refresh(force=True) + self.library = library + self.library_state = f"ready ({time.time() - started:.0f} s)" + except Exception as error: # the page still works from the catalog samples + self.library_state = f"failed: {error}" + print(f"[review] library {self.library_state}") + + def _builder_yaml(self) -> Dict[str, Any]: + if yaml is None: + return {} + try: + with open(BUILDERS, "r", encoding="utf-8") as handle: + return yaml.safe_load(handle) or {} + except OSError: + return {} + + def sections(self) -> List[Dict[str, Any]]: + """The Composer's prompt sections (lines), in the builder's order, with the button headings each shows.""" + out = [] + for block in self.buttons(): + groups = [] + for button in block["buttons"]: + if button["group"] and button["group"] not in groups: + groups.append(button["group"]) + if MY_GROUP not in groups: + groups.insert(0, MY_GROUP) + out.append({"section": block["section"], "label": block["label"], "groups": groups}) + listed = {s["section"] for s in out} + for part in self._builder_yaml().get("parts") or []: + if part.get("section") not in listed: + out.append({"section": part.get("section"), "label": part.get("label", part.get("section")), "groups": [MY_GROUP]}) + return out + + def buttons(self) -> List[Dict[str, Any]]: + """The Composer's buttons by section: your own (my_buttons.yaml, first) and the shipped ones (prompt.yaml).""" + data = self._builder_yaml() + labels = {part.get("section"): part.get("label", part.get("section")) for part in data.get("parts") or []} + everything = [dict(id=b["id"], label=b["label"], group=MY_GROUP, mine=True, section=b.get("section")) + for b in read_buttons(MY_BUTTONS)] + for block in data.get("slots") or []: + everything += [dict(id=s.get("id"), label=s.get("label", s.get("id")), group=s.get("group", ""), mine=False, + section=block.get("section")) for s in block.get("slots") or []] + try: + moved = (read_layout(LAYOUT) or {}).get("moved") or {} # your Arrange moves (section / heading) + except Exception: + moved = {} + for button in everything: + move = moved.get(button["id"]) or {} + button["section"] = move.get("section") or button["section"] + if "group" in move: + button["group"] = move["group"] or (MY_GROUP if button["mine"] else "") + out = [] + for section in list(labels) + sorted({b["section"] for b in everything} - set(labels)): + own = [b for b in everything if b["section"] == section] + if own: + own.sort(key=lambda b: (b["group"] != MY_GROUP, )) # yours first, the rest in their shipped order + out.append({"section": section, "label": labels.get(section, section), + "buttons": [{k: b[k] for k in ("id", "label", "group", "mine")} for b in own]}) + return out + + def create_button(self, label: str, section: str, key: str = "", group: str = "") -> Dict[str, Any]: + """A new button in my_buttons.yaml with this list on it, shown under `group` (a heading of that section).""" + label = (label or "").strip() + if not label: + return {"ok": False, "error": "a button needs a label"} + if section not in {s["section"] for s in self.sections()}: + return {"ok": False, "error": f"unknown section '{section}'"} + name = L.strip_nsfw_prefix(key) if key else "" + raw = {"label": label, "section": section, "lists": [name] if name else [], "search": "" if name else label} + button = clean_button(raw) + if button is None: + return {"ok": False, "error": "could not make that button"} + buttons = read_buttons(MY_BUTTONS) + taken = {b["id"] for b in buttons} | {s.get("id") for block in self._builder_yaml().get("slots") or [] + for s in block.get("slots") or []} + button["id"] = button_id(label, taken) + buttons.append(button) + write_buttons(MY_BUTTONS, buttons) + group = (group or "").strip() + if group and group != MY_GROUP: + # Under one of the section's headings: the same record the Composer's Arrange mode writes. + write_layout(LAYOUT, layout_move(read_layout(LAYOUT), button["id"], section=section, group=group)) + return {"ok": True, "button": dict(button, group=group or MY_GROUP)} + + # -- data -------------------------------------------------------------------------------------------------------- + def rows(self, fresh: bool = False) -> List[Dict[str, Any]]: + """Every list under review with its latest judgement and your review (cached a few seconds).""" + with self.lock: + if self._rows is not None and not fresh and time.time() - self._rows_at < 5: + return self._rows + sql = f""" + select t.key, t.file, t.pack, t.display, t.kind, t.entries, t.nsfw as catalog_nsfw, t.facet as catalog_facet, + {SOURCE_SQL} as source, + j.model, j.dimension, j.facet, j.coherence, j.visual, j.slot_form, j.era_specific, j.verdict, + j.reason, j.confidence, + r.decision, r.same_as, r.button, r.name, r.nsfw as review_nsfw, r.how, r.decided_at, r.applied_at, + r.moved_to, r.renamed_from, r.merge_with, r.merge_name, r.merge_drop, r.merged_into + from triage t + join judgements j on j.key = t.key + and j.judged_at = (select max(j2.judged_at) from judgements j2 where j2.key = t.key) + left join reviews r on r.key = t.key + """ + with self.connect() as connection: + rows = [dict(row) for row in connection.execute(sql)] + seen, unique = set(), [] + for row in rows: # two models judged the same list at the same minute: keep one + if row["key"] not in seen: + seen.add(row["key"]) + unique.append(row) + with self.lock: + self._rows, self._rows_at = unique, time.time() + return unique + + def _forget_rows(self) -> None: + with self.lock: + self._rows = None + + def meta(self) -> Dict[str, Any]: + rows = self.rows() + facets: Dict[str, int] = {} + decided: Dict[str, int] = {} + hidden = moved = renamed = 0 + for row in rows: + facets[row["facet"] or "(none)"] = facets.get(row["facet"] or "(none)", 0) + 1 + if row["decision"]: + decided[row["decision"]] = decided.get(row["decision"], 0) + 1 + if row["decision"] in ("dump", "same", "merge") and row["applied_at"]: + hidden += 1 + moved += bool(row["moved_to"]) + renamed += bool(row["renamed_from"]) + return {"total": len(rows), "decided": decided, "facets": sorted(facets.items()), + "hidden": hidden, "moved": moved, "renamed": renamed, + "sources": {s: sum(1 for r in rows if r["source"] == s) for s in ("candidate", "small", "nsfw")}, + "buttons": self.buttons(), "sections": self.sections(), + "library": self.library_state, "levels": list(L.NSFW_LEVELS)} + + def queue(self, order: str, facet: str, verdict: str, status: str, source: str, query: str) -> List[Dict[str, Any]]: + items = [] + wanted = (query or "").strip().lower() + for row in self.rows(): + if facet and (row["facet"] or "(none)") != facet: + continue + if verdict and row["verdict"] != verdict: + continue + if source and row["source"] != source: + continue + if status == "todo" and row["decision"]: + continue + if status in DECISIONS and row["decision"] != status: + continue + if wanted and wanted not in row["key"] and wanted not in (row["dimension"] or "").lower(): + continue + items.append({"key": row["key"], "dim": row["dimension"] or "", "facet": row["facet"] or "", + "verdict": row["verdict"] or "", "visual": row["visual"], "coherence": row["coherence"], + "entries": row["entries"], "source": row["source"], "decision": row["decision"] or "", + "pack": row["pack"] or "", "moved": bool(row["moved_to"]), + "hidden": row["decision"] in ("dump", "same", "merge") and bool(row["applied_at"])}) + if order == "junk": # most likely junk first: judge says purge, then not visual, then incoherent + items.sort(key=lambda i: (i["verdict"] != "purge", i["visual"] or 0, i["coherence"] or 0, i["key"])) + elif order == "pack": + items.sort(key=lambda i: (i["pack"], i["key"])) + elif order == "size": + items.sort(key=lambda i: (-(i["entries"] or 0), i["key"])) + else: # facet, then what the lists are, so look-alikes sit together + items.sort(key=lambda i: (i["facet"], i["dim"].lower(), i["key"])) + return items + + def _entries(self, key: str) -> List[Any]: + library = self.library + if library is None: + return [] + try: + return library._entries_for_key(key, nsfw=L.NSFW_EXPLICIT, apply_drops=False) + except Exception: + return [] + + def _samples(self, row: Dict[str, Any]) -> List[str]: + with self.connect() as connection: + found = connection.execute("select samples from triage where key = ?", (row["key"],)).fetchone() + try: + return [str(s) for s in json.loads((found["samples"] if found else None) or "[]")] + except ValueError: + return [] + + def similar(self, row: Dict[str, Any], limit: int = 10) -> List[Dict[str, Any]]: + """Lists that look like the same thing: same facet, shared words in what the judge says they are and in the name.""" + mine = _words(row["dimension"]) | _words(row["key"].rsplit("/", 1)[-1]) + if not mine: + return [] + scored = [] + for other in self.rows(): + if other["key"] == row["key"] or other["facet"] != row["facet"]: + continue + theirs = _words(other["dimension"]) | _words(other["key"].rsplit("/", 1)[-1]) + shared = len(mine & theirs) + if shared: + scored.append((shared / len(mine | theirs), other)) + scored.sort(key=lambda pair: -pair[0]) + return [{"key": o["key"], "dim": o["dimension"], "entries": o["entries"], "decision": o["decision"] or "", + "verdict": o["verdict"]} for _s, o in scored[:limit]] + + def rename_target(self, key: str, file: str, name: str) -> Optional[Dict[str, Any]]: + """What renaming this list to `name` does: {kind: rename|split, old, new, path, conflict}. None = no change.""" + new_leaf = slug(name) + if not new_leaf or not file: + return None + path = _abs(file) + if new_leaf == key.rsplit("/", 1)[-1]: + return None + target = os.path.join(os.path.dirname(path), new_leaf + ".txt") + new_key = L.normalise_key(os.path.splitext(_rel(target))[0]) + kind = "rename" if path.lower().endswith(".txt") else "split" + conflict = "" + if os.path.exists(target): + conflict = f"{_rel(target)} already exists" + elif self.library is not None and self.library.real_keys(L.strip_nsfw_prefix(new_key)): + conflict = f"a list called {L.strip_nsfw_prefix(new_key)} already exists" + elif kind == "rename" and not os.path.exists(path): + conflict = f"{_rel(path)} is not there (moved?)" + return {"kind": kind, "old": key, "new": new_key, "path": _rel(target), "from": _rel(path), "conflict": conflict} + + def list_view(self, key: str, seed: int) -> Optional[Dict[str, Any]]: + row = next((r for r in self.rows() if r["key"] == key), None) + if row is None: + return None + entries = [] if row["moved_to"] else self._entries(key) + library = self.library + if entries: + rng = random.Random(seed) + shown = entries if len(entries) <= SAMPLE else [entries[i] for i in sorted(rng.sample(range(len(entries)), SAMPLE))] + gone = self._dropped(key) + sample = [{"text": e.text, "rating": library.entry_rating(e), "dropped": e.text.strip().casefold() in gone} + for e in shown] + total = len(entries) + rated = [0, 0, 0, 0] + for entry in entries: + rated[library.entry_rating(entry)] += 1 + list_rating = library._view(L.NSFW_EXPLICIT).rating.get(L.strip_nsfw_prefix(key), 0) + else: + sample = [{"text": t, "rating": L.nsfw_word_tier(t)} for t in self._samples(row)] + total, rated, list_rating = row["entries"], None, None + file = row["file"] or "" + return { + "key": key, "name": L.strip_nsfw_prefix(key), "file": _rel(_abs(file)) if file else "", + "kind_of_file": "txt" if file.lower().endswith(".txt") else "structured", + "pack": row["pack"], "kind": row["kind"], "entries": total, "source": row["source"], + "catalog_nsfw": row["catalog_nsfw"], "list_rating": list_rating, "rated": rated, + "judge": {k: row[k] for k in ("model", "dimension", "facet", "coherence", "visual", "slot_form", + "era_specific", "verdict", "reason", "confidence")}, + "review": {k: row[k] for k in ("decision", "same_as", "button", "name", "review_nsfw", "decided_at", + "applied_at", "moved_to", "renamed_from", "merge_with", "merge_name", + "merge_drop", "merged_into")}, + "sample": sample, "from_library": bool(entries), "similar": self.similar(row), + } + + def entries(self, key: str, query: str, offset: int, limit: int) -> Dict[str, Any]: + entries = self._entries(key) + wanted = (query or "").strip().casefold() + matched = [e for e in entries if not wanted or wanted in e.text.casefold()] + library = self.library + gone = self._dropped(key) + return {"total": len(entries), "matched": len(matched), "dropped": len(gone), + "entries": [{"text": e.text, "rating": library.entry_rating(e) if library else 0, + "dropped": e.text.strip().casefold() in gone} + for e in matched[offset:offset + limit]]} + + # -- quick edits: drops (saved at once) ------------------------------------------------------------------------------ + def _overlay(self) -> UserOverlay: + overlay = getattr(self, "_overlay_cache", None) + if overlay is None: + overlay = self._overlay_cache = UserOverlay(LIBRARY_ROOT) + overlay.reload_if_changed() + return overlay + + def _dropped(self, key: str) -> set: + entry = self._overlay().list(L.strip_nsfw_prefix(key)) + return {str(t).strip().casefold() for t in entry.get("drop") or [] if str(t).strip()} + + def drop(self, key: str, texts: List[str], undrop: bool = False) -> Dict[str, Any]: + """Drop (or bring back) lines of a list: _catalog_user.yaml drop:, saved now. The file is not edited.""" + name = L.strip_nsfw_prefix(key) + with self.lock: + overlay = self._overlay() + entry = dict(overlay.list(name)) + drops = [str(t) for t in entry.get("drop") or []] + folded = {t.strip().casefold() for t in drops} + for text in texts: + text = str(text).strip() + if not text: + continue + if undrop: + drops = [t for t in drops if t.strip().casefold() != text.casefold()] + folded.discard(text.casefold()) + elif text.casefold() not in folded: + drops.append(text) + folded.add(text.casefold()) + if drops: + entry["drop"] = drops + else: + entry.pop("drop", None) + if entry: + overlay.data["lists"][name] = entry + else: + overlay.data["lists"].pop(name, None) + overlay.save() + return {"ok": True, "dropped": len(drops)} + + def drop_matching(self, key: str, pattern: str, regex: bool, do_it: bool) -> Dict[str, Any]: + """The lines that have `pattern` (a word, or a regex); with do_it, drop them.""" + pattern = (pattern or "").strip() + if not pattern: + return {"ok": True, "matches": []} + try: + test = re.compile(pattern, re.I).search if regex else (lambda text: pattern.casefold() in text.casefold()) + except re.error as error: + return {"ok": False, "error": f"bad pattern: {error}"} + matches = [e.text for e in self._entries(key) if test(e.text)] + if do_it and matches: + self.drop(key, matches) + return {"ok": True, "matches": matches[:500], "count": len(matches), "dropped": bool(do_it)} + + # -- the local LLM: Suggest junk, New list, Extend ------------------------------------------------------------------- + def suggest_junk(self, key: str, model: str) -> Dict[str, Any]: + row = next((r for r in self.rows() if r["key"] == key), None) + meant = (row or {}).get("dimension") or key.rsplit("/", 1)[-1].replace("_", " ") + gone = self._dropped(key) + texts = [e.text for e in self._entries(key) if e.text.strip().casefold() not in gone][:JUNK_MAX] + schema = {"type": "object", "properties": {"off": {"type": "array", "items": {"type": "integer"}}}, + "required": ["off"], "additionalProperties": False} + flagged: List[str] = [] + for start in range(0, len(texts), JUNK_CHUNK): + chunk = texts[start:start + JUNK_CHUNK] + user = (f"The list is meant to hold: {meant}\nIts name: {key}\n\nLines:\n" + + "\n".join(f"{i + 1}. {t}" for i, t in enumerate(chunk))) + answer = llm_json(model, JUNK_SYSTEM, user, schema, max_tokens=600) + for number in answer.get("off") or []: + if isinstance(number, int) and 1 <= number <= len(chunk): + flagged.append(chunk[number - 1]) + return {"ok": True, "flagged": flagged, "read": len(texts), "meant": meant} + + def generate(self, body: Dict[str, Any]) -> Dict[str, Any]: + """One batch of new lines for a list that is being made or extended; repeats are folded away.""" + name = str(body.get("name") or "").strip() + description = str(body.get("description") or "").strip() + samples = [str(s).strip() for s in body.get("samples") or [] if str(s).strip()] + have = [str(s).strip() for s in body.get("have") or [] if str(s).strip()] + style = STYLES.get(str(body.get("style") or "short"), STYLES["short"]) + count = max(5, min(80, int(body.get("count") or GEN_BATCH))) + if not (description or samples or name): + return {"ok": False, "error": "say what the list holds, or give a few samples"} + shown_samples = samples[:25] + already = (have + samples)[-150:] + user = (f"List name: {name or '(none)'}\nWhat it holds: {description or '(see the examples)'}\n" + f"Each item is {style}.\n\nExamples of the kind and form wanted:\n" + + "\n".join(f"- {s}" for s in shown_samples) + + ("\n\nAlready in the list, do not repeat or rephrase any of these:\n" + "\n".join(f"- {s}" for s in already) + if already else "") + + f"\n\nWrite {count} NEW items.") + schema = {"type": "object", "properties": {"items": {"type": "array", "items": {"type": "string"}}}, + "required": ["items"], "additionalProperties": False} + answer = llm_json(str(body.get("model") or ""), GEN_SYSTEM, user, schema, max_tokens=60 * count + 200, + temperature=float(body.get("temperature") or 0.9)) + seen = {fold_key(s) for s in have + samples} + fresh = [] + for item in answer.get("items") or []: + text = re.sub(r"^\s*(?:[-*\u2022]|\d+[.)])\s*", "", str(item)).strip().strip('"').strip() + fold = fold_key(text) + if text and fold and fold not in seen and len(text) <= 300: + seen.add(fold) + fresh.append(text) + warn = bool(NAMEY.search(f"{name} {description}")) + return {"ok": True, "items": fresh, "asked": count, "warn_names": warn} + + def save_list(self, body: Dict[str, Any]) -> Dict[str, Any]: + """Save a made or extended list to mine//.txt and put it on its buttons.""" + entries = [str(e).strip() for e in body.get("entries") or [] if str(e).strip()] + extend = str(body.get("extend") or "").strip() + ids = button_ids(body.get("buttons")) + name = str(body.get("name") or "").strip() or (extend.rsplit("/", 1)[-1] if extend else "") + if not name: + return {"ok": False, "error": "the list needs a name"} + if extend: + gone = self._dropped(extend) + entries = [e.text for e in self._entries(extend) if e.text.strip().casefold() not in gone] + entries + kept, seen = [], set() + for text in entries: + fold = fold_key(text) + if fold and fold not in seen: + seen.add(fold) + kept.append(text.replace("\n", " ")) + if not kept: + return {"ok": False, "error": "nothing to save"} + slot = ids[0] if ids else "generated" + path = merged_path(LIBRARY_ROOT, slot, name) + if os.path.exists(path): + return {"ok": False, "error": f"{_rel(path)} already exists: pick another name"} + new_name = merged_key(slot, name) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + handle.write(f"# Made in the wildcard review on {time.strftime('%Y-%m-%d')} with {body.get('model') or LLM_DEFAULT_MODEL}" + + (f", extending {extend}" if extend else "") + f": {len(kept)} entries.\n") + if body.get("description"): + handle.write(f"# {str(body.get('description')).strip()[:300]}\n") + handle.write("\n".join(kept) + "\n") + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + with self.lock: + overlay = UserOverlay(LIBRARY_ROOT) + settings = {} + if ids: + settings["slot"] = ids if len(ids) > 1 else ids[0] + if body.get("nsfw"): + settings["nsfw"] = str(body["nsfw"]) + if settings: + overlay.data["lists"][new_name] = settings + old_settings, partner_reviews = {}, {} + if extend: + source = L.strip_nsfw_prefix(extend) + old_settings[source] = dict(overlay.list(source)) + hidden = dict(overlay.list(source), hidden=True, merged_into=new_name) + hidden.pop("slot", None) + overlay.data["lists"][source] = hidden + overlay.save() + if extend: + with self.connect() as connection: + old = connection.execute("select * from reviews where key = ?", (extend,)).fetchone() + partner_reviews[extend] = dict(old) if old else None + connection.execute( + """insert into reviews (key, decision, same_as, how, decided_at, applied_at, merged_into) + values (?, 'same', ?, 'extend', ?, ?, ?) + on conflict(key) do update set decision='same', same_as=excluded.same_as, how='extend', + decided_at=excluded.decided_at, applied_at=excluded.applied_at, merged_into=excluded.merged_into""", + (extend, new_name, stamp, stamp, new_name)) + manifest = {"made": stamp, "root": LIBRARY_ROOT, "kind": "made", "renames": [], "moves": [], + "merges": [{"key": "", "with": [], "into": new_name, "path": _rel(path), "count": len(kept), + "old_settings": old_settings, "partner_reviews": partner_reviews}]} + manifest_path = self._save_manifest(manifest) + self._forget_rows() + if self.library is not None: + self.library.refresh(force=True) + return {"ok": True, "key": new_name, "path": _rel(path), "count": len(kept), "manifest": manifest_path} + + def page(self, keys: List[str]) -> List[Dict[str, Any]]: + by_key = {r["key"]: r for r in self.rows()} + out = [] + with self.connect() as connection: + for key in keys: + row = by_key.get(key) + if row is None: + continue + found = connection.execute("select samples from triage where key = ?", (key,)).fetchone() + try: + samples = json.loads((found["samples"] if found else None) or "[]") + except ValueError: + samples = [] + out.append({"key": key, "dim": row["dimension"], "facet": row["facet"], "verdict": row["verdict"], + "reason": row["reason"], "visual": row["visual"], "coherence": row["coherence"], + "entries": row["entries"], "source": row["source"], "decision": row["decision"] or "", + "samples": [str(s)[:120] for s in samples[:10]]}) + return out + + # -- saving ------------------------------------------------------------------------------------------------------ + def save(self, body: Dict[str, Any], how: str = "card") -> Dict[str, Any]: + key = str(body.get("key") or "") + decision = str(body.get("decision") or "") + if decision and decision not in DECISIONS: + return {"ok": False, "error": f"unknown decision {decision}"} + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + if "button" in body: + body = dict(body, button=",".join(button_ids(body.get("button")))) + with self.connect() as connection: + if not connection.execute("select 1 from triage where key = ?", (key,)).fetchone(): + return {"ok": False, "error": "unknown list"} + old = connection.execute("select * from reviews where key = ?", (key,)).fetchone() + old = dict(old) if old else {} + values = {name: (body[name] if name in body else old.get(name)) for name in ("same_as", "button", "name", "nsfw", "note")} + if decision != "same": + values["same_as"] = None + merge = {"merge_with": ",".join(button_ids(body.get("merge_with"))) if "merge_with" in body else old.get("merge_with"), + "merge_name": body.get("merge_name", old.get("merge_name")), + "merge_drop": json.dumps(body["merge_drop"], ensure_ascii=False) if isinstance(body.get("merge_drop"), list) + else old.get("merge_drop")} + if decision == "merge" and not merge["merge_with"]: + return {"ok": False, "error": "a merge needs at least one other list"} + connection.execute( + """insert into reviews (key, decision, same_as, button, name, nsfw, note, how, decided_at, applied_at) + values (?,?,?,?,?,?,?,?,?,?) + on conflict(key) do update set decision=excluded.decision, same_as=excluded.same_as, + button=excluded.button, name=excluded.name, nsfw=excluded.nsfw, note=excluded.note, + how=excluded.how, decided_at=excluded.decided_at""", + (key, decision or None, values["same_as"], values["button"] or None, values["name"] or None, + values["nsfw"] or None, values["note"] or None, how, stamp, old.get("applied_at"))) + connection.execute("update reviews set merge_with = ?, merge_name = ?, merge_drop = ? where key = ?", + (merge["merge_with"] or None, merge["merge_name"] or None, merge["merge_drop"] or None, key)) + self._forget_rows() + return {"ok": True} + + def save_many(self, items: List[Dict[str, Any]]) -> Dict[str, Any]: + saved = 0 + for item in items: + if self.save({"key": item.get("key"), "decision": item.get("decision")}, how="page").get("ok"): + saved += 1 + return {"ok": True, "saved": saved} + + # -- apply ------------------------------------------------------------------------------------------------------- + def _decided(self, connection) -> List[Dict[str, Any]]: + return [dict(row) for row in connection.execute( + """select r.*, t.file from reviews r left join triage t on t.key = r.key + where r.decision is not null and r.decision != 'unsure'""")] + + def _settings(self, row: Dict[str, Any]) -> Dict[str, Any]: + settings: Dict[str, Any] = {} + if row["decision"] == "merge": + settings["hidden"] = True + settings["merged_into"] = L.strip_nsfw_prefix(self._merge_target(row)["key"]) + elif row["decision"] in ("dump", "same"): + settings["hidden"] = True + if row["same_as"]: + settings["same_as"] = L.strip_nsfw_prefix(row["same_as"]) + else: + settings["hidden"] = False + ids = button_ids(row["button"]) + if ids: + settings["slot"] = ids if len(ids) > 1 else ids[0] + if row["nsfw"]: + settings["nsfw"] = row["nsfw"] + return settings + + def _renames(self, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + out = [] + for row in rows: + if row["decision"] == "keep" and row["name"]: + target = self.rename_target(row["key"], row["file"] or "", row["name"]) + if target: + out.append(target) + return out + + def plan(self) -> Dict[str, Any]: + """What Apply would write for every reviewed list (only the settings each decision owns) and rename.""" + with self.connect() as connection: + rows = self._decided(connection) + changes = [{"key": r["key"], "name": L.strip_nsfw_prefix(r["key"]), "decision": r["decision"], + "settings": self._settings(r), "applied_at": r["applied_at"], "decided_at": r["decided_at"]} + for r in rows] + counts: Dict[str, int] = {} + for change in changes: + counts[change["decision"]] = counts.get(change["decision"], 0) + 1 + pending = [c for c in changes if not c["applied_at"] or c["applied_at"] < c["decided_at"]] + renames = self._renames(rows) + merges = self._merges(rows) + return {"counts": counts, "pending": len(pending), "total": len(changes), "changes": pending[:40], + "renames": renames, "merges": merges, "file": os.path.join(LIBRARY_ROOT, "_catalog_user.yaml")} + + def apply(self) -> Dict[str, Any]: + with self.connect() as connection: + rows = self._decided(connection) + planned = self._renames(rows) + renames = [r for r in planned if not r["conflict"]] + overlay = UserOverlay(LIBRARY_ROOT) + overlay.reload_if_changed() + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + manifest = {"made": stamp, "root": LIBRARY_ROOT, "kind": "apply", "renames": [], "moves": [], "merges": []} + done = self._do_renames(renames, overlay, manifest) if renames else [] + merged = self._do_merges(overlay, manifest, stamp) + done = done + merged + written = 0 + with self.connect() as connection: + for row in self._decided(connection): # again: renamed lists now carry their new keys + name = L.strip_nsfw_prefix(row["key"]) + entry = dict(overlay.list(name)) + if row["decision"] == "merge": + entry["hidden"] = True + entry.pop("slot", None) + if row["merged_into"]: + entry["merged_into"] = L.strip_nsfw_prefix(row["merged_into"]) + elif row["decision"] in ("dump", "same"): + entry["hidden"] = True + if row["same_as"]: + entry["same_as"] = L.strip_nsfw_prefix(row["same_as"]) + entry.pop("slot", None) + else: + entry.pop("hidden", None) + entry.pop("same_as", None) + entry.pop("name", None) # the old display-name field: renames replace it + ids = button_ids(row["button"]) + if ids: + entry["slot"] = ids if len(ids) > 1 else ids[0] + else: + entry.pop("slot", None) + if row["nsfw"]: + entry["nsfw"] = row["nsfw"] + if entry: + overlay.data["lists"][name] = entry + else: + overlay.data["lists"].pop(name, None) + written += 1 + overlay.save() + connection.execute("update reviews set applied_at = ? where decision is not null and decision != 'unsure'", + (stamp,)) + manifest_path = self._save_manifest(manifest) if done else "" + self._forget_rows() + if done and self.library is not None: + self.library.refresh(force=True) + return {"ok": True, "written": written, "file": overlay.path, "renamed": len(done) - len(merged), + "merged": len(merged), + "skipped": [r for r in planned if r["conflict"]], "manifest": manifest_path} + + def _do_renames(self, renames: List[Dict[str, Any]], overlay: UserOverlay, manifest: Dict[str, Any]) -> List[Dict[str, Any]]: + """Rename .txt lists in place; write YAML / JSON keys out as their own .txt. Settings, buttons, review follow.""" + buttons = read_buttons(MY_BUTTONS) + buttons_changed = False + done = [] + for item in renames: + old_key, new_key = item["old"], item["new"] + old_name, new_name = L.strip_nsfw_prefix(old_key), L.strip_nsfw_prefix(new_key) + source, target = _abs(item["from"]), _abs(item["path"]) + if item["kind"] == "rename": + os.rename(source, target) + else: + entries = self._entries(old_key) + if not entries: + continue + with open(target, "w", encoding="utf-8") as handle: + for entry in entries: + weight = "" if abs(entry.weight - 1.0) < 1e-9 else f"{entry.weight:g}::" + handle.write(weight + entry.text.replace("\n", " ") + "\n") + old_entry = dict(overlay.list(old_name)) + moved_settings = {k: v for k, v in old_entry.items() if k not in ("hidden", "moved_to", "name")} + if moved_settings: + overlay.data["lists"][new_name] = dict(overlay.list(new_name), **moved_settings) + if item["kind"] == "rename": + overlay.data["lists"].pop(old_name, None) + overlay.data.setdefault("aliases", {})[old_name] = new_name + else: + overlay.data["lists"][old_name] = {"hidden": True, "moved_to": new_name} + for button in buttons: + if old_name in (button.get("lists") or []): + button["lists"] = [new_name if k == old_name else k for k in button["lists"]] + buttons_changed = True + with self.connect() as connection: + connection.execute("update triage set key = ?, file = ? where key = ?", (new_key, item["path"], old_key)) + connection.execute("update judgements set key = ? where key = ?", (new_key, old_key)) + connection.execute("update reviews set key = ?, renamed_from = ? where key = ?", (new_key, old_key, old_key)) + record = dict(item, old_settings=old_entry) + manifest["renames"].append(record) + done.append(record) + if buttons_changed: + write_buttons(MY_BUTTONS, buttons) + return done + + # -- merging ----------------------------------------------------------------------------------------------------- + def merge_preview(self, key: str, partners: List[str]) -> Dict[str, Any]: + """Every entry of this list and its partners, near-duplicates folded, shared ones first; which lists have each.""" + sources = [key] + [k for k in partners if k and k != key] + lists = [[e.text for e in self._entries(k)] for k in sources] + rows = merge_entries(lists) + by_key = {r["key"]: r for r in self.rows()} + return {"sources": [{"key": k, "dim": (by_key.get(k) or {}).get("dimension") or "", "entries": len(lists[i])} + for i, k in enumerate(sources)], + "rows": [row.to_dict() for row in rows[:MERGE_PREVIEW_MAX]], "total": len(rows), + "shared": sum(1 for row in rows if row.lists > 1)} + + def _merge_target(self, row: Dict[str, Any]) -> Dict[str, Any]: + """Where a merge goes: mine//.txt.""" + ids = button_ids(row.get("button")) + slot = ids[0] if ids else "review" + name = row.get("merge_name") or (row.get("key") or "merged").rsplit("/", 1)[-1] + return {"key": merged_key(slot, name), "path": merged_path(LIBRARY_ROOT, slot, name), "slot": slot, + "name": merged_name(name)} + + def _merges(self, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + out = [] + for row in rows: + if row["decision"] != "merge" or not row["merge_with"]: + continue + target = self._merge_target(row) + done = bool(row.get("merged_into")) and row["merged_into"] == target["key"] and os.path.exists(target["path"]) + conflict = "" if done or not os.path.exists(target["path"]) else f"{_rel(target['path'])} already exists" + out.append({"key": row["key"], "with": button_ids(row["merge_with"]), "into": target["key"], + "path": _rel(target["path"]), "done": done, "conflict": conflict, + "drop": len(json.loads(row["merge_drop"] or "[]"))}) + return out + + def _do_merges(self, overlay: UserOverlay, manifest: Dict[str, Any], stamp: str) -> List[Dict[str, Any]]: + """Write each pending merge to mine/, hide its sources (merged_into), put it on its buttons.""" + with self.connect() as connection: + rows = self._decided(connection) + done = [] + for item in self._merges(rows): + if item["done"] or item["conflict"]: + continue + row = next(r for r in rows if r["key"] == item["key"]) + target = self._merge_target(row) + sources = [row["key"]] + item["with"] + drop = {str(t).casefold() for t in json.loads(row["merge_drop"] or "[]")} + merged = merge_entries([[e.text for e in self._entries(k)] for k in sources]) + kept = [r.text for r in merged if r.text.casefold() not in drop] + try: + count = write_merged(target["path"], kept, sources, slot_label=target["slot"]) + except (ValueError, FileExistsError): + continue + new_name = target["key"] + old_settings = {} + for source in sources: + name = L.strip_nsfw_prefix(source) + old_settings[name] = dict(overlay.list(name)) + entry = dict(overlay.list(name), hidden=True, merged_into=new_name) + entry.pop("slot", None) + overlay.data["lists"][name] = entry + settings = {} + ids = button_ids(row["button"]) + if ids: + settings["slot"] = ids if len(ids) > 1 else ids[0] + if row["nsfw"]: + settings["nsfw"] = row["nsfw"] + if settings: + overlay.data["lists"][new_name] = dict(overlay.list(new_name), **settings) + partner_reviews = {} + with self.connect() as connection: + for partner in item["with"]: + old = connection.execute("select * from reviews where key = ?", (partner,)).fetchone() + partner_reviews[partner] = dict(old) if old else None + if not connection.execute("select 1 from triage where key = ?", (partner,)).fetchone(): + continue + connection.execute( + """insert into reviews (key, decision, same_as, how, decided_at, applied_at, merged_into) + values (?, 'same', ?, 'merge', ?, ?, ?) + on conflict(key) do update set decision='same', same_as=excluded.same_as, how='merge', + decided_at=excluded.decided_at, applied_at=excluded.applied_at, merged_into=excluded.merged_into""", + (partner, new_name, stamp, stamp, new_name)) + connection.execute("update reviews set merged_into = ? where key = ?", (new_name, row["key"])) + record = {"key": row["key"], "with": item["with"], "into": new_name, "path": _rel(target["path"]), + "count": count, "old_settings": old_settings, "partner_reviews": partner_reviews} + manifest["merges"].append(record) + done.append(record) + return done + + # -- moving dumped files ------------------------------------------------------------------------------------------- + @staticmethod + def _file_keys(path: str) -> Optional[set]: + """ + Every list key a YAML / JSON file holds, read from the file itself (the triage can be out of date: a key the + review split out by a rename still sits in its file). None = the file could not be read (then it stays). + """ + try: + text = L.read_text_file(path) + data = json.loads(text) if path.lower().endswith(".json") else (yaml.safe_load(text) if yaml else None) + except Exception: + return None + found: set = set() + + def walk(node, parts): + if isinstance(node, dict): + for k, v in node.items(): + walk(v, parts + [str(k)]) + elif parts and (isinstance(node, (list, str, int, float)) and not isinstance(node, bool)): + found.add(L.normalise_key("/".join(parts))) + walk(data, []) + return found + + def dump_plan(self) -> Dict[str, Any]: + with self.connect() as connection: + rows = [dict(r) for r in connection.execute( + """select r.key, t.file from reviews r join triage t on t.key = r.key + where r.decision = 'dump' and (r.moved_to is null or r.moved_to = '')""")] + by_file: Dict[str, List[str]] = {} + for row in rows: + if row["file"]: + by_file.setdefault(_abs(row["file"]), []).append(row["key"]) + moves, stay, missing = [], [], [] + for path, keys in sorted(by_file.items()): + if not os.path.exists(path): + missing.append(_rel(path)) + continue + relative = _rel(path) + target = os.path.join(LIBRARY_ROOT, DUMPED_FOLDER, *relative.split("/")) + if path.lower().endswith(STRUCTURED): + holds = self._file_keys(path) + if not holds or not holds <= set(keys): + stay.append({"file": relative, "dumped": len(keys), "lists": len(holds or ())}) + continue + if os.path.exists(target): + stay.append({"file": relative, "dumped": len(keys), "lists": len(keys), "why": "already in _dumped"}) + continue + moves.append({"file": relative, "to": _rel(target), "keys": keys}) + return {"moves": moves, "stay": stay, "missing": missing, "folder": os.path.join(LIBRARY_ROOT, DUMPED_FOLDER)} + + def dump_move(self) -> Dict[str, Any]: + plan = self.dump_plan() + manifest = {"made": time.strftime("%Y-%m-%d %H:%M:%S"), "root": LIBRARY_ROOT, "kind": "dump_move", + "renames": [], "moves": []} + path = "" + for move in plan["moves"]: + source, target = _abs(move["file"]), _abs(move["to"]) + os.makedirs(os.path.dirname(target), exist_ok=True) + shutil.move(source, target) + manifest["moves"].append(move) + with self.connect() as connection: + connection.executemany("update reviews set moved_to = ? where key = ?", [(move["to"], k) for k in move["keys"]]) + if len(manifest["moves"]) % 200 == 0: + path = self._save_manifest(manifest, path) + if manifest["moves"]: + path = self._save_manifest(manifest, path) + self._forget_rows() + if manifest["moves"] and self.library is not None: + self.library.refresh(force=True) + return {"ok": True, "moved": len(manifest["moves"]), "stayed_hidden": len(plan["stay"]), "manifest": path} + + def _save_manifest(self, manifest: Dict[str, Any], path: str = "") -> str: + os.makedirs(MANIFESTS, exist_ok=True) + if not path: # a new manifest never replaces an older one (two Applies in one second) + stem = os.path.join(MANIFESTS, f"{manifest['kind']}_{time.strftime('%Y%m%d_%H%M%S')}") + path, n = stem + ".json", 1 + while os.path.exists(path): + n += 1 + path = f"{stem}_{n}.json" + with open(path, "w", encoding="utf-8") as handle: + json.dump(manifest, handle, indent=1, ensure_ascii=False) + return path + + +def undo(manifest_path: str) -> None: + """Put back what a manifest records: moved files, renamed lists (files, settings, aliases, buttons, review keys).""" + with open(manifest_path, "r", encoding="utf-8") as handle: + manifest = json.load(handle) + review = Review(load_library=False) + back = 0 + for move in reversed(manifest.get("moves") or []): + source, target = _abs(move["file"]), _abs(move["to"]) + if os.path.exists(target) and not os.path.exists(source): + os.makedirs(os.path.dirname(source), exist_ok=True) + shutil.move(target, source) + back += 1 + with review.connect() as connection: + connection.executemany("update reviews set moved_to = null where key = ?", [(k,) for k in move["keys"]]) + overlay = UserOverlay(LIBRARY_ROOT) + overlay.reload_if_changed() + buttons = read_buttons(MY_BUTTONS) + for item in reversed(manifest.get("renames") or []): + old_key, new_key = item["old"], item["new"] + old_name, new_name = L.strip_nsfw_prefix(old_key), L.strip_nsfw_prefix(new_key) + source, target = _abs(item["from"]), _abs(item["path"]) + if item["kind"] == "rename": + if os.path.exists(target) and not os.path.exists(source): + os.rename(target, source) + elif os.path.exists(target): + os.remove(target) + overlay.data["lists"].pop(new_name, None) + if item.get("old_settings"): + overlay.data["lists"][old_name] = item["old_settings"] + else: + overlay.data["lists"].pop(old_name, None) + (overlay.data.get("aliases") or {}).pop(old_name, None) + for button in buttons: + if new_name in (button.get("lists") or []): + button["lists"] = [old_name if k == new_name else k for k in button["lists"]] + with review.connect() as connection: + connection.execute("update triage set key = ?, file = ? where key = ?", (old_key, item["from"], new_key)) + connection.execute("update judgements set key = ? where key = ?", (old_key, new_key)) + connection.execute("update reviews set key = ?, renamed_from = null where key = ?", (old_key, new_key)) + back += 1 + for item in reversed(manifest.get("merges") or []): + path = _abs(item["path"]) + if os.path.exists(path): + os.remove(path) + overlay.data["lists"].pop(L.strip_nsfw_prefix(item["into"]), None) + for name, settings in (item.get("old_settings") or {}).items(): + if settings: + overlay.data["lists"][name] = settings + else: + overlay.data["lists"].pop(name, None) + with review.connect() as connection: + connection.execute("update reviews set merged_into = null, applied_at = null where key = ?", (item["key"],)) + for partner, old in (item.get("partner_reviews") or {}).items(): + if old is None: + connection.execute("delete from reviews where key = ?", (partner,)) + else: + columns = [c for c in old if c != "key"] + connection.execute(f"update reviews set {', '.join(c + ' = ?' for c in columns)} where key = ?", + [old[c] for c in columns] + [partner]) + back += 1 + overlay.save() + write_buttons(MY_BUTTONS, buttons) + print(f"undone: {back} changes from {manifest_path}") + + +def make_handler(review: Review): + class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + + def _send(self, code: int, body: bytes, kind: str) -> None: + self.send_response(code) + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _json(self, data: Any, code: int = 200) -> None: + self._send(code, json.dumps(data, ensure_ascii=False).encode("utf-8"), "application/json; charset=utf-8") + + def do_GET(self): + url = urlparse(self.path) + q = {k: v[0] for k, v in parse_qs(url.query).items()} + try: + if url.path == "/favicon.ico": + self._send(204, b"", "image/x-icon") + elif url.path in ("/", "/index.html"): + with open(os.path.join(HERE, "review.html"), "rb") as handle: + self._send(200, handle.read(), "text/html; charset=utf-8") + elif url.path == "/api/meta": + self._json(review.meta()) + elif url.path == "/api/queue": + self._json(review.queue(q.get("order", "facet"), q.get("facet", ""), q.get("verdict", ""), + q.get("status", "todo"), q.get("source", ""), q.get("q", ""))) + elif url.path == "/api/list": + found = review.list_view(q.get("key", ""), int(q.get("seed", "0") or 0)) + self._json(found if found else {"error": "not found"}, 200 if found else 404) + elif url.path == "/api/entries": + self._json(review.entries(q.get("key", ""), q.get("q", ""), int(q.get("offset", 0) or 0), + min(500, int(q.get("limit", 200) or 200)))) + elif url.path == "/api/page": + self._json(review.page([k for k in q.get("keys", "").split(",") if k])) + elif url.path == "/api/apply_preview": + self._json(review.plan()) + elif url.path == "/api/dump_preview": + self._json(review.dump_plan()) + elif url.path == "/api/llm_models": + try: + self._json({"models": llm_models(), "default": LLM_DEFAULT_MODEL}) + except Exception as error: + self._json({"models": [], "default": LLM_DEFAULT_MODEL, "error": f"LM Studio: {error}"}) + elif url.path == "/api/merge_preview": + self._json(review.merge_preview(q.get("key", ""), [k for k in q.get("with", "").split(",") if k])) + else: + self._send(404, b"not found", "text/plain") + except Exception as error: + self._json({"error": f"{type(error).__name__}: {error}"}, 500) + + def do_POST(self): + url = urlparse(self.path) + try: + length = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(length) or b"{}") + if url.path == "/api/review": + self._json(review.save(body)) + elif url.path == "/api/review_many": + self._json(review.save_many(body.get("items") or [])) + elif url.path == "/api/button": + self._json(review.create_button(str(body.get("label", "")), str(body.get("section", "")), + str(body.get("key", "")), str(body.get("group", "")))) + elif url.path == "/api/apply": + self._json(review.apply()) + elif url.path == "/api/dump_move": + self._json(review.dump_move()) + elif url.path == "/api/drop": + self._json(review.drop(str(body.get("key", "")), list(body.get("texts") or []), bool(body.get("undrop")))) + elif url.path == "/api/drop_matching": + self._json(review.drop_matching(str(body.get("key", "")), str(body.get("pattern", "")), + bool(body.get("regex")), bool(body.get("drop")))) + elif url.path == "/api/suggest_junk": + self._json(review.suggest_junk(str(body.get("key", "")), str(body.get("model", "")))) + elif url.path == "/api/generate": + self._json(review.generate(body)) + elif url.path == "/api/save_list": + self._json(review.save_list(body)) + else: + self._send(404, b"not found", "text/plain") + except Exception as error: + self._json({"ok": False, "error": f"{type(error).__name__}: {error}"}, 500) + + return Handler + + +def serve(port: int = PORT, open_browser: bool = True) -> None: + review = Review() + server = ThreadingHTTPServer(("127.0.0.1", port), make_handler(review)) + url = f"http://127.0.0.1:{port}/" + print(f"[review] {url} (Ctrl+C to stop) triage: {TRIAGE}") + if open_browser: + threading.Timer(0.8, lambda: webbrowser.open(url)).start() + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + + +if __name__ == "__main__": + if "--undo" in sys.argv: + undo(sys.argv[sys.argv.index("--undo") + 1]) + else: + serve(int(sys.argv[sys.argv.index("--port") + 1]) if "--port" in sys.argv else PORT, + open_browser="--no-browser" not in sys.argv) diff --git a/tools/wildcard_review/wildcard_review.bat b/tools/wildcard_review/wildcard_review.bat new file mode 100644 index 0000000..0be34dc --- /dev/null +++ b/tools/wildcard_review/wildcard_review.bat @@ -0,0 +1,6 @@ +@echo off +rem Wildcard review: page through the wildcard lists, Keep / Dump / Same as. Opens http://127.0.0.1:8766 +cd /d "A:\Comfy25\ComfyUI_windows_portable\ComfyUI\custom_nodes\video_prompter" +A:\Comfy25\ComfyUI_windows_portable\python_embeded\python.exe -X utf8 tools\wildcard_review\server.py %* +pause + diff --git a/web/prompt_builder.js b/web/prompt_builder.js new file mode 100644 index 0000000..e536aec --- /dev/null +++ b/web/prompt_builder.js @@ -0,0 +1,2178 @@ +/** + * prompt_builder.js - video_prompter + * + * The panel on the Wildcard Prompt / Character Composer nodes: a composer where you type text between + * wildcard pills, slot buttons that insert pills, one row of settings per pill, a live preview, + * and a list viewer to fix, subset or exclude entries. A Find box searches every list by name + * (and, through the catalog, by entries); "+ Button" / ☆ save your own buttons to + * builders_user/my_buttons.yaml, shared by all builder nodes. The template bar saves the whole + * composer to builders_user/templates/.json and loads it back on any composer node. + * "Merge lists…" in the list viewer folds a slot's lists into one list of your own + * (/mine//.txt), shared entries first, for pruning by eye. + * "More lists…" under the palette shows every usable list no button offers, grouped by path words. + * + * State lives as JSON in the node's hidden `composer` widget (format: core/builder/compose.py), + * so it saves with the workflow. Everything the panel shows comes from the routes in + * prompt_builder_support.py; the Python side builds the wildcard syntax, so preview and run match. + * + * Author: Eric Hiss (GitHub: EricRollei) + */ + +import { app } from "../../scripts/app.js"; +import { api } from "../../scripts/api.js"; + +const ROUTE = "/eric_prompt_builder"; +const MODES = [ + ["random", "Random", "Any entry of the list."], + ["fixed", "Fixed", "Always the first entry you picked, or your own text."], + ["subset", "Subset", "Random among the entries you picked."], + ["exclude", "Exclude", "Random, never the entries you picked."], +]; +const KEEPS = [ + ["meaning", "Meaning", "The LLM polish may reword it but must keep the meaning."], + ["exact", "Exact", "Kept word for word (names, LoRA triggers)."], + ["loose", "Loose", "Inspiration only; the LLM may drop it."], +]; +const MAX_COUNT = 8; +const MY_GROUP = "My buttons"; +// Seed order of the sections (core/builder/compose.py SECTIONS): new sections are only appended. +const SEED_ORDER = ["subject", "action", "setting", "lighting", "style", "camera", "format", "composition", "text", "prompt"]; +const FALLBACK_ORDER = ["format", "prompt", "subject", "action", "setting", "lighting", "composition", "camera", "style", "text"]; +const SUBJECT_KINDS = [["", "Any type"], ["person", "Person"], ["animal", "Animal"], ["creature", "Creature"], ["object", "Object"], + ["vehicle", "Vehicle"], ["food", "Food"], ["scene", "Scene only"]]; +const PANELS = new Set(); // every open builder panel, so a saved button shows up on all of them + +// ------------------------------------------------------------------ small helpers +const cache = { builders: null, sections: {}, platforms: null, slots: new Map(), templates: null }; + +async function templates(refresh = false) { + if (refresh || !cache.templates) { + cache.templates = getJSON(`${ROUTE}/templates`).then((d) => d.templates || []).catch((e) => { cache.templates = null; throw e; }); + } + return cache.templates; +} + +/** After a template is saved or deleted: every panel's template menu shows the new list. */ +async function reloadTemplates() { + await templates(true); + for (const panel of PANELS) panel.renderTemplates(); +} + +async function getJSON(path) { + const response = await api.fetchApi(path); + if (!response.ok) throw new Error(`${path}: HTTP ${response.status}`); + return response.json(); +} + +async function postJSON(path, body) { + const response = await api.fetchApi(path, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!response.ok) { + let detail = {}; + try { detail = await response.json(); } catch (e) { /* not JSON */ } + const error = new Error(detail.error || `${path}: HTTP ${response.status}`); + error.status = response.status; + error.detail = detail; + throw error; + } + return response.json(); +} + +async function builders() { + if (!cache.builders) { + const data = await getJSON(`${ROUTE}/builders`); + cache.builders = Object.fromEntries(data.builders.map((b) => [b.id, b])); + cache.sections = data.sections || {}; + cache.defaultOrder = data.default_order || FALLBACK_ORDER; + } + return cache.builders; +} + +async function platforms() { + if (!cache.platforms) { + const data = await getJSON(`${ROUTE}/platforms`); + cache.platforms = Object.fromEntries(data.platforms.map((p) => [p.id, p])); + } + return cache.platforms; +} + +async function slotLists(builderId, slotId, nsfw, gender = "", violence = 0) { + const key = `${builderId}|${slotId}|${nsfw}|${gender}|${violence}`; + if (!cache.slots.has(key)) { + cache.slots.set(key, getJSON(`${ROUTE}/slot?builder=${encodeURIComponent(builderId)}&slot=${encodeURIComponent(slotId)}&nsfw=${nsfw}&gender=${gender}&violence=${violence}`) + .then((d) => d.lists) + .catch((e) => { cache.slots.delete(key); throw e; })); + } + return cache.slots.get(key); +} + +/** After a button is saved or deleted: fetch the definitions again and redraw every panel. */ +async function reloadBuilders() { + cache.builders = null; + cache.slots.clear(); + const defs = await builders(); + for (const panel of PANELS) { + if (!defs[panel.builderId]) continue; + panel.def = defs[panel.builderId]; + panel.countsKey = null; + panel.render(); + panel.loadSlotCounts(); + } +} + +/** One line describing a list in a dropdown or the Find results. */ +function listLabel(l) { + return `${l.nsfw ? `${nsfwName(l.nsfw)} · ` : ""}${l.stars ? "★".repeat(l.stars) + " " : ""}${l.display || displayName(l.key)}${l.gender ? (l.gender === "female" ? " · women" : " · men") : ""} (${l.count}${l.dialect === "tags" ? " · tags" : l.length ? ` · ${l.length}` : ""}${l.quality >= 0 ? ` · q${Math.round(l.quality * 100)}` : ""})`; +} + +/** A short pill label from a list key: "pack_v10/black_female_names" -> "Black Female Names". */ +function shortLabel(key) { + return (displayName(key).split(" · ")[0] || key).slice(0, 28); +} + +// NSFW levels, as the node's nsfw dropdown names them (core/wildcards/library.py NSFW_LEVELS). +const NSFW_LEVELS = ["off", "suggestive", "nudity", "explicit"]; +const nsfwName = (rating) => NSFW_LEVELS[rating] || ""; + +function el(tag, attrs = {}, ...children) { + const node = document.createElement(tag); + for (const [k, v] of Object.entries(attrs)) { + if (v === undefined || v === null || v === false) continue; + if (k === "class") node.className = v; + else if (k === "style") node.style.cssText = v; + else if (k.startsWith("on")) node.addEventListener(k.slice(2), v); + else if (k === "text") node.textContent = v; + else node.setAttribute(k, v === true ? "" : v); + } + for (const child of children.flat()) { + if (child === null || child === undefined || child === false) continue; + node.append(child instanceof Node ? child : document.createTextNode(String(child))); + } + return node; +} + +function titleCase(s) { + return s.replace(/[_\-]+/g, " ").replace(/\s+/g, " ").trim().replace(/\b\w/g, (c) => c.toUpperCase()); +} + +/** "pack_v10/sub/cops_wc_hair-color" -> "Hair Color · Pack" */ +function displayName(key) { + const parts = key.split("/"); + const leaf = titleCase(parts[parts.length - 1].replace(/^cops_(bp_)?(male_|female_)?wc[-_]?/, "").replace(/^(cyber|bk|rd|fg3|vc|tr)_/, "")); + if (parts.length === 1) return `${leaf} · top level`; + const pack = parts[0].replace(/_v\d+.*$/, "").replace(/wildcards?/g, "").replace(/_+$/, ""); + return `${leaf} · ${titleCase(pack) || parts[0]}`; +} + +function debounce(fn, ms) { + let timer = null; + return (...args) => { clearTimeout(timer); timer = setTimeout(() => fn(...args), ms); }; +} + +// ------------------------------------------------------------------ styles (once) +function injectStyles() { + if (document.getElementById("epb-styles")) return; + const css = ` +.epb{--epb-accent:#f0b454;--epb-accent-ink:#1b1b1f;--epb-sel:#6ea8fe;--epb-bg:var(--comfy-input-bg,#1e1e22);--epb-fg:var(--input-text,#ece8e1); + --epb-muted:var(--descrip-text,#a8a39b);--epb-line:var(--border-color,#3a3a41);--epb-panel:var(--comfy-menu-bg,#232327); + box-sizing:border-box;width:100%;height:100%;display:flex;flex-direction:column;gap:8px;padding:6px 8px;overflow:auto; + font:12px/1.35 system-ui,-apple-system,"Segoe UI",sans-serif;color:var(--epb-fg)} +.epb *{box-sizing:border-box} +.epb h4{margin:0;font-size:10.5px;font-weight:600;letter-spacing:.07em;text-transform:uppercase;color:var(--epb-muted)} +.epb-sections{display:flex;flex-wrap:wrap;gap:4px;align-items:center} +.epb-chip{padding:2px 7px;border-radius:6px;border:1px dashed var(--epb-line);color:var(--epb-muted);font-size:11px} +.epb-chip.mine{border:1px solid var(--epb-line);color:var(--epb-fg);background:none} +.epb-arrow{color:var(--epb-muted);font-size:11px} +.epb-composer{display:flex;flex-wrap:wrap;gap:4px;align-items:center} +.epb-gap{height:26px;min-width:2.2ch;padding:0 5px;border-radius:5px;border:1px dashed var(--epb-line);background:transparent;color:var(--epb-fg);font:inherit;font-size:12.5px} +.epb-gap.filled{border-style:solid;border-color:transparent;background:rgba(255,255,255,.04)} +.epb-gap.caret{border:1px solid var(--epb-accent);box-shadow:0 0 0 2px rgba(240,180,84,.18)} +.epb-gap:focus{outline:none;border-color:var(--epb-accent)} +.epb-pill{display:inline-flex;align-items:center;height:26px;border-radius:13px;border:1px solid #b98a3a;background:#3a3120;color:#f7d9a4;font-size:12px} +.epb-pill.sel{border-color:var(--epb-sel);background:#1f3350;color:#dce9ff} +.epb-pill[draggable=true]{cursor:grab} +.epb-pill.dragging{opacity:.35} +.epb-pill.drop-before{box-shadow:-3px 0 0 0 var(--epb-sel)} +.epb-gap.drop-here{border:1px solid var(--epb-sel)!important;background:rgba(110,168,254,.15)!important;min-width:4ch} +.epb-pill.missing{border-color:#b9565d;background:#3a2224;color:#f3c1c4} +.epb-pill button{background:none;border:0;color:inherit;font:inherit;cursor:pointer;height:100%} +.epb-pill .lab{padding:0 3px 0 9px;font-weight:600} +.epb-pill .badge{padding:0 4px 0 0;font-family:ui-monospace,Consolas,monospace;font-size:10.5px;opacity:.85} +.epb-pill .x{padding:0 7px 0 3px;opacity:.7} +.epb-groups{display:flex;flex-direction:column;gap:4px} +.epb-group{display:flex;flex-direction:column;gap:2px} +.epb-group-name{font-size:10px;letter-spacing:.06em;text-transform:uppercase;color:var(--epb-muted);opacity:.8} +.epb-seg.small button{height:18px;padding:0 5px;font-size:10.5px} +.epb-palette{display:grid;grid-template-columns:repeat(auto-fill,minmax(88px,1fr));gap:4px} +.epb-slot{height:26px;padding:0 7px;border-radius:6px;border:1px solid var(--epb-line);background:var(--epb-panel);color:var(--epb-fg);font:inherit;font-size:11.5px;text-align:left;cursor:pointer;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.epb-slot:hover{border-color:var(--epb-accent)} +.epb-rows{display:flex;flex-direction:column;gap:3px} +.epb-row{display:flex;align-items:center;gap:6px;padding:3px 6px;border-radius:6px;border:1px solid var(--epb-line);background:rgba(255,255,255,.02)} +.epb-row.sel{border-color:#4b76b3;background:rgba(110,168,254,.08)} +.epb-row .who{width:128px;flex-shrink:0;background:none;border:0;padding:0;color:inherit;text-align:left;cursor:pointer;font:inherit} +.epb-row .who b{display:block;font-size:12px} +.epb-row .who small{display:block;color:var(--epb-muted);font-size:10px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-family:ui-monospace,Consolas,monospace} +.epb-seg{display:flex;padding:1px;border-radius:6px;border:1px solid var(--epb-line);background:var(--epb-bg)} +.epb-seg button{height:22px;padding:0 6px;border:0;border-radius:4px;background:none;color:var(--epb-muted);font:inherit;font-size:11px;cursor:pointer} +.epb-seg button.on{background:var(--epb-accent);color:var(--epb-accent-ink);font-weight:600} +.epb-count{display:flex;align-items:center;border:1px solid var(--epb-line);border-radius:6px;background:var(--epb-bg)} +.epb-count button{width:20px;height:22px;border:0;background:none;color:var(--epb-fg);cursor:pointer} +.epb-count span{width:18px;text-align:center;font-family:ui-monospace,Consolas,monospace;font-size:11px} +.epb-now{margin-left:auto;min-width:0;flex:1;text-align:right;color:var(--epb-muted);white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-size:11.5px} +.epb-preview{padding:6px 8px;border-radius:7px;border:1px solid #36513a;background:#1f2a1f;color:#eef6ee} +.epb-preview .bar{display:flex;align-items:center;gap:8px;margin-bottom:3px} +.epb-preview .bar h4{color:#bfe0bf} +.epb-preview .meta{color:#a9c7a9;font-family:ui-monospace,Consolas,monospace;font-size:10.5px} +.epb-preview p{margin:0;font-size:13px;line-height:1.45} +.epb-preview .warn{margin-top:3px;color:#f0b454;font-size:11px} +.epb-btn{height:24px;padding:0 9px;border-radius:6px;border:1px solid var(--epb-line);background:var(--epb-panel);color:var(--epb-fg);font:inherit;font-size:11.5px;cursor:pointer} +.epb-btn.primary{border-color:var(--epb-accent);color:#f7d9a4;background:#3a3120} +.epb-btn.green{border-color:#4d6f50;background:#2c3d2c;color:#e3f2e3;margin-left:auto} +.epb-empty{padding:8px;border:1px dashed var(--epb-line);border-radius:6px;color:var(--epb-muted)} +.epb-tabs{display:flex;gap:4px} +.epb-chip{cursor:default;background:none;font:inherit} +button.epb-chip{cursor:pointer} +.epb-chip.active{background:var(--epb-accent);color:var(--epb-accent-ink);border-color:var(--epb-accent);font-weight:600} +.epb-chip.filled:not(.active){border-color:#b98a3a;color:#f7d9a4;background:rgba(240,180,84,.08)} +.epb-frame{display:flex;flex-direction:column;gap:4px;padding:6px;background:var(--epb-bg);border:1px solid var(--epb-line);border-radius:8px} +.epb-lane{border:1px solid transparent;border-radius:6px;padding:3px 4px} +.epb-lane.active{border-color:#b98a3a;background:rgba(240,180,84,.05)} +.epb-lane .head{display:flex;align-items:center;gap:6px;min-height:20px} +.epb-lane .head .name{background:none;border:0;padding:0;color:var(--epb-muted);font:inherit;font-size:10.5px;font-weight:600;letter-spacing:.07em;text-transform:uppercase;cursor:pointer} +.epb-lane.active .head .name{color:var(--epb-accent)} +.epb-lane .head .hint{color:var(--epb-muted);font-size:10.5px;opacity:.75;flex:1;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.epb-toggle{height:18px;padding:0 6px;border-radius:9px;border:1px solid var(--epb-line);background:none;color:var(--epb-muted);font:inherit;font-size:10.5px;cursor:pointer;display:inline-flex;align-items:center;gap:3px} +.epb-toggle.on{border-color:var(--epb-sel);color:#dce9ff;background:#1f3350} +.epb-toggle.fill.on{border-color:#7fb77e;color:#e3f2e3;background:#2c3d2c} +.epb-lane .flow{display:flex;flex-wrap:wrap;gap:4px;align-items:center;min-height:30px} +.epb-lane .llm{color:#a9c7a9;font-size:11px;font-style:italic;padding:0 4px} +.epb-rowshead{display:flex;align-items:baseline;gap:8px} +.epb-dialog{position:fixed;z-index:10000;width:460px;max-height:80vh;display:flex;flex-direction:column;background:var(--comfy-menu-bg,#202024); + color:var(--input-text,#ece8e1);border:1px solid var(--border-color,#3a3a41);border-radius:10px;box-shadow:0 16px 48px rgba(0,0,0,.55); + font:12.5px/1.4 system-ui,-apple-system,"Segoe UI",sans-serif} +.epb-dialog header{display:flex;align-items:baseline;gap:8px;padding:10px 12px 6px;cursor:move} +.epb-dialog header b{font-size:15px} +.epb-dialog header span{color:var(--descrip-text,#a8a39b);font-size:11.5px;flex:1} +.epb-dialog .close{background:none;border:0;color:inherit;font-size:16px;cursor:pointer} +.epb-dialog .body{padding:0 12px;display:flex;flex-direction:column;gap:6px} +.epb-dialog select,.epb-dialog input{width:100%;height:28px;padding:0 8px;border-radius:6px;border:1px solid var(--border-color,#44444c);background:var(--comfy-input-bg,#26262b);color:inherit;font:inherit} +.epb-dialog label{display:flex;flex-direction:column;gap:3px;color:var(--descrip-text,#a8a39b);font-size:11px} +.epb-dialog .two{display:flex;gap:8px}.epb-dialog .two label{flex:1} +.epb-dialog .info{display:flex;justify-content:space-between;color:var(--descrip-text,#a8a39b);font-size:11px;padding:4px 12px} +.epb-dialog .entries{flex:1;overflow:auto;padding:0 12px 8px;display:flex;flex-direction:column;gap:3px;min-height:120px} +.epb-entry{display:flex;align-items:center;gap:8px;min-height:28px;padding:4px 8px;border-radius:6px;border:1px solid transparent;background:rgba(255,255,255,.03);color:inherit;font:inherit;text-align:left;cursor:pointer} +.epb-entry .box{flex-shrink:0;width:16px;height:16px;border-radius:4px;border:1px solid #5a5a63;display:flex;align-items:center;justify-content:center;font-size:11px;font-weight:700} +.epb-entry.on{border-color:#b98a3a;background:#3a3120}.epb-entry.on .box{background:#f0b454;border-color:#f0b454;color:#1b1b1f} +.epb-entry.off{border-color:#b9565d;background:#3a2224}.epb-entry.off .box{background:#e0747b;border-color:#e0747b;color:#1b1b1f} +.epb-dialog{--epb-accent:#f0b454;--epb-accent-ink:#1b1b1f;--epb-sel:#6ea8fe;--epb-fg:var(--input-text,#ece8e1); + --epb-muted:var(--descrip-text,#a8a39b);--epb-line:var(--border-color,#3a3a41)} +.epb-entry-row{display:flex;gap:4px;align-items:stretch} +.epb-entry-row .epb-entry{flex:1;min-width:0} +.epb-entry-row.dropped .txt{text-decoration:line-through;opacity:.55} +.epb-drop{flex-shrink:0;width:56px;border-radius:6px;border:1px solid transparent;background:none;color:var(--descrip-text,#a8a39b);font:inherit;font-size:11px;cursor:pointer;opacity:.6} +.epb-entry-row:hover .epb-drop{opacity:1;border-color:var(--border-color,#44444c)} +.epb-stars{display:flex;align-items:center;gap:2px} +.epb-nsfw{margin-left:8px;padding:1px 6px;border-radius:5px;background:#5a2227;color:#f3c1c4;font-size:10.5px;font-weight:600} +.epb-nsfw select{margin-left:4px;background:#3a1a1d;color:#f3c1c4;border:1px solid #6a3036;border-radius:4px;font-size:10.5px} +.epb-rate{flex-shrink:0;max-width:96px;background:none;color:#e0a4a8;border:1px solid #5a2227;border-radius:5px;font-size:10.5px} +.epb-entry-row.above .epb-entry{opacity:.45} +.epb-entry-row.above .epb-entry .txt{text-decoration:line-through dotted} +.epb-more{padding:6px;text-align:center;color:var(--descrip-text,#a8a39b);font-size:11px} +.epb-stars button{background:none;border:0;padding:0 2px;font-size:16px;line-height:1;color:#5a5a63;cursor:pointer} +.epb-stars button.on{color:#f0b454} +.epb-stars .epb-toggle{margin-left:8px;font-size:11px} +.epb-stars .q{margin-left:auto;color:var(--descrip-text,#a8a39b);font-size:11px} +.epb-dialog footer{display:flex;flex-wrap:wrap;gap:6px;padding:10px 12px;border-top:1px solid var(--border-color,#3a3a41)} +.epb-tpl{display:flex;flex-wrap:wrap;gap:4px;align-items:center} +.epb-tpl select,.epb-tpl input{height:24px;padding:0 6px;border-radius:6px;border:1px solid var(--epb-line);background:var(--epb-bg);color:var(--epb-fg);font:inherit;font-size:11.5px} +.epb-tpl select{max-width:260px} +.epb-tpl input{width:200px} +.epb-tpl input:focus,.epb-tpl select:focus{outline:none;border-color:var(--epb-accent)} +.epb-tpl .msg{color:var(--epb-muted);font-size:11px;flex:1;min-width:120px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.epb-tpl .msg.warn{color:#f0b454} +.epb-tpl .epb-btn{height:24px} +.epb-dialog.epb-merge{width:660px;max-height:88vh} +.epb-merge .srcs{max-height:130px;overflow:auto;display:flex;flex-direction:column;gap:1px;padding:4px;border:1px solid var(--border-color,#3a3a41);border-radius:6px} +.epb-merge .srcs label{flex-direction:row;align-items:center;gap:6px;color:inherit;font-size:11.5px;cursor:pointer} +.epb-merge .srcs input{width:auto;height:auto} +.epb-merge .srcs small{color:var(--descrip-text,#a8a39b)} +.epb-merge .bar{display:flex;flex-wrap:wrap;gap:6px;align-items:center;font-size:11px;color:var(--descrip-text,#a8a39b)} +.epb-merge .bar input[type=search]{flex:1;min-width:140px} +.epb-merge .stats{font-size:11.5px;color:var(--descrip-text,#a8a39b)} +.epb-merge .stats b{color:var(--input-text,#ece8e1)} +.epb-merge .mrows{flex:1;overflow:auto;padding:0 12px 8px;display:flex;flex-direction:column;gap:2px;min-height:160px;max-height:46vh} +.epb-mrow{display:flex;align-items:center;gap:8px;min-height:24px;padding:2px 8px;border-radius:5px;border:1px solid transparent;background:rgba(255,255,255,.03);color:inherit;font:inherit;font-size:12px;text-align:left;cursor:pointer} +.epb-mrow .box{flex-shrink:0;width:15px;height:15px;border-radius:4px;border:1px solid #5a5a63;display:flex;align-items:center;justify-content:center;font-size:10px;font-weight:700;background:#f0b454;border-color:#f0b454;color:#1b1b1f} +.epb-mrow.off{opacity:.45}.epb-mrow.off .box{background:none;border-color:#5a5a63;color:transparent}.epb-mrow.off .txt{text-decoration:line-through} +.epb-mrow .txt{flex:1;min-width:0} +.epb-mrow .n{flex-shrink:0;font-family:ui-monospace,Consolas,monospace;font-size:10.5px;color:var(--descrip-text,#a8a39b)} +.epb-mrow .n.core{color:#f7d9a4} +.epb-merge footer input[type=text]{width:180px;height:26px} +.epb-merge footer label{flex-direction:row;align-items:center;gap:5px} +.epb-merge footer label input{width:auto;height:auto} +.epb-slot.more{border-style:dashed;color:var(--epb-muted);text-align:center} +.epb-mine.arr{cursor:grab}.epb-mine.arr .epb-slot{cursor:grab;border-color:var(--epb-line)} +.epb-mine.arr .edit{width:auto;padding:0 5px;font-size:10px;border-color:var(--epb-line)} +.epb-mine.arr.is-hidden{opacity:.45} +.epb-mine.drop-before{box-shadow:-3px 0 0 0 var(--epb-sel)} +.epb-group-name.drop-on,.epb-chip.drop-on{outline:1px dashed var(--epb-sel);outline-offset:2px} +.epb-kind{height:18px;border-radius:9px;border:1px solid var(--epb-line);background:var(--epb-bg);color:var(--epb-fg);font:inherit;font-size:10.5px;padding:0 4px} +.epb-more-groups{display:flex;flex-wrap:wrap;gap:4px;max-height:150px;overflow:auto;padding:2px} +.epb-more-groups button{height:22px;padding:0 8px;border-radius:11px;border:1px solid var(--border-color,#44444c);background:none;color:inherit;font:inherit;font-size:11px;cursor:pointer} +.epb-more-groups button.on{background:#f0b454;border-color:#f0b454;color:#1b1b1f;font-weight:600} +.epb-more-groups button small{opacity:.7;margin-left:4px} +.epb-more-list .row{display:flex;gap:4px;align-items:stretch} +.epb-more-list .add{flex:1;min-width:0;display:flex;gap:8px;align-items:baseline;padding:3px 8px;border-radius:5px;border:1px solid transparent;background:rgba(255,255,255,.03);color:inherit;font:inherit;font-size:12px;text-align:left;cursor:pointer} +.epb-more-list .add:hover{border-color:#b98a3a} +.epb-more-list .add b{font-weight:600;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.epb-more-list .add small{color:var(--descrip-text,#a8a39b);font-family:ui-monospace,Consolas,monospace;font-size:10.5px;white-space:nowrap} +.epb-more-list .keep{flex-shrink:0;border-radius:5px;border:1px solid var(--border-color,#44444c);background:none;color:var(--descrip-text,#a8a39b);font:inherit;font-size:11px;cursor:pointer;padding:0 7px} +.epb-more-list .tag{padding:0 5px;border-radius:4px;background:#5a2227;color:#f3c1c4;font-size:10px;font-weight:600} +.epb-rowshead .epb-find{margin-left:auto;width:220px;height:24px;padding:0 8px;border-radius:6px;border:1px solid var(--epb-line);background:var(--epb-bg);color:var(--epb-fg);font:inherit;font-size:11.5px} +.epb-rowshead .epb-find:focus{outline:none;border-color:var(--epb-accent)} +.epb-rowshead{align-items:center} +.epb-found{display:flex;flex-direction:column;gap:2px;padding:4px;border:1px solid var(--epb-line);border-radius:6px;background:var(--epb-bg)} +.epb-found .row{display:flex;gap:4px;align-items:stretch} +.epb-found-list{display:flex;flex-direction:column;gap:2px;max-height:380px;overflow-y:auto;padding-right:2px} +.epb-found .add{flex:1;min-width:0;display:flex;gap:8px;align-items:baseline;padding:3px 7px;border-radius:5px;border:1px solid transparent;background:rgba(255,255,255,.03);color:var(--epb-fg);font:inherit;font-size:11.5px;text-align:left;cursor:pointer} +.epb-found .add:hover{border-color:var(--epb-accent)} +.epb-found .add b{font-weight:600;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.epb-found .add small{color:var(--epb-muted);font-size:10.5px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-family:ui-monospace,Consolas,monospace} +.epb-found .keep{flex-shrink:0;border-radius:5px;border:1px solid var(--epb-line);background:none;color:var(--epb-muted);font:inherit;font-size:11px;cursor:pointer;padding:0 7px} +.epb-found .keep:hover{color:#f7d9a4;border-color:#b98a3a} +.epb-found .foot{display:flex;gap:6px;align-items:center;padding:3px 2px 0;color:var(--epb-muted);font-size:11px} +.epb-found .tag{padding:0 5px;border-radius:4px;background:#5a2227;color:#f3c1c4;font-size:10px;font-weight:600} +.epb-mine{display:flex;min-width:0} +.epb-mine .epb-slot{flex:1;min-width:0;border-top-right-radius:0;border-bottom-right-radius:0;border-color:#8a6a33} +.epb-mine .edit{width:22px;flex-shrink:0;border:1px solid #8a6a33;border-left:0;border-radius:0 6px 6px 0;background:var(--epb-panel);color:var(--epb-muted);cursor:pointer;font-size:11px} +.epb-mine .edit:hover{color:#f7d9a4} +.epb-mine.empty .epb-slot{opacity:.55;border-style:dashed} +.epb-dialog .kinds{display:flex;gap:6px} +.epb-dialog .keys{display:flex;flex-direction:column;gap:3px;max-height:160px;overflow:auto} +.epb-dialog .keys div{display:flex;gap:6px;align-items:center;padding:3px 7px;border-radius:5px;background:rgba(255,255,255,.04);font-family:ui-monospace,Consolas,monospace;font-size:11px} +.epb-dialog .keys div span{flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +.epb-dialog .keys button{background:none;border:0;color:inherit;cursor:pointer;opacity:.7} +.epb-dialog .matches{color:var(--descrip-text,#a8a39b);font-size:11px;padding:2px 0 6px;max-height:120px;overflow:auto} +`; + document.head.append(el("style", { id: "epb-styles", text: css })); +} + +// ------------------------------------------------------------------ state helpers +function emptyState() { + return { v: 1, parts: {}, tokens: {}, next: 1, locks: {}, fill: {} }; +} + +const baseSection = (section) => section.replace(/_\d+$/, ""); + +function parseState(raw) { + if (!raw || !String(raw).trim()) return null; + try { + const s = JSON.parse(raw); + if (s && typeof s === "object" && s.parts && s.tokens) { + s.next = s.next || 1 + Math.max(0, ...Object.keys(s.tokens).map(Number).filter(Number.isFinite)); + s.locks = s.locks || {}; + s.fill = s.fill || {}; + return s; + } + } catch (e) { /* not JSON: shown as one text gap */ } + return { v: 1, parts: { subject: [{ t: "text", v: String(raw) }] }, tokens: {}, next: 1 }; +} + +/** Keep every part as text, tok, text, tok, ..., text (merges neighbouring text, drops dead pills). */ +function normalise(state, sections) { + for (const section of sections) { + const segs = state.parts[section] || []; + const out = []; + for (const seg of segs) { + if (seg.t === "tok" && !state.tokens[String(seg.id)]) continue; + const last = out[out.length - 1]; + if (seg.t !== "tok") { + const v = seg.v || ""; + if (last && last.t !== "tok") last.v = [last.v, v].filter((x) => x.trim()).join(" "); + else out.push({ t: "text", v }); + } else { + if (!last || last.t === "tok") out.push({ t: "text", v: "" }); + out.push({ t: "tok", id: seg.id }); + } + } + if (!out.length || out[out.length - 1].t === "tok") out.push({ t: "text", v: "" }); + state.parts[section] = out; + } + const used = new Set(Object.values(state.parts).flat().filter((s) => s.t === "tok").map((s) => String(s.id))); + for (const id of Object.keys(state.tokens)) if (!used.has(id)) delete state.tokens[id]; + return state; +} + +// ------------------------------------------------------------------ the panel +class BuilderPanel { + constructor(node, builderId) { + this.node = node; + this.builderId = builderId; + this.def = null; + this.state = emptyState(); + this.part = null; // section shown in the composer + this.caret = null; // {section, index} of the text gap that receives new pills + this.sel = null; // selected token id + this.preview = { text: "", picks: {}, warnings: [] }; + this.dialog = null; + this.root = el("div", { class: "epb" }); + this.root.addEventListener("keydown", (e) => e.stopPropagation()); + this.root.addEventListener("wheel", (e) => { if (this.root.scrollHeight > this.root.clientHeight) e.stopPropagation(); }, { passive: true }); + this.schedulePreview = debounce(() => this.refreshPreview(), 250); + this.find = { q: "", lists: [], busy: false, error: "" }; + this.runFind = debounce(() => this.doFind(), 300); + this.editor = null; + this.merge = null; + this.more = null; + this.arrange = false; // Arrange mode: drag / hide buttons + this.dragSlot = null; // a button being dragged (Arrange) + this.dragSection = null; // a section chip being dragged + this.tpl = { mode: "", name: "", msg: "", warn: false }; // mode: '' | 'save' | 'replace' | 'delete' + this.undo = null; // {state, platform} before the last template load + this.lostKeys = new Set(); // list keys a loaded template names that this library lacks + PANELS.add(this); + } + + widget(name) { return this.node.widgets?.find((w) => w.name === name); } + /** The node's nsfw level. When the nsfw input is wired, the value of the widget it is wired from. */ + get nsfw() { + const own = this.widget("nsfw")?.value || "off"; + const input = this.node.inputs?.find((i) => i.name === "nsfw" || i.widget?.name === "nsfw"); + if (input?.link == null) return own; + try { + const graph = this.node.graph || app.graph; + const link = graph?.links?.get ? graph.links.get(input.link) : graph?.links?.[input.link]; + const source = link ? graph.getNodeById(link.origin_id) : null; + const value = source?.widgets?.find((w) => NSFW_LEVELS.includes(w.value))?.value; + return value || own; + } catch (e) { + return own; + } + } + /** The node's violence_filter level: 0 off .. 3 (index in the widget's options, as the backend numbers them). */ + get violence() { + const w = this.widget("violence_filter"); + const values = w?.options?.values || []; + return Math.max(0, values.indexOf(w?.value)); + } + /** This builder's sections, plus any Subject 2, 3 ... the user added. */ + get sections() { + const parts = this.def?.parts || [{ section: "subject" }]; + const all = parts.map((p) => p.section); + const own = parts.filter((p) => !p.optional || (this.state?.parts || {})[p.section]).map((p) => p.section); + const extra = Object.keys(this.state?.parts || {}).filter((k) => !all.includes(k) && all.includes(baseSection(k))); + return [...own, ...extra]; + } + + /** Optional sections (Text) this node could add with "+ Section". */ + get addableSections() { + return (this.def?.parts || []).filter((p) => p.optional && !(this.state?.parts || {})[p.section]).map((p) => p.section); + } + + /** The node's order: its own (dragged) order first, then the platform's. */ + get sectionOrder() { + const platform = cache.platforms?.[this.widget("platform")?.value]?.order || cache.defaultOrder || FALLBACK_ORDER; + const own = (this.state?.order || []).filter((s) => SEED_ORDER.includes(s)); + return [...own, ...platform.filter((s) => !own.includes(s))]; + } + + kindOf(section) { + return (this.state.kinds || {})[section] || ""; + } + + setKind(section, kind) { + this.state.kinds = this.state.kinds || {}; + if (kind) this.state.kinds[section] = kind; + else delete this.state.kinds[section]; + if (kind && kind !== "person") this.setGender(section, ""); // Woman / Man only for people + this.save(); + this.render(); + } + + /** Drag a section chip onto another: this node gets its own order (saved with it and its templates). */ + moveSection(from, to) { + const order = this.sectionOrder.filter((s) => s !== from); + order.splice(order.indexOf(to), 0, from); + this.state.order = order; + this.save(); + this.render(); + } + + addSection(section) { + this.state.parts[section] = [{ t: "text", v: "" }]; + this.save(); + this.setPart(section, true); + } + + partDef(section) { + return this.def?.parts.find((p) => p.section === baseSection(section)) || { section, label: section, slots: [section], hint: "" }; + } + + sectionLabel(section) { + const m = section.match(/^(.+)_(\d+)$/); + const base = this.partDef(section).label || cache.sections[baseSection(section)] || section; + return m ? `${base} ${m[2]}` : base; + } + + /** Sections in the platform's order; Subject 2, 3 ... right after Subject. */ + orderedSections() { + const order = this.sectionOrder; + const rank = (s) => { + const i = order.indexOf(baseSection(s)); + const n = (s.match(/_(\d+)$/) || [0, 1])[1]; + return (i < 0 ? 99 : i) * 100 + Number(n); + }; + return [...this.sections].sort((a, b) => rank(a) - rank(b)); + } + + setPart(section, focusEnd = false) { + this.part = section; + const segs = this.state.parts[section] || []; + this.caret = { section, index: Math.max(0, segs.length - 1) }; + this.render(); + if (focusEnd) this.focusGap(section, this.caret.index); + } + + addSubject() { + let n = 2; + while (this.state.parts[`subject_${n}`]) n++; + this.state.parts[`subject_${n}`] = [{ t: "text", v: "" }]; + this.save(); + this.setPart(`subject_${n}`, true); + } + + removeSection(section) { + delete this.state.parts[section]; + delete this.state.locks[section]; + delete this.state.fill[section]; + if (this.state.kinds) delete this.state.kinds[section]; + if (this.part === section) this.part = "subject"; + this.caret = null; + this.save(); + this.render(); + } + + toggleLock(section) { + if (section in this.state.locks) { + delete this.state.locks[section]; + } else { + // Hold the seed the section last rolled with (same rule as section_seed_offset in core/builder/compose.py). + let seed = this.preview.seeds?.[section]; + if (seed === undefined) { + const canonical = SEED_ORDER; + const n = Number((section.match(/_(\d+)$/) || [0, 1])[1]); + seed = Number(this.widget("seed")?.value || 0) + Math.max(0, canonical.indexOf(baseSection(section))) + 100 * (n - 1); + } + this.state.locks[section] = seed; + } + this.save(); + this.render(); + } + + genderOf(section) { + return (this.state.gender || {})[section] || ""; + } + + sectionOfToken(id) { + for (const section of this.sections) { + if ((this.state.parts[section] || []).some((g) => g.t === "tok" && String(g.id) === String(id))) return section; + } + return this.part; + } + + /** Woman / Man / Any for one section: lists named for the other are left out, matching ones come first. + * Pills still on Random move to the best list for the new choice; pills with picks keep their list. */ + async setGender(section, gender) { + this.state.gender = this.state.gender || {}; + if (gender) this.state.gender[section] = gender; + else delete this.state.gender[section]; + const pills = (this.state.parts[section] || []).filter((g) => g.t === "tok").map((g) => this.state.tokens[String(g.id)]); + for (const token of pills) { + if (!token || !token.slot || token.mode !== "random" || token.picks.length) continue; + try { + const lists = await slotLists(this.builderId, token.slot, this.nsfw, gender, this.violence); + if (lists[0]) token.key = lists[0].key; + } catch (e) { /* keep its list */ } + } + this.save(); + this.render(); + } + + /** Lists behind each slot button (buttons with none are hidden). */ + async loadSlotCounts() { + const key = `${this.builderId}|${this.nsfw}|${this.violence}`; + if (this.countsKey === key) return; + this.countsKey = key; + try { + const data = await getJSON(`${ROUTE}/slot_counts?builder=${encodeURIComponent(this.builderId)}&nsfw=${this.nsfw}&violence=${this.violence}`); + if (this.countsKey === key) { + this.slotCounts = data.counts || null; + this.nsfwOnly = new Set(data.nsfw_only || []); // your buttons whose lists are all rated above off: hidden at nsfw off + this.render(); + } + } catch (e) { this.slotCounts = null; } + } + + toggleFill(section) { + if (this.state.fill[section]) delete this.state.fill[section]; + else this.state.fill[section] = true; + this.save(); + this.render(); + } + + async init() { + injectStyles(); + try { + this.def = (await builders())[this.builderId]; + } catch (e) { + this.root.replaceChildren(el("div", { class: "epb-empty" }, `Builder routes unavailable: ${e.message}. Restart ComfyUI after installing.`)); + return; + } + if (!this.def) { + this.root.replaceChildren(el("div", { class: "epb-empty" }, `No builder definition "${this.builderId}" (builders/*.yaml).`)); + return; + } + this.part = this.sections[0]; + for (const name of ["platform", "nsfw", "violence_filter", "seed"]) { + const w = this.widget(name); + if (!w) continue; + const original = w.callback; + w.callback = (...args) => { + const r = original?.apply(w, args); + if (name === "nsfw" || name === "violence_filter") { cache.slots.clear(); this.countsKey = null; this.loadSlotCounts(); } + if (name === "nsfw") { this.dialog?.reload?.(); this.prompts?.reload?.(); } + this.render(); + this.schedulePreview(); + return r; + }; + } + await this.loadFromWidget(); + this.loadSlotCounts(); + } + + /** Read the composer widget (after creation or after a workflow loads). */ + async loadFromWidget() { + if (!this.def) return; + const raw = this.widget("composer")?.value; + let state = parseState(raw); + if (!state) { + try { + state = await getJSON(`${ROUTE}/starter?builder=${encodeURIComponent(this.builderId)}&nsfw=${this.nsfw}&violence=${this.violence}`); + } catch (e) { + state = emptyState(); + } + } + this.state = state; + normalise(this.state, this.sections); + if (!this.state.tokens[String(this.sel)]) this.sel = null; + this.save(false); + this.render(); + this.refreshPreview(); + } + + save(preview = true) { + normalise(this.state, this.sections); + const w = this.widget("composer"); + if (w) w.value = JSON.stringify(this.state); + this.node.setDirtyCanvas?.(true, true); + if (preview) this.schedulePreview(); + } + + // -------------------------------------------------- edits + /** Put a new pill at the amber gap. Returns {id, section, index of the gap after it}. */ + insertToken(fields) { + const section = this.caret && this.caret.section === this.part ? this.caret.section : this.part; + const segs = this.state.parts[section] || (this.state.parts[section] = [{ t: "text", v: "" }]); + let at = this.caret && this.caret.section === section ? this.caret.index : segs.length - 1; + if (!segs[at] || segs[at].t === "tok") at = segs.length - 1; + const id = this.state.next++; + this.state.tokens[String(id)] = { slot: "", key: "", mode: "random", picks: [], count: 1, keep: "meaning", ...fields }; + segs.splice(at + 1, 0, { t: "tok", id }, { t: "text", v: "" }); + this.caret = { section, index: at + 2 }; + this.sel = id; + return { id, section, at }; + } + + /** A pill straight from a Find result: that one list, no slot button behind it. */ + addList(info) { + const { section, at } = this.insertToken({ key: info.key, label: info.label || shortLabel(info.key) }); + this.save(); + this.render(); + this.focusGap(section, at + 2); + } + + async addSlot(slot) { + const { id, section, at } = this.insertToken({ slot: slot.id }); + this.save(false); + this.render(); + try { + const lists = await slotLists(this.builderId, slot.id, this.nsfw, this.genderOf(section), this.violence); + if (this.state.tokens[String(id)]) this.state.tokens[String(id)].key = lists[0]?.key || ""; + } catch (e) { console.warn("[Wildcard Composer]", e); } + this.save(); + this.render(); + this.focusGap(section, at + 2); + } + + removeToken(section, index) { + const segs = this.state.parts[section]; + const token = segs[index]; + const before = segs[index - 1]?.v || ""; + const after = segs[index + 1]?.v || ""; + segs.splice(index - 1, 3, { t: "text", v: [before, after].filter((x) => x.trim()).join(" ") }); + delete this.state.tokens[String(token.id)]; + if (this.sel === token.id) this.sel = null; + this.caret = { section, index: index - 1 }; + this.save(); + this.render(); + if (this.dialog && this.dialog.tokenId === token.id) this.closeDialog(); + } + + updateToken(id, patch) { + const token = this.state.tokens[String(id)]; + if (!token) return; + Object.assign(token, patch); + this.save(); + this.render(); + this.dialog?.refresh(); + } + + focusGap(section, index) { + requestAnimationFrame(() => { + const input = this.root.querySelector(`input[data-section="${section}"][data-index="${index}"]`); + input?.focus(); + }); + } + + // -------------------------------------------------- preview + async refreshPreview() { + if (!this.def) return; + try { + this.preview = await postJSON(`${ROUTE}/preview`, { + builder: this.builderId, + composer: this.widget("composer")?.value || "", + seed: Number(this.widget("seed")?.value || 0), + platform: this.widget("platform")?.value || "generic", + nsfw: this.nsfw, + violence: this.violence, + }); + } catch (e) { + this.preview = { text: "", picks: {}, warnings: [`preview failed: ${e.message}`] }; + } + this.renderPreview(); + this.renderRows(); + this.fitNode(); + } + + // -------------------------------------------------- render + render() { + if (!this.def) return; + if (!this.sections.includes(this.part)) this.part = this.sections[0]; + const shows = this.partDef(this.part).slots || [baseSection(this.part)]; + const counts = this.slotCounts; + const kind = baseSection(this.part) === "subject" ? this.kindOf(this.part) : ""; + const slots = this.def.slots.filter((slot) => shows.includes(slot.section || "subject") + && !(counts && counts[slot.id] === 0 && !slot.custom && !this.arrange) + && !(this.nsfw === "off" && this.nsfwOnly?.has(slot.id)) + && (this.arrange || !slot.hidden) + // The Subject type picks the subject buttons: Scene only shows none, other types their own. + && !(kind && slot.section === "subject" && (kind === "scene" || ((slot.kinds || []).length && !slot.kinds.includes(kind))))); + const find = el("input", { + class: "epb-find", value: this.find.q, placeholder: "Find a list… (e.g. famous buildings)", + title: "Search every wildcard list by name (and by entries, when the catalog is built). Click a result to insert it; ☆ saves it as a button.", + }); + find.addEventListener("input", () => { this.find.q = find.value; this.runFind(); }); + find.addEventListener("keydown", (e) => { if (e.key === "Escape") { this.find = { q: "", lists: [] }; find.value = ""; this.renderFind(); } }); + this.root.replaceChildren( + (this.tplBox = el("div", { class: "epb-tpl" })), + this.sectionStrip(), + this.composer(), + el("div", { class: "epb-rowshead" }, el("h4", {}, `${this.sectionLabel(this.part)} · add a wildcard`), find, + el("button", { class: "epb-btn", title: "Make your own button: one list, or every list whose name has some words", onclick: () => this.openEditor(this.find.q.trim() ? { search: this.find.q.trim(), label: titleCase(this.find.q.trim()) } : {}) }, "+ Button"), + el("button", { class: `epb-btn${this.arrange ? " primary" : ""}`, + title: "Arrange the buttons for every node: drag one onto another button, a group heading or a section chip; hide the ones you never use", + onclick: () => { this.arrange = !this.arrange; this.render(); } }, this.arrange ? "Done" : "Arrange"), + this.arrange ? el("button", { class: "epb-btn", title: "Back to the shipped layout (deletes builders_user/button_layout.yaml)", + onclick: () => this.resetLayout() }, "Reset layout") : null), + (this.findBox = el("div", { class: "epb-found", style: "display:none" })), + this.palette(slots, counts), + (this.rowsBox = el("div", { class: "epb-rows" })), + (this.previewBox = el("div", { class: "epb-preview" })), + ); + this.renderTemplates(); + this.renderFind(); + this.renderRows(); + this.renderPreview(); + this.fitNode(); + } + + // -------------------------------------------------- templates + /** The template bar: load a saved template, save this composer as one, update, delete, undo a load. */ + async renderTemplates() { + const box = this.tplBox; + if (!box) return; + let list = []; + try { list = await templates(); } catch (e) { this.tpl.msg = `templates unavailable: ${e.message}`; this.tpl.warn = true; } + if (box !== this.tplBox) return; // redrawn meanwhile + const current = this.state.template; // {file, name} of the template this composer came from + const select = el("select", { + title: "Load a saved template (replaces what is on this node; Undo brings it back)", + onchange: (e) => { if (e.target.value) this.loadTemplate(e.target.value); }, + }, + el("option", { value: "" }, list.length ? "Load template…" : "No saved templates yet"), + list.map((t) => el("option", { value: t.file, selected: current?.file === t.file, + title: `${t.starter ? "Starter template · " : ""}${t.sections.map((s) => cache.sections[s.replace(/_\d+$/, "")] || s).join(", ")} · ${t.pills} wildcards${t.saved ? ` · saved ${t.saved}` : ""}` }, + `${t.name}${t.starter ? " · starter" : ""}${t.builder && t.builder !== this.builderId ? ` (${t.builder})` : ""}`))); + const items = [select]; + const mode = this.tpl.mode; + if (mode === "save" || mode === "replace") { + const name = el("input", { value: this.tpl.name, placeholder: "Template name" }); + name.addEventListener("input", () => { this.tpl.name = name.value; if (this.tpl.mode === "replace") { this.tpl.mode = "save"; this.renderTemplates(); } }); + name.addEventListener("keydown", (e) => { if (e.key === "Enter") this.saveTemplate(this.tpl.name, this.tpl.mode === "replace"); }); + items.push(name, + el("button", { class: "epb-btn primary", onclick: () => this.saveTemplate(this.tpl.name, mode === "replace") }, mode === "replace" ? "Replace it" : "Save"), + el("button", { class: "epb-btn", onclick: () => { this.tpl.mode = ""; this.tpl.msg = ""; this.renderTemplates(); } }, "Cancel")); + requestAnimationFrame(() => { if (document.activeElement?.tagName !== "INPUT" || !this.root.contains(document.activeElement)) name.focus(); }); + } else if (mode === "delete") { + items.push(el("span", { class: "msg warn" }, `Delete the template “${current?.name}”?`), + el("button", { class: "epb-btn", onclick: () => this.deleteTemplate() }, "Delete"), + el("button", { class: "epb-btn", onclick: () => { this.tpl.mode = ""; this.renderTemplates(); } }, "Keep")); + } else { + items.push(el("button", { class: "epb-btn", title: "Save everything on this node as a named template", + onclick: () => { this.tpl = { mode: "save", name: "", msg: "", warn: false }; this.renderTemplates(); } }, "Save as…"), + el("button", { class: "epb-btn", title: "Empty every field on this node, ready to start fresh (Undo brings it back)", + onclick: () => this.clearComposer() }, "Clear")); + if (current && !current.starter) { + items.push(el("button", { class: "epb-btn", title: `Save the current node over “${current.name}”`, onclick: () => this.saveTemplate(current.name, true) }, "Update"), + el("button", { class: "epb-btn", title: `Delete the template “${current.name}”`, onclick: () => { this.tpl.mode = "delete"; this.renderTemplates(); } }, "Delete")); + } + if (this.undo) items.push(el("button", { class: "epb-btn", title: "Put back what was on the node before the last template load or clear", onclick: () => this.undoTemplate() }, "Undo")); + } + if (mode !== "delete") items.push(el("span", { class: `msg${this.tpl.warn ? " warn" : ""}`, title: this.tpl.msg }, this.tpl.msg)); + box.replaceChildren(...items); + } + + async saveTemplate(name, overwrite = false) { + name = (name || "").trim(); + if (!name) { this.tpl.msg = "Give the template a name."; this.tpl.warn = true; this.renderTemplates(); return; } + try { + const data = await postJSON(`${ROUTE}/templates`, { + action: "save", name, overwrite, builder: this.builderId, + platform: this.widget("platform")?.value || "", composer: this.state, + }); + this.state.template = { file: data.template.file, name: data.template.name }; + this.save(false); + this.tpl = { mode: "", name: "", msg: `Saved “${data.template.name}”.`, warn: false }; + await reloadTemplates(); + } catch (e) { + if (e.status === 409) this.tpl = { mode: "replace", name, msg: `“${name}” exists already.`, warn: true }; + else this.tpl = { ...this.tpl, msg: `Could not save: ${e.message}`, warn: true }; + this.renderTemplates(); + } + } + + async deleteTemplate() { + const current = this.state.template; + if (!current || current.starter) return; + try { + await postJSON(`${ROUTE}/templates`, { action: "delete", file: current.file }); + delete this.state.template; + this.save(false); + this.tpl = { mode: "", name: "", msg: `Deleted “${current.name}”.`, warn: false }; + } catch (e) { + this.tpl = { mode: "", name: "", msg: `Could not delete: ${e.message}`, warn: true }; + } + for (const panel of PANELS) if (panel.state.template?.file === current.file && panel !== this) delete panel.state.template; + await reloadTemplates(); + } + + /** Replace this node's composer with a saved template. Sections this node doesn't have are left out. */ + async loadTemplate(file) { + let data; + try { + data = await getJSON(`${ROUTE}/template?file=${encodeURIComponent(file)}&nsfw=${this.nsfw}`); + } catch (e) { + this.tpl = { mode: "", name: "", msg: `Could not load: ${e.message}`, warn: true }; + this.renderTemplates(); + return; + } + const platformWidget = this.widget("platform"); + this.undo = { state: JSON.stringify(this.state), platform: platformWidget?.value }; + const state = parseState(JSON.stringify(data.composer || {})) || emptyState(); + const own = new Set((this.def.parts || []).map((p) => p.section)); + const left = Object.keys(state.parts).filter((section) => !own.has(baseSection(section)) + && (state.parts[section] || []).some((g) => g.t === "tok" || (g.v || "").trim())); + for (const section of Object.keys(state.parts)) if (!own.has(baseSection(section))) delete state.parts[section]; + state.locks = {}; + for (const key of ["fill", "gender"]) { + for (const section of Object.keys(state[key] || {})) if (!own.has(baseSection(section))) delete state[key][section]; + } + state.template = { file, name: data.name || file, starter: !!data.starter }; + const choices = platformWidget?.options?.values; + if (data.platform && platformWidget && (!Array.isArray(choices) || choices.includes(data.platform))) platformWidget.value = data.platform; + this.lostKeys = new Set((data.missing || []).map((k) => k.toLowerCase())); + this.state = state; + normalise(this.state, this.sections); + this.sel = null; + this.caret = null; + this.part = this.sections[0]; + const notes = [`Loaded “${state.template.name}”.`]; + if (left.length) notes.push(`Left out (not on this node): ${left.map((s) => cache.sections[baseSection(s)] || s).join(", ")}.`); + if (this.lostKeys.size) notes.push(`${this.lostKeys.size} list${this.lostKeys.size > 1 ? "s" : ""} not in this library (red pills).`); + this.tpl = { mode: "", name: "", msg: notes.join(" "), warn: left.length > 0 || this.lostKeys.size > 0 }; + this.save(false); + this.render(); + this.refreshPreview(); + } + + undoTemplate() { + if (!this.undo) return; + const { state, platform } = this.undo; + this.undo = null; + this.state = parseState(state) || emptyState(); + const platformWidget = this.widget("platform"); + if (platformWidget && platform !== undefined) platformWidget.value = platform; + this.lostKeys = new Set(); + normalise(this.state, this.sections); + this.sel = null; + this.caret = null; + this.tpl = { mode: "", name: "", msg: "Undone.", warn: false }; + this.save(false); + this.render(); + this.refreshPreview(); + } + + /** Empty every field, keeping the platform. Undo restores what was here. */ + clearComposer() { + this.undo = { state: JSON.stringify(this.state), platform: this.widget("platform")?.value }; + this.state = emptyState(); + this.lostKeys = new Set(); + normalise(this.state, this.sections); + this.sel = null; + this.caret = null; + this.part = this.sections[0]; + this.tpl = { mode: "", name: "", msg: "Cleared. Undo brings it back.", warn: false }; + this.save(false); + this.render(); + this.refreshPreview(); + } + + // -------------------------------------------------- find + async doFind() { + const q = this.find.q.trim(); + if (q.length < 2) { this.find.lists = []; this.find.error = ""; this.renderFind(); return; } + this.find.busy = true; + this.renderFind(); + try { + const gender = this.genderOf(this.part); + const data = await getJSON(`${ROUTE}/find?q=${encodeURIComponent(q)}&nsfw=${this.nsfw}&gender=${gender}&limit=400&violence=${this.violence}`); + if (this.find.q.trim() !== q) return; // typed on meanwhile + this.find.lists = data.lists || []; + this.find.capped = !!data.capped; + this.find.shown = 60; + this.find.scroll = 0; + this.find.error = ""; + } catch (e) { + this.find.lists = []; + this.find.error = e.message; + } + this.find.busy = false; + this.renderFind(); + this.fitNode(); + } + + renderFind() { + const box = this.findBox; + if (!box) return; + const q = this.find.q.trim(); + if (q.length < 2) { box.style.display = "none"; box.replaceChildren(); return; } + box.style.display = ""; + const all = this.find.lists || []; + const shown = this.find.shown || 60; + const rows = all.slice(0, shown).map((l) => el("div", { class: "row" }, + el("button", { class: "add", title: `${l.key}\nInsert at the amber gap`, onclick: () => this.addList(l) }, + l.nsfw ? el("span", { class: "tag" }, nsfwName(l.nsfw)) : null, + el("b", {}, l.display || displayName(l.key)), + el("small", {}, `${l.count}${l.dialect === "tags" ? " · tags" : ` · ${l.length}`}${l.quality >= 0 ? ` · q${Math.round(l.quality * 100)}` : ""}${l.gender ? (l.gender === "female" ? " · women" : " · men") : ""}`)), + el("button", { class: "keep", title: "Save this list as a button", onclick: () => this.openEditor({ lists: [l.key], label: shortLabel(l.key) }) }, "☆ Button"))); + const more = this.find.capped ? "+ lists \u00b7 the likeliest; add a word to narrow them" : " lists"; + // Every hit, in a box that scrolls; the next 60 rows load as you near the end. + const list = el("div", { class: "epb-found-list" }, ...rows, + all.length > shown ? el("div", { class: "foot" }, `scroll for more (${shown} of ${all.length})`) : null); + list.addEventListener("wheel", (e) => e.stopPropagation(), { passive: true }); + list.addEventListener("scroll", () => { + this.find.scroll = list.scrollTop; + if (all.length > (this.find.shown || 60) && list.scrollTop + list.clientHeight > list.scrollHeight - 120) { + this.find.shown = (this.find.shown || 60) + 60; + this.renderFind(); + } + }); + requestAnimationFrame(() => { list.scrollTop = this.find.scroll || 0; }); + box.replaceChildren(...(rows.length ? [list] : []), + el("div", { class: "foot" }, + el("span", {}, this.find.busy ? "searching…" : this.find.error ? this.find.error + : rows.length ? `${all.length}${more}` : "no list names have all these words"), + el("button", { class: "epb-btn", style: "margin-left:auto", title: "A button that always offers every list whose name has these words", + onclick: () => this.openEditor({ search: q, label: titleCase(q) }) }, `☆ Button for “${q}”`), + el("button", { class: "epb-btn", onclick: () => { this.find = { q: "", lists: [] }; this.render(); } }, "Close"))); + } + + // -------------------------------------------------- my buttons + /** Make or edit one of your buttons. initial: {id?, label, section?, lists?, search?, tokenId?}. */ + openEditor(initial = {}) { + this.closeEditor(); + const panel = this; + const b = { + id: initial.id || "", + label: initial.label || "", + section: initial.section || baseSection(this.part || "subject"), + lists: [...(initial.lists || [])], + search: initial.search || "", + kind: (initial.lists || []).length && !initial.search ? "lists" : "search", + matches: null, + error: "", + }; + const box = el("div", { class: "epb-dialog", role: "dialog" }); + box.addEventListener("keydown", (e) => { e.stopPropagation(); if (e.key === "Escape") panel.closeEditor(); }); + const rect = this.root.getBoundingClientRect(); + box.style.left = `${Math.max(8, Math.min(window.innerWidth - 480, rect.right + 12))}px`; + box.style.top = `${Math.max(8, Math.min(window.innerHeight - 460, rect.top + 40))}px`; + document.body.append(box); + this.editor = box; + + const countMatches = debounce(async () => { + const q = b.search.trim(); + if (q.length < 2) { b.matches = null; draw(); return; } + try { + const data = await getJSON(`${ROUTE}/find?q=${encodeURIComponent(q)}&names=1&nsfw=${panel.nsfw}&limit=100&violence=${panel.violence}`); + if (b.search.trim() === q) { b.matches = data.lists || []; draw(false); } + } catch (e) { b.matches = null; } + }, 300); + + const save = async () => { + const button = { id: b.id, label: b.label.trim(), section: b.section }; + if (b.kind === "lists") button.lists = b.lists; else button.search = b.search.trim(); + try { + const data = await postJSON(`${ROUTE}/buttons`, { action: "save", button }); + // A pill the button was made from now belongs to it. + const token = initial.tokenId !== undefined ? panel.state.tokens[String(initial.tokenId)] : null; + if (token && !token.slot) { token.slot = data.button.id; delete token.label; panel.save(); } + panel.closeEditor(); + await reloadBuilders(); + } catch (e) { b.error = `could not save: ${e.message}`; draw(); } + }; + const remove = async () => { + try { + await postJSON(`${ROUTE}/buttons`, { action: "delete", id: b.id }); + panel.closeEditor(); + await reloadBuilders(); + } catch (e) { b.error = `could not delete: ${e.message}`; draw(); } + }; + + const draw = (full = true) => { + if (!full && box.querySelector(".matches")) { + box.querySelector(".matches").replaceChildren(...matchLines()); + return; + } + const header = el("header", {}, el("b", {}, b.id ? "Edit button" : "New button"), + el("span", {}, "Saved in builders_user/my_buttons.yaml, shared by every builder node"), + el("button", { class: "close", "aria-label": "Close", onclick: () => panel.closeEditor() }, "✕")); + makeDraggable(box, header); + const label = el("input", { value: b.label, placeholder: "Button label" }); + label.addEventListener("input", () => { b.label = label.value; }); + const sectionSelect = el("select", { onchange: (e) => { b.section = e.target.value; } }, + (cache.defaultOrder || FALLBACK_ORDER).map((s) => el("option", { value: s, selected: s === b.section }, cache.sections[s] || s))); + const kinds = el("div", { class: "epb-seg kinds" }, + el("button", { class: b.kind === "lists" ? "on" : "", disabled: !b.lists.length, title: b.lists.length ? "Always these lists" : "Add lists from Find with ☆ Button", + onclick: () => { b.kind = "lists"; draw(); } }, `Specific list${b.lists.length > 1 ? "s" : ""} (${b.lists.length})`), + el("button", { class: b.kind === "search" ? "on" : "", title: "Every list whose name has all these words, now and as you add packs", + onclick: () => { b.kind = "search"; draw(); countMatches(); } }, "Category: name words")); + let body; + if (b.kind === "lists") { + body = el("div", { class: "keys" }, b.lists.map((key) => el("div", {}, el("span", { title: key }, key), + el("button", { title: "Remove", onclick: () => { b.lists = b.lists.filter((k) => k !== key); if (!b.lists.length) b.kind = "search"; draw(); } }, "✕")))); + } else { + const words = el("input", { value: b.search, placeholder: "e.g. famous buildings or black female names" }); + words.addEventListener("input", () => { b.search = words.value; countMatches(); }); + body = el("label", {}, el("span", {}, "Words that must all be in a list's name (folders count; 'female' / 'male' follow the list's gender)"), words); + } + box.replaceChildren(header, + el("div", { class: "body" }, + el("div", { class: "two" }, el("label", {}, el("span", {}, "Label"), label), el("label", {}, el("span", {}, "Section"), sectionSelect)), + kinds, body, + b.kind === "search" ? el("div", { class: "matches" }, ...matchLines()) : null, + b.error ? el("div", { class: "warn", style: "color:#f0b454" }, b.error) : null), + el("footer", {}, + el("button", { class: "epb-btn primary", onclick: save }, "Save button"), + b.id ? el("button", { class: "epb-btn", onclick: remove }, "Delete") : null, + el("button", { class: "epb-btn", style: "margin-left:auto", onclick: () => panel.closeEditor() }, "Cancel"))); + if (full) (b.label ? null : label)?.focus(); + }; + const matchLines = () => { + if (b.matches === null) return [b.search.trim().length < 2 ? "Type at least one word." : "counting…"]; + if (!b.matches.length) return ["No list names have all these words yet."]; + return [`${b.matches.length}${b.matches.length >= 100 ? "+" : ""} lists: `, + b.matches.slice(0, 8).map((l) => l.display || displayName(l.key)).join(" · ") + (b.matches.length > 8 ? " …" : "")]; + }; + draw(); + if (b.kind === "search") countMatches(); + } + + closeEditor() { + this.editor?.remove(); + this.editor = null; + } + + // -------------------------------------------------- more lists + closeMore() { + this.more?.element.remove(); + this.more = null; + } + + /** Every usable list no button offers for this part's sections, grouped by path words. */ + async openMore() { + this.closeMore(); + const panel = this; + const ownSections = this.partDef(this.part).slots || [baseSection(this.part)]; + let sections = ownSections; + const m = { element: el("div", { class: "epb-dialog epb-merge", role: "dialog" }), data: null, group: 0, q: "", shown: 200, error: "", unsorted: false }; + this.more = m; + m.element.addEventListener("keydown", (e) => { e.stopPropagation(); if (e.key === "Escape") panel.closeMore(); }); + m.element.style.left = `${Math.max(8, (window.innerWidth - 660) / 2)}px`; + m.element.style.top = `${Math.max(8, window.innerHeight * 0.06)}px`; + document.body.append(m.element); + + const lists = () => { + const q = m.q.trim().toLowerCase(); + if (!m.data) return []; + if (q) return m.data.groups.flatMap((g) => g.lists).filter((l) => (l.key + " " + (l.display || "")).toLowerCase().includes(q)); + return m.data.groups[m.group]?.lists || []; + }; + const draw = () => { + const label = m.unsorted ? "no clear section" : sections.map((s) => cache.sections[s] || s).join(" + "); + const header = el("header", {}, el("b", {}, `More lists · ${label}`), + el("span", {}, m.data ? `${m.data.total} lists no button offers, in ${m.data.groups.length} groups` : m.error || "sorting your lists…"), + el("button", { class: "close", "aria-label": "Close", onclick: () => panel.closeMore() }, "✕")); + makeDraggable(m.element, header); + const d = m.data; + const search = el("input", { type: "search", value: m.q, placeholder: "filter by name…" }); + search.addEventListener("input", () => { m.q = search.value; m.shown = 200; drawList(); }); + const chips = el("div", { class: "epb-more-groups" }, (d?.groups || []).map((g, i) => el("button", { + class: !m.q && i === m.group ? "on" : "", + onclick: () => { m.group = i; m.q = ""; m.shown = 200; draw(); }, + }, g.label, el("small", {}, String(g.count))))); + const g = d?.groups[m.group]; + const groupBar = el("div", { class: "bar" }, + g && !m.q ? el("button", { class: "epb-btn", title: "A button that offers every list of this group (in My buttons)", + onclick: () => panel.openEditor({ lists: g.lists.map((l) => l.key), label: g.label, section: ownSections[0] }) }, + `☆ Button for all ${g.count} “${g.label}” lists`) : null, + d && (d.unsorted || m.unsorted) ? el("button", { class: "epb-btn", style: "margin-left:auto", + title: "Lists whose name, folder and content don't point to one section", + onclick: () => { m.unsorted = !m.unsorted; load(); } }, m.unsorted ? "← Back to this section" : `${d.unsorted} lists with no clear section →`) : null); + m.listBox = el("div", { class: "mrows epb-more-list" }); + m.listBox.addEventListener("scroll", () => { + const box = m.listBox; + if (box.scrollTop + box.clientHeight > box.scrollHeight - 200 && m.shown < lists().length) { m.shown += 200; drawList(); } + }); + m.element.replaceChildren(header, el("div", { class: "body" }, search, chips, groupBar), m.listBox, + el("footer", {}, el("span", { class: "stats" }, "Click a list to insert it at the amber gap; ☆ saves one as a button."), + el("button", { class: "epb-btn", style: "margin-left:auto", onclick: () => panel.closeMore() }, "Close"))); + drawList(); + }; + const drawList = () => { + if (!m.listBox) return; + const all = lists(); + const top = m.listBox.scrollTop; + m.listBox.replaceChildren(...all.slice(0, m.shown).map((l) => el("div", { class: "row" }, + el("button", { class: "add", title: `${l.key}\nInsert at the amber gap`, onclick: () => panel.addList(l) }, + l.nsfw ? el("span", { class: "tag" }, nsfwName(l.nsfw)) : null, + el("b", {}, l.display || displayName(l.key)), + el("small", {}, `${l.count}${l.dialect === "tags" ? " · tags" : ` · ${l.length}`}${l.quality >= 0 ? ` · q${Math.round(l.quality * 100)}` : ""}${l.gender ? (l.gender === "female" ? " · women" : " · men") : ""}`)), + el("button", { class: "keep", title: "Save this list as a button", onclick: () => panel.openEditor({ lists: [l.key], label: shortLabel(l.key), section: ownSections[0] }) }, "☆"))), + all.length > m.shown ? el("div", { class: "epb-more" }, `scroll for more (${m.shown} of ${all.length})`) : null, + m.data && !all.length ? el("div", { class: "epb-more" }, m.q ? "no list matches" : "nothing here") : null); + m.listBox.scrollTop = top; + }; + const load = async () => { + sections = m.unsorted ? ["_unsorted"] : ownSections; + m.data = null; m.group = 0; m.q = ""; m.shown = 200; m.error = ""; + draw(); + try { + m.data = await getJSON(`${ROUTE}/more?builder=${encodeURIComponent(panel.builderId)}§ions=${sections.join(",")}&nsfw=${panel.nsfw}&violence=${panel.violence}`); + if (m.data.note) m.error = m.data.note; + } catch (e) { m.error = e.message; } + if (panel.more === m) draw(); + }; + load(); + } + + // -------------------------------------------------- prompt files (_prompts): the Prompt line's browser + closePrompts() { + this.prompts?.element.remove(); + this.prompts = null; + } + + /** + * Walk the prompt files (_prompts///...), look at a file's prompts, and insert a file or a whole + * folder at the amber gap. replaceId: swap that pill's file instead (from the pill dialog). + */ + async openPrompts(replaceId = null) { + this.closePrompts(); + if (!document.getElementById("epb-prompts-css")) { + document.head.append(el("style", { id: "epb-prompts-css" }, ` + .epb-crumbs { display: flex; flex-wrap: wrap; gap: 4px; margin: 4px 0; } + .epb-prompt-preview { max-height: 240px; overflow: auto; margin: 0 0 8px 18px; padding: 4px 8px; + border-left: 2px solid rgba(255,190,80,.6); font-size: 11px; line-height: 1.35; } + .epb-prompt-line { padding: 3px 0; border-bottom: 1px solid rgba(255,255,255,.07); white-space: normal; } + .epb-prompt-line.above { opacity: .4; } + .epb-prompt-line.gone { text-decoration: line-through; opacity: .5; }`)); + } + const panel = this; + const p = { element: el("div", { class: "epb-dialog epb-merge", role: "dialog" }), folder: "prompts", q: "", + data: null, error: "", open: null, preview: null, listBox: null, stats: null }; + this.prompts = p; + p.reload = () => load(true); + p.element.addEventListener("keydown", (e) => { e.stopPropagation(); if (e.key === "Escape") panel.closePrompts(); }); + p.element.style.left = `${Math.max(8, (window.innerWidth - 660) / 2)}px`; + p.element.style.top = `${Math.max(8, window.innerHeight * 0.06)}px`; + document.body.append(p.element); + + const leafName = (key) => titleCase(key.split("/").pop() || key); + const choose = (key, label) => { + const id = replaceId === null ? null : String(replaceId); + if (id !== null && panel.state.tokens[id]) { + panel.updateToken(replaceId, { key, label, picks: [], mode: "random" }); + if (panel.dialog && String(panel.dialog.tokenId) === id) panel.openDialog(replaceId); + } else { + panel.addList({ key, label }); + } + panel.closePrompts(); + }; + const statsText = () => { + const d = p.data; + if (!d) return p.error || "reading your prompt files\u2026"; + if (p.q.trim()) return `${d.here} files match${d.more ? ` (first ${d.lists.length} shown)` : ""}`; + return `${d.folders.length} folders \u00b7 ${d.lists.length} files here${d.more ? ` (+${d.more} more)` : ""} \u00b7 ${d.here} files in all`; + }; + const previewBox = () => { + const pr = p.preview; + if (!pr) return el("div", { class: "epb-prompt-preview" }, "loading\u2026"); + return el("div", { class: "epb-prompt-preview" }, + pr.entries.map((e) => el("div", { class: `epb-prompt-line${e.above ? " above" : ""}${e.dropped ? " gone" : ""}`, + title: e.above ? `left out at ${panel.nsfw}` : "" }, e.text)), + pr.total > pr.entries.length ? el("div", { class: "epb-more" }, + `${pr.entries.length} of ${pr.total}. Use the file, then click its pill to see, search and pick from all of them.`) : null, + !pr.total ? el("div", { class: "epb-more" }, "no prompts at this NSFW level") : null); + }; + const togglePreview = async (key) => { + if (p.open === key) { p.open = null; drawList(); return; } + p.open = key; + p.preview = null; + drawList(); + try { + p.preview = await getJSON(`${ROUTE}/list?key=${encodeURIComponent(key)}&limit=40&nsfw=${panel.nsfw}`); + } catch (e) { + p.preview = { entries: [], total: 0 }; + p.error = e.message; + } + if (panel.prompts === p && p.open === key) drawList(); + }; + const drawList = () => { + if (!p.listBox) return; + if (p.stats) p.stats.textContent = statsText(); + const d = p.data; + const rows = []; + if (d && !p.q.trim() && d.here && d.folder !== "prompts") { + rows.push(el("div", { class: "row" }, + el("button", { class: "add", title: `${d.folder}/*\nEach roll draws a prompt from any file in this folder`, + onclick: () => choose(`${d.folder}/*`, `${leafName(d.folder)} (all)`) }, + el("b", {}, `Use the whole folder: ${leafName(d.folder)}`), el("small", {}, `${d.here} files`)))); + } + for (const f of d?.folders || []) { + rows.push(el("div", { class: "row" }, + el("button", { class: "add", title: `${f.key}\nOpen this folder`, onclick: () => { p.folder = f.key; p.q = ""; load(true); } }, + el("b", {}, `\u{1F4C1} ${titleCase(f.name)}`), el("small", {}, `${f.lists} files`)))); + } + for (const l of d?.lists || []) { + rows.push(el("div", { class: "row" }, + el("button", { class: "add", title: `${l.key}\nClick to read its prompts`, onclick: () => togglePreview(l.key) }, + l.nsfw ? el("span", { class: "tag" }, nsfwName(l.nsfw)) : null, + el("b", {}, p.q.trim() ? l.name : titleCase(l.name)), el("small", {}, `${l.count} prompts`)), + el("button", { class: "keep", title: replaceId === null ? "Insert this file at the amber gap" : "Use this file for the pill", + onclick: () => choose(l.key, shortLabel(l.key)) }, "Use"))); + if (p.open === l.key) rows.push(previewBox()); + } + if (d && !rows.length) rows.push(el("div", { class: "epb-more" }, p.q.trim() ? "no prompt file matches" : "nothing here")); + const top = p.listBox.scrollTop; + p.listBox.replaceChildren(...rows); + p.listBox.scrollTop = top; + }; + const draw = () => { + p.stats = el("span", {}, statsText()); + const header = el("header", {}, el("b", { title: "The node's nsfw setting: files and prompts above it are left out" }, + `${replaceId === null ? "Prompt files" : "Prompt files \u00b7 swap this pill"} \u00b7 browsing at: ${panel.nsfw}`), p.stats, + el("button", { class: "close", "aria-label": "Close", onclick: () => panel.closePrompts() }, "\u00d7")); + makeDraggable(p.element, header); + const search = el("input", { type: "search", value: p.q, placeholder: "find prompt files by name, in every folder\u2026" }); + search.addEventListener("input", debounce(() => { p.q = search.value; load(false); }, 250)); + const parts = p.folder.split("/"); + const crumbs = el("div", { class: "epb-crumbs" }, parts.map((part, i) => el("button", { + class: `epb-btn${i === parts.length - 1 ? " primary" : ""}`, + onclick: () => { p.folder = parts.slice(0, i + 1).join("/"); p.q = ""; load(true); }, + }, i ? titleCase(part) : "All prompts"))); + p.listBox = el("div", { class: "mrows epb-more-list" }); + p.element.replaceChildren(header, el("div", { class: "body" }, search, crumbs), p.listBox, + el("footer", {}, el("span", { class: "stats" }, "Click a file to read its prompts; Use inserts it. A whole folder draws from all its files."), + el("button", { class: "epb-btn", style: "margin-left:auto", onclick: () => panel.closePrompts() }, "Close"))); + drawList(); + }; + const load = async (redraw) => { + p.data = null; p.open = null; p.error = ""; + if (redraw) draw(); else drawList(); + try { + p.data = await getJSON(`${ROUTE}/prompts?folder=${encodeURIComponent(p.folder)}&q=${encodeURIComponent(p.q.trim())}&nsfw=${panel.nsfw}`); + } catch (e) { p.error = e.message; } + if (panel.prompts === p) drawList(); + }; + load(true); + } + + // -------------------------------------------------- merging a slot's lists + closeMerge() { + this.merge?.element.remove(); + this.merge = null; + } + + /** Fold the lists behind a pill's slot into one list of your own (mine//). */ + async openMerge(tokenId, lists) { + this.closeMerge(); + const panel = this; + const token = this.state.tokens[String(tokenId)]; + if (!token?.slot) return; + const slotDef = this.def.slots.find((s) => s.id === token.slot) || { id: token.slot, label: token.slot }; + if (!lists?.length) { + try { lists = await slotLists(this.builderId, token.slot, this.nsfw, this.genderOf(this.sectionOfToken(tokenId)), this.violence); } catch (e) { lists = []; } + } + lists = lists.filter((l) => !l.key.startsWith(`mine/${token.slot}/`)); // not your earlier merges + const m = { + element: el("div", { class: "epb-dialog epb-merge", role: "dialog" }), + chosen: new Set(lists.map((l) => l.key)), + data: null, off: new Set(), min: 1, len: "any", q: "", shown: 300, + name: "merged", hide: false, busy: false, error: "", exists: false, + }; + this.merge = m; + m.element.addEventListener("keydown", (e) => { e.stopPropagation(); if (e.key === "Escape") panel.closeMerge(); }); + m.element.style.left = `${Math.max(8, (window.innerWidth - 660) / 2)}px`; + m.element.style.top = `${Math.max(8, window.innerHeight * 0.06)}px`; + document.body.append(m.element); + + const lengthOf = (row) => (row.words <= 4 ? "short" : row.words <= 8 ? "medium" : "long"); + const kept = () => (m.data?.rows || []).filter((r) => r.lists >= m.min && (m.len === "any" || lengthOf(r) === m.len)); + const shown = () => { const q = m.q.trim().toLowerCase(); return q ? kept().filter((r) => r.text.toLowerCase().includes(q)) : kept(); }; + + const build = async () => { + m.busy = true; m.error = ""; draw(); + try { + m.data = await postJSON(`${ROUTE}/merge/preview`, { builder: panel.builderId, slot: token.slot, keys: lists.filter((l) => m.chosen.has(l.key)).map((l) => l.key), nsfw: panel.nsfw }); + } catch (e) { m.data = null; m.error = e.message; } + m.busy = false; m.shown = 300; draw(); + }; + const rebuild = debounce(build, 400); + + const save = async (overwrite = false) => { + const entries = kept().filter((r) => !m.off.has(r.text)).map((r) => r.text); + try { + const result = await postJSON(`${ROUTE}/merge/save`, { + builder: panel.builderId, slot: token.slot, name: m.name, entries, sources: m.data?.keys || [], + hide_sources: m.hide, overwrite, nsfw: panel.nsfw, + }); + cache.slots.clear(); + panel.closeMerge(); + panel.updateToken(tokenId, { key: result.key, picks: [], mode: "random" }); + panel.countsKey = null; panel.loadSlotCounts(); + panel.openDialog(tokenId); + } catch (e) { + m.exists = e.status === 409; + m.error = m.exists ? `“${m.name}” exists for this button.` : `Could not save: ${e.message}`; + draw(); + } + }; + + const rowButton = (r) => el("button", { + class: `epb-mrow${m.off.has(r.text) ? " off" : ""}`, + title: `${r.lists} of ${m.data.keys.length} lists: ${r.sources.slice(0, 6).map((i) => displayName(m.data.keys[i])).join(", ")}${r.sources.length > 6 ? " …" : ""}${r.variants.length ? `\nAlso spelled: ${r.variants.join(" | ")}` : ""}\nClick to leave it out`, + onclick: (e) => { + if (m.off.has(r.text)) m.off.delete(r.text); else m.off.add(r.text); + e.currentTarget.classList.toggle("off", m.off.has(r.text)); + drawCounts(); + }, + }, el("span", { class: "box" }, "✓"), el("span", { class: "txt" }, r.text), + r.variants.length ? el("span", { class: "n" }, `+${r.variants.length}`) : null, + el("span", { class: `n${r.lists >= 3 ? " core" : ""}` }, `×${r.lists}`)); + + const drawCounts = () => { + const box = m.element.querySelector(".keepcount"); + if (!box) return; + const k = kept(); + const saving = k.filter((r) => !m.off.has(r.text)).length; + box.textContent = `Saves ${saving} of ${k.length} entries${m.min > 1 ? ` (in ${m.min}+ lists)` : ""}${m.len !== "any" ? `, ${m.len} only` : ""}`; + }; + + const draw = () => { + const header = el("header", {}, el("b", {}, `Merge lists · ${slotDef.label}`), + el("span", {}, "Shared entries first. Untick what you don't want; save as your own list."), + el("button", { class: "close", "aria-label": "Close", onclick: () => panel.closeMerge() }, "✕")); + makeDraggable(m.element, header); + const srcs = el("div", { class: "srcs" }, lists.map((l) => { + const box = el("input", { type: "checkbox" }); + box.checked = m.chosen.has(l.key); + box.addEventListener("change", () => { if (box.checked) m.chosen.add(l.key); else m.chosen.delete(l.key); rebuild(); }); + return el("label", { title: l.key }, box, listLabel(l)); + })); + const d = m.data; + const stats = el("div", { class: "stats" }, m.busy ? "merging…" : m.error && !d ? m.error : d + ? [el("b", {}, String(d.keys.length)), " lists · ", el("b", {}, String(d.entries)), " entries → ", el("b", {}, String(d.rows.length)), + " unique · ", el("b", {}, String(d.rows.filter((r) => r.lists >= 3).length)), " in 3+ lists · ", el("b", {}, String(d.shared)), " in 2+"] + : ""); + const search = el("input", { type: "search", value: m.q, placeholder: "find in the merged list…" }); + search.addEventListener("input", () => { m.q = search.value; m.shown = 300; drawRows(); }); + const seg = (values, current, onSet) => el("div", { class: "epb-seg" }, values.map(([v, label]) => + el("button", { class: current === v ? "on" : "", onclick: () => { onSet(v); draw(); } }, label))); + const bar = el("div", { class: "bar" }, search, + "in", seg([[1, "any"], [2, "2+"], [3, "3+"], [5, "5+"]], m.min, (v) => { m.min = v; m.shown = 300; }), "lists", + seg([["any", "Any"], ["short", "Short"], ["medium", "Medium"], ["long", "Long"]], m.len, (v) => { m.len = v; m.shown = 300; })); + const bulk = el("div", { class: "bar" }, + el("button", { class: "epb-btn", title: "Leave out every entry shown (use with the search box)", onclick: () => { shown().forEach((r) => m.off.add(r.text)); drawRows(); } }, "Untick shown"), + el("button", { class: "epb-btn", title: "Keep every entry shown", onclick: () => { shown().forEach((r) => m.off.delete(r.text)); drawRows(); } }, "Tick shown"), + el("span", { class: "keepcount", style: "margin-left:auto" })); + m.rowsBox = el("div", { class: "mrows" }); + m.rowsBox.addEventListener("scroll", () => { + const box = m.rowsBox; + if (box.scrollTop + box.clientHeight > box.scrollHeight - 200 && m.shown < shown().length) { m.shown += 300; drawRows(true); } + }); + const name = el("input", { type: "text", value: m.name, title: "Saved as mine/