From 07b0f9a18b04b39832cb7ea9a20be70800aff398 Mon Sep 17 00:00:00 2001 From: Crespin Alexandre <114501859+kunail0804@users.noreply.github.com> Date: Tue, 7 Jul 2026 19:23:50 +0200 Subject: [PATCH] feat(knowledge-base): define KB file format, entry rules, and categories (#8) (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Formalize the Knowledge Base on-disk format and tighten the entry schema per MASTER_SPEC §15.3-§15.10, §18.2. - compiler/common/categories.py: single source of truth for the 19 canonical categories (lowercase identifiers of the §18.2 list) plus is_valid_category. - knowledge_base_entry.schema.json (v1.1): category restricted to the 19; tags require at least one item; expand entries must be snake_case canonical ids; aliases uniqueItems. - knowledge_base/README.md: by-domain file layout, the "each file is a JSON array of entries" format, and contributor field rules (id/alias global uniqueness, tag order, expansion targets, one-tag minimum). - examples/knowledge_base.example.json: worked file demonstrating an alias and an expansion. Cross-entry rules (global uniqueness, expand targets exist) are deferred to the KB validation tooling (#9); this issue defines per-entry structure only. Tests: 19-category constant and schema-enum consistency, entry validation (category/tags/id/expand/unknown-field), and the worked example (array shape, per-entry validity, alias+expansion present, expand targets resolve). Co-authored-by: kunail0804 Co-authored-by: Claude Opus 4.8 --- compiler/common/categories.py | 44 +++++++ examples/knowledge_base.example.json | 16 +++ knowledge_base/.gitkeep | 0 knowledge_base/README.md | 73 +++++++++++ schemas/json/knowledge_base_entry.schema.json | 46 ++++++- tests/test_knowledge_base_format.py | 122 ++++++++++++++++++ 6 files changed, 295 insertions(+), 6 deletions(-) create mode 100644 compiler/common/categories.py create mode 100644 examples/knowledge_base.example.json delete mode 100644 knowledge_base/.gitkeep create mode 100644 knowledge_base/README.md create mode 100644 tests/test_knowledge_base_format.py diff --git a/compiler/common/categories.py b/compiler/common/categories.py new file mode 100644 index 0000000..8202a1b --- /dev/null +++ b/compiler/common/categories.py @@ -0,0 +1,44 @@ +"""The 19 canonical compiler categories (MASTER_SPEC §18.2, Appendix C.1). + +Categories are defined by the compiler, not by the Knowledge Base (§15.5). This +module is their single source of truth: the Knowledge Base entry schema, the +Knowledge Base validation tooling, and the Category Splitter all draw the valid +category set from here so they cannot drift apart. + +The §18.2 list is authoritative and uses Title-Case display names; we use the +lowercase identifier form here for machine use (Category Map keys and +``ResolvedTag.category``). The two forms correspond one-to-one, in order. +""" + +from __future__ import annotations + +# Ordered lowercase identifiers for the 19 categories of §18.2. Order is the +# authoritative spec order and is significant (e.g. category ordering downstream). +CANONICAL_CATEGORIES: tuple[str, ...] = ( + "character", + "appearance", + "hair", + "face", + "eyes", + "expression", + "body", + "clothing", + "accessories", + "pose", + "action", + "interaction", + "objects", + "environment", + "camera", + "lighting", + "style", + "quality", + "miscellaneous", +) + +_CATEGORY_SET = frozenset(CANONICAL_CATEGORIES) + + +def is_valid_category(category: str) -> bool: + """Return True when ``category`` is one of the 19 canonical categories.""" + return category in _CATEGORY_SET diff --git a/examples/knowledge_base.example.json b/examples/knowledge_base.example.json new file mode 100644 index 0000000..77b2bdd --- /dev/null +++ b/examples/knowledge_base.example.json @@ -0,0 +1,16 @@ +[ + { + "id": "long_hair", + "aliases": ["lengthy hair"], + "tags": ["long hair"], + "category": "hair" + }, + { + "id": "twin_braids", + "aliases": ["twintail braids", "double braids"], + "tags": ["twin braids"], + "category": "hair", + "expand": ["long_hair"], + "notes": "Composite hairstyle: resolving 'twin braids' also adds 'long hair' via expansion." + } +] diff --git a/knowledge_base/.gitkeep b/knowledge_base/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/knowledge_base/README.md b/knowledge_base/README.md new file mode 100644 index 0000000..fc8e82f --- /dev/null +++ b/knowledge_base/README.md @@ -0,0 +1,73 @@ +# Knowledge Base + +The Knowledge Base is **data only** — no executable code (MASTER_SPEC §15). It is +the single source of truth for how semantic concepts map to Illustrious tags. The +compiler loads every file here once during initialization. + +## File organization + +Concepts are split into multiple JSON files **by domain**, for readability and +maintenance (§15.3). The compiler loads and merges all of them: + +``` +knowledge_base/ + appearance.json clothing.json anatomy.json expressions.json + poses.json actions.json interactions.json objects.json + environments.json camera.json lighting.json quality.json style.json +``` + +Domain files are a maintenance convenience; a concept's **category** (below), not +the file it lives in, determines its behaviour. + +## File format + +Each file is a **JSON array of entries**: + +```json +[ + { "id": "long_hair", "tags": ["long hair"], "category": "hair" }, + { "id": "blue_eyes", "tags": ["blue eyes"], "category": "eyes" } +] +``` + +A complete worked example (with an alias and an expansion) is in +[`examples/knowledge_base.example.json`](../examples/knowledge_base.example.json). + +## Entry fields + +Each entry validates against the `knowledge_base_entry` schema +([`schemas/json/knowledge_base_entry.schema.json`](../schemas/json/knowledge_base_entry.schema.json)). + +| Field | Required | Rule | +|---|---|---| +| `id` | yes | Canonical ID: lowercase `snake_case`, **globally unique** across the whole Knowledge Base. | +| `tags` | yes | The Illustrious tags this concept generates. **At least one.** Order is preserved. | +| `category` | yes | Exactly one of the 19 canonical categories (see below). | +| `aliases` | no | Alternative expressions resolving **directly** to this entry (no alias→alias chains). **Globally unique**, and must not collide with any `id`. | +| `expand` | no | Additional canonical `id`s automatically added during resolution. References **canonical ids only**, never raw tags; targets must exist. | +| `deprecated` | no | Marks an obsolete concept; it keeps working but the compiler emits a warning. | +| `notes` | no | Free-form documentation for contributors; ignored during compilation. | + +### Contributor rules + +- **`id` uniqueness** — every canonical `id` is unique across all files. +- **`alias` uniqueness** — every alias is unique across all files and never equal + to any `id`; aliases resolve directly to their entry (no chains). +- **Tag order** — the order of `tags` is significant and preserved in output. +- **Category** — must be one of the 19 categories; the authoritative set is + defined once in `compiler/common/categories.py`. +- **Expansion** — `expand` lists canonical `id`s only; every target must exist. +- **At least one tag** — every entry generates one or more tags. + +> Per-entry structure is enforced by the entry schema. Cross-entry rules (global +> uniqueness, expand targets exist, no alias chains) are enforced by the Knowledge +> Base validation tooling. + +## The 19 categories + +`character`, `appearance`, `hair`, `face`, `eyes`, `expression`, `body`, +`clothing`, `accessories`, `pose`, `action`, `interaction`, `objects`, +`environment`, `camera`, `lighting`, `style`, `quality`, `miscellaneous`. + +`style` and `quality` are reserved for workflow integration (not generated +automatically in V1); `miscellaneous` is a fallback that should stay nearly empty. diff --git a/schemas/json/knowledge_base_entry.schema.json b/schemas/json/knowledge_base_entry.schema.json index 2de5172..42b3932 100644 --- a/schemas/json/knowledge_base_entry.schema.json +++ b/schemas/json/knowledge_base_entry.schema.json @@ -1,18 +1,52 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "knowledge_base_entry", - "version": "1.0", + "version": "1.1", "title": "Knowledge Base Entry", - "description": "A canonical Knowledge Base concept. MASTER_SPEC §11.6 / §15.5.", + "description": "A canonical Knowledge Base concept. MASTER_SPEC §11.6, §15.4-§15.10. Cross-entry rules (global uniqueness of ids/aliases, expand targets exist) are enforced by the KB validation tooling, not this per-entry schema.", "type": "object", "required": ["id", "tags", "category"], "additionalProperties": false, "properties": { "id": { "type": "string", "pattern": "^[a-z0-9]+(_[a-z0-9]+)*$" }, - "aliases": { "type": "array", "items": { "type": "string" } }, - "tags": { "type": "array", "items": { "type": "string" } }, - "category": { "type": "string" }, - "expand": { "type": "array", "items": { "type": "string" } }, + "aliases": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true + }, + "tags": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "category": { + "type": "string", + "enum": [ + "character", + "appearance", + "hair", + "face", + "eyes", + "expression", + "body", + "clothing", + "accessories", + "pose", + "action", + "interaction", + "objects", + "environment", + "camera", + "lighting", + "style", + "quality", + "miscellaneous" + ] + }, + "expand": { + "type": "array", + "items": { "type": "string", "pattern": "^[a-z0-9]+(_[a-z0-9]+)*$" } + }, "deprecated": { "type": "boolean" }, "notes": { "type": "string" } } diff --git a/tests/test_knowledge_base_format.py b/tests/test_knowledge_base_format.py new file mode 100644 index 0000000..751c5c2 --- /dev/null +++ b/tests/test_knowledge_base_format.py @@ -0,0 +1,122 @@ +"""Tests for the Knowledge Base file format and entry schema (issue #8, §15, §18.2).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from compiler.common.categories import CANONICAL_CATEGORIES, is_valid_category +from schemas.validation import validate_document + +REPO_ROOT = Path(__file__).resolve().parent.parent +EXAMPLE_FILE = REPO_ROOT / "examples" / "knowledge_base.example.json" +ENTRY_SCHEMA_FILE = REPO_ROOT / "schemas" / "json" / "knowledge_base_entry.schema.json" + + +# --- categories (single source of truth) ----------------------------------- + + +def test_there_are_nineteen_categories() -> None: + assert len(CANONICAL_CATEGORIES) == 19 + assert len(set(CANONICAL_CATEGORIES)) == 19 # all unique + + +def test_expected_categories_present() -> None: + for name in ("character", "hair", "eyes", "clothing", "miscellaneous"): + assert name in CANONICAL_CATEGORIES + assert is_valid_category(name) + + +def test_unknown_category_is_invalid() -> None: + assert not is_valid_category("not_a_category") + + +def test_schema_category_enum_matches_constant() -> None: + schema = json.loads(ENTRY_SCHEMA_FILE.read_text(encoding="utf-8")) + assert set(schema["properties"]["category"]["enum"]) == set(CANONICAL_CATEGORIES) + + +# --- entry schema ---------------------------------------------------------- + + +def valid_entry() -> dict: + return {"id": "long_hair", "tags": ["long hair"], "category": "hair"} + + +def test_valid_entry_passes() -> None: + assert validate_document(valid_entry(), "knowledge_base_entry") == [] + + +def test_entry_category_must_be_one_of_the_19() -> None: + entry = valid_entry() + entry["category"] = "hairstyle" + assert validate_document(entry, "knowledge_base_entry") != [] + + +def test_entry_requires_at_least_one_tag() -> None: + entry = valid_entry() + entry["tags"] = [] + assert validate_document(entry, "knowledge_base_entry") != [] + + +def test_entry_id_must_be_snake_case() -> None: + entry = valid_entry() + entry["id"] = "Long Hair" + assert validate_document(entry, "knowledge_base_entry") != [] + + +def test_entry_expand_must_reference_snake_case_ids() -> None: + entry = valid_entry() + entry["expand"] = ["Not Snake"] + assert validate_document(entry, "knowledge_base_entry") != [] + + +def test_entry_unknown_field_rejected() -> None: + entry = valid_entry() + entry["colour"] = "brown" + assert validate_document(entry, "knowledge_base_entry") != [] + + +def test_full_entry_with_aliases_and_expand_valid() -> None: + entry = { + "id": "twin_braids", + "aliases": ["twintail braids", "double braids"], + "tags": ["twin braids"], + "category": "hair", + "expand": ["long_hair"], + "deprecated": False, + "notes": "Composite hairstyle.", + } + assert validate_document(entry, "knowledge_base_entry") == [] + + +# --- worked example file --------------------------------------------------- + + +def load_example_entries() -> list[dict]: + return json.loads(EXAMPLE_FILE.read_text(encoding="utf-8")) + + +def test_example_file_is_an_array_of_entries() -> None: + entries = load_example_entries() + assert isinstance(entries, list) + assert len(entries) >= 2 + + +def test_every_example_entry_validates() -> None: + for entry in load_example_entries(): + assert validate_document(entry, "knowledge_base_entry") == [] + + +def test_example_demonstrates_alias_and_expansion() -> None: + entries = load_example_entries() + assert any(entry.get("aliases") for entry in entries), "no entry demonstrates an alias" + assert any(entry.get("expand") for entry in entries), "no entry demonstrates an expansion" + + +def test_example_expand_targets_exist_in_file() -> None: + entries = load_example_entries() + ids = {entry["id"] for entry in entries} + for entry in entries: + for target in entry.get("expand", []): + assert target in ids, f"expand target '{target}' missing from example"