feat(schemas): add JSON Schemas and validation helper for inter-stage documents (#2) (#46)

Define the nine JSON Schemas (draft 2020-12) exchanged between compiler stages
per MASTER_SPEC §10–§11, plus a reusable validation helper.

Schemas (schemas/json/, cross-referenced by $id):
- scene, character, concept, interaction, metadata, knowledge_base_entry,
  resolved_tag, category_map, prompt_output
- Scene requires all seven sections (collections may be empty); Concept is a
  bare string or an object with required name; Character has an integer id plus
  seven concept arrays; Interaction has integer participants and a concept.
- Each schema carries a major.minor version; unknown fields rejected by default.

Helper (schemas/validation.py):
- validate_document(doc, schema_name, *, allow_unknown=False) -> list of
  structured ValidationIssue (empty = valid); collects all errors.
- Loads all schemas into a referencing registry so cross-schema $ref resolves.
- allow_unknown relaxes additionalProperties per config without duplicating
  schemas (schema-valued additionalProperties, e.g. Category Map, is preserved).

Tests: valid + several invalid documents per schema, unknown-field toggle,
unknown-schema error (34 assertions). Bumps jsonschema to >=4.18 for referencing.

Co-authored-by: kunail0804 <kunail0804@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Crespin Alexandre
2026-07-07 18:52:32 +02:00
committed by GitHub
co-authored by kunail0804 Claude Opus 4.8
parent 72a60aa28b
commit 03dd578435
12 changed files with 552 additions and 1 deletions
+1 -1
View File
@@ -14,7 +14,7 @@ authors = [{ name = "kunail0804" }]
# Runtime dependencies are kept minimal; prefer the standard library (MASTER_SPEC §25.9).
# jsonschema is the single validation dependency, used by the schema layer (issue #2 onward).
dependencies = [
"jsonschema>=4.0",
"jsonschema>=4.18",
]
[project.optional-dependencies]
+12
View File
@@ -0,0 +1,12 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "category_map",
"version": "1.0",
"title": "Category Map",
"description": "Maps each category name to an ordered array of Resolved Tags. MASTER_SPEC §9.9 / §11.8. Category completeness is guaranteed by the Category Splitter, not this schema; missing categories are empty arrays.",
"type": "object",
"additionalProperties": {
"type": "array",
"items": { "$ref": "resolved_tag" }
}
}
+29
View File
@@ -0,0 +1,29 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "character",
"version": "1.0",
"title": "Character",
"description": "A human, creature, or identifiable subject. MASTER_SPEC §9.2 / §10.2 / §11.2. All fields are required; concept arrays MAY be empty.",
"type": "object",
"required": [
"id",
"identity",
"appearance",
"clothing",
"accessories",
"pose",
"expression",
"actions"
],
"additionalProperties": false,
"properties": {
"id": { "type": "integer" },
"identity": { "type": "array", "items": { "$ref": "concept" } },
"appearance": { "type": "array", "items": { "$ref": "concept" } },
"clothing": { "type": "array", "items": { "$ref": "concept" } },
"accessories": { "type": "array", "items": { "$ref": "concept" } },
"pose": { "type": "array", "items": { "$ref": "concept" } },
"expression": { "type": "array", "items": { "$ref": "concept" } },
"actions": { "type": "array", "items": { "$ref": "concept" } }
}
}
+21
View File
@@ -0,0 +1,21 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "concept",
"version": "1.0",
"title": "Concept",
"description": "A semantic concept. MASTER_SPEC §9.3 / §11.3. Serialized either as a bare string (equivalent to {\"name\": <string>}) or as an object.",
"oneOf": [
{ "type": "string" },
{
"type": "object",
"required": ["name"],
"additionalProperties": false,
"properties": {
"name": { "type": "string" },
"category": { "type": "string" },
"source": { "type": "string" },
"metadata": { "type": "object" }
}
}
]
}
+14
View File
@@ -0,0 +1,14 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "interaction",
"version": "1.0",
"title": "Interaction",
"description": "A relationship between two or more characters. MASTER_SPEC §9.4 / §11.4. participants reference existing Character ids.",
"type": "object",
"required": ["participants", "concept"],
"additionalProperties": false,
"properties": {
"participants": { "type": "array", "items": { "type": "integer" } },
"concept": { "type": "string" }
}
}
@@ -0,0 +1,19 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "knowledge_base_entry",
"version": "1.0",
"title": "Knowledge Base Entry",
"description": "A canonical Knowledge Base concept. MASTER_SPEC §11.6 / §15.5.",
"type": "object",
"required": ["id", "tags", "category"],
"additionalProperties": false,
"properties": {
"id": { "type": "string", "pattern": "^[a-z0-9]+(_[a-z0-9]+)*$" },
"aliases": { "type": "array", "items": { "type": "string" } },
"tags": { "type": "array", "items": { "type": "string" } },
"category": { "type": "string" },
"expand": { "type": "array", "items": { "type": "string" } },
"deprecated": { "type": "boolean" },
"notes": { "type": "string" }
}
}
+16
View File
@@ -0,0 +1,16 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "metadata",
"version": "1.0",
"title": "Metadata",
"description": "Compiler metadata. MASTER_SPEC §9.7 / §11.5. MUST NOT affect compilation.",
"type": "object",
"required": ["schema_version"],
"additionalProperties": false,
"properties": {
"schema_version": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+$" },
"compiler_version": { "type": "string" },
"language": { "type": "string" },
"warnings": { "type": "array", "items": { "type": "string" } }
}
}
+14
View File
@@ -0,0 +1,14 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "prompt_output",
"version": "1.0",
"title": "Prompt Output",
"description": "One final workflow output. MASTER_SPEC §9.10 / §11.9. value is always a string.",
"type": "object",
"required": ["name", "value"],
"additionalProperties": false,
"properties": {
"name": { "type": "string" },
"value": { "type": "string" }
}
}
+16
View File
@@ -0,0 +1,16 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "resolved_tag",
"version": "1.0",
"title": "Resolved Tag",
"description": "A generated Illustrious tag with full traceability. MASTER_SPEC §9.8 / §11.7.",
"type": "object",
"required": ["tag", "category", "source_concept", "knowledge_base_entry"],
"additionalProperties": false,
"properties": {
"tag": { "type": "string" },
"category": { "type": "string" },
"source_concept": { "type": "string" },
"knowledge_base_entry": { "type": "string" }
}
}
+27
View File
@@ -0,0 +1,27 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "scene",
"version": "1.0",
"title": "Scene",
"description": "The Scene JSON root. MASTER_SPEC §10.1 / §11.1. All sections are required; collections MAY be empty.",
"type": "object",
"required": [
"characters",
"interactions",
"objects",
"environment",
"camera",
"lighting",
"metadata"
],
"additionalProperties": false,
"properties": {
"characters": { "type": "array", "items": { "$ref": "character" } },
"interactions": { "type": "array", "items": { "$ref": "interaction" } },
"objects": { "type": "array", "items": { "$ref": "concept" } },
"environment": { "type": "array", "items": { "$ref": "concept" } },
"camera": { "type": "array", "items": { "$ref": "concept" } },
"lighting": { "type": "array", "items": { "$ref": "concept" } },
"metadata": { "$ref": "metadata" }
}
}
+134
View File
@@ -0,0 +1,134 @@
"""Reusable JSON Schema validation helper (MASTER_SPEC §11).
Validates a document against one of the named inter-stage schemas and returns a
list of structured :class:`ValidationIssue` objects (an empty list means the
document is valid). Every JSON exchanged between compiler stages MUST validate
against its schema before continuing through the pipeline (§11.0).
Schemas live as JSON files in ``schemas/json/`` and reference one another by
``$id`` (e.g. Scene references Character references Concept); they are loaded
into a shared :mod:`referencing` registry so cross-schema ``$ref`` resolves.
Unknown fields are rejected by default (``additionalProperties: false``). Passing
``allow_unknown=True`` relaxes that check per configuration, satisfying the
"(per config) unknown fields" acceptance criterion without duplicating schemas.
"""
from __future__ import annotations
import functools
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from jsonschema import Draft202012Validator
from referencing import Registry, Resource
from referencing.jsonschema import DRAFT202012
SCHEMA_DIR = Path(__file__).parent / "json"
@dataclass(frozen=True)
class ValidationIssue:
"""A single schema violation.
Attributes:
path: JSON pointer (slash-separated) to the offending location; empty
string for a violation at the document root.
message: Human-readable explanation of what failed.
validator: The JSON Schema keyword that produced the error
(e.g. ``"required"``, ``"type"``, ``"additionalProperties"``).
"""
path: str
message: str
validator: str
def _relax_unknown_fields(schema: Any) -> Any:
"""Return a deep copy of ``schema`` with ``additionalProperties: false`` removed.
Only the strict boolean form is relaxed; a schema-valued
``additionalProperties`` (e.g. the Category Map value constraint) is
preserved so value shapes stay enforced.
"""
if isinstance(schema, dict):
return {
key: _relax_unknown_fields(value)
for key, value in schema.items()
if not (key == "additionalProperties" and value is False)
}
if isinstance(schema, list):
return [_relax_unknown_fields(item) for item in schema]
return schema
@functools.cache
def _load_schemas() -> dict[str, dict]:
"""Load every ``*.schema.json`` file, keyed by its ``$id``."""
schemas: dict[str, dict] = {}
for path in sorted(SCHEMA_DIR.glob("*.schema.json")):
schema = json.loads(path.read_text(encoding="utf-8"))
schema_id = schema.get("$id")
if not schema_id:
raise ValueError(f"Schema file {path.name} is missing a required '$id'.")
if schema_id in schemas:
raise ValueError(f"Duplicate schema '$id' '{schema_id}' in {path.name}.")
schemas[schema_id] = schema
return schemas
@functools.cache
def _registry(allow_unknown: bool) -> Registry:
"""Build a referencing registry of all schemas, optionally unknown-field-relaxed."""
resources = []
for schema_id, schema in _load_schemas().items():
contents = _relax_unknown_fields(schema) if allow_unknown else schema
resource = Resource.from_contents(contents, default_specification=DRAFT202012)
resources.append((schema_id, resource))
return Registry().with_resources(resources)
def list_schemas() -> list[str]:
"""Return the sorted names ($id) of all registered schemas."""
return sorted(_load_schemas())
def validate_document(
document: Any,
schema_name: str,
*,
allow_unknown: bool = False,
) -> list[ValidationIssue]:
"""Validate ``document`` against the named schema.
Args:
document: The parsed JSON value to validate.
schema_name: The ``$id`` of the target schema (see :func:`list_schemas`).
allow_unknown: When True, unknown object fields are permitted.
Returns:
A list of :class:`ValidationIssue` objects, ordered deterministically.
An empty list means the document is valid.
Raises:
KeyError: If ``schema_name`` is not a registered schema.
"""
if schema_name not in _load_schemas():
raise KeyError(f"Unknown schema '{schema_name}'. Available schemas: {list_schemas()}.")
registry = _registry(allow_unknown)
schema = registry.contents(schema_name)
validator = Draft202012Validator(schema, registry=registry)
issues = [
ValidationIssue(
path="/".join(str(part) for part in error.absolute_path),
message=error.message,
validator=str(error.validator),
)
for error in validator.iter_errors(document)
]
issues.sort(key=lambda issue: (issue.path, issue.validator, issue.message))
return issues
+249
View File
@@ -0,0 +1,249 @@
"""Tests for the JSON Schema validation helper (issue #2, MASTER_SPEC §11).
Covers, per schema, at least one valid document and several invalid documents
(wrong types, missing required fields, unknown fields), plus the configurable
unknown-field behaviour.
"""
from __future__ import annotations
import pytest
from schemas.validation import ValidationIssue, list_schemas, validate_document
# --- helpers ---------------------------------------------------------------
def valid_metadata() -> dict:
return {"schema_version": "1.0"}
def valid_scene() -> dict:
return {
"characters": [],
"interactions": [],
"objects": [],
"environment": [],
"camera": [],
"lighting": [],
"metadata": valid_metadata(),
}
def valid_character() -> dict:
return {
"id": 0,
"identity": ["female"],
"appearance": [{"name": "long_hair", "category": "hair"}],
"clothing": [],
"accessories": [],
"pose": [],
"expression": ["smile"],
"actions": [],
}
# --- generic behaviour -----------------------------------------------------
def test_valid_scene_passes() -> None:
assert validate_document(valid_scene(), "scene") == []
def test_returns_structured_issue_objects() -> None:
issues = validate_document({"characters": []}, "scene")
assert issues
assert all(isinstance(i, ValidationIssue) for i in issues)
assert all(i.message for i in issues)
def test_unknown_schema_name_raises() -> None:
with pytest.raises(KeyError):
validate_document({}, "does_not_exist")
def test_all_nine_schemas_are_registered() -> None:
expected = {
"scene",
"character",
"concept",
"interaction",
"metadata",
"knowledge_base_entry",
"resolved_tag",
"category_map",
"prompt_output",
}
assert expected.issubset(set(list_schemas()))
# --- scene -----------------------------------------------------------------
def test_scene_missing_required_field_fails() -> None:
doc = valid_scene()
del doc["metadata"]
issues = validate_document(doc, "scene")
assert any("metadata" in i.message or "metadata" in i.path for i in issues)
def test_scene_unknown_field_rejected_by_default() -> None:
doc = valid_scene()
doc["extra"] = 1
assert validate_document(doc, "scene") != []
def test_scene_unknown_field_allowed_when_configured() -> None:
doc = valid_scene()
doc["extra"] = 1
assert validate_document(doc, "scene", allow_unknown=True) == []
def test_scene_with_populated_nested_documents_passes() -> None:
doc = valid_scene()
doc["characters"] = [valid_character()]
doc["interactions"] = [{"participants": [0], "concept": "holding hands"}]
doc["objects"] = ["chair"]
doc["environment"] = ["classroom"]
assert validate_document(doc, "scene") == []
# --- character -------------------------------------------------------------
def test_valid_character_passes() -> None:
assert validate_document(valid_character(), "character") == []
def test_character_id_must_be_integer() -> None:
doc = valid_character()
doc["id"] = "0"
assert validate_document(doc, "character") != []
def test_character_missing_field_fails() -> None:
doc = valid_character()
del doc["identity"]
assert validate_document(doc, "character") != []
# --- concept ---------------------------------------------------------------
def test_concept_bare_string_valid() -> None:
assert validate_document("blue_eyes", "concept") == []
def test_concept_object_valid() -> None:
assert validate_document({"name": "blue_eyes", "category": "eyes"}, "concept") == []
def test_concept_object_without_name_fails() -> None:
assert validate_document({"category": "eyes"}, "concept") != []
def test_concept_unknown_field_rejected_by_default() -> None:
assert validate_document({"name": "x", "bogus": 1}, "concept") != []
def test_concept_unknown_field_allowed_when_configured() -> None:
assert validate_document({"name": "x", "bogus": 1}, "concept", allow_unknown=True) == []
# --- interaction -----------------------------------------------------------
def test_valid_interaction_passes() -> None:
assert validate_document({"participants": [0, 1], "concept": "hug"}, "interaction") == []
def test_interaction_participants_must_be_integers() -> None:
assert validate_document({"participants": ["a"], "concept": "hug"}, "interaction") != []
def test_interaction_missing_concept_fails() -> None:
assert validate_document({"participants": [0]}, "interaction") != []
# --- metadata --------------------------------------------------------------
def test_valid_metadata_passes() -> None:
assert validate_document({"schema_version": "1.0", "warnings": []}, "metadata") == []
def test_metadata_missing_schema_version_fails() -> None:
assert validate_document({"warnings": []}, "metadata") != []
def test_metadata_warnings_must_be_array() -> None:
assert validate_document({"schema_version": "1.0", "warnings": "oops"}, "metadata") != []
# --- knowledge base entry --------------------------------------------------
def valid_kb_entry() -> dict:
return {"id": "blonde_hair", "tags": ["blonde hair"], "category": "hair"}
def test_valid_kb_entry_passes() -> None:
assert validate_document(valid_kb_entry(), "knowledge_base_entry") == []
def test_kb_entry_id_must_be_snake_case() -> None:
doc = valid_kb_entry()
doc["id"] = "Blonde Hair"
assert validate_document(doc, "knowledge_base_entry") != []
def test_kb_entry_category_must_be_string() -> None:
doc = valid_kb_entry()
doc["category"] = 5
assert validate_document(doc, "knowledge_base_entry") != []
# --- resolved tag ----------------------------------------------------------
def valid_resolved_tag() -> dict:
return {
"tag": "blue eyes",
"category": "eyes",
"source_concept": "blue eyes",
"knowledge_base_entry": "blue_eyes",
}
def test_valid_resolved_tag_passes() -> None:
assert validate_document(valid_resolved_tag(), "resolved_tag") == []
def test_resolved_tag_missing_field_fails() -> None:
doc = valid_resolved_tag()
del doc["knowledge_base_entry"]
assert validate_document(doc, "resolved_tag") != []
# --- category map ----------------------------------------------------------
def test_valid_category_map_passes() -> None:
doc = {"eyes": [valid_resolved_tag()], "hair": []}
assert validate_document(doc, "category_map") == []
def test_category_map_values_must_be_resolved_tag_arrays() -> None:
assert validate_document({"eyes": ["not_a_tag_object"]}, "category_map") != []
# --- prompt output ---------------------------------------------------------
def test_valid_prompt_output_passes() -> None:
doc = {"name": "positive", "value": "blue eyes, smile"}
assert validate_document(doc, "prompt_output") == []
def test_prompt_output_value_must_be_string() -> None:
assert validate_document({"name": "positive", "value": 5}, "prompt_output") != []