diff --git a/CHANGELOG.md b/CHANGELOG.md index bafca49..376a4b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ All notable changes to Scene Compiler are documented here. The project follows ### Added +- **Knowledge Base versioning (epic #36).** A Knowledge Base directory can now + ship a `manifest.json` (new `knowledge_base_manifest` schema) recording a + dataset `version`, a deterministic `content_hash`, the `source_vocab` digest, + and an optional `entry_schema_version`. Workflows can pin a version via the new + `resolver.knowledge_base_version` config option (Configuration node + Knowledge + Base Loader node `version` input); an unavailable version fails with a clear + error. Older datasets are adapted to the current entry schema **in memory** at + load time (pinned files are never rewritten); versions too old to adapt error + clearly. A missing manifest defaults to an implicit `v1`, so existing installs + load unchanged. - **Concept fidelity (epic #86).** The compiler no longer silently drops meaningful parts of the input: - The Resolver now recovers leading modifiers the head-noun reduction drops: diff --git a/compiler/common/config.py b/compiler/common/config.py index 6d2f968..0b9da7b 100644 --- a/compiler/common/config.py +++ b/compiler/common/config.py @@ -65,7 +65,12 @@ class AnalyzerConfig: @dataclass(frozen=True) class ResolverConfig: - """Resolver options (§23.2).""" + """Resolver options (§23.2). + + ``knowledge_base_version`` pins the Knowledge Base dataset version to load + (matched against the dataset manifest, epic #36). Unset (``None``) means "use + whatever is on the configured path", preserving the pre-versioning behaviour. + """ knowledge_base: str = "knowledge_base/" strict_mode: bool = True @@ -73,9 +78,10 @@ class ResolverConfig: expansion_enabled: bool = True max_expansion_depth: int = 8 include_nsfw: bool = False + knowledge_base_version: str | None = None def to_json(self) -> dict[str, Any]: - return { + result: dict[str, Any] = { "knowledge_base": self.knowledge_base, "strict_mode": self.strict_mode, "allow_aliases": self.allow_aliases, @@ -83,6 +89,9 @@ class ResolverConfig: "max_expansion_depth": self.max_expansion_depth, "include_nsfw": self.include_nsfw, } + if self.knowledge_base_version is not None: + result["knowledge_base_version"] = self.knowledge_base_version + return result @dataclass(frozen=True) diff --git a/compiler/common/kb_manifest.py b/compiler/common/kb_manifest.py new file mode 100644 index 0000000..4488527 --- /dev/null +++ b/compiler/common/kb_manifest.py @@ -0,0 +1,132 @@ +"""Knowledge Base dataset manifest (MASTER_SPEC §30.2, epic #36). + +A Knowledge Base directory may ship a ``manifest.json`` that pins the dataset's +``version`` so a workflow can select and reproduce a specific Knowledge Base +build. The manifest is *data about the dataset*, not an entry file, so the loader +skips it when reading entries. + +The manifest is deterministic: its ``content_hash`` is a stable digest of every +entry file (sorted by name), and ``source_vocab`` is the digest of the committed +Danbooru vocabulary snapshot the generated entries came from. Regenerating an +unchanged Knowledge Base therefore produces a byte-identical manifest — there is +no wall-clock build date. A directory without a manifest defaults to the implicit +``v1`` version so existing installs keep loading unchanged. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +MANIFEST_FILENAME = "manifest.json" + +# The manifest document's own schema version (bumped if the manifest shape +# changes), distinct from the dataset ``version`` it records. +MANIFEST_SCHEMA_VERSION = "1.0" + +# The version assumed when a Knowledge Base ships no manifest (backward compat). +IMPLICIT_VERSION = "1.0.0" + + +@dataclass(frozen=True) +class KnowledgeBaseManifest: + """The parsed manifest of a Knowledge Base dataset.""" + + version: str + content_hash: str + source_vocab: str | None = None + entry_schema_version: str | None = None + manifest_schema_version: str = MANIFEST_SCHEMA_VERSION + + def to_json(self) -> dict[str, Any]: + result: dict[str, Any] = { + "manifest_schema_version": self.manifest_schema_version, + "version": self.version, + "content_hash": self.content_hash, + } + if self.source_vocab is not None: + result["source_vocab"] = self.source_vocab + if self.entry_schema_version is not None: + result["entry_schema_version"] = self.entry_schema_version + return result + + +def _entry_files(directory: Path) -> list[Path]: + """Every Knowledge Base entry file (``*.json`` except the manifest), sorted.""" + return [p for p in sorted(directory.glob("*.json")) if p.name != MANIFEST_FILENAME] + + +def compute_content_hash(directory: str | Path) -> str: + """Return a deterministic ``sha256:`` digest of all entry files in a directory. + + The digest covers each entry file's name and raw bytes in sorted order, so it + is stable across runs and changes only when an entry file's content changes. + """ + directory = Path(directory) + digest = hashlib.sha256() + for path in _entry_files(directory): + digest.update(path.name.encode("utf-8")) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return f"sha256:{digest.hexdigest()}" + + +def _hash_file(path: Path) -> str | None: + if not path.is_file(): + return None + return f"sha256:{hashlib.sha256(path.read_bytes()).hexdigest()}" + + +def build_manifest( + directory: str | Path, version: str, source_vocab_path: str | Path +) -> KnowledgeBaseManifest: + """Build a manifest for a directory, hashing its entries and the source vocab.""" + directory = Path(directory) + return KnowledgeBaseManifest( + version=version, + content_hash=compute_content_hash(directory), + source_vocab=_hash_file(Path(source_vocab_path)), + ) + + +def write_manifest( + directory: str | Path, version: str, source_vocab_path: str | Path +) -> KnowledgeBaseManifest: + """Stamp ``manifest.json`` into ``directory`` deterministically and return it.""" + directory = Path(directory) + manifest = build_manifest(directory, version, source_vocab_path) + path = directory / MANIFEST_FILENAME + path.write_text( + json.dumps(manifest.to_json(), indent=2, ensure_ascii=False, sort_keys=True) + "\n", + encoding="utf-8", + ) + return manifest + + +def load_manifest(directory: str | Path) -> KnowledgeBaseManifest: + """Load the manifest of a Knowledge Base directory. + + A missing manifest yields the implicit ``v1`` version (with the content hash + computed on the fly), so a directory without a manifest still reports a + coherent version and loads unchanged. + """ + directory = Path(directory) + path = directory / MANIFEST_FILENAME + if not path.is_file(): + return KnowledgeBaseManifest( + version=IMPLICIT_VERSION, + content_hash=compute_content_hash(directory), + source_vocab=None, + ) + data = json.loads(path.read_text(encoding="utf-8")) + return KnowledgeBaseManifest( + version=data["version"], + content_hash=data["content_hash"], + source_vocab=data.get("source_vocab"), + entry_schema_version=data.get("entry_schema_version"), + manifest_schema_version=data.get("manifest_schema_version", MANIFEST_SCHEMA_VERSION), + ) diff --git a/compiler/common/kb_migration.py b/compiler/common/kb_migration.py new file mode 100644 index 0000000..cdb9536 --- /dev/null +++ b/compiler/common/kb_migration.py @@ -0,0 +1,105 @@ +"""Cross-version Knowledge Base entry migration (MASTER_SPEC §30.2, epic #36). + +A pinned Knowledge Base may have been produced against an older entry-schema +version than the running code. To keep pinned datasets loadable as the format +evolves, entries are *adapted in memory at load time* — the on-disk (pinned) +files are never rewritten. + +Compatibility policy +-------------------- +- The current entry-schema version is :data:`CURRENT_ENTRY_SCHEMA_VERSION` (it + matches ``schemas/json/knowledge_base_entry.schema.json``'s ``version``). +- A dataset declares its entry-schema version via the manifest's + ``entry_schema_version`` field. A dataset with **no** manifest, or a manifest + without that field, is assumed to already be at the current version — so the + shipped reference Knowledge Base loads with no adaptation. +- Each supported older version has a forward adapter that upgrades a raw entry + dict to the next version. Adapters are applied in sequence up to the current + version, then the entry is validated and materialised as usual. +- Versions older than :data:`MIN_SUPPORTED_ENTRY_SCHEMA_VERSION` cannot be adapted + and raise a clear error rather than loading a misinterpreted dataset. + +Adapters must be pure and deterministic: the same pinned dataset always adapts to +the same in-memory entries. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +# Keep in sync with schemas/json/knowledge_base_entry.schema.json "version". +CURRENT_ENTRY_SCHEMA_VERSION = "1.1" +MIN_SUPPORTED_ENTRY_SCHEMA_VERSION = "1.0" + + +class UnsupportedKnowledgeBaseVersion(Exception): + """Raised when a dataset's entry-schema version is too old to adapt.""" + + +def _version_tuple(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + +def _adapt_1_0_to_1_1(entry: dict[str, Any]) -> dict[str, Any]: + """Entry schema 1.0 → 1.1. + + In 1.0 the explicit/general distinction was carried by a boolean ``nsfw`` + field; 1.1 replaced it with the ``rating`` enum (``general``/``explicit``). + Translate the old field so a 1.0 dataset keeps its NSFW gating. + """ + if "nsfw" in entry: + entry.setdefault("rating", "explicit" if entry["nsfw"] else "general") + del entry["nsfw"] + return entry + + +# Forward adapters, keyed by the version they upgrade FROM. Each returns the +# entry at the next version. Extend this chain as the entry schema evolves. +_ADAPTERS: dict[str, tuple[str, Callable[[dict[str, Any]], dict[str, Any]]]] = { + "1.0": ("1.1", _adapt_1_0_to_1_1), +} + + +def adapt_entries( + entries: list[Mapping[str, Any]], source_version: str | None +) -> list[dict[str, Any]]: + """Adapt raw entry dicts from ``source_version`` to the current entry schema. + + Args: + entries: The raw (parsed JSON) entries as loaded from disk. + source_version: The dataset's entry-schema version, or ``None`` to assume + the current version (no adaptation). + + Returns: + New entry dicts at the current entry-schema version. Input dicts are not + mutated. + + Raises: + UnsupportedKnowledgeBaseVersion: If ``source_version`` is older than the + minimum supported version, or references an unknown version with no + adapter path to the current one. + """ + version = source_version or CURRENT_ENTRY_SCHEMA_VERSION + adapted = [dict(entry) for entry in entries] + if version == CURRENT_ENTRY_SCHEMA_VERSION: + return adapted + + if _version_tuple(version) < _version_tuple(MIN_SUPPORTED_ENTRY_SCHEMA_VERSION): + raise UnsupportedKnowledgeBaseVersion( + f"Knowledge Base entry-schema version '{version}' is older than the " + f"minimum supported '{MIN_SUPPORTED_ENTRY_SCHEMA_VERSION}' and cannot be adapted." + ) + + # Walk the adapter chain forward to the current version. + while version != CURRENT_ENTRY_SCHEMA_VERSION: + step = _ADAPTERS.get(version) + if step is None: + raise UnsupportedKnowledgeBaseVersion( + f"No migration path from Knowledge Base entry-schema version '{version}' " + f"to '{CURRENT_ENTRY_SCHEMA_VERSION}'." + ) + next_version, adapter = step + adapted = [adapter(entry) for entry in adapted] + version = next_version + return adapted diff --git a/compiler/common/knowledge_base.py b/compiler/common/knowledge_base.py index 1f52899..fdb5351 100644 --- a/compiler/common/knowledge_base.py +++ b/compiler/common/knowledge_base.py @@ -21,6 +21,8 @@ from pathlib import Path from types import MappingProxyType from typing import Any +from compiler.common.kb_manifest import IMPLICIT_VERSION, MANIFEST_FILENAME, load_manifest +from compiler.common.kb_migration import UnsupportedKnowledgeBaseVersion, adapt_entries from compiler.common.kb_validation import validate_knowledge_base from compiler.common.log import StructuredLogger from compiler.common.result import Message, Severity @@ -65,7 +67,7 @@ class KnowledgeBaseEntry: class KnowledgeBase: """An immutable, validated Knowledge Base with id and alias lookups.""" - def __init__(self, entries: list[KnowledgeBaseEntry]) -> None: + def __init__(self, entries: list[KnowledgeBaseEntry], version: str = IMPLICIT_VERSION) -> None: by_id: dict[str, KnowledgeBaseEntry] = {} alias_to_id: dict[str, str] = {} for entry in entries: @@ -74,6 +76,12 @@ class KnowledgeBase: alias_to_id[alias] = entry.id self._by_id: Mapping[str, KnowledgeBaseEntry] = MappingProxyType(by_id) self._alias_to_id: Mapping[str, str] = MappingProxyType(alias_to_id) + self._version = version + + @property + def version(self) -> str: + """The dataset version from the manifest (implicit ``v1`` when absent).""" + return self._version @property def by_id(self) -> Mapping[str, KnowledgeBaseEntry]: @@ -107,6 +115,8 @@ def _read_entries(directory: Path) -> tuple[list[dict], list[Message]]: entries: list[dict] = [] problems: list[Message] = [] for path in sorted(directory.glob("*.json")): + if path.name == MANIFEST_FILENAME: + continue # the manifest is dataset metadata, not an entry array try: data = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: @@ -134,15 +144,59 @@ def _load_problem(description: str, source: str) -> Message: def load_knowledge_base( directory: str | Path, logger: StructuredLogger | None = None, + requested_version: str | None = None, ) -> KnowledgeBase: """Load and validate a Knowledge Base directory into an in-memory index. + Args: + directory: The Knowledge Base directory to load. + logger: Optional structured logger. + requested_version: If set, the dataset manifest ``version`` must match it + exactly; otherwise a Fatal ``SC0004`` is raised (the requested version + is unavailable at this path). ``None`` loads whatever is on the path. + Raises: - KnowledgeBaseError: If any file cannot be read or the Knowledge Base - fails validation. Nothing is loaded in that case. + KnowledgeBaseError: If any file cannot be read, the Knowledge Base fails + validation, or the requested version is unavailable. Nothing is loaded + in that case. """ directory = Path(directory) + manifest = load_manifest(directory) + if requested_version is not None and manifest.version != requested_version: + raise KnowledgeBaseError( + Message( + code="SC0004", + severity=Severity.FATAL, + title="Knowledge Base Load Failure", + description=( + f"Requested Knowledge Base version '{requested_version}' is not " + f"available at '{directory}' (found version '{manifest.version}')." + ), + context={ + "id": str(directory), + "requested_version": requested_version, + "available_version": manifest.version, + }, + ), + findings=[], + ) raw_entries, problems = _read_entries(directory) + try: + raw_entries = adapt_entries(raw_entries, manifest.entry_schema_version) + except UnsupportedKnowledgeBaseVersion as exc: + raise KnowledgeBaseError( + Message( + code="SC0004", + severity=Severity.FATAL, + title="Knowledge Base Load Failure", + description=str(exc), + context={ + "id": str(directory), + "entry_schema_version": manifest.entry_schema_version, + }, + ), + findings=[], + ) from exc findings = problems + validate_knowledge_base(raw_entries) if findings: raise KnowledgeBaseError( @@ -160,24 +214,37 @@ def load_knowledge_base( ) entries = [KnowledgeBaseEntry.from_json(entry) for entry in raw_entries] - kb = KnowledgeBase(entries) + kb = KnowledgeBase(entries, version=manifest.version) if logger is not None: - logger.basic("knowledge_base_loaded", directory=str(directory), entries=len(kb)) + logger.basic( + "knowledge_base_loaded", + directory=str(directory), + entries=len(kb), + version=kb.version, + ) return kb class KnowledgeBaseLoader: """Loads a Knowledge Base once and caches it; reload is explicit and safe.""" - def __init__(self, directory: str | Path, logger: StructuredLogger | None = None) -> None: + def __init__( + self, + directory: str | Path, + logger: StructuredLogger | None = None, + requested_version: str | None = None, + ) -> None: self._directory = Path(directory) self._logger = logger + self._requested_version = requested_version self._cache: KnowledgeBase | None = None def get(self) -> KnowledgeBase: """Return the cached Knowledge Base, loading it on first use.""" if self._cache is None: - self._cache = load_knowledge_base(self._directory, self._logger) + self._cache = load_knowledge_base( + self._directory, self._logger, self._requested_version + ) return self._cache def reload(self) -> KnowledgeBase: @@ -186,6 +253,6 @@ class KnowledgeBaseLoader: On failure the previously cached Knowledge Base is left untouched (the load raises before the cache is replaced). """ - kb = load_knowledge_base(self._directory, self._logger) + kb = load_knowledge_base(self._directory, self._logger, self._requested_version) self._cache = kb return kb diff --git a/config/default_config.json b/config/default_config.json index 1bdd8d3..b787fac 100644 --- a/config/default_config.json +++ b/config/default_config.json @@ -13,7 +13,8 @@ "allow_aliases": true, "expansion_enabled": true, "max_expansion_depth": 8, - "include_nsfw": false + "include_nsfw": false, + "knowledge_base_version": null }, "validator": { "allow_unknown_fields": false diff --git a/knowledge_base/manifest.json b/knowledge_base/manifest.json new file mode 100644 index 0000000..3622a60 --- /dev/null +++ b/knowledge_base/manifest.json @@ -0,0 +1,6 @@ +{ + "content_hash": "sha256:a9f132307076670a20999a5be768537c2a6b93f7a0afdd41ce46a3da08175b68", + "manifest_schema_version": "1.0", + "source_vocab": "sha256:d9238cecb70736aa81e1297b174d2ed0c4555c0b214651d33c955d9e5ce0e88b", + "version": "1.0.0" +} diff --git a/nodes/configuration_node.py b/nodes/configuration_node.py index efc0872..60b32b8 100644 --- a/nodes/configuration_node.py +++ b/nodes/configuration_node.py @@ -48,8 +48,11 @@ class ConfigurationNode: "debug_enabled": ("BOOLEAN", {"default": False}), "debug_level": (_DEBUG_LEVELS, {"default": "basic"}), }, + # Appended at the END so saved workflows keep their positional widget + # values (ComfyUI stores widget values positionally). "optional": { "analyzer_system_prompt": ("STRING", {"multiline": True, "default": ""}), + "resolver_knowledge_base_version": ("STRING", {"default": ""}), }, } @@ -72,6 +75,7 @@ class ConfigurationNode: debug_enabled: bool, debug_level: str, analyzer_system_prompt: str = "", + resolver_knowledge_base_version: str = "", ) -> tuple[Any, str]: analyzer: dict[str, Any] = { "model": analyzer_model, @@ -91,6 +95,11 @@ class ConfigurationNode: "expansion_enabled": resolver_expansion_enabled, "max_expansion_depth": resolver_max_expansion_depth, "include_nsfw": resolver_include_nsfw, + **( + {"knowledge_base_version": resolver_knowledge_base_version} + if resolver_knowledge_base_version + else {} + ), }, "validator": {"allow_unknown_fields": validator_allow_unknown_fields}, "prompt_builder": { diff --git a/nodes/knowledge_base_loader_node.py b/nodes/knowledge_base_loader_node.py index cca97b6..efa51a7 100644 --- a/nodes/knowledge_base_loader_node.py +++ b/nodes/knowledge_base_loader_node.py @@ -40,13 +40,19 @@ class KnowledgeBaseLoaderNode: def INPUT_TYPES(cls) -> dict[str, Any]: return { "required": {"path": ("STRING", {"default": "knowledge_base/"})}, - "optional": {"reload": ("INT", {"default": 0, "min": 0})}, + # New widgets are appended at the END so saved workflows keep their + # positional widget values (ComfyUI stores widgets positionally). + "optional": { + "reload": ("INT", {"default": 0, "min": 0}), + "version": ("STRING", {"default": ""}), + }, } - def run(self, path: str, reload: int = 0) -> tuple[Any, str, str, str]: + def run(self, path: str, reload: int = 0, version: str = "") -> tuple[Any, str, str, str]: resolved = _resolve_path(path) + requested_version = version or None try: - knowledge_base = load_knowledge_base(resolved) + knowledge_base = load_knowledge_base(resolved, requested_version=requested_version) except KnowledgeBaseError as error: errors = format_messages([error.message, *error.findings]) return (None, "", errors, f"Failed to load Knowledge Base from: {resolved}") diff --git a/schemas/json/configuration.schema.json b/schemas/json/configuration.schema.json index 9a89dc9..5101873 100644 --- a/schemas/json/configuration.schema.json +++ b/schemas/json/configuration.schema.json @@ -29,7 +29,8 @@ "allow_aliases": { "type": "boolean" }, "expansion_enabled": { "type": "boolean" }, "max_expansion_depth": { "type": "integer", "minimum": 1 }, - "include_nsfw": { "type": "boolean" } + "include_nsfw": { "type": "boolean" }, + "knowledge_base_version": { "type": ["string", "null"] } } }, "validator": { diff --git a/schemas/json/knowledge_base_manifest.schema.json b/schemas/json/knowledge_base_manifest.schema.json new file mode 100644 index 0000000..401fc43 --- /dev/null +++ b/schemas/json/knowledge_base_manifest.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "knowledge_base_manifest", + "version": "1.0", + "title": "Knowledge Base Manifest", + "description": "Metadata for a versioned Knowledge Base dataset. MASTER_SPEC §30.2 (epic #36). Deterministic: content_hash and source_vocab are content digests, so regenerating an unchanged dataset yields a byte-identical manifest.", + "type": "object", + "required": ["manifest_schema_version", "version", "content_hash"], + "additionalProperties": false, + "properties": { + "manifest_schema_version": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+$" }, + "version": { "type": "string", "pattern": "^[0-9]+(\\.[0-9]+)*$" }, + "content_hash": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, + "source_vocab": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, + "entry_schema_version": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+$" } + } +} diff --git a/scripts/generate_kb_from_vocab.py b/scripts/generate_kb_from_vocab.py index 82dc969..811d331 100644 --- a/scripts/generate_kb_from_vocab.py +++ b/scripts/generate_kb_from_vocab.py @@ -20,13 +20,22 @@ Usage: from __future__ import annotations import json +import sys from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from compiler.common.kb_manifest import write_manifest # noqa: E402 + _REPO_ROOT = Path(__file__).resolve().parent.parent _VOCAB = _REPO_ROOT / "data" / "danbooru_vocab.txt" _KB_DIR = _REPO_ROOT / "knowledge_base" _GEN_PREFIX = "gen_" +# The dataset version stamped into the manifest on each rebuild. Bump this when a +# regeneration changes the dataset in a way workflows should be able to pin. +_DATASET_VERSION = "1.0.0" + # Raffle category -> one of the 19 canonical SceneCompiler categories. _CATEGORY_MAP = { "clothes_and_accessories": "clothing", @@ -138,8 +147,11 @@ def generate() -> dict[str, int]: out.write_text(json.dumps(entries, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") counts[sc_category] = len(entries) + manifest = write_manifest(_KB_DIR, _DATASET_VERSION, _VOCAB) + total = sum(counts.values()) print(f"generated {total} entries across {len(counts)} category files") + print(f" stamped manifest version {manifest.version} ({manifest.content_hash[:19]}...)") print(f" skipped {skipped_reserved} (already hand-curated), {skipped_unmapped} (unmapped)") for cat, n in sorted(counts.items(), key=lambda x: -x[1]): print(f" {n:6} gen_{cat}.json") diff --git a/tests/test_config.py b/tests/test_config.py index 97cba31..3f6c8c4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -81,6 +81,18 @@ def test_partial_config_fills_defaults() -> None: assert c.resolver.max_expansion_depth == 8 +def test_knowledge_base_version_defaults_to_none_and_is_omitted() -> None: + c = Config() + assert c.resolver.knowledge_base_version is None + assert "knowledge_base_version" not in c.resolver.to_json() # omitted when unset + + +def test_knowledge_base_version_roundtrips_when_set() -> None: + c = Config.from_json({"resolver": {"knowledge_base_version": "2.0.0"}}) + assert c.resolver.knowledge_base_version == "2.0.0" + assert c.to_json()["resolver"]["knowledge_base_version"] == "2.0.0" + + def test_load_config_reads_file(tmp_path) -> None: path = tmp_path / "config.json" path.write_text(json.dumps(EXAMPLE_CONFIG), encoding="utf-8") diff --git a/tests/test_kb_manifest.py b/tests/test_kb_manifest.py new file mode 100644 index 0000000..c690db2 --- /dev/null +++ b/tests/test_kb_manifest.py @@ -0,0 +1,109 @@ +"""Tests for the Knowledge Base dataset manifest (issue #127, epic #36).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from compiler.common.kb_manifest import ( + IMPLICIT_VERSION, + MANIFEST_FILENAME, + compute_content_hash, + load_manifest, + write_manifest, +) +from compiler.common.knowledge_base import load_knowledge_base +from schemas.validation import validate_document + +REPO_ROOT = Path(__file__).resolve().parent.parent +KB_DIR = REPO_ROOT / "knowledge_base" + + +# --- shipped reference manifest -------------------------------------------- + + +def test_reference_manifest_exists_and_validates() -> None: + data = json.loads((KB_DIR / MANIFEST_FILENAME).read_text(encoding="utf-8")) + assert validate_document(data, "knowledge_base_manifest") == [] + assert data["version"] == "1.0.0" + + +def test_loader_reads_manifest_version() -> None: + kb = load_knowledge_base(KB_DIR) + assert kb.version == "1.0.0" + + +def test_manifest_content_hash_matches_current_entries() -> None: + # The committed manifest is a faithful, deterministic stamp of the dataset. + manifest = load_manifest(KB_DIR) + assert manifest.content_hash == compute_content_hash(KB_DIR) + + +# --- backward compatibility ------------------------------------------------- + + +def test_missing_manifest_defaults_to_implicit_v1(tmp_path) -> None: + (tmp_path / "hair.json").write_text( + '[{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]', + encoding="utf-8", + ) + assert load_manifest(tmp_path).version == IMPLICIT_VERSION + assert load_knowledge_base(tmp_path).version == IMPLICIT_VERSION + + +# --- determinism ------------------------------------------------------------ + + +def test_write_manifest_is_deterministic(tmp_path) -> None: + vocab = tmp_path / "vocab.txt" + vocab.write_text("clothing\tshirt\n", encoding="utf-8") + (tmp_path / "clothing.json").write_text( + '[{"id": "shirt", "tags": ["shirt"], "category": "clothing"}]', + encoding="utf-8", + ) + first = write_manifest(tmp_path, "2.0.0", vocab) + first_bytes = (tmp_path / MANIFEST_FILENAME).read_bytes() + second = write_manifest(tmp_path, "2.0.0", vocab) + second_bytes = (tmp_path / MANIFEST_FILENAME).read_bytes() + assert first == second + assert first_bytes == second_bytes # byte-identical regeneration + + +def test_requesting_available_version_loads(tmp_path) -> None: + (tmp_path / "hair.json").write_text( + '[{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]', + encoding="utf-8", + ) + write_manifest(tmp_path, "3.1.0", tmp_path / "missing_vocab.txt") + kb = load_knowledge_base(tmp_path, requested_version="3.1.0") + assert kb.version == "3.1.0" + assert kb.get("long_hair") is not None + + +def test_requesting_unavailable_version_errors_clearly(tmp_path) -> None: + from compiler.common.knowledge_base import KnowledgeBaseError + + (tmp_path / "hair.json").write_text( + '[{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]', + encoding="utf-8", + ) + write_manifest(tmp_path, "3.1.0", tmp_path / "missing_vocab.txt") + try: + load_knowledge_base(tmp_path, requested_version="9.9.9") + except KnowledgeBaseError as error: + assert error.message.code == "SC0004" + assert "9.9.9" in error.message.description + assert error.message.context["available_version"] == "3.1.0" + else: + raise AssertionError("expected KnowledgeBaseError for unavailable version") + + +def test_manifest_is_not_read_as_an_entry_file(tmp_path) -> None: + # A manifest.json in the KB dir must not break loading (it is not an entry array). + (tmp_path / "hair.json").write_text( + '[{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]', + encoding="utf-8", + ) + write_manifest(tmp_path, "1.0.0", tmp_path / "missing_vocab.txt") + kb = load_knowledge_base(tmp_path) + assert len(kb) == 1 # the manifest was skipped, only the one entry loaded diff --git a/tests/test_kb_migration.py b/tests/test_kb_migration.py new file mode 100644 index 0000000..8ae222c --- /dev/null +++ b/tests/test_kb_migration.py @@ -0,0 +1,105 @@ +"""Tests for cross-version Knowledge Base loading (issue #129, epic #36).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from compiler.common.kb_manifest import ( + MANIFEST_FILENAME, + compute_content_hash, + load_manifest, +) +from compiler.common.kb_migration import ( + CURRENT_ENTRY_SCHEMA_VERSION, + UnsupportedKnowledgeBaseVersion, + adapt_entries, +) +from compiler.common.knowledge_base import KnowledgeBaseError, load_knowledge_base + +REPO_ROOT = Path(__file__).resolve().parent.parent +KB_DIR = REPO_ROOT / "knowledge_base" + + +def _write_manifest(directory: Path, entry_schema_version: str) -> None: + (directory / MANIFEST_FILENAME).write_text( + json.dumps( + { + "manifest_schema_version": "1.0", + "version": "1.0.0", + "content_hash": compute_content_hash(directory), + "entry_schema_version": entry_schema_version, + } + ), + encoding="utf-8", + ) + + +# --- adapter unit level ----------------------------------------------------- + + +def test_no_source_version_is_treated_as_current() -> None: + entries = [{"id": "long_hair", "tags": ["long hair"], "category": "hair"}] + assert adapt_entries(entries, None) == entries + + +def test_adapter_1_0_translates_nsfw_to_rating() -> None: + entries = [ + {"id": "a", "tags": ["a"], "category": "body", "nsfw": True}, + {"id": "b", "tags": ["b"], "category": "body", "nsfw": False}, + ] + adapted = adapt_entries(entries, "1.0") + assert adapted[0]["rating"] == "explicit" and "nsfw" not in adapted[0] + assert adapted[1]["rating"] == "general" and "nsfw" not in adapted[1] + assert entries[0]["nsfw"] is True # inputs not mutated + + +def test_too_old_version_raises() -> None: + with pytest.raises(UnsupportedKnowledgeBaseVersion): + adapt_entries([], "0.9") + + +# --- loader level ----------------------------------------------------------- + + +def test_current_reference_kb_loads_without_adaptation() -> None: + assert load_manifest(KB_DIR).entry_schema_version is None + kb = load_knowledge_base(KB_DIR) # loads clean, no migration applied + assert len(kb) > 100 + + +def test_older_dataset_adapts_and_resolves(tmp_path) -> None: + (tmp_path / "body.json").write_text( + json.dumps( + [{"id": "bare_shoulders", "tags": ["bare shoulders"], "category": "body", "nsfw": True}] + ), + encoding="utf-8", + ) + _write_manifest(tmp_path, "1.0") + kb = load_knowledge_base(tmp_path) + entry = kb.get("bare_shoulders") + assert entry is not None + assert entry.rating == "explicit" # adapted from nsfw=true + + +def test_unsupported_old_dataset_errors_clearly(tmp_path) -> None: + (tmp_path / "hair.json").write_text( + json.dumps([{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]), + encoding="utf-8", + ) + _write_manifest(tmp_path, "0.9") + with pytest.raises(KnowledgeBaseError) as exc: + load_knowledge_base(tmp_path) + assert exc.value.message.code == "SC0004" + assert "0.9" in exc.value.message.description + + +def test_current_version_string_matches_entry_schema_file() -> None: + schema = json.loads( + (REPO_ROOT / "schemas" / "json" / "knowledge_base_entry.schema.json").read_text( + encoding="utf-8" + ) + ) + assert schema["version"] == CURRENT_ENTRY_SCHEMA_VERSION