Closes #127, #128, #129. - manifest.json (new knowledge_base_manifest schema) records a dataset version, deterministic content_hash, source_vocab digest, optional entry_schema_version. Loader reads/skips it; regeneration stamps it deterministically. - resolver.knowledge_base_version config option (Configuration + KB Loader nodes) pins a version; unavailable version -> clear SC0004. - In-memory entry migration adapts older datasets to the current entry schema at load time without rewriting pinned files; too-old versions error clearly. Co-authored-by: kunail0804 <kunail0804@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Crespin Alexandre
co-authored by
kunail0804
Claude Opus 4.8
kunail0804
Claude Opus 4.8
parent
af775d48c0
commit
bb846c8aa0
@@ -7,6 +7,16 @@ All notable changes to Scene Compiler are documented here. The project follows
|
||||
|
||||
### Added
|
||||
|
||||
- **Knowledge Base versioning (epic #36).** A Knowledge Base directory can now
|
||||
ship a `manifest.json` (new `knowledge_base_manifest` schema) recording a
|
||||
dataset `version`, a deterministic `content_hash`, the `source_vocab` digest,
|
||||
and an optional `entry_schema_version`. Workflows can pin a version via the new
|
||||
`resolver.knowledge_base_version` config option (Configuration node + Knowledge
|
||||
Base Loader node `version` input); an unavailable version fails with a clear
|
||||
error. Older datasets are adapted to the current entry schema **in memory** at
|
||||
load time (pinned files are never rewritten); versions too old to adapt error
|
||||
clearly. A missing manifest defaults to an implicit `v1`, so existing installs
|
||||
load unchanged.
|
||||
- **Concept fidelity (epic #86).** The compiler no longer silently drops
|
||||
meaningful parts of the input:
|
||||
- The Resolver now recovers leading modifiers the head-noun reduction drops:
|
||||
|
||||
@@ -65,7 +65,12 @@ class AnalyzerConfig:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolverConfig:
|
||||
"""Resolver options (§23.2)."""
|
||||
"""Resolver options (§23.2).
|
||||
|
||||
``knowledge_base_version`` pins the Knowledge Base dataset version to load
|
||||
(matched against the dataset manifest, epic #36). Unset (``None``) means "use
|
||||
whatever is on the configured path", preserving the pre-versioning behaviour.
|
||||
"""
|
||||
|
||||
knowledge_base: str = "knowledge_base/"
|
||||
strict_mode: bool = True
|
||||
@@ -73,9 +78,10 @@ class ResolverConfig:
|
||||
expansion_enabled: bool = True
|
||||
max_expansion_depth: int = 8
|
||||
include_nsfw: bool = False
|
||||
knowledge_base_version: str | None = None
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
result: dict[str, Any] = {
|
||||
"knowledge_base": self.knowledge_base,
|
||||
"strict_mode": self.strict_mode,
|
||||
"allow_aliases": self.allow_aliases,
|
||||
@@ -83,6 +89,9 @@ class ResolverConfig:
|
||||
"max_expansion_depth": self.max_expansion_depth,
|
||||
"include_nsfw": self.include_nsfw,
|
||||
}
|
||||
if self.knowledge_base_version is not None:
|
||||
result["knowledge_base_version"] = self.knowledge_base_version
|
||||
return result
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Knowledge Base dataset manifest (MASTER_SPEC §30.2, epic #36).
|
||||
|
||||
A Knowledge Base directory may ship a ``manifest.json`` that pins the dataset's
|
||||
``version`` so a workflow can select and reproduce a specific Knowledge Base
|
||||
build. The manifest is *data about the dataset*, not an entry file, so the loader
|
||||
skips it when reading entries.
|
||||
|
||||
The manifest is deterministic: its ``content_hash`` is a stable digest of every
|
||||
entry file (sorted by name), and ``source_vocab`` is the digest of the committed
|
||||
Danbooru vocabulary snapshot the generated entries came from. Regenerating an
|
||||
unchanged Knowledge Base therefore produces a byte-identical manifest — there is
|
||||
no wall-clock build date. A directory without a manifest defaults to the implicit
|
||||
``v1`` version so existing installs keep loading unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
MANIFEST_FILENAME = "manifest.json"
|
||||
|
||||
# The manifest document's own schema version (bumped if the manifest shape
|
||||
# changes), distinct from the dataset ``version`` it records.
|
||||
MANIFEST_SCHEMA_VERSION = "1.0"
|
||||
|
||||
# The version assumed when a Knowledge Base ships no manifest (backward compat).
|
||||
IMPLICIT_VERSION = "1.0.0"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KnowledgeBaseManifest:
|
||||
"""The parsed manifest of a Knowledge Base dataset."""
|
||||
|
||||
version: str
|
||||
content_hash: str
|
||||
source_vocab: str | None = None
|
||||
entry_schema_version: str | None = None
|
||||
manifest_schema_version: str = MANIFEST_SCHEMA_VERSION
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {
|
||||
"manifest_schema_version": self.manifest_schema_version,
|
||||
"version": self.version,
|
||||
"content_hash": self.content_hash,
|
||||
}
|
||||
if self.source_vocab is not None:
|
||||
result["source_vocab"] = self.source_vocab
|
||||
if self.entry_schema_version is not None:
|
||||
result["entry_schema_version"] = self.entry_schema_version
|
||||
return result
|
||||
|
||||
|
||||
def _entry_files(directory: Path) -> list[Path]:
|
||||
"""Every Knowledge Base entry file (``*.json`` except the manifest), sorted."""
|
||||
return [p for p in sorted(directory.glob("*.json")) if p.name != MANIFEST_FILENAME]
|
||||
|
||||
|
||||
def compute_content_hash(directory: str | Path) -> str:
|
||||
"""Return a deterministic ``sha256:`` digest of all entry files in a directory.
|
||||
|
||||
The digest covers each entry file's name and raw bytes in sorted order, so it
|
||||
is stable across runs and changes only when an entry file's content changes.
|
||||
"""
|
||||
directory = Path(directory)
|
||||
digest = hashlib.sha256()
|
||||
for path in _entry_files(directory):
|
||||
digest.update(path.name.encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(path.read_bytes())
|
||||
digest.update(b"\0")
|
||||
return f"sha256:{digest.hexdigest()}"
|
||||
|
||||
|
||||
def _hash_file(path: Path) -> str | None:
|
||||
if not path.is_file():
|
||||
return None
|
||||
return f"sha256:{hashlib.sha256(path.read_bytes()).hexdigest()}"
|
||||
|
||||
|
||||
def build_manifest(
|
||||
directory: str | Path, version: str, source_vocab_path: str | Path
|
||||
) -> KnowledgeBaseManifest:
|
||||
"""Build a manifest for a directory, hashing its entries and the source vocab."""
|
||||
directory = Path(directory)
|
||||
return KnowledgeBaseManifest(
|
||||
version=version,
|
||||
content_hash=compute_content_hash(directory),
|
||||
source_vocab=_hash_file(Path(source_vocab_path)),
|
||||
)
|
||||
|
||||
|
||||
def write_manifest(
|
||||
directory: str | Path, version: str, source_vocab_path: str | Path
|
||||
) -> KnowledgeBaseManifest:
|
||||
"""Stamp ``manifest.json`` into ``directory`` deterministically and return it."""
|
||||
directory = Path(directory)
|
||||
manifest = build_manifest(directory, version, source_vocab_path)
|
||||
path = directory / MANIFEST_FILENAME
|
||||
path.write_text(
|
||||
json.dumps(manifest.to_json(), indent=2, ensure_ascii=False, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return manifest
|
||||
|
||||
|
||||
def load_manifest(directory: str | Path) -> KnowledgeBaseManifest:
|
||||
"""Load the manifest of a Knowledge Base directory.
|
||||
|
||||
A missing manifest yields the implicit ``v1`` version (with the content hash
|
||||
computed on the fly), so a directory without a manifest still reports a
|
||||
coherent version and loads unchanged.
|
||||
"""
|
||||
directory = Path(directory)
|
||||
path = directory / MANIFEST_FILENAME
|
||||
if not path.is_file():
|
||||
return KnowledgeBaseManifest(
|
||||
version=IMPLICIT_VERSION,
|
||||
content_hash=compute_content_hash(directory),
|
||||
source_vocab=None,
|
||||
)
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return KnowledgeBaseManifest(
|
||||
version=data["version"],
|
||||
content_hash=data["content_hash"],
|
||||
source_vocab=data.get("source_vocab"),
|
||||
entry_schema_version=data.get("entry_schema_version"),
|
||||
manifest_schema_version=data.get("manifest_schema_version", MANIFEST_SCHEMA_VERSION),
|
||||
)
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Cross-version Knowledge Base entry migration (MASTER_SPEC §30.2, epic #36).
|
||||
|
||||
A pinned Knowledge Base may have been produced against an older entry-schema
|
||||
version than the running code. To keep pinned datasets loadable as the format
|
||||
evolves, entries are *adapted in memory at load time* — the on-disk (pinned)
|
||||
files are never rewritten.
|
||||
|
||||
Compatibility policy
|
||||
--------------------
|
||||
- The current entry-schema version is :data:`CURRENT_ENTRY_SCHEMA_VERSION` (it
|
||||
matches ``schemas/json/knowledge_base_entry.schema.json``'s ``version``).
|
||||
- A dataset declares its entry-schema version via the manifest's
|
||||
``entry_schema_version`` field. A dataset with **no** manifest, or a manifest
|
||||
without that field, is assumed to already be at the current version — so the
|
||||
shipped reference Knowledge Base loads with no adaptation.
|
||||
- Each supported older version has a forward adapter that upgrades a raw entry
|
||||
dict to the next version. Adapters are applied in sequence up to the current
|
||||
version, then the entry is validated and materialised as usual.
|
||||
- Versions older than :data:`MIN_SUPPORTED_ENTRY_SCHEMA_VERSION` cannot be adapted
|
||||
and raise a clear error rather than loading a misinterpreted dataset.
|
||||
|
||||
Adapters must be pure and deterministic: the same pinned dataset always adapts to
|
||||
the same in-memory entries.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any
|
||||
|
||||
# Keep in sync with schemas/json/knowledge_base_entry.schema.json "version".
|
||||
CURRENT_ENTRY_SCHEMA_VERSION = "1.1"
|
||||
MIN_SUPPORTED_ENTRY_SCHEMA_VERSION = "1.0"
|
||||
|
||||
|
||||
class UnsupportedKnowledgeBaseVersion(Exception):
|
||||
"""Raised when a dataset's entry-schema version is too old to adapt."""
|
||||
|
||||
|
||||
def _version_tuple(version: str) -> tuple[int, ...]:
|
||||
return tuple(int(part) for part in version.split("."))
|
||||
|
||||
|
||||
def _adapt_1_0_to_1_1(entry: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Entry schema 1.0 → 1.1.
|
||||
|
||||
In 1.0 the explicit/general distinction was carried by a boolean ``nsfw``
|
||||
field; 1.1 replaced it with the ``rating`` enum (``general``/``explicit``).
|
||||
Translate the old field so a 1.0 dataset keeps its NSFW gating.
|
||||
"""
|
||||
if "nsfw" in entry:
|
||||
entry.setdefault("rating", "explicit" if entry["nsfw"] else "general")
|
||||
del entry["nsfw"]
|
||||
return entry
|
||||
|
||||
|
||||
# Forward adapters, keyed by the version they upgrade FROM. Each returns the
|
||||
# entry at the next version. Extend this chain as the entry schema evolves.
|
||||
_ADAPTERS: dict[str, tuple[str, Callable[[dict[str, Any]], dict[str, Any]]]] = {
|
||||
"1.0": ("1.1", _adapt_1_0_to_1_1),
|
||||
}
|
||||
|
||||
|
||||
def adapt_entries(
|
||||
entries: list[Mapping[str, Any]], source_version: str | None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Adapt raw entry dicts from ``source_version`` to the current entry schema.
|
||||
|
||||
Args:
|
||||
entries: The raw (parsed JSON) entries as loaded from disk.
|
||||
source_version: The dataset's entry-schema version, or ``None`` to assume
|
||||
the current version (no adaptation).
|
||||
|
||||
Returns:
|
||||
New entry dicts at the current entry-schema version. Input dicts are not
|
||||
mutated.
|
||||
|
||||
Raises:
|
||||
UnsupportedKnowledgeBaseVersion: If ``source_version`` is older than the
|
||||
minimum supported version, or references an unknown version with no
|
||||
adapter path to the current one.
|
||||
"""
|
||||
version = source_version or CURRENT_ENTRY_SCHEMA_VERSION
|
||||
adapted = [dict(entry) for entry in entries]
|
||||
if version == CURRENT_ENTRY_SCHEMA_VERSION:
|
||||
return adapted
|
||||
|
||||
if _version_tuple(version) < _version_tuple(MIN_SUPPORTED_ENTRY_SCHEMA_VERSION):
|
||||
raise UnsupportedKnowledgeBaseVersion(
|
||||
f"Knowledge Base entry-schema version '{version}' is older than the "
|
||||
f"minimum supported '{MIN_SUPPORTED_ENTRY_SCHEMA_VERSION}' and cannot be adapted."
|
||||
)
|
||||
|
||||
# Walk the adapter chain forward to the current version.
|
||||
while version != CURRENT_ENTRY_SCHEMA_VERSION:
|
||||
step = _ADAPTERS.get(version)
|
||||
if step is None:
|
||||
raise UnsupportedKnowledgeBaseVersion(
|
||||
f"No migration path from Knowledge Base entry-schema version '{version}' "
|
||||
f"to '{CURRENT_ENTRY_SCHEMA_VERSION}'."
|
||||
)
|
||||
next_version, adapter = step
|
||||
adapted = [adapter(entry) for entry in adapted]
|
||||
version = next_version
|
||||
return adapted
|
||||
@@ -21,6 +21,8 @@ from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
from compiler.common.kb_manifest import IMPLICIT_VERSION, MANIFEST_FILENAME, load_manifest
|
||||
from compiler.common.kb_migration import UnsupportedKnowledgeBaseVersion, adapt_entries
|
||||
from compiler.common.kb_validation import validate_knowledge_base
|
||||
from compiler.common.log import StructuredLogger
|
||||
from compiler.common.result import Message, Severity
|
||||
@@ -65,7 +67,7 @@ class KnowledgeBaseEntry:
|
||||
class KnowledgeBase:
|
||||
"""An immutable, validated Knowledge Base with id and alias lookups."""
|
||||
|
||||
def __init__(self, entries: list[KnowledgeBaseEntry]) -> None:
|
||||
def __init__(self, entries: list[KnowledgeBaseEntry], version: str = IMPLICIT_VERSION) -> None:
|
||||
by_id: dict[str, KnowledgeBaseEntry] = {}
|
||||
alias_to_id: dict[str, str] = {}
|
||||
for entry in entries:
|
||||
@@ -74,6 +76,12 @@ class KnowledgeBase:
|
||||
alias_to_id[alias] = entry.id
|
||||
self._by_id: Mapping[str, KnowledgeBaseEntry] = MappingProxyType(by_id)
|
||||
self._alias_to_id: Mapping[str, str] = MappingProxyType(alias_to_id)
|
||||
self._version = version
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
"""The dataset version from the manifest (implicit ``v1`` when absent)."""
|
||||
return self._version
|
||||
|
||||
@property
|
||||
def by_id(self) -> Mapping[str, KnowledgeBaseEntry]:
|
||||
@@ -107,6 +115,8 @@ def _read_entries(directory: Path) -> tuple[list[dict], list[Message]]:
|
||||
entries: list[dict] = []
|
||||
problems: list[Message] = []
|
||||
for path in sorted(directory.glob("*.json")):
|
||||
if path.name == MANIFEST_FILENAME:
|
||||
continue # the manifest is dataset metadata, not an entry array
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as exc:
|
||||
@@ -134,15 +144,59 @@ def _load_problem(description: str, source: str) -> Message:
|
||||
def load_knowledge_base(
|
||||
directory: str | Path,
|
||||
logger: StructuredLogger | None = None,
|
||||
requested_version: str | None = None,
|
||||
) -> KnowledgeBase:
|
||||
"""Load and validate a Knowledge Base directory into an in-memory index.
|
||||
|
||||
Args:
|
||||
directory: The Knowledge Base directory to load.
|
||||
logger: Optional structured logger.
|
||||
requested_version: If set, the dataset manifest ``version`` must match it
|
||||
exactly; otherwise a Fatal ``SC0004`` is raised (the requested version
|
||||
is unavailable at this path). ``None`` loads whatever is on the path.
|
||||
|
||||
Raises:
|
||||
KnowledgeBaseError: If any file cannot be read or the Knowledge Base
|
||||
fails validation. Nothing is loaded in that case.
|
||||
KnowledgeBaseError: If any file cannot be read, the Knowledge Base fails
|
||||
validation, or the requested version is unavailable. Nothing is loaded
|
||||
in that case.
|
||||
"""
|
||||
directory = Path(directory)
|
||||
manifest = load_manifest(directory)
|
||||
if requested_version is not None and manifest.version != requested_version:
|
||||
raise KnowledgeBaseError(
|
||||
Message(
|
||||
code="SC0004",
|
||||
severity=Severity.FATAL,
|
||||
title="Knowledge Base Load Failure",
|
||||
description=(
|
||||
f"Requested Knowledge Base version '{requested_version}' is not "
|
||||
f"available at '{directory}' (found version '{manifest.version}')."
|
||||
),
|
||||
context={
|
||||
"id": str(directory),
|
||||
"requested_version": requested_version,
|
||||
"available_version": manifest.version,
|
||||
},
|
||||
),
|
||||
findings=[],
|
||||
)
|
||||
raw_entries, problems = _read_entries(directory)
|
||||
try:
|
||||
raw_entries = adapt_entries(raw_entries, manifest.entry_schema_version)
|
||||
except UnsupportedKnowledgeBaseVersion as exc:
|
||||
raise KnowledgeBaseError(
|
||||
Message(
|
||||
code="SC0004",
|
||||
severity=Severity.FATAL,
|
||||
title="Knowledge Base Load Failure",
|
||||
description=str(exc),
|
||||
context={
|
||||
"id": str(directory),
|
||||
"entry_schema_version": manifest.entry_schema_version,
|
||||
},
|
||||
),
|
||||
findings=[],
|
||||
) from exc
|
||||
findings = problems + validate_knowledge_base(raw_entries)
|
||||
if findings:
|
||||
raise KnowledgeBaseError(
|
||||
@@ -160,24 +214,37 @@ def load_knowledge_base(
|
||||
)
|
||||
|
||||
entries = [KnowledgeBaseEntry.from_json(entry) for entry in raw_entries]
|
||||
kb = KnowledgeBase(entries)
|
||||
kb = KnowledgeBase(entries, version=manifest.version)
|
||||
if logger is not None:
|
||||
logger.basic("knowledge_base_loaded", directory=str(directory), entries=len(kb))
|
||||
logger.basic(
|
||||
"knowledge_base_loaded",
|
||||
directory=str(directory),
|
||||
entries=len(kb),
|
||||
version=kb.version,
|
||||
)
|
||||
return kb
|
||||
|
||||
|
||||
class KnowledgeBaseLoader:
|
||||
"""Loads a Knowledge Base once and caches it; reload is explicit and safe."""
|
||||
|
||||
def __init__(self, directory: str | Path, logger: StructuredLogger | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
directory: str | Path,
|
||||
logger: StructuredLogger | None = None,
|
||||
requested_version: str | None = None,
|
||||
) -> None:
|
||||
self._directory = Path(directory)
|
||||
self._logger = logger
|
||||
self._requested_version = requested_version
|
||||
self._cache: KnowledgeBase | None = None
|
||||
|
||||
def get(self) -> KnowledgeBase:
|
||||
"""Return the cached Knowledge Base, loading it on first use."""
|
||||
if self._cache is None:
|
||||
self._cache = load_knowledge_base(self._directory, self._logger)
|
||||
self._cache = load_knowledge_base(
|
||||
self._directory, self._logger, self._requested_version
|
||||
)
|
||||
return self._cache
|
||||
|
||||
def reload(self) -> KnowledgeBase:
|
||||
@@ -186,6 +253,6 @@ class KnowledgeBaseLoader:
|
||||
On failure the previously cached Knowledge Base is left untouched (the
|
||||
load raises before the cache is replaced).
|
||||
"""
|
||||
kb = load_knowledge_base(self._directory, self._logger)
|
||||
kb = load_knowledge_base(self._directory, self._logger, self._requested_version)
|
||||
self._cache = kb
|
||||
return kb
|
||||
|
||||
@@ -13,7 +13,8 @@
|
||||
"allow_aliases": true,
|
||||
"expansion_enabled": true,
|
||||
"max_expansion_depth": 8,
|
||||
"include_nsfw": false
|
||||
"include_nsfw": false,
|
||||
"knowledge_base_version": null
|
||||
},
|
||||
"validator": {
|
||||
"allow_unknown_fields": false
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"content_hash": "sha256:a9f132307076670a20999a5be768537c2a6b93f7a0afdd41ce46a3da08175b68",
|
||||
"manifest_schema_version": "1.0",
|
||||
"source_vocab": "sha256:d9238cecb70736aa81e1297b174d2ed0c4555c0b214651d33c955d9e5ce0e88b",
|
||||
"version": "1.0.0"
|
||||
}
|
||||
@@ -48,8 +48,11 @@ class ConfigurationNode:
|
||||
"debug_enabled": ("BOOLEAN", {"default": False}),
|
||||
"debug_level": (_DEBUG_LEVELS, {"default": "basic"}),
|
||||
},
|
||||
# Appended at the END so saved workflows keep their positional widget
|
||||
# values (ComfyUI stores widget values positionally).
|
||||
"optional": {
|
||||
"analyzer_system_prompt": ("STRING", {"multiline": True, "default": ""}),
|
||||
"resolver_knowledge_base_version": ("STRING", {"default": ""}),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -72,6 +75,7 @@ class ConfigurationNode:
|
||||
debug_enabled: bool,
|
||||
debug_level: str,
|
||||
analyzer_system_prompt: str = "",
|
||||
resolver_knowledge_base_version: str = "",
|
||||
) -> tuple[Any, str]:
|
||||
analyzer: dict[str, Any] = {
|
||||
"model": analyzer_model,
|
||||
@@ -91,6 +95,11 @@ class ConfigurationNode:
|
||||
"expansion_enabled": resolver_expansion_enabled,
|
||||
"max_expansion_depth": resolver_max_expansion_depth,
|
||||
"include_nsfw": resolver_include_nsfw,
|
||||
**(
|
||||
{"knowledge_base_version": resolver_knowledge_base_version}
|
||||
if resolver_knowledge_base_version
|
||||
else {}
|
||||
),
|
||||
},
|
||||
"validator": {"allow_unknown_fields": validator_allow_unknown_fields},
|
||||
"prompt_builder": {
|
||||
|
||||
@@ -40,13 +40,19 @@ class KnowledgeBaseLoaderNode:
|
||||
def INPUT_TYPES(cls) -> dict[str, Any]:
|
||||
return {
|
||||
"required": {"path": ("STRING", {"default": "knowledge_base/"})},
|
||||
"optional": {"reload": ("INT", {"default": 0, "min": 0})},
|
||||
# New widgets are appended at the END so saved workflows keep their
|
||||
# positional widget values (ComfyUI stores widgets positionally).
|
||||
"optional": {
|
||||
"reload": ("INT", {"default": 0, "min": 0}),
|
||||
"version": ("STRING", {"default": ""}),
|
||||
},
|
||||
}
|
||||
|
||||
def run(self, path: str, reload: int = 0) -> tuple[Any, str, str, str]:
|
||||
def run(self, path: str, reload: int = 0, version: str = "") -> tuple[Any, str, str, str]:
|
||||
resolved = _resolve_path(path)
|
||||
requested_version = version or None
|
||||
try:
|
||||
knowledge_base = load_knowledge_base(resolved)
|
||||
knowledge_base = load_knowledge_base(resolved, requested_version=requested_version)
|
||||
except KnowledgeBaseError as error:
|
||||
errors = format_messages([error.message, *error.findings])
|
||||
return (None, "", errors, f"Failed to load Knowledge Base from: {resolved}")
|
||||
|
||||
@@ -29,7 +29,8 @@
|
||||
"allow_aliases": { "type": "boolean" },
|
||||
"expansion_enabled": { "type": "boolean" },
|
||||
"max_expansion_depth": { "type": "integer", "minimum": 1 },
|
||||
"include_nsfw": { "type": "boolean" }
|
||||
"include_nsfw": { "type": "boolean" },
|
||||
"knowledge_base_version": { "type": ["string", "null"] }
|
||||
}
|
||||
},
|
||||
"validator": {
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "knowledge_base_manifest",
|
||||
"version": "1.0",
|
||||
"title": "Knowledge Base Manifest",
|
||||
"description": "Metadata for a versioned Knowledge Base dataset. MASTER_SPEC §30.2 (epic #36). Deterministic: content_hash and source_vocab are content digests, so regenerating an unchanged dataset yields a byte-identical manifest.",
|
||||
"type": "object",
|
||||
"required": ["manifest_schema_version", "version", "content_hash"],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"manifest_schema_version": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+$" },
|
||||
"version": { "type": "string", "pattern": "^[0-9]+(\\.[0-9]+)*$" },
|
||||
"content_hash": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
|
||||
"source_vocab": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
|
||||
"entry_schema_version": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+$" }
|
||||
}
|
||||
}
|
||||
@@ -20,13 +20,22 @@ Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from compiler.common.kb_manifest import write_manifest # noqa: E402
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
_VOCAB = _REPO_ROOT / "data" / "danbooru_vocab.txt"
|
||||
_KB_DIR = _REPO_ROOT / "knowledge_base"
|
||||
_GEN_PREFIX = "gen_"
|
||||
|
||||
# The dataset version stamped into the manifest on each rebuild. Bump this when a
|
||||
# regeneration changes the dataset in a way workflows should be able to pin.
|
||||
_DATASET_VERSION = "1.0.0"
|
||||
|
||||
# Raffle category -> one of the 19 canonical SceneCompiler categories.
|
||||
_CATEGORY_MAP = {
|
||||
"clothes_and_accessories": "clothing",
|
||||
@@ -138,8 +147,11 @@ def generate() -> dict[str, int]:
|
||||
out.write_text(json.dumps(entries, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
counts[sc_category] = len(entries)
|
||||
|
||||
manifest = write_manifest(_KB_DIR, _DATASET_VERSION, _VOCAB)
|
||||
|
||||
total = sum(counts.values())
|
||||
print(f"generated {total} entries across {len(counts)} category files")
|
||||
print(f" stamped manifest version {manifest.version} ({manifest.content_hash[:19]}...)")
|
||||
print(f" skipped {skipped_reserved} (already hand-curated), {skipped_unmapped} (unmapped)")
|
||||
for cat, n in sorted(counts.items(), key=lambda x: -x[1]):
|
||||
print(f" {n:6} gen_{cat}.json")
|
||||
|
||||
@@ -81,6 +81,18 @@ def test_partial_config_fills_defaults() -> None:
|
||||
assert c.resolver.max_expansion_depth == 8
|
||||
|
||||
|
||||
def test_knowledge_base_version_defaults_to_none_and_is_omitted() -> None:
|
||||
c = Config()
|
||||
assert c.resolver.knowledge_base_version is None
|
||||
assert "knowledge_base_version" not in c.resolver.to_json() # omitted when unset
|
||||
|
||||
|
||||
def test_knowledge_base_version_roundtrips_when_set() -> None:
|
||||
c = Config.from_json({"resolver": {"knowledge_base_version": "2.0.0"}})
|
||||
assert c.resolver.knowledge_base_version == "2.0.0"
|
||||
assert c.to_json()["resolver"]["knowledge_base_version"] == "2.0.0"
|
||||
|
||||
|
||||
def test_load_config_reads_file(tmp_path) -> None:
|
||||
path = tmp_path / "config.json"
|
||||
path.write_text(json.dumps(EXAMPLE_CONFIG), encoding="utf-8")
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Tests for the Knowledge Base dataset manifest (issue #127, epic #36)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from compiler.common.kb_manifest import (
|
||||
IMPLICIT_VERSION,
|
||||
MANIFEST_FILENAME,
|
||||
compute_content_hash,
|
||||
load_manifest,
|
||||
write_manifest,
|
||||
)
|
||||
from compiler.common.knowledge_base import load_knowledge_base
|
||||
from schemas.validation import validate_document
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
KB_DIR = REPO_ROOT / "knowledge_base"
|
||||
|
||||
|
||||
# --- shipped reference manifest --------------------------------------------
|
||||
|
||||
|
||||
def test_reference_manifest_exists_and_validates() -> None:
|
||||
data = json.loads((KB_DIR / MANIFEST_FILENAME).read_text(encoding="utf-8"))
|
||||
assert validate_document(data, "knowledge_base_manifest") == []
|
||||
assert data["version"] == "1.0.0"
|
||||
|
||||
|
||||
def test_loader_reads_manifest_version() -> None:
|
||||
kb = load_knowledge_base(KB_DIR)
|
||||
assert kb.version == "1.0.0"
|
||||
|
||||
|
||||
def test_manifest_content_hash_matches_current_entries() -> None:
|
||||
# The committed manifest is a faithful, deterministic stamp of the dataset.
|
||||
manifest = load_manifest(KB_DIR)
|
||||
assert manifest.content_hash == compute_content_hash(KB_DIR)
|
||||
|
||||
|
||||
# --- backward compatibility -------------------------------------------------
|
||||
|
||||
|
||||
def test_missing_manifest_defaults_to_implicit_v1(tmp_path) -> None:
|
||||
(tmp_path / "hair.json").write_text(
|
||||
'[{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]',
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert load_manifest(tmp_path).version == IMPLICIT_VERSION
|
||||
assert load_knowledge_base(tmp_path).version == IMPLICIT_VERSION
|
||||
|
||||
|
||||
# --- determinism ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_write_manifest_is_deterministic(tmp_path) -> None:
|
||||
vocab = tmp_path / "vocab.txt"
|
||||
vocab.write_text("clothing\tshirt\n", encoding="utf-8")
|
||||
(tmp_path / "clothing.json").write_text(
|
||||
'[{"id": "shirt", "tags": ["shirt"], "category": "clothing"}]',
|
||||
encoding="utf-8",
|
||||
)
|
||||
first = write_manifest(tmp_path, "2.0.0", vocab)
|
||||
first_bytes = (tmp_path / MANIFEST_FILENAME).read_bytes()
|
||||
second = write_manifest(tmp_path, "2.0.0", vocab)
|
||||
second_bytes = (tmp_path / MANIFEST_FILENAME).read_bytes()
|
||||
assert first == second
|
||||
assert first_bytes == second_bytes # byte-identical regeneration
|
||||
|
||||
|
||||
def test_requesting_available_version_loads(tmp_path) -> None:
|
||||
(tmp_path / "hair.json").write_text(
|
||||
'[{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]',
|
||||
encoding="utf-8",
|
||||
)
|
||||
write_manifest(tmp_path, "3.1.0", tmp_path / "missing_vocab.txt")
|
||||
kb = load_knowledge_base(tmp_path, requested_version="3.1.0")
|
||||
assert kb.version == "3.1.0"
|
||||
assert kb.get("long_hair") is not None
|
||||
|
||||
|
||||
def test_requesting_unavailable_version_errors_clearly(tmp_path) -> None:
|
||||
from compiler.common.knowledge_base import KnowledgeBaseError
|
||||
|
||||
(tmp_path / "hair.json").write_text(
|
||||
'[{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]',
|
||||
encoding="utf-8",
|
||||
)
|
||||
write_manifest(tmp_path, "3.1.0", tmp_path / "missing_vocab.txt")
|
||||
try:
|
||||
load_knowledge_base(tmp_path, requested_version="9.9.9")
|
||||
except KnowledgeBaseError as error:
|
||||
assert error.message.code == "SC0004"
|
||||
assert "9.9.9" in error.message.description
|
||||
assert error.message.context["available_version"] == "3.1.0"
|
||||
else:
|
||||
raise AssertionError("expected KnowledgeBaseError for unavailable version")
|
||||
|
||||
|
||||
def test_manifest_is_not_read_as_an_entry_file(tmp_path) -> None:
|
||||
# A manifest.json in the KB dir must not break loading (it is not an entry array).
|
||||
(tmp_path / "hair.json").write_text(
|
||||
'[{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]',
|
||||
encoding="utf-8",
|
||||
)
|
||||
write_manifest(tmp_path, "1.0.0", tmp_path / "missing_vocab.txt")
|
||||
kb = load_knowledge_base(tmp_path)
|
||||
assert len(kb) == 1 # the manifest was skipped, only the one entry loaded
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Tests for cross-version Knowledge Base loading (issue #129, epic #36)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from compiler.common.kb_manifest import (
|
||||
MANIFEST_FILENAME,
|
||||
compute_content_hash,
|
||||
load_manifest,
|
||||
)
|
||||
from compiler.common.kb_migration import (
|
||||
CURRENT_ENTRY_SCHEMA_VERSION,
|
||||
UnsupportedKnowledgeBaseVersion,
|
||||
adapt_entries,
|
||||
)
|
||||
from compiler.common.knowledge_base import KnowledgeBaseError, load_knowledge_base
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
KB_DIR = REPO_ROOT / "knowledge_base"
|
||||
|
||||
|
||||
def _write_manifest(directory: Path, entry_schema_version: str) -> None:
|
||||
(directory / MANIFEST_FILENAME).write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"manifest_schema_version": "1.0",
|
||||
"version": "1.0.0",
|
||||
"content_hash": compute_content_hash(directory),
|
||||
"entry_schema_version": entry_schema_version,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
# --- adapter unit level -----------------------------------------------------
|
||||
|
||||
|
||||
def test_no_source_version_is_treated_as_current() -> None:
|
||||
entries = [{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]
|
||||
assert adapt_entries(entries, None) == entries
|
||||
|
||||
|
||||
def test_adapter_1_0_translates_nsfw_to_rating() -> None:
|
||||
entries = [
|
||||
{"id": "a", "tags": ["a"], "category": "body", "nsfw": True},
|
||||
{"id": "b", "tags": ["b"], "category": "body", "nsfw": False},
|
||||
]
|
||||
adapted = adapt_entries(entries, "1.0")
|
||||
assert adapted[0]["rating"] == "explicit" and "nsfw" not in adapted[0]
|
||||
assert adapted[1]["rating"] == "general" and "nsfw" not in adapted[1]
|
||||
assert entries[0]["nsfw"] is True # inputs not mutated
|
||||
|
||||
|
||||
def test_too_old_version_raises() -> None:
|
||||
with pytest.raises(UnsupportedKnowledgeBaseVersion):
|
||||
adapt_entries([], "0.9")
|
||||
|
||||
|
||||
# --- loader level -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_current_reference_kb_loads_without_adaptation() -> None:
|
||||
assert load_manifest(KB_DIR).entry_schema_version is None
|
||||
kb = load_knowledge_base(KB_DIR) # loads clean, no migration applied
|
||||
assert len(kb) > 100
|
||||
|
||||
|
||||
def test_older_dataset_adapts_and_resolves(tmp_path) -> None:
|
||||
(tmp_path / "body.json").write_text(
|
||||
json.dumps(
|
||||
[{"id": "bare_shoulders", "tags": ["bare shoulders"], "category": "body", "nsfw": True}]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
_write_manifest(tmp_path, "1.0")
|
||||
kb = load_knowledge_base(tmp_path)
|
||||
entry = kb.get("bare_shoulders")
|
||||
assert entry is not None
|
||||
assert entry.rating == "explicit" # adapted from nsfw=true
|
||||
|
||||
|
||||
def test_unsupported_old_dataset_errors_clearly(tmp_path) -> None:
|
||||
(tmp_path / "hair.json").write_text(
|
||||
json.dumps([{"id": "long_hair", "tags": ["long hair"], "category": "hair"}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
_write_manifest(tmp_path, "0.9")
|
||||
with pytest.raises(KnowledgeBaseError) as exc:
|
||||
load_knowledge_base(tmp_path)
|
||||
assert exc.value.message.code == "SC0004"
|
||||
assert "0.9" in exc.value.message.description
|
||||
|
||||
|
||||
def test_current_version_string_matches_entry_schema_file() -> None:
|
||||
schema = json.loads(
|
||||
(REPO_ROOT / "schemas" / "json" / "knowledge_base_entry.schema.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
assert schema["version"] == CURRENT_ENTRY_SCHEMA_VERSION
|
||||
Reference in New Issue
Block a user