50e7b0a8a4
* feat(kb): automatic KB builder, ingestion & validation tooling (#35) Closes #118, #119, #120, #121, #122. - Ingest committed Danbooru alias/implication snapshots during generation: synonyms reach canonicals (long black stockings -> thighhighs), implications become expand relations. Curated-wins preserved. - Automatic candidate validator (structural rules reused from kb_validation + confidence heuristic, no approval gate; SC0022). - Duplicate/conflict detection tool (scripts/detect_kb_conflicts.py) + reusable detect_conflicts; clean reference KB reports zero. - KB coverage benchmark harness (scripts/coverage_benchmark.py) + JSON summary. - Single fully-automated, CI-runnable build pipeline (scripts/build_knowledge_base.py) that fails fast on rejections/conflicts and never prompts. Docs added. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * test(kb): use pytest.approx for confidence float assertions (SonarCloud reliability) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: kunail0804 <kunail0804@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
59 lines
2.0 KiB
Python
59 lines
2.0 KiB
Python
"""CLI to scan a Knowledge Base for duplicates and conflicts (issue #120).
|
|
|
|
Loads every entry file under a Knowledge Base directory (curated ids are taken
|
|
from the non-``gen_*`` files so curated-wins semantics are respected), runs the
|
|
conflict detector, writes a machine-readable JSON report to stdout, and prints a
|
|
one-line human summary to stderr. Exit code is 0 when the Knowledge Base is clean,
|
|
1 when any conflict is found — so it can serve as a CI gate.
|
|
|
|
Usage:
|
|
python scripts/detect_kb_conflicts.py knowledge_base/
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from compiler.common.kb_conflicts import detect_conflicts # noqa: E402
|
|
|
|
_GEN_PREFIX = "gen_"
|
|
_MANIFEST = "manifest.json"
|
|
|
|
|
|
def _load(directory: Path) -> tuple[list[dict], set[str]]:
|
|
"""Return (all entries, curated ids) from a Knowledge Base directory."""
|
|
entries: list[dict] = []
|
|
curated_ids: set[str] = set()
|
|
for path in sorted(directory.glob("*.json")):
|
|
if path.name == _MANIFEST:
|
|
continue
|
|
file_entries = json.loads(path.read_text(encoding="utf-8"))
|
|
entries.extend(file_entries)
|
|
if not path.name.startswith(_GEN_PREFIX):
|
|
curated_ids.update(e["id"] for e in file_entries if isinstance(e.get("id"), str))
|
|
return entries, curated_ids
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
if len(argv) != 1:
|
|
sys.stderr.write("usage: detect_kb_conflicts.py <knowledge_base_dir>\n")
|
|
return 2
|
|
directory = Path(argv[0])
|
|
if not directory.is_dir():
|
|
sys.stderr.write(f"not a directory: {directory}\n")
|
|
return 2
|
|
|
|
entries, curated_ids = _load(directory)
|
|
report = detect_conflicts(entries, curated_ids)
|
|
sys.stdout.write(json.dumps(report.to_json(), indent=2, sort_keys=True) + "\n")
|
|
sys.stderr.write(f"{report.total} conflict(s) found across {len(entries)} entries.\n")
|
|
return 0 if report.total == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main(sys.argv[1:]))
|