Fixes from the 2026-10-26 bench critiques (chat side): - Claim checks: "No <kind> card came" notes count the kinds a card changes or deletes, request gaps skip delete and tidy answers, and new notes mark a weight or factor no card sets, a bundle move the card doesn't make and "created successfully" before Apply (engine/ai/claims.py). Tool names in replies are said in plain words. - Deletes only target items the user named (a bulk ask still covers more), a category or tag delete never deletes an item of that name, and with deleting off the reply says where to turn it on. - edit_bundle: "switch off" switches off instead of removing; new move op (after/before). - A new item or rename onto a taken name warns; a rewritten derivation rule keeps its ELSE; a typo tag fix adds the tag it meant. - run_test runs a stack item with this answer's pending card and adds the items that set what a combine or derivation reads. - Only is described as a guest list (chat guide, docs, matrix legend). - read_page names the settings section the way open takes it, section aliases, more_tools hints carry the group's contents, library tags merge case, and Explain on a selection is read-only.
186 lines
8.0 KiB
Python
186 lines
8.0 KiB
Python
"""What the user asked for that this answer's cards don't have yet.
|
|
|
|
Before an answer with cards stands, code compares the request with the
|
|
cards, the other way round from the claim checks (which compare what the
|
|
answer SAYS with what was made). Mingbird's finish gate (harness research
|
|
H1, `/pr-drafts/reports/Small-model harness ideas.md`) was the largest of
|
|
its four mechanisms on a 2B model; the 9B used a boost where the user said
|
|
"only gets" (smoke s-only, 2026-10-23).
|
|
|
|
Only features whose words are exact and whose presence a card shows are
|
|
checked: a constraint, an "only" rule, a fallback option, negatives, a
|
|
tag-filtered ref ("only the warm colours from Palette"). Counts
|
|
belong to the plan loop; kinds of item are left to it too.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections.abc import Callable, Iterator
|
|
from typing import Any
|
|
|
|
_CONSTRAINT_TOOLS = frozenset({"create_constraint", "edit_constraint"})
|
|
#: Cards that delete or tidy items: an answer of only these makes nothing the
|
|
#: feature checks look for. "Delete the Mood → weather constraint" asks for
|
|
#: no constraint (bench X7, R7, T6: "No card has a constraint" under deletes).
|
|
TIDY_TOOLS = frozenset({"delete_items", "update_many", "update_info", "edit_tags",
|
|
"edit_categories"})
|
|
|
|
|
|
def _walk(value: Any) -> Iterator[tuple[str, Any]]:
|
|
"""Every (key, value) pair in a card's op, at any depth."""
|
|
if isinstance(value, dict):
|
|
for k, v in value.items():
|
|
yield str(k), v
|
|
yield from _walk(v)
|
|
elif isinstance(value, list):
|
|
for v in value:
|
|
yield from _walk(v)
|
|
|
|
|
|
def _has_constraint(cards: list[dict[str, Any]]) -> bool:
|
|
return any(c.get("tool") in _CONSTRAINT_TOOLS for c in cards)
|
|
|
|
|
|
def _has_only(cards: list[dict[str, Any]]) -> bool:
|
|
return any(k in ("effect", "mode") and v == "only"
|
|
for c in cards if c.get("tool") in _CONSTRAINT_TOOLS
|
|
for k, v in _walk(c.get("op")))
|
|
|
|
|
|
def _has_fallback(cards: list[dict[str, Any]]) -> bool:
|
|
return any(k == "fallback" and v is True for c in cards for k, v in _walk(c.get("op")))
|
|
|
|
|
|
def _has_tag_ref(cards: list[dict[str, Any]]) -> bool:
|
|
return any(isinstance(v, str) and _TAG_REF.search(v)
|
|
for c in cards for _k, v in _walk(c.get("op")))
|
|
|
|
|
|
_TAG_REF = re.compile(r"@\{[^}]*:")
|
|
|
|
|
|
def _has_negative(cards: list[dict[str, Any]]) -> bool:
|
|
return any((k == "negative" and isinstance(v, str) and v.strip())
|
|
or (k == "mode" and v in ("negative", "negative_replace"))
|
|
for c in cards for k, v in _walk(c.get("op")))
|
|
|
|
|
|
#: (what the note calls it, the request words, whether the cards have it).
|
|
FEATURES: tuple[tuple[str, re.Pattern[str], Callable[[list[dict[str, Any]]], bool]], ...] = (
|
|
("a constraint", re.compile(
|
|
r"\bconstraints?\b|\bgoes with\b|\bgo with\b|\bonly gets?\b|\bgets? only\b|"
|
|
r"\bexclud\w*|"
|
|
r"\brule[sd]? out\b|\bnever\b.{0,60}?\b(?:when|with|for|to|if|on|in)\b", re.I),
|
|
_has_constraint),
|
|
('an "only" rule (effect only: the target may use just what the row names)', re.compile(
|
|
r"\bonly (?:gets?|allow\w*|with|uses?)\b|\b(?:gets?|uses?|allows?) only\b|"
|
|
r"\bgoes with\b|\bgo with\b", re.I),
|
|
_has_only),
|
|
("a fallback option", re.compile(r"\bfallback\b|\bnever (?:be )?empty\b", re.I),
|
|
_has_fallback),
|
|
("negatives", re.compile(r"\bnegatives?\b", re.I), _has_negative),
|
|
("a tag-filtered ref (an option such as \"a @{Palette:warm} sky\", not the colours "
|
|
"copied in)", re.compile(r"\bonly\b[^.]{0,40}?\bfrom\b|\btagged \w+ from\b", re.I),
|
|
_has_tag_ref),
|
|
)
|
|
|
|
#: Phrases that read like an ask above but aren't: "only the warm colours
|
|
#: from Palette" is a tag filter, not an only rule; "when rules rule out
|
|
#: everything" is how a fallback is described (smoke 2026-10-23, both
|
|
#: nudged wrongly).
|
|
_NOT_ASKS = re.compile(r"\bonly\b[^.]{0,40}?\bfrom\b|\brules?\s+rule[sd]?\s+out\b", re.I)
|
|
|
|
#: Chips name items; their names must not count as asks ("[[id#Never Rain]]").
|
|
_CHIP = re.compile(r"\[\[[^\]]*\]\]")
|
|
|
|
|
|
#: `FEATURES` by the request frame's `wants` keys (`engine/ai/lang.py`).
|
|
WANT_KEYS = ("constraint", "only_rule", "fallback", "negatives", "tag_ref")
|
|
|
|
|
|
def gaps(request: str, cards: list[dict[str, Any]],
|
|
wants: list[str] | None = None) -> list[str]:
|
|
"""What `request` asks for that `cards` (this answer's proposals) lack,
|
|
in `FEATURES` order. An "only" rule counts as the constraint too.
|
|
``wants``: what a request frame says the ask wants, for a request in
|
|
another language; it decides instead of the words."""
|
|
if not cards or (not request and wants is None) \
|
|
or all(c.get("tool") in TIDY_TOOLS for c in cards):
|
|
return []
|
|
if wants is not None:
|
|
out = [name for key, (name, _words, has) in zip(WANT_KEYS, FEATURES, strict=True)
|
|
if key in wants and not has(cards)]
|
|
else:
|
|
plain = _CHIP.sub(" ", request)
|
|
asks = _NOT_ASKS.sub(" ", plain)
|
|
out = [name for name, words, has in FEATURES
|
|
if (words.search(plain) if name.startswith("a tag-filtered")
|
|
else words.search(asks))
|
|
and not has(cards)]
|
|
if len(out) > 1 and out[0] == "a constraint" and out[1].startswith('an "only"'):
|
|
out.pop(0)
|
|
return out
|
|
|
|
|
|
def nudge(missing: list[str], request: str) -> str:
|
|
"""The one note an answer gets when its cards miss part of the request."""
|
|
asked = request.strip()
|
|
if len(asked) > 400:
|
|
asked = asked[:400].rstrip() + "…"
|
|
return (f"Before you finish, check the cards against the request: it asks for "
|
|
f"{' and '.join(missing)}, and no card in this answer has "
|
|
f"{'it' if len(missing) == 1 else 'them'}. Propose "
|
|
f"{'it' if len(missing) == 1 else 'them'} now (or change the card that should "
|
|
f"have it), or say why it isn't needed.\nThe request: \"{asked}\"")
|
|
|
|
|
|
#: The tools that add each feature, besides re-sending the card's own tool.
|
|
_FIX_TOOLS = (
|
|
("a constraint", ("create_constraint", "edit_constraint")),
|
|
('an "only" rule', ("create_constraint", "edit_constraint")),
|
|
("a fallback option", ("edit_options",)),
|
|
("negatives", ("edit_options",)),
|
|
("a tag-filtered ref", ("edit_options", "add_options")),
|
|
)
|
|
|
|
|
|
def fix_tools(missing: list[str], cards: list[dict[str, Any]]) -> list[str]:
|
|
"""What the step after the nudge offers: the cards' own tools (sent
|
|
again with the fix) and the tools that add what is missing. On an empty
|
|
library the 9B answered the nudge with more_tools, then opened its own
|
|
card twice and claimed the fix (smoke --empty, 2026-10-23)."""
|
|
out = [str(c.get("tool")) for c in cards if c.get("tool")]
|
|
for name in missing:
|
|
out += [t for prefix, tools in _FIX_TOOLS if name.startswith(prefix) for t in tools]
|
|
return list(dict.fromkeys(out))
|
|
|
|
|
|
#: How an answer talks about each feature as if it were made.
|
|
_SAYS = (
|
|
("a constraint", re.compile(r"\bconstraints?\b", re.I)),
|
|
('an "only" rule', re.compile(r"\bonly\b.{0,30}\brule\b|\beffect\b.{0,10}\bonly\b", re.I)),
|
|
("a fallback option", re.compile(r"\bfallback\b", re.I)),
|
|
("negatives", re.compile(r"\bnegatives?\b", re.I)),
|
|
("a tag-filtered ref", re.compile(r"@\{[^}]*:")),
|
|
)
|
|
|
|
|
|
_NOT_BEFORE = re.compile(r"\b(?:no|not|without|isn't|aren't|doesn't|don't|didn't|"
|
|
r"skip(?:ped)?|instead of)\b[^.\n]{0,30}$", re.I)
|
|
|
|
|
|
def claimed(missing: list[str], message: str) -> list[str]:
|
|
"""The missing features the answer talks about anyway: on an empty
|
|
library the 9B said "clear sky (fallback)" and "a @{Palette:warm} sky"
|
|
beside cards with neither (smoke --empty, 2026-10-23). "No fallback is
|
|
needed" says why it's left out, which is the answer the note asked for."""
|
|
text = message or ""
|
|
return [name for name in missing for prefix, says in _SAYS
|
|
if name.startswith(prefix) and any(
|
|
not _NOT_BEFORE.search(text[:m.start()]) for m in says.finditer(text))]
|
|
|
|
|
|
def short_name(name: str) -> str:
|
|
"""A feature's name without its explanation in brackets."""
|
|
return name.split(" (", 1)[0]
|