DumiFlex
58f5aff720
Fixes from the 2026-10-26 bench critiques (chat side): - Claim checks: "No <kind> card came" notes count the kinds a card changes or deletes, request gaps skip delete and tidy answers, and new notes mark a weight or factor no card sets, a bundle move the card doesn't make and "created successfully" before Apply (engine/ai/claims.py). Tool names in replies are said in plain words. - Deletes only target items the user named (a bulk ask still covers more), a category or tag delete never deletes an item of that name, and with deleting off the reply says where to turn it on. - edit_bundle: "switch off" switches off instead of removing; new move op (after/before). - A new item or rename onto a taken name warns; a rewritten derivation rule keeps its ELSE; a typo tag fix adds the tag it meant. - run_test runs a stack item with this answer's pending card and adds the items that set what a combine or derivation reads. - Only is described as a guest list (chat guide, docs, matrix legend). - read_page names the settings section the way open takes it, section aliases, more_tools hints carry the group's contents, library tags merge case, and Explain on a selection is read-only.
241 lines
12 KiB
Python
241 lines
12 KiB
Python
"""An answer whose cards miss a feature the request names gets one note
|
|
(harness research H1; smoke s-only, 2026-10-23)."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from tests.wp_api.test_ai_api import _chat
|
|
from tests.wp_api.test_ai_chats import _on
|
|
|
|
ASK = "make a weapon wildcard with a fallback option"
|
|
WEAPON = {"name": "Weapon", "var": "weapon",
|
|
"options": [{"text": "sword"}, {"text": "bow"}]}
|
|
|
|
|
|
def _told(fake_llm, n):
|
|
return " ".join(str(m["content"]) for x in fake_llm.seen[n:]
|
|
for m in x["body"]["messages"] if m["role"] == "user")
|
|
|
|
|
|
async def test_a_missing_feature_is_asked_for_once(wp_client, fake_llm):
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(WEAPON)}]},
|
|
{"message": "Proposed Weapon.", "calls": []},
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(
|
|
{**WEAPON, "options": [*WEAPON["options"], {"text": "fists", "fallback": True}]})}]},
|
|
{"message": "Proposed Weapon with a fallback.", "calls": []},
|
|
])
|
|
out = await _chat(wp_client, ASK)
|
|
told = _told(fake_llm, len(fake_llm.seen) - 1) # the last ask carries the whole log
|
|
assert told.count("check the cards against the request") == 1
|
|
assert 'a fallback option' in told and f'The request: "{ASK}"' in told
|
|
[card] = out["proposals"]
|
|
assert any(o.get("fallback") for o in card["op"]["body"]["payload"]["options"])
|
|
|
|
|
|
async def test_cards_that_have_it_get_no_note(wp_client, fake_llm):
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(
|
|
{**WEAPON, "options": [{"text": "sword"}, {"text": "fists", "fallback": True}]})}]},
|
|
{"message": "Proposed Weapon.", "calls": []},
|
|
])
|
|
n = len(fake_llm.seen)
|
|
await _chat(wp_client, ASK)
|
|
assert "check the cards against the request" not in _told(fake_llm, n)
|
|
|
|
|
|
async def test_the_step_after_the_note_offers_only_the_fix(wp_client, fake_llm):
|
|
# Smoke --empty: the 9B met the note with more_tools, then opened its
|
|
# own card twice and claimed the fallback.
|
|
from tests.wp_api.test_ai_narrow_api import _acts
|
|
from tests.wp_api.test_ai_plan_items import _offered
|
|
from tests.wp_api.test_ai_prefix_stable import _on as _on_typed
|
|
|
|
_on_typed(fake_llm)
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(WEAPON)}]},
|
|
{"message": "Proposed Weapon.", "calls": []},
|
|
{"message": "No fallback needed.", "calls": []},
|
|
])
|
|
n = len(fake_llm.seen)
|
|
await _chat(wp_client, ASK)
|
|
acts = _acts(fake_llm, n)
|
|
assert "more_tools" in _offered(acts[1])
|
|
after = _offered(acts[2])
|
|
assert set(after) == {"create_wildcard", "edit_options"}
|
|
|
|
|
|
async def test_an_answer_that_claims_the_missing_feature_is_marked(wp_client, fake_llm):
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(WEAPON)}]},
|
|
{"message": "Proposed Weapon.", "calls": []},
|
|
{"message": "Proposed Weapon with fists as the fallback.", "calls": []},
|
|
{"message": "Proposed Weapon with fists as the fallback.", "calls": []},
|
|
])
|
|
out = await _chat(wp_client, ASK)
|
|
assert "You asked for a fallback option, but no card in this answer has it" \
|
|
in out["message"]
|
|
|
|
|
|
async def test_a_forced_answer_still_gets_the_note(wp_client, fake_llm):
|
|
# 9B smoke s-tag-filter: two refused opens force the answer, which
|
|
# used to skip the note, so plain colours stood without the tag ref.
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
sky = {"name": "Sky", "var": "", "options": [{"text": "crimson sky"}]}
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(sky)}]},
|
|
{"message": "", "calls": [{"tool": "open", "args": json.dumps({"proposal": "p1"})}]},
|
|
{"message": "", "calls": [{"tool": "open", "args": json.dumps({"proposal": "p1"})}]},
|
|
{"message": "Proposed Sky.", "calls": []},
|
|
{"message": "Proposed Sky.", "calls": []},
|
|
{"message": "Proposed Sky.", "calls": []},
|
|
])
|
|
await _chat(wp_client, "Make a Sky wildcard whose options use only warm colours from Palette.")
|
|
told = _told(fake_llm, len(fake_llm.seen) - 1)
|
|
assert told.count("check the cards against the request") == 1
|
|
|
|
|
|
async def test_a_step_after_the_card_that_changes_nothing_gets_the_note(wp_client, fake_llm):
|
|
# 9B smoke d25d274: card, more_tools, open, then the answer, and only
|
|
# then the note. The note now comes with the first step that changes nothing.
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(WEAPON)}]},
|
|
{"message": "", "calls": [{"tool": "more_tools", "args": json.dumps({"group": "web"})}]},
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(
|
|
{**WEAPON, "options": [*WEAPON["options"], {"text": "fists", "fallback": True}]})}]},
|
|
{"message": "Proposed Weapon with a fallback.", "calls": []},
|
|
])
|
|
out = await _chat(wp_client, ASK)
|
|
log = " ".join(str(m["content"]) for m in fake_llm.seen[-1]["body"]["messages"])
|
|
assert log.count("check the cards against the request") == 1
|
|
assert log.index("check the cards against the request") < log.index("fists")
|
|
[card] = out["proposals"]
|
|
assert any(o.get("fallback") for o in card["op"]["body"]["payload"]["options"])
|
|
|
|
|
|
async def test_an_answer_on_the_last_step_still_gets_the_note(wp_client, fake_llm, monkeypatch):
|
|
# 9B smoke d25d274 full library: the answer came on the last step, so
|
|
# the note had no step left for the fix and never ran.
|
|
from engine.ai import chat as ai_chat
|
|
monkeypatch.setattr(ai_chat, "MAX_STEPS", 2)
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(WEAPON)}]},
|
|
{"message": "Proposed Weapon.", "calls": []},
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(
|
|
{**WEAPON, "options": [*WEAPON["options"], {"text": "fists", "fallback": True}]})}]},
|
|
{"message": "Proposed Weapon with a fallback.", "calls": []},
|
|
])
|
|
out = await _chat(wp_client, ASK)
|
|
[card] = out["proposals"]
|
|
assert any(o.get("fallback") for o in card["op"]["body"]["payload"]["options"])
|
|
assert "ran out of steps" not in out["message"]
|
|
|
|
|
|
async def test_the_same_card_after_the_note_is_told_what_it_lacks(wp_client, fake_llm):
|
|
# 9B smoke s-tag-filter: three runs answered the note with the same card.
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
same = {"tool": "create_wildcard", "args": json.dumps(WEAPON)}
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [same]},
|
|
{"message": "", "calls": [{"tool": "more_tools", "args": json.dumps({"group": "web"})}]},
|
|
{"message": "", "calls": [same]},
|
|
{"message": "Proposed Weapon.", "calls": []},
|
|
])
|
|
await _chat(wp_client, ASK)
|
|
log = " ".join(str(m["content"]) for m in fake_llm.seen[-1]["body"]["messages"])
|
|
assert "It still lacks a fallback option" in log
|
|
[tail] = [str(m["content"]) for m in fake_llm.seen[-1]["body"]["messages"]
|
|
if "check the cards against" in str(m["content"])]
|
|
assert "don't propose these again" not in tail
|
|
|
|
|
|
async def test_a_repeat_before_any_note_gets_the_check(wp_client, fake_llm):
|
|
# 9B smoke 234bce5 full s42: the same card twice, then constraints;
|
|
# the note came only at the answer, with create_wildcard closed.
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
same = {"tool": "create_wildcard", "args": json.dumps(WEAPON)}
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [same]},
|
|
{"message": "", "calls": [same]},
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(
|
|
{**WEAPON, "options": [*WEAPON["options"], {"text": "fists", "fallback": True}]})}]},
|
|
{"message": "Proposed Weapon with a fallback.", "calls": []},
|
|
])
|
|
out = await _chat(wp_client, ASK)
|
|
log = " ".join(str(m["content"]) for m in fake_llm.seen[-1]["body"]["messages"])
|
|
assert "It still lacks a fallback option" in log
|
|
assert "check the cards against the request" not in log
|
|
assert any(o.get("fallback") for p in out["proposals"]
|
|
for o in p["op"]["body"]["payload"]["options"])
|
|
|
|
|
|
async def test_a_repeat_of_a_card_that_has_it_gets_the_plain_refusal(wp_client, fake_llm):
|
|
# 234bce5 empty s42: the fixed card sent again was told it lacked the ref.
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
fixed = {"tool": "create_wildcard", "args": json.dumps(
|
|
{**WEAPON, "options": [*WEAPON["options"], {"text": "fists", "fallback": True}]})}
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [fixed]},
|
|
{"message": "", "calls": [fixed]},
|
|
{"message": "Proposed Weapon with a fallback.", "calls": []},
|
|
])
|
|
await _chat(wp_client, ASK)
|
|
log = " ".join(str(m["content"]) for m in fake_llm.seen[-1]["body"]["messages"])
|
|
assert "still lacks" not in log and "you already proposed exactly this" in log
|
|
|
|
|
|
async def test_a_card_that_drops_the_asked_feature_is_told(wp_client, fake_llm):
|
|
# 9B smoke e760fd2 empty s42: the fixed Sky card rewritten back to
|
|
# plain colours to get rid of a duplicate warning.
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
good = {**WEAPON, "options": [*WEAPON["options"], {"text": "fists", "fallback": True}]}
|
|
bad = {**WEAPON, "options": [*WEAPON["options"], {"text": "fists"}]}
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(good)}]},
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(bad)}]},
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(good)}]},
|
|
{"message": "Proposed Weapon with a fallback.", "calls": []},
|
|
])
|
|
await _chat(wp_client, ASK)
|
|
log = " ".join(str(m["content"]) for m in fake_llm.seen[-1]["body"]["messages"])
|
|
assert log.count("This card drops a fallback option") == 1
|
|
|
|
|
|
async def test_a_card_resent_with_only_its_tags_changed_still_lacks_it(wp_client, fake_llm):
|
|
# 9B smoke e760fd2 full s7: Sky re-sent with a category, then with its
|
|
# option tags flipped, each accepted, never with the ref.
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(WEAPON)}]},
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(
|
|
{**WEAPON, "category": "Props"})}]},
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(
|
|
{**WEAPON, "options": [*WEAPON["options"], {"text": "fists", "fallback": True}]})}]},
|
|
{"message": "Proposed Weapon with a fallback.", "calls": []},
|
|
])
|
|
await _chat(wp_client, ASK)
|
|
log = " ".join(str(m["content"]) for m in fake_llm.seen[-1]["body"]["messages"])
|
|
assert "It still lacks a fallback option" in log
|
|
assert "check the cards against the request" not in log
|
|
|
|
|
|
async def test_claims_the_cards_do_not_back_are_marked(wp_client, fake_llm):
|
|
# Bench B3 / W13 / C5: "created successfully" before Apply, a weight no
|
|
# card sets, and a tool name in the reply.
|
|
_on(fake_llm, model_size="small", tool_format="json")
|
|
fake_llm.chat_steps.extend([
|
|
{"message": "", "calls": [{"tool": "create_wildcard", "args": json.dumps(WEAPON)}]},
|
|
*[{"message": "Weapon has been created successfully with the create_wildcard tool; "
|
|
"bow is at weight 0.5.", "calls": []}] * 4,
|
|
])
|
|
out = await _chat(wp_client, "make a weapon wildcard, bows rare")
|
|
msg = out["message"]
|
|
assert "Nothing has changed yet" in msg
|
|
assert "names a weight or factor (0.5) that no card sets" in msg
|
|
assert "create_wildcard" not in msg and "a new wildcard" in msg
|