DemonGatanjieu-Anomalous_Model_Browser/api/ai_panels.py

"""Panels the AI draws in its task (show_panel), from Anomalous's own blocks, and the answers
the user gives on them.

A panel is a step of the AI's current task (ai_tasks.py) holding a title and a list of blocks:
heading, text, images (from ComfyUI's output folder, with marks: circles or boxes and a label
at places given as fractions of the picture), slider (two pictures over each other, a line to
drag between them), picks (pictures with a reason and "take this one"), models (the user's, by
model_id), diagram (boxes and arrows, laid out by the page; a box can name a canvas node),
table, list, params, choices (buttons; one or several picks) and input (a line of text),
prompts (the user's prompt box: texts and reference pictures the AI gets as tokens, ai_sealed.py),
progress, divider; and area (the user's repaint area beside the AI's, ai_repaint.py). Nothing the AI sends is run or rendered as markup, and no drawing of its own
(SVG) is taken: every block is checked here to be one of these shapes, with plain strings and
numbers of bounded size, pictures only from the output folder and models only from the user's
folders (the parts: ai_panel_parts.py). The page draws them (ui_ai_panel.js).

The user's answers come back through POST /anomalous/ai_tasks/{task}/answer and wait in the
step until the AI asks for them (get_answers, which may wait a while for one) or makes its next
successful MCP tool call (attach_pending). The user can also write to the AI in a task
(POST /anomalous/ai_tasks/{task}/message): the message is a step of its own and reaches the AI
the same way.
"""

import re
import threading
import time
import uuid

from . import ai_marks, ai_tasks
from .ai_panel_parts import (  # noqa: F401 - PanelError and the limits are read as ai_panels.*
    MAX_ITEMS, MAX_NODES, PanelError, _diagram, _image, _list, _model, _text, _zoom,
)
from .mcp_tools import tool_text

MAX_BLOCKS = 30
MAX_ROWS = 30
MAX_COLUMNS = 8
MAX_WAIT = 55
TONES = ("normal", "muted", "tip", "warning")
LAYOUTS = ("row", "grid", "compare")
INTERACTIVE = ("choices", "input", "picks", "prompts")
MAX_PROMPT_BOXES = 8
INPAINT_NOTE = (
    "inpaint_image is the picture with the area to repaint as its alpha, already in ComfyUI's input folder: do not "
    "check it with a command (get_image_info(image=inpaint_image, folder='input', include_image=true) shows it). "
    "Repaint with a workflow, never by painting the file with code. On a workflow that makes this picture (its model "
    "and prompt: get_image_info on the picture says them), add_nodes(use=\"repaint\", image=inpaint_image) "
    "builds the repaint in one call, wired into its sampler (the best repaint this ComfyUI can run); set_prompt "
    "the sampler's prompt to what belongs in the area (\"a hand, five fingers\"), not the whole picture. Keep the block's low denoise (it keeps the thing's size; higher paints a new one filling the whole "
    "area). run_workflow with batch 4 (each try gets its own seed) and let the user pick in a picks panel zoomed on the "
    "area (zoom {x, y, w, h} around it), then show the old and new with a slider panel, zoomed the same. Say in "
    "report what you set and why."
)

_answered = threading.Condition(ai_tasks._lock)


def clean_block(block, at, taken):
    """One block as the page may draw it, or PanelError."""
    where = f"blocks[{at}]"
    if not isinstance(block, dict) or not isinstance(block.get("type"), str):
        raise PanelError(f"{where} must be an object with a type")
    kind = block["type"]
    out = {"type": kind}
    if kind == "heading":
        out["text"] = _text(block.get("text"), 120, f"{where}.text", True)
    elif kind == "text":
        out["text"] = _text(block.get("text"), 4000, f"{where}.text", True)
        out["tone"] = block.get("tone") if block.get("tone") in TONES else "normal"
    elif kind == "images":
        items = _list(block.get("items"), MAX_ITEMS, f"{where}.items")
        if not items:
            raise PanelError(f"{where}.items is empty")
        out["items"] = [_image(item, f"{where}.items[{i}]") for i, item in enumerate(items)]
        out["layout"] = block.get("layout") if block.get("layout") in LAYOUTS else "row"
    elif kind == "slider":
        out.update(before=_image(block.get("before"), f"{where}.before"), after=_image(block.get("after"), f"{where}.after"),
                   before_label=_text(block.get("before_label"), 40, f"{where}.before_label"),
                   after_label=_text(block.get("after_label"), 40, f"{where}.after_label"))
        if block.get("zoom"):
            out["zoom"] = _zoom(block["zoom"], f"{where}.zoom")
    elif kind == "picks":
        items, ids = [], set()
        raw = _list(block.get("items"), MAX_ITEMS, f"{where}.items")
        if not raw:  # one is "take this one" (a single base, ai_bases.py)
            raise PanelError(f"{where}.items needs 1 to {MAX_ITEMS} pictures")
        for i, item in enumerate(raw):
            here = f"{where}.items[{i}]"
            if not isinstance(item, dict):
                raise PanelError(f"{here} must be {{id, image, label, reason, recommended}}")
            item_id = _text(item.get("id") or f"p{i + 1}", 40, f"{here}.id")
            if item_id in ids:
                raise PanelError(f"{here}.id {item_id!r} twice")
            ids.add(item_id)
            picture = _image(item, here)
            items.append({**picture, "id": item_id, "label": _text(item.get("label"), 60, f"{here}.label"),
                          "reason": _text(item.get("reason"), 200, f"{here}.reason"), "recommended": item.get("recommended") is True})
        out.update(question=_text(block.get("question"), 200, f"{where}.question"), items=items,
                   multiple=block.get("multiple") is True)
        if _page_word(block.get("question_key")):
            out["question_key"] = block["question_key"]
        if block.get("zoom"):
            out["zoom"] = _zoom(block["zoom"], f"{where}.zoom")
    elif kind == "run_map":  # a run's real shape (ai_run_map.py), drawn by the page from the step; only show_run_map makes it
        out["step"] = _text(block.get("step"), 20, f"{where}.step", True)
    elif kind == "area":  # the user's repaint area and the AI's, pictures in the task's marks folder; only ai_repaint makes it
        if not all(ai_marks.FILE_NAME.fullmatch(str(block.get(side))) for side in ("yours", "ai")):
            raise PanelError(f"{where}: an area block shows pictures Anomalous made")
        out.update({side: block[side] for side in ("yours", "ai")}, grow=int(block.get("grow") or 0), feather=int(block.get("feather") or 0))
    elif kind == "diagram":
        out.update(_diagram(block, where))
    elif kind == "models":
        items = _list(block.get("items"), MAX_ITEMS, f"{where}.items")
        if not items:
            raise PanelError(f"{where}.items is empty")
        out["items"] = [_model(item, f"{where}.items[{i}]") for i, item in enumerate(items)]
    elif kind == "table":
        columns = [_text(c, 60, f"{where}.columns") for c in _list(block.get("columns"), MAX_COLUMNS, f"{where}.columns")]
        rows = []
        for r, row in enumerate(_list(block.get("rows"), MAX_ROWS, f"{where}.rows")):
            cells = _list(row, MAX_COLUMNS, f"{where}.rows[{r}]")
            rows.append([_text(cell, 200, f"{where}.rows[{r}]") for cell in cells])
        if not rows:
            raise PanelError(f"{where}.rows is empty")
        out.update(columns=columns, rows=rows)
    elif kind == "list":
        items = [_text(item, 300, f"{where}.items") for item in _list(block.get("items"), MAX_ROWS, f"{where}.items")]
        if not items:
            raise PanelError(f"{where}.items is empty")
        out.update(items=items, ordered=block.get("ordered") is True)
    elif kind == "params":
        items = []
        for i, item in enumerate(_list(block.get("items"), MAX_ROWS, f"{where}.items")):
            if not isinstance(item, dict):
                raise PanelError(f"{where}.items[{i}] must be {{label, value}}")
            items.append({"label": _text(item.get("label"), 60, f"{where}.items[{i}].label", True),
                          "value": _text(item.get("value"), 300, f"{where}.items[{i}].value")})
        if not items:
            raise PanelError(f"{where}.items is empty")
        out["items"] = items
    elif kind == "choices":
        options, ids = [], set()
        raw = _list(block.get("options"), 8, f"{where}.options")
        if len(raw) < 2:
            raise PanelError(f"{where}.options needs 2 to 8 options")
        for i, option in enumerate(raw):
            if not isinstance(option, dict):
                option = {"label": option}
            label = _text(option.get("label"), 80, f"{where}.options[{i}].label", True)
            option_id = _text(option.get("id") or f"o{i + 1}", 40, f"{where}.options[{i}].id")
            if option_id in ids:
                raise PanelError(f"{where}.options: id {option_id!r} twice")
            ids.add(option_id)
            options.append({"id": option_id, "label": label, "detail": _text(option.get("detail"), 200, f"{where}.options[{i}].detail"),
                            **({"recommended": True} if option.get("recommended") is True else {}),
                            **({key: option[key] for key in ("label_key", "detail_key") if _page_word(option.get(key))}),
                            **({"values": {str(key): _text(value, 120, f"{where}.options[{i}].values") for key, value in option["values"].items()}}
                               if isinstance(option.get("values"), dict) and _page_word(option.get("label_key")) else {}),
                            **({"sample": option["sample"]} if re.fullmatch(r"[a-z_]{1,40}", str(option.get("sample"))) else {})})
        # other: a line for an answer none of the options is
        out.update(question=_text(block.get("question"), 200, f"{where}.question"), options=options,
                   multiple=block.get("multiple") is True, other=block.get("other") is True)
    elif kind == "input":
        out.update(question=_text(block.get("question"), 200, f"{where}.question", True),
                   placeholder=_text(block.get("placeholder"), 80, f"{where}.placeholder"))
    elif kind == "prompts":  # filled in Anomalous only: the texts are sealed and the AI gets tokens
        count = block.get("count", 1)
        if not isinstance(count, int) or isinstance(count, bool) or not 1 <= count <= MAX_PROMPT_BOXES:
            raise PanelError(f"{where}.count: 1 to {MAX_PROMPT_BOXES} prompts")
        out.update(question=_text(block.get("question"), 200, f"{where}.question"), count=count,
                   negative=block.get("negative") is True, pictures=block.get("pictures") is True)
    elif kind == "progress":
        value, top = block.get("value"), block.get("max", 100)
        if not all(isinstance(n, (int, float)) and not isinstance(n, bool) for n in (value, top)) or top <= 0:
            raise PanelError(f"{where}: value and max must be numbers, max above 0")
        out.update(label=_text(block.get("label"), 120, f"{where}.label"), value=max(0, min(value, top)), max=top)
    elif kind == "divider":
        pass
    else:
        raise PanelError(f"{where}.type {kind!r} is not one of heading, text, images, slider, picks, models, diagram, "
                         "table, list, params, choices, input, prompts, progress, divider")
    if kind in INTERACTIVE or block.get("id") is not None:
        block_id = _text(block.get("id") or f"q{at + 1}", 40, f"{where}.id")
        if block_id in taken:
            raise PanelError(f"{where}.id {block_id!r} twice")
        taken.add(block_id)
        out["id"] = block_id
    return out


def clean_blocks(blocks):
    blocks = _list(blocks, MAX_BLOCKS, "blocks")
    if not blocks:
        raise PanelError("blocks is empty")
    taken = set()
    return [clean_block(block, at, taken) for at, block in enumerate(blocks)]


def _block_ids(blocks):
    """Stable handles for deliberate edits; old records get them only when updated."""
    taken = {block["id"] for block in blocks if block.get("id")}
    out = []
    for block in blocks:
        block = dict(block)
        if not block.get("id"):
            block_id = uuid.uuid4().hex[:10]
            while block_id in taken:
                block_id = uuid.uuid4().hex[:10]
            block["id"] = block_id
            taken.add(block_id)
        out.append(block)
    return out


def _merge_blocks(existing, incoming):
    """Keep the order already seen, append new output, edit only identified blocks in place."""
    merged = list(existing)
    by_id = {block["id"]: at for at, block in enumerate(existing) if block.get("id")}
    used = set()
    for raw, block in zip(incoming, clean_blocks(incoming)):
        if block["type"] in INTERACTIVE and not raw.get("id"):
            # Positional q1/q2 defaults cannot identify a question in a partial append.
            block.pop("id")
        at = by_id.get(block.get("id"))
        if at is None and not block.get("id"):
            # A client may send its entire previous layout again. That cannot reorder it.
            at = next((i for i, old in enumerate(existing) if i not in used
                       and {key: value for key, value in old.items() if key != "id"} == block), None)
            if at is None and block["type"] == "progress":
                matches = [i for i, old in enumerate(existing) if old["type"] == "progress"
                           and old.get("label") == block.get("label")]
                if len(matches) == 1:
                    at = matches[0]
        if at is None:
            merged.append(block)
        else:
            if existing[at]["type"] != block["type"]:
                raise PanelError("An existing block id must keep its type; use a new id for new output.")
            merged[at] = {**block, **({"id": existing[at]["id"]} if existing[at].get("id") else {})}
            used.add(at)
    if len(merged) > MAX_BLOCKS:
        raise PanelError(f"The panel would exceed {MAX_BLOCKS} blocks; use show_panel for the next stage of work.")
    return _block_ids(merged)


def _find(panel_id):
    """(task, step) holding panel `panel_id`, or (None, None). Call under the lock."""
    for task in ai_tasks._load().values():
        for step in task.get("steps") or []:
            if step.get("kind") == "panel" and step.get("panel", {}).get("id") == panel_id:
                return task, step
    return None, None


def _page_word(key):
    """A word of the page (locales.js) Anomalous's own panels name, shown in the user's language."""
    return bool(re.fullmatch(r"ai[A-Z]\w{1,40}", str(key)))


def show(client, title, blocks, now=None, session="", task_id=None, tool="show_panel", extra=None, title_key=None):
    title = _text(title, 120, "title", True)
    panel = {"id": uuid.uuid4().hex[:10], "title": title, "blocks": _block_ids(clean_blocks(blocks)), "closed": False,
             **({"title_key": title_key} if _page_word(title_key) else {})}
    now = time.time() if now is None else now
    step = {"id": uuid.uuid4().hex[:10], "at": now, "ms": 0, "tool": tool, "kind": "panel", "ok": True,
            "summary": f"{tool}: {title}", "panel": panel, "answers": {}, "results": [], **(extra or {})}
    with ai_tasks._lock:
        task = ai_tasks.current_task(client, now, session=session, task_id=task_id, follow_up=True)
        ai_tasks._add_step(task, step)
        ai_tasks._save(task)
    ai_tasks._notify(task["id"], {"panel": title, "asks": any(b["type"] in INTERACTIVE for b in panel["blocks"])})
    return panel


def update(panel_id, title=None, blocks=None, closed=None):
    with ai_tasks._lock:
        task, step = _find(str(panel_id))
        if not step:
            raise PanelError(f"No panel {panel_id!r}; show_panel returns its id.")
        # Validate the whole update before replacing the saved panel.
        panel = dict(step["panel"])
        if title is not None:
            panel["title"] = _text(title, 120, "title", True)
        if blocks is not None:
            panel["blocks"] = _merge_blocks(panel["blocks"], blocks)
            panel["changed"] = time.time()  # when its numbers were written: the page greys a progress bar the task went past
        if closed is not None:
            panel["closed"] = bool(closed)
        step["panel"] = panel
        task["updated"] = max(task.get("updated", 0), time.time())
        ai_tasks._save(task)
    ai_tasks._notify(task["id"])
    return panel


def answer(task_id, step_id, block_id, choice=None, text=None, now=None, skip=False, extra=None):
    """The user's answer to one block of a panel (from the page), or `skip` it, with `extra` for the AI
    (ai_repaint.prepare). Returns the step's answers."""
    with _answered:
        task = ai_tasks._load().get(task_id)
        step = next((item for item in (task or {}).get("steps") or [] if item.get("id") == step_id), None)
        if not step or step.get("kind") != "panel":
            raise PanelError("No such panel")
        if step["panel"].get("closed"):
            raise PanelError("The AI closed this panel")
        block = next((b for b in step["panel"]["blocks"] if b.get("id") == block_id), None)
        if not block or block["type"] not in INTERACTIVE:
            raise PanelError("No such question")
        if block_id in step.setdefault("answers", {}):
            raise PanelError("Already answered")
        entry = {"at": time.time() if now is None else now, "delivered": False, **(extra or {})}
        if skip:
            entry["skipped"] = True
        elif block["type"] == "choices" and block.get("other") and not choice and isinstance(text, str) and text.strip():
            entry.update(choice=["other"], labels=["other"], text=_text(text, 2000, "text", True))
        elif block["type"] == "prompts":
            if not (extra or {}).get("prompts"):
                raise PanelError("Prompts are written in Anomalous's prompt box")
        elif block["type"] in ("choices", "picks"):
            picked = choice if isinstance(choice, list) else [choice]
            options = block["options"] if block["type"] == "choices" else block["items"]
            known = {option["id"]: option.get("label") or option["id"] for option in options}
            picked = [str(item) for item in picked if str(item) in known]
            if not picked or (len(picked) > 1 and not block.get("multiple")):
                raise PanelError("Pick one of the options")
            entry.update(choice=picked, labels=[known[item] for item in picked])
            if (step.get("routes") or {}).get(picked[0]):  # what to call for a route (mcp_routes.py)
                entry["note"] = step["routes"][picked[0]]
        else:
            value = _text(text, 2000, "text", True)
            entry["text"] = value
        step["answers"][block_id] = entry
        task["updated"] = max(task.get("updated", 0), entry["at"])
        ai_tasks._save(task)
        _answered.notify_all()
        answers = dict(step["answers"])
    ai_tasks._notify(task_id)
    return answers


MAX_MESSAGE = 2000


def leave_message(task_id, text, refs=None, now=None):
    """A message from the user to the AI working on task `task_id`, with what its "@" point at and
    the pictures attached (ai_message_refs.py); words may be left out when something is attached.
    Returns the step."""
    from . import ai_message_refs

    try:
        refs = ai_message_refs.clean(refs)
    except ai_message_refs.RefsError as error:
        raise PanelError(str(error))
    text = _text(text, MAX_MESSAGE, "text", not refs)
    now = time.time() if now is None else now
    step = {"id": uuid.uuid4().hex[:10], "at": now, "ms": 0, "tool": "user_message", "kind": "user", "ok": True,
            "summary": "user_message", "text": text, "delivered": False, "results": [], **({"refs": refs} if refs else {})}
    with _answered:
        task = ai_tasks._load().get(task_id)
        if not task:
            raise PanelError("No such task")
        task["steps"].append(step)
        del task["steps"][:-ai_tasks.MAX_STEPS]
        task["updated"] = max(task.get("updated", 0), now)
        task["ended"] = None  # writing to the AI goes on with the task: its next call comes back here
        ai_tasks._save(task)
        _answered.notify_all()
    ai_tasks._notify(task_id)
    return step


def _collect(client, panel_id, task_id=None):
    """Answers and messages not given to the AI yet (its own tasks, task `task_id`, or panel
    `panel_id` and the messages of that panel's task), marked as given. Under the lock."""
    out, waiting, messages, changed = [], [], [], set()
    tasks = ai_tasks._load()
    panel_task = _find(panel_id)[0] if panel_id is not None else None
    named_task = tasks.get(task_id) if task_id else None
    for task in tasks.values():
        if panel_id is not None:
            if task is not panel_task:
                continue
        elif task_id is not None:
            if task is not named_task:
                continue
        elif task.get("client") != client:
            continue
        for step in task.get("steps") or []:
            if step.get("kind") == "user" and not step.get("delivered") and (panel_id is None or task is panel_task):
                step.update(delivered=True, delivered_at=time.time())
                changed.add(task["id"])
                message = {"id": step["id"], "task": task["id"], "text": step["text"], "at": step["at"]}
                if step.get("tool") == "user_activity":
                    message["kind"] = "change"  # something the user changed in Anomalous, not words to the AI
                elif step.get("tool") == "user_pictures":
                    from . import ai_pictures
                    message.update(ai_pictures.message_of(step))  # the pictures themselves go with it
                elif step.get("refs"):
                    from . import ai_message_refs
                    message.update(ai_message_refs.message_of(step))  # what its "@" point at, pictures attached
                elif step.get("tool") == "user_stop":
                    message["kind"] = "stop"  # the user stopped the AI's run (ai_run_control.py)
                elif step.get("tool") == "user_toolbox":
                    message["kind"] = "toolbox"  # what the user picked for the AI to work with (ai_toolbox.py)
                elif step.get("tool") == "user_marks":
                    # Marks on a picture: the picture's path for the AI, and the marked one attached (ai_marks.image_items).
                    message.update(kind="marks", image="/".join(part for part in (step["image"].get("subfolder"), step["image"]["filename"]) if part),
                                   mode=step.get("mode") or ("local" if step.get("inpaint") else "auto"),
                                   _marked=ai_marks.path_of(task["id"], step.get("marked")))
                    if step.get("inpaint"):
                        message.update(inpaint_image=step["inpaint"], inpaint_note=INPAINT_NOTE)
                messages.append(message)
                continue
            asked = step.get("gate") or {}
            if asked.get("timed_out") and asked.get("decision") and not asked.get("delivered") and panel_id is None:
                asked["delivered"] = True
                changed.add(task["id"])
                messages.append({"id": step["id"], "task": task["id"], "kind": "gate", "at": asked.get("at"),
                                 "text": (f"The user allowed {step['tool']}: call it again with the same arguments now."
                                          if asked["decision"] == "allow" else
                                          f"The user said no to {step['tool']}." + (f" They said: {asked['text']}" if asked.get("text") else ""))})
                continue
            panel = step.get("panel")
            if step.get("kind") != "panel" or (panel_id is not None and panel["id"] != panel_id):
                continue
            for block in panel["blocks"]:
                if block["type"] not in INTERACTIVE:
                    continue
                given = step.get("answers", {}).get(block["id"])
                if not given:
                    if not panel.get("closed"):
                        waiting.append({"panel": panel["id"], "question": block["id"]})
                    continue
                if given.get("delivered"):
                    continue
                given.update(delivered=True, delivered_at=time.time())
                changed.add(task["id"])
                item = {"panel": panel["id"], "question": block["id"], "asked": block.get("question", "")}
                item.update({key: given[key] for key in ("choice", "labels", "text", "skipped", "inpaint_image", "note", "recommended_on_timeout", "prompts", "negative", "pictures") if key in given})
                out.append(item)
    for tid in changed:
        ai_tasks._save(tasks[tid])
    return out, waiting, messages, changed


def wake():
    """Anything waiting in get_answers looks again (the user added something to a task)."""
    with _answered:
        _answered.notify_all()


def get_answers(client, panel_id=None, wait_seconds=0, task_id=None):
    """The user's new answers and messages; with `wait_seconds`, waits up to that long (MAX_WAIT)
    for one: for a panel's question still open, or (no panel named) for anything the user sends."""
    deadline = time.monotonic() + max(0, min(float(wait_seconds or 0), MAX_WAIT))
    panel_id = str(panel_id) if panel_id else None
    notified = set()
    try:
        with _answered:
            if panel_id is not None and _find(panel_id)[1] is None:
                raise PanelError(f"No panel {panel_id!r}; show_panel returns its id.")
            while True:
                answers, waiting, messages, changed = _collect(client, panel_id, task_id=task_id)
                notified.update(changed)
                left = deadline - time.monotonic()
                if answers or messages or left <= 0 or (panel_id is not None and not waiting):
                    answer = {"answers": answers, "still_waiting": waiting}
                    if messages:
                        answer["messages"] = messages
                    return answer
                _answered.wait(left)
    finally:
        for tid in notified:
            ai_tasks._notify(tid)


def attach_pending(result, client, task_id=None):
    """Attaches unread user messages and panel answers to a successful tool result so an AI
    that does not call get_answers still receives them once."""
    if not isinstance(result, dict) or result.get("isError"):
        return result
    with _answered:
        answers, waiting, messages, changed = _collect(client, None, task_id=task_id)
    for tid in changed:
        ai_tasks._notify(tid)
    if not answers and not messages:
        return result
    data = result.get("structuredContent")
    data = dict(data) if isinstance(data, dict) else {}
    if messages:
        data["user_messages"] = messages
    if answers:
        data["user_answers"] = answers
    if waiting:
        data["still_waiting"] = waiting
    data["user_notice"] = (
        "The user sent new input in the AI workbench. Read user_messages / user_answers and follow them; you may "
        "reply in your chat or in Anomalous (report(reply_to=<its id>)), but once you and the user settle what to do, "
        "record it with log_decision so the decision stays in Anomalous; "
        "kind=change messages are changes the user made themselves while you worked: keep them, do not undo them; "
        "kind=marks messages are places the user marked on a picture to change (the marked picture is attached): "
        "say what you see and how you would fix it (show_panel can show the picture with your own marks), "
        "ask_user which fix to do, and log_decision the fix chosen (related: the marks' id); "
        "kind=toolbox messages are the nodes, combos, workflows, prompts, models and ways the user picked for you to use; "
        "kind=stop: the user stopped your run, so do not run it again until they say what to change; "
        "kind=pictures: pictures the user handed you (attached), with made_with read from each picture's own workflow."
    )
    pictures = ai_marks.image_items(messages)
    result["structuredContent"] = data
    content = list(result.get("content") or [])
    if content and isinstance(content[0], dict) and content[0].get("type") == "text":
        content[0] = {"type": "text", "text": tool_text(data)}
    else:
        content.insert(0, {"type": "text", "text": tool_text(data)})
    result["content"] = content + pictures
    return result