"""Panels the AI draws in its task (show_panel), from Anomalous's own blocks, and the answers the user gives on them. A panel is a step of the AI's current task (ai_tasks.py) holding a title and a list of blocks: heading, text, images (from ComfyUI's output folder, with marks: circles or boxes and a label at places given as fractions of the picture), slider (two pictures over each other, a line to drag between them), picks (pictures with a reason and "take this one"), models (the user's, by model_id), diagram (boxes and arrows, laid out by the page; a box can name a canvas node), table, list, params, choices (buttons; one or several picks) and input (a line of text), prompts (the user's prompt box: texts and reference pictures the AI gets as tokens, ai_sealed.py), progress, divider; and area (the user's repaint area beside the AI's, ai_repaint.py). Nothing the AI sends is run or rendered as markup, and no drawing of its own (SVG) is taken: every block is checked here to be one of these shapes, with plain strings and numbers of bounded size, pictures only from the output folder and models only from the user's folders (the parts: ai_panel_parts.py). The page draws them (ui_ai_panel.js). The user's answers come back through POST /anomalous/ai_tasks/{task}/answer and wait in the step until the AI asks for them (get_answers, which may wait a while for one) or makes its next successful MCP tool call (attach_pending). The user can also write to the AI in a task (POST /anomalous/ai_tasks/{task}/message): the message is a step of its own and reaches the AI the same way. """ import re import threading import time import uuid from . import ai_marks, ai_tasks from .ai_panel_parts import ( # noqa: F401 - PanelError and the limits are read as ai_panels.* MAX_ITEMS, MAX_NODES, PanelError, _diagram, _image, _list, _model, _text, _zoom, ) from .mcp_tools import tool_text MAX_BLOCKS = 30 MAX_ROWS = 30 MAX_COLUMNS = 8 MAX_WAIT = 55 TONES = ("normal", "muted", "tip", "warning") LAYOUTS = ("row", "grid", "compare") INTERACTIVE = ("choices", "input", "picks", "prompts") MAX_PROMPT_BOXES = 8 INPAINT_NOTE = ( "inpaint_image is the picture with the area to repaint as its alpha, already in ComfyUI's input folder: do not " "check it with a command (get_image_info(image=inpaint_image, folder='input', include_image=true) shows it). " "Repaint with a workflow, never by painting the file with code. On a workflow that makes this picture (its model " "and prompt: get_image_info on the picture says them), add_nodes(use=\"repaint\", image=inpaint_image) " "builds the repaint in one call, wired into its sampler (the best repaint this ComfyUI can run); set_prompt " "the sampler's prompt to what belongs in the area (\"a hand, five fingers\"), not the whole picture. Keep the block's low denoise (it keeps the thing's size; higher paints a new one filling the whole " "area). run_workflow with batch 4 (each try gets its own seed) and let the user pick in a picks panel zoomed on the " "area (zoom {x, y, w, h} around it), then show the old and new with a slider panel, zoomed the same. Say in " "report what you set and why." ) _answered = threading.Condition(ai_tasks._lock) def clean_block(block, at, taken): """One block as the page may draw it, or PanelError.""" where = f"blocks[{at}]" if not isinstance(block, dict) or not isinstance(block.get("type"), str): raise PanelError(f"{where} must be an object with a type") kind = block["type"] out = {"type": kind} if kind == "heading": out["text"] = _text(block.get("text"), 120, f"{where}.text", True) elif kind == "text": out["text"] = _text(block.get("text"), 4000, f"{where}.text", True) out["tone"] = block.get("tone") if block.get("tone") in TONES else "normal" elif kind == "images": items = _list(block.get("items"), MAX_ITEMS, f"{where}.items") if not items: raise PanelError(f"{where}.items is empty") out["items"] = [_image(item, f"{where}.items[{i}]") for i, item in enumerate(items)] out["layout"] = block.get("layout") if block.get("layout") in LAYOUTS else "row" elif kind == "slider": out.update(before=_image(block.get("before"), f"{where}.before"), after=_image(block.get("after"), f"{where}.after"), before_label=_text(block.get("before_label"), 40, f"{where}.before_label"), after_label=_text(block.get("after_label"), 40, f"{where}.after_label")) if block.get("zoom"): out["zoom"] = _zoom(block["zoom"], f"{where}.zoom") elif kind == "picks": items, ids = [], set() raw = _list(block.get("items"), MAX_ITEMS, f"{where}.items") if not raw: # one is "take this one" (a single base, ai_bases.py) raise PanelError(f"{where}.items needs 1 to {MAX_ITEMS} pictures") for i, item in enumerate(raw): here = f"{where}.items[{i}]" if not isinstance(item, dict): raise PanelError(f"{here} must be {{id, image, label, reason, recommended}}") item_id = _text(item.get("id") or f"p{i + 1}", 40, f"{here}.id") if item_id in ids: raise PanelError(f"{here}.id {item_id!r} twice") ids.add(item_id) picture = _image(item, here) items.append({**picture, "id": item_id, "label": _text(item.get("label"), 60, f"{here}.label"), "reason": _text(item.get("reason"), 200, f"{here}.reason"), "recommended": item.get("recommended") is True}) out.update(question=_text(block.get("question"), 200, f"{where}.question"), items=items, multiple=block.get("multiple") is True) if _page_word(block.get("question_key")): out["question_key"] = block["question_key"] if block.get("zoom"): out["zoom"] = _zoom(block["zoom"], f"{where}.zoom") elif kind == "run_map": # a run's real shape (ai_run_map.py), drawn by the page from the step; only show_run_map makes it out["step"] = _text(block.get("step"), 20, f"{where}.step", True) elif kind == "area": # the user's repaint area and the AI's, pictures in the task's marks folder; only ai_repaint makes it if not all(ai_marks.FILE_NAME.fullmatch(str(block.get(side))) for side in ("yours", "ai")): raise PanelError(f"{where}: an area block shows pictures Anomalous made") out.update({side: block[side] for side in ("yours", "ai")}, grow=int(block.get("grow") or 0), feather=int(block.get("feather") or 0)) elif kind == "diagram": out.update(_diagram(block, where)) elif kind == "models": items = _list(block.get("items"), MAX_ITEMS, f"{where}.items") if not items: raise PanelError(f"{where}.items is empty") out["items"] = [_model(item, f"{where}.items[{i}]") for i, item in enumerate(items)] elif kind == "table": columns = [_text(c, 60, f"{where}.columns") for c in _list(block.get("columns"), MAX_COLUMNS, f"{where}.columns")] rows = [] for r, row in enumerate(_list(block.get("rows"), MAX_ROWS, f"{where}.rows")): cells = _list(row, MAX_COLUMNS, f"{where}.rows[{r}]") rows.append([_text(cell, 200, f"{where}.rows[{r}]") for cell in cells]) if not rows: raise PanelError(f"{where}.rows is empty") out.update(columns=columns, rows=rows) elif kind == "list": items = [_text(item, 300, f"{where}.items") for item in _list(block.get("items"), MAX_ROWS, f"{where}.items")] if not items: raise PanelError(f"{where}.items is empty") out.update(items=items, ordered=block.get("ordered") is True) elif kind == "params": items = [] for i, item in enumerate(_list(block.get("items"), MAX_ROWS, f"{where}.items")): if not isinstance(item, dict): raise PanelError(f"{where}.items[{i}] must be {{label, value}}") items.append({"label": _text(item.get("label"), 60, f"{where}.items[{i}].label", True), "value": _text(item.get("value"), 300, f"{where}.items[{i}].value")}) if not items: raise PanelError(f"{where}.items is empty") out["items"] = items elif kind == "choices": options, ids = [], set() raw = _list(block.get("options"), 8, f"{where}.options") if len(raw) < 2: raise PanelError(f"{where}.options needs 2 to 8 options") for i, option in enumerate(raw): if not isinstance(option, dict): option = {"label": option} label = _text(option.get("label"), 80, f"{where}.options[{i}].label", True) option_id = _text(option.get("id") or f"o{i + 1}", 40, f"{where}.options[{i}].id") if option_id in ids: raise PanelError(f"{where}.options: id {option_id!r} twice") ids.add(option_id) options.append({"id": option_id, "label": label, "detail": _text(option.get("detail"), 200, f"{where}.options[{i}].detail"), **({"recommended": True} if option.get("recommended") is True else {}), **({key: option[key] for key in ("label_key", "detail_key") if _page_word(option.get(key))}), **({"values": {str(key): _text(value, 120, f"{where}.options[{i}].values") for key, value in option["values"].items()}} if isinstance(option.get("values"), dict) and _page_word(option.get("label_key")) else {}), **({"sample": option["sample"]} if re.fullmatch(r"[a-z_]{1,40}", str(option.get("sample"))) else {})}) # other: a line for an answer none of the options is out.update(question=_text(block.get("question"), 200, f"{where}.question"), options=options, multiple=block.get("multiple") is True, other=block.get("other") is True) elif kind == "input": out.update(question=_text(block.get("question"), 200, f"{where}.question", True), placeholder=_text(block.get("placeholder"), 80, f"{where}.placeholder")) elif kind == "prompts": # filled in Anomalous only: the texts are sealed and the AI gets tokens count = block.get("count", 1) if not isinstance(count, int) or isinstance(count, bool) or not 1 <= count <= MAX_PROMPT_BOXES: raise PanelError(f"{where}.count: 1 to {MAX_PROMPT_BOXES} prompts") out.update(question=_text(block.get("question"), 200, f"{where}.question"), count=count, negative=block.get("negative") is True, pictures=block.get("pictures") is True) elif kind == "progress": value, top = block.get("value"), block.get("max", 100) if not all(isinstance(n, (int, float)) and not isinstance(n, bool) for n in (value, top)) or top <= 0: raise PanelError(f"{where}: value and max must be numbers, max above 0") out.update(label=_text(block.get("label"), 120, f"{where}.label"), value=max(0, min(value, top)), max=top) elif kind == "divider": pass else: raise PanelError(f"{where}.type {kind!r} is not one of heading, text, images, slider, picks, models, diagram, " "table, list, params, choices, input, prompts, progress, divider") if kind in INTERACTIVE or block.get("id") is not None: block_id = _text(block.get("id") or f"q{at + 1}", 40, f"{where}.id") if block_id in taken: raise PanelError(f"{where}.id {block_id!r} twice") taken.add(block_id) out["id"] = block_id return out def clean_blocks(blocks): blocks = _list(blocks, MAX_BLOCKS, "blocks") if not blocks: raise PanelError("blocks is empty") taken = set() return [clean_block(block, at, taken) for at, block in enumerate(blocks)] def _block_ids(blocks): """Stable handles for deliberate edits; old records get them only when updated.""" taken = {block["id"] for block in blocks if block.get("id")} out = [] for block in blocks: block = dict(block) if not block.get("id"): block_id = uuid.uuid4().hex[:10] while block_id in taken: block_id = uuid.uuid4().hex[:10] block["id"] = block_id taken.add(block_id) out.append(block) return out def _merge_blocks(existing, incoming): """Keep the order already seen, append new output, edit only identified blocks in place.""" merged = list(existing) by_id = {block["id"]: at for at, block in enumerate(existing) if block.get("id")} used = set() for raw, block in zip(incoming, clean_blocks(incoming)): if block["type"] in INTERACTIVE and not raw.get("id"): # Positional q1/q2 defaults cannot identify a question in a partial append. block.pop("id") at = by_id.get(block.get("id")) if at is None and not block.get("id"): # A client may send its entire previous layout again. That cannot reorder it. at = next((i for i, old in enumerate(existing) if i not in used and {key: value for key, value in old.items() if key != "id"} == block), None) if at is None and block["type"] == "progress": matches = [i for i, old in enumerate(existing) if old["type"] == "progress" and old.get("label") == block.get("label")] if len(matches) == 1: at = matches[0] if at is None: merged.append(block) else: if existing[at]["type"] != block["type"]: raise PanelError("An existing block id must keep its type; use a new id for new output.") merged[at] = {**block, **({"id": existing[at]["id"]} if existing[at].get("id") else {})} used.add(at) if len(merged) > MAX_BLOCKS: raise PanelError(f"The panel would exceed {MAX_BLOCKS} blocks; use show_panel for the next stage of work.") return _block_ids(merged) def _find(panel_id): """(task, step) holding panel `panel_id`, or (None, None). Call under the lock.""" for task in ai_tasks._load().values(): for step in task.get("steps") or []: if step.get("kind") == "panel" and step.get("panel", {}).get("id") == panel_id: return task, step return None, None def _page_word(key): """A word of the page (locales.js) Anomalous's own panels name, shown in the user's language.""" return bool(re.fullmatch(r"ai[A-Z]\w{1,40}", str(key))) def show(client, title, blocks, now=None, session="", task_id=None, tool="show_panel", extra=None, title_key=None): title = _text(title, 120, "title", True) panel = {"id": uuid.uuid4().hex[:10], "title": title, "blocks": _block_ids(clean_blocks(blocks)), "closed": False, **({"title_key": title_key} if _page_word(title_key) else {})} now = time.time() if now is None else now step = {"id": uuid.uuid4().hex[:10], "at": now, "ms": 0, "tool": tool, "kind": "panel", "ok": True, "summary": f"{tool}: {title}", "panel": panel, "answers": {}, "results": [], **(extra or {})} with ai_tasks._lock: task = ai_tasks.current_task(client, now, session=session, task_id=task_id, follow_up=True) ai_tasks._add_step(task, step) ai_tasks._save(task) ai_tasks._notify(task["id"], {"panel": title, "asks": any(b["type"] in INTERACTIVE for b in panel["blocks"])}) return panel def update(panel_id, title=None, blocks=None, closed=None): with ai_tasks._lock: task, step = _find(str(panel_id)) if not step: raise PanelError(f"No panel {panel_id!r}; show_panel returns its id.") # Validate the whole update before replacing the saved panel. panel = dict(step["panel"]) if title is not None: panel["title"] = _text(title, 120, "title", True) if blocks is not None: panel["blocks"] = _merge_blocks(panel["blocks"], blocks) panel["changed"] = time.time() # when its numbers were written: the page greys a progress bar the task went past if closed is not None: panel["closed"] = bool(closed) step["panel"] = panel task["updated"] = max(task.get("updated", 0), time.time()) ai_tasks._save(task) ai_tasks._notify(task["id"]) return panel def answer(task_id, step_id, block_id, choice=None, text=None, now=None, skip=False, extra=None): """The user's answer to one block of a panel (from the page), or `skip` it, with `extra` for the AI (ai_repaint.prepare). Returns the step's answers.""" with _answered: task = ai_tasks._load().get(task_id) step = next((item for item in (task or {}).get("steps") or [] if item.get("id") == step_id), None) if not step or step.get("kind") != "panel": raise PanelError("No such panel") if step["panel"].get("closed"): raise PanelError("The AI closed this panel") block = next((b for b in step["panel"]["blocks"] if b.get("id") == block_id), None) if not block or block["type"] not in INTERACTIVE: raise PanelError("No such question") if block_id in step.setdefault("answers", {}): raise PanelError("Already answered") entry = {"at": time.time() if now is None else now, "delivered": False, **(extra or {})} if skip: entry["skipped"] = True elif block["type"] == "choices" and block.get("other") and not choice and isinstance(text, str) and text.strip(): entry.update(choice=["other"], labels=["other"], text=_text(text, 2000, "text", True)) elif block["type"] == "prompts": if not (extra or {}).get("prompts"): raise PanelError("Prompts are written in Anomalous's prompt box") elif block["type"] in ("choices", "picks"): picked = choice if isinstance(choice, list) else [choice] options = block["options"] if block["type"] == "choices" else block["items"] known = {option["id"]: option.get("label") or option["id"] for option in options} picked = [str(item) for item in picked if str(item) in known] if not picked or (len(picked) > 1 and not block.get("multiple")): raise PanelError("Pick one of the options") entry.update(choice=picked, labels=[known[item] for item in picked]) if (step.get("routes") or {}).get(picked[0]): # what to call for a route (mcp_routes.py) entry["note"] = step["routes"][picked[0]] else: value = _text(text, 2000, "text", True) entry["text"] = value step["answers"][block_id] = entry task["updated"] = max(task.get("updated", 0), entry["at"]) ai_tasks._save(task) _answered.notify_all() answers = dict(step["answers"]) ai_tasks._notify(task_id) return answers MAX_MESSAGE = 2000 def leave_message(task_id, text, refs=None, now=None): """A message from the user to the AI working on task `task_id`, with what its "@" point at and the pictures attached (ai_message_refs.py); words may be left out when something is attached. Returns the step.""" from . import ai_message_refs try: refs = ai_message_refs.clean(refs) except ai_message_refs.RefsError as error: raise PanelError(str(error)) text = _text(text, MAX_MESSAGE, "text", not refs) now = time.time() if now is None else now step = {"id": uuid.uuid4().hex[:10], "at": now, "ms": 0, "tool": "user_message", "kind": "user", "ok": True, "summary": "user_message", "text": text, "delivered": False, "results": [], **({"refs": refs} if refs else {})} with _answered: task = ai_tasks._load().get(task_id) if not task: raise PanelError("No such task") task["steps"].append(step) del task["steps"][:-ai_tasks.MAX_STEPS] task["updated"] = max(task.get("updated", 0), now) task["ended"] = None # writing to the AI goes on with the task: its next call comes back here ai_tasks._save(task) _answered.notify_all() ai_tasks._notify(task_id) return step def _collect(client, panel_id, task_id=None): """Answers and messages not given to the AI yet (its own tasks, task `task_id`, or panel `panel_id` and the messages of that panel's task), marked as given. Under the lock.""" out, waiting, messages, changed = [], [], [], set() tasks = ai_tasks._load() panel_task = _find(panel_id)[0] if panel_id is not None else None named_task = tasks.get(task_id) if task_id else None for task in tasks.values(): if panel_id is not None: if task is not panel_task: continue elif task_id is not None: if task is not named_task: continue elif task.get("client") != client: continue for step in task.get("steps") or []: if step.get("kind") == "user" and not step.get("delivered") and (panel_id is None or task is panel_task): step.update(delivered=True, delivered_at=time.time()) changed.add(task["id"]) message = {"id": step["id"], "task": task["id"], "text": step["text"], "at": step["at"]} if step.get("tool") == "user_activity": message["kind"] = "change" # something the user changed in Anomalous, not words to the AI elif step.get("tool") == "user_pictures": from . import ai_pictures message.update(ai_pictures.message_of(step)) # the pictures themselves go with it elif step.get("refs"): from . import ai_message_refs message.update(ai_message_refs.message_of(step)) # what its "@" point at, pictures attached elif step.get("tool") == "user_stop": message["kind"] = "stop" # the user stopped the AI's run (ai_run_control.py) elif step.get("tool") == "user_toolbox": message["kind"] = "toolbox" # what the user picked for the AI to work with (ai_toolbox.py) elif step.get("tool") == "user_marks": # Marks on a picture: the picture's path for the AI, and the marked one attached (ai_marks.image_items). message.update(kind="marks", image="/".join(part for part in (step["image"].get("subfolder"), step["image"]["filename"]) if part), mode=step.get("mode") or ("local" if step.get("inpaint") else "auto"), _marked=ai_marks.path_of(task["id"], step.get("marked"))) if step.get("inpaint"): message.update(inpaint_image=step["inpaint"], inpaint_note=INPAINT_NOTE) messages.append(message) continue asked = step.get("gate") or {} if asked.get("timed_out") and asked.get("decision") and not asked.get("delivered") and panel_id is None: asked["delivered"] = True changed.add(task["id"]) messages.append({"id": step["id"], "task": task["id"], "kind": "gate", "at": asked.get("at"), "text": (f"The user allowed {step['tool']}: call it again with the same arguments now." if asked["decision"] == "allow" else f"The user said no to {step['tool']}." + (f" They said: {asked['text']}" if asked.get("text") else ""))}) continue panel = step.get("panel") if step.get("kind") != "panel" or (panel_id is not None and panel["id"] != panel_id): continue for block in panel["blocks"]: if block["type"] not in INTERACTIVE: continue given = step.get("answers", {}).get(block["id"]) if not given: if not panel.get("closed"): waiting.append({"panel": panel["id"], "question": block["id"]}) continue if given.get("delivered"): continue given.update(delivered=True, delivered_at=time.time()) changed.add(task["id"]) item = {"panel": panel["id"], "question": block["id"], "asked": block.get("question", "")} item.update({key: given[key] for key in ("choice", "labels", "text", "skipped", "inpaint_image", "note", "recommended_on_timeout", "prompts", "negative", "pictures") if key in given}) out.append(item) for tid in changed: ai_tasks._save(tasks[tid]) return out, waiting, messages, changed def wake(): """Anything waiting in get_answers looks again (the user added something to a task).""" with _answered: _answered.notify_all() def get_answers(client, panel_id=None, wait_seconds=0, task_id=None): """The user's new answers and messages; with `wait_seconds`, waits up to that long (MAX_WAIT) for one: for a panel's question still open, or (no panel named) for anything the user sends.""" deadline = time.monotonic() + max(0, min(float(wait_seconds or 0), MAX_WAIT)) panel_id = str(panel_id) if panel_id else None notified = set() try: with _answered: if panel_id is not None and _find(panel_id)[1] is None: raise PanelError(f"No panel {panel_id!r}; show_panel returns its id.") while True: answers, waiting, messages, changed = _collect(client, panel_id, task_id=task_id) notified.update(changed) left = deadline - time.monotonic() if answers or messages or left <= 0 or (panel_id is not None and not waiting): answer = {"answers": answers, "still_waiting": waiting} if messages: answer["messages"] = messages return answer _answered.wait(left) finally: for tid in notified: ai_tasks._notify(tid) def attach_pending(result, client, task_id=None): """Attaches unread user messages and panel answers to a successful tool result so an AI that does not call get_answers still receives them once.""" if not isinstance(result, dict) or result.get("isError"): return result with _answered: answers, waiting, messages, changed = _collect(client, None, task_id=task_id) for tid in changed: ai_tasks._notify(tid) if not answers and not messages: return result data = result.get("structuredContent") data = dict(data) if isinstance(data, dict) else {} if messages: data["user_messages"] = messages if answers: data["user_answers"] = answers if waiting: data["still_waiting"] = waiting data["user_notice"] = ( "The user sent new input in the AI workbench. Read user_messages / user_answers and follow them; you may " "reply in your chat or in Anomalous (report(reply_to=)), but once you and the user settle what to do, " "record it with log_decision so the decision stays in Anomalous; " "kind=change messages are changes the user made themselves while you worked: keep them, do not undo them; " "kind=marks messages are places the user marked on a picture to change (the marked picture is attached): " "say what you see and how you would fix it (show_panel can show the picture with your own marks), " "ask_user which fix to do, and log_decision the fix chosen (related: the marks' id); " "kind=toolbox messages are the nodes, combos, workflows, prompts, models and ways the user picked for you to use; " "kind=stop: the user stopped your run, so do not run it again until they say what to change; " "kind=pictures: pictures the user handed you (attached), with made_with read from each picture's own workflow." ) pictures = ai_marks.image_items(messages) result["structuredContent"] = data content = list(result.get("content") or []) if content and isinstance(content[0], dict) and content[0].get("type") == "text": content[0] = {"type": "text", "text": tool_text(data)} else: content.insert(0, {"type": "text", "text": tool_text(data)}) result["content"] = content + pictures return result