From b0346e71ccc21e16d60a37a38b8f0ded1beab827 Mon Sep 17 00:00:00 2001 From: Ken Simpson Date: Fri, 17 Apr 2026 12:41:22 -0700 Subject: [PATCH] feat: show modal execution state on canvas --- README.md | 9 +- api_intercept.py | 85 +++++++ docs/LEARNINGS.md | 1 + tests/test_api_intercept.py | 46 ++++ web/modal_toggle.js | 448 +++++++++++++++++++++++++++++++++--- 5 files changed, 559 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 0dd476f..ae659e6 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ## What is implemented -- A frontend extension in [web/modal_toggle.js](/home/user/git/ComfyUI-Modal/web/modal_toggle.js) that injects a `Run on Modal` toggle into every non-internal node, draws a blue border around remote nodes, and submits queued prompts to `/modal/queue_prompt`. +- A frontend extension in [web/modal_toggle.js](/home/user/git/ComfyUI-Modal/web/modal_toggle.js) that injects a `Run on Modal` toggle into every non-internal node, draws live remote-state overlays on marked nodes, and submits queued prompts to `/modal/queue_prompt`. - A backend route in [api_intercept.py](/home/user/git/ComfyUI-Modal/api_intercept.py) that reads the workflow snapshot from `extra_pnginfo.workflow`, identifies connected remote-marked node components, syncs referenced assets, and rewrites each component into one signature-preserving Modal proxy node. - A dynamic proxy registry in [modal_executor_node.py](/home/user/git/ComfyUI-Modal/modal_executor_node.py) that mirrors the exported output count and output types of each remote component so ComfyUI validation still succeeds after rewrite. - A content-addressable sync engine in [sync_engine.py](/home/user/git/ComfyUI-Modal/sync_engine.py) that mirrors model files and a zipped `custom_nodes/` bundle into storage keyed by SHA256. @@ -145,6 +145,13 @@ The frontend extension intercepts queue submission and sends the prompt to `/mod 4. If custom-node syncing is enabled, the local `custom_nodes/` directory is zipped and mirrored if its content hash changed. 5. The rewritten prompt is submitted to the normal ComfyUI prompt queue. +While that is happening, the frontend now shows remote-node state directly on the canvas: + +- blue border: marked for Modal, idle +- orange pulsing border: queued or still being prepared for remote execution +- green border with light green shading: the remote component is actively executing +- red border: queue-time or execution failure detected + ### 6. What happens during execution When execution reaches a rewritten component proxy: diff --git a/api_intercept.py b/api_intercept.py index 52eac35..28bd859 100644 --- a/api_intercept.py +++ b/api_intercept.py @@ -88,6 +88,7 @@ class RewriteSummary: remote_node_ids: list[str] = field(default_factory=list) remote_component_ids: list[str] = field(default_factory=list) + component_node_ids_by_representative: dict[str, list[str]] = field(default_factory=dict) rewritten_node_id_map: dict[str, str] = field(default_factory=dict) synced_assets: list[SyncedAsset] = field(default_factory=list) custom_nodes_bundle: SyncedAsset | None = None @@ -97,6 +98,42 @@ class ModalPromptValidationError(ValueError): """Raised when a prompt cannot be executed with the current Modal transport.""" +def _emit_modal_status( + prompt_server: Any, + phase: str, + *, + client_id: str | None, + prompt_id: str | None, + node_ids: list[str], + component_node_ids_by_representative: dict[str, list[str]] | None = None, + error_message: str | None = None, +) -> None: + """Send a Modal execution status event to the active websocket client.""" + if client_id is None: + logger.debug("Skipping Modal status event %s because no client id is available.", phase) + return + + payload: dict[str, Any] = { + "phase": phase, + "prompt_id": prompt_id, + "node_ids": list(node_ids), + } + if component_node_ids_by_representative: + payload["components"] = [ + { + "representative_node_id": representative_node_id, + "node_ids": list(component_node_ids), + } + for representative_node_id, component_node_ids in sorted( + component_node_ids_by_representative.items() + ) + ] + if error_message is not None: + payload["error_message"] = error_message + + prompt_server.send_sync("modal_status", payload, client_id) + + def _get_nodes_module() -> Any: """Import the ComfyUI nodes module lazily.""" import nodes @@ -605,6 +642,9 @@ def rewrite_prompt_for_modal( for component in components: summary.remote_component_ids.append(component.representative_node_id) + summary.component_node_ids_by_representative[component.representative_node_id] = list( + component.node_ids + ) for node_id in component.node_ids: summary.rewritten_node_id_map[node_id] = component.representative_node_id @@ -717,11 +757,20 @@ def setup_modal_queue_route( async def modal_queue_prompt(request: web.Request) -> web.Response: """Handle prompt queue requests that include Modal remote markers.""" logger.info("Received Modal queue request.") + json_data: dict[str, Any] | None = None + workflow: dict[str, Any] | None = None + remote_node_ids: list[str] = [] + summary = RewriteSummary() try: request_started_at = time.perf_counter() json_data = await request.json() + json_data.setdefault("prompt_id", str(uuid.uuid4())) + json_data.setdefault("extra_data", {}) + if json_data.get("client_id") is not None: + json_data["extra_data"]["client_id"] = json_data["client_id"] extra_pnginfo = ((json_data.get("extra_data") or {}).get("extra_pnginfo") or {}) workflow = extra_pnginfo.get("workflow") + remote_node_ids = sorted(extract_remote_node_ids(workflow, resolved_settings)) if "prompt" in json_data: rewrite_started_at = time.perf_counter() rewritten_prompt, summary = rewrite_prompt_for_modal( @@ -737,6 +786,7 @@ def setup_modal_queue_route( len(summary.remote_node_ids), len(summary.remote_component_ids), ) + remote_node_ids = list(summary.remote_node_ids) json_data["prompt"] = rewritten_prompt if json_data.get("partial_execution_targets"): rewritten_targets = { @@ -754,6 +804,14 @@ def setup_modal_queue_route( json_data["extra_data"]["modal"]["custom_nodes_bundle"] = ( summary.custom_nodes_bundle.remote_path ) + _emit_modal_status( + prompt_server=prompt_server, + phase="setup", + client_id=str(json_data.get("client_id")) if json_data.get("client_id") else None, + prompt_id=str(json_data.get("prompt_id")) if json_data.get("prompt_id") else None, + node_ids=remote_node_ids, + component_node_ids_by_representative=summary.component_node_ids_by_representative, + ) response = await _queue_prompt_json(prompt_server, json_data) logger.info( "Modal queue request completed in %.3fs.", @@ -762,12 +820,39 @@ def setup_modal_queue_route( return response except FileNotFoundError as exc: logger.exception("Modal asset sync failed.") + if json_data is not None: + _emit_modal_status( + prompt_server=prompt_server, + phase="error", + client_id=str(json_data.get("client_id")) if json_data.get("client_id") else None, + prompt_id=str(json_data.get("prompt_id")) if json_data.get("prompt_id") else None, + node_ids=remote_node_ids, + error_message=str(exc), + ) return web.json_response({"error": str(exc), "node_errors": []}, status=400) except ModalPromptValidationError as exc: logger.exception("Modal prompt validation failed.") + if json_data is not None: + _emit_modal_status( + prompt_server=prompt_server, + phase="error", + client_id=str(json_data.get("client_id")) if json_data.get("client_id") else None, + prompt_id=str(json_data.get("prompt_id")) if json_data.get("prompt_id") else None, + node_ids=remote_node_ids, + error_message=str(exc), + ) return web.json_response({"error": str(exc), "node_errors": []}, status=400) except Exception as exc: logger.exception("Modal queue handler failed.") + if json_data is not None: + _emit_modal_status( + prompt_server=prompt_server, + phase="error", + client_id=str(json_data.get("client_id")) if json_data.get("client_id") else None, + prompt_id=str(json_data.get("prompt_id")) if json_data.get("prompt_id") else None, + node_ids=remote_node_ids, + error_message=str(exc), + ) return web.json_response({"error": str(exc), "node_errors": []}, status=500) _ROUTE_REGISTERED = True diff --git a/docs/LEARNINGS.md b/docs/LEARNINGS.md index 6c32989..c0ef8e9 100644 --- a/docs/LEARNINGS.md +++ b/docs/LEARNINGS.md @@ -13,3 +13,4 @@ - Modal objects defined in-process are not directly callable from an arbitrary long-running app like ComfyUI unless they are inside a hydrated `app.run()` session or looked up from a deployed app with `modal.Cls.from_name(...)`. Remote mode should prefer deployed lookup and only fall back to `app.run()` when no deployment is available. - ComfyUI loads custom node packs from directories under synthetic module names derived from filesystem paths, and for directory-based packs that module key is not guaranteed to be a valid Python import path. Modal container startup requires a real importable module name for `@app.cls` services, so the cloud entrypoint has to live in a separate, stable module such as `comfyui_modal_sync_cloud.py` rather than on the custom-node package module itself. - Shipping the ComfyUI source tree into the Modal image is not enough by itself. The container also needs ComfyUI's import-time Python dependencies such as `psutil`, `aiohttp`, `einops`, `torchsde`, `torchvision`, and related runtime packages, or the remote worker fails before it can execute any graph nodes. +- The frontend can reuse ComfyUI's native `executing`, `execution_error`, and `execution_success` websocket events for remote component visuals as long as the queue route also publishes the representative-node-to-original-node mapping after rewrite. That keeps the UI in sync with actual proxy execution instead of inventing a separate polling channel. diff --git a/tests/test_api_intercept.py b/tests/test_api_intercept.py index b5bbf19..f88dcb7 100644 --- a/tests/test_api_intercept.py +++ b/tests/test_api_intercept.py @@ -132,6 +132,7 @@ def test_rewrite_groups_connected_remote_nodes_into_single_proxy( assert rewritten_prompt["3"]["inputs"]["latent"] == ["1", 0] assert summary.remote_node_ids == ["1", "2"] assert summary.remote_component_ids == ["1"] + assert summary.component_node_ids_by_representative == {"1": ["1", "2"]} assert summary.rewritten_node_id_map == {"1": "1", "2": "1"} assert len(summary.synced_assets) == 1 @@ -281,3 +282,48 @@ def test_rewrite_rejects_non_transportable_remote_outputs( assert "exports node 1 (RemoteClip) output index 0 of type 'CLIP'" in message assert "cannot cross the current local/remote boundary" in message + + +def test_emit_modal_status_targets_prompt_client( + api_intercept_module: Any, +) -> None: + """Modal status events should preserve prompt and component metadata for the UI.""" + + class FakePromptServer: + """Capture websocket events emitted by the queue route.""" + + def __init__(self) -> None: + """Initialize the event sink.""" + self.messages: list[tuple[str, dict[str, Any], str | None]] = [] + + def send_sync(self, event: str, data: dict[str, Any], sid: str | None) -> None: + """Record an emitted websocket message.""" + self.messages.append((event, data, sid)) + + prompt_server = FakePromptServer() + api_intercept_module._emit_modal_status( + prompt_server=prompt_server, + phase="setup", + client_id="client-1", + prompt_id="prompt-1", + node_ids=["4", "5"], + component_node_ids_by_representative={"4": ["4", "5"]}, + ) + + assert prompt_server.messages == [ + ( + "modal_status", + { + "phase": "setup", + "prompt_id": "prompt-1", + "node_ids": ["4", "5"], + "components": [ + { + "representative_node_id": "4", + "node_ids": ["4", "5"], + } + ], + }, + "client-1", + ) + ] diff --git a/web/modal_toggle.js b/web/modal_toggle.js index 94b3aee..4637206 100644 --- a/web/modal_toggle.js +++ b/web/modal_toggle.js @@ -3,9 +3,27 @@ import { PromptExecutionError, api } from "../../scripts/api.js"; const REMOTE_PROPERTY = "is_modal_remote"; const MODAL_ROUTE = "/modal/queue_prompt"; -const REMOTE_BORDER_COLOR = "#1d9bf0"; const INTERNAL_NODE_PREFIX = "ModalUniversalExecutor"; +const IDLE_BORDER_COLOR = "#1d9bf0"; +const SETUP_BORDER_COLOR = "#f59e0b"; +const ACTIVE_BORDER_COLOR = "#22c55e"; +const ERROR_BORDER_COLOR = "#ef4444"; + +const STATE_SETUP = "setup"; +const STATE_EXECUTING = "executing"; +const STATE_COMPLETE = "complete"; +const STATE_ERROR = "error"; + +const COMPLETION_CLEAR_DELAY_MS = 1800; +const ERROR_CLEAR_DELAY_MS = 5000; + +const modalNodeStates = new Map(); +const modalNodeClearTimers = new Map(); +const modalPromptStates = new Map(); + +let animationFrameHandle = null; + /** * Return whether a node should show the Modal toggle. * @param {LGraphNode} node @@ -24,6 +42,218 @@ function isRemoteNode(node) { return Boolean(node?.properties?.[REMOTE_PROPERTY]); } +/** + * Return a websocket event detail payload. + * @param {CustomEvent | object} event + * @returns {Record} + */ +function eventDetail(event) { + return event?.detail ?? event ?? {}; +} + +/** + * Build a prompt id for queue requests. + * @returns {string} + */ +function createPromptId() { + if (globalThis.crypto?.randomUUID) { + return globalThis.crypto.randomUUID(); + } + return `modal-${Date.now()}-${Math.random().toString(16).slice(2)}`; +} + +/** + * Return the current monotonic clock value for UI state ordering. + * @returns {number} + */ +function nowMs() { + return Date.now(); +} + +/** + * Read a node id as a stable string. + * @param {LGraphNode} node + * @returns {string} + */ +function nodeId(node) { + return String(node?.id ?? ""); +} + +/** + * Return the prompt metadata bucket, creating it if needed. + * @param {string} promptId + * @returns {{ startedAt: number, remoteNodeIds: string[], componentsByRepresentative: Map }} + */ +function ensurePromptState(promptId) { + if (!modalPromptStates.has(promptId)) { + modalPromptStates.set(promptId, { + startedAt: nowMs(), + remoteNodeIds: [], + componentsByRepresentative: new Map(), + }); + } + return modalPromptStates.get(promptId); +} + +/** + * Return whether an incoming prompt update should replace the current node state. + * @param {string} nodeIdValue + * @param {string} promptId + * @returns {boolean} + */ +function shouldApplyPromptState(nodeIdValue, promptId) { + const incomingPromptState = modalPromptStates.get(promptId); + if (!incomingPromptState) { + return true; + } + + const currentState = modalNodeStates.get(nodeIdValue); + if (!currentState?.promptId) { + return true; + } + + const currentPromptState = modalPromptStates.get(currentState.promptId); + if (!currentPromptState) { + return true; + } + + return incomingPromptState.startedAt >= currentPromptState.startedAt; +} + +/** + * Mark the canvas dirty and keep animation alive while visual states are active. + */ +function refreshCanvasAnimation() { + app.graph?.setDirtyCanvas(true, true); + const hasAnimatedState = Array.from(modalNodeStates.values()).some((state) => + [STATE_SETUP, STATE_EXECUTING, STATE_COMPLETE, STATE_ERROR].includes(state.phase), + ); + if (!hasAnimatedState) { + animationFrameHandle = null; + return; + } + animationFrameHandle = requestAnimationFrame(refreshCanvasAnimation); +} + +/** + * Ensure the redraw loop is running while remote visual effects are active. + */ +function ensureAnimationLoop() { + if (animationFrameHandle !== null) { + return; + } + animationFrameHandle = requestAnimationFrame(refreshCanvasAnimation); +} + +/** + * Clear any pending visual-state timeout for a node. + * @param {string} nodeIdValue + */ +function clearNodeTimer(nodeIdValue) { + const timerId = modalNodeClearTimers.get(nodeIdValue); + if (timerId !== undefined) { + clearTimeout(timerId); + modalNodeClearTimers.delete(nodeIdValue); + } +} + +/** + * Schedule a node visual state to clear after a delay. + * @param {string} nodeIdValue + * @param {string} promptId + * @param {number} delayMs + */ +function scheduleNodeClear(nodeIdValue, promptId, delayMs) { + clearNodeTimer(nodeIdValue); + const timerId = setTimeout(() => { + const state = modalNodeStates.get(nodeIdValue); + if (state?.promptId !== promptId) { + return; + } + modalNodeStates.delete(nodeIdValue); + modalNodeClearTimers.delete(nodeIdValue); + ensureAnimationLoop(); + app.graph?.setDirtyCanvas(true, true); + }, delayMs); + modalNodeClearTimers.set(nodeIdValue, timerId); +} + +/** + * Set the current visual phase for a list of remote nodes. + * @param {string[]} nodeIds + * @param {string} phase + * @param {string} promptId + * @param {string | undefined} errorMessage + */ +function setNodesPhase(nodeIds, phase, promptId, errorMessage) { + for (const currentNodeId of nodeIds) { + if (!shouldApplyPromptState(currentNodeId, promptId)) { + continue; + } + clearNodeTimer(currentNodeId); + modalNodeStates.set(currentNodeId, { + phase, + promptId, + errorMessage, + updatedAt: nowMs(), + }); + if (phase === STATE_COMPLETE) { + scheduleNodeClear(currentNodeId, promptId, COMPLETION_CLEAR_DELAY_MS); + } else if (phase === STATE_ERROR) { + scheduleNodeClear(currentNodeId, promptId, ERROR_CLEAR_DELAY_MS); + } + } + ensureAnimationLoop(); + app.graph?.setDirtyCanvas(true, true); +} + +/** + * Register remote component membership for a prompt. + * @param {string} promptId + * @param {string[]} remoteNodeIds + * @param {{ representative_node_id: string, node_ids: string[] }[]} components + */ +function registerPromptComponents(promptId, remoteNodeIds, components) { + const promptState = ensurePromptState(promptId); + promptState.remoteNodeIds = [...remoteNodeIds]; + promptState.componentsByRepresentative.clear(); + for (const component of components) { + promptState.componentsByRepresentative.set( + String(component.representative_node_id), + component.node_ids.map((nodeIdValue) => String(nodeIdValue)), + ); + } +} + +/** + * Return remote component node ids for an executing proxy node event. + * @param {string} promptId + * @param {string} representativeNodeId + * @returns {string[] | null} + */ +function resolveComponentNodeIds(promptId, representativeNodeId) { + const promptState = modalPromptStates.get(promptId); + if (!promptState) { + return null; + } + return promptState.componentsByRepresentative.get(representativeNodeId) ?? null; +} + +/** + * Extract remote node ids from a workflow snapshot. + * @param {object | undefined} workflow + * @returns {string[]} + */ +function extractRemoteNodeIds(workflow) { + const remoteNodeIds = []; + for (const node of workflow?.nodes ?? []) { + if (node?.properties?.[REMOTE_PROPERTY]) { + remoteNodeIds.push(String(node.id)); + } + } + return remoteNodeIds; +} + /** * Update the node state and redraw the canvas. * @param {LGraphNode} node @@ -35,6 +265,81 @@ function setRemoteFlag(node, value) { app.graph.setDirtyCanvas(true, true); } +/** + * Return the current remote visual state for a node. + * @param {LGraphNode} node + * @returns {{ phase: string, promptId: string } | null} + */ +function getRemoteVisualState(node) { + return modalNodeStates.get(nodeId(node)) ?? null; +} + +/** + * Draw the remote execution border and shading for a node. + * @param {LGraphNode} node + * @param {CanvasRenderingContext2D} ctx + */ +function drawRemoteNodeDecoration(node, ctx) { + if (!isRemoteNode(node)) { + return; + } + + const state = getRemoteVisualState(node); + const titleHeight = node.constructor?.title_height ?? LiteGraph.NODE_TITLE_HEIGHT ?? 24; + const scale = app.canvas?.ds?.scale ?? 1; + const borderWidth = 3 / scale; + const elapsed = performance.now() / 1000; + + let borderColor = IDLE_BORDER_COLOR; + let shadowColor = "rgba(29, 155, 240, 0.35)"; + let fillColor = null; + + if (state?.phase === STATE_SETUP) { + const pulse = (Math.sin(elapsed * 5) + 1) / 2; + borderColor = `${SETUP_BORDER_COLOR}${Math.round((0.65 + pulse * 0.35) * 255) + .toString(16) + .padStart(2, "0")}`; + shadowColor = `rgba(245, 158, 11, ${0.25 + pulse * 0.35})`; + } else if (state?.phase === STATE_EXECUTING) { + const pulse = (Math.sin(elapsed * 6) + 1) / 2; + borderColor = ACTIVE_BORDER_COLOR; + shadowColor = "rgba(34, 197, 94, 0.35)"; + fillColor = `rgba(134, 239, 172, ${0.12 + pulse * 0.08})`; + } else if (state?.phase === STATE_COMPLETE) { + borderColor = ACTIVE_BORDER_COLOR; + shadowColor = "rgba(34, 197, 94, 0.28)"; + fillColor = "rgba(134, 239, 172, 0.14)"; + } else if (state?.phase === STATE_ERROR) { + const pulse = (Math.sin(elapsed * 6) + 1) / 2; + borderColor = `${ERROR_BORDER_COLOR}${Math.round((0.7 + pulse * 0.3) * 255) + .toString(16) + .padStart(2, "0")}`; + shadowColor = `rgba(239, 68, 68, ${0.22 + pulse * 0.28})`; + } + + ctx.save(); + if (fillColor) { + ctx.fillStyle = fillColor; + ctx.fillRect( + borderWidth, + -titleHeight + borderWidth, + Math.max(0, node.size[0] - borderWidth * 2), + Math.max(0, node.size[1] + titleHeight - borderWidth * 2), + ); + } + ctx.strokeStyle = borderColor; + ctx.lineWidth = borderWidth; + ctx.shadowColor = shadowColor; + ctx.shadowBlur = 8 / scale; + ctx.strokeRect( + -borderWidth, + -titleHeight, + node.size[0] + borderWidth * 2, + node.size[1] + titleHeight + borderWidth, + ); + ctx.restore(); +} + /** * Inject the toggle widget and remote border behavior into a node. * @param {LGraphNode} node @@ -66,25 +371,96 @@ function decorateNode(node) { const originalDrawForeground = node.onDrawForeground; node.onDrawForeground = function onDrawForeground(ctx) { originalDrawForeground?.apply(this, arguments); - if (!isRemoteNode(this)) { + drawRemoteNodeDecoration(this, ctx); + }; +} + +/** + * Apply an incoming Modal websocket status event. + * @param {CustomEvent} event + */ +function handleModalStatus(event) { + const detail = eventDetail(event); + const promptId = String(detail.prompt_id ?? ""); + if (!promptId) { + return; + } + + const nodeIds = (detail.node_ids ?? []).map((value) => String(value)); + const components = detail.components ?? []; + if (components.length > 0 || nodeIds.length > 0) { + registerPromptComponents(promptId, nodeIds, components); + } + + if (detail.phase === STATE_SETUP) { + setNodesPhase(nodeIds, STATE_SETUP, promptId); + return; + } + + if (detail.phase === STATE_ERROR) { + setNodesPhase(nodeIds, STATE_ERROR, promptId, detail.error_message); + } +} + +/** + * Update remote component visuals from a native ComfyUI execution event. + * @param {CustomEvent} event + * @param {string} phase + */ +function handleExecutionPhase(event, phase) { + const detail = eventDetail(event); + const promptId = String(detail.prompt_id ?? ""); + const representativeNodeId = String(detail.display_node ?? detail.node ?? detail.node_id ?? ""); + if (!promptId || !representativeNodeId) { + return; + } + + const componentNodeIds = resolveComponentNodeIds(promptId, representativeNodeId); + if (!componentNodeIds) { + return; + } + setNodesPhase(componentNodeIds, phase, promptId, detail.exception_message); +} + +/** + * Apply a queue-time failure to all remote nodes in the just-submitted workflow. + * @param {string[]} remoteNodeIds + * @param {string} promptId + * @param {Error} error + */ +function markQueueFailure(remoteNodeIds, promptId, error) { + if (remoteNodeIds.length === 0) { + return; + } + setNodesPhase(remoteNodeIds, STATE_ERROR, promptId, String(error?.message ?? error)); +} + +/** + * Register websocket listeners for Modal and execution status updates. + */ +function registerExecutionListeners() { + if (api.__modalExecutionListenersRegistered) { + return; + } + + api.addEventListener("modal_status", handleModalStatus); + api.addEventListener("executing", (event) => handleExecutionPhase(event, STATE_EXECUTING)); + api.addEventListener("executed", (event) => handleExecutionPhase(event, STATE_COMPLETE)); + api.addEventListener("execution_error", (event) => handleExecutionPhase(event, STATE_ERROR)); + api.addEventListener("execution_interrupted", (event) => handleExecutionPhase(event, STATE_ERROR)); + api.addEventListener("execution_success", (event) => { + const detail = eventDetail(event); + const promptId = String(detail.prompt_id ?? ""); + if (!promptId) { return; } - - const titleHeight = this.constructor?.title_height ?? LiteGraph.NODE_TITLE_HEIGHT ?? 24; - const borderWidth = 3 / app.canvas.ds.scale; - ctx.save(); - ctx.strokeStyle = REMOTE_BORDER_COLOR; - ctx.lineWidth = borderWidth; - ctx.shadowColor = "rgba(29, 155, 240, 0.35)"; - ctx.shadowBlur = 8 / app.canvas.ds.scale; - ctx.strokeRect( - -borderWidth, - -titleHeight, - this.size[0] + borderWidth * 2, - this.size[1] + titleHeight + borderWidth, - ); - ctx.restore(); - }; + const promptState = modalPromptStates.get(promptId); + if (!promptState) { + return; + } + setNodesPhase(promptState.remoteNodeIds, STATE_COMPLETE, promptId); + }); + api.__modalExecutionListenersRegistered = true; } /** @@ -97,8 +473,16 @@ function patchQueuePrompt() { api.queuePrompt = async function modalQueuePrompt(number, data, options) { const { output: prompt, workflow } = data; + const promptId = createPromptId(); + const remoteNodeIds = extractRemoteNodeIds(workflow); + registerPromptComponents(promptId, remoteNodeIds, []); + if (remoteNodeIds.length > 0) { + setNodesPhase(remoteNodeIds, STATE_SETUP, promptId); + } + const body = { client_id: this.clientId ?? "", + prompt_id: promptId, prompt, ...(options?.partialExecutionTargets && { partial_execution_targets: options.partialExecutionTargets, @@ -116,19 +500,24 @@ function patchQueuePrompt() { body.number = number; } - const response = await this.fetchApi(MODAL_ROUTE, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(body), - }); + try { + const response = await this.fetchApi(MODAL_ROUTE, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); - if (response.status !== 200) { - throw new PromptExecutionError(await response.json()); + if (response.status !== 200) { + throw new PromptExecutionError(await response.json()); + } + + return await response.json(); + } catch (error) { + markQueueFailure(remoteNodeIds, promptId, error); + throw error; } - - return await response.json(); }; api.__modalQueuePromptPatched = true; @@ -139,6 +528,7 @@ app.registerExtension({ async init() { patchQueuePrompt(); + registerExecutionListeners(); }, async nodeCreated(node) {