Show multi-environment execution locations
This commit is contained in:
Ken Simpson
@@ -257,7 +257,7 @@ These state colours work with both the legacy LiteGraph renderer and ComfyUI's e
|
||||
|
||||
Subgraph nodes inherit a derived decoration whenever any nested descendant is set to run on Modal, including through multiple levels of nested subgraphs. The container remains a local organizational node: the derived highlight does not change its `Run on Modal` setting. During execution, a `Σ` badge identifies the decoration as a descendant summary and its tooltip lists the descendant phase counts. When descendants have conflicting phases, the border uses the most operationally important state in this order: error, cancelling, active, starting, setup, ready, finalizing, then complete. For example, a subgraph containing one active node and one starting node is purple for active work while the tooltip preserves both counts.
|
||||
|
||||
Remote sampler-style progress is rendered in a small temporary panel near the node. Each active progress bar includes a smoothed iterations-per-second (`it/s`) rate derived from successive streamed progress samples; LLM token streams use engine-reported `tok/s`, show token count and TTFT, and reuse the header for compact startup-stage text. Parallel mapped lanes report their rates independently. Static progress redraws happen only when progress events arrive, while indeterminate LLM work, pulsing node phases, setup lane placeholders, and short fade-outs use a throttled canvas animation loop. Preview images and ComfyUI UI payloads emitted by remote nodes are streamed back into the local PromptServer while the remote component is still running. When the browser regains focus, Modal-Sync replays recent UI events and reconciles them against ComfyUI queue/history state so cancelled or completed prompts do not leave stale progress bars behind.
|
||||
Remote sampler-style progress is rendered in a small temporary panel near the node. Each active progress bar includes a smoothed iterations-per-second (`it/s`) rate derived from successive streamed progress samples; LLM token streams use engine-reported `tok/s`, show token count and TTFT, and reuse the header for compact startup-stage text. Parallel mapped lanes report their rates independently. When one prompt is actually scheduled across more than one remote environment, the panel adds a compact execution-location row beneath its bars: the Modal mark precedes the exact running container ID once known, while the SSH mark precedes the configured host name. Long identities are clipped to the panel width. Static progress redraws happen only when progress events arrive, while indeterminate LLM work, pulsing node phases, setup lane placeholders, and short fade-outs use a throttled canvas animation loop. Preview images and ComfyUI UI payloads emitted by remote nodes are streamed back into the local PromptServer while the remote component is still running. When the browser regains focus, Modal-Sync replays recent UI events and reconciles them against ComfyUI queue/history state so cancelled or completed prompts do not leave stale progress bars behind.
|
||||
|
||||
Cancelling a local prompt propagates a targeted interrupt to the active Modal work. If Modal is still deploying, provisioning, or slow to observe the interrupt, the local proxy releases the ComfyUI prompt after the configured grace period while remote cleanup continues.
|
||||
|
||||
|
||||
+73
-36
@@ -1097,11 +1097,14 @@ def _stamp_execution_assignment(
|
||||
assignment: ExecutionAssignment,
|
||||
worker_index: int = 0,
|
||||
execution_history_signature: str | None = None,
|
||||
execution_location: str | None = None,
|
||||
) -> None:
|
||||
"""Attach provider placement to a payload and every nested proxy phase."""
|
||||
payload["execution_provider"] = assignment.provider.value
|
||||
payload["execution_environment_id"] = assignment.environment_id
|
||||
payload["execution_worker_index"] = worker_index
|
||||
if execution_location:
|
||||
payload["execution_location"] = execution_location
|
||||
if execution_history_signature:
|
||||
payload["execution_history_signature"] = execution_history_signature
|
||||
split_payloads = payload.get("split_proxy_payloads")
|
||||
@@ -1124,9 +1127,29 @@ def _stamp_execution_assignment(
|
||||
assignment,
|
||||
worker_index,
|
||||
execution_history_signature,
|
||||
execution_location,
|
||||
)
|
||||
|
||||
|
||||
def _ssh_hostname(ssh_target: str) -> str:
|
||||
"""Return the host portion of one validated OpenSSH destination."""
|
||||
host = ssh_target.rsplit("@", 1)[-1].strip()
|
||||
if host.startswith("[") and "]" in host:
|
||||
return host[1 : host.index("]")]
|
||||
return host
|
||||
|
||||
|
||||
def _execution_location_for_assignment(
|
||||
assignment: ExecutionAssignment,
|
||||
ssh_hosts_by_id: Mapping[str, SshHostConfig],
|
||||
) -> str | None:
|
||||
"""Return the runtime location label known before remote dispatch."""
|
||||
if assignment.provider is ExecutionProvider.MODAL:
|
||||
return None
|
||||
host = ssh_hosts_by_id.get(assignment.environment_id)
|
||||
return _ssh_hostname(host.ssh_target) if host is not None else assignment.environment_id
|
||||
|
||||
|
||||
def _ensure_remote_sync_backend(
|
||||
settings: ModalSyncSettings,
|
||||
sync_engine: ModalAssetSyncEngine,
|
||||
@@ -6216,6 +6239,7 @@ def rewrite_prompt_for_modal(
|
||||
0,
|
||||
),
|
||||
_component_execution_signature(component, prompt),
|
||||
_execution_location_for_assignment(assignment, ssh_hosts_by_id),
|
||||
)
|
||||
implicitly_mapped_output_sources = _implicitly_mapped_boundary_output_sources(
|
||||
component=component,
|
||||
@@ -6233,6 +6257,14 @@ def rewrite_prompt_for_modal(
|
||||
payload=payload,
|
||||
nodes_module=resolved_nodes_module,
|
||||
)
|
||||
for proxy_node_id in proxy_node_ids:
|
||||
summary.execution_assignments_by_representative[proxy_node_id] = assignment
|
||||
summary.execution_worker_indices_by_representative[proxy_node_id] = (
|
||||
summary.execution_worker_indices_by_representative.get(
|
||||
component.representative_node_id,
|
||||
0,
|
||||
)
|
||||
)
|
||||
split_proxy_payloads = payload.get("split_proxy_payloads")
|
||||
if isinstance(split_proxy_payloads, dict):
|
||||
static_proxy_node_id, mapped_proxy_node_id = proxy_node_ids
|
||||
@@ -6374,6 +6406,39 @@ async def rewrite_prompt_for_modal_async(
|
||||
)
|
||||
|
||||
|
||||
def _execution_assignments_payload(
|
||||
summary: RewriteSummary,
|
||||
settings: ModalSyncSettings,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Return planner assignments with member nodes and safe runtime labels."""
|
||||
ssh_hosts_by_id = {
|
||||
host.environment_id: host for host in _configured_ssh_hosts(settings)
|
||||
}
|
||||
return {
|
||||
component_id: {
|
||||
"provider": assignment.provider.value,
|
||||
"environment_id": assignment.environment_id,
|
||||
"execution_location": _execution_location_for_assignment(
|
||||
assignment,
|
||||
ssh_hosts_by_id,
|
||||
),
|
||||
"node_ids": list(
|
||||
summary.component_node_ids_by_representative.get(component_id, [])
|
||||
),
|
||||
"predicted_cost_usd": assignment.predicted_cost_usd,
|
||||
"predicted_completion_seconds": assignment.predicted_completion_seconds,
|
||||
"worker_index": summary.execution_worker_indices_by_representative.get(
|
||||
component_id,
|
||||
0,
|
||||
),
|
||||
"reasons": list(assignment.reasons),
|
||||
}
|
||||
for component_id, assignment in sorted(
|
||||
summary.execution_assignments_by_representative.items()
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
async def _queue_prompt_json(
|
||||
prompt_server: Any,
|
||||
json_data: dict[str, Any],
|
||||
@@ -7206,24 +7271,10 @@ def setup_modal_queue_route(
|
||||
asset.remote_path for asset in summary.synced_assets
|
||||
]
|
||||
json_data["extra_data"]["remote_execution"] = {
|
||||
"assignments": {
|
||||
component_id: {
|
||||
"provider": assignment.provider.value,
|
||||
"environment_id": assignment.environment_id,
|
||||
"predicted_cost_usd": assignment.predicted_cost_usd,
|
||||
"predicted_completion_seconds": (
|
||||
assignment.predicted_completion_seconds
|
||||
),
|
||||
"worker_index": summary.execution_worker_indices_by_representative.get(
|
||||
component_id,
|
||||
0,
|
||||
),
|
||||
"reasons": list(assignment.reasons),
|
||||
}
|
||||
for component_id, assignment in sorted(
|
||||
summary.execution_assignments_by_representative.items()
|
||||
)
|
||||
}
|
||||
"assignments": _execution_assignments_payload(
|
||||
summary,
|
||||
request_settings,
|
||||
)
|
||||
}
|
||||
if summary.custom_nodes_bundle is not None:
|
||||
json_data["extra_data"]["modal"][
|
||||
@@ -7252,24 +7303,10 @@ def setup_modal_queue_route(
|
||||
"modal_parallel_local_branch_node_ids": list(
|
||||
summary.parallel_local_branch_node_ids
|
||||
),
|
||||
"remote_execution_assignments": {
|
||||
component_id: {
|
||||
"provider": assignment.provider.value,
|
||||
"environment_id": assignment.environment_id,
|
||||
"predicted_cost_usd": assignment.predicted_cost_usd,
|
||||
"predicted_completion_seconds": (
|
||||
assignment.predicted_completion_seconds
|
||||
),
|
||||
"worker_index": summary.execution_worker_indices_by_representative.get(
|
||||
component_id,
|
||||
0,
|
||||
),
|
||||
"reasons": list(assignment.reasons),
|
||||
}
|
||||
for component_id, assignment in sorted(
|
||||
summary.execution_assignments_by_representative.items()
|
||||
)
|
||||
},
|
||||
"remote_execution_assignments": _execution_assignments_payload(
|
||||
summary,
|
||||
request_settings,
|
||||
),
|
||||
"modal_components": [
|
||||
{
|
||||
"representative_node_id": representative_node_id,
|
||||
|
||||
@@ -481,3 +481,4 @@
|
||||
- Replacing a provider image's entrypoint does not replace its Docker health check. The llama-server base probes HTTP port 8080, while the managed SSH worker initially serves only its Unix socket; explicitly emit `HEALTHCHECK NONE` in the derived image so an otherwise ready worker is not trapped in an unhealthy startup state.
|
||||
- Copying an executable from a provider image does not make its sibling shared libraries discoverable after the derived image overrides `LD_LIBRARY_PATH`. Preserve `/app` in the SSH image library path and also prefix the configured llama-server binary directory in the spawned process environment; validate with `ldd` or an actual server launch because image construction alone does not reveal the missing dynamic dependency.
|
||||
- Provider-aware placement must happen before remote-session bridging. Cut explicit backend boundaries during component formation, preserve an acyclic coarse graph when later expensive-edge merges would wrap around that cut, and materialize cross-environment values through ComfyUI; otherwise session co-location silently overwrites the scheduler's provider decisions. Modal speculative prewarm and local-gap keepalive metadata must carry and match both provider and environment identity, because recursively embedded SSH payloads can otherwise stage self-hosted-only models on Modal.
|
||||
- A node execution-location badge needs both queue-time and runtime identity. Return the planner's provider/environment assignment keyed to rewritten proxy representatives so every original member node can render immediately, then replace the Modal environment label with the exact task/container ID learned from the remote log stream. For SSH, derive the display hostname from the configured target instead of exposing the scheduler's environment key. Gate the row on the count of distinct environments actually assigned to that prompt, not the number configured, so a one-environment plan stays visually quiet.
|
||||
|
||||
@@ -3679,6 +3679,9 @@ def _emit_local_modal_progress(
|
||||
time_to_first_token_seconds: float | None = None,
|
||||
tokens_per_second: float | None = None,
|
||||
pre_gpu: bool = False,
|
||||
execution_provider: str | None = None,
|
||||
execution_environment_id: str | None = None,
|
||||
execution_location: str | None = None,
|
||||
) -> None:
|
||||
"""Forward remote numeric and stage progress into the local websocket stream."""
|
||||
if client_id is None:
|
||||
@@ -3730,6 +3733,12 @@ def _emit_local_modal_progress(
|
||||
payload["tokens_per_second"] = float(tokens_per_second)
|
||||
if pre_gpu:
|
||||
payload["pre_gpu"] = True
|
||||
if execution_provider:
|
||||
payload["execution_provider"] = execution_provider
|
||||
if execution_environment_id:
|
||||
payload["execution_environment_id"] = execution_environment_id
|
||||
if execution_location:
|
||||
payload["execution_location"] = execution_location
|
||||
_record_local_modal_ui_event("modal_progress", payload, client_id)
|
||||
prompt_server.send_sync("modal_progress", payload, client_id)
|
||||
|
||||
@@ -4435,6 +4444,7 @@ def _consume_remote_payload_stream(
|
||||
suppress_status_stream = bool(payload.get("suppress_status_stream"))
|
||||
result_payload: bytes | bytearray | None = None
|
||||
suppressed_progress_node_metadata: dict[str, dict[str, str | None]] = {}
|
||||
active_remote_task_id: str | None = None
|
||||
active_remote_log_task_id: str | None = None
|
||||
should_close_stream = False
|
||||
component_id = str(payload.get("component_id") or "payload")
|
||||
@@ -4466,6 +4476,8 @@ def _consume_remote_payload_stream(
|
||||
event_kind = str(stream_event.get("kind", ""))
|
||||
if event_kind == "remote_logs":
|
||||
task_id = _coerce_modal_task_id(stream_event.get("task_id"))
|
||||
if task_id is not None and active_remote_task_id is None:
|
||||
active_remote_task_id = task_id
|
||||
if (
|
||||
task_id is not None
|
||||
and active_remote_log_task_id is None
|
||||
@@ -4577,6 +4589,27 @@ def _consume_remote_payload_stream(
|
||||
),
|
||||
"aggregate_only": aggregate_only,
|
||||
}
|
||||
raw_execution_provider = payload.get("execution_provider")
|
||||
execution_provider = str(
|
||||
raw_execution_provider
|
||||
or ("modal" if active_remote_task_id else "")
|
||||
).strip()
|
||||
execution_environment_id = str(
|
||||
payload.get("execution_environment_id") or ""
|
||||
).strip()
|
||||
execution_location = (
|
||||
active_remote_task_id
|
||||
if execution_provider == "modal" and active_remote_task_id
|
||||
else str(payload.get("execution_location") or "").strip()
|
||||
)
|
||||
if execution_provider:
|
||||
progress_kwargs["execution_provider"] = execution_provider
|
||||
if execution_environment_id:
|
||||
progress_kwargs[
|
||||
"execution_environment_id"
|
||||
] = execution_environment_id
|
||||
if execution_location:
|
||||
progress_kwargs["execution_location"] = execution_location
|
||||
for string_field in ("stage", "message", "unit"):
|
||||
if stream_event.get(string_field) is not None:
|
||||
progress_kwargs[string_field] = str(
|
||||
|
||||
@@ -42,6 +42,7 @@ const transformedSource = `${[
|
||||
"globalThis.__modalToggleExports = {",
|
||||
" ensurePromptState,",
|
||||
" registerPromptComponents,",
|
||||
" registerPromptExecutionAssignments,",
|
||||
" resolveComponentNodeIds,",
|
||||
" handleModalProgress,",
|
||||
" handleModalStatus,",
|
||||
@@ -213,6 +214,119 @@ const progressCanvasContext = {
|
||||
assert.doesNotThrow(() => remoteProgressNode.onDrawForeground(progressCanvasContext));
|
||||
assert.match(progressPanelStrokeStyles.at(-1), /^rgba\(168, 85, 247, 0\.44/);
|
||||
|
||||
resetFrontendState();
|
||||
const mixedLocationNode = {
|
||||
id: 177,
|
||||
comfyClass: "KSampler",
|
||||
properties: { is_modal_remote: true },
|
||||
size: [132, 80],
|
||||
};
|
||||
modalToggle.registerPromptComponents("prompt-mixed-location", ["177", "178"], [
|
||||
{ representative_node_id: "177", node_ids: ["177"] },
|
||||
{ representative_node_id: "178", node_ids: ["178"] },
|
||||
]);
|
||||
modalToggle.registerPromptExecutionAssignments("prompt-mixed-location", {
|
||||
"177": {
|
||||
provider: "modal",
|
||||
environment_id: "modal:B300",
|
||||
node_ids: ["177"],
|
||||
},
|
||||
"178": {
|
||||
provider: "ssh_docker",
|
||||
environment_id: "spark-one",
|
||||
execution_location: "spark-one.internal.example",
|
||||
node_ids: ["178"],
|
||||
},
|
||||
});
|
||||
modalToggle.handleModalProgress({
|
||||
detail: {
|
||||
prompt_id: "prompt-mixed-location",
|
||||
node_id: "177",
|
||||
value: 2,
|
||||
max: 10,
|
||||
execution_provider: "modal",
|
||||
execution_environment_id: "modal:B300",
|
||||
execution_location: "ta-01K3M0DALCONTAINERIDENTITY",
|
||||
},
|
||||
});
|
||||
const mixedLocationState = modalToggle.getRemoteVisualState(mixedLocationNode);
|
||||
assert.equal(mixedLocationState?.scheduledEnvironmentCount, 2);
|
||||
assert.deepEqual(mixedLocationState?.executionLocation, {
|
||||
provider: "modal",
|
||||
label: "ta-01K3M0DALCONTAINERIDENTITY",
|
||||
});
|
||||
const mixedLocationLabels = [];
|
||||
const mixedLocationCanvasContext = {
|
||||
globalAlpha: 1,
|
||||
save() {},
|
||||
restore() {},
|
||||
beginPath() {},
|
||||
roundRect() {},
|
||||
arc() {},
|
||||
fill() {},
|
||||
fillRect() {},
|
||||
stroke() {},
|
||||
fillText(text) {
|
||||
mixedLocationLabels.push(String(text));
|
||||
},
|
||||
measureText(text) {
|
||||
return { width: String(text).length * 6 };
|
||||
},
|
||||
};
|
||||
modalToggle.drawModalNodeDecoration(mixedLocationNode, mixedLocationCanvasContext);
|
||||
assert.equal(mixedLocationLabels.some((label) => label.endsWith("…")), true);
|
||||
|
||||
modalToggle.handleModalProgress({
|
||||
detail: {
|
||||
prompt_id: "prompt-mixed-location",
|
||||
node_id: "178",
|
||||
value: 1,
|
||||
max: 4,
|
||||
execution_provider: "ssh_docker",
|
||||
execution_environment_id: "spark-one",
|
||||
execution_location: "spark-one.internal.example",
|
||||
},
|
||||
});
|
||||
assert.deepEqual(modalToggle.getRemoteVisualState({ id: 178 })?.executionLocation, {
|
||||
provider: "ssh_docker",
|
||||
label: "spark-one.internal.example",
|
||||
});
|
||||
|
||||
resetFrontendState();
|
||||
modalToggle.registerPromptComponents("prompt-single-location", ["179"], [
|
||||
{ representative_node_id: "179", node_ids: ["179"] },
|
||||
]);
|
||||
modalToggle.registerPromptExecutionAssignments("prompt-single-location", {
|
||||
"179": {
|
||||
provider: "ssh_docker",
|
||||
environment_id: "spark-one",
|
||||
execution_location: "spark-one.internal.example",
|
||||
node_ids: ["179"],
|
||||
},
|
||||
});
|
||||
modalToggle.handleModalProgress({
|
||||
detail: {
|
||||
prompt_id: "prompt-single-location",
|
||||
node_id: "179",
|
||||
value: 1,
|
||||
max: 4,
|
||||
},
|
||||
});
|
||||
const singleLocationLabels = [];
|
||||
modalToggle.drawModalNodeDecoration(
|
||||
{ id: 179, size: [132, 80] },
|
||||
{
|
||||
...mixedLocationCanvasContext,
|
||||
fillText(text) {
|
||||
singleLocationLabels.push(String(text));
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
singleLocationLabels.some((label) => label.includes("spark-one")),
|
||||
false,
|
||||
);
|
||||
|
||||
resetFrontendState();
|
||||
modalToggle.registerPromptComponents("prompt-a", ["10", "11", "12"], [
|
||||
{
|
||||
|
||||
@@ -13,6 +13,14 @@ from typing import Any
|
||||
import pytest
|
||||
|
||||
|
||||
def test_ssh_hostname_extracts_safe_runtime_badge_label(
|
||||
api_intercept_module: Any,
|
||||
) -> None:
|
||||
"""Planner UI metadata should show the host rather than an SSH user target."""
|
||||
assert api_intercept_module._ssh_hostname("worker@example.internal") == "example.internal"
|
||||
assert api_intercept_module._ssh_hostname("[2001:db8::17]") == "2001:db8::17"
|
||||
|
||||
|
||||
class _FakeRemoteModelNode:
|
||||
"""Fake node that produces a non-transportable MODEL output."""
|
||||
|
||||
|
||||
@@ -196,3 +196,43 @@ def test_llm_stream_progress_preserves_stage_and_token_metrics(
|
||||
"tokens_per_second": 9.75,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_stream_progress_reports_exact_modal_container_identity(monkeypatch: Any) -> None:
|
||||
"""A Modal task id should follow its component progress into the node badge."""
|
||||
modal_app = _load_modal_app_module()
|
||||
emitted_progress: list[dict[str, Any]] = []
|
||||
monkeypatch.setattr(
|
||||
modal_app,
|
||||
"_emit_local_modal_progress",
|
||||
lambda **kwargs: emitted_progress.append(kwargs),
|
||||
)
|
||||
payload = {
|
||||
"prompt_id": "prompt-location",
|
||||
"component_id": "node-162",
|
||||
"component_node_ids": ["node-162"],
|
||||
"execution_provider": "modal",
|
||||
"execution_environment_id": "modal:B300",
|
||||
"extra_data": {"client_id": "client-1"},
|
||||
}
|
||||
|
||||
modal_app._consume_remote_payload_stream(
|
||||
payload,
|
||||
iter(
|
||||
[
|
||||
{"kind": "remote_logs", "task_id": "ta-01K3MODAL"},
|
||||
{
|
||||
"kind": "progress",
|
||||
"event_type": "node_progress",
|
||||
"node_id": "node-162",
|
||||
"value": 1,
|
||||
"max": 8,
|
||||
},
|
||||
{"kind": "result", "outputs": ["done"]},
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
assert emitted_progress[0]["execution_provider"] == "modal"
|
||||
assert emitted_progress[0]["execution_environment_id"] == "modal:B300"
|
||||
assert emitted_progress[0]["execution_location"] == "ta-01K3MODAL"
|
||||
|
||||
@@ -417,6 +417,19 @@ def test_streamed_modal_node_progress_updates_active_overlay() -> None:
|
||||
assert "function fadeNodeProgress(nodeIdValue, promptId)" in source
|
||||
assert "function progressVisualOpacity(progressState)" in source
|
||||
assert 'api.addEventListener("modal_progress", handleModalProgress);' in source
|
||||
|
||||
|
||||
def test_mixed_remote_environments_render_clipped_runtime_locations() -> None:
|
||||
"""Mixed planner assignments should add provider identity beneath progress bars."""
|
||||
source = _modal_toggle_source()
|
||||
|
||||
assert "function registerPromptExecutionAssignments(promptId, assignments)" in source
|
||||
assert "promptState.scheduledEnvironmentCount = environmentIds.size;" in source
|
||||
assert "updateNodeExecutionLocation(promptId, progressNodeId, detail);" in source
|
||||
assert "function drawRemoteExecutionLocation(" in source
|
||||
assert "Number(state?.scheduledEnvironmentCount ?? 0) > 1" in source
|
||||
assert "fitCanvasText(ctx, label, availableTextWidth)" in source
|
||||
assert "REMOTE_LOCATION_ICON_SOURCES" in source
|
||||
assert "function handleModalProgress(event)" in source
|
||||
assert "if (detail.aggregate_only) {" in source
|
||||
assert "setNodeBatchProgress(" in source
|
||||
|
||||
+202
-7
@@ -56,6 +56,14 @@ const FINALIZING_NODE_BORDER_COLOR = "#00358A";
|
||||
const ERROR_BORDER_COLOR = "#ef4444";
|
||||
const LOCAL_BOTTLENECK_BADGE_BORDER_COLOR = "rgba(148, 163, 184, 0.72)";
|
||||
const LOCAL_BOTTLENECK_TOOLTIP = "Did you mean to make this node execute on Modal?";
|
||||
const REMOTE_LOCATION_ICON_SOURCES = {
|
||||
modal: `data:image/svg+xml;charset=utf-8,${encodeURIComponent(
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 368 192"><path fill="#34D058" d="M0 128 73 0l37 64-73 128z"/><path fill="#00B75A" d="m110 64 74 0-74 128H37z"/><path fill="#8AEF74" d="M73 0h74l37 64h-74z"/><path fill="#34D058" d="M184 64 220 0l74 128-37 64z"/><path fill="#A7F59A" d="M220 0h74l74 128h-74z"/><path fill="#00B75A" d="m294 128h74l-37 64h-74z"/></svg>',
|
||||
)}`,
|
||||
ssh_docker: `data:image/svg+xml;charset=utf-8,${encodeURIComponent(
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 73 73"><rect x="1" y="1" width="71" height="71" rx="14" fill="#fff" stroke="#000" stroke-width="2"/><path d="M60.24 17.02H12.66A1.66 1.66 0 0 0 11 18.68v8.26h50.9v-8.26a1.66 1.66 0 0 0-1.66-1.66Zm-17.2 7.18a2.21 2.21 0 1 1 0-4.43 2.21 2.21 0 0 1 0 4.43Zm6.03 0a2.21 2.21 0 1 1 0-4.43 2.21 2.21 0 0 1 0 4.43Zm6.03 0a2.21 2.21 0 1 1 0-4.43 2.21 2.21 0 0 1 0 4.43ZM11 30.26v23.83c0 .92.74 1.66 1.66 1.66h47.58c.92 0 1.66-.74 1.66-1.66V30.26H11Zm23.41 14.01-5.44 4.62a1.66 1.66 0 0 1-2.15-2.53l3.95-3.35-3.95-3.36a1.66 1.66 0 1 1 2.15-2.53l5.44 4.62a1.66 1.66 0 0 1 0 2.53ZM45 49.29h-6.55a1.66 1.66 0 1 1 0-3.32H45a1.66 1.66 0 1 1 0 3.32Z"/></svg>',
|
||||
)}`,
|
||||
};
|
||||
|
||||
const STATE_SETUP = "setup";
|
||||
const STATE_STARTING = "starting";
|
||||
@@ -92,6 +100,7 @@ const modalSandwichedLocalNodeIds = new Set();
|
||||
const modalRemoteDescendantNodeIdsByAncestor = new Map();
|
||||
const syntheticPromptUiStates = new Map();
|
||||
const modalGlobalStatusStates = new Map();
|
||||
const remoteLocationIconImages = new Map();
|
||||
|
||||
let animationFrameHandle = null;
|
||||
let modalLastAnimationRedrawAt = 0;
|
||||
@@ -296,6 +305,27 @@ function fitCanvasText(ctx, text, maxWidth) {
|
||||
return `${safeText.slice(0, end)}…`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a lazily loaded provider icon for the canvas location line.
|
||||
* @param {string | null | undefined} provider
|
||||
* @returns {HTMLImageElement | null}
|
||||
*/
|
||||
function remoteLocationIcon(provider) {
|
||||
const safeProvider = String(provider ?? "");
|
||||
const source = REMOTE_LOCATION_ICON_SOURCES[safeProvider];
|
||||
if (!source || typeof Image !== "function") {
|
||||
return null;
|
||||
}
|
||||
if (!remoteLocationIconImages.has(safeProvider)) {
|
||||
const icon = new Image();
|
||||
icon.decoding = "async";
|
||||
icon.src = source;
|
||||
icon.addEventListener?.("load", refreshNodeDecorations, { once: true });
|
||||
remoteLocationIconImages.set(safeProvider, icon);
|
||||
}
|
||||
return remoteLocationIconImages.get(safeProvider) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw an iteration-rate label over a progress bar with a tight opaque backing.
|
||||
* @param {CanvasRenderingContext2D} ctx
|
||||
@@ -1755,7 +1785,7 @@ function refreshModalUiAfterVisibilityChange() {
|
||||
/**
|
||||
* Return the prompt metadata bucket, creating it if needed.
|
||||
* @param {string} promptId
|
||||
* @returns {{ startedAt: number, modalGpu: string | null, executionLabel: string | null, remoteNodeIds: string[], componentsByRepresentative: Map<string, string[]>, componentNodeIdsByMember: Map<string, string[]>, representativeNodeIdByMember: Map<string, string>, componentLabelByMember: Map<string, string>, laneNodeIdsByLane: Map<string, string> }}
|
||||
* @returns {{ startedAt: number, modalGpu: string | null, executionLabel: string | null, scheduledEnvironmentCount: number, remoteNodeIds: string[], componentsByRepresentative: Map<string, string[]>, componentNodeIdsByMember: Map<string, string[]>, representativeNodeIdByMember: Map<string, string>, componentLabelByMember: Map<string, string>, executionAssignmentsByRepresentative: Map<string, object>, executionAssignmentByMember: Map<string, object>, laneNodeIdsByLane: Map<string, string> }}
|
||||
*/
|
||||
function ensurePromptState(promptId) {
|
||||
if (isPromptTerminal(promptId)) {
|
||||
@@ -1767,11 +1797,14 @@ function ensurePromptState(promptId) {
|
||||
startedAt,
|
||||
modalGpu: null,
|
||||
executionLabel: null,
|
||||
scheduledEnvironmentCount: 0,
|
||||
remoteNodeIds: [],
|
||||
componentsByRepresentative: new Map(),
|
||||
componentNodeIdsByMember: new Map(),
|
||||
representativeNodeIdByMember: new Map(),
|
||||
componentLabelByMember: new Map(),
|
||||
executionAssignmentsByRepresentative: new Map(),
|
||||
executionAssignmentByMember: new Map(),
|
||||
descendantNodeIdsByAncestor: new Map(),
|
||||
laneNodeIdsByLane: new Map(),
|
||||
activeNodeId: null,
|
||||
@@ -2094,6 +2127,82 @@ function registerPromptComponents(promptId, remoteNodeIds, components) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store planner assignments by both representative and visible member node.
|
||||
* @param {string} promptId
|
||||
* @param {Record<string, Record<string, any>>} assignments
|
||||
*/
|
||||
function registerPromptExecutionAssignments(promptId, assignments) {
|
||||
const promptState = ensurePromptState(promptId);
|
||||
if (!promptState) {
|
||||
return;
|
||||
}
|
||||
promptState.executionAssignmentsByRepresentative.clear();
|
||||
promptState.executionAssignmentByMember.clear();
|
||||
const environmentIds = new Set();
|
||||
|
||||
for (const [representativeNodeIdValue, rawAssignment] of Object.entries(assignments ?? {})) {
|
||||
if (!rawAssignment || typeof rawAssignment !== "object") {
|
||||
continue;
|
||||
}
|
||||
const representativeNodeId = String(representativeNodeIdValue);
|
||||
const environmentId = String(rawAssignment.environment_id ?? "").trim();
|
||||
const assignment = {
|
||||
provider: String(rawAssignment.provider ?? "modal"),
|
||||
environmentId,
|
||||
location: String(rawAssignment.execution_location ?? "").trim() || null,
|
||||
};
|
||||
if (environmentId) {
|
||||
environmentIds.add(environmentId);
|
||||
}
|
||||
promptState.executionAssignmentsByRepresentative.set(representativeNodeId, assignment);
|
||||
const assignedNodeIds = Array.isArray(rawAssignment.node_ids) && rawAssignment.node_ids.length > 0
|
||||
? rawAssignment.node_ids.map((nodeIdValue) => String(nodeIdValue))
|
||||
: (promptState.componentNodeIdsByMember.get(representativeNodeId) ?? [representativeNodeId]);
|
||||
for (const assignedNodeId of assignedNodeIds) {
|
||||
promptState.executionAssignmentByMember.set(assignedNodeId, assignment);
|
||||
}
|
||||
promptState.executionAssignmentByMember.set(representativeNodeId, assignment);
|
||||
}
|
||||
promptState.scheduledEnvironmentCount = environmentIds.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update one component's runtime identity from its streamed progress metadata.
|
||||
* @param {string} promptId
|
||||
* @param {string} nodeIdValue
|
||||
* @param {Record<string, any>} metadata
|
||||
*/
|
||||
function updateNodeExecutionLocation(promptId, nodeIdValue, metadata) {
|
||||
const promptState = modalPromptStates.get(promptId);
|
||||
const location = String(metadata.execution_location ?? "").trim();
|
||||
if (!promptState || !location) {
|
||||
return;
|
||||
}
|
||||
const safeNodeIdValue = String(nodeIdValue);
|
||||
const representativeNodeId =
|
||||
promptState.representativeNodeIdByMember.get(safeNodeIdValue) ?? safeNodeIdValue;
|
||||
const existingAssignment =
|
||||
promptState.executionAssignmentByMember.get(safeNodeIdValue) ??
|
||||
promptState.executionAssignmentsByRepresentative.get(representativeNodeId) ??
|
||||
{};
|
||||
const assignment = {
|
||||
...existingAssignment,
|
||||
provider: String(metadata.execution_provider ?? existingAssignment.provider ?? "modal"),
|
||||
environmentId: String(
|
||||
metadata.execution_environment_id ?? existingAssignment.environmentId ?? "",
|
||||
),
|
||||
location,
|
||||
};
|
||||
promptState.executionAssignmentsByRepresentative.set(representativeNodeId, assignment);
|
||||
const componentNodeIds =
|
||||
promptState.componentNodeIdsByMember.get(safeNodeIdValue) ?? [safeNodeIdValue];
|
||||
for (const componentNodeId of componentNodeIds) {
|
||||
promptState.executionAssignmentByMember.set(String(componentNodeId), assignment);
|
||||
}
|
||||
promptState.executionAssignmentByMember.set(representativeNodeId, assignment);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return ancestor node ids for a composed subgraph prompt id like `24:23`.
|
||||
* @param {string} nodeIdValue
|
||||
@@ -3490,6 +3599,8 @@ function getRemoteVisualState(node) {
|
||||
const cachedState = nodeCachedState(visualNodeId, state.promptId);
|
||||
const batchProgressState = modalNodeBatchProgress.get(visualNodeId) ?? null;
|
||||
const hasLiveProgress = hasLiveNodeProgress(visualNodeId, state.promptId);
|
||||
const executionAssignment =
|
||||
promptState?.executionAssignmentByMember.get(visualNodeId) ?? null;
|
||||
return {
|
||||
...state,
|
||||
phase: deriveRemoteNodePhase(state.phase, hasLiveProgress),
|
||||
@@ -3503,6 +3614,13 @@ function getRemoteVisualState(node) {
|
||||
progress: progressState,
|
||||
batchProgress: batchProgressState?.promptId === state.promptId ? batchProgressState : null,
|
||||
progressLanes,
|
||||
scheduledEnvironmentCount: promptState?.scheduledEnvironmentCount ?? 0,
|
||||
executionLocation: executionAssignment?.location
|
||||
? {
|
||||
provider: executionAssignment.provider,
|
||||
label: executionAssignment.location,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3939,6 +4057,51 @@ function updateLegacyModalTooltip(graphCanvas, tooltip) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw a clipped provider icon and runtime identity beneath node progress bars.
|
||||
* @param {CanvasRenderingContext2D} ctx
|
||||
* @param {{ provider?: string, label?: string }} location
|
||||
* @param {number} rowY
|
||||
* @param {number} rowHeight
|
||||
* @param {number} nodeWidth
|
||||
* @param {number} paddingX
|
||||
* @param {number} scale
|
||||
*/
|
||||
function drawRemoteExecutionLocation(
|
||||
ctx,
|
||||
location,
|
||||
rowY,
|
||||
rowHeight,
|
||||
nodeWidth,
|
||||
paddingX,
|
||||
scale,
|
||||
) {
|
||||
const provider = String(location?.provider ?? "");
|
||||
const label = String(location?.label ?? "");
|
||||
if (!label) {
|
||||
return;
|
||||
}
|
||||
const icon = remoteLocationIcon(provider);
|
||||
const iconHeight = 10 / scale;
|
||||
const iconWidth = (provider === "modal" ? 19 : 10) / scale;
|
||||
const iconX = paddingX;
|
||||
const iconY = rowY + (rowHeight - iconHeight) / 2;
|
||||
if (icon?.complete && Number(icon.naturalWidth) > 0 && typeof ctx.drawImage === "function") {
|
||||
ctx.drawImage(icon, iconX, iconY, iconWidth, iconHeight);
|
||||
}
|
||||
const textX = iconX + iconWidth + 5 / scale;
|
||||
const availableTextWidth = Math.max(0, nodeWidth - paddingX - textX);
|
||||
ctx.fillStyle = "#cbd5e1";
|
||||
ctx.font = `${Math.max(8 / scale, 6.5)}px ui-sans-serif, system-ui, sans-serif`;
|
||||
ctx.textAlign = "left";
|
||||
ctx.textBaseline = "middle";
|
||||
ctx.fillText(
|
||||
fitCanvasText(ctx, label, availableTextWidth),
|
||||
textX,
|
||||
rowY + rowHeight / 2,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw the Modal execution decoration or neutral local-bottleneck badge for a node.
|
||||
* @param {LGraphNode} node
|
||||
@@ -4065,13 +4228,25 @@ function drawModalNodeDecoration(node, ctx) {
|
||||
const visibleLaneProgress = hasLaneProgress ? visibleActiveProgressLanes : setupProgressLanes;
|
||||
const hasVisibleLaneProgress = visibleLaneProgress.length > 0;
|
||||
const hasIterationRateLabels = hasAggregateProgress || hasLaneProgress;
|
||||
const executionLocation =
|
||||
Number(state?.scheduledEnvironmentCount ?? 0) > 1 ? state?.executionLocation : null;
|
||||
const hasExecutionLocation = Boolean(executionLocation?.label);
|
||||
const progressBarWidth = barWidth;
|
||||
const laneBlockHeight = hasVisibleLaneProgress
|
||||
? visibleLaneProgress.length * laneHeight + (visibleLaneProgress.length - 1) * laneGap
|
||||
: 0;
|
||||
const bodyHeight =
|
||||
const progressBodyHeight =
|
||||
(hasVisibleLaneProgress ? laneBlockHeight + laneGap : 0) + (hasAggregateProgress ? aggregateHeight : 0);
|
||||
const panelHeight = panelPaddingY * 2 + headerHeight + bodyHeight;
|
||||
const progressTopGap = progressBodyHeight > 0 ? laneGap : 0;
|
||||
const locationRowHeight = 13 / scale;
|
||||
const locationTopGap = hasExecutionLocation ? 5 / scale : 0;
|
||||
const panelHeight =
|
||||
panelPaddingY * 2 +
|
||||
headerHeight +
|
||||
progressTopGap +
|
||||
progressBodyHeight +
|
||||
locationTopGap +
|
||||
(hasExecutionLocation ? locationRowHeight : 0);
|
||||
const laneColors = [
|
||||
"rgba(196, 181, 253, 0.94)",
|
||||
"rgba(147, 197, 253, 0.94)",
|
||||
@@ -4154,7 +4329,7 @@ function drawModalNodeDecoration(node, ctx) {
|
||||
ctx.fillText(badgeText, badgeX + badgeWidth / 2, badgeY + badgeHeight / 2);
|
||||
}
|
||||
|
||||
const barY = panelY + panelPaddingY + headerHeight + laneGap;
|
||||
const barY = panelY + panelPaddingY + headerHeight + progressTopGap;
|
||||
|
||||
if (hasIterationRateLabels) {
|
||||
ctx.font = `${Math.max(9 / scale, 7)}px ui-sans-serif, system-ui, sans-serif`;
|
||||
@@ -4219,6 +4394,24 @@ function drawModalNodeDecoration(node, ctx) {
|
||||
state.progress.unit,
|
||||
);
|
||||
}
|
||||
if (hasExecutionLocation) {
|
||||
const locationY =
|
||||
panelY +
|
||||
panelPaddingY +
|
||||
headerHeight +
|
||||
progressTopGap +
|
||||
progressBodyHeight +
|
||||
locationTopGap;
|
||||
drawRemoteExecutionLocation(
|
||||
ctx,
|
||||
executionLocation,
|
||||
locationY,
|
||||
locationRowHeight,
|
||||
node.size[0],
|
||||
panelPaddingX,
|
||||
scale,
|
||||
);
|
||||
}
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
@@ -4513,6 +4706,7 @@ function handleModalProgress(event) {
|
||||
if (!detail.cached_hit && !detail.pre_gpu) {
|
||||
promptState.hasRemoteExecutionStarted = true;
|
||||
}
|
||||
updateNodeExecutionLocation(promptId, progressNodeId, detail);
|
||||
const componentNodeIds = resolveComponentNodeIds(promptId, progressNodeId);
|
||||
const readyNodeIds = (componentNodeIds ?? []).filter((nodeIdValue) => nodeIdValue !== progressNodeId);
|
||||
promptState.hasStreamedProgress = true;
|
||||
@@ -4999,9 +5193,9 @@ function patchQueuePrompt() {
|
||||
const acceptedModalGpu = MODAL_GPU_TYPES.includes(responsePayload.modal_gpu)
|
||||
? responsePayload.modal_gpu
|
||||
: modalGpu;
|
||||
const executionAssignments = Object.values(
|
||||
responsePayload.remote_execution_assignments ?? {},
|
||||
);
|
||||
const executionAssignmentsByRepresentative =
|
||||
responsePayload.remote_execution_assignments ?? {};
|
||||
const executionAssignments = Object.values(executionAssignmentsByRepresentative);
|
||||
const usesSshExecution = executionAssignments.some(
|
||||
(assignment) => assignment?.provider === "ssh_docker",
|
||||
);
|
||||
@@ -5023,6 +5217,7 @@ function patchQueuePrompt() {
|
||||
if (resolvedRemoteNodeIds.length > 0 || resolvedComponents.length > 0) {
|
||||
registerPromptComponents(promptId, resolvedRemoteNodeIds, resolvedComponents);
|
||||
}
|
||||
registerPromptExecutionAssignments(promptId, executionAssignmentsByRepresentative);
|
||||
const promptState = ensurePromptState(promptId);
|
||||
if (!promptState) {
|
||||
return responsePayload;
|
||||
|
||||
Reference in New Issue
Block a user