Show provider icons beside progress bars

README.md

@@ -242,7 +242,7 @@ Remote state is shown directly on the canvas, in both the legacy LiteGraph rende
 
 Subgraph containers inherit a derived summary decoration (marked with a `Σ` badge) whenever any nested descendant runs remotely, using the most operationally important descendant state for the border color.
 
-Active remote nodes show a small progress panel with per-bar `it/s` rates, human-readable `B/s` through `TB/s` transfer rates for file downloads, and `tok/s`, token counts, and TTFT for LLM nodes. When a prompt is scheduled across more than one remote environment, the panel also names where each component is running — the Modal container id or the SSH host name. Preview images and UI payloads from remote nodes stream back into the local session while the component is still running, and progress state is reconciled against queue history when the browser regains focus so cancelled prompts do not leave stale bars behind.
+Active remote nodes show a small progress panel with a Modal, SSH terminal, or Vast.ai icon beside every bar, per-bar `it/s` rates, human-readable `B/s` through `TB/s` transfer rates for file downloads, and `tok/s`, token counts, and TTFT for LLM nodes. When a prompt is scheduled across more than one remote environment, the panel also names where each component is running — the Modal container id or the SSH host name. Preview images and UI payloads from remote nodes stream back into the local session while the component is still running, and progress state is reconciled against queue history when the browser regains focus so cancelled prompts do not leave stale bars behind.
 
 Remote planning and capacity acquisition count as active queue work even though they happen before the rewritten prompt enters ComfyUI's native execution queue. During Vast startup, the canvas receives readable updates for rental, worker-image download, SSH readiness, and runtime initialization. A provider-side terminal image-pull failure is reported immediately rather than appearing as an empty queue until the startup timeout expires. A rental that never produces an SSH-usable worker is marked unusable and destroyed after the bounded initialization wait so a later workflow cannot silently reuse it.
 

docs/LEARNINGS.md

@@ -579,3 +579,4 @@
 
 - Live free VRAM is the correct admission signal for an otherwise idle SSH host, but it is misleading when the same ComfyUI queue already has a workflow assigned to that host. Preserve each queued prompt's provider assignment in queue metadata, identify the exact SSH environments owned by earlier prompts, and admit later serialized work against those hosts' reserve-adjusted total capacity. Do not generalize this to every busy GPU: unrelated workloads still need the conservative live-free-memory check, and physically undersized hardware must remain incompatible.
 - A persistent managed worker can retain model allocations after its originating ComfyUI process has exited, so a new process has no queue metadata proving where the VRAM went. Configurator-backed planning needs the same ownership-checked idle-worker reclamation as legacy planning: only when live capacity fails but reserve-adjusted nameplate capacity succeeds, inspect the extension-owned worker, remove it only if no execution or staging process is active, re-probe, and then rebuild candidate states.
+- Progress telemetry should retain provider identity independently of the optional execution-location row. A single-environment plan suppresses that row, and mapped worker lanes can outlive aggregate progress, so each aggregate or lane event needs a provider fallback from its planner assignment before the canvas can reliably label every progress bar with the correct backend icon.

tests/modal_toggle_frontend.test.mjs

@@ -31,6 +31,15 @@ globalThis.CustomEvent = class CustomEvent {
     this.detail = init.detail;
   }
 };
+globalThis.Image = class Image {
+  constructor() {
+    this.complete = true;
+    this.naturalWidth = 1;
+    this.src = "";
+  }
+
+  addEventListener() {}
+};
 class FakeElement {
   constructor(tagName) {
     this.tagName = tagName;
@@ -160,6 +169,7 @@ const transformedSource = `${[
   "  stampModalGpuOnWorkflow,",
   "  installModalContextMenu,",
   "  MODAL_GPU_TYPES,",
+  "  REMOTE_LOCATION_ICON_SOURCES,",
   "  clearPromptRemoteStates,",
   "  getRemoteVisualState,",
   "  rebuildRemoteDescendantIndex,",
@@ -296,16 +306,25 @@ modalToggle.registerPromptComponents("prompt-decoration", ["176"], [
     node_ids: ["176"],
   },
 ]);
+modalToggle.registerPromptExecutionAssignments("prompt-decoration", {
+  "176": {
+    provider: "modal",
+    environment_id: "modal:H100",
+    node_ids: ["176"],
+  },
+});
 modalToggle.handleModalProgress({
   detail: {
     prompt_id: "prompt-decoration",
     node_id: "176",
     value: 2,
     max: 10,
+    execution_provider: "modal",
   },
 });
 modalToggle.decorateNode(remoteProgressNode);
 const progressPanelStrokeStyles = [];
+const progressProviderIcons = [];
 const progressCanvasContext = {
   globalAlpha: 1,
   save() {},
@@ -315,6 +334,9 @@ const progressCanvasContext = {
   arc() {},
   fill() {},
   fillRect() {},
+  drawImage(image, x, y, width, height) {
+    progressProviderIcons.push({ src: image.src, x, y, width, height });
+  },
   fillText() {},
   measureText(text) {
     return { width: String(text).length * 6 };
@@ -325,6 +347,16 @@ const progressCanvasContext = {
 };
 assert.doesNotThrow(() => remoteProgressNode.onDrawForeground(progressCanvasContext));
 assert.match(progressPanelStrokeStyles.at(-1), /^rgba\(168, 85, 247, 0\.44/);
+assert.equal(progressProviderIcons.length, 1);
+assert.match(decodeURIComponent(progressProviderIcons[0].src), /viewBox="0 0 368 192"/);
+assert.match(
+  decodeURIComponent(modalToggle.REMOTE_LOCATION_ICON_SOURCES.vast),
+  /viewBox="0 0 54\.3 46\.28"/,
+);
+assert.doesNotMatch(
+  decodeURIComponent(modalToggle.REMOTE_LOCATION_ICON_SOURCES.vast),
+  /viewBox="0 0 173\.64 46\.28"/,
+);
 
 resetFrontendState();
 const mixedLocationNode = {
@@ -420,6 +452,7 @@ const mixedLocationCanvasContext = {
   arc() {},
   fill() {},
   fillRect() {},
+  drawImage() {},
   stroke() {},
   fillText(text) {
     mixedLocationLabels.push(String(text));
@@ -655,6 +688,13 @@ modalToggle.registerPromptComponents("prompt-lane-rate", ["21"], [
     node_ids: ["21"],
   },
 ]);
+modalToggle.registerPromptExecutionAssignments("prompt-lane-rate", {
+  "21": {
+    provider: "ssh_docker",
+    environment_id: "ssh-one",
+    node_ids: ["21"],
+  },
+});
 for (const [laneId, value] of [["0", 1], ["1", 2]]) {
   modalToggle.handleModalProgress({
     detail: {
@@ -663,6 +703,7 @@ for (const [laneId, value] of [["0", 1], ["1", 2]]) {
       value,
       max: 10,
       lane_id: laneId,
+      execution_provider: "ssh_docker",
     },
   });
 }
@@ -675,11 +716,28 @@ for (const [laneId, value] of [["0", 3], ["1", 6]]) {
       value,
       max: 10,
       lane_id: laneId,
+      execution_provider: "ssh_docker",
     },
   });
 }
 assert.equal(modalToggle.modalNodeProgressLanes.get("21")?.lanes.get("0")?.iterationRate, 2);
 assert.equal(modalToggle.modalNodeProgressLanes.get("21")?.lanes.get("1")?.iterationRate, 4);
+const laneProviderIcons = [];
+modalToggle.drawModalNodeDecoration(
+  { id: 21, properties: { is_modal_remote: true }, size: [160, 80] },
+  {
+    ...progressCanvasContext,
+    drawImage(image, x, y, width, height) {
+      laneProviderIcons.push({ src: image.src, x, y, width, height });
+    },
+  },
+);
+assert.equal(laneProviderIcons.length, 2);
+assert.equal(
+  laneProviderIcons.every(({ src }) =>
+    decodeURIComponent(src).includes('viewBox="0 0 73 73"')),
+  true,
+);
 Date.now = originalDateNow;
 
 resetFrontendState();

tests/test_web_modal_toggle.py

@@ -495,6 +495,9 @@ def test_mixed_remote_environments_render_clipped_runtime_locations() -> None:
     assert "Number(state?.scheduledEnvironmentCount ?? 0) > 1" in source
     assert "fitCanvasText(ctx, label, availableTextWidth)" in source
     assert "REMOTE_LOCATION_ICON_SOURCES" in source
+    assert 'viewBox="0 0 54.3 46.28"' in source
+    assert 'viewBox="0 0 173.64 46.28"' not in source
+    assert "function drawProgressProviderIcon(" in source
     assert "function handleModalProgress(event)" in source
     assert "if (detail.aggregate_only) {" in source
     assert "setNodeBatchProgress(" in source
@@ -510,7 +513,8 @@ def test_mixed_remote_environments_render_clipped_runtime_locations() -> None:
     assert "if (state.progress.indeterminate) {" in source
     assert "preGpu: Boolean(metadata.pre_gpu)" in source
     assert "state.phase === STATE_STARTING && state.progress.preGpu" in source
-    assert "function setNodeProgressLane(nodeIdValue, promptId, laneId, value, maxValue, itemIndex, setupOnly = false)" in source
+    assert "function setNodeProgressLane(" in source
+    assert "metadata = {}," in source
     assert "function clearNodeProgressLane(nodeIdValue, promptId, laneId)" in source
     assert "function clearNodeProgress(nodeIdValue, promptId)" in source
     assert "fadeNodeProgress(nodeIdValue, promptId);" in source
@@ -518,7 +522,8 @@ def test_mixed_remote_environments_render_clipped_runtime_locations() -> None:
     assert "function formatIterationRate(iterationRate, unit = null)" in source
     assert "function drawIterationRateOverlay(" in source
     assert 'ctx.fillStyle = "rgba(0, 0, 0, 0.9)";' in source
-    assert "const progressBarWidth = barWidth;" in source
+    assert "const progressIconColumnWidth = hasProgressProviderIcon ? 24 / scale : 0;" in source
+    assert "const progressBarWidth = Math.max(0, barWidth - progressIconColumnWidth);" in source
     assert "iterationRateColumnWidth" not in source
     assert ": progressIterationRate(" in source
     assert "laneProgress.iterationRate," in source
@@ -625,7 +630,8 @@ def test_mapped_lane_setup_events_render_placeholder_lane_bars() -> None:
     """Lane setup events should render translucent placeholder bars before real node progress arrives."""
     source = _modal_toggle_source()
 
-    assert "function setNodeProgressLane(nodeIdValue, promptId, laneId, value, maxValue, itemIndex, setupOnly = false)" in source
+    assert "function setNodeProgressLane(" in source
+    assert "provider: String(" in source
     assert "setupOnly: Boolean(setupOnly)," in source
     assert "Boolean(detail.setup_only)," in source
     assert "const hasSetupLaneProgress = setupProgressLanes.length > 0;" in source

web/modal_toggle.js

@@ -78,7 +78,15 @@ const REMOTE_LOCATION_ICON_SOURCES = {
   ssh_docker: `data:image/svg+xml;charset=utf-8,${encodeURIComponent(
     '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 73 73"><rect x="1" y="1" width="71" height="71" rx="14" fill="#fff" stroke="#000" stroke-width="2"/><path d="M60.24 17.02H12.66A1.66 1.66 0 0 0 11 18.68v8.26h50.9v-8.26a1.66 1.66 0 0 0-1.66-1.66Zm-17.2 7.18a2.21 2.21 0 1 1 0-4.43 2.21 2.21 0 0 1 0 4.43Zm6.03 0a2.21 2.21 0 1 1 0-4.43 2.21 2.21 0 0 1 0 4.43Zm6.03 0a2.21 2.21 0 1 1 0-4.43 2.21 2.21 0 0 1 0 4.43ZM11 30.26v23.83c0 .92.74 1.66 1.66 1.66h47.58c.92 0 1.66-.74 1.66-1.66V30.26H11Zm23.41 14.01-5.44 4.62a1.66 1.66 0 0 1-2.15-2.53l3.95-3.35-3.95-3.36a1.66 1.66 0 1 1 2.15-2.53l5.44 4.62a1.66 1.66 0 0 1 0 2.53ZM45 49.29h-6.55a1.66 1.66 0 1 1 0-3.32H45a1.66 1.66 0 1 1 0 3.32Z"/></svg>',
   )}`,
+  vast: `data:image/svg+xml;charset=utf-8,${encodeURIComponent(
+    '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 54.3 46.28"><path fill="#fff" d="M45.46.04l-12.43,25.42-7.71,16.46c-.16.47-.49.84-.94,1.06-.44.22-.95.25-1.42.09-.47-.16-.85-.49-1.07-.94L4.1,5.73l3.34-1.63,17.08,34.94,1.67-3.56L8.84,0,0,4.32l19.14,39.15c.58,1.18,1.58,2.06,2.82,2.49,1.24.42,2.58.34,3.76-.24.25-.12.5-.27.73-.44.3-.22.56-.53.74-.91.5-1.04,1.81-3.74,1.87-3.85L46.86,4.14l3.34,1.63-17.8,36.4c-.22.44-.59.78-1.07.94-.47.16-.97.13-1.42-.09-.02,0-.04-.03-.06-.04l-1.3,2.78s.01,0,.02.01c.68.34,1.42.5,2.16.5.54,0,1.08-.09,1.6-.27,1.24-.43,2.24-1.31,2.82-2.49L54.3,4.36,45.46.04Z"/></svg>',
+  )}`,
 };
+const REMOTE_LOCATION_ICON_ASPECT_RATIOS = Object.freeze({
+  modal: 368 / 192,
+  ssh_docker: 1,
+  vast: 54.3 / 46.28,
+});
 
 const STATE_SETUP = "setup";
 const STATE_STARTING = "starting";
@@ -382,6 +390,37 @@ function remoteLocationIcon(provider) {
   return remoteLocationIconImages.get(safeProvider) ?? null;
 }
 
+/**
+ * Draw a provider icon in the column immediately before a progress bar.
+ * @param {CanvasRenderingContext2D} ctx
+ * @param {string | null | undefined} provider
+ * @param {number} columnX
+ * @param {number} centerY
+ * @param {number} columnWidth
+ * @param {number} barHeight
+ * @param {number} scale
+ */
+function drawProgressProviderIcon(
+  ctx,
+  provider,
+  columnX,
+  centerY,
+  columnWidth,
+  barHeight,
+  scale,
+) {
+  const safeProvider = String(provider ?? "");
+  const icon = remoteLocationIcon(safeProvider);
+  if (!icon?.complete || !icon.naturalWidth || typeof ctx.drawImage !== "function") {
+    return;
+  }
+  const iconHeight = Math.max(0, Math.min(10 / scale, barHeight - 2 / scale));
+  const aspectRatio = REMOTE_LOCATION_ICON_ASPECT_RATIOS[safeProvider] ?? 1;
+  const iconWidth = Math.min(columnWidth - 4 / scale, iconHeight * aspectRatio);
+  const iconX = columnX + (columnWidth - iconWidth) / 2;
+  ctx.drawImage(icon, iconX, centerY - iconHeight / 2, iconWidth, iconHeight);
+}
+
 /**
  * Draw an iteration-rate label over a progress bar with a tight opaque backing.
  * @param {CanvasRenderingContext2D} ctx
@@ -2935,6 +2974,7 @@ function setNodeProgress(nodeIdValue, promptId, value, maxValue, metadata = {})
       stage: String(metadata.stage ?? ""),
       message: String(metadata.message ?? ""),
       unit: String(metadata.unit ?? ""),
+      provider: String(metadata.execution_provider ?? existingProgress?.provider ?? ""),
       indeterminate: Boolean(metadata.indeterminate),
       preGpu: Boolean(metadata.pre_gpu),
       elapsedSeconds: Number.isFinite(Number(metadata.elapsed_seconds))
@@ -2967,8 +3007,18 @@ function setNodeProgress(nodeIdValue, promptId, value, maxValue, metadata = {})
  * @param {number} maxValue
  * @param {number | null | undefined} itemIndex
  * @param {boolean} setupOnly
+ * @param {Record<string, unknown>} metadata
  */
-function setNodeProgressLane(nodeIdValue, promptId, laneId, value, maxValue, itemIndex, setupOnly = false) {
+function setNodeProgressLane(
+  nodeIdValue,
+  promptId,
+  laneId,
+  value,
+  maxValue,
+  itemIndex,
+  setupOnly = false,
+  metadata = {},
+) {
   const safeLaneId = String(laneId ?? "");
   if (!safeLaneId) {
     return;
@@ -3009,6 +3059,10 @@ function setNodeProgressLane(nodeIdValue, promptId, laneId, value, maxValue, ite
       itemIndex: Number.isFinite(Number(itemIndex)) ? Number(itemIndex) : null,
       updatedAt,
       setupOnly: Boolean(setupOnly),
+      unit: String(metadata.unit ?? existingLaneProgress?.unit ?? ""),
+      provider: String(
+        metadata.execution_provider ?? existingLaneProgress?.provider ?? "",
+      ),
       iterationRate: setupOnly
         ? null
         : progressIterationRate(
@@ -4743,6 +4797,11 @@ function getRemoteVisualState(node) {
     progress: progressState,
     batchProgress: batchProgressState?.promptId === state.promptId ? batchProgressState : null,
     progressLanes,
+    executionProvider:
+      executionAssignment?.provider ??
+      progressState?.provider ??
+      progressLanes.find((laneProgress) => laneProgress.provider)?.provider ??
+      null,
     scheduledEnvironmentCount: promptState?.scheduledEnvironmentCount ?? 0,
     executionLocation: executionAssignment?.location
       ? {
@@ -5425,7 +5484,16 @@ function drawModalNodeDecoration(node, ctx) {
     hasAggregateProgress || hasVisibleLaneProgress || hasBatchBadge;
   const headerHeight = hasProgressContent ? 16 / scale : 0;
   const hasIterationRateLabels = hasAggregateProgress || hasLaneProgress;
-  const progressBarWidth = barWidth;
+  const aggregateProvider = state?.progress?.provider || state?.executionProvider || null;
+  const hasProgressProviderIcon = [
+    aggregateProvider,
+    ...visibleLaneProgress.map(
+      (laneProgress) => laneProgress.provider || state?.executionProvider || null,
+    ),
+  ].some((provider) => Boolean(REMOTE_LOCATION_ICON_SOURCES[String(provider ?? "")]));
+  const progressIconColumnWidth = hasProgressProviderIcon ? 24 / scale : 0;
+  const progressBarX = -borderWidth + progressIconColumnWidth;
+  const progressBarWidth = Math.max(0, barWidth - progressIconColumnWidth);
   const laneBlockHeight = hasVisibleLaneProgress
     ? visibleLaneProgress.length * laneHeight + (visibleLaneProgress.length - 1) * laneGap
     : 0;
@@ -5540,14 +5608,23 @@ function drawModalNodeDecoration(node, ctx) {
         : Math.max(0, Math.min(1, laneProgress.value / laneProgress.max));
       const laneWidth = Math.max(0, progressBarWidth * laneRatio);
       ctx.fillStyle = "rgba(15, 23, 42, 0.66)";
-      ctx.fillRect(-borderWidth, laneY, progressBarWidth, laneHeight);
+      ctx.fillRect(progressBarX, laneY, progressBarWidth, laneHeight);
       const laneColor = laneColors[laneIndex % laneColors.length];
       if (laneProgress.setupOnly) {
         ctx.fillStyle = laneColor.replace("0.94)", `${0.28 + elapsedPulse * 0.22})`);
       } else {
         ctx.fillStyle = laneColor;
       }
-      ctx.fillRect(-borderWidth, laneY, laneWidth, laneHeight);
+      ctx.fillRect(progressBarX, laneY, laneWidth, laneHeight);
+      drawProgressProviderIcon(
+        ctx,
+        laneProgress.provider || state?.executionProvider,
+        -borderWidth,
+        laneY + laneHeight / 2,
+        progressIconColumnWidth,
+        laneHeight,
+        scale,
+      );
       if (!laneProgress.setupOnly) {
         drawIterationRateOverlay(
           ctx,
@@ -5568,16 +5645,25 @@ function drawModalNodeDecoration(node, ctx) {
     const progressWidth = Math.max(0, progressBarWidth * progressRatio);
     const aggregateY = hasVisibleLaneProgress ? barY + laneBlockHeight + laneGap : barY;
     ctx.fillStyle = "rgba(15, 23, 42, 0.72)";
-    ctx.fillRect(-borderWidth, aggregateY, progressBarWidth, aggregateHeight);
+    ctx.fillRect(progressBarX, aggregateY, progressBarWidth, aggregateHeight);
     ctx.fillStyle = "rgba(216, 180, 254, 0.92)";
     if (state.progress.indeterminate) {
       const pulseWidth = Math.max(progressBarWidth * 0.18, 12 / scale);
       const travelWidth = Math.max(0, progressBarWidth - pulseWidth);
-      const pulseX = -borderWidth + ((elapsed * 0.65) % 1) * travelWidth;
+      const pulseX = progressBarX + ((elapsed * 0.65) % 1) * travelWidth;
       ctx.fillRect(pulseX, aggregateY, pulseWidth, aggregateHeight);
     } else {
-      ctx.fillRect(-borderWidth, aggregateY, progressWidth, aggregateHeight);
+      ctx.fillRect(progressBarX, aggregateY, progressWidth, aggregateHeight);
     }
+    drawProgressProviderIcon(
+      ctx,
+      aggregateProvider,
+      -borderWidth,
+      aggregateY + aggregateHeight / 2,
+      progressIconColumnWidth,
+      aggregateHeight,
+      scale,
+    );
     drawIterationRateOverlay(
       ctx,
       state.progress.iterationRate,
@@ -6019,6 +6105,7 @@ function handleModalProgress(event) {
       Number(detail.max ?? 1),
       detail.item_index,
       Boolean(detail.setup_only),
+      detail,
     );
     return;
   }