From 194df484d1ff56aa77faef606644d3ec6bb95906 Mon Sep 17 00:00:00 2001 From: Ken Simpson Date: Mon, 24 Aug 2026 21:58:36 -0700 Subject: [PATCH] Recover stale Vast offers and persist UI failures --- README.md | 2 +- docs/LEARNINGS.md | 2 ++ tests/modal_toggle_frontend.test.mjs | 34 +++++++++++++++++++++++++ tests/test_vast_leases.py | 4 +++ tests/test_web_modal_toggle.py | 1 + vast_api.py | 27 +++++++++++++++++--- vast_simulator.py | 7 +++++- web/modal_toggle.js | 37 ++++++++++++++++++++++++---- 8 files changed, 104 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c8f3c1c..a81e59d 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Active remote nodes show a small progress panel with per-bar `it/s` rates (LLM n Remote planning and capacity acquisition count as active queue work even though they happen before the rewritten prompt enters ComfyUI's native execution queue. During Vast startup, the canvas receives readable updates for rental, worker-image download, SSH readiness, and runtime initialization. A provider-side terminal image-pull failure is reported immediately rather than appearing as an empty queue until the startup timeout expires. A rental that never produces an SSH-usable worker is marked unusable and destroyed after the bounded initialization wait so a later workflow cannot silently reuse it. -For workflows using a **Remote Execution Configurator**, that node is also the primary planning surface. As soon as scheduling finishes, the scheduler result is streamed to the panel before any Vast rental or SSH-readiness wait begins. The browser retains that plan across late node-panel mounts, a full page refresh, tab refocus, queue failure, and extension reloads, while removing any orphaned prior panel before remounting. Configuration names, selected targets, predicted completion time, and predicted cost therefore remain inspectable even when capacity never becomes ready. Each selected environment has its own status and progress lane, so parallel Vast, Modal, and self-hosted setup or execution updates cannot overwrite each other; once a newer prompt owns the panel, delayed events from an older prompt remain isolated. Live preparation and execution status use the same durable event stream. Queue-history reconciliation reads the prompt's actual terminal result, preserving a failure or interruption in the Configurator and replacing stale submission text when the prompt is no longer executing. Legacy workflows without a configurator continue to use the global status pill. +For workflows using a **Remote Execution Configurator**, that node is also the primary planning surface. As soon as scheduling finishes, the scheduler result is streamed to the panel before any Vast rental or SSH-readiness wait begins. The browser retains that plan across late node-panel mounts, a full page refresh, tab refocus, queue failure, and extension reloads, while removing any orphaned prior panel before remounting. Configuration names, selected targets, predicted completion time, and predicted cost therefore remain inspectable even when capacity never becomes ready. Each selected environment has its own status and progress lane, so parallel Vast, Modal, and self-hosted setup or execution updates cannot overwrite each other; once a newer prompt owns the panel, delayed events from an older prompt remain isolated. A prompt-wide failure persistently terminates every environment lane and clears its in-progress meter, including when a later plan render or panel remount occurs. Live preparation and execution status use the same durable event stream. Queue-history reconciliation reads the prompt's actual terminal result, preserving a failure or interruption in the Configurator and replacing stale submission text when the prompt is no longer executing. Legacy workflows without a configurator continue to use the global status pill. Queue-time preparation appears in ComfyUI's queue through the server-side preparation bridge, while the Configurator panel and global status pill show its detailed phase. Preparation deliberately does not emit fake native node-execution events: doing so starts ComfyUI's execution watchdog, which can incorrectly mark a node as stalled or failed during a healthy multi-minute asset download. A preparation failure remains a queue/setup error with its original message rather than being recast as a node execution crash. diff --git a/docs/LEARNINGS.md b/docs/LEARNINGS.md index 94d87cd..4d35298 100644 --- a/docs/LEARNINGS.md +++ b/docs/LEARNINGS.md @@ -558,3 +558,5 @@ - Do not distribute permanent object-store credentials to rented or arbitrary workers. Keep the S3-compatible client on the controller, issue expiring exact-object GET/PUT or multipart-part URLs over protected stdin, validate their HTTPS origin remotely, and verify byte size plus SHA-256 before atomically publishing downloaded content. - Content-addressing makes one R2 bucket safely reusable across Vast instances, SSH hosts, and installations without a mutable filename catalog. Write-back can run outside queue-time critical latency, while a synchronous mode remains useful when short-lived capacity may disappear before background population completes. - Marketplace network capacity is not geographically uniform. A host can accept inbound SSH and reach ordinary HTTPS sites while DNS interception or regional filtering makes Hugging Face's CloudFront endpoints unreachable; expose explicit country exclusions in the workflow profile, send them as `geolocation.notin`, and revalidate returned offers locally rather than treating advertised bandwidth as proof of origin reachability. +- Vast may report an offer-disappearance race as a non-404 HTTP response containing provider code `3603` and `no_such_ask`. Classify the provider code as well as HTTP 404 so the lease manager invalidates the stale ask, force-refreshes marketplace results, and tries another candidate. +- Configurator terminal rendering must update retained per-environment state, not only the currently mounted DOM rows. Otherwise a later plan render, graph remount, or replay reconstructs the last healthy setup meter after the workflow has already failed; terminal transitions should also clear numeric progress so a failed lane cannot look active. diff --git a/tests/modal_toggle_frontend.test.mjs b/tests/modal_toggle_frontend.test.mjs index 8b9b917..faf7020 100644 --- a/tests/modal_toggle_frontend.test.mjs +++ b/tests/modal_toggle_frontend.test.mjs @@ -1622,6 +1622,40 @@ assert.equal(modalEnvironmentRow.statusText.textContent, "Starting Lambda"); assert.equal(modalEnvironmentRow.progressValue.textContent, "1/4"); assert.equal(modalEnvironmentRow.progressFill.style.width, "25%"); +modalToggle.handleModalStatus({ + detail: { + phase: "error", + prompt_id: "plan-before-capacity", + node_ids: ["14", "15", "24"], + configurator_node_id: "381", + error_message: "Hugging Face could not be reached", + }, +}); +const failedPromptState = modalToggle.modalPromptStates.get("plan-before-capacity"); +assert.equal( + failedPromptState.remoteEnvironmentStatuses.get("vast:profile:slot-0")?.phase, + "error", +); +assert.equal( + failedPromptState.remoteEnvironmentStatuses.get("modal:lambda")?.statusMessage, + "Hugging Face could not be reached", +); +assert.equal(vastEnvironmentRow.progress.hidden, true); +assert.equal(modalEnvironmentRow.progress.hidden, true); +modalToggle.registerRemoteConfiguratorPlan( + "plan-before-capacity", + failedPromptState.remoteExecutionPlanAssignments, + failedPromptState.remoteExecutionConfigurations, +); +assert.equal( + retainedPanel.environmentRows.get("vast:profile:slot-0").root.dataset.phase, + "error", +); +assert.equal( + retainedPanel.environmentRows.get("modal:lambda").statusText.textContent, + "Hugging Face could not be reached", +); + modalToggle.registerPromptConfigurator("newer-prompt", "381"); modalToggle.modalPromptStates.get("plan-before-capacity").startedAt = 1; modalToggle.modalPromptStates.get("newer-prompt").startedAt = 2; diff --git a/tests/test_vast_leases.py b/tests/test_vast_leases.py index 72f6248..33f81b6 100644 --- a/tests/test_vast_leases.py +++ b/tests/test_vast_leases.py @@ -89,6 +89,10 @@ def test_manager_recovers_offer_race_then_reuses_persisted_lease( async def scenario() -> None: state = vast_simulator_module.VastSimulatorState( create_failures_remaining={1001: 1}, + create_failure_status=400, + create_failure_message=( + "error 404/3603: no_such_ask Instance type by id 1001 is not available." + ), polls_until_running=1, ) async with _running_simulator( diff --git a/tests/test_web_modal_toggle.py b/tests/test_web_modal_toggle.py index 78305e9..f749458 100644 --- a/tests/test_web_modal_toggle.py +++ b/tests/test_web_modal_toggle.py @@ -75,6 +75,7 @@ def test_configurator_renders_out_of_band_plan_and_status() -> None: assert "function renderRemoteConfiguratorStatus(panel, state)" in source assert "function renderRemoteConfiguratorEnvironments(panel, assignments, configurations)" in source assert "function updateRemoteConfiguratorEnvironmentStatus(promptId, detail)" in source + assert "promptState?.remoteEnvironmentStatuses.set(environmentId" in source assert "function renderRemoteConfiguratorPlan(panel, assignments, configurations)" in source assert 'widget = node.addDOMWidget(\n "remote_execution_plan"' in source assert "serialize: false" in source diff --git a/vast_api.py b/vast_api.py index 868231e..ac35e82 100644 --- a/vast_api.py +++ b/vast_api.py @@ -39,6 +39,7 @@ _RETRIABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504}) VAST_OFFER_CACHE_TTL_SECONDS = 60.0 * 60.0 VAST_UNKNOWN_INSTANCE_STATE_TIMEOUT_SECONDS = 120.0 _VAST_OFFER_CACHE_MAX_ENTRIES = 256 +_VAST_OFFER_UNAVAILABLE_MARKERS = frozenset({"no_such_ask", "3603"}) class VastApiError(RuntimeError): @@ -295,7 +296,13 @@ class VastApiClient: offer_creation=True, ) if not payload.get("success", False): - raise VastApiError(_safe_error_message(payload, "Vast did not create the instance.")) + error_message = _safe_error_message( + payload, + "Vast did not create the instance.", + ) + if _is_offer_unavailable_response(200, error_message): + raise VastOfferUnavailableError(error_message) + raise VastApiError(error_message) try: instance_id = int(payload["new_contract"]) except (KeyError, TypeError, ValueError) as exc: @@ -461,9 +468,12 @@ class VastApiClient: ) if status in {401, 403}: raise VastAuthenticationError(error_message) + if offer_creation and _is_offer_unavailable_response( + status, + error_message, + ): + raise VastOfferUnavailableError(error_message) if status == 404: - if offer_creation: - raise VastOfferUnavailableError(error_message) if path.startswith("/api/v0/instances/"): raise VastInstanceNotFoundError(error_message) raise VastApiError(error_message) @@ -543,6 +553,17 @@ def _safe_error_message(payload: Mapping[str, Any], fallback: str) -> str: return fallback +def _is_offer_unavailable_response(status: int, error_message: str) -> bool: + """Recognize Vast's HTTP and provider-code forms of a stale marketplace ask.""" + if status == 404: + return True + normalized_message = error_message.casefold() + return any( + marker in normalized_message + for marker in _VAST_OFFER_UNAVAILABLE_MARKERS + ) + + def _retry_delay(attempt: int, retry_after: str | None) -> float: """Return one bounded retry delay with Retry-After precedence.""" if retry_after: diff --git a/vast_simulator.py b/vast_simulator.py index 0a7c5cd..cc3c401 100644 --- a/vast_simulator.py +++ b/vast_simulator.py @@ -100,6 +100,8 @@ class VastSimulatorState: destroyed_instance_ids: list[int] = field(default_factory=list) request_log: list[dict[str, Any]] = field(default_factory=list) create_failures_remaining: dict[int, int] = field(default_factory=dict) + create_failure_status: int = 404 + create_failure_message: str = "Offer is no longer available" polls_until_running: int = 2 next_instance_id: int = 9001 @@ -201,7 +203,10 @@ class VastApiSimulator: failures_remaining = self.state.create_failures_remaining.get(offer_id, 0) if failures_remaining > 0: self.state.create_failures_remaining[offer_id] = failures_remaining - 1 - return web.json_response({"error": "Offer is no longer available"}, status=404) + return web.json_response( + {"error": self.state.create_failure_message}, + status=self.state.create_failure_status, + ) if not body.get("cancel_unavail", False): return web.json_response( {"error": "Simulator requires cancel_unavail for deterministic tests."}, diff --git a/web/modal_toggle.js b/web/modal_toggle.js index 0ed9aa4..4c8f041 100644 --- a/web/modal_toggle.js +++ b/web/modal_toggle.js @@ -3734,18 +3734,45 @@ function refreshRemoteConfiguratorPanelForPrompt(promptId) { */ function setRemoteConfiguratorTerminalStatus(promptId, phase, message = null) { const promptState = modalPromptStates.get(promptId); - const panel = + const candidatePanel = remoteConfiguratorPanel(promptState?.configuratorNodeId) ?? Array.from(remoteConfiguratorPanels.values()).find( (candidate) => candidate?.promptId === promptId, ); - if (!panel || panel.promptId !== promptId) { + const panel = candidatePanel?.promptId === promptId ? candidatePanel : null; + if (!promptState && !panel) { + return; + } + const terminalState = { + phase, + statusMessage: message, + statusCurrent: null, + statusTotal: null, + updatedAt: nowMs(), + }; + const environmentIds = new Set([ + ...(promptState?.remoteEnvironmentStatuses.keys() ?? []), + ...remoteConfiguratorEnvironmentEntries( + promptState?.remoteExecutionPlanAssignments ?? {}, + promptState?.remoteExecutionConfigurations ?? [], + ).map((entry) => entry.environmentId), + ...(panel?.environmentRows.keys() ?? []), + ]); + for (const environmentId of environmentIds) { + promptState?.remoteEnvironmentStatuses.set(environmentId, { + ...(promptState.remoteEnvironmentStatuses.get(environmentId) ?? {}), + ...terminalState, + }); + } + if (!panel) { return; } - const terminalState = { phase, statusMessage: message }; renderRemoteConfiguratorStatus(panel, terminalState); - for (const row of panel.environmentRows.values()) { - renderRemoteConfiguratorEnvironmentStatus(row, terminalState); + for (const [environmentId, row] of panel.environmentRows.entries()) { + renderRemoteConfiguratorEnvironmentStatus( + row, + promptState?.remoteEnvironmentStatuses.get(environmentId) ?? terminalState, + ); } }