nodes/robertvoy-ComfyUI-Distributed
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge feat/runpod-serverless-workers into feat/dynamic-image-list-queue

Browse Source
This commit is contained in:
Robert Wojciechowski
2026-10-13 21:33:28 +00:00
parent dcddf23db2 06be0e5dee
commit 00a9ba05a3
17 changed files with 407 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/config_routes.py
+9 -13
View File
@@ -218,8 +218,7 @@ def _bulk_serverless_changes(old_workers, new_workers):
if new is None or new.get("type") != "serverless":
_ensure_pool_idle(worker_id)
secret_ids.add(worker_id)
elif (old.get("endpoint_id") != new.get("endpoint_id")
or old.get("concurrency", 1) != new.get("concurrency", 1)):
elif old.get("endpoint_id") != new.get("endpoint_id"):
_ensure_pool_idle(worker_id)
elif new is not None and new.get("type") == "serverless":
secret_ids.add(worker_id)
@@ -299,7 +298,7 @@ async def update_config_endpoint(request):
if isinstance(worker, dict) and worker.get("type") == "serverless" and worker.get("enabled", False):
store = store or _serverless_secret_store()
if worker["id"] in changed_ids or not store.has(worker["id"]):
return _error_response("Save an API key before enabling this pool", 400)
return _error_response("Save an API key before enabling this worker", 400)
if changed_ids:
store = store or _serverless_secret_store()
for worker_id in sorted(changed_ids):
@@ -316,7 +315,7 @@ async def update_config_endpoint(request):
secret_changes.commit()
return response
except _ActivePoolCheckError:
return _error_response("Finish or reconcile unresolved serverless jobs before changing this pool", 409)
return _error_response("Finish or reconcile unresolved serverless jobs before changing this worker", 409)
except Exception:
return _error_response("Unable to update config", 500)
@@ -401,7 +400,8 @@ async def _update_worker_endpoint(request, secret_changes):
candidate["type"] = "serverless"
candidate.setdefault("provider", "runpod")
candidate.setdefault("enabled", False)
for field in ("provider", "endpoint_id", "jobs_per_run", "concurrency", "timeout_seconds", "enabled", "name"):
# Legacy "concurrency" is accepted (see _SERVERLESS_CONFIG_FIELDS) but ignored.
for field in ("provider", "endpoint_id", "jobs_per_run", "timeout_seconds", "enabled", "name"):
if field in data:
candidate[field] = data[field]
candidate = _clean_serverless_worker(candidate)
@@ -424,17 +424,13 @@ async def _update_worker_endpoint(request, secret_changes):
current_type == "serverless"
and existing.get("endpoint_id") != validated.get("endpoint_id")
)
concurrency_changed = (
current_type == "serverless"
and existing.get("concurrency", 1) != validated.get("concurrency", 1)
)
if current_type == "serverless" and (
endpoint_changed or concurrency_changed or next_secret != prior_secret
endpoint_changed or next_secret != prior_secret
):
_ensure_pool_idle(worker_id)
if validated.get("enabled", False) and next_secret is None:
return _error_response("Save an API key before enabling this pool; disable it before clearing the key", 400)
return _error_response("Save an API key before enabling this worker; turn the worker off before clearing its key", 400)
if next_secret != prior_secret:
if next_secret is None:
@@ -489,7 +485,7 @@ async def _update_worker_endpoint(request, secret_changes):
return web.json_response(result)
except _ActivePoolCheckError:
return _error_response("Finish or reconcile unresolved serverless jobs before changing this pool", 409)
return _error_response("Finish or reconcile unresolved serverless jobs before changing this worker", 409)
except ValueError:
return _error_response("Invalid worker configuration", 400)
except Exception:
@@ -537,7 +533,7 @@ async def _delete_worker_endpoint(request, secret_changes):
return web.json_response(result)
except _ActivePoolCheckError:
return _error_response("Finish or reconcile unresolved serverless jobs before changing this pool", 409)
return _error_response("Finish or reconcile unresolved serverless jobs before changing this worker", 409)
except Exception:
return _error_response("Unable to delete worker", 500)
api/orchestration/serverless_runtime.py
+18 -13
View File
@@ -12,6 +12,15 @@ from .serverless_state import (
)
# Fixed per-worker cap on this master's outstanding Runpod jobs; equals the
# Images per run maximum (MAX_JOBS_PER_RUN in utils/serverless_config.py), so
# every image of a run is submitted at once. Deliberately not configurable or
# derived from jobs_per_run: unresolved jobs from earlier runs (ambiguous
# submissions, unconfirmed cancels) still count against it, and a constant
# never has to change while such jobs exist.
PER_WORKER_JOB_LIMIT = 16
async def decode_results(output, manifest):
from .serverless_media import collect_images
import torch
@@ -85,7 +94,7 @@ class ServerlessManager:
self._check_master(run)
outstanding = sum(belongs_to_pool(record, pool_id) and occupies_capacity(record)
for record in self.ledger.all())
if outstanding < run["worker"]["concurrency"]:
if outstanding < PER_WORKER_JOB_LIMIT:
# Claim atomically before any await that could submit a billed job.
self.ledger.update(run["record_id"], status="SUBMITTING", updated_at=time.time())
return
@@ -122,18 +131,14 @@ class ServerlessManager:
return result
def _semaphore(self, worker):
# The limit is a constant, so it never has to be swapped while this
# worker still has unresolved jobs.
pool_id = worker.get("pool_id", worker["id"])
limit = worker["concurrency"]
current = self.pool_limits.get(pool_id)
if current is None:
current = (limit, asyncio.Semaphore(limit))
self.pool_limits[pool_id] = current
elif current[0] != limit:
if self.pool_is_active(pool_id):
raise ValueError("Wait for this serverless pool's current jobs before changing concurrency")
current = (limit, asyncio.Semaphore(limit))
self.pool_limits[pool_id] = current
return current[1]
semaphore = self.pool_limits.get(pool_id)
if semaphore is None:
semaphore = asyncio.Semaphore(PER_WORKER_JOB_LIMIT)
self.pool_limits[pool_id] = semaphore
return semaphore
def ensure_capacity(self, count):
if sum(not task.done() for task in self.tasks.values()) + count > 64:
@@ -320,7 +325,7 @@ class ServerlessManager:
async with self.recovery_lock:
record = next((r for r in self._detached_records(worker_id) if r["record_id"] == record_id), None)
if record is None:
raise ValueError("Unresolved inactive job not found for this pool")
raise ValueError("Unresolved inactive job not found for this worker")
self.ledger.update(record_id, status="RESOLVED", updated_at=time.time(),
error="Recovery explicitly acknowledged after operator checked Runpod; not provider-confirmed")
await self._notify_capacity()
api/serverless_routes.py
+1 -1
View File
@@ -83,7 +83,7 @@ async def _recover(request, *, acknowledge=False):
result = {"resolved": await manager.reconcile(worker_id=worker_id)}
return web.json_response(dict(result, ok=True, jobs=manager.public_jobs(worker_id)))
except ValueError:
return web.json_response({"ok": False, "message": "Unresolved inactive job not found for this pool"}, status=409)
return web.json_response({"ok": False, "message": "Unresolved inactive job not found for this worker"}, status=409)
except Exception:
return web.json_response({"ok": False, "message": "Recovery could not be confirmed; retained jobs remain protected"}, status=502)
api/worker_routes.py
+1 -1
View File
@@ -461,7 +461,7 @@ async def launch_worker_endpoint(request):
return await handle_api_error(request, f"Worker {worker_id} not found", 404)
if worker.get("type") == "serverless":
return await handle_api_error(request, "Serverless pools are managed by Runpod, not local processes", 400)
return await handle_api_error(request, "Serverless workers are managed by Runpod, not local processes", 400)
# Ensure consistent string ID
worker_id_str = worker_id