feat: dynamically distribute image lists across direct workers (#98)
* feat: add Runpod serverless image collector pools * fix: prevent worker settings clipping and remove preview copy * fix: initialize worker settings before mounting the form * fix: validate complete serverless requests against 10 MB limit * fix: simplify serverless cards and keep status dots circular * fix: match serverless endpoint box to master card * fix: preserve serverless capacity and recovery across job failures Retain durable reservations until provider termination or explicit recovery, protect unresolved pools and credentials, and add scoped reconciliation. Consolidate lifecycle and secret policies and split prompt/media stages with regression coverage. * feat: add dynamic per-image list queue with Start and End nodes * fix(image-list): reject upstream data bypasses around Start * fix(image-list): make optional End text a connection-only input * fix(image-list): isolate unavailable workers during preflight * fix(image-list): bound acknowledgements and configure run deadline * fix(media-sync): preserve output and temp source roots * fix(image-list): own job lifecycle and preserve worker preparation * feat(image-list): let master participate unless orchestrator-only * feat(image-list)!: replace fixed Start extras with growable value_N slots Distributed Image List Start no longer has reference_image, text, strength, seed or per-slot mode dropdowns. It now takes images, mask and up to 10 generic value_N inputs (IMAGE, MASK, STRING, INT, FLOAT, BOOLEAN) that grow via native V3 Autogrow, each with a matching output. - One value is shared by every image; one per image pairs in order; any other count fails before dispatch. Only images is split into frames. - Outputs are declared up front (image, mask, item_index, context, value_1..value_10). web/imageListStart.js hides unused value outputs, mirrors input types, and follows Autogrow compaction so downstream links keep their value. - Image-list protocol bumped to v2; older workers fail preflight. Breaking: saved workflows using the old Start inputs must recreate Start. * fix(serverless): disable Clear saved API key while the worker is enabled The checkbox read "Clear saved API key (disable pool first)", but the panel never defines "pool" and saving with it ticked on an enabled worker was rejected by the backend. - Disable the checkbox while the worker is enabled, with a tooltip saying to turn the worker off first. Also disable it when no key is saved. - Use "worker" instead of "pool" in user-facing serverless errors, the serverless guide and the README entry. * feat(serverless): replace Concurrent jobs with a single Images per run setting - Remove the separate Concurrent jobs field; rename Jobs per run to Images per run (1-16) with a tooltip explaining that Runpod's max workers decides how many GPUs run them. - Backend uses a fixed per-worker limit of 16 outstanding jobs, so all images of a run are submitted at once. Unresolved jobs still count against it. The limit is constant, so it never has to change while jobs are unresolved. - Legacy 'concurrency' in saved configs and older browser tabs is accepted and ignored/dropped; it no longer triggers the busy guard. - Revert the enable toggle and show the reason when the backend rejects it. - Trim a pasted api.runpod.ai/v2/<id> URL to the endpoint ID; plainer endpoint-ID error otherwise. - Confirm before a type change deletes a saved Runpod API key. - Update docs and tests.
This commit is contained in:
@@ -4,3 +4,4 @@ from . import tunnel_routes # noqa: F401
|
||||
from . import worker_routes # noqa: F401
|
||||
from . import job_routes # noqa: F401
|
||||
from . import usdu_routes # noqa: F401
|
||||
from . import image_list_routes # noqa: F401
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Bounded, typed HTTP transport for dynamic image-list worker tasks."""
|
||||
import json
|
||||
|
||||
from aiohttp import web
|
||||
import server
|
||||
|
||||
from ..image_list.runtime import (
|
||||
MAX_REQUEST_BYTES,
|
||||
PROTOCOL_VERSION,
|
||||
decode_result,
|
||||
delete_worker_item,
|
||||
set_master_result,
|
||||
store_worker_item,
|
||||
)
|
||||
|
||||
prompt_server = server.PromptServer.instance
|
||||
|
||||
|
||||
async def _json_body(request):
|
||||
length = request.content_length
|
||||
if length is not None and length > MAX_REQUEST_BYTES:
|
||||
raise ValueError(f"Request exceeds the {MAX_REQUEST_BYTES}-byte image-list limit")
|
||||
raw = bytearray()
|
||||
async for chunk in request.content.iter_chunked(64 * 1024):
|
||||
raw.extend(chunk)
|
||||
if len(raw) > MAX_REQUEST_BYTES:
|
||||
raise ValueError(f"Request exceeds the {MAX_REQUEST_BYTES}-byte image-list limit")
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ValueError(f"Invalid image-list JSON body: {exc}") from exc
|
||||
|
||||
|
||||
@prompt_server.routes.get("/distributed/image-list/capabilities")
|
||||
async def image_list_capabilities_endpoint(_request):
|
||||
return web.json_response({
|
||||
"supported": True,
|
||||
"protocol_version": PROTOCOL_VERSION,
|
||||
"node_ids": ["DistributedImageListStart", "DistributedImageListEnd"],
|
||||
"transport": "typed-float-tensor-json-v1",
|
||||
})
|
||||
|
||||
|
||||
@prompt_server.routes.post("/distributed/image-list/item")
|
||||
async def image_list_item_endpoint(request):
|
||||
try:
|
||||
data = await _json_body(request)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("Expected an image-list item object")
|
||||
context_id = data.get("context_id")
|
||||
if not isinstance(context_id, str) or not context_id:
|
||||
raise ValueError("context_id must be a non-empty string")
|
||||
item = data.get("item")
|
||||
store_worker_item(data.get("token"), item)
|
||||
return web.json_response({"status": "stored", "item_index": item["index"]})
|
||||
except (ValueError, TypeError) as exc:
|
||||
return web.json_response({"error": str(exc)}, status=400)
|
||||
except Exception as exc:
|
||||
return web.json_response({"error": str(exc)}, status=500)
|
||||
|
||||
|
||||
@prompt_server.routes.delete("/distributed/image-list/item/{token}")
|
||||
async def image_list_delete_item_endpoint(request):
|
||||
token = request.match_info.get("token", "")
|
||||
return web.json_response({"status": "deleted", "removed": delete_worker_item(token)})
|
||||
|
||||
|
||||
@prompt_server.routes.post("/distributed/image-list/result")
|
||||
async def image_list_result_endpoint(request):
|
||||
try:
|
||||
data = await _json_body(request)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("Expected an image-list result object")
|
||||
context_id = data.get("context_id")
|
||||
token = data.get("token")
|
||||
item_index = data.get("item_index")
|
||||
if not isinstance(context_id, str) or not context_id:
|
||||
raise ValueError("context_id must be a non-empty string")
|
||||
if not isinstance(token, str) or not token:
|
||||
raise ValueError("token must be a non-empty string")
|
||||
if isinstance(item_index, bool) or not isinstance(item_index, int) or item_index < 0:
|
||||
raise ValueError("item_index must be a non-negative integer")
|
||||
result = decode_result(data.get("result"))
|
||||
set_master_result(context_id, item_index, token, result)
|
||||
return web.json_response({"status": "accepted", "item_index": item_index})
|
||||
except (ValueError, TypeError) as exc:
|
||||
return web.json_response({"error": str(exc)}, status=400)
|
||||
except Exception as exc:
|
||||
return web.json_response({"error": str(exc)}, status=500)
|
||||
+135
-22
@@ -6,6 +6,9 @@ import aiohttp
|
||||
|
||||
from ...utils.logging import debug_log, log
|
||||
from ...utils.network import build_worker_url, get_client_session, probe_worker
|
||||
# Must match image_list.runtime.PROTOCOL_VERSION (kept local so this module
|
||||
# stays importable without the image-list runtime; a test pins them equal).
|
||||
IMAGE_LIST_PROTOCOL_VERSION = 2
|
||||
try:
|
||||
from ...utils.trace_logger import trace_debug, trace_info
|
||||
except ImportError:
|
||||
@@ -77,27 +80,44 @@ async def _dispatch_via_websocket(worker_url, payload, client_id, timeout=60.0):
|
||||
ws_url = worker_url.replace("http://", "ws://").replace("https://", "wss://")
|
||||
ws_url = f"{ws_url}/distributed/worker_ws"
|
||||
session = await get_client_session()
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout
|
||||
ws = None
|
||||
|
||||
async with session.ws_connect(ws_url, heartbeat=20, timeout=timeout) as ws:
|
||||
await ws.send_json(ws_payload)
|
||||
async for msg in ws:
|
||||
if msg.type == aiohttp.WSMsgType.TEXT:
|
||||
data = json.loads(msg.data or "{}")
|
||||
if data.get("type") == "dispatch_ack" and data.get("request_id") == request_id:
|
||||
if data.get("ok"):
|
||||
return
|
||||
error_text = data.get("error") or "Worker rejected websocket dispatch."
|
||||
validation_error = data.get("validation_error")
|
||||
node_errors = data.get("node_errors")
|
||||
if validation_error:
|
||||
error_text = f"{error_text} | validation_error={validation_error}"
|
||||
if node_errors:
|
||||
error_text = f"{error_text} | node_errors={node_errors}"
|
||||
raise RuntimeError(error_text)
|
||||
elif msg.type in (aiohttp.WSMsgType.ERROR, aiohttp.WSMsgType.CLOSED):
|
||||
raise RuntimeError(f"Worker websocket closed unexpectedly: {msg.type}")
|
||||
|
||||
raise RuntimeError("Worker websocket closed before dispatch_ack was received.")
|
||||
try:
|
||||
async with asyncio.timeout_at(deadline):
|
||||
ws = await session.ws_connect(ws_url, heartbeat=20)
|
||||
await ws.send_json(ws_payload)
|
||||
while True:
|
||||
msg = await ws.receive()
|
||||
if msg.type == aiohttp.WSMsgType.TEXT:
|
||||
data = json.loads(msg.data or "{}")
|
||||
if data.get("type") == "dispatch_ack" and data.get("request_id") == request_id:
|
||||
if data.get("ok"):
|
||||
return data.get("prompt_id")
|
||||
error_text = data.get("error") or "Worker rejected websocket dispatch."
|
||||
validation_error = data.get("validation_error")
|
||||
node_errors = data.get("node_errors")
|
||||
if validation_error:
|
||||
error_text = f"{error_text} | validation_error={validation_error}"
|
||||
if node_errors:
|
||||
error_text = f"{error_text} | node_errors={node_errors}"
|
||||
raise RuntimeError(error_text)
|
||||
elif msg.type in (aiohttp.WSMsgType.ERROR, aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED):
|
||||
raise RuntimeError(f"Worker websocket closed before acknowledgement: {msg.type}; submission status is ambiguous")
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise RuntimeError(
|
||||
"Worker websocket dispatch timed out before dispatch_ack; submission status is ambiguous."
|
||||
) from exc
|
||||
finally:
|
||||
if ws is not None:
|
||||
# Closing must not discard an already acknowledged prompt ID or
|
||||
# replace a definite rejection with an ambiguous timeout.
|
||||
try:
|
||||
async with asyncio.timeout_at(min(deadline, loop.time() + 1.0)):
|
||||
await ws.close()
|
||||
except (asyncio.TimeoutError, aiohttp.ClientError):
|
||||
pass
|
||||
|
||||
|
||||
async def dispatch_worker_prompt(
|
||||
@@ -107,6 +127,7 @@ async def dispatch_worker_prompt(
|
||||
client_id=None,
|
||||
use_websocket=False,
|
||||
trace_execution_id=None,
|
||||
return_prompt_id=False,
|
||||
):
|
||||
"""Send the prepared prompt to a worker ComfyUI instance."""
|
||||
worker_url = build_worker_url(worker)
|
||||
@@ -120,7 +141,7 @@ async def dispatch_worker_prompt(
|
||||
|
||||
if use_websocket:
|
||||
try:
|
||||
await _dispatch_via_websocket(
|
||||
prompt_id = await _dispatch_via_websocket(
|
||||
worker_url,
|
||||
{
|
||||
"prompt": prompt_obj,
|
||||
@@ -128,7 +149,9 @@ async def dispatch_worker_prompt(
|
||||
},
|
||||
client_id,
|
||||
)
|
||||
return
|
||||
if return_prompt_id and (not isinstance(prompt_id, str) or not prompt_id):
|
||||
raise RuntimeError("Worker websocket acknowledgement omitted prompt_id")
|
||||
return prompt_id if return_prompt_id else None
|
||||
except Exception as exc:
|
||||
worker_id = worker.get("id")
|
||||
if trace_execution_id:
|
||||
@@ -144,6 +167,96 @@ async def dispatch_worker_prompt(
|
||||
timeout=aiohttp.ClientTimeout(total=60),
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
if not return_prompt_id:
|
||||
return None
|
||||
try:
|
||||
response_data = await resp.json()
|
||||
except Exception as exc:
|
||||
raise RuntimeError("Worker /prompt response did not contain JSON acknowledgement") from exc
|
||||
prompt_id = response_data.get("prompt_id") if isinstance(response_data, dict) else None
|
||||
if not isinstance(prompt_id, str) or not prompt_id:
|
||||
raise RuntimeError("Worker /prompt response omitted prompt_id")
|
||||
return prompt_id if return_prompt_id else None
|
||||
|
||||
|
||||
async def preflight_image_list_workers(workers, require_idle=True, timeout=3.0):
|
||||
"""Require protocol/node support and report which direct workers are idle.
|
||||
|
||||
This performs read-only GETs only; it never submits a prompt or contacts a
|
||||
serverless provider. Unsupported worker releases fail before master inputs run.
|
||||
"""
|
||||
if not workers:
|
||||
raise RuntimeError("Distributed Image List requires at least one selected direct worker")
|
||||
session = await get_client_session()
|
||||
required_nodes = {"DistributedImageListStart", "DistributedImageListEnd"}
|
||||
semaphore = asyncio.Semaphore(8)
|
||||
|
||||
async def inspect(worker):
|
||||
worker_id = str(worker.get("id") or "unknown")
|
||||
check = "capability"
|
||||
try:
|
||||
async with semaphore:
|
||||
cap_url = build_worker_url(worker, "/distributed/image-list/capabilities")
|
||||
async with session.get(cap_url, timeout=aiohttp.ClientTimeout(total=timeout)) as response:
|
||||
if response.status != 200:
|
||||
raise RuntimeError(
|
||||
f"Image-list worker {worker_id} does not expose the capability endpoint "
|
||||
f"(HTTP {response.status}); upgrade that worker before running this feature"
|
||||
)
|
||||
capability = await response.json()
|
||||
if not isinstance(capability, dict):
|
||||
raise RuntimeError(f"Image-list worker {worker_id} returned malformed capability data")
|
||||
supported = capability.get("supported")
|
||||
protocol_version = capability.get("protocol_version")
|
||||
nodes = capability.get("node_ids")
|
||||
if (
|
||||
not isinstance(supported, bool)
|
||||
or type(protocol_version) is not int
|
||||
or not isinstance(nodes, list)
|
||||
or any(not isinstance(node_id, str) for node_id in nodes)
|
||||
):
|
||||
raise RuntimeError(f"Image-list worker {worker_id} returned malformed capability data")
|
||||
if (
|
||||
supported is not True
|
||||
or protocol_version != IMAGE_LIST_PROTOCOL_VERSION
|
||||
or not required_nodes.issubset(nodes)
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Image-list worker {worker_id} is incompatible: requires protocol "
|
||||
f"v{IMAGE_LIST_PROTOCOL_VERSION} and nodes "
|
||||
f"{sorted(required_nodes)}; no work was submitted"
|
||||
)
|
||||
|
||||
check = "queue"
|
||||
queue_url = build_worker_url(worker, "/queue")
|
||||
async with session.get(queue_url, timeout=aiohttp.ClientTimeout(total=timeout)) as response:
|
||||
if response.status != 200:
|
||||
raise RuntimeError(f"Image-list worker {worker_id} queue preflight returned HTTP {response.status}")
|
||||
queue = await response.json()
|
||||
if (
|
||||
not isinstance(queue, dict)
|
||||
or not isinstance(queue.get("queue_running"), list)
|
||||
or not isinstance(queue.get("queue_pending"), list)
|
||||
):
|
||||
raise RuntimeError(f"Image-list worker {worker_id} returned malformed /queue state")
|
||||
idle = not queue["queue_running"] and not queue["queue_pending"]
|
||||
return worker_id, idle
|
||||
except (asyncio.TimeoutError, aiohttp.ClientConnectionError) as exc:
|
||||
log(
|
||||
f"[Distributed] Image-list worker {worker_id} unavailable during {check} check; "
|
||||
f"skipping: {exc}"
|
||||
)
|
||||
return worker_id, None
|
||||
|
||||
statuses = await asyncio.gather(*(inspect(worker) for worker in workers))
|
||||
idle_workers = {
|
||||
worker_id: True
|
||||
for worker_id, idle in statuses
|
||||
if idle is True
|
||||
}
|
||||
if require_idle and not idle_workers:
|
||||
raise RuntimeError("No selected feature-compatible direct worker is idle; unrelated worker prompts were left untouched")
|
||||
return idle_workers
|
||||
|
||||
|
||||
async def select_active_workers(
|
||||
|
||||
@@ -29,6 +29,11 @@ def _normalize_media_reference(value):
|
||||
return None
|
||||
cleaned = re.sub(r"\s*\[\w+\]$", "", value).strip().replace("\\", "/")
|
||||
if MEDIA_FILE_RE.search(cleaned):
|
||||
# Output/temp annotations select a different master-side media root.
|
||||
# Preserve them through discovery/loading and as distinct rewrite keys.
|
||||
annotation = re.search(r"\s*\[(output|temp)\]$", value)
|
||||
if annotation:
|
||||
return f"{cleaned} [{annotation.group(1)}]"
|
||||
return cleaned
|
||||
return None
|
||||
|
||||
@@ -171,6 +176,12 @@ async def _upload_media_to_worker(worker, filename, file_bytes, file_hash, mime_
|
||||
"""Upload one media file to worker iff missing or hash-mismatched."""
|
||||
session = await get_client_session()
|
||||
normalized = filename.replace("\\", "/")
|
||||
annotation = re.search(r"\s*\[(output|temp)\]$", normalized)
|
||||
if annotation:
|
||||
# Worker uploads live under input/. Do not collide across master roots,
|
||||
# or overwrite a different reference with the same basename.
|
||||
basename = normalized[:annotation.start()].rstrip().rsplit("/", 1)[-1]
|
||||
normalized = f"distributed-media/{annotation.group(1)}/{file_hash}/{basename}"
|
||||
|
||||
check_url = build_worker_url(worker, "/distributed/check_file")
|
||||
try:
|
||||
|
||||
@@ -458,11 +458,55 @@ def prepare_delegate_master_prompt(prompt_obj, collector_ids):
|
||||
return pruned_prompt
|
||||
|
||||
|
||||
def _resolve_direct_workers_for_image_list(config, requested_ids):
|
||||
"""Resolve enabled workers by default, or exact explicitly selected direct workers."""
|
||||
requested = None if requested_ids is None else {str(worker_id) for worker_id in requested_ids}
|
||||
configured_workers = (config or {}).get("workers", [])
|
||||
configured_ids = {
|
||||
str(worker.get("id") or "").strip()
|
||||
for worker in configured_workers
|
||||
if str(worker.get("id") or "").strip()
|
||||
}
|
||||
if requested is not None:
|
||||
unknown_ids = requested - configured_ids
|
||||
if unknown_ids:
|
||||
raise RuntimeError(
|
||||
"Unknown explicitly selected image-list worker ID(s): "
|
||||
+ ", ".join(sorted(unknown_ids))
|
||||
)
|
||||
direct = []
|
||||
for worker in configured_workers:
|
||||
worker_id = str(worker.get("id") or "").strip()
|
||||
if not worker_id:
|
||||
continue
|
||||
if requested is None:
|
||||
if not worker.get("enabled", False):
|
||||
continue
|
||||
elif worker_id not in requested:
|
||||
continue
|
||||
if str(worker.get("type") or "local").strip().lower() == "serverless":
|
||||
continue
|
||||
try:
|
||||
port = int(worker.get("port", worker.get("listen_port", 8188)) or 8188)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RuntimeError(f"Invalid port for image-list worker {worker_id}") from exc
|
||||
direct.append({
|
||||
"id": worker_id,
|
||||
"name": str(worker.get("name") or worker_id),
|
||||
"host": worker.get("host"),
|
||||
"port": port,
|
||||
"type": worker.get("type", "local"),
|
||||
})
|
||||
return direct
|
||||
|
||||
|
||||
def generate_job_id_map(prompt_index, prefix):
|
||||
"""Create stable per-node job IDs for distributed nodes."""
|
||||
job_map = {}
|
||||
distributed_nodes = prompt_index.nodes_for_class("DistributedCollector") + prompt_index.nodes_for_class(
|
||||
"UltimateSDUpscaleDistributed"
|
||||
distributed_nodes = (
|
||||
prompt_index.nodes_for_class("DistributedCollector")
|
||||
+ prompt_index.nodes_for_class("UltimateSDUpscaleDistributed")
|
||||
+ prompt_index.nodes_for_class("DistributedImageListStart")
|
||||
)
|
||||
for node_id in distributed_nodes:
|
||||
job_map[node_id] = f"{prefix}_{node_id}"
|
||||
@@ -557,6 +601,22 @@ def _override_value_nodes(prompt_copy, prompt_index, is_master, participant_id,
|
||||
inputs["worker_id"] = f"worker_{worker_index_map.get(participant_id, 0)}"
|
||||
|
||||
|
||||
def _override_image_list_nodes(prompt_copy, prompt_index, participant_id, worker_ids,
|
||||
job_id_map, master_url, delegate_master, workflow_meta=None):
|
||||
"""Attach protocol context to Start without changing its visible V3 sockets."""
|
||||
for node_id in prompt_index.nodes_for_class("DistributedImageListStart"):
|
||||
node = prompt_copy.get(node_id)
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
inputs = node.setdefault("inputs", {})
|
||||
inputs["image_list_enabled_worker_ids"] = json.dumps(list(worker_ids))
|
||||
inputs["image_list_master_participates"] = participant_id == "master" and not delegate_master
|
||||
inputs["image_list_master_callback"] = str(master_url or "")
|
||||
inputs["image_list_execution_id"] = job_id_map.get(node_id, node_id)
|
||||
if workflow_meta is not None:
|
||||
inputs["image_list_workflow_meta"] = workflow_meta
|
||||
|
||||
|
||||
def apply_participant_overrides(
|
||||
prompt_copy,
|
||||
participant_id,
|
||||
@@ -565,6 +625,7 @@ def apply_participant_overrides(
|
||||
master_url,
|
||||
delegate_master,
|
||||
prompt_index,
|
||||
workflow_meta=None,
|
||||
):
|
||||
"""Return a prompt copy with hidden inputs configured for master/worker."""
|
||||
is_master = participant_id == "master"
|
||||
@@ -592,5 +653,15 @@ def apply_participant_overrides(
|
||||
master_url,
|
||||
enabled_json,
|
||||
)
|
||||
_override_image_list_nodes(
|
||||
prompt_copy,
|
||||
prompt_index,
|
||||
participant_id,
|
||||
enabled_worker_ids,
|
||||
job_id_map,
|
||||
master_url,
|
||||
delegate_master,
|
||||
workflow_meta=workflow_meta,
|
||||
)
|
||||
|
||||
return prompt_copy
|
||||
|
||||
+65
-14
@@ -35,7 +35,9 @@ from .orchestration.prompt_transform import (
|
||||
prepare_delegate_master_prompt,
|
||||
prune_prompt_for_worker,
|
||||
WORKER_SINK_CLASS,
|
||||
_resolve_direct_workers_for_image_list,
|
||||
)
|
||||
from ..image_list.runtime import validate_image_list_regions
|
||||
|
||||
|
||||
prompt_server = server.PromptServer.instance
|
||||
@@ -252,6 +254,8 @@ async def orchestrate_distributed_execution(
|
||||
execution_trace_id = trace_execution_id or _generate_execution_trace_id()
|
||||
|
||||
config = load_config()
|
||||
if delegate_master is None:
|
||||
delegate_master = bool(config.get("settings", {}).get("master_delegate_only", False))
|
||||
use_websocket = bool(config.get("settings", {}).get("websocket_orchestration", False))
|
||||
master_url = build_master_url(config=config, prompt_server_instance=prompt_server)
|
||||
(
|
||||
@@ -261,8 +265,42 @@ async def orchestrate_distributed_execution(
|
||||
media_sync_timeout_seconds,
|
||||
) = _resolve_orchestration_limits(config)
|
||||
requested_ids = enabled_worker_ids if enabled_worker_ids is not None else None
|
||||
workers = _resolve_enabled_workers(config, requested_ids)
|
||||
prompt_index = PromptIndex(prompt_obj)
|
||||
image_list_start_ids = prompt_index.nodes_for_class("DistributedImageListStart")
|
||||
image_list_mode = bool(image_list_start_ids)
|
||||
if image_list_mode:
|
||||
# Validate before probing or queueing anything so malformed boundaries and
|
||||
# old worker releases fail before any upstream image is materialized.
|
||||
validate_image_list_regions(prompt_obj)
|
||||
other_distributed = (
|
||||
prompt_index.nodes_for_class("DistributedCollector")
|
||||
+ prompt_index.nodes_for_class("UltimateSDUpscaleDistributed")
|
||||
)
|
||||
if other_distributed:
|
||||
raise ValueError(
|
||||
"Distributed Image List v1 cannot be mixed with Distributed Collector or "
|
||||
"Ultimate SD Upscale nodes in one prompt"
|
||||
)
|
||||
configured_serverless = [
|
||||
str(worker.get("id")) for worker in config.get("workers", [])
|
||||
if worker.get("enabled", False)
|
||||
and str(worker.get("type") or "local").lower() == "serverless"
|
||||
and (requested_ids is None or str(worker.get("id")) in {str(x) for x in requested_ids})
|
||||
]
|
||||
if configured_serverless:
|
||||
trace_debug(
|
||||
execution_trace_id,
|
||||
"Image List v1 explicitly excludes unsupported serverless pools: "
|
||||
+ ", ".join(configured_serverless),
|
||||
)
|
||||
workers = _resolve_direct_workers_for_image_list(config, requested_ids)
|
||||
if not workers and delegate_master:
|
||||
raise RuntimeError(
|
||||
"Distributed Image List in orchestrator-only mode requires a selected direct worker; "
|
||||
"serverless pools are unsupported"
|
||||
)
|
||||
else:
|
||||
workers = _resolve_enabled_workers(config, requested_ids)
|
||||
load_balance_requested = _prompt_requests_load_balance(prompt_index)
|
||||
trace_debug(
|
||||
execution_trace_id,
|
||||
@@ -278,26 +316,37 @@ async def orchestrate_distributed_execution(
|
||||
),
|
||||
)
|
||||
|
||||
# Respect master delegate-only configuration
|
||||
if delegate_master is None:
|
||||
delegate_master = bool(config.get("settings", {}).get("master_delegate_only", False))
|
||||
|
||||
if not workers and delegate_master:
|
||||
# Keep the legacy Collector fallback, but never override image-list
|
||||
# orchestrator-only mode merely because its direct workers are unavailable.
|
||||
if not image_list_mode and not workers and delegate_master:
|
||||
trace_debug(
|
||||
execution_trace_id,
|
||||
"Delegate-only requested but no workers are enabled. Falling back to master execution.",
|
||||
)
|
||||
delegate_master = False
|
||||
|
||||
active_workers, delegate_master = await select_active_workers(
|
||||
active_workers, selected_delegate_master = await select_active_workers(
|
||||
workers,
|
||||
use_websocket,
|
||||
delegate_master,
|
||||
False if image_list_mode else delegate_master,
|
||||
trace_execution_id=execution_trace_id,
|
||||
probe_concurrency=worker_probe_concurrency,
|
||||
)
|
||||
if image_list_mode:
|
||||
from .orchestration.dispatch import preflight_image_list_workers
|
||||
if not active_workers and delegate_master:
|
||||
raise RuntimeError(
|
||||
"No selected direct worker is reachable for Distributed Image List in orchestrator-only mode"
|
||||
)
|
||||
idle_ids = (await preflight_image_list_workers(active_workers, require_idle=delegate_master)
|
||||
if active_workers else {})
|
||||
active_workers = [worker for worker in active_workers if worker["id"] in idle_ids]
|
||||
if not active_workers and delegate_master:
|
||||
raise RuntimeError("No feature-compatible direct worker is idle in orchestrator-only mode")
|
||||
else:
|
||||
delegate_master = selected_delegate_master
|
||||
|
||||
if load_balance_requested:
|
||||
if load_balance_requested and not image_list_mode:
|
||||
candidate_workers = list(active_workers)
|
||||
if not delegate_master:
|
||||
# Include master in load balancing only when master participation is enabled.
|
||||
@@ -377,9 +426,10 @@ async def orchestrate_distributed_execution(
|
||||
master_url,
|
||||
delegate_master,
|
||||
prompt_index,
|
||||
workflow_meta=workflow_meta,
|
||||
)
|
||||
|
||||
if delegate_master:
|
||||
if delegate_master and not image_list_mode:
|
||||
collector_ids = find_nodes_by_class(master_prompt, "DistributedCollector")
|
||||
upscale_nodes = find_nodes_by_class(master_prompt, "UltimateSDUpscaleDistributed")
|
||||
if upscale_nodes:
|
||||
@@ -400,7 +450,7 @@ async def orchestrate_distributed_execution(
|
||||
+ ", ".join(f"{worker['name']} ({worker['id']})" for worker in active_workers),
|
||||
)
|
||||
worker_payloads = []
|
||||
if active_workers:
|
||||
if active_workers and not image_list_mode:
|
||||
worker_prep_semaphore = asyncio.Semaphore(worker_prep_concurrency)
|
||||
media_sync_semaphore = asyncio.Semaphore(media_sync_concurrency)
|
||||
worker_payloads = await asyncio.gather(
|
||||
@@ -431,8 +481,9 @@ async def orchestrate_distributed_execution(
|
||||
manager.ensure_capacity(len(serverless_jobs))
|
||||
|
||||
# Do not leave collector queues behind when offline compilation/media validation fails.
|
||||
for job_id in job_id_map.values():
|
||||
await _ensure_distributed_queue(job_id)
|
||||
if not image_list_mode:
|
||||
for job_id in job_id_map.values():
|
||||
await _ensure_distributed_queue(job_id)
|
||||
|
||||
if regular_payloads:
|
||||
await asyncio.gather(
|
||||
@@ -483,4 +534,4 @@ async def orchestrate_distributed_execution(
|
||||
execution_trace_id,
|
||||
f"Orchestration complete: prompt_id={prompt_id}, dispatched_workers={len(worker_payloads)}, delegate_master={delegate_master}",
|
||||
)
|
||||
return prompt_id, prompt_number, len(worker_payloads), node_errors
|
||||
return prompt_id, prompt_number, (len(active_workers) if image_list_mode else len(worker_payloads)), node_errors
|
||||
|
||||
Reference in New Issue
Block a user