feat: add Runpod serverless image collector pools
This commit is contained in:
Robert Wojciechowski
+30
-4
@@ -7,7 +7,7 @@ export function createApiClient(baseUrl) {
|
||||
const request = async (
|
||||
endpoint,
|
||||
options = {},
|
||||
{ retries = TIMEOUTS.MAX_RETRIES, retry = true } = {},
|
||||
{ retries = TIMEOUTS.MAX_RETRIES, retry = true, redactErrors = false } = {},
|
||||
) => {
|
||||
const maxAttempts = retry ? retries : 1;
|
||||
let lastError;
|
||||
@@ -36,7 +36,7 @@ export function createApiClient(baseUrl) {
|
||||
return await response.json();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
console.log(`API Error (attempt ${attempt + 1}/${maxAttempts}): ${endpoint} - ${error.message}`);
|
||||
console.log(`API Error (attempt ${attempt + 1}/${maxAttempts}): ${endpoint} - ${redactErrors ? "[redacted]" : error.message}`);
|
||||
if (attempt < maxAttempts - 1) {
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
delay *= 2; // Exponential backoff
|
||||
@@ -86,9 +86,35 @@ export function createApiClient(baseUrl) {
|
||||
},
|
||||
|
||||
async updateWorker(workerId, data) {
|
||||
return request('/distributed/config/update_worker', {
|
||||
const apiKey = data?.api_key;
|
||||
try {
|
||||
return await request('/distributed/config/update_worker', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ worker_id: workerId, ...data })
|
||||
}, { retry: false, redactErrors: Boolean(apiKey) });
|
||||
} catch (error) {
|
||||
if (apiKey && typeof error?.message === "string") {
|
||||
error.message = error.message.split(apiKey).join("[redacted]");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
|
||||
async testServerlessWorker(workerId) {
|
||||
return request('/distributed/serverless/test', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ worker_id: workerId, ...data })
|
||||
body: JSON.stringify({ worker_id: workerId }),
|
||||
}, { retry: false });
|
||||
},
|
||||
|
||||
async getServerlessStatus(workerId) {
|
||||
return request(`/distributed/serverless/status/${encodeURIComponent(workerId)}`);
|
||||
},
|
||||
|
||||
async cancelServerlessWorker(workerId) {
|
||||
return request('/distributed/serverless/cancel', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ worker_id: workerId }),
|
||||
}, { retry: false });
|
||||
},
|
||||
|
||||
|
||||
@@ -112,6 +112,21 @@ export async function performPreflightCheck(extension, workers) {
|
||||
const startTime = Date.now();
|
||||
|
||||
const checkPromises = workers.map(async (worker) => {
|
||||
if (worker.type === "serverless") {
|
||||
try {
|
||||
const providerStatus = await extension.api.getServerlessStatus(worker.id);
|
||||
if (providerStatus?.ok) {
|
||||
extension.log(`Serverless worker ${worker.name} is ready to scale`, "debug");
|
||||
return { worker, active: true };
|
||||
}
|
||||
extension.log(`Serverless worker ${worker.name} is not ready: ${providerStatus?.message || "unavailable"}`, "debug");
|
||||
return { worker, active: false };
|
||||
} catch (error) {
|
||||
extension.log(`Serverless worker ${worker.name} status check failed: ${error.message}`, "debug");
|
||||
return { worker, active: false };
|
||||
}
|
||||
}
|
||||
|
||||
const workerUrl = getWorkerUrl(extension, worker);
|
||||
|
||||
extension.log(`Pre-flight checking ${worker.name} at: ${workerUrl}`, "debug");
|
||||
|
||||
@@ -18,6 +18,19 @@ export function updateWorkerCard(extension, workerId, newStatus = {}) {
|
||||
const workerState = extension.state.getWorker(workerId);
|
||||
const isLaunching = Boolean(workerState?.launching);
|
||||
|
||||
if (worker.type === "serverless") {
|
||||
const providerStatus = workerState?.serverlessStatus || {};
|
||||
if (newStatus.online && newStatus.processing) {
|
||||
extension.ui.updateStatusDot(workerId, STATUS_COLORS.PROCESSING_YELLOW, `Runpod active (${newStatus.queueCount || 0} queued)`, false);
|
||||
} else if (newStatus.online) {
|
||||
extension.ui.updateStatusDot(workerId, STATUS_COLORS.ONLINE_GREEN, providerStatus.message || "Runpod ready to scale", false);
|
||||
} else if (worker.enabled) {
|
||||
extension.ui.updateStatusDot(workerId, STATUS_COLORS.OFFLINE_RED, providerStatus.message || "Runpod endpoint unavailable", false);
|
||||
}
|
||||
updateWorkerControls(extension, workerId);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isLaunching && !newStatus.online) {
|
||||
extension.ui.updateStatusDot(workerId, STATUS_COLORS.PROCESSING_YELLOW, "Launching...", true);
|
||||
} else if (newStatus.online && newStatus.processing) {
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("../../scripts/api.js", () => ({ api: { clientId: "test-client" } }), { virtual: true });
|
||||
|
||||
import { createApiClient } from "../apiClient.js";
|
||||
import { createWorkerSettingsForm } from "../ui/settingsForm.js";
|
||||
import { DistributedUI } from "../ui.js";
|
||||
import { checkWorkerStatus, getWorkerUrl, launchWorker, stopWorker, viewWorkerLog } from "../workerLifecycle.js";
|
||||
import { handleClearMemory, handleInterruptWorkers } from "../workerUtils.js";
|
||||
import { isRemoteWorker, saveWorkerSettings, testServerlessWorker } from "../workerSettings.js";
|
||||
import { performPreflightCheck } from "../executionUtils.js";
|
||||
|
||||
function createDocumentMock() {
|
||||
const elements = new Map();
|
||||
const makeElement = (tagName) => {
|
||||
const element = {
|
||||
tagName,
|
||||
children: [],
|
||||
style: {},
|
||||
classList: {
|
||||
values: new Set(),
|
||||
add(...names) { names.forEach((name) => this.values.add(name)); },
|
||||
remove(...names) { names.forEach((name) => this.values.delete(name)); },
|
||||
toggle(name, force) {
|
||||
const enabled = force ?? !this.values.has(name);
|
||||
if (enabled) this.values.add(name);
|
||||
else this.values.delete(name);
|
||||
return enabled;
|
||||
},
|
||||
},
|
||||
appendChild(child) { this.children.push(child); child.parentNode = this; return child; },
|
||||
dispatchEvent() { this.onchange?.({ target: this }); },
|
||||
};
|
||||
Object.defineProperty(element, "id", {
|
||||
get() { return this._id || ""; },
|
||||
set(value) { this._id = value; elements.set(value, this); },
|
||||
});
|
||||
return element;
|
||||
};
|
||||
return {
|
||||
elements,
|
||||
document: {
|
||||
createElement: makeElement,
|
||||
getElementById: (id) => elements.get(id) || null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createFormUi() {
|
||||
return {
|
||||
createFormGroup(label, value, id, type = "text", placeholder = "") {
|
||||
const group = document.createElement("div");
|
||||
const input = document.createElement("input");
|
||||
input.id = id;
|
||||
input.value = String(value ?? "");
|
||||
input.type = type;
|
||||
input.placeholder = placeholder;
|
||||
group.appendChild(input);
|
||||
return { group, input, label };
|
||||
},
|
||||
createButton(text, onclick) {
|
||||
const button = document.createElement("button");
|
||||
button.textContent = text;
|
||||
button.onclick = onclick;
|
||||
return button;
|
||||
},
|
||||
createButtonGroup(buttons) {
|
||||
const group = document.createElement("div");
|
||||
buttons.forEach((button) => group.appendChild(button));
|
||||
return group;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeSaveExtension(worker, updateWorker = vi.fn().mockResolvedValue({})) {
|
||||
return {
|
||||
config: { workers: [worker] },
|
||||
api: { updateWorker },
|
||||
state: { updateWorker: vi.fn(), setWorkerExpanded: vi.fn() },
|
||||
app: { extensionManager: { toast: { add: vi.fn() } } },
|
||||
panelElement: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("Runpod serverless worker panel", () => {
|
||||
let originalDocument;
|
||||
let originalWindow;
|
||||
let elements;
|
||||
|
||||
beforeEach(() => {
|
||||
originalDocument = globalThis.document;
|
||||
originalWindow = globalThis.window;
|
||||
const mock = createDocumentMock();
|
||||
elements = mock.elements;
|
||||
globalThis.document = mock.document;
|
||||
globalThis.window = {
|
||||
location: { hostname: "localhost", protocol: "http:", port: "8188", origin: "http://localhost:8188" },
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.document = originalDocument;
|
||||
globalThis.window = originalWindow;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("classifies Runpod serverless as remote for worker allocation", () => {
|
||||
expect(isRemoteWorker({}, { type: "serverless" })).toBe(true);
|
||||
});
|
||||
|
||||
it("shows only serverless settings and keeps credentials write-only", () => {
|
||||
const worker = {
|
||||
id: "w1", name: "Runpod", type: "remote", host: "worker.example", port: 8188,
|
||||
cuda_device: 0, extra_args: "--test", api_key_configured: true,
|
||||
};
|
||||
const form = createWorkerSettingsForm(createFormUi(), {}, worker);
|
||||
const type = elements.get("worker-type-w1");
|
||||
type.value = "serverless";
|
||||
type.dispatchEvent();
|
||||
|
||||
expect(type.children.map((option) => [option.value, option.textContent])).toContainEqual([
|
||||
"serverless", "Serverless (Runpod)",
|
||||
]);
|
||||
for (const id of ["host-group-w1", "port-group-w1", "cuda-group-w1", "args-group-w1"]) {
|
||||
expect(elements.get(id).style.display).toBe("none");
|
||||
}
|
||||
for (const id of ["endpoint-id-w1", "api-key-w1", "jobs-per-run-w1", "concurrency-w1", "timeout-seconds-w1"]) {
|
||||
expect(elements.has(id)).toBe(true);
|
||||
}
|
||||
for (const id of ["endpoint-id-group-w1", "api-key-group-w1", "clear-api-key-group-w1", "jobs-per-run-group-w1", "concurrency-group-w1", "timeout-seconds-group-w1", "serverless-test-group-w1"]) {
|
||||
expect(elements.get(id).style.display).toBe("flex");
|
||||
}
|
||||
expect(elements.get("serverless-help-w1").textContent).toContain("image Collectors only");
|
||||
expect(elements.get("api-key-w1").type).toBe("password");
|
||||
expect(elements.get("api-key-w1").value).toBe("");
|
||||
expect(elements.get("api-key-w1").placeholder).toMatch(/leave blank/i);
|
||||
expect(elements.get("jobs-per-run-w1").value).toBe("1");
|
||||
expect(elements.get("jobs-per-run-w1").min).toBe("1");
|
||||
expect(elements.get("jobs-per-run-w1").max).toBe("16");
|
||||
expect(elements.get("concurrency-w1").value).toBe("1");
|
||||
expect(elements.get("concurrency-w1").min).toBe("1");
|
||||
expect(elements.get("concurrency-w1").max).toBe("16");
|
||||
expect(elements.get("timeout-seconds-w1").value).toBe("1800");
|
||||
expect(elements.get("timeout-seconds-w1").min).toBe("30");
|
||||
expect(elements.get("timeout-seconds-w1").max).toBe("86400");
|
||||
expect(form.children.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("saves fixed Runpod settings without enabling or retaining the API key", async () => {
|
||||
const worker = { id: "w2", name: "Old", type: "remote", enabled: false, api_key_configured: false };
|
||||
const extension = makeSaveExtension(worker);
|
||||
elements.set("name-w2", { value: "Runpod worker" });
|
||||
elements.set("worker-type-w2", { value: "serverless" });
|
||||
elements.set("endpoint-id-w2", { value: "ep_123" });
|
||||
elements.set("api-key-w2", { value: "secret-token" });
|
||||
elements.set("clear-api-key-w2", { checked: false });
|
||||
elements.set("jobs-per-run-w2", { value: "4" });
|
||||
elements.set("concurrency-w2", { value: "3" });
|
||||
elements.set("timeout-seconds-w2", { value: "2400" });
|
||||
|
||||
await saveWorkerSettings(extension, "w2");
|
||||
|
||||
expect(extension.api.updateWorker).toHaveBeenCalledWith("w2", {
|
||||
name: "Runpod worker", type: "serverless", provider: "runpod", endpoint_id: "ep_123",
|
||||
api_key: "secret-token", jobs_per_run: 4, concurrency: 3, timeout_seconds: 2400, enabled: false,
|
||||
});
|
||||
expect(worker).toMatchObject({ type: "serverless", provider: "runpod", enabled: false, api_key_configured: true });
|
||||
expect(worker).not.toHaveProperty("api_key");
|
||||
expect(JSON.stringify(extension.config)).not.toContain("secret-token");
|
||||
expect(elements.get("api-key-w2").value).toBe("");
|
||||
expect(extension.state.updateWorker).toHaveBeenCalledWith("w2", { enabled: false });
|
||||
expect(JSON.stringify(extension.state.updateWorker.mock.calls)).not.toContain("secret-token");
|
||||
});
|
||||
|
||||
it("omits a blank API key so the saved key is preserved", async () => {
|
||||
const worker = { id: "w-blank", name: "Runpod", type: "serverless", provider: "runpod", enabled: true, api_key_configured: true };
|
||||
const extension = makeSaveExtension(worker);
|
||||
elements.set("name-w-blank", { value: "Runpod" });
|
||||
elements.set("worker-type-w-blank", { value: "serverless" });
|
||||
elements.set("endpoint-id-w-blank", { value: "ep_123" });
|
||||
elements.set("api-key-w-blank", { value: "" });
|
||||
elements.set("clear-api-key-w-blank", { checked: false });
|
||||
elements.set("jobs-per-run-w-blank", { value: "1" });
|
||||
elements.set("concurrency-w-blank", { value: "1" });
|
||||
elements.set("timeout-seconds-w-blank", { value: "1800" });
|
||||
|
||||
await saveWorkerSettings(extension, "w-blank");
|
||||
|
||||
const payload = extension.api.updateWorker.mock.calls[0][1];
|
||||
expect(payload).not.toHaveProperty("api_key");
|
||||
expect(payload).not.toHaveProperty("clear_api_key");
|
||||
expect(worker.api_key_configured).toBe(true);
|
||||
expect(worker).not.toHaveProperty("api_key");
|
||||
});
|
||||
|
||||
it("omits a blank key to preserve it and sends an explicit clear flag when requested", async () => {
|
||||
const worker = { id: "w3", name: "Runpod", type: "serverless", provider: "runpod", enabled: true, api_key_configured: true };
|
||||
const extension = makeSaveExtension(worker);
|
||||
elements.set("name-w3", { value: "Runpod" });
|
||||
elements.set("worker-type-w3", { value: "serverless" });
|
||||
elements.set("endpoint-id-w3", { value: "ep_123" });
|
||||
elements.set("api-key-w3", { value: "" });
|
||||
elements.set("clear-api-key-w3", { checked: true });
|
||||
elements.set("jobs-per-run-w3", { value: "1" });
|
||||
elements.set("concurrency-w3", { value: "1" });
|
||||
elements.set("timeout-seconds-w3", { value: "1800" });
|
||||
|
||||
await saveWorkerSettings(extension, "w3");
|
||||
|
||||
expect(extension.api.updateWorker).toHaveBeenCalledWith("w3", {
|
||||
name: "Runpod", type: "serverless", provider: "runpod", endpoint_id: "ep_123",
|
||||
clear_api_key: true, jobs_per_run: 1, concurrency: 1, timeout_seconds: 1800, enabled: true,
|
||||
});
|
||||
expect(worker.api_key_configured).toBe(false);
|
||||
expect(worker).not.toHaveProperty("api_key");
|
||||
});
|
||||
|
||||
it("uses provider status instead of a direct ComfyUI URL probe", async () => {
|
||||
const statusElement = { textContent: "" };
|
||||
elements.set("serverless-status-w4", statusElement);
|
||||
const status = {
|
||||
ok: true, queueRemaining: 0, message: "Ready to scale",
|
||||
jobs: [{ job_id: "job-1", status: "RUNNING", worker_id: "w4", execution_id: "exec-1" }],
|
||||
};
|
||||
const extension = {
|
||||
config: { workers: [{ id: "w4", name: "Runpod", type: "serverless", enabled: true }] },
|
||||
api: { getServerlessStatus: vi.fn().mockResolvedValue(status), probeWorker: vi.fn() },
|
||||
statusCheckAbortController: null,
|
||||
state: {
|
||||
setWorkerStatus: vi.fn(),
|
||||
updateWorker: vi.fn(),
|
||||
getWorkerStatus: vi.fn().mockReturnValue({ online: true, processing: false, queueCount: 0 }),
|
||||
getWorker: vi.fn().mockReturnValue({}),
|
||||
isWorkerLaunching: vi.fn().mockReturnValue(false),
|
||||
},
|
||||
updateWorkerCard: vi.fn().mockReturnValue(true),
|
||||
log: vi.fn(),
|
||||
};
|
||||
|
||||
await checkWorkerStatus(extension, extension.config.workers[0]);
|
||||
|
||||
expect(extension.api.getServerlessStatus).toHaveBeenCalledWith("w4");
|
||||
expect(extension.api.probeWorker).not.toHaveBeenCalled();
|
||||
expect(extension.state.setWorkerStatus).toHaveBeenCalledWith("w4", {
|
||||
online: true, processing: true, queueCount: 0,
|
||||
});
|
||||
expect(statusElement.textContent).toContain("Ready to scale");
|
||||
expect(statusElement.textContent).toContain("RUNNING");
|
||||
});
|
||||
|
||||
it("considers a healthy zero-worker Runpod pool ready and never builds a worker URL", async () => {
|
||||
const worker = { id: "w-ready", name: "Runpod", type: "serverless", enabled: true };
|
||||
const extension = {
|
||||
api: {
|
||||
getServerlessStatus: vi.fn().mockResolvedValue({ ok: true, queueRemaining: 0, message: "Ready to scale" }),
|
||||
probeWorker: vi.fn(),
|
||||
},
|
||||
log: vi.fn(),
|
||||
};
|
||||
|
||||
await expect(performPreflightCheck(extension, [worker])).resolves.toEqual([worker]);
|
||||
expect(extension.api.getServerlessStatus).toHaveBeenCalledWith("w-ready");
|
||||
expect(extension.api.probeWorker).not.toHaveBeenCalled();
|
||||
expect(getWorkerUrl(extension, worker)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not render or invoke direct ComfyUI launch, stop, or log controls", async () => {
|
||||
const ui = new DistributedUI();
|
||||
const worker = { id: "w-controls", name: "Runpod", type: "serverless", enabled: true };
|
||||
const controls = ui.createControlsSection({ dynamic: true }, worker, {}, true);
|
||||
const collect = (element) => [element, ...(element.children || []).flatMap(collect)];
|
||||
const elementsInControls = collect(controls);
|
||||
const extension = {
|
||||
config: { workers: [worker] },
|
||||
api: {
|
||||
launchWorker: vi.fn(), stopWorker: vi.fn(), getRemoteWorkerLog: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
expect(elementsInControls.filter((element) => element.tagName === "button")).toHaveLength(0);
|
||||
await launchWorker(extension, worker.id);
|
||||
await stopWorker(extension, worker.id);
|
||||
await viewWorkerLog(extension, worker.id, true);
|
||||
expect(extension.api.launchWorker).not.toHaveBeenCalled();
|
||||
expect(extension.api.stopWorker).not.toHaveBeenCalled();
|
||||
expect(extension.api.getRemoteWorkerLog).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the serverless status, test, and cancel routes with the frozen request shapes", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const statusPayload = {
|
||||
ok: true,
|
||||
queueRemaining: 0,
|
||||
message: "Ready to scale",
|
||||
jobs: [{ job_id: "job-1", status: "RUNNING", worker_id: "w5", execution_id: "exec-1" }],
|
||||
};
|
||||
globalThis.fetch = vi.fn()
|
||||
.mockResolvedValueOnce({ ok: true, status: 200, json: vi.fn().mockResolvedValue({ ok: true, message: "Endpoint test passed" }) })
|
||||
.mockResolvedValueOnce({ ok: true, status: 200, json: vi.fn().mockResolvedValue(statusPayload) })
|
||||
.mockResolvedValueOnce({ ok: true, status: 200, json: vi.fn().mockResolvedValue({ ok: true }) });
|
||||
try {
|
||||
const client = createApiClient("http://localhost:8188");
|
||||
await client.testServerlessWorker("w5");
|
||||
await expect(client.getServerlessStatus("w5")).resolves.toEqual(statusPayload);
|
||||
await client.cancelServerlessWorker("w5");
|
||||
|
||||
expect(globalThis.fetch).toHaveBeenNthCalledWith(1, "http://localhost:8188/distributed/serverless/test", expect.objectContaining({
|
||||
method: "POST", body: JSON.stringify({ worker_id: "w5" }),
|
||||
}));
|
||||
expect(globalThis.fetch).toHaveBeenNthCalledWith(2, "http://localhost:8188/distributed/serverless/status/w5", expect.any(Object));
|
||||
expect(globalThis.fetch).toHaveBeenNthCalledWith(3, "http://localhost:8188/distributed/serverless/cancel", expect.objectContaining({
|
||||
method: "POST", body: JSON.stringify({ worker_id: "w5" }),
|
||||
}));
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
it("tests only the saved Runpod endpoint and reports the backend message", async () => {
|
||||
const button = document.createElement("button");
|
||||
button.id = "test-serverless-w6";
|
||||
button.textContent = "Test Saved Endpoint";
|
||||
const extension = {
|
||||
api: {
|
||||
testServerlessWorker: vi.fn().mockResolvedValue({ ok: true, message: "Endpoint reachable" }),
|
||||
queueDistributed: vi.fn(),
|
||||
},
|
||||
app: { extensionManager: { toast: { add: vi.fn() } } },
|
||||
};
|
||||
|
||||
await expect(testServerlessWorker(extension, "w6")).resolves.toEqual({ ok: true, message: "Endpoint reachable" });
|
||||
|
||||
expect(extension.api.testServerlessWorker).toHaveBeenCalledWith("w6");
|
||||
expect(extension.api.queueDistributed).not.toHaveBeenCalled();
|
||||
expect(extension.app.extensionManager.toast.add).toHaveBeenCalledWith(expect.objectContaining({
|
||||
severity: "success", detail: "Endpoint reachable",
|
||||
}));
|
||||
expect(button.textContent).toBe("Test Saved Endpoint");
|
||||
expect(button.disabled).toBe(false);
|
||||
});
|
||||
|
||||
it("redacts API-key-bearing worker update errors from logs and user-visible exceptions", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
const secret = "never-log-this-runpod-key";
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
status: 400,
|
||||
json: vi.fn().mockResolvedValue({ message: `Rejected key ${secret}` }),
|
||||
});
|
||||
try {
|
||||
const client = createApiClient("http://localhost:8188");
|
||||
await expect(client.updateWorker("w-secret", { api_key: secret })).rejects.toThrow("[redacted]");
|
||||
expect(logSpy.mock.calls.flat().join(" ")).not.toContain(secret);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
it("never sends VRAM operations to serverless workers", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = vi.fn();
|
||||
vi.useFakeTimers();
|
||||
const button = {
|
||||
textContent: "Clear Worker VRAM", style: { cssText: "" }, disabled: false,
|
||||
classList: { names: new Set(), add(...names) { names.forEach((name) => this.names.add(name)); }, remove(...names) { names.forEach((name) => this.names.delete(name)); } },
|
||||
};
|
||||
const extension = { enabledWorkers: [{ id: "s-vram", name: "Runpod", type: "serverless", enabled: true }] };
|
||||
try {
|
||||
await handleClearMemory(extension, button);
|
||||
expect(globalThis.fetch).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.runOnlyPendingTimers();
|
||||
vi.useRealTimers();
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels each enabled serverless worker once without changing normal worker interrupts", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({ ok: true });
|
||||
vi.useFakeTimers();
|
||||
const button = {
|
||||
textContent: "Interrupt Workers", style: { cssText: "" }, disabled: false,
|
||||
classList: { names: new Set(), add(...names) { names.forEach((name) => this.names.add(name)); }, remove(...names) { names.forEach((name) => this.names.delete(name)); } },
|
||||
};
|
||||
const workers = [
|
||||
{ id: "s1", name: "Serverless 1", type: "serverless", enabled: true },
|
||||
{ id: "s2", name: "Serverless 2", type: "serverless", enabled: true },
|
||||
{ id: "s3", name: "Disabled", type: "serverless", enabled: false },
|
||||
{ id: "local", name: "Local", type: "local", enabled: true, port: 8189 },
|
||||
];
|
||||
const extension = {
|
||||
enabledWorkers: workers.filter((worker) => worker.enabled),
|
||||
api: { cancelServerlessWorker: vi.fn().mockResolvedValue({ ok: true }) },
|
||||
log: vi.fn(),
|
||||
};
|
||||
try {
|
||||
await handleInterruptWorkers(extension, button);
|
||||
|
||||
expect(extension.api.cancelServerlessWorker).toHaveBeenCalledTimes(2);
|
||||
expect(extension.api.cancelServerlessWorker).toHaveBeenNthCalledWith(1, "s1");
|
||||
expect(extension.api.cancelServerlessWorker).toHaveBeenNthCalledWith(2, "s2");
|
||||
expect(globalThis.fetch).toHaveBeenCalledTimes(1);
|
||||
expect(globalThis.fetch.mock.calls[0][0]).toBe("http://localhost:8189/interrupt");
|
||||
} finally {
|
||||
vi.runOnlyPendingTimers();
|
||||
vi.useRealTimers();
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,14 @@ import { createLogModal } from './ui/logModal.js';
|
||||
import { launchWorker, stopWorker, updateWorkerControls, viewWorkerLog } from './workerLifecycle.js';
|
||||
import { isRemoteWorker } from './workerSettings.js';
|
||||
|
||||
const escapeWorkerInfo = (value) => String(value ?? "").replace(/[&<>"']/g, (character) => ({
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
'"': """,
|
||||
"'": "'",
|
||||
}[character]));
|
||||
|
||||
const cardConfigs = {
|
||||
master: {
|
||||
checkbox: {
|
||||
@@ -58,7 +66,15 @@ const cardConfigs = {
|
||||
infoText: (data, extension) => {
|
||||
const isRemote = isRemoteWorker(extension, data);
|
||||
const isCloud = data.type === 'cloud';
|
||||
|
||||
const isServerless = data.type === 'serverless';
|
||||
|
||||
if (isServerless) {
|
||||
const endpoint = data.endpoint_id || "Endpoint not configured";
|
||||
const jobsPerRun = data.jobs_per_run ?? 1;
|
||||
const concurrency = data.concurrency ?? 1;
|
||||
return `<span class="dist-worker-info__title">${escapeWorkerInfo(data.name)}</span><br><span class="dist-worker-info__meta">Runpod Serverless • ${escapeWorkerInfo(endpoint)} • ${escapeWorkerInfo(jobsPerRun)} jobs/run • ${escapeWorkerInfo(concurrency)} slots</span>`;
|
||||
}
|
||||
|
||||
if (isCloud) {
|
||||
// For cloud workers, don't show port (it's always 443)
|
||||
return `<span class="dist-worker-info__title">${data.name}</span><br><span class="dist-worker-info__meta">${data.host}</span>`;
|
||||
@@ -591,18 +607,24 @@ export class DistributedUI {
|
||||
controlsWrapper.appendChild(badge);
|
||||
} else if (config.dynamic && data) {
|
||||
if (isRemote) {
|
||||
const isCloud = data.type === 'cloud';
|
||||
const workerTypeText = isCloud ? "Cloud worker" : "Remote worker";
|
||||
const workerTypeBadge = this.createInfoBox(workerTypeText);
|
||||
workerTypeBadge.title = "Worker is externally hosted";
|
||||
controlsWrapper.appendChild(workerTypeBadge);
|
||||
if (data.type === "serverless") {
|
||||
const workerTypeBadge = this.createInfoBox("Runpod Serverless worker");
|
||||
workerTypeBadge.title = "Provider-managed workers scale on demand";
|
||||
controlsWrapper.appendChild(workerTypeBadge);
|
||||
} else {
|
||||
const isCloud = data.type === 'cloud';
|
||||
const workerTypeText = isCloud ? "Cloud worker" : "Remote worker";
|
||||
const workerTypeBadge = this.createInfoBox(workerTypeText);
|
||||
workerTypeBadge.title = "Worker is externally hosted";
|
||||
controlsWrapper.appendChild(workerTypeBadge);
|
||||
|
||||
const logBtn = this.createButton('View Log', () => viewWorkerLog(extension, data.id, true));
|
||||
logBtn.id = `log-${data.id}`;
|
||||
logBtn.style.cssText = BUTTON_STYLES.base + BUTTON_STYLES.workerControl;
|
||||
logBtn.classList.add("btn--log");
|
||||
logBtn.title = "View remote worker log";
|
||||
controlsWrapper.appendChild(logBtn);
|
||||
const logBtn = this.createButton('View Log', () => viewWorkerLog(extension, data.id, true));
|
||||
logBtn.id = `log-${data.id}`;
|
||||
logBtn.style.cssText = BUTTON_STYLES.base + BUTTON_STYLES.workerControl;
|
||||
logBtn.classList.add("btn--log");
|
||||
logBtn.title = "View remote worker log";
|
||||
controlsWrapper.appendChild(logBtn);
|
||||
}
|
||||
} else {
|
||||
const controls = this.createWorkerControls(data.id, {
|
||||
launch: () => launchWorker(extension, data.id),
|
||||
|
||||
@@ -1,6 +1,20 @@
|
||||
import { updateWorkerControls, toggleWorkerExpanded } from "../workerLifecycle.js";
|
||||
import { isRemoteWorker } from "../workerSettings.js";
|
||||
|
||||
function formatServerlessStatus(status) {
|
||||
if (!status) return "Runpod status pending";
|
||||
const message = status.message || (status.ok ? "Ready to scale" : "Runpod status unavailable");
|
||||
const queue = Number(status.queueRemaining);
|
||||
const parts = [message];
|
||||
if (Number.isFinite(queue)) parts.push(`${Math.max(0, queue)} queued`);
|
||||
const jobs = (Array.isArray(status.jobs) ? status.jobs : [])
|
||||
.slice(0, 2)
|
||||
.map((job) => String(job?.status || "").trim())
|
||||
.filter(Boolean);
|
||||
if (jobs.length) parts.push(`Jobs: ${jobs.join(", ")}`);
|
||||
return parts.join(" • ");
|
||||
}
|
||||
|
||||
export function renderEntityCard(ui, cardConfigs, entityType, data, extension) {
|
||||
const config = cardConfigs[entityType] || {};
|
||||
const isPlaceholder = entityType === 'blueprint' || entityType === 'add';
|
||||
@@ -74,6 +88,16 @@ export function renderEntityCard(ui, cardConfigs, entityType, data, extension) {
|
||||
infoSpan.innerHTML = config.infoText(data, extension);
|
||||
workerContent.appendChild(infoSpan);
|
||||
|
||||
if (isWorker && data.type === "serverless") {
|
||||
const statusMessage = document.createElement("small");
|
||||
statusMessage.id = `serverless-status-${data.id}`;
|
||||
statusMessage.className = "dist-worker-info__meta dist-serverless-status";
|
||||
statusMessage.style.display = "block";
|
||||
statusMessage.setAttribute("aria-live", "polite");
|
||||
statusMessage.textContent = formatServerlessStatus(extension.state.getWorker(data.id)?.serverlessStatus);
|
||||
infoSpan.appendChild(statusMessage);
|
||||
}
|
||||
|
||||
infoRow.appendChild(workerContent);
|
||||
|
||||
let settingsArrow;
|
||||
|
||||
+103
-3
@@ -1,5 +1,5 @@
|
||||
import { BUTTON_STYLES } from '../constants.js';
|
||||
import { cancelWorkerSettings, deleteWorker, isRemoteWorker, saveWorkerSettings } from '../workerSettings.js';
|
||||
import { cancelWorkerSettings, deleteWorker, isRemoteWorker, saveWorkerSettings, testServerlessWorker } from '../workerSettings.js';
|
||||
|
||||
export function createWorkerSettingsForm(ui, extension, worker) {
|
||||
const form = document.createElement("div");
|
||||
@@ -33,9 +33,14 @@ export function createWorkerSettingsForm(ui, extension, worker) {
|
||||
cloudOption.value = "cloud";
|
||||
cloudOption.textContent = "Cloud";
|
||||
|
||||
const serverlessOption = document.createElement("option");
|
||||
serverlessOption.value = "serverless";
|
||||
serverlessOption.textContent = "Serverless (Runpod)";
|
||||
|
||||
typeSelect.appendChild(localOption);
|
||||
typeSelect.appendChild(remoteOption);
|
||||
typeSelect.appendChild(cloudOption);
|
||||
typeSelect.appendChild(serverlessOption);
|
||||
|
||||
const runpodText = document.createElement("a");
|
||||
runpodText.id = `runpod-text-${worker.id}`;
|
||||
@@ -54,18 +59,32 @@ export function createWorkerSettingsForm(ui, extension, worker) {
|
||||
const cudaGroup = document.getElementById(`cuda-group-${worker.id}`);
|
||||
const argsGroup = document.getElementById(`args-group-${worker.id}`);
|
||||
const runpodTextElem = document.getElementById(`runpod-text-${worker.id}`);
|
||||
const serverlessGroups = [
|
||||
`endpoint-id-group-${worker.id}`,
|
||||
`api-key-group-${worker.id}`,
|
||||
`clear-api-key-group-${worker.id}`,
|
||||
`jobs-per-run-group-${worker.id}`,
|
||||
`concurrency-group-${worker.id}`,
|
||||
`timeout-seconds-group-${worker.id}`,
|
||||
`serverless-test-group-${worker.id}`,
|
||||
`serverless-help-${worker.id}`,
|
||||
].map((id) => document.getElementById(id));
|
||||
|
||||
if (!hostGroup || !portGroup || !cudaGroup || !argsGroup || !runpodTextElem || !hostInput || !portInput) {
|
||||
if (!hostGroup || !portGroup || !cudaGroup || !argsGroup || !runpodTextElem || !hostInput || !portInput || serverlessGroups.some((group) => !group)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const hideServerless = () => serverlessGroups.forEach((group) => { group.style.display = "none"; });
|
||||
|
||||
if (workerType === "local") {
|
||||
hideServerless();
|
||||
hostGroup.style.display = "none";
|
||||
portGroup.style.display = "flex";
|
||||
cudaGroup.style.display = "flex";
|
||||
argsGroup.style.display = "flex";
|
||||
runpodTextElem.style.display = "none";
|
||||
} else if (workerType === "remote") {
|
||||
hideServerless();
|
||||
hostGroup.style.display = "flex";
|
||||
portGroup.style.display = "flex";
|
||||
cudaGroup.style.display = "none";
|
||||
@@ -76,6 +95,7 @@ export function createWorkerSettingsForm(ui, extension, worker) {
|
||||
hostInput.value = "";
|
||||
}
|
||||
} else if (workerType === "cloud") {
|
||||
hideServerless();
|
||||
hostGroup.style.display = "flex";
|
||||
portGroup.style.display = "flex";
|
||||
cudaGroup.style.display = "none";
|
||||
@@ -86,6 +106,13 @@ export function createWorkerSettingsForm(ui, extension, worker) {
|
||||
if (hostInput.value === "localhost" || hostInput.value === "127.0.0.1") {
|
||||
hostInput.value = "";
|
||||
}
|
||||
} else if (workerType === "serverless") {
|
||||
hostGroup.style.display = "none";
|
||||
portGroup.style.display = "none";
|
||||
cudaGroup.style.display = "none";
|
||||
argsGroup.style.display = "none";
|
||||
runpodTextElem.style.display = "none";
|
||||
serverlessGroups.forEach((group) => { group.style.display = "flex"; });
|
||||
}
|
||||
};
|
||||
};
|
||||
@@ -114,6 +141,77 @@ export function createWorkerSettingsForm(ui, extension, worker) {
|
||||
argsGroup.group.style.display = (isRemoteWorker(extension, worker) || worker.type === "cloud") ? "none" : "flex";
|
||||
form.appendChild(argsGroup.group);
|
||||
|
||||
const serverlessHelp = document.createElement("small");
|
||||
serverlessHelp.id = `serverless-help-${worker.id}`;
|
||||
serverlessHelp.style.display = "none";
|
||||
serverlessHelp.textContent = "Preview: image Collectors only. The endpoint must provide the generic workflow + images handler and required models/nodes. Enabling it permits paid Runpod jobs.";
|
||||
form.appendChild(serverlessHelp);
|
||||
|
||||
const endpointGroup = ui.createFormGroup("Endpoint ID:", worker.endpoint_id || "", `endpoint-id-${worker.id}`, "text", "Runpod endpoint ID");
|
||||
endpointGroup.group.id = `endpoint-id-group-${worker.id}`;
|
||||
endpointGroup.group.style.display = "none";
|
||||
form.appendChild(endpointGroup.group);
|
||||
|
||||
const apiKeyGroup = ui.createFormGroup("Runpod API Key (write-only):", "", `api-key-${worker.id}`, "password", worker.api_key_configured ? "Leave blank to keep saved API key" : "Runpod API key");
|
||||
apiKeyGroup.group.id = `api-key-group-${worker.id}`;
|
||||
apiKeyGroup.input.autocomplete = "new-password";
|
||||
apiKeyGroup.input.spellcheck = false;
|
||||
apiKeyGroup.group.style.display = "none";
|
||||
form.appendChild(apiKeyGroup.group);
|
||||
|
||||
const clearApiKeyGroup = document.createElement("div");
|
||||
clearApiKeyGroup.id = `clear-api-key-group-${worker.id}`;
|
||||
clearApiKeyGroup.style.cssText = "display: none; align-items: center; gap: 8px;";
|
||||
const clearApiKeyInput = document.createElement("input");
|
||||
clearApiKeyInput.type = "checkbox";
|
||||
clearApiKeyInput.id = `clear-api-key-${worker.id}`;
|
||||
const clearApiKeyLabel = document.createElement("label");
|
||||
clearApiKeyLabel.htmlFor = clearApiKeyInput.id;
|
||||
clearApiKeyLabel.textContent = "Clear saved API key (disable pool first)";
|
||||
clearApiKeyGroup.appendChild(clearApiKeyInput);
|
||||
clearApiKeyGroup.appendChild(clearApiKeyLabel);
|
||||
clearApiKeyInput.onchange = () => {
|
||||
if (clearApiKeyInput.checked) apiKeyGroup.input.value = "";
|
||||
apiKeyGroup.input.disabled = clearApiKeyInput.checked;
|
||||
};
|
||||
form.appendChild(clearApiKeyGroup);
|
||||
|
||||
const jobsPerRunGroup = ui.createFormGroup("Jobs per run:", worker.jobs_per_run ?? 1, `jobs-per-run-${worker.id}`, "number");
|
||||
jobsPerRunGroup.group.id = `jobs-per-run-group-${worker.id}`;
|
||||
jobsPerRunGroup.input.min = "1";
|
||||
jobsPerRunGroup.input.max = "16";
|
||||
jobsPerRunGroup.input.step = "1";
|
||||
jobsPerRunGroup.input.title = "Maximum jobs submitted in one run";
|
||||
jobsPerRunGroup.group.style.display = "none";
|
||||
form.appendChild(jobsPerRunGroup.group);
|
||||
|
||||
const concurrencyGroup = ui.createFormGroup("Concurrent jobs (slots):", worker.concurrency ?? 1, `concurrency-${worker.id}`, "number");
|
||||
concurrencyGroup.group.id = `concurrency-group-${worker.id}`;
|
||||
concurrencyGroup.input.min = "1";
|
||||
concurrencyGroup.input.max = "16";
|
||||
concurrencyGroup.input.step = "1";
|
||||
concurrencyGroup.input.title = "Maximum concurrent jobs per serverless worker";
|
||||
concurrencyGroup.group.style.display = "none";
|
||||
form.appendChild(concurrencyGroup.group);
|
||||
|
||||
const timeoutGroup = ui.createFormGroup("Timeout (seconds):", worker.timeout_seconds ?? 1800, `timeout-seconds-${worker.id}`, "number");
|
||||
timeoutGroup.group.id = `timeout-seconds-group-${worker.id}`;
|
||||
timeoutGroup.input.min = "30";
|
||||
timeoutGroup.input.max = "86400";
|
||||
timeoutGroup.input.step = "1";
|
||||
timeoutGroup.group.style.display = "none";
|
||||
form.appendChild(timeoutGroup.group);
|
||||
|
||||
const testServerlessButton = ui.createButton("Test Saved Endpoint", () => testServerlessWorker(extension, worker.id));
|
||||
testServerlessButton.id = `test-serverless-${worker.id}`;
|
||||
testServerlessButton.title = "Test the saved Runpod endpoint without submitting inference";
|
||||
testServerlessButton.style.cssText = BUTTON_STYLES.base + BUTTON_STYLES.workerControl;
|
||||
const testServerlessGroup = document.createElement("div");
|
||||
testServerlessGroup.id = `serverless-test-group-${worker.id}`;
|
||||
testServerlessGroup.style.cssText = "display: none;";
|
||||
testServerlessGroup.appendChild(testServerlessButton);
|
||||
form.appendChild(testServerlessGroup);
|
||||
|
||||
const saveBtn = ui.createButton("Save", () => saveWorkerSettings(extension, worker.id), "background-color: #4a7c4a;");
|
||||
saveBtn.style.cssText = BUTTON_STYLES.base + BUTTON_STYLES.success;
|
||||
|
||||
@@ -128,7 +226,9 @@ export function createWorkerSettingsForm(ui, extension, worker) {
|
||||
|
||||
typeSelect.onchange = createOnChangeHandler();
|
||||
|
||||
if (worker.type === "cloud") {
|
||||
if (worker.type === "serverless") {
|
||||
typeSelect.value = "serverless";
|
||||
} else if (worker.type === "cloud") {
|
||||
typeSelect.value = "cloud";
|
||||
runpodText.style.display = "block";
|
||||
} else if (isRemoteWorker(extension, worker)) {
|
||||
|
||||
+93
-1
@@ -41,6 +41,28 @@ function setButtonVisibility(button, visible) {
|
||||
button.style.display = visible ? "" : "none";
|
||||
}
|
||||
|
||||
function updateServerlessStatusMessage(workerId, status) {
|
||||
const messageElement = document.getElementById(`serverless-status-${workerId}`);
|
||||
if (!messageElement) return;
|
||||
|
||||
const message = typeof status?.message === "string" && status.message
|
||||
? status.message
|
||||
: status?.ok ? "Ready to scale" : "Runpod status unavailable";
|
||||
const queueRemaining = Number(status?.queueRemaining);
|
||||
const parts = [message];
|
||||
if (Number.isFinite(queueRemaining)) {
|
||||
parts.push(`${Math.max(0, queueRemaining)} queued`);
|
||||
}
|
||||
const jobStatuses = (Array.isArray(status?.jobs) ? status.jobs : [])
|
||||
.slice(0, 2)
|
||||
.map((job) => String(job?.status || "").trim())
|
||||
.filter(Boolean);
|
||||
if (jobStatuses.length) {
|
||||
parts.push(`Jobs: ${jobStatuses.join(", ")}`);
|
||||
}
|
||||
messageElement.textContent = parts.join(" • ");
|
||||
}
|
||||
|
||||
export async function checkAllWorkerStatuses(extension) {
|
||||
if (_statusCheckRunning || !extension.panelElement) {
|
||||
return;
|
||||
@@ -136,11 +158,60 @@ export async function checkMasterStatus(extension) {
|
||||
|
||||
// Helper to build worker URL
|
||||
export function getWorkerUrl(extension, worker, endpoint = "") {
|
||||
if (worker?.type === "serverless") {
|
||||
return null;
|
||||
}
|
||||
return buildWorkerUrl(worker, endpoint, window.location);
|
||||
}
|
||||
|
||||
export async function checkWorkerStatus(extension, worker) {
|
||||
// Assume caller ensured enabled; proceed with check
|
||||
if (worker.type === "serverless") {
|
||||
try {
|
||||
const result = await extension.api.getServerlessStatus(worker.id);
|
||||
const queueRemaining = Number(result?.queueRemaining);
|
||||
const normalizedQueue = Number.isFinite(queueRemaining) ? Math.max(0, queueRemaining) : 0;
|
||||
const normalizedStatus = {
|
||||
ok: Boolean(result?.ok),
|
||||
queueRemaining: normalizedQueue,
|
||||
message: typeof result?.message === "string" ? result.message : "",
|
||||
jobs: (Array.isArray(result?.jobs) ? result.jobs : []).map((job) => ({
|
||||
job_id: String(job?.job_id || ""),
|
||||
status: String(job?.status || ""),
|
||||
worker_id: String(job?.worker_id || ""),
|
||||
execution_id: String(job?.execution_id || ""),
|
||||
})),
|
||||
};
|
||||
const hasActiveJobs = normalizedStatus.jobs.some((job) =>
|
||||
/^(waiting|submitting|in_queue|queued|pending|running|processing|starting|in_progress|downloading|cancel_requested)$/i.test(job.status.trim())
|
||||
);
|
||||
const status = {
|
||||
online: normalizedStatus.ok,
|
||||
processing: normalizedQueue > 0 || hasActiveJobs,
|
||||
queueCount: normalizedQueue,
|
||||
};
|
||||
extension.state.updateWorker(worker.id, { serverlessStatus: normalizedStatus });
|
||||
extension.state.setWorkerStatus(worker.id, status);
|
||||
updateServerlessStatusMessage(worker.id, normalizedStatus);
|
||||
const updatedInPlace = extension.updateWorkerCard?.(worker.id, status);
|
||||
if (!updatedInPlace) {
|
||||
updateWorkerControls(extension, worker.id);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error?.name === "AbortError") return;
|
||||
const failedStatus = { ok: false, queueRemaining: 0, message: error?.message || "Runpod status unavailable", jobs: [] };
|
||||
extension.state.updateWorker(worker.id, { serverlessStatus: failedStatus });
|
||||
extension.state.setWorkerStatus(worker.id, { online: false, processing: false, queueCount: 0 });
|
||||
updateServerlessStatusMessage(worker.id, failedStatus);
|
||||
const updatedInPlace = extension.updateWorkerCard?.(worker.id, extension.state.getWorkerStatus(worker.id));
|
||||
if (!updatedInPlace) {
|
||||
updateWorkerControls(extension, worker.id);
|
||||
}
|
||||
extension.log(`Serverless worker ${worker.id} status check failed: ${error.message}`, "debug");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const workerUrl = getWorkerUrl(extension, worker);
|
||||
|
||||
try {
|
||||
@@ -206,6 +277,9 @@ export async function checkWorkerStatus(extension, worker) {
|
||||
|
||||
export async function launchWorker(extension, workerId) {
|
||||
const worker = extension.config.workers.find((w) => w.id === workerId);
|
||||
if (!worker || worker.type === "serverless") {
|
||||
return;
|
||||
}
|
||||
|
||||
// If worker is disabled, enable it first
|
||||
if (!worker.enabled) {
|
||||
@@ -273,6 +347,9 @@ export async function launchWorker(extension, workerId) {
|
||||
|
||||
export async function stopWorker(extension, workerId) {
|
||||
const worker = extension.config.workers.find((w) => w.id === workerId);
|
||||
if (!worker || worker.type === "serverless") {
|
||||
return;
|
||||
}
|
||||
const stopBtn = document.querySelectorAll(`#controls-${workerId} button`)[1];
|
||||
|
||||
// Provide immediate feedback
|
||||
@@ -349,8 +426,12 @@ export async function loadManagedWorkers(extension) {
|
||||
try {
|
||||
const result = await extension.api.getManagedWorkers();
|
||||
|
||||
// Check for launching workers
|
||||
const serverlessWorkerIds = new Set(
|
||||
(extension.config?.workers || []).filter((worker) => worker.type === "serverless").map((worker) => String(worker.id))
|
||||
);
|
||||
// Managed-worker lifecycle state is ComfyUI-process-only; serverless is provider-managed.
|
||||
for (const [workerId, info] of Object.entries(result.managed_workers)) {
|
||||
if (serverlessWorkerIds.has(String(workerId))) continue;
|
||||
extension.state.setWorkerManaged(workerId, info);
|
||||
|
||||
// If worker is marked as launching, add to launchingWorkers set
|
||||
@@ -386,6 +467,13 @@ export function updateWorkerControls(extension, workerId) {
|
||||
const stopBtn = document.getElementById(`stop-${workerId}`);
|
||||
const logBtn = document.getElementById(`log-${workerId}`);
|
||||
|
||||
if (worker.type === "serverless") {
|
||||
setButtonVisibility(launchBtn, false);
|
||||
setButtonVisibility(stopBtn, false);
|
||||
setButtonVisibility(logBtn, false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (isRemoteWorker(extension, worker)) {
|
||||
setButtonVisibility(launchBtn, false);
|
||||
setButtonVisibility(stopBtn, false);
|
||||
@@ -438,6 +526,9 @@ export function updateWorkerControls(extension, workerId) {
|
||||
|
||||
export async function viewWorkerLog(extension, workerId, isRemote = false) {
|
||||
const worker = extension.config.workers.find((w) => w.id === workerId);
|
||||
if (worker?.type === "serverless") {
|
||||
return;
|
||||
}
|
||||
const isRemoteLog = isRemote || (worker ? isRemoteWorker(extension, worker) : false);
|
||||
const managedInfo = extension.state.getWorker(workerId).managed;
|
||||
if (!isRemoteLog && !managedInfo?.log_file) {
|
||||
@@ -498,6 +589,7 @@ export async function refreshLog(extension, workerId, silent = false) {
|
||||
|
||||
try {
|
||||
const worker = extension.config.workers.find((w) => w.id === workerId);
|
||||
if (worker?.type === "serverless") return;
|
||||
const isRemoteLog = worker ? isRemoteWorker(extension, worker) : false;
|
||||
const data = isRemoteLog
|
||||
? await extension.api.getRemoteWorkerLog(workerId, 300)
|
||||
|
||||
+149
-2
@@ -18,7 +18,7 @@ export function isRemoteWorker(extension, worker) {
|
||||
const workerType = String(worker?.type || "").toLowerCase();
|
||||
|
||||
// Explicit type always wins over host heuristics.
|
||||
if (workerType === "cloud" || workerType === "remote") {
|
||||
if (workerType === "cloud" || workerType === "remote" || workerType === "serverless") {
|
||||
return true;
|
||||
}
|
||||
if (workerType === "local") {
|
||||
@@ -37,6 +37,134 @@ export function isCloudWorker(extension, worker) {
|
||||
return worker.type === "cloud";
|
||||
}
|
||||
|
||||
function showServerlessValidationError(extension, detail) {
|
||||
extension.app.extensionManager.toast.add({
|
||||
severity: "error",
|
||||
summary: "Validation Error",
|
||||
detail,
|
||||
life: 3000,
|
||||
});
|
||||
}
|
||||
|
||||
async function saveServerlessWorkerSettings(extension, worker, workerId, name) {
|
||||
const endpointId = String(document.getElementById(`endpoint-id-${workerId}`).value || "").trim();
|
||||
const apiKey = document.getElementById(`api-key-${workerId}`).value || "";
|
||||
const clearApiKey = Boolean(document.getElementById(`clear-api-key-${workerId}`).checked);
|
||||
const jobsPerRun = Number(document.getElementById(`jobs-per-run-${workerId}`).value);
|
||||
const concurrency = Number(document.getElementById(`concurrency-${workerId}`).value);
|
||||
const timeoutSeconds = Number(document.getElementById(`timeout-seconds-${workerId}`).value);
|
||||
|
||||
if (!name.trim()) {
|
||||
showServerlessValidationError(extension, "Worker name is required");
|
||||
return;
|
||||
}
|
||||
if (!endpointId) {
|
||||
showServerlessValidationError(extension, "Endpoint ID is required for Runpod serverless workers");
|
||||
return;
|
||||
}
|
||||
if (!Number.isInteger(jobsPerRun) || jobsPerRun < 1 || jobsPerRun > 16) {
|
||||
showServerlessValidationError(extension, "Jobs per run must be an integer between 1 and 16");
|
||||
return;
|
||||
}
|
||||
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 16) {
|
||||
showServerlessValidationError(extension, "Concurrency must be an integer between 1 and 16");
|
||||
return;
|
||||
}
|
||||
if (!Number.isInteger(timeoutSeconds) || timeoutSeconds < 30 || timeoutSeconds > 86400) {
|
||||
showServerlessValidationError(extension, "Timeout must be an integer between 30 and 86400 seconds");
|
||||
return;
|
||||
}
|
||||
|
||||
const updateData = {
|
||||
name: name.trim(),
|
||||
type: "serverless",
|
||||
provider: "runpod",
|
||||
endpoint_id: endpointId,
|
||||
...(apiKey.length > 0 && !clearApiKey ? { api_key: apiKey } : {}),
|
||||
...(clearApiKey ? { clear_api_key: true } : {}),
|
||||
jobs_per_run: jobsPerRun,
|
||||
concurrency,
|
||||
timeout_seconds: timeoutSeconds,
|
||||
// Never enable a serverless worker as a side effect of saving credentials.
|
||||
enabled: worker.enabled,
|
||||
};
|
||||
|
||||
try {
|
||||
await extension.api.updateWorker(workerId, updateData);
|
||||
document.getElementById(`api-key-${workerId}`).value = "";
|
||||
|
||||
worker.name = name.trim();
|
||||
worker.type = "serverless";
|
||||
worker.provider = "runpod";
|
||||
worker.endpoint_id = endpointId;
|
||||
worker.jobs_per_run = jobsPerRun;
|
||||
worker.concurrency = concurrency;
|
||||
worker.timeout_seconds = timeoutSeconds;
|
||||
worker.api_key_configured = clearApiKey
|
||||
? false
|
||||
: apiKey.length > 0 || Boolean(worker.api_key_configured);
|
||||
// The credential is write-only: never copy it into extension config/state.
|
||||
delete worker.api_key;
|
||||
delete worker.host;
|
||||
delete worker.port;
|
||||
delete worker.cuda_device;
|
||||
delete worker.extra_args;
|
||||
|
||||
extension.state.updateWorker(workerId, { enabled: worker.enabled });
|
||||
emitWorkersChanged(extension);
|
||||
extension.app.extensionManager.toast.add({
|
||||
severity: "success",
|
||||
summary: "Settings Saved",
|
||||
detail: `Worker ${worker.name} settings updated`,
|
||||
life: 3000,
|
||||
});
|
||||
extension.state.setWorkerExpanded(workerId, false);
|
||||
if (extension.panelElement) {
|
||||
renderSidebarContent(extension, extension.panelElement);
|
||||
}
|
||||
} catch (error) {
|
||||
extension.app.extensionManager.toast.add({
|
||||
severity: "error",
|
||||
summary: "Save Failed",
|
||||
detail: error.message,
|
||||
life: 5000,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function testServerlessWorker(extension, workerId) {
|
||||
const button = document.getElementById(`test-serverless-${workerId}`);
|
||||
const originalText = button?.textContent;
|
||||
if (button) {
|
||||
button.disabled = true;
|
||||
button.textContent = "Testing...";
|
||||
}
|
||||
try {
|
||||
// This endpoint only checks the saved endpoint configuration; it does not enqueue inference.
|
||||
const result = await extension.api.testServerlessWorker(workerId);
|
||||
extension.app.extensionManager.toast.add({
|
||||
severity: result?.ok ? "success" : "error",
|
||||
summary: result?.ok ? "Runpod Endpoint Ready" : "Runpod Endpoint Test Failed",
|
||||
detail: result?.message || (result?.ok ? "Saved endpoint is reachable" : "Endpoint test failed"),
|
||||
life: 4000,
|
||||
});
|
||||
return result;
|
||||
} catch (error) {
|
||||
extension.app.extensionManager.toast.add({
|
||||
severity: "error",
|
||||
summary: "Runpod Endpoint Test Failed",
|
||||
detail: error.message,
|
||||
life: 5000,
|
||||
});
|
||||
return null;
|
||||
} finally {
|
||||
if (button) {
|
||||
button.disabled = false;
|
||||
button.textContent = originalText;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveWorkerSettings(extension, workerId) {
|
||||
const worker = extension.config.workers.find((w) => w.id === workerId);
|
||||
if (!worker) {
|
||||
@@ -44,8 +172,12 @@ export async function saveWorkerSettings(extension, workerId) {
|
||||
}
|
||||
|
||||
// Get form values
|
||||
const name = document.getElementById(`name-${workerId}`).value;
|
||||
const workerType = document.getElementById(`worker-type-${workerId}`).value;
|
||||
if (workerType === "serverless") {
|
||||
await saveServerlessWorkerSettings(extension, worker, workerId, document.getElementById(`name-${workerId}`).value);
|
||||
return;
|
||||
}
|
||||
const name = document.getElementById(`name-${workerId}`).value;
|
||||
const isRemote = workerType === "remote" || workerType === "cloud";
|
||||
const isCloud = workerType === "cloud";
|
||||
const rawHost = isRemote ? document.getElementById(`host-${workerId}`).value : window.location.hostname;
|
||||
@@ -209,6 +341,21 @@ export function cancelWorkerSettings(extension, workerId) {
|
||||
document.getElementById(`port-${workerId}`).value = worker.port;
|
||||
document.getElementById(`cuda-${workerId}`).value = worker.cuda_device || 0;
|
||||
document.getElementById(`args-${workerId}`).value = worker.extra_args || "";
|
||||
const endpointInput = document.getElementById(`endpoint-id-${workerId}`);
|
||||
if (endpointInput) endpointInput.value = worker.endpoint_id || "";
|
||||
const apiKeyInput = document.getElementById(`api-key-${workerId}`);
|
||||
if (apiKeyInput) {
|
||||
apiKeyInput.value = "";
|
||||
apiKeyInput.disabled = false;
|
||||
}
|
||||
const clearApiKeyInput = document.getElementById(`clear-api-key-${workerId}`);
|
||||
if (clearApiKeyInput) clearApiKeyInput.checked = false;
|
||||
const jobsPerRunInput = document.getElementById(`jobs-per-run-${workerId}`);
|
||||
if (jobsPerRunInput) jobsPerRunInput.value = worker.jobs_per_run ?? 1;
|
||||
const concurrencyInput = document.getElementById(`concurrency-${workerId}`);
|
||||
if (concurrencyInput) concurrencyInput.value = worker.concurrency ?? 1;
|
||||
const timeoutInput = document.getElementById(`timeout-seconds-${workerId}`);
|
||||
if (timeoutInput) timeoutInput.value = worker.timeout_seconds ?? 1800;
|
||||
|
||||
// Reset remote checkbox
|
||||
const remoteCheckbox = document.getElementById(`remote-${workerId}`);
|
||||
|
||||
+20
-10
@@ -19,12 +19,17 @@ export async function handleWorkerOperation(extension, button, operation, succes
|
||||
setButtonStateClass("btn--working");
|
||||
|
||||
try {
|
||||
const urlsToProcess = extension.enabledWorkers.map(w => ({
|
||||
name: w.name,
|
||||
url: getWorkerUrl(extension, w)
|
||||
const workersToProcess = extension.enabledWorkers.filter((worker) =>
|
||||
operation.endpoint !== ENDPOINTS.CLEAR_MEMORY || worker.type !== "serverless"
|
||||
);
|
||||
const targets = workersToProcess.map((worker) => ({
|
||||
name: worker.name,
|
||||
worker,
|
||||
serverless: worker.type === "serverless",
|
||||
url: worker.type === "serverless" ? null : getWorkerUrl(extension, worker),
|
||||
}));
|
||||
|
||||
if (urlsToProcess.length === 0) {
|
||||
|
||||
if (targets.length === 0) {
|
||||
button.textContent = "No Workers";
|
||||
setButtonStateClass("btn--error");
|
||||
setTimeout(() => {
|
||||
@@ -37,14 +42,19 @@ export async function handleWorkerOperation(extension, button, operation, succes
|
||||
return;
|
||||
}
|
||||
|
||||
const promises = urlsToProcess.map(target =>
|
||||
fetch(`${target.url}${operation.endpoint}`, {
|
||||
method: 'POST',
|
||||
const promises = targets.map((target) => {
|
||||
if (target.serverless && operation.name === "Interrupt") {
|
||||
return extension.api.cancelServerlessWorker(target.worker.id)
|
||||
.then((result) => ({ ok: result?.ok !== false, name: target.name }))
|
||||
.catch(() => ({ ok: false, name: target.name }));
|
||||
}
|
||||
return fetch(`${target.url}${operation.endpoint}`, {
|
||||
method: 'POST',
|
||||
mode: 'cors'
|
||||
})
|
||||
.then(response => ({ ok: response.ok, name: target.name }))
|
||||
.catch(() => ({ ok: false, name: target.name }))
|
||||
);
|
||||
.catch(() => ({ ok: false, name: target.name }));
|
||||
});
|
||||
|
||||
const results = await Promise.all(promises);
|
||||
const failures = results.filter(r => !r.ok);
|
||||
|
||||
Reference in New Issue
Block a user