562 lines
20 KiB
JavaScript
562 lines
20 KiB
JavaScript
import { BUTTON_STYLES, TIMEOUTS, STATUS_COLORS, ENDPOINTS } from './constants.js';
|
|
|
|
export async function checkAllWorkerStatuses(extension) {
|
|
// Don't continue if panel is closed
|
|
if (!extension.panelElement) {
|
|
return;
|
|
}
|
|
|
|
// Create new abort controller for this round of checks
|
|
extension.statusCheckAbortController = new AbortController();
|
|
|
|
// Check master status
|
|
checkMasterStatus(extension);
|
|
|
|
if (!extension.config || !extension.config.workers) {
|
|
return;
|
|
}
|
|
|
|
for (const worker of extension.config.workers) {
|
|
// Check status for enabled workers OR workers that are launching
|
|
if (worker.enabled || extension.state.isWorkerLaunching(worker.id)) {
|
|
checkWorkerStatus(extension, worker);
|
|
}
|
|
}
|
|
|
|
// Determine next interval based on current state
|
|
let isActive = extension.state.getMasterStatus() === "processing"; // Master is busy
|
|
|
|
// Check workers for activity
|
|
extension.config.workers.forEach((worker) => {
|
|
const ws = extension.state.getWorker(worker.id); // Get worker state
|
|
if (ws.launching || ws.status?.processing) {
|
|
// Launching or processing
|
|
isActive = true;
|
|
}
|
|
});
|
|
|
|
// Set next delay: 1s if active, 5s if idle
|
|
const nextInterval = isActive ? 1000 : 5000;
|
|
|
|
// Schedule the next check
|
|
extension.statusCheckTimeout = setTimeout(() => checkAllWorkerStatuses(extension), nextInterval);
|
|
}
|
|
|
|
export async function checkMasterStatus(extension) {
|
|
try {
|
|
const response = await fetch(`${window.location.origin}${ENDPOINTS.PROMPT}`, {
|
|
method: "GET",
|
|
signal: AbortSignal.timeout(TIMEOUTS.STATUS_CHECK),
|
|
});
|
|
|
|
if (response.ok) {
|
|
const data = await response.json();
|
|
const queueRemaining = data.exec_info?.queue_remaining || 0;
|
|
const isProcessing = queueRemaining > 0;
|
|
|
|
// Update master status in state
|
|
extension.state.setMasterStatus(isProcessing ? "processing" : "online");
|
|
|
|
// Update master status dot
|
|
const statusDot = document.getElementById("master-status");
|
|
if (statusDot) {
|
|
if (!extension.isMasterParticipating()) {
|
|
if (isProcessing) {
|
|
statusDot.style.backgroundColor = STATUS_COLORS.PROCESSING_YELLOW;
|
|
statusDot.title = `Orchestrating (${queueRemaining} in queue)`;
|
|
} else {
|
|
statusDot.style.backgroundColor = STATUS_COLORS.DISABLED_GRAY;
|
|
statusDot.title = "Master orchestrator only";
|
|
}
|
|
} else if (isProcessing) {
|
|
statusDot.style.backgroundColor = STATUS_COLORS.PROCESSING_YELLOW;
|
|
statusDot.title = `Processing (${queueRemaining} in queue)`;
|
|
} else {
|
|
statusDot.style.backgroundColor = STATUS_COLORS.ONLINE_GREEN;
|
|
statusDot.title = "Online";
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
// Master is always online (we're running on it), so keep it green
|
|
const statusDot = document.getElementById("master-status");
|
|
if (statusDot) {
|
|
statusDot.style.backgroundColor = extension.isMasterParticipating() ? STATUS_COLORS.ONLINE_GREEN : STATUS_COLORS.DISABLED_GRAY;
|
|
statusDot.title = extension.isMasterParticipating() ? "Online" : "Master orchestrator only";
|
|
}
|
|
}
|
|
}
|
|
|
|
// Helper to build worker URL
|
|
export function getWorkerUrl(extension, worker, endpoint = "") {
|
|
const parsed = extension._parseHostInput(worker.host || window.location.hostname);
|
|
const host = parsed.host || window.location.hostname;
|
|
const resolvedPort = parsed.port || worker.port;
|
|
|
|
// Cloud workers always use HTTPS
|
|
const isCloud = worker.type === "cloud";
|
|
|
|
// Detect if we're running on Runpod (for local workers on Runpod infrastructure)
|
|
const isRunpodProxy = host.endsWith(".proxy.runpod.net");
|
|
|
|
// For local workers on Runpod, construct the port-specific proxy URL
|
|
let finalHost = host;
|
|
if (!worker.host && isRunpodProxy) {
|
|
const match = host.match(/^(.*)\.proxy\.runpod\.net$/);
|
|
if (match) {
|
|
const podId = match[1];
|
|
const domain = "proxy.runpod.net";
|
|
finalHost = `${podId}-${worker.port}.${domain}`;
|
|
} else {
|
|
// Fallback or log error if no match (shouldn't happen)
|
|
console.error(`[Distributed] Failed to parse Runpod proxy host: ${host}`);
|
|
}
|
|
}
|
|
|
|
// Determine protocol: HTTPS for cloud, Runpod proxies, or port 443
|
|
const useHttps = isCloud || isRunpodProxy || resolvedPort === 443;
|
|
const protocol = useHttps ? "https" : "http";
|
|
|
|
// Only add port if non-standard
|
|
const defaultPort = useHttps ? 443 : 80;
|
|
const needsPort = !isRunpodProxy && resolvedPort !== defaultPort;
|
|
const portStr = needsPort ? `:${resolvedPort}` : "";
|
|
|
|
return `${protocol}://${finalHost}${portStr}${endpoint}`;
|
|
}
|
|
|
|
export async function checkWorkerStatus(extension, worker) {
|
|
// Assume caller ensured enabled; proceed with check
|
|
const url = getWorkerUrl(extension, worker, ENDPOINTS.PROMPT);
|
|
|
|
try {
|
|
// Combine timeout with abort controller signal
|
|
const timeoutSignal = AbortSignal.timeout(TIMEOUTS.STATUS_CHECK);
|
|
const signal = extension.statusCheckAbortController
|
|
? AbortSignal.any([timeoutSignal, extension.statusCheckAbortController.signal])
|
|
: timeoutSignal;
|
|
|
|
const response = await fetch(url, {
|
|
method: "GET",
|
|
mode: "cors",
|
|
signal,
|
|
});
|
|
|
|
if (response.ok) {
|
|
const data = await response.json();
|
|
const queueRemaining = data.exec_info?.queue_remaining || 0;
|
|
const isProcessing = queueRemaining > 0;
|
|
|
|
// Update status
|
|
extension.state.setWorkerStatus(worker.id, {
|
|
online: true,
|
|
processing: isProcessing,
|
|
queueCount: queueRemaining,
|
|
});
|
|
|
|
// Update status dot based on processing state
|
|
if (isProcessing) {
|
|
extension.ui.updateStatusDot(worker.id, "#f0ad4e", `Online - Processing (${queueRemaining} in queue)`, false);
|
|
} else {
|
|
extension.ui.updateStatusDot(worker.id, "#3ca03c", "Online - Idle", false);
|
|
}
|
|
|
|
// Clear launching state since worker is now online
|
|
if (extension.state.isWorkerLaunching(worker.id)) {
|
|
extension.state.setWorkerLaunching(worker.id, false);
|
|
clearLaunchingFlag(extension, worker.id);
|
|
}
|
|
} else {
|
|
throw new Error(`HTTP ${response.status}`);
|
|
}
|
|
} catch (error) {
|
|
// Don't process aborted requests
|
|
if (error.name === "AbortError") {
|
|
return;
|
|
}
|
|
|
|
// Worker is offline or unreachable
|
|
extension.state.setWorkerStatus(worker.id, {
|
|
online: false,
|
|
processing: false,
|
|
queueCount: 0,
|
|
});
|
|
|
|
// Check if worker is launching
|
|
if (extension.state.isWorkerLaunching(worker.id)) {
|
|
extension.ui.updateStatusDot(worker.id, "#f0ad4e", "Launching...", true);
|
|
} else if (worker.enabled) {
|
|
// Only update to red if not currently launching AND still enabled
|
|
extension.ui.updateStatusDot(worker.id, "#c04c4c", "Offline - Cannot connect", false);
|
|
}
|
|
// If disabled, don't update the dot (leave it gray)
|
|
|
|
extension.log(`Worker ${worker.id} status check failed: ${error.message}`, "debug");
|
|
}
|
|
|
|
// Update control buttons based on new status
|
|
updateWorkerControls(extension, worker.id);
|
|
}
|
|
|
|
export async function launchWorker(extension, workerId) {
|
|
const worker = extension.config.workers.find((w) => w.id === workerId);
|
|
|
|
// If worker is disabled, enable it first
|
|
if (!worker.enabled) {
|
|
await extension.updateWorkerEnabled(workerId, true);
|
|
|
|
// Update the checkbox UI
|
|
const checkbox = document.getElementById(`gpu-${workerId}`);
|
|
if (checkbox) {
|
|
checkbox.checked = true;
|
|
}
|
|
}
|
|
|
|
// Re-query button AFTER updateWorkerEnabled (which may re-render sidebar)
|
|
const launchBtn = document.querySelector(`#controls-${workerId} button`);
|
|
|
|
extension.ui.updateStatusDot(workerId, "#f0ad4e", "Launching...", true);
|
|
extension.state.setWorkerLaunching(workerId, true);
|
|
|
|
// Allow 90 seconds for worker to launch (model loading can take time)
|
|
setTimeout(() => {
|
|
extension.state.setWorkerLaunching(workerId, false);
|
|
}, TIMEOUTS.LAUNCH);
|
|
|
|
if (!launchBtn) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
// Disable button immediately
|
|
launchBtn.disabled = true;
|
|
|
|
const result = await extension.api.launchWorker(workerId);
|
|
if (result) {
|
|
extension.log(`Launched ${worker.name} (PID: ${result.pid})`, "info");
|
|
if (result.log_file) {
|
|
extension.log(`Log file: ${result.log_file}`, "debug");
|
|
}
|
|
|
|
extension.state.setWorkerManaged(workerId, {
|
|
pid: result.pid,
|
|
log_file: result.log_file,
|
|
started_at: Date.now(),
|
|
});
|
|
|
|
// Update controls immediately to hide launch button and show stop/log buttons
|
|
updateWorkerControls(extension, workerId);
|
|
setTimeout(() => checkWorkerStatus(extension, worker), TIMEOUTS.STATUS_CHECK);
|
|
}
|
|
} catch (error) {
|
|
// Check if worker was already running
|
|
if (error.message && error.message.includes("already running")) {
|
|
extension.log(`Worker ${worker.name} is already running`, "info");
|
|
updateWorkerControls(extension, workerId);
|
|
setTimeout(() => checkWorkerStatus(extension, worker), TIMEOUTS.STATUS_CHECK_DELAY);
|
|
} else {
|
|
extension.log(`Error launching worker: ${error.message || error}`, "error");
|
|
|
|
// Re-enable button on error
|
|
if (launchBtn) {
|
|
launchBtn.disabled = false;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function stopWorker(extension, workerId) {
|
|
const worker = extension.config.workers.find((w) => w.id === workerId);
|
|
const stopBtn = document.querySelectorAll(`#controls-${workerId} button`)[1];
|
|
|
|
// Provide immediate feedback
|
|
if (stopBtn) {
|
|
stopBtn.disabled = true;
|
|
stopBtn.textContent = "Stopping...";
|
|
stopBtn.style.backgroundColor = "#666";
|
|
}
|
|
|
|
try {
|
|
const result = await extension.api.stopWorker(workerId);
|
|
if (result) {
|
|
extension.log(`Stopped worker: ${result.message}`, "info");
|
|
extension.state.setWorkerManaged(workerId, null);
|
|
|
|
// Immediately update status to offline
|
|
extension.ui.updateStatusDot(workerId, "#c04c4c", "Offline");
|
|
extension.state.setWorkerStatus(workerId, { online: false });
|
|
|
|
// Flash success feedback
|
|
if (stopBtn) {
|
|
stopBtn.style.backgroundColor = BUTTON_STYLES.success;
|
|
stopBtn.textContent = "Stopped!";
|
|
setTimeout(() => {
|
|
updateWorkerControls(extension, workerId);
|
|
}, TIMEOUTS.FLASH_SHORT);
|
|
}
|
|
|
|
// Verify status after a short delay
|
|
setTimeout(() => checkWorkerStatus(extension, worker), TIMEOUTS.STATUS_CHECK);
|
|
} else {
|
|
extension.log(`Failed to stop worker: ${result.message}`, "error");
|
|
|
|
// Flash error feedback
|
|
if (stopBtn) {
|
|
stopBtn.style.backgroundColor = BUTTON_STYLES.error;
|
|
stopBtn.textContent = result.message.includes("already stopped") ? "Not Running" : "Failed";
|
|
|
|
// If already stopped, update status immediately
|
|
if (result.message.includes("already stopped")) {
|
|
extension.ui.updateStatusDot(workerId, "#c04c4c", "Offline");
|
|
extension.state.setWorkerStatus(workerId, { online: false });
|
|
}
|
|
|
|
setTimeout(() => {
|
|
updateWorkerControls(extension, workerId);
|
|
}, TIMEOUTS.FLASH_MEDIUM);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
extension.log(`Error stopping worker: ${error}`, "error");
|
|
|
|
// Reset button on error
|
|
if (stopBtn) {
|
|
stopBtn.style.backgroundColor = BUTTON_STYLES.error;
|
|
stopBtn.textContent = "Error";
|
|
setTimeout(() => {
|
|
updateWorkerControls(extension, workerId);
|
|
}, TIMEOUTS.FLASH_MEDIUM);
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function clearLaunchingFlag(extension, workerId) {
|
|
try {
|
|
await extension.api.clearLaunchingFlag(workerId);
|
|
extension.log(`Cleared launching flag for worker ${workerId}`, "debug");
|
|
} catch (error) {
|
|
extension.log(`Error clearing launching flag: ${error.message || error}`, "error");
|
|
}
|
|
}
|
|
|
|
export async function loadManagedWorkers(extension) {
|
|
try {
|
|
const result = await extension.api.getManagedWorkers();
|
|
|
|
// Check for launching workers
|
|
for (const [workerId, info] of Object.entries(result.managed_workers)) {
|
|
extension.state.setWorkerManaged(workerId, info);
|
|
|
|
// If worker is marked as launching, add to launchingWorkers set
|
|
if (info.launching) {
|
|
extension.state.setWorkerLaunching(workerId, true);
|
|
extension.log(`Worker ${workerId} is in launching state`, "debug");
|
|
}
|
|
}
|
|
|
|
// Update UI for all workers
|
|
if (extension.config?.workers) {
|
|
extension.config.workers.forEach((w) => updateWorkerControls(extension, w.id));
|
|
}
|
|
} catch (error) {
|
|
extension.log(`Error loading managed workers: ${error}`, "error");
|
|
}
|
|
}
|
|
|
|
export function updateWorkerControls(extension, workerId) {
|
|
const controlsDiv = document.getElementById(`controls-${workerId}`);
|
|
|
|
if (!controlsDiv) {
|
|
return;
|
|
}
|
|
|
|
const worker = extension.config.workers.find((w) => w.id === workerId);
|
|
if (!worker) {
|
|
return;
|
|
}
|
|
|
|
// Skip button updates for remote workers
|
|
if (extension.isRemoteWorker(worker)) {
|
|
return;
|
|
}
|
|
|
|
// Ensure we check for string ID
|
|
const managedInfo = extension.state.getWorker(workerId).managed;
|
|
const status = extension.state.getWorkerStatus(workerId);
|
|
|
|
// Update button states - buttons are now inside a wrapper div
|
|
const launchBtn = document.getElementById(`launch-${workerId}`);
|
|
const stopBtn = document.getElementById(`stop-${workerId}`);
|
|
const logBtn = document.getElementById(`log-${workerId}`);
|
|
|
|
// Show log button immediately if we have log file info (even if worker is still starting)
|
|
if (managedInfo?.log_file && logBtn) {
|
|
logBtn.style.display = "";
|
|
} else if (logBtn && !managedInfo) {
|
|
logBtn.style.display = "none";
|
|
}
|
|
|
|
if (status?.online || managedInfo) {
|
|
// Worker is running or we just launched it
|
|
launchBtn.style.display = "none"; // Hide launch button when running
|
|
|
|
if (managedInfo) {
|
|
// Only show stop button if we manage this worker
|
|
stopBtn.style.display = "";
|
|
stopBtn.disabled = false;
|
|
stopBtn.textContent = "Stop";
|
|
stopBtn.style.backgroundColor = "#7c4a4a"; // Red when enabled
|
|
} else {
|
|
// Hide stop button for workers launched outside UI
|
|
stopBtn.style.display = "none";
|
|
}
|
|
} else {
|
|
// Worker is not running
|
|
launchBtn.style.display = ""; // Show launch button
|
|
launchBtn.disabled = false;
|
|
launchBtn.textContent = "Launch";
|
|
launchBtn.style.backgroundColor = "#4a7c4a"; // Always green
|
|
|
|
stopBtn.style.display = "none"; // Hide stop button when not running
|
|
}
|
|
}
|
|
|
|
export async function viewWorkerLog(extension, workerId) {
|
|
const managedInfo = extension.state.getWorker(workerId).managed;
|
|
if (!managedInfo?.log_file) {
|
|
return;
|
|
}
|
|
|
|
const logBtn = document.getElementById(`log-${workerId}`);
|
|
|
|
// Provide immediate feedback
|
|
if (logBtn) {
|
|
logBtn.disabled = true;
|
|
logBtn.textContent = "Loading...";
|
|
logBtn.style.backgroundColor = "#666";
|
|
}
|
|
|
|
try {
|
|
// Fetch log content
|
|
const data = await extension.api.getWorkerLog(workerId, 1000);
|
|
|
|
// Create modal dialog
|
|
extension.ui.showLogModal(extension, workerId, data);
|
|
|
|
// Restore button
|
|
if (logBtn) {
|
|
logBtn.disabled = false;
|
|
logBtn.textContent = "View Log";
|
|
logBtn.style.backgroundColor = "#685434"; // Keep the yellow color
|
|
}
|
|
} catch (error) {
|
|
extension.log("Error viewing log: " + error.message, "error");
|
|
extension.app.extensionManager.toast.add({
|
|
severity: "error",
|
|
summary: "Error",
|
|
detail: `Failed to load log: ${error.message}`,
|
|
life: 5000,
|
|
});
|
|
|
|
// Flash error and restore button
|
|
if (logBtn) {
|
|
logBtn.style.backgroundColor = BUTTON_STYLES.error;
|
|
logBtn.textContent = "Error";
|
|
setTimeout(() => {
|
|
logBtn.disabled = false;
|
|
logBtn.textContent = "View Log";
|
|
logBtn.style.backgroundColor = "#685434"; // Keep the yellow color
|
|
}, TIMEOUTS.FLASH_LONG);
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function refreshLog(extension, workerId, silent = false) {
|
|
const logContent = document.getElementById("distributed-log-content");
|
|
if (!logContent) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const data = await extension.api.getWorkerLog(workerId, 1000);
|
|
|
|
// Update content
|
|
const shouldAutoScroll = logContent.scrollTop + logContent.clientHeight >= logContent.scrollHeight - 50;
|
|
logContent.textContent = data.content;
|
|
|
|
// Auto-scroll if was at bottom
|
|
if (shouldAutoScroll) {
|
|
logContent.scrollTop = logContent.scrollHeight;
|
|
}
|
|
|
|
// Only show toast if not in silent mode (manual refresh)
|
|
if (!silent) {
|
|
extension.app.extensionManager.toast.add({
|
|
severity: "success",
|
|
summary: "Log Refreshed",
|
|
detail: "Log content updated",
|
|
life: 2000,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
// Only show error toast if not in silent mode
|
|
if (!silent) {
|
|
extension.app.extensionManager.toast.add({
|
|
severity: "error",
|
|
summary: "Refresh Failed",
|
|
detail: error.message,
|
|
life: 3000,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
export function startLogAutoRefresh(extension, workerId) {
|
|
// Stop any existing auto-refresh
|
|
stopLogAutoRefresh(extension);
|
|
|
|
// Refresh every 2 seconds
|
|
extension.logAutoRefreshInterval = setInterval(() => {
|
|
refreshLog(extension, workerId, true); // silent mode
|
|
}, TIMEOUTS.LOG_REFRESH);
|
|
}
|
|
|
|
export function stopLogAutoRefresh(extension) {
|
|
if (extension.logAutoRefreshInterval) {
|
|
clearInterval(extension.logAutoRefreshInterval);
|
|
extension.logAutoRefreshInterval = null;
|
|
}
|
|
}
|
|
|
|
export function toggleWorkerExpanded(extension, workerId) {
|
|
const settingsDiv = document.getElementById(`settings-${workerId}`);
|
|
const gpuDiv = settingsDiv?.closest('[style*="margin-bottom: 12px"]');
|
|
const settingsArrow = gpuDiv?.querySelector(".settings-arrow");
|
|
|
|
if (!settingsDiv) {
|
|
return;
|
|
}
|
|
|
|
if (extension.state.isWorkerExpanded(workerId)) {
|
|
extension.state.setWorkerExpanded(workerId, false);
|
|
settingsDiv.classList.remove("expanded");
|
|
if (settingsArrow) {
|
|
settingsArrow.style.transform = "rotate(0deg)";
|
|
}
|
|
// Animate padding to 0
|
|
settingsDiv.style.padding = "0 12px";
|
|
settingsDiv.style.marginTop = "0";
|
|
settingsDiv.style.marginBottom = "0";
|
|
} else {
|
|
extension.state.setWorkerExpanded(workerId, true);
|
|
settingsDiv.classList.add("expanded");
|
|
if (settingsArrow) {
|
|
settingsArrow.style.transform = "rotate(90deg)";
|
|
}
|
|
// Animate padding to full
|
|
settingsDiv.style.padding = "12px";
|
|
settingsDiv.style.marginTop = "8px";
|
|
settingsDiv.style.marginBottom = "8px";
|
|
}
|
|
}
|