Files

601 lines
23 KiB
TypeScript

import { APP_CONFIG } from "../../app/config.js";
import { EVENTS } from "../../app/events.js";
import { createExecutionAssetBuffer } from "./executionAssetBuffer.js";
import { createJobFinalizeQueue } from "./finalizeQueue.js";
import { createLiveAssetGate } from "./liveAssetGate.js";
import { createTTLCache } from "../../utils/ttlCache.js";
import { findGraphNodeById, getHostRootGraph } from "../../app/graphTraversal.js";
const DEDUPE_TTL_MS = 2000;
const RECENT_FILES_MAX = 2000;
const INDEX_RETRYABLE_CODES = new Set([
"DB_MAINTENANCE",
"TIMEOUT",
"NETWORK_ERROR",
"SERVICE_UNAVAILABLE",
]);
const INDEX_RETRY_MAX_ATTEMPTS = 8;
const INDEX_RETRY_BASE_DELAY_MS = 1200;
const INDEX_RETRY_MAX_DELAY_MS = 15_000;
const EXECUTION_START_TTL_MS = 10 * 60 * 1000;
const EXECUTION_STARTS_MAX = 500;
const ORPHAN_GENERATION_RETRY_MAX_ATTEMPTS = 6;
const ORPHAN_GENERATION_ABSOLUTE_TIMEOUT_MS = 30_000;
const POST_EXECUTION_BUFFER_FLUSH_FAST_MS = 500;
export function createExecutionRuntimeController({
post,
ENDPOINTS,
reportError,
extractOutputFiles,
ensureExecutionRuntime,
emitRuntimeStatus,
refreshGeneratedFeedHosts,
getActiveGridContainer: _getActiveGridContainer,
}: Record<string, any>): Record<string, any> {
const recentFiles = createTTLCache({ ttlMs: DEDUPE_TTL_MS, maxSize: RECENT_FILES_MAX });
const executionStarts = createTTLCache({
ttlMs: EXECUTION_START_TTL_MS,
maxSize: EXECUTION_STARTS_MAX,
});
const executionIdleGraceMs = Math.max(0, Number(APP_CONFIG.EXECUTION_IDLE_GRACE_MS) || 6000);
const orphanGenerationEntries = new Map();
const executionAssetBuffer = createExecutionAssetBuffer();
const liveAssetGate = createLiveAssetGate();
let lastStacksUpdateSignature = "";
let lastStacksUpdateAt = 0;
let postExecutionWorkTimer: any = null;
function isJobTrackingDebugEnabled() {
try {
if (typeof window === "undefined") return false;
if ((window as any).__MJR_DEBUG_JOB_TRACKING__ === true) return true;
return (
String(window.localStorage?.getItem("mjr.debug.jobTracking") || "").trim() === "1"
);
} catch (e) {
console.debug?.(e);
return false;
}
}
function debugJobTracking(stage: any, detail: Record<string, any> = {}) {
if (!isJobTrackingDebugEnabled()) return;
try {
console.debug("[Majoor][job-tracking]", stage, detail);
} catch (e) {
console.debug?.(e);
}
}
function isExecutionGroupingEnabled() {
return !!APP_CONFIG.EXECUTION_GROUPING_ENABLED;
}
function getExecutionRuntimePolicy() {
const groupingEnabled = isExecutionGroupingEnabled();
return {
groupingEnabled,
useExecutionBuffer: groupingEnabled,
// Post-execution directory scans race the watcher and live file indexing,
// then emit scan-complete reloads while the grid is already showing assets.
usePostExecutionScan: false,
allowLiveStackFinalize: groupingEnabled,
};
}
function notifyStacksUpdated(detail: Record<string, any> = {}) {
debugJobTracking("stacks-updated", {
targeted_job_id: String(detail?.targeted_job_id || ""),
stacks: Array.isArray(detail?.stacks)
? detail.stacks.map((item: any) => ({
job_id: String(item?.job_id || ""),
stack_id: String(item?.stack_id || ""),
asset_count: Number(item?.asset_count || 0) || 0,
}))
: [],
});
try {
const targeted = String(detail?.targeted_job_id || "").trim();
if (targeted) liveAssetGate.clearPendingJob(targeted);
const stacks = Array.isArray(detail?.stacks) ? detail.stacks : [];
for (const item of stacks) {
const jobId = String(item?.job_id || "").trim();
if (jobId) liveAssetGate.clearPendingJob(jobId);
}
} catch (e) {
console.debug?.(e);
}
try {
const stacks = Array.isArray(detail?.stacks) ? detail.stacks : [];
const signature = JSON.stringify({
targeted_job_id: String(detail?.targeted_job_id || ""),
stacks: stacks.map((item: any) => ({
job_id: String(item?.job_id || ""),
stack_id: String(item?.stack_id || ""),
asset_count: Number(item?.asset_count || 0) || 0,
})),
});
const now = Date.now();
if (
signature &&
signature === lastStacksUpdateSignature &&
now - lastStacksUpdateAt < 1500
) {
return;
}
lastStacksUpdateSignature = signature;
lastStacksUpdateAt = now;
} catch (e) {
console.debug?.(e);
}
try {
window.dispatchEvent(
new CustomEvent("mjr:stacks-updated", { detail: { ...(detail || {}) } }),
);
} catch (e) {
console.debug?.(e);
}
try {
refreshGeneratedFeedHosts();
} catch (e) {
console.debug?.(e);
}
}
const stackFinalizeQueue = createJobFinalizeQueue({
defaultDelayMs: 900,
async postJob(targetJobId) {
debugJobTracking("auto-stack:start", { job_id: targetJobId });
try {
const res = await post(ENDPOINTS.STACKS_AUTO_STACK, {
mode: "job_id",
job_id: targetJobId,
});
if (!res?.ok) {
liveAssetGate.clearPendingJob(targetJobId);
reportError(
new Error(String(res?.error || "Auto stack failed")),
"executionRuntime.execution_end.auto_stack",
);
return;
}
debugJobTracking("auto-stack:done", {
job_id: targetJobId,
stacks: Array.isArray(res?.data?.stacks) ? res.data.stacks.length : 0,
});
notifyStacksUpdated(res?.data || { targeted_job_id: targetJobId });
} catch (error) {
reportError(error, "executionRuntime.execution_end.auto_stack");
}
},
});
function rememberExecutionStart(promptId: any, timestampMs: any) {
if (!promptId) return;
const ts = Number(timestampMs) || Date.now();
executionStarts.set(String(promptId), ts, { at: ts });
executionStarts.prune();
}
function dedupeFiles(files: any) {
recentFiles.prune();
const fresh = [];
for (const f of files) {
const key = `${f.type || ""}|${f.subfolder || ""}|${f.filename || ""}`.toLowerCase();
if (!recentFiles.has(key)) {
recentFiles.set(key, true);
fresh.push(f);
}
}
return fresh;
}
function indexRetryDelayMs(attempt: any) {
const a = Math.max(1, Number(attempt) || 1);
const exp = Math.min(6, a - 1);
return Math.min(INDEX_RETRY_MAX_DELAY_MS, INDEX_RETRY_BASE_DELAY_MS * 2 ** exp);
}
function clearPostExecutionWorkTimer() {
if (postExecutionWorkTimer) {
clearTimeout(postExecutionWorkTimer);
postExecutionWorkTimer = null;
}
}
function schedulePostExecutionWork(work: any, delayMs = executionIdleGraceMs) {
const waitMs = Math.max(0, Number(delayMs) || 0);
clearPostExecutionWorkTimer();
postExecutionWorkTimer = setTimeout(() => {
postExecutionWorkTimer = null;
try {
work?.();
} catch (error) {
reportError(error, "executionRuntime.post_execution");
}
}, waitMs);
}
function shouldRetryIndexResponse(res: any) {
const code = String(res?.code || "")
.trim()
.toUpperCase();
return INDEX_RETRYABLE_CODES.has(code);
}
function scheduleGenerationIndex(files: any, attempt = 1, meta: Record<string, any> = {}) {
const payloadFiles = Array.isArray(files) ? files : [];
if (!payloadFiles.length) return;
const payload: Record<string, any> = { files: payloadFiles, origin: "generation" };
if (meta?.prompt_id || meta?.promptId) {
payload.prompt_id = String(meta.prompt_id || meta.promptId);
}
debugJobTracking("index:direct", {
prompt_id: String(payload.prompt_id || ""),
attempt,
files: payloadFiles.map((item: any) => String(item?.filename || item?.filepath || "")),
});
post(ENDPOINTS.INDEX_FILES, payload)
.then((res: any) => {
if (res?.ok) return;
if (shouldRetryIndexResponse(res) && attempt < INDEX_RETRY_MAX_ATTEMPTS) {
const delayMs = indexRetryDelayMs(attempt);
setTimeout(
() => scheduleGenerationIndex(payloadFiles, attempt + 1, meta),
delayMs,
);
return;
}
try {
const code = String(res?.code || "INDEX_FAILED");
const msg = String(res?.error || "Indexing generated files failed");
reportError(new Error(`${code}: ${msg}`), "executionRuntime.executed.index");
} catch (e) {
console.debug?.(e);
}
})
.catch((error: any) => reportError(error, "executionRuntime.executed.index"));
}
function resolveGenerationPromptId(meta: Record<string, any> = {}) {
const explicitPromptId = String(meta?.prompt_id || meta?.promptId || "").trim();
if (explicitPromptId) return explicitPromptId;
try {
return String(ensureExecutionRuntime()?.active_prompt_id || "").trim();
} catch (e) {
console.debug?.(e);
return "";
}
}
function orphanGenerationKey(files: any) {
if (!Array.isArray(files) || !files.length) return "";
return files
.map((item: any) => String(item?.filepath || item?.filename || ""))
.filter(Boolean)
.sort()
.join("|");
}
function enqueueOrphanGenerationIndex(files: any, meta = {}, attempt = 1) {
const payloadFiles = Array.isArray(files) ? files : [];
if (!payloadFiles.length) return;
const key = orphanGenerationKey(payloadFiles);
if (!key) {
scheduleGenerationIndex(payloadFiles, 1, meta);
return;
}
const existing = orphanGenerationEntries.get(key);
if (existing?.timer) clearTimeout(existing.timer);
const firstAttemptAt = existing?.firstAttemptAt ?? Date.now();
if (Date.now() - firstAttemptAt >= ORPHAN_GENERATION_ABSOLUTE_TIMEOUT_MS) {
scheduleGenerationIndex(payloadFiles, 1, meta);
return;
}
const nextAttempt = Math.max(1, Number(attempt) || 1);
const delayMs = Math.min(2000, 250 * nextAttempt);
const timer = setTimeout(() => {
orphanGenerationEntries.delete(key);
bufferGenerationIndex(payloadFiles, meta, nextAttempt + 1);
}, delayMs);
orphanGenerationEntries.set(key, {
files: payloadFiles,
meta,
attempt: nextAttempt,
timer,
firstAttemptAt,
});
}
function flushOrphanGenerationEntries(promptId: any) {
const normalizedPromptId = String(promptId || "").trim();
if (!normalizedPromptId || orphanGenerationEntries.size <= 0) return;
for (const [key, entry] of Array.from(orphanGenerationEntries.entries())) {
try {
if (entry?.timer) clearTimeout(entry.timer);
} catch (e) {
console.debug?.(e);
}
orphanGenerationEntries.delete(key);
executionAssetBuffer.add(normalizedPromptId, entry?.files || []);
}
}
function clearOrphanGenerationEntries() {
for (const [key, entry] of Array.from(orphanGenerationEntries.entries())) {
try {
if (entry?.timer) clearTimeout(entry.timer);
} catch (e) {
console.debug?.(e);
}
orphanGenerationEntries.delete(key);
}
}
function bufferGenerationIndex(files: any, meta: Record<string, any> = {}, attempt = 1) {
const promptId = resolveGenerationPromptId(meta);
debugJobTracking("buffer:add", {
prompt_id: promptId,
explicit_prompt_id: String(meta?.prompt_id || meta?.promptId || ""),
attempt,
file_count: Array.isArray(files) ? files.length : 0,
files: Array.isArray(files)
? files.map((item: any) => String(item?.filename || item?.filepath || ""))
: [],
});
if (!promptId) {
if (attempt < ORPHAN_GENERATION_RETRY_MAX_ATTEMPTS) {
enqueueOrphanGenerationIndex(files, meta, attempt);
return;
}
scheduleGenerationIndex(files, 1, meta);
return;
}
executionAssetBuffer.add(promptId, files);
}
function scheduleFinalizeExecutionStacks(jobId: any, delayMs = 900) {
if (!isExecutionGroupingEnabled()) return;
debugJobTracking("finalize:schedule", { job_id: String(jobId || ""), delay_ms: delayMs });
stackFinalizeQueue.schedule(jobId, delayMs);
}
function flushBufferedGenerationIndex(jobId: any) {
const policy = getExecutionRuntimePolicy();
const promptId = String(jobId || "").trim();
if (!promptId) return;
const files = executionAssetBuffer.take(promptId);
debugJobTracking("buffer:flush", {
prompt_id: promptId,
file_count: files.length,
files: files.map((item: any) => String(item?.filename || item?.filepath || "")),
});
if (policy.allowLiveStackFinalize) {
liveAssetGate.markPendingJob(promptId, { files, expectedCount: files.length });
}
if (!files.length) {
if (policy.allowLiveStackFinalize) {
scheduleFinalizeExecutionStacks(promptId);
}
return;
}
post(ENDPOINTS.INDEX_FILES, {
files,
origin: "generation",
prompt_id: promptId,
})
.then((res: any) => {
if (!res?.ok) {
if (policy.allowLiveStackFinalize) {
liveAssetGate.clearPendingJob(promptId);
}
reportError(
new Error(String(res?.error || "Indexing generated files failed")),
"executionRuntime.executed.index",
);
return;
}
debugJobTracking("index:flush:done", {
prompt_id: promptId,
file_count: files.length,
});
if (policy.allowLiveStackFinalize) {
scheduleFinalizeExecutionStacks(promptId, 1800);
}
})
.catch((error: any) => {
if (policy.allowLiveStackFinalize) {
liveAssetGate.clearPendingJob(promptId);
}
reportError(error, "executionRuntime.executed.index");
});
}
function hasBufferedGenerationFiles(jobId: any) {
const promptId = String(jobId || "").trim();
if (!promptId) return false;
try {
const bucket = executionAssetBuffer?._jobs?.get?.(promptId);
return !!bucket && typeof bucket.size === "number" && bucket.size > 0;
} catch (e) {
console.debug?.(e);
return false;
}
}
function prepareLiveAssetEvent(detail: any) {
if (!isExecutionGroupingEnabled()) {
return { detail, jobId: "", defer: false };
}
try {
return liveAssetGate.prepare(detail);
} catch (e) {
console.debug?.(e);
return { detail, jobId: "", defer: false };
}
}
function isRenderableLiveAsset(detail: any) {
const kind = String(detail?.kind || "").trim();
const filename = String(detail?.filename || "").trim();
const filepath = String(detail?.filepath || "").trim();
return !!kind && (!!filename || !!filepath);
}
function handleExecutedEvent(event: any, { appRef }: { appRef: any }) {
const policy = getExecutionRuntimePolicy();
if (!policy.useExecutionBuffer) return;
const outputFiles = dedupeFiles(extractOutputFiles(event?.detail?.output));
if (!outputFiles.length) return;
const promptId = event?.detail?.prompt_id || event?.detail?.promptId;
const sourceNodeId = String(
event?.detail?.node || event?.detail?.display_node || "",
).trim();
const startTs = promptId ? executionStarts.get(String(promptId)) : null;
const genTimeMs = startTs ? Math.max(0, Date.now() - (startTs as number)) : 0;
let sourceNodeType = "";
try {
if (sourceNodeId && typeof appRef !== "undefined") {
const graphNode = findGraphNodeById(getHostRootGraph(appRef), sourceNodeId);
if (graphNode) {
sourceNodeType = String(graphNode.type || graphNode.comfyClass || "").trim();
}
}
} catch (e) {
console.debug?.(e);
}
const files = outputFiles.map((f) => ({
...f,
...(genTimeMs > 0 ? { generation_time_ms: genTimeMs } : {}),
...(sourceNodeId ? { source_node_id: sourceNodeId } : {}),
...(sourceNodeType ? { source_node_type: sourceNodeType } : {}),
}));
debugJobTracking("executed:event", {
prompt_id: String(promptId || ""),
source_node_id: sourceNodeId,
source_node_type: sourceNodeType,
file_count: files.length,
files: files.map((item: any) => String(item?.filename || item?.filepath || "")),
});
try {
window.dispatchEvent(
new CustomEvent(EVENTS.NEW_GENERATION_OUTPUT, {
detail: {
files,
prompt_id: String(promptId || ""),
promptId: String(promptId || ""),
},
}),
);
} catch (e) {
console.debug?.(e);
}
const immediateFiles = files.length > 0 ? [files[0]] : [];
const bufferedFiles = files.length > 1 ? files.slice(1) : [];
// Prioritize the first renderable asset from each executed payload so the
// placeholder is replaced by a real card almost immediately, while the
// remaining outputs continue through the grouped post-execution flow.
if (immediateFiles.length > 0) {
scheduleGenerationIndex(immediateFiles, 1, { prompt_id: promptId });
}
if (bufferedFiles.length > 0) {
bufferGenerationIndex(bufferedFiles, { prompt_id: promptId });
}
}
function handleExecutionStart(event: any, { setCurrentJobId }: { setCurrentJobId: any }) {
const promptId = event?.detail?.prompt_id || event?.detail?.promptId;
const ts = event?.detail?.timestamp;
debugJobTracking("execution:start", {
prompt_id: String(promptId || ""),
timestamp: Number(ts) || null,
});
rememberExecutionStart(promptId, ts);
clearPostExecutionWorkTimer();
setCurrentJobId?.(promptId || null);
emitRuntimeStatus({
active_prompt_id: promptId || null,
cached_nodes: [],
progress_node: null,
progress_value: null,
progress_max: null,
});
if (promptId) {
flushOrphanGenerationEntries(promptId);
}
}
function handleExecutionEnd(event: any, { setCurrentJobId }: { setCurrentJobId: any }) {
const policy = getExecutionRuntimePolicy();
const promptId = event?.detail?.prompt_id || event?.detail?.promptId;
debugJobTracking("execution:end", {
type: String(event?.type || ""),
prompt_id: String(promptId || ""),
});
if (promptId) {
executionStarts.delete(String(promptId));
}
executionStarts.prune();
setCurrentJobId?.(null);
emitRuntimeStatus({
active_prompt_id: null,
cached_nodes: [],
progress_node: null,
progress_value: null,
progress_max: null,
});
if (promptId && String(event?.type || "") !== "execution_success") {
debugJobTracking("buffer:clear", {
prompt_id: String(promptId || ""),
reason: String(event?.type || ""),
});
clearPostExecutionWorkTimer();
executionAssetBuffer.clear(promptId);
if (policy.allowLiveStackFinalize) {
liveAssetGate.clearPendingJob(promptId);
}
clearOrphanGenerationEntries();
}
if (String(event?.type || "") === "execution_success") {
const flushDelayMs = hasBufferedGenerationFiles(promptId)
? Math.min(executionIdleGraceMs, POST_EXECUTION_BUFFER_FLUSH_FAST_MS)
: executionIdleGraceMs;
schedulePostExecutionWork(() => {
if (promptId) {
flushOrphanGenerationEntries(promptId);
}
flushBufferedGenerationIndex(promptId);
}, flushDelayMs);
}
}
function handleNodeOutputsUpdated(nodeOutputs: any) {
if (!isExecutionGroupingEnabled()) return;
const files = [];
for (const [, outputs] of nodeOutputs instanceof Map ? nodeOutputs.entries() : []) {
const found = extractOutputFiles(outputs);
if (found?.length) {
for (const f of found) files.push(f);
}
}
if (!files.length || !isExecutionGroupingEnabled()) return;
debugJobTracking("node-outputs:updated", {
prompt_id: resolveGenerationPromptId(),
file_count: files.length,
});
bufferGenerationIndex(files, { prompt_id: resolveGenerationPromptId() });
}
return {
notifyStacksUpdated,
prepareLiveAssetEvent,
isRenderableLiveAsset,
handleExecutedEvent,
handleExecutionStart,
handleExecutionEnd,
handleNodeOutputsUpdated,
};
}