From e1022529f57987bc2e5e81ecf07b02e886ecd37e Mon Sep 17 00:00:00 2001 From: vindennt Date: Mon, 15 Jul 2024 17:54:29 -0700 Subject: [PATCH 01/19] adds chatbot caller and simple ui --- js/chatbot/chatbot.js | 31 +++++++++ js/chatbot/ext.js | 156 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 js/chatbot/chatbot.js create mode 100644 js/chatbot/ext.js diff --git a/js/chatbot/chatbot.js b/js/chatbot/chatbot.js new file mode 100644 index 0000000..dffafb7 --- /dev/null +++ b/js/chatbot/chatbot.js @@ -0,0 +1,31 @@ +// import { endpoint } from "../constants"; + +const endpoint = 'http://localhost:4000/'; // Switch to live as needed + +export async function getBotResponse(message) { + // TODO: add auth + console.log("calling the gpt bot woith message", message); + try { + const response = await fetch(`${endpoint}chat/send-message`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ message: message, origin: "ComfyUI Chat" }) + }); + + if (!response.ok) { + throw new Error('Network response was not ok ' + response.statusText); + } + + console.log('response', response); + + const responseData = await response.json(); + + return responseData; + } catch (error) { + console.error('There was a problem with the fetch operation:', error); + throw error; + } +} + diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js new file mode 100644 index 0000000..920e832 --- /dev/null +++ b/js/chatbot/ext.js @@ -0,0 +1,156 @@ +// import { api, app } from './comfy/comfy.js' +import {app} from "../comfy/comfy.js" +import { getBotResponse } from './chatbot.js'; + +const CHAT_BUTTON_ID = 'chat-button'; +const CHAT_BOX_ID = 'chat-box'; +const CHAT_INPUT_ID = 'chat-input'; +const CHAT_MESSAGES_ID = 'chat-messages'; + +const createChatButton = () => { + const chatButton = document.createElement('div'); + chatButton.id = CHAT_BUTTON_ID; + chatButton.innerHTML = `
`; + chatButton.style.position = 'fixed'; + chatButton.style.bottom = '0'; + chatButton.style.right = '0'; + chatButton.style.backgroundColor = 'green'; + chatButton.style.color = 'white'; + chatButton.style.borderRadius = '50%'; + chatButton.style.width = '100px'; + chatButton.style.height = '100px'; + chatButton.style.zIndex = 9999; + chatButton.style.display = 'flex'; + chatButton.style.alignItems = 'center'; + chatButton.style.justifyContent = 'center'; + chatButton.style.cursor = 'pointer'; + chatButton.innerText = 'Chat'; + return chatButton; +} + +const createChatBox = () => { + const chatBox = document.createElement('div'); + chatBox.id = CHAT_BOX_ID; + chatBox.style.position = 'fixed'; + chatBox.style.bottom = '100px'; + chatBox.style.right = '0'; + chatBox.style.backgroundColor = 'white'; + chatBox.style.width = '300px'; + chatBox.style.height = '500px'; + chatBox.style.zIndex = 9999; + chatBox.style.display = 'none'; // Initially hidden + + // Messages section + const chatMessages = document.createElement('div'); + chatMessages.id = CHAT_MESSAGES_ID; + chatMessages.style.height = '80%'; + chatMessages.style.overflowY = 'auto'; + chatMessages.style.padding = '10px'; + chatMessages.style.color = 'black'; + chatBox.appendChild(chatMessages); + + // Input section + const chatInputContainer = document.createElement('div'); + chatInputContainer.style.position = 'absolute'; + chatInputContainer.style.bottom = '0'; + chatInputContainer.style.width = '100%'; + chatInputContainer.style.padding = '10px'; + chatInputContainer.style.boxSizing = 'border-box'; + chatInputContainer.style.backgroundColor = '#f1f1f1'; + + const chatInput = document.createElement('input'); + chatInput.id = CHAT_INPUT_ID; + chatInput.type = 'text'; + chatInput.style.width = 'calc(100% - 60px)'; // Leave space for the send button + chatInput.style.padding = '10px'; + chatInput.style.boxSizing = 'border-box'; + chatInput.style.border = '1px solid #ccc'; + chatInput.style.borderRadius = '4px'; + + const chatSendButton = document.createElement('button'); + chatSendButton.innerText = 'Send'; + chatSendButton.style.width = '50px'; + chatSendButton.style.marginLeft = '10px'; + chatSendButton.style.padding = '10px'; + chatSendButton.style.border = 'none'; + chatSendButton.style.backgroundColor = 'green'; + chatSendButton.style.color = 'white'; + chatSendButton.style.borderRadius = '4px'; + chatSendButton.style.cursor = 'pointer'; + + chatSendButton.addEventListener('click', sendMessage); + + chatInputContainer.appendChild(chatInput); + chatInputContainer.appendChild(chatSendButton); + chatBox.appendChild(chatInputContainer); + + return chatBox; +} + +const sendMessage = () => { + const chatInput = document.getElementById(CHAT_INPUT_ID); + const chatMessages = document.getElementById(CHAT_MESSAGES_ID); + const message = chatInput.value.trim(); + + if (message) { + const messageElement = document.createElement('div'); + messageElement.innerText = message; + messageElement.style.marginBottom = '10px'; + messageElement.style.padding = '10px'; + messageElement.style.backgroundColor = '#e1ffc7'; + messageElement.style.borderRadius = '4px'; + + chatMessages.appendChild(messageElement); + chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + chatInput.value = ''; + // // get bot response + getBotResponse(message).then((response) => { + console.log('ext got rresponse', response); + const botMessageElement = document.createElement('div'); + botMessageElement.innerText = response.message; + botMessageElement.style.marginBottom = '10px'; + botMessageElement.style.padding = '10px'; + botMessageElement.style.backgroundColor = '#c7e1ff'; + botMessageElement.style.borderRadius = '4px'; + + chatMessages.appendChild(botMessageElement); + chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + }).catch((error) => { + console.error(error); + }); + } +} + +const toggleChatBox = () => { + const chatBox = document.getElementById(CHAT_BOX_ID); + if (chatBox.style.display === 'none') { + chatBox.style.display = 'block'; + } else { + chatBox.style.display = 'none'; + } +} + +const registerChat = () => { + + // With js, append an bottom that opens up a chatbox in the bottom right hand corner + const chatButton = createChatButton(); + const chatBox = createChatBox() + + document.body.appendChild(chatBox); + document.body.appendChild(chatButton); + chatButton.addEventListener('click', toggleChatBox); +} + + +/** @typedef {import('../../../web/types/comfy.js').ComfyExtension} ComfyExtension*/ +/** @type {ComfyExtension} */ +const ext = { + name: 'nathannlu.ComfyUI-Chat', + + // ComfyUI extension init + init() { + registerChat() + }, +} + +app.registerExtension(ext) From cdfc64d65ad3d7d9dea54f9abaa17e080a2c2746 Mon Sep 17 00:00:00 2001 From: vindennt Date: Tue, 16 Jul 2024 14:09:18 -0700 Subject: [PATCH 02/19] adds test button for node isnertion and ran code formatter --- js/chatbot/ext.js | 215 ++++++++++++++++++++++++++-------------------- 1 file changed, 121 insertions(+), 94 deletions(-) diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index 920e832..7d9af17 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -1,156 +1,183 @@ // import { api, app } from './comfy/comfy.js' -import {app} from "../comfy/comfy.js" -import { getBotResponse } from './chatbot.js'; +import { app } from "../comfy/comfy.js"; +import { getBotResponse } from "./chatbot.js"; -const CHAT_BUTTON_ID = 'chat-button'; -const CHAT_BOX_ID = 'chat-box'; -const CHAT_INPUT_ID = 'chat-input'; -const CHAT_MESSAGES_ID = 'chat-messages'; +const CHAT_BUTTON_ID = "chat-button"; +const CHAT_BOX_ID = "chat-box"; +const CHAT_INPUT_ID = "chat-input"; +const CHAT_MESSAGES_ID = "chat-messages"; const createChatButton = () => { - const chatButton = document.createElement('div'); + const chatButton = document.createElement("div"); chatButton.id = CHAT_BUTTON_ID; chatButton.innerHTML = `
`; - chatButton.style.position = 'fixed'; - chatButton.style.bottom = '0'; - chatButton.style.right = '0'; - chatButton.style.backgroundColor = 'green'; - chatButton.style.color = 'white'; - chatButton.style.borderRadius = '50%'; - chatButton.style.width = '100px'; - chatButton.style.height = '100px'; + chatButton.style.position = "fixed"; + chatButton.style.bottom = "0"; + chatButton.style.right = "0"; + chatButton.style.backgroundColor = "green"; + chatButton.style.color = "white"; + chatButton.style.borderRadius = "50%"; + chatButton.style.width = "100px"; + chatButton.style.height = "100px"; chatButton.style.zIndex = 9999; - chatButton.style.display = 'flex'; - chatButton.style.alignItems = 'center'; - chatButton.style.justifyContent = 'center'; - chatButton.style.cursor = 'pointer'; - chatButton.innerText = 'Chat'; + chatButton.style.display = "flex"; + chatButton.style.alignItems = "center"; + chatButton.style.justifyContent = "center"; + chatButton.style.cursor = "pointer"; + chatButton.innerText = "Chat"; return chatButton; -} +}; const createChatBox = () => { - const chatBox = document.createElement('div'); + const chatBox = document.createElement("div"); chatBox.id = CHAT_BOX_ID; - chatBox.style.position = 'fixed'; - chatBox.style.bottom = '100px'; - chatBox.style.right = '0'; - chatBox.style.backgroundColor = 'white'; - chatBox.style.width = '300px'; - chatBox.style.height = '500px'; + chatBox.style.position = "fixed"; + chatBox.style.bottom = "100px"; + chatBox.style.right = "0"; + chatBox.style.backgroundColor = "white"; + chatBox.style.width = "300px"; + chatBox.style.height = "500px"; chatBox.style.zIndex = 9999; - chatBox.style.display = 'none'; // Initially hidden + chatBox.style.display = "none"; // Initially hidden // Messages section - const chatMessages = document.createElement('div'); + const chatMessages = document.createElement("div"); chatMessages.id = CHAT_MESSAGES_ID; - chatMessages.style.height = '80%'; - chatMessages.style.overflowY = 'auto'; - chatMessages.style.padding = '10px'; - chatMessages.style.color = 'black'; + chatMessages.style.height = "80%"; + chatMessages.style.overflowY = "auto"; + chatMessages.style.padding = "10px"; + chatMessages.style.color = "black"; chatBox.appendChild(chatMessages); // Input section - const chatInputContainer = document.createElement('div'); - chatInputContainer.style.position = 'absolute'; - chatInputContainer.style.bottom = '0'; - chatInputContainer.style.width = '100%'; - chatInputContainer.style.padding = '10px'; - chatInputContainer.style.boxSizing = 'border-box'; - chatInputContainer.style.backgroundColor = '#f1f1f1'; + const chatInputContainer = document.createElement("div"); + chatInputContainer.style.position = "absolute"; + chatInputContainer.style.bottom = "0"; + chatInputContainer.style.width = "100%"; + chatInputContainer.style.padding = "10px"; + chatInputContainer.style.boxSizing = "border-box"; + chatInputContainer.style.backgroundColor = "#f1f1f1"; - const chatInput = document.createElement('input'); + const chatInput = document.createElement("input"); chatInput.id = CHAT_INPUT_ID; - chatInput.type = 'text'; - chatInput.style.width = 'calc(100% - 60px)'; // Leave space for the send button - chatInput.style.padding = '10px'; - chatInput.style.boxSizing = 'border-box'; - chatInput.style.border = '1px solid #ccc'; - chatInput.style.borderRadius = '4px'; + chatInput.type = "text"; + chatInput.style.width = "calc(100% - 60px)"; // Leave space for the send button + chatInput.style.padding = "10px"; + chatInput.style.boxSizing = "border-box"; + chatInput.style.border = "1px solid #ccc"; + chatInput.style.borderRadius = "4px"; - const chatSendButton = document.createElement('button'); - chatSendButton.innerText = 'Send'; - chatSendButton.style.width = '50px'; - chatSendButton.style.marginLeft = '10px'; - chatSendButton.style.padding = '10px'; - chatSendButton.style.border = 'none'; - chatSendButton.style.backgroundColor = 'green'; - chatSendButton.style.color = 'white'; - chatSendButton.style.borderRadius = '4px'; - chatSendButton.style.cursor = 'pointer'; + const chatSendButton = document.createElement("button"); + chatSendButton.innerText = "Send"; + chatSendButton.style.width = "50px"; + chatSendButton.style.marginLeft = "10px"; + chatSendButton.style.padding = "10px"; + chatSendButton.style.border = "none"; + chatSendButton.style.backgroundColor = "green"; + chatSendButton.style.color = "white"; + chatSendButton.style.borderRadius = "4px"; + chatSendButton.style.cursor = "pointer"; - chatSendButton.addEventListener('click', sendMessage); + chatSendButton.addEventListener("click", sendMessage); chatInputContainer.appendChild(chatInput); chatInputContainer.appendChild(chatSendButton); chatBox.appendChild(chatInputContainer); - return chatBox; -} + // New button section + const newButton = document.createElement("button"); + newButton.innerText = "New Button"; + newButton.style.position = "absolute"; + newButton.style.top = "10px"; + newButton.style.right = "10px"; + newButton.style.padding = "10px"; + newButton.style.backgroundColor = "blue"; + newButton.style.color = "white"; + newButton.style.border = "none"; + newButton.style.borderRadius = "4px"; + newButton.style.cursor = "pointer"; + newButton.addEventListener("click", () => { + console.log("sets graph to null"); + console.log(app.graph) + + // TODO: insert a parsed node + const newNode = LiteGraph.createNode("LoadImage"); + app.graph.add(newNode); + + app.graph.change() + console.log(app.graph) + // console.log('New button clicked!'); + }); + + chatBox.appendChild(newButton); + + return chatBox; +}; const sendMessage = () => { const chatInput = document.getElementById(CHAT_INPUT_ID); const chatMessages = document.getElementById(CHAT_MESSAGES_ID); const message = chatInput.value.trim(); if (message) { - const messageElement = document.createElement('div'); + const messageElement = document.createElement("div"); messageElement.innerText = message; - messageElement.style.marginBottom = '10px'; - messageElement.style.padding = '10px'; - messageElement.style.backgroundColor = '#e1ffc7'; - messageElement.style.borderRadius = '4px'; + messageElement.style.marginBottom = "10px"; + messageElement.style.padding = "10px"; + messageElement.style.backgroundColor = "#e1ffc7"; + messageElement.style.borderRadius = "4px"; chatMessages.appendChild(messageElement); chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom - chatInput.value = ''; + chatInput.value = ""; // // get bot response - getBotResponse(message).then((response) => { - console.log('ext got rresponse', response); - const botMessageElement = document.createElement('div'); - botMessageElement.innerText = response.message; - botMessageElement.style.marginBottom = '10px'; - botMessageElement.style.padding = '10px'; - botMessageElement.style.backgroundColor = '#c7e1ff'; - botMessageElement.style.borderRadius = '4px'; + getBotResponse(message) + .then((response) => { + console.log("ext got rresponse", response); + const botMessageElement = document.createElement("div"); + botMessageElement.innerText = response.message; + botMessageElement.style.marginBottom = "10px"; + botMessageElement.style.padding = "10px"; + botMessageElement.style.backgroundColor = "#c7e1ff"; + botMessageElement.style.borderRadius = "4px"; - chatMessages.appendChild(botMessageElement); - chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom - }).catch((error) => { - console.error(error); - }); + chatMessages.appendChild(botMessageElement); + chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + }) + .catch((error) => { + console.error(error); + }); } -} +}; const toggleChatBox = () => { const chatBox = document.getElementById(CHAT_BOX_ID); - if (chatBox.style.display === 'none') { - chatBox.style.display = 'block'; + if (chatBox.style.display === "none") { + chatBox.style.display = "block"; } else { - chatBox.style.display = 'none'; + chatBox.style.display = "none"; } -} +}; const registerChat = () => { - // With js, append an bottom that opens up a chatbox in the bottom right hand corner const chatButton = createChatButton(); - const chatBox = createChatBox() + const chatBox = createChatBox(); document.body.appendChild(chatBox); document.body.appendChild(chatButton); - chatButton.addEventListener('click', toggleChatBox); -} - + chatButton.addEventListener("click", toggleChatBox); +}; /** @typedef {import('../../../web/types/comfy.js').ComfyExtension} ComfyExtension*/ /** @type {ComfyExtension} */ const ext = { - name: 'nathannlu.ComfyUI-Chat', + name: "nathannlu.ComfyUI-Chat", // ComfyUI extension init init() { - registerChat() + registerChat(); }, -} +}; -app.registerExtension(ext) +app.registerExtension(ext); From 2aef122187a891f101e3d033bb7f99d4fe33d6e4 Mon Sep 17 00:00:00 2001 From: vindennt Date: Wed, 17 Jul 2024 10:17:15 -0700 Subject: [PATCH 03/19] adds working chatgpt input loader --- js/chatbot/ext.js | 30 +++- js/chatbot/workflow.js | 333 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 355 insertions(+), 8 deletions(-) create mode 100644 js/chatbot/workflow.js diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index 7d9af17..72a0780 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -1,6 +1,7 @@ // import { api, app } from './comfy/comfy.js' import { app } from "../comfy/comfy.js"; import { getBotResponse } from "./chatbot.js"; +import { runWorkflowLoader } from "./workflow.js"; const CHAT_BUTTON_ID = "chat-button"; const CHAT_BOX_ID = "chat-box"; @@ -86,7 +87,7 @@ const createChatBox = () => { // New button section const newButton = document.createElement("button"); - newButton.innerText = "New Button"; + newButton.innerText = "Run test fxn"; newButton.style.position = "absolute"; newButton.style.top = "10px"; newButton.style.right = "10px"; @@ -98,19 +99,32 @@ const createChatBox = () => { newButton.style.cursor = "pointer"; newButton.addEventListener("click", () => { - console.log("sets graph to null"); - console.log(app.graph) + runWorkflowLoader(); + }); - // TODO: insert a parsed node - const newNode = LiteGraph.createNode("LoadImage"); - app.graph.add(newNode); + // New button section + const otherButton = document.createElement("button"); + otherButton.innerText = "Log graph"; + otherButton.style.position = "absolute"; + otherButton.style.top = "100px"; + otherButton.style.right = "10px"; + otherButton.style.padding = "10px"; + otherButton.style.backgroundColor = "blue"; + otherButton.style.color = "white"; + otherButton.style.border = "none"; + otherButton.style.borderRadius = "4px"; + otherButton.style.cursor = "pointer"; - app.graph.change() - console.log(app.graph) + otherButton.addEventListener("click", () => { + // const newNode = LiteGraph.createNode("LoadImage"); + // app.graph.add(newNode); + console.log(app.graph); + app.graph.change(); // console.log('New button clicked!'); }); chatBox.appendChild(newButton); + chatBox.appendChild(otherButton); return chatBox; }; diff --git a/js/chatbot/workflow.js b/js/chatbot/workflow.js new file mode 100644 index 0000000..b0a7462 --- /dev/null +++ b/js/chatbot/workflow.js @@ -0,0 +1,333 @@ +import { app } from "../comfy/comfy.js"; + +// const defaultGraph = { +// last_node_id: 9, +// last_link_id: 9, +// nodes: [ +// { +// id: 7, +// type: "CLIPTextEncode", +// pos: [413, 389], +// size: { 0: 425.27801513671875, 1: 180.6060791015625 }, +// flags: {}, +// order: 3, +// mode: 0, +// inputs: [{ name: "clip", type: "CLIP", link: 5 }], +// outputs: [ +// { +// name: "CONDITIONING", +// type: "CONDITIONING", +// links: [6], +// slot_index: 0, +// }, +// ], +// properties: {}, +// widgets_values: ["text, watermark"], +// }, +// { +// id: 6, +// type: "CLIPTextEncode", +// pos: [415, 186], +// size: { 0: 422.84503173828125, 1: 164.31304931640625 }, +// flags: {}, +// order: 2, +// mode: 0, +// inputs: [{ name: "clip", type: "CLIP", link: 3 }], +// outputs: [ +// { +// name: "CONDITIONING", +// type: "CONDITIONING", +// links: [4], +// slot_index: 0, +// }, +// ], +// properties: {}, +// widgets_values: [ +// "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", +// ], +// }, +// { +// id: 5, +// type: "EmptyLatentImage", +// pos: [473, 609], +// size: { 0: 315, 1: 106 }, +// flags: {}, +// order: 1, +// mode: 0, +// outputs: [{ name: "LATENT", type: "LATENT", links: [2], slot_index: 0 }], +// properties: {}, +// widgets_values: [512, 512, 1], +// }, +// { +// id: 3, +// type: "KSampler", +// pos: [863, 186], +// size: { 0: 315, 1: 262 }, +// flags: {}, +// order: 4, +// mode: 0, +// inputs: [ +// { name: "model", type: "MODEL", link: 1 }, +// { name: "positive", type: "CONDITIONING", link: 4 }, +// { name: "negative", type: "CONDITIONING", link: 6 }, +// { name: "latent_image", type: "LATENT", link: 2 }, +// ], +// outputs: [{ name: "LATENT", type: "LATENT", links: [7], slot_index: 0 }], +// properties: {}, +// widgets_values: [156680208700286, true, 20, 8, "euler", "normal", 1], +// }, +// { +// id: 8, +// type: "VAEDecode", +// pos: [1209, 188], +// size: { 0: 210, 1: 46 }, +// flags: {}, +// order: 5, +// mode: 0, +// inputs: [ +// { name: "samples", type: "LATENT", link: 7 }, +// { name: "vae", type: "VAE", link: 8 }, +// ], +// outputs: [{ name: "IMAGE", type: "IMAGE", links: [9], slot_index: 0 }], +// properties: {}, +// }, +// { +// id: 9, +// type: "SaveImage", +// pos: [1451, 189], +// size: { 0: 210, 1: 26 }, +// flags: {}, +// order: 6, +// mode: 0, +// inputs: [{ name: "images", type: "IMAGE", link: 9 }], +// properties: {}, +// }, +// { +// id: 4, +// type: "CheckpointLoaderSimple", +// pos: [26, 474], +// size: { 0: 315, 1: 98 }, +// flags: {}, +// order: 0, +// mode: 0, +// outputs: [ +// { name: "MODEL", type: "MODEL", links: [1], slot_index: 0 }, +// { name: "CLIP", type: "CLIP", links: [3, 5], slot_index: 1 }, +// { name: "VAE", type: "VAE", links: [8], slot_index: 2 }, +// ], +// properties: {}, +// widgets_values: ["v1-5-pruned-emaonly.ckpt"], +// }, +// ], +// links: [ +// [1, 4, 0, 3, 0, "MODEL"], +// [2, 5, 0, 3, 3, "LATENT"], +// [3, 4, 1, 6, 0, "CLIP"], +// [4, 6, 0, 3, 1, "CONDITIONING"], +// [5, 4, 1, 7, 0, "CLIP"], +// [6, 7, 0, 3, 2, "CONDITIONING"], +// [7, 3, 0, 8, 0, "LATENT"], +// [8, 4, 2, 8, 1, "VAE"], +// [9, 8, 0, 9, 0, "IMAGE"], +// ], +// groups: [], +// config: {}, +// extra: {}, +// version: 0.4, +// }; + +const input = { + nodes: [ + { id: 1, class_type: "CLIPTextEncode", pos: [100, 100] }, + { id: 2, class_type: "KSampler", pos: [300, 100] }, + { id: 3, class_type: "VAEDecode", pos: [500, 100] }, + { id: 4, class_type: "SaveImage", pos: [700, 100] }, + ], + edges: [ + { from: 1, to: 2, input: 0, output: 0 }, + { from: 2, to: 3, input: 0, output: 0 }, + { from: 3, to: 4, input: 0, output: 0 }, + ], +}; + +function getInputType(name) { + const nameTypeMap = { + model: "MODEL", + positive: "CONDITIONING", + negative: "CONDITIONING", + latent_image: "LATENT", + images: "IMAGE", + }; + + return nameTypeMap[name] || "XXX UNKNOWN INPUT TYPE XXX"; +} + +function getOutputIndex(node, inputName) { + return node.outputs.findIndex((output) => output.name === inputName); +} + +function getInputIndex(node, inputName) { + return node.inputs.findIndex((input) => input.name === inputName); +} + +// TODO: WIP +// translates chatgpt input to immediately loadable graph format +function transformInputToGraph(input) { + const baseGraph = { + last_node_id: input.nodes.length, + last_link_id: input.edges.length, + nodes: [], + links: [], + groups: [], + config: {}, + extra: {}, + version: 0.4, + }; + + console.log("baseGraph"); + console.log(baseGraph); + + const xStep = 300; // step for node positions + const nodesById = {}; // pointer of nodes by id for easier/faster search + + let x = 100; // starting x post + // Create nodes with positions in order + input.nodes.forEach((node, index) => { + const newNode = { + id: node.id, + type: node.class_type, + pos: [x, 100], + size: [200, 50], + flags: {}, + order: index, + mode: 0, + inputs: [], + outputs: [], + properties: {}, + widgets_values: [], // TODO: look into how widgets work + }; + + baseGraph.nodes.push(newNode); + nodesById[node.id] = newNode; + x += xStep; + }); + + console.log("baseGraph after nodes insertion"); + console.log(baseGraph); + + // Insert edges + input.edges.forEach((edgeData, index) => { + const fromNode = nodesById[edgeData.from]; + const toNode = nodesById[edgeData.to]; + + // Adding output to fromNode + let outputIndex = getOutputIndex(fromNode, edgeData.input); + + // add inputs + let inputIndex = getInputIndex(toNode, edgeData.input); + + // if input doesnt already exist + if (inputIndex < 0) { + toNode.inputs.push({ + name: edgeData.input, + type: getInputType(edgeData.input), + link: index + 1, + }); + } + + console.log("toNode"); + console.log(toNode); + + // add outputs by inferring the inputs + // if output doesnt already exist + if (outputIndex < 0) { + fromNode.outputs.push({ + name: edgeData.input, + // type: edgeData.input.toUpperCase(), + type: getInputType(edgeData.output), + links: [], + slot_index: fromNode.outputs.length, + }); + } + + outputIndex = fromNode.outputs.length - 1; + inputIndex = fromNode.inputs.length - 1; + + console.log("fromNode"); + console.log(fromNode); + + // add links + const link = [ + index + 1, // Link id + fromNode.id, // Origin node id + getOutputIndex(fromNode, edgeData.input), // origin index + toNode.id, // Destination node id + getInputIndex(toNode, edgeData.input), // destination index + getInputType(edgeData.input), + ]; + + console.log("link"); + console.log(link); + + fromNode.outputs[outputIndex].links.push(link[0]); + baseGraph.links.push(link); + }); + + console.log("baseGraph after edges insertion"); + console.log(baseGraph); + + return baseGraph; +} + +// const node = LiteGraph.createNode("LoadImage"); +// graph.add(node); + +// input: chat gpt's generated workflow +function addNodesAndEdges(graph, input) { + const translatedData = transformInputToGraph(input); + console.log("Translated Graph Data: "); + console.log(translatedData); + app.loadGraphData(translatedData, true); + + + // OUTDATED MANUAL INSERTION METHOD + // const n = input.nodes; + // const missingNodeTypes = []; + // const nodesById = {}; + + // TODO: register nodes. e.g. LiteGraph.registered_node_types["LoadImage"] = LoadImageNode; + + // code block from loadGraphData to find unregistered nodes + // if (!(n.type in LiteGraph.registered_node_types)) { + // missingNodeTypes.push(n.type); + // // n.type = sanitizeNodeName(n.type); + // } + + // insert the nodes one by one. This code is working + // n.forEach((nodeData) => { + // const node = LiteGraph.createNode(nodeData.class_type); + // node.id = nodeData.id; + // node.pos = nodeData.pos || [100, 100]; // Default position. TODO: add increment for position + // graph.add(node); + // // nodesById[nodeData.id] = node; + // }); + + // TODO: add input edges + + // TODO: add output edges + + // TODO: without manually adding the edges, try to get input data into defaultGraph's format + // app.loadGraphData(defaultGraph, true) +} + +export function runWorkflowLoader() { + console.log("Initial graph"); + console.log(app.graph); + + // app.graph.clear(); // clear graph + addNodesAndEdges(app.graph, input); + + app.graph.change(); + console.log("Graph after change"); + console.log(app.graph); +} From f06fa60dc59f9c84c454b6cd7207008c5f209572 Mon Sep 17 00:00:00 2001 From: vindennt Date: Thu, 18 Jul 2024 19:24:52 -0700 Subject: [PATCH 04/19] fixes broken output type naming and a running sample input workflow --- js/chatbot/workflow.js | 185 ++++++++--------------------------------- 1 file changed, 35 insertions(+), 150 deletions(-) diff --git a/js/chatbot/workflow.js b/js/chatbot/workflow.js index b0a7462..cbf0f56 100644 --- a/js/chatbot/workflow.js +++ b/js/chatbot/workflow.js @@ -1,152 +1,25 @@ import { app } from "../comfy/comfy.js"; -// const defaultGraph = { -// last_node_id: 9, -// last_link_id: 9, -// nodes: [ -// { -// id: 7, -// type: "CLIPTextEncode", -// pos: [413, 389], -// size: { 0: 425.27801513671875, 1: 180.6060791015625 }, -// flags: {}, -// order: 3, -// mode: 0, -// inputs: [{ name: "clip", type: "CLIP", link: 5 }], -// outputs: [ -// { -// name: "CONDITIONING", -// type: "CONDITIONING", -// links: [6], -// slot_index: 0, -// }, -// ], -// properties: {}, -// widgets_values: ["text, watermark"], -// }, -// { -// id: 6, -// type: "CLIPTextEncode", -// pos: [415, 186], -// size: { 0: 422.84503173828125, 1: 164.31304931640625 }, -// flags: {}, -// order: 2, -// mode: 0, -// inputs: [{ name: "clip", type: "CLIP", link: 3 }], -// outputs: [ -// { -// name: "CONDITIONING", -// type: "CONDITIONING", -// links: [4], -// slot_index: 0, -// }, -// ], -// properties: {}, -// widgets_values: [ -// "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", -// ], -// }, -// { -// id: 5, -// type: "EmptyLatentImage", -// pos: [473, 609], -// size: { 0: 315, 1: 106 }, -// flags: {}, -// order: 1, -// mode: 0, -// outputs: [{ name: "LATENT", type: "LATENT", links: [2], slot_index: 0 }], -// properties: {}, -// widgets_values: [512, 512, 1], -// }, -// { -// id: 3, -// type: "KSampler", -// pos: [863, 186], -// size: { 0: 315, 1: 262 }, -// flags: {}, -// order: 4, -// mode: 0, -// inputs: [ -// { name: "model", type: "MODEL", link: 1 }, -// { name: "positive", type: "CONDITIONING", link: 4 }, -// { name: "negative", type: "CONDITIONING", link: 6 }, -// { name: "latent_image", type: "LATENT", link: 2 }, -// ], -// outputs: [{ name: "LATENT", type: "LATENT", links: [7], slot_index: 0 }], -// properties: {}, -// widgets_values: [156680208700286, true, 20, 8, "euler", "normal", 1], -// }, -// { -// id: 8, -// type: "VAEDecode", -// pos: [1209, 188], -// size: { 0: 210, 1: 46 }, -// flags: {}, -// order: 5, -// mode: 0, -// inputs: [ -// { name: "samples", type: "LATENT", link: 7 }, -// { name: "vae", type: "VAE", link: 8 }, -// ], -// outputs: [{ name: "IMAGE", type: "IMAGE", links: [9], slot_index: 0 }], -// properties: {}, -// }, -// { -// id: 9, -// type: "SaveImage", -// pos: [1451, 189], -// size: { 0: 210, 1: 26 }, -// flags: {}, -// order: 6, -// mode: 0, -// inputs: [{ name: "images", type: "IMAGE", link: 9 }], -// properties: {}, -// }, -// { -// id: 4, -// type: "CheckpointLoaderSimple", -// pos: [26, 474], -// size: { 0: 315, 1: 98 }, -// flags: {}, -// order: 0, -// mode: 0, -// outputs: [ -// { name: "MODEL", type: "MODEL", links: [1], slot_index: 0 }, -// { name: "CLIP", type: "CLIP", links: [3, 5], slot_index: 1 }, -// { name: "VAE", type: "VAE", links: [8], slot_index: 2 }, -// ], -// properties: {}, -// widgets_values: ["v1-5-pruned-emaonly.ckpt"], -// }, -// ], -// links: [ -// [1, 4, 0, 3, 0, "MODEL"], -// [2, 5, 0, 3, 3, "LATENT"], -// [3, 4, 1, 6, 0, "CLIP"], -// [4, 6, 0, 3, 1, "CONDITIONING"], -// [5, 4, 1, 7, 0, "CLIP"], -// [6, 7, 0, 3, 2, "CONDITIONING"], -// [7, 3, 0, 8, 0, "LATENT"], -// [8, 4, 2, 8, 1, "VAE"], -// [9, 8, 0, 9, 0, "IMAGE"], -// ], -// groups: [], -// config: {}, -// extra: {}, -// version: 0.4, -// }; - const input = { nodes: [ - { id: 1, class_type: "CLIPTextEncode", pos: [100, 100] }, - { id: 2, class_type: "KSampler", pos: [300, 100] }, - { id: 3, class_type: "VAEDecode", pos: [500, 100] }, - { id: 4, class_type: "SaveImage", pos: [700, 100] }, + { id: 4, class_type: "CheckpointLoaderSimple" }, + { id: 6, class_type: "CLIPTextEncode" }, + { id: 7, class_type: "CLIPTextEncode" }, + { id: 5, class_type: "EmptyLatentImage" }, + { id: 3, class_type: "KSampler" }, + { id: 8, class_type: "VAEDecode" }, + { id: 9, class_type: "SaveImage" }, ], edges: [ - { from: 1, to: 2, input: 0, output: 0 }, - { from: 2, to: 3, input: 0, output: 0 }, - { from: 3, to: 4, input: 0, output: 0 }, + { from: 4, to: 3, input: "model" }, + { from: 5, to: 3, input: "latent_image" }, + { from: 4, to: 6, input: "clip" }, + { from: 6, to: 3, input: "positive" }, + { from: 4, to: 7, input: "clip" }, + { from: 7, to: 3, input: "negative" }, + { from: 3, to: 8, input: "samples" }, + { from: 4, to: 8, input: "vae" }, + { from: 8, to: 9, input: "images" }, ], }; @@ -156,18 +29,31 @@ function getInputType(name) { positive: "CONDITIONING", negative: "CONDITIONING", latent_image: "LATENT", + samples: "LATENT", images: "IMAGE", + clip: "CLIP", + vae: "VAE", }; - return nameTypeMap[name] || "XXX UNKNOWN INPUT TYPE XXX"; + const matchName = nameTypeMap[name]; + if (matchName) { + return matchName; + } else { + console.log("not matching name, will return unknown: ", name); + return "XXX UNKNOWN INPUT TYPE XXX"; + } } function getOutputIndex(node, inputName) { - return node.outputs.findIndex((output) => output.name === inputName); + return node.outputs.findIndex( + (output) => output.name === getInputType(inputName) + ); } function getInputIndex(node, inputName) { - return node.inputs.findIndex((input) => input.name === inputName); + return node.inputs.findIndex( + (input) => input.name === getInputType(inputName) + ); } // TODO: WIP @@ -242,9 +128,9 @@ function transformInputToGraph(input) { // if output doesnt already exist if (outputIndex < 0) { fromNode.outputs.push({ - name: edgeData.input, - // type: edgeData.input.toUpperCase(), - type: getInputType(edgeData.output), + name: getInputType(edgeData.input), + // name: edgeData.input, + type: getInputType(edgeData.input), links: [], slot_index: fromNode.outputs.length, }); @@ -289,7 +175,6 @@ function addNodesAndEdges(graph, input) { console.log(translatedData); app.loadGraphData(translatedData, true); - // OUTDATED MANUAL INSERTION METHOD // const n = input.nodes; // const missingNodeTypes = []; From 60293aecbea52ac2feb36bbb9d9d828a0ceb5143 Mon Sep 17 00:00:00 2001 From: vindennt Date: Thu, 18 Jul 2024 19:30:21 -0700 Subject: [PATCH 05/19] adds workflow spacer function --- js/chatbot/workflow.js | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/js/chatbot/workflow.js b/js/chatbot/workflow.js index cbf0f56..cb6a6db 100644 --- a/js/chatbot/workflow.js +++ b/js/chatbot/workflow.js @@ -74,16 +74,20 @@ function transformInputToGraph(input) { console.log(baseGraph); const xStep = 300; // step for node positions + const yStep = 300 // step for node positions + const yBase = 100; // base y position const nodesById = {}; // pointer of nodes by id for easier/faster search let x = 100; // starting x post + let y = yBase; // starting y post + let columnCounter = 0; // counter for columns // Create nodes with positions in order input.nodes.forEach((node, index) => { const newNode = { id: node.id, type: node.class_type, - pos: [x, 100], - size: [200, 50], + pos: [x, y], + size: [200, 50], // TODO: write a function to get the node's actual height for better placement flags: {}, order: index, mode: 0, @@ -95,7 +99,15 @@ function transformInputToGraph(input) { baseGraph.nodes.push(newNode); nodesById[node.id] = newNode; - x += xStep; + + if (columnCounter % 3 === 0) { + x += xStep; + y = yBase; + } else { + // y += newNode.size[1] + yStep; + y += yStep; + } + columnCounter++; }); console.log("baseGraph after nodes insertion"); From 893caa0d705dd73d2eeba1300466978e51e71e93 Mon Sep 17 00:00:00 2001 From: vindennt Date: Thu, 18 Jul 2024 20:13:28 -0700 Subject: [PATCH 06/19] adds hardcoded output input name list and logic to scan it --- js/chatbot/edgeTypes.js | 406 ++++++++++++++++++++++++++++++++++++++++ js/chatbot/ext.js | 2 +- js/chatbot/workflow.js | 28 +-- 3 files changed, 411 insertions(+), 25 deletions(-) create mode 100644 js/chatbot/edgeTypes.js diff --git a/js/chatbot/edgeTypes.js b/js/chatbot/edgeTypes.js new file mode 100644 index 0000000..1a94310 --- /dev/null +++ b/js/chatbot/edgeTypes.js @@ -0,0 +1,406 @@ +/* eslint-disable no-unused-vars */ +export function getInputType(name) { + const nameTypeMap = { + LATENT: "LATENT", + MODEL: "MODEL", + CLIP: "CLIP", + VAE: "VAE", + CONDITIONING: "CONDITIONING", + IMAGE: "IMAGE", + MASK: "MASK", + CLIP_VISION_OUTPUT: "CLIP_VISION_OUTPUT", + positive: "CONDITIONING", + negative: "CONDITIONING", + CONTROL_NET: "CONTROL_NET", + STYLE_MODEL: "STYLE_MODEL", + CLIP_VISION: "CLIP_VISION", + GLIGEN: "GLIGEN", + latent: "LATENT", + UPSCALE_MODEL: "UPSCALE_MODEL", + output: "LATENT", + denoised_output: "LATENT", + SIGMAS: "SIGMAS", + SAMPLER: "SAMPLER", + high_sigmas: "SIGMAS", + low_sigmas: "SIGMAS", + GUIDER: "GUIDER", + NOISE: "NOISE", + PHOTOMAKER: "PHOTOMAKER", + stage_c: "LATENT", + stage_b: "LATENT", + controlnet_input: "IMAGE", + STRING: "STRING", + "*": "*", + INT: "INT", + FLOAT: "FLOAT", + BLIP_MODEL: "BLIP_MODEL", + bus: "BUS", + model: "MODEL", + clip: "CLIP", + vae: "VAE", + latent_filename: "STRING", + latent_image: "LATENT", + image_filename: "STRING", + conditioning_filename: "STRING", + NAME_STRING: "STRING", + conditioning: "CONDITIONING", + parsed_text: "STRING", + raw_text: "STRING", + NUMBER: "NUMBER", + image_a_pass: "IMAGE", + image_b_pass: "IMAGE", + filepath_text: "STRING", + filename_text: "STRING", + MASK_IMAGE: "IMAGE", + clipseg_model: "CLIPSEG_MODEL", + IMAGES_BATCH: "IMAGE", + MASKS_BATCH: "MASK", + MASK_IMAGES_BATCH: "IMAGE", + IMAGES: "IMAGE", + DICT: "DICT", + BOOLEAN: "BOOLEAN", + composited_images: "IMAGE", + ssao_images: "IMAGE", + specular_mask_images: "IMAGE", + ssdo_images: "IMAGE", + ssdo_image_masks: "IMAGE", + light_source_image_masks: "IMAGE", + aspect_number: "NUMBER", + aspect_float: "FLOAT", + is_landscape_bool: "NUMBER", + aspect_ratio_common: "STRING", + aspect_type: "STRING", + image: "IMAGE", + images: "IMAGE", + color_palettes: "LIST", + CROP_DATA: "CROP_DATA", + mask: "MASK", + shadow_map: "IMAGE", + highlight_map: "IMAGE", + width_num: "NUMBER", + height_num: "NUMBER", + width_float: "FLOAT", + height_float: "FLOAT", + width_int: "INT", + height_int: "INT", + MASKS: "MASK", + INT_PLACES: "INT", + "latent(s)": "LATENT", + tensor_w_num: "NUMBER", + tensor_h_num: "NUMBER", + masks: "MASK", + cropped_mask: "MASK", + crop_data: "CROP_DATA", + top_int: "INT", + left_int: "INT", + right_int: "INT", + bottom_int: "INT", + midas_model: "MIDAS_MODEL", + number: "NUMBER", + float: "FLOAT", + int: "INT", + SEED: "SEED", + positive_string: "STRING", + negative_string: "STRING", + seed: "SEED", + SAM_MODEL: "SAM_MODEL", + SAM_PARAMETERS: "SAM_PARAMETERS", + samples: "LATENT", + IMAGE_BOUNDS: "IMAGE_BOUNDS", + LIST: "LIST", + TEXT_A_PASS: "STRING", + TEXT_B_PASS: "STRING", + SCORE_NUMBER: "NUMBER", + COMPARISON_TEXT: "STRING", + result_text: "STRING", + replacement_count_number: "NUMBER", + replacement_count_float: "FLOAT", + replacement_count_int: "INT", + f: "B", + o: "O", + u: "O", + n: "L", + d: "E", + line_text: "STRING", + dictionary: "DICT", + MODEL_NAME_TEXT: "STRING", + image_pass: "IMAGE", + IMAGE_PASS: "IMAGE", + output_path: "STRING", + processed_count: "NUMBER", + BBOX_DETECTOR: "BBOX_DETECTOR", + SEGS: "SEGS", + cropped: "IMAGE", + cropped_refined: "IMAGE", + cropped_refined_alpha: "IMAGE", + cnet_images: "IMAGE", + segs: "SEGS", + basic_pipe: "BASIC_PIPE", + combined_mask: "MASK", + batch_masks: "MASK", + cropped_enhanced_alpha: "IMAGE", + detailer_pipe: "DETAILER_PIPE", + refiner_basic_pipe_opt: "BASIC_PIPE", + bbox_detector: "BBOX_DETECTOR", + sam_model_opt: "SAM_MODEL", + segm_detector_opt: "SEGM_DETECTOR", + detailer_hook: "DETAILER_HOOK", + refiner_model: "MODEL", + refiner_clip: "CLIP", + refiner_positive: "CONDITIONING", + refiner_negative: "CONDITIONING", + base_basic_pipe: "BASIC_PIPE", + refiner_basic_pipe: "BASIC_PIPE", + UPSCALER: "UPSCALER", + PK_HOOK: "PK_HOOK", + DETAILER_HOOK: "DETAILER_HOOK", + UPSCALER_HOOK: "UPSCALER_HOOK", + SEGS_HEADER: "SEGS_HEADER", + SEG_ELT: "SEG_ELT", + seg_elt: "SEG_ELT", + cropped_image: "IMAGE", + crop_region: "SEG_ELT_crop_region", + bbox: "SEG_ELT_bbox", + control_net_wrapper: "SEG_ELT_control_net_wrapper", + confidence: "FLOAT", + label: "STRING", + left: "INT", + top: "INT", + right: "INT", + bottom: "INT", + KSAMPLER: "KSAMPLER", + KSAMPLER_ADVANCED: "KSAMPLER_ADVANCED", + selected_value: "*", + selected_label: "STRING", + selected_index: "INT", + output1: "*", + populated_text: "STRING", + BASIC_PIPE: "BASIC_PIPE", + REGIONAL_PROMPTS: "REGIONAL_PROMPTS", + filtered_SEGS: "SEGS", + remained_SEGS: "SEGS", + signal_opt: "*", + bool: "BOOLEAN", + signal: "*", + batch: "INT", + height: "INT", + width: "INT", + channel: "INT", + wildcard: "STRING", + segs_labels: "STRING", + count: "INT", + total: "INT", + value: "*", + TRANSFORMERS_CLASSIFIER: "TRANSFORMERS_CLASSIFIER", + scheduler: [ + "normal", + "karras", + "exponential", + "sgm_uniform", + "simple", + "ddim_uniform", + ], + SEGM_DETECTOR: "SEGM_DETECTOR", + TIMESTEP_KF: "TIMESTEP_KEYFRAME", + LATENT_KF: "LATENT_KEYFRAME", + model_opt: "MODEL", + CN_WEIGHTS: "CONTROL_NET_WEIGHTS", + TK_SHORTCUT: "TIMESTEP_KEYFRAME", + proc_IMAGE: "IMAGE", + SPARSE_METHOD: "SPARSE_METHOD", + cn_extras: "CN_WEIGHTS_EXTRAS", + "ABG_CHARACTER_MASK (MASK)": "MASK", + POSE_KEYPOINT: "POSE_KEYPOINT", + tracking: "TRACKING", + prompt: "STRING", + OPTICAL_FLOW: "OPTICAL_FLOW", + PREVIEW_IMAGE: "IMAGE", + INPAINTING_MASK: "MASK", + preprocessor: [ + "none", + "TilePreprocessor", + "TTPlanet_TileGF_Preprocessor", + "TTPlanet_TileSimple_Preprocessor", + "AnimeFace_SemSegPreprocessor", + "BinaryPreprocessor", + "Zoe-DepthMapPreprocessor", + "ColorPreprocessor", + "DepthAnythingPreprocessor", + "Zoe_DepthAnythingPreprocessor", + "DensePosePreprocessor", + "OneFormer-COCO-SemSegPreprocessor", + "OneFormer-ADE20K-SemSegPreprocessor", + "M-LSDPreprocessor", + "AnyLineArtPreprocessor_aux", + "DWPreprocessor", + "AnimalPosePreprocessor", + "DSINE-NormalMapPreprocessor", + "MediaPipe-FaceMeshPreprocessor", + "LineArtPreprocessor", + "MiDaS-NormalMapPreprocessor", + "MiDaS-DepthMapPreprocessor", + "CannyEdgePreprocessor", + "ShufflePreprocessor", + "Metric3D-DepthMapPreprocessor", + "Metric3D-NormalMapPreprocessor", + "AnimeLineArtPreprocessor", + "SAMPreprocessor", + "HEDPreprocessor", + "FakeScribblePreprocessor", + "ScribblePreprocessor", + "Scribble_XDoG_Preprocessor", + "Scribble_PiDiNet_Preprocessor", + "DiffusionEdge_Preprocessor", + "SavePoseKpsAsJsonFile", + "FacialPartColoringFromPoseKps", + "UpperBodyTrackingFromPoseKps", + "PiDiNetPreprocessor", + "OpenposePreprocessor", + "LeReS-DepthMapPreprocessor", + "Unimatch_OptFlowPreprocessor", + "MaskOptFlow", + "ImageLuminanceDetector", + "ImageIntensityDetector", + "UniFormer-SemSegPreprocessor", + "SemSegPreprocessor", + "MeshGraphormer-DepthMapPreprocessor", + "MeshGraphormer+ImpactDetector-DepthMapPreprocessor", + "Manga2Anime_LineArt_Preprocessor", + "BAE-NormalMapPreprocessor", + "TEEDPreprocessor", + "LineartStandardPreprocessor", + ], + "RESOLUTION (INT)": "INT", + "IMAGE_GEN_WIDTH (INT)": "INT", + "IMAGE_GEN_HEIGHT (INT)": "INT", + "Full prompt": "STRING", + "Short prompt": "STRING", + "Compiled prompt": "CONDITIONING", + Value: "STRING", + Value2: "STRING", + mix: "INT", + Filenames: "VHS_FILENAMES", + frame_count: "INT", + audio: "VHS_AUDIO", + video_info: "VHS_VIDEOINFO", + meta_batch: "VHS_BatchManager", + "source_fps🟨": "FLOAT", + "source_frame_count🟨": "INT", + "source_duration🟨": "FLOAT", + "source_width🟨": "INT", + "source_height🟨": "INT", + "loaded_fps🟦": "FLOAT", + "loaded_frame_count🟦": "INT", + "loaded_duration🟦": "FLOAT", + "loaded_width🟦": "INT", + "loaded_height🟦": "INT", + "fps🟨": "FLOAT", + "frame_count🟨": "INT", + "duration🟨": "FLOAT", + "width🟨": "INT", + "height🟨": "INT", + "fps🟦": "FLOAT", + "frame_count🟦": "INT", + "duration🟦": "FLOAT", + "width🟦": "INT", + "height🟦": "INT", + LATENT_A: "LATENT", + A_count: "INT", + LATENT_B: "LATENT", + B_count: "INT", + IMAGE_A: "IMAGE", + IMAGE_B: "IMAGE", + MASK_A: "MASK", + MASK_B: "MASK", + modelname: "STRING", + sampler_name: [ + "euler", + "euler_ancestral", + "heun", + "heunpp2", + "dpm_2", + "dpm_2_ancestral", + "lms", + "dpm_fast", + "dpm_adaptive", + "dpmpp_2s_ancestral", + "dpmpp_sde", + "dpmpp_sde_gpu", + "dpmpp_2m", + "dpmpp_2m_sde", + "dpmpp_2m_sde_gpu", + "dpmpp_3m_sde", + "dpmpp_3m_sde_gpu", + "ddpm", + "lcm", + "ddim", + "uni_pc", + "uni_pc_bh2", + ], + scheduler_name: "STRING", + s: "STRING", + x: "INT", + y: "INT", + REMBG_SESSION: "REMBG_SESSION", + red: "MASK", + green: "MASK", + blue: "MASK", + cyan: "MASK", + magenta: "MASK", + yellow: "MASK", + black: "MASK", + white: "MASK", + CLIP_SEG: "CLIP_SEG", + description: "STRING", + response: "STRING", + context: "STRING", + }; + + const matchName = nameTypeMap[name]; + if (matchName) { + return matchName; + } else { + console.log("not matching name, will return unknown: ", name); + return "XXX UNKNOWN INPUT TYPE XXX"; + } +} + +// originally was in in file that imported app +function scanNodeInputOutput() { + const nameTypeMap = new Map(); + + function updateNameTypeMap(input, output) { + const maxLength = Math.min(input.length, output.length); + + for (let i = 0; i < maxLength; i++) { + const inputName = input[i]; + const outputName = output[i]; + + // Add to nameTypeMap if inputName is not already present + if (!nameTypeMap.has(inputName)) { + nameTypeMap[inputName] = outputName; + } + } + } + + + console.log(LiteGraph.registered_node_types); + if (LiteGraph.registered_node_types) { + // Get the first node in the list + for (const [key, nodeType] of Object.entries( + LiteGraph.registered_node_types + )) { + console.log(key); + if (nodeType.nodeData?.output !== undefined) { + updateNameTypeMap( + nodeType.nodeData.output_name, + nodeType.nodeData.output + ); + } + } + console.log(JSON.stringify(nameTypeMap)); + } else { + console.log("No registered nodes found."); + } + return nameTypeMap; +} diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index 72a0780..6400139 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -120,7 +120,7 @@ const createChatBox = () => { // app.graph.add(newNode); console.log(app.graph); app.graph.change(); - // console.log('New button clicked!'); + }); chatBox.appendChild(newButton); diff --git a/js/chatbot/workflow.js b/js/chatbot/workflow.js index cb6a6db..d3c4ccd 100644 --- a/js/chatbot/workflow.js +++ b/js/chatbot/workflow.js @@ -1,4 +1,5 @@ import { app } from "../comfy/comfy.js"; +import { getInputType } from "./edgeTypes.js"; const input = { nodes: [ @@ -23,27 +24,6 @@ const input = { ], }; -function getInputType(name) { - const nameTypeMap = { - model: "MODEL", - positive: "CONDITIONING", - negative: "CONDITIONING", - latent_image: "LATENT", - samples: "LATENT", - images: "IMAGE", - clip: "CLIP", - vae: "VAE", - }; - - const matchName = nameTypeMap[name]; - if (matchName) { - return matchName; - } else { - console.log("not matching name, will return unknown: ", name); - return "XXX UNKNOWN INPUT TYPE XXX"; - } -} - function getOutputIndex(node, inputName) { return node.outputs.findIndex( (output) => output.name === getInputType(inputName) @@ -74,7 +54,7 @@ function transformInputToGraph(input) { console.log(baseGraph); const xStep = 300; // step for node positions - const yStep = 300 // step for node positions + const yStep = 300; // step for node positions const yBase = 100; // base y position const nodesById = {}; // pointer of nodes by id for easier/faster search @@ -104,8 +84,8 @@ function transformInputToGraph(input) { x += xStep; y = yBase; } else { - // y += newNode.size[1] + yStep; - y += yStep; + // y += newNode.size[1] + yStep; + y += yStep; } columnCounter++; }); From 042f4f29f39eca3649473ca01c06e9021dad9599 Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 09:35:19 -0700 Subject: [PATCH 07/19] refactoring and adding new sample data structure --- js/chatbot/ext.js | 40 ++++++++-------- js/chatbot/workflow.js | 102 ++++++++++++++++++++++++----------------- 2 files changed, 80 insertions(+), 62 deletions(-) diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index 6400139..b68bbaf 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -102,29 +102,26 @@ const createChatBox = () => { runWorkflowLoader(); }); - // New button section - const otherButton = document.createElement("button"); - otherButton.innerText = "Log graph"; - otherButton.style.position = "absolute"; - otherButton.style.top = "100px"; - otherButton.style.right = "10px"; - otherButton.style.padding = "10px"; - otherButton.style.backgroundColor = "blue"; - otherButton.style.color = "white"; - otherButton.style.border = "none"; - otherButton.style.borderRadius = "4px"; - otherButton.style.cursor = "pointer"; + // log button + // const otherButton = document.createElement("button"); + // otherButton.innerText = "Log graph"; + // otherButton.style.position = "absolute"; + // otherButton.style.top = "100px"; + // otherButton.style.right = "10px"; + // otherButton.style.padding = "10px"; + // otherButton.style.backgroundColor = "blue"; + // otherButton.style.color = "white"; + // otherButton.style.border = "none"; + // otherButton.style.borderRadius = "4px"; + // otherButton.style.cursor = "pointer"; - otherButton.addEventListener("click", () => { - // const newNode = LiteGraph.createNode("LoadImage"); - // app.graph.add(newNode); - console.log(app.graph); - app.graph.change(); - - }); + // otherButton.addEventListener("click", () => { + // console.log(app.graph); + // app.graph.change(); + // }); chatBox.appendChild(newButton); - chatBox.appendChild(otherButton); + // chatBox.appendChild(otherButton); return chatBox; }; @@ -157,6 +154,9 @@ const sendMessage = () => { chatMessages.appendChild(botMessageElement); chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + + runWorkflowLoader() + // addNodesAndEdges(app.graph, response.message); }) .catch((error) => { console.error(error); diff --git a/js/chatbot/workflow.js b/js/chatbot/workflow.js index d3c4ccd..b642d36 100644 --- a/js/chatbot/workflow.js +++ b/js/chatbot/workflow.js @@ -1,7 +1,7 @@ import { app } from "../comfy/comfy.js"; import { getInputType } from "./edgeTypes.js"; -const input = { +const sampleInput = { nodes: [ { id: 4, class_type: "CheckpointLoaderSimple" }, { id: 6, class_type: "CLIPTextEncode" }, @@ -24,6 +24,59 @@ const input = { ], }; +// const newSampleInput = { +// nodes: [ +// { +// id: 4, +// class_type: "CheckpointLoaderSimple", +// widget_values: { checkpoint: "v1-5-pruned-emaonly.ckpt" }, +// }, +// { +// id: 5, +// class_type: "EmptyLatentImage", +// widget_values: { width: 512, height: 512, channels: 1 }, +// }, +// { +// id: 6, +// class_type: "CLIPTextEncode", +// widget_values: { +// text: "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", +// }, +// }, +// { +// id: 7, +// class_type: "CLIPTextEncode", +// widget_values: { text: "text, watermark" }, +// }, +// { +// id: 3, +// class_type: "KSampler", +// widget_values: { +// seed: 156680208700286, +// continuous: true, +// steps: 20, +// denoising: 8, +// scheduler: "euler", +// noise: "normal", +// scale: 1, +// }, +// }, +// { id: 8, class_type: "VAEDecode" }, +// { id: 9, class_type: "SaveImage" }, +// ], +// edges: [ +// { from: 4, to: 3, from_output: 0, to_input: "model" }, +// { from: 5, to: 3, from_output: 0, to_input: "latent_image" }, +// { from: 4, to: 6, from_output: 1, to_input: "clip" }, +// { from: 6, to: 3, from_output: 0, to_input: "positive" }, +// { from: 7, to: 3, from_output: 0, to_input: "negative" }, +// { from: 4, to: 7, from_output: 1, to_input: "clip" }, +// { from: 3, to: 8, from_output: 0, to_input: "samples" }, +// { from: 4, to: 8, from_output: 2, to_input: "vae" }, +// { from: 8, to: 9, from_output: 0, to_input: "images" }, +// ], +// }; + function getOutputIndex(node, inputName) { return node.outputs.findIndex( (output) => output.name === getInputType(inputName) @@ -36,7 +89,6 @@ function getInputIndex(node, inputName) { ); } -// TODO: WIP // translates chatgpt input to immediately loadable graph format function transformInputToGraph(input) { const baseGraph = { @@ -157,54 +209,20 @@ function transformInputToGraph(input) { return baseGraph; } -// const node = LiteGraph.createNode("LoadImage"); -// graph.add(node); - -// input: chat gpt's generated workflow -function addNodesAndEdges(graph, input) { - const translatedData = transformInputToGraph(input); - console.log("Translated Graph Data: "); - console.log(translatedData); +export function loadGraphFromPrompt(generatedWorkflow) { + const translatedData = transformInputToGraph(generatedWorkflow); + // console.log("Translated Graph Data: "); + // console.log(translatedData); app.loadGraphData(translatedData, true); - - // OUTDATED MANUAL INSERTION METHOD - // const n = input.nodes; - // const missingNodeTypes = []; - // const nodesById = {}; - - // TODO: register nodes. e.g. LiteGraph.registered_node_types["LoadImage"] = LoadImageNode; - - // code block from loadGraphData to find unregistered nodes - // if (!(n.type in LiteGraph.registered_node_types)) { - // missingNodeTypes.push(n.type); - // // n.type = sanitizeNodeName(n.type); - // } - - // insert the nodes one by one. This code is working - // n.forEach((nodeData) => { - // const node = LiteGraph.createNode(nodeData.class_type); - // node.id = nodeData.id; - // node.pos = nodeData.pos || [100, 100]; // Default position. TODO: add increment for position - // graph.add(node); - // // nodesById[nodeData.id] = node; - // }); - - // TODO: add input edges - - // TODO: add output edges - - // TODO: without manually adding the edges, try to get input data into defaultGraph's format - // app.loadGraphData(defaultGraph, true) + app.graph.change(); } export function runWorkflowLoader() { console.log("Initial graph"); console.log(app.graph); - // app.graph.clear(); // clear graph - addNodesAndEdges(app.graph, input); + loadGraphFromPrompt(sampleInput); - app.graph.change(); console.log("Graph after change"); console.log(app.graph); } From 9ccbfa0bb0ab4131290944fea5486e1b45e7b753 Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 10:20:44 -0700 Subject: [PATCH 08/19] updates workflow loader to use new data structure --- js/chatbot/workflow.js | 135 +++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 79 deletions(-) diff --git a/js/chatbot/workflow.js b/js/chatbot/workflow.js index b642d36..3bf1af1 100644 --- a/js/chatbot/workflow.js +++ b/js/chatbot/workflow.js @@ -1,82 +1,59 @@ import { app } from "../comfy/comfy.js"; import { getInputType } from "./edgeTypes.js"; -const sampleInput = { +const newSampleInput = { nodes: [ - { id: 4, class_type: "CheckpointLoaderSimple" }, - { id: 6, class_type: "CLIPTextEncode" }, - { id: 7, class_type: "CLIPTextEncode" }, - { id: 5, class_type: "EmptyLatentImage" }, - { id: 3, class_type: "KSampler" }, + { + id: 4, + class_type: "CheckpointLoaderSimple", + widget_values: { checkpoint: "v1-5-pruned-emaonly.safetensors" }, + }, + { + id: 5, + class_type: "EmptyLatentImage", + widget_values: { width: 512, height: 512, channels: 1 }, + }, + { + id: 6, + class_type: "CLIPTextEncode", + widget_values: { + text: "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", + }, + }, + { + id: 7, + class_type: "CLIPTextEncode", + widget_values: { text: "text, watermark" }, + }, + { + id: 3, + class_type: "KSampler", + widget_values: { + seed: 156680208700286, + continuous: true, + steps: 20, + denoising: 8, + scheduler: "euler", + noise: "normal", + scale: 1, + }, + }, { id: 8, class_type: "VAEDecode" }, { id: 9, class_type: "SaveImage" }, ], edges: [ - { from: 4, to: 3, input: "model" }, - { from: 5, to: 3, input: "latent_image" }, - { from: 4, to: 6, input: "clip" }, - { from: 6, to: 3, input: "positive" }, - { from: 4, to: 7, input: "clip" }, - { from: 7, to: 3, input: "negative" }, - { from: 3, to: 8, input: "samples" }, - { from: 4, to: 8, input: "vae" }, - { from: 8, to: 9, input: "images" }, + { from: 4, to: 3, from_output: 0, to_input: "model" }, + { from: 5, to: 3, from_output: 0, to_input: "latent_image" }, + { from: 4, to: 6, from_output: 1, to_input: "clip" }, + { from: 6, to: 3, from_output: 0, to_input: "positive" }, + { from: 7, to: 3, from_output: 0, to_input: "negative" }, + { from: 4, to: 7, from_output: 1, to_input: "clip" }, + { from: 3, to: 8, from_output: 0, to_input: "samples" }, + { from: 4, to: 8, from_output: 2, to_input: "vae" }, + { from: 8, to: 9, from_output: 0, to_input: "images" }, ], }; -// const newSampleInput = { -// nodes: [ -// { -// id: 4, -// class_type: "CheckpointLoaderSimple", -// widget_values: { checkpoint: "v1-5-pruned-emaonly.ckpt" }, -// }, -// { -// id: 5, -// class_type: "EmptyLatentImage", -// widget_values: { width: 512, height: 512, channels: 1 }, -// }, -// { -// id: 6, -// class_type: "CLIPTextEncode", -// widget_values: { -// text: "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", -// }, -// }, -// { -// id: 7, -// class_type: "CLIPTextEncode", -// widget_values: { text: "text, watermark" }, -// }, -// { -// id: 3, -// class_type: "KSampler", -// widget_values: { -// seed: 156680208700286, -// continuous: true, -// steps: 20, -// denoising: 8, -// scheduler: "euler", -// noise: "normal", -// scale: 1, -// }, -// }, -// { id: 8, class_type: "VAEDecode" }, -// { id: 9, class_type: "SaveImage" }, -// ], -// edges: [ -// { from: 4, to: 3, from_output: 0, to_input: "model" }, -// { from: 5, to: 3, from_output: 0, to_input: "latent_image" }, -// { from: 4, to: 6, from_output: 1, to_input: "clip" }, -// { from: 6, to: 3, from_output: 0, to_input: "positive" }, -// { from: 7, to: 3, from_output: 0, to_input: "negative" }, -// { from: 4, to: 7, from_output: 1, to_input: "clip" }, -// { from: 3, to: 8, from_output: 0, to_input: "samples" }, -// { from: 4, to: 8, from_output: 2, to_input: "vae" }, -// { from: 8, to: 9, from_output: 0, to_input: "images" }, -// ], -// }; - function getOutputIndex(node, inputName) { return node.outputs.findIndex( (output) => output.name === getInputType(inputName) @@ -126,7 +103,7 @@ function transformInputToGraph(input) { inputs: [], outputs: [], properties: {}, - widgets_values: [], // TODO: look into how widgets work + widgets_values: node.widget_values ? Object.values(node.widget_values) : [] }; baseGraph.nodes.push(newNode); @@ -151,16 +128,16 @@ function transformInputToGraph(input) { const toNode = nodesById[edgeData.to]; // Adding output to fromNode - let outputIndex = getOutputIndex(fromNode, edgeData.input); + let outputIndex = getOutputIndex(fromNode, edgeData.to_input); // add inputs - let inputIndex = getInputIndex(toNode, edgeData.input); + let inputIndex = getInputIndex(toNode, edgeData.to_input); // if input doesnt already exist if (inputIndex < 0) { toNode.inputs.push({ - name: edgeData.input, - type: getInputType(edgeData.input), + name: edgeData.to_input, + type: getInputType(edgeData.to_input), link: index + 1, }); } @@ -172,9 +149,9 @@ function transformInputToGraph(input) { // if output doesnt already exist if (outputIndex < 0) { fromNode.outputs.push({ - name: getInputType(edgeData.input), + name: getInputType(edgeData.to_input), // name: edgeData.input, - type: getInputType(edgeData.input), + type: getInputType(edgeData.to_input), links: [], slot_index: fromNode.outputs.length, }); @@ -190,10 +167,10 @@ function transformInputToGraph(input) { const link = [ index + 1, // Link id fromNode.id, // Origin node id - getOutputIndex(fromNode, edgeData.input), // origin index + getOutputIndex(fromNode, edgeData.to_input), // origin index toNode.id, // Destination node id - getInputIndex(toNode, edgeData.input), // destination index - getInputType(edgeData.input), + getInputIndex(toNode, edgeData.to_input), // destination index + getInputType(edgeData.to_input), ]; console.log("link"); @@ -221,7 +198,7 @@ export function runWorkflowLoader() { console.log("Initial graph"); console.log(app.graph); - loadGraphFromPrompt(sampleInput); + loadGraphFromPrompt(newSampleInput); console.log("Graph after change"); console.log(app.graph); From cc2aad458a17440ac1ccec1a19479b92bb531e94 Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 13:33:35 -0700 Subject: [PATCH 09/19] increases robustness of edge alg --- js/chatbot/workflow.js | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/js/chatbot/workflow.js b/js/chatbot/workflow.js index 3bf1af1..caafb55 100644 --- a/js/chatbot/workflow.js +++ b/js/chatbot/workflow.js @@ -54,12 +54,6 @@ const newSampleInput = { ], }; -function getOutputIndex(node, inputName) { - return node.outputs.findIndex( - (output) => output.name === getInputType(inputName) - ); -} - function getInputIndex(node, inputName) { return node.inputs.findIndex( (input) => input.name === getInputType(inputName) @@ -128,8 +122,7 @@ function transformInputToGraph(input) { const toNode = nodesById[edgeData.to]; // Adding output to fromNode - let outputIndex = getOutputIndex(fromNode, edgeData.to_input); - + let outputIndex = edgeData.from_output; // add inputs let inputIndex = getInputIndex(toNode, edgeData.to_input); @@ -145,17 +138,15 @@ function transformInputToGraph(input) { console.log("toNode"); console.log(toNode); - // add outputs by inferring the inputs - // if output doesnt already exist - if (outputIndex < 0) { - fromNode.outputs.push({ - name: getInputType(edgeData.to_input), - // name: edgeData.input, - type: getInputType(edgeData.to_input), - links: [], - slot_index: fromNode.outputs.length, - }); - } + // add outputs + fromNode.outputs.push({ + name: getInputType(edgeData.to_input), + // name: edgeData.input, + type: getInputType(edgeData.to_input), + links: [], + slot_index: fromNode.outputs.length, + }); + outputIndex = fromNode.outputs.length - 1; inputIndex = fromNode.inputs.length - 1; @@ -167,7 +158,8 @@ function transformInputToGraph(input) { const link = [ index + 1, // Link id fromNode.id, // Origin node id - getOutputIndex(fromNode, edgeData.to_input), // origin index + edgeData.from_output, + // getOutputIndex(fromNode, edgeData.to_input), // origin index toNode.id, // Destination node id getInputIndex(toNode, edgeData.to_input), // destination index getInputType(edgeData.to_input), From 61e1c8c9c34b69ebe556ff824bca21a6403a24af Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 13:35:12 -0700 Subject: [PATCH 10/19] adds auth requirement before chatbot message can be sent --- js/chatbot/ext.js | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index b68bbaf..362ef5b 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -2,6 +2,9 @@ import { app } from "../comfy/comfy.js"; import { getBotResponse } from "./chatbot.js"; import { runWorkflowLoader } from "./workflow.js"; +import { setButtonDefault } from "../button/ui.js"; +import { authDialog } from "../auth/index.js"; +import { getApiToken } from "../utils.js"; const CHAT_BUTTON_ID = "chat-button"; const CHAT_BOX_ID = "chat-box"; @@ -103,29 +106,37 @@ const createChatBox = () => { }); // log button - // const otherButton = document.createElement("button"); - // otherButton.innerText = "Log graph"; - // otherButton.style.position = "absolute"; - // otherButton.style.top = "100px"; - // otherButton.style.right = "10px"; - // otherButton.style.padding = "10px"; - // otherButton.style.backgroundColor = "blue"; - // otherButton.style.color = "white"; - // otherButton.style.border = "none"; - // otherButton.style.borderRadius = "4px"; - // otherButton.style.cursor = "pointer"; + const otherButton = document.createElement("button"); + otherButton.innerText = "Log graph"; + otherButton.style.position = "absolute"; + otherButton.style.top = "100px"; + otherButton.style.right = "10px"; + otherButton.style.padding = "10px"; + otherButton.style.backgroundColor = "blue"; + otherButton.style.color = "white"; + otherButton.style.border = "none"; + otherButton.style.borderRadius = "4px"; + otherButton.style.cursor = "pointer"; - // otherButton.addEventListener("click", () => { - // console.log(app.graph); - // app.graph.change(); - // }); + otherButton.addEventListener("click", () => { + console.log(app.graph); + app.graph.change(); + }); chatBox.appendChild(newButton); - // chatBox.appendChild(otherButton); + chatBox.appendChild(otherButton); return chatBox; }; const sendMessage = () => { + const apiToken = getApiToken(); + const doesApiTokenExist = !!apiToken; + if (!doesApiTokenExist) { + toggleChatBox() + setButtonDefault(); + return authDialog.show(); + } + const chatInput = document.getElementById(CHAT_INPUT_ID); const chatMessages = document.getElementById(CHAT_MESSAGES_ID); const message = chatInput.value.trim(); @@ -155,7 +166,7 @@ const sendMessage = () => { chatMessages.appendChild(botMessageElement); chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom - runWorkflowLoader() + runWorkflowLoader(); // addNodesAndEdges(app.graph, response.message); }) .catch((error) => { @@ -164,7 +175,7 @@ const sendMessage = () => { } }; -const toggleChatBox = () => { +const toggleChatBox = async () => { const chatBox = document.getElementById(CHAT_BOX_ID); if (chatBox.style.display === "none") { chatBox.style.display = "block"; From edb86fd3b337db2ef199577971b8963bfae1e21f Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 19:43:33 -0700 Subject: [PATCH 11/19] auth check before calling chatbot --- js/chatbot/chatbot.js | 4 ++-- js/chatbot/ext.js | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/js/chatbot/chatbot.js b/js/chatbot/chatbot.js index dffafb7..5a45256 100644 --- a/js/chatbot/chatbot.js +++ b/js/chatbot/chatbot.js @@ -2,14 +2,14 @@ const endpoint = 'http://localhost:4000/'; // Switch to live as needed -export async function getBotResponse(message) { - // TODO: add auth +export async function getBotResponse(message, token) { console.log("calling the gpt bot woith message", message); try { const response = await fetch(`${endpoint}chat/send-message`, { method: 'POST', headers: { 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}`, }, body: JSON.stringify({ message: message, origin: "ComfyUI Chat" }) }); diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index 362ef5b..d457822 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -11,6 +11,9 @@ const CHAT_BOX_ID = "chat-box"; const CHAT_INPUT_ID = "chat-input"; const CHAT_MESSAGES_ID = "chat-messages"; +const apiToken = getApiToken(); +const doesApiTokenExist = !!apiToken; + const createChatButton = () => { const chatButton = document.createElement("div"); chatButton.id = CHAT_BUTTON_ID; @@ -129,8 +132,6 @@ const createChatBox = () => { return chatBox; }; const sendMessage = () => { - const apiToken = getApiToken(); - const doesApiTokenExist = !!apiToken; if (!doesApiTokenExist) { toggleChatBox() setButtonDefault(); @@ -153,7 +154,7 @@ const sendMessage = () => { chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom chatInput.value = ""; // // get bot response - getBotResponse(message) + getBotResponse(message, apiToken) .then((response) => { console.log("ext got rresponse", response); const botMessageElement = document.createElement("div"); From 7cf65e98a2d9f863cac0d719daedfe9753a1be8d Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 20:10:19 -0700 Subject: [PATCH 12/19] adds automatic loader for generated workflows --- js/chatbot/chatbot.js | 3 +++ js/chatbot/ext.js | 39 +++++++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/js/chatbot/chatbot.js b/js/chatbot/chatbot.js index 5a45256..349af89 100644 --- a/js/chatbot/chatbot.js +++ b/js/chatbot/chatbot.js @@ -29,3 +29,6 @@ export async function getBotResponse(message, token) { } } +export function isValidWorkflow(workflow) { + return workflow && workflow.nodes && workflow.edges; +} \ No newline at end of file diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index d457822..63a2239 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -1,7 +1,7 @@ // import { api, app } from './comfy/comfy.js' import { app } from "../comfy/comfy.js"; -import { getBotResponse } from "./chatbot.js"; -import { runWorkflowLoader } from "./workflow.js"; +import { getBotResponse, isValidWorkflow } from "./chatbot.js"; +import { runWorkflowLoader, loadGraphFromPrompt } from "./workflow.js"; import { setButtonDefault } from "../button/ui.js"; import { authDialog } from "../auth/index.js"; import { getApiToken } from "../utils.js"; @@ -133,7 +133,7 @@ const createChatBox = () => { }; const sendMessage = () => { if (!doesApiTokenExist) { - toggleChatBox() + toggleChatBox(); setButtonDefault(); return authDialog.show(); } @@ -156,19 +156,30 @@ const sendMessage = () => { // // get bot response getBotResponse(message, apiToken) .then((response) => { - console.log("ext got rresponse", response); - const botMessageElement = document.createElement("div"); - botMessageElement.innerText = response.message; - botMessageElement.style.marginBottom = "10px"; - botMessageElement.style.padding = "10px"; - botMessageElement.style.backgroundColor = "#c7e1ff"; - botMessageElement.style.borderRadius = "4px"; + console.log("got response", response); + const botResponse = response.responses.bot; + console.log("ext got rresponse", botResponse); + const cleanedResponse = botResponse.replace(/^\s*```json\s*/, '').replace(/\s*```\s*$/, ''); - chatMessages.appendChild(botMessageElement); - chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + if (isValidWorkflow(JSON.parse(cleanedResponse))) { + console.log("Valid workflow that could be loaded"); + const botMessageElement = document.createElement("div"); + botMessageElement.innerText = botResponse; - runWorkflowLoader(); - // addNodesAndEdges(app.graph, response.message); + botMessageElement.style.marginBottom = "10px"; + botMessageElement.style.padding = "10px"; + botMessageElement.style.backgroundColor = "#c7e1ff"; + botMessageElement.style.borderRadius = "4px"; + + chatMessages.appendChild(botMessageElement); + chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + + // runWorkflowLoader(); + loadGraphFromPrompt(JSON.parse(cleanedResponse)); + // addNodesAndEdges(app.graph, response.message); + } else { + console.log("Not a valid workflow that could be loaded"); + } }) .catch((error) => { console.error(error); From 0e1fa1280d0c6ce00511697c49f39b86b96fc8f9 Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 20:45:30 -0700 Subject: [PATCH 13/19] refactor: bot message sending --- js/chatbot/chatbot.js | 58 ++++++++++--------- js/chatbot/ext.js | 131 +++++++++++++++++++++--------------------- 2 files changed, 98 insertions(+), 91 deletions(-) diff --git a/js/chatbot/chatbot.js b/js/chatbot/chatbot.js index 349af89..3cdc91f 100644 --- a/js/chatbot/chatbot.js +++ b/js/chatbot/chatbot.js @@ -1,34 +1,40 @@ // import { endpoint } from "../constants"; - -const endpoint = 'http://localhost:4000/'; // Switch to live as needed +const endpoint = "http://localhost:4000/"; // Switch to live as needed export async function getBotResponse(message, token) { - console.log("calling the gpt bot woith message", message); - try { - const response = await fetch(`${endpoint}chat/send-message`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${token}`, - }, - body: JSON.stringify({ message: message, origin: "ComfyUI Chat" }) - }); + console.log("calling the gpt bot woith message", message); + try { + const response = await fetch(`${endpoint}chat/send-message`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ message: message, origin: "ComfyUI Chat" }), + }); - if (!response.ok) { - throw new Error('Network response was not ok ' + response.statusText); - } - - console.log('response', response); - - const responseData = await response.json(); - - return responseData; - } catch (error) { - console.error('There was a problem with the fetch operation:', error); - throw error; + if (!response.ok) { + throw new Error("Network response was not ok " + response.statusText); } + + console.log("response", response); + + const responseData = await response.json(); + + return responseData; + } catch (error) { + console.error("There was a problem with the fetch operation:", error); + throw error; + } } export function isValidWorkflow(workflow) { - return workflow && workflow.nodes && workflow.edges; -} \ No newline at end of file + return workflow && workflow.nodes && workflow.edges; +} + +export function parseWorkflowFromBot(response) { + const cleanedResponse = response.responses.bot + .replace(/^\s*```json\s*/, "") + .replace(/\s*```\s*$/, ""); + return JSON.parse(cleanedResponse); +} diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index 63a2239..fc8b94a 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -1,6 +1,10 @@ // import { api, app } from './comfy/comfy.js' import { app } from "../comfy/comfy.js"; -import { getBotResponse, isValidWorkflow } from "./chatbot.js"; +import { + getBotResponse, + isValidWorkflow, + parseWorkflowFromBot, +} from "./chatbot.js"; import { runWorkflowLoader, loadGraphFromPrompt } from "./workflow.js"; import { setButtonDefault } from "../button/ui.js"; import { authDialog } from "../auth/index.js"; @@ -14,6 +18,67 @@ const CHAT_MESSAGES_ID = "chat-messages"; const apiToken = getApiToken(); const doesApiTokenExist = !!apiToken; +const toggleChatBox = async () => { + const chatBox = document.getElementById(CHAT_BOX_ID); + if (chatBox.style.display === "none") { + chatBox.style.display = "block"; + } else { + chatBox.style.display = "none"; + } +}; + +const sendMessage = () => { + if (!doesApiTokenExist) { + toggleChatBox(); + setButtonDefault(); + return authDialog.show(); + } + + const chatInput = document.getElementById(CHAT_INPUT_ID); + const chatMessages = document.getElementById(CHAT_MESSAGES_ID); + const message = chatInput.value.trim(); + + if (message) { + const messageElement = document.createElement("div"); + messageElement.innerText = message; + messageElement.style.marginBottom = "10px"; + messageElement.style.padding = "10px"; + messageElement.style.backgroundColor = "#e1ffc7"; + messageElement.style.borderRadius = "4px"; + + chatMessages.appendChild(messageElement); + chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + chatInput.value = ""; + + // get bot response + getBotResponse(message, apiToken) + .then((response) => { + const parsedBotResponse = parseWorkflowFromBot(response); + console.log("got response", parsedBotResponse); + if (isValidWorkflow(parsedBotResponse)) { + console.log("Valid workflow that could be loaded"); + + const botMessageElement = document.createElement("div"); + botMessageElement.innerText = JSON.stringify(parsedBotResponse); + botMessageElement.style.marginBottom = "10px"; + botMessageElement.style.padding = "10px"; + botMessageElement.style.backgroundColor = "#c7e1ff"; + botMessageElement.style.borderRadius = "4px"; + + chatMessages.appendChild(botMessageElement); + chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + + loadGraphFromPrompt(parsedBotResponse); + } else { + console.log("Not a valid workflow that could be loaded"); + } + }) + .catch((error) => { + console.error(error); + }); + } +}; + const createChatButton = () => { const chatButton = document.createElement("div"); chatButton.id = CHAT_BUTTON_ID; @@ -131,70 +196,6 @@ const createChatBox = () => { return chatBox; }; -const sendMessage = () => { - if (!doesApiTokenExist) { - toggleChatBox(); - setButtonDefault(); - return authDialog.show(); - } - - const chatInput = document.getElementById(CHAT_INPUT_ID); - const chatMessages = document.getElementById(CHAT_MESSAGES_ID); - const message = chatInput.value.trim(); - - if (message) { - const messageElement = document.createElement("div"); - messageElement.innerText = message; - messageElement.style.marginBottom = "10px"; - messageElement.style.padding = "10px"; - messageElement.style.backgroundColor = "#e1ffc7"; - messageElement.style.borderRadius = "4px"; - - chatMessages.appendChild(messageElement); - chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom - chatInput.value = ""; - // // get bot response - getBotResponse(message, apiToken) - .then((response) => { - console.log("got response", response); - const botResponse = response.responses.bot; - console.log("ext got rresponse", botResponse); - const cleanedResponse = botResponse.replace(/^\s*```json\s*/, '').replace(/\s*```\s*$/, ''); - - if (isValidWorkflow(JSON.parse(cleanedResponse))) { - console.log("Valid workflow that could be loaded"); - const botMessageElement = document.createElement("div"); - botMessageElement.innerText = botResponse; - - botMessageElement.style.marginBottom = "10px"; - botMessageElement.style.padding = "10px"; - botMessageElement.style.backgroundColor = "#c7e1ff"; - botMessageElement.style.borderRadius = "4px"; - - chatMessages.appendChild(botMessageElement); - chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom - - // runWorkflowLoader(); - loadGraphFromPrompt(JSON.parse(cleanedResponse)); - // addNodesAndEdges(app.graph, response.message); - } else { - console.log("Not a valid workflow that could be loaded"); - } - }) - .catch((error) => { - console.error(error); - }); - } -}; - -const toggleChatBox = async () => { - const chatBox = document.getElementById(CHAT_BOX_ID); - if (chatBox.style.display === "none") { - chatBox.style.display = "block"; - } else { - chatBox.style.display = "none"; - } -}; const registerChat = () => { // With js, append an bottom that opens up a chatbox in the bottom right hand corner From 12a953fa845d6e3974dde1c7317ee049f53e0f89 Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 20:53:06 -0700 Subject: [PATCH 14/19] refactor: conforms chatbot to nombis endpoint builder --- js/chatbot/chatbot.js | 35 +++++++----------------- js/resource/endpoints.js | 58 +++++++++++++++++++++------------------- 2 files changed, 41 insertions(+), 52 deletions(-) diff --git a/js/chatbot/chatbot.js b/js/chatbot/chatbot.js index 3cdc91f..1d204d7 100644 --- a/js/chatbot/chatbot.js +++ b/js/chatbot/chatbot.js @@ -1,31 +1,16 @@ -// import { endpoint } from "../constants"; -const endpoint = "http://localhost:4000/"; // Switch to live as needed +import { nimbus } from "../resource/index.js"; -export async function getBotResponse(message, token) { - console.log("calling the gpt bot woith message", message); - try { - const response = await fetch(`${endpoint}chat/send-message`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - }, - body: JSON.stringify({ message: message, origin: "ComfyUI Chat" }), - }); +export async function getBotResponse(message) { + try { + const data = await nimbus.chatbot.sendMessage({ + message: message, + origin: "ComfyUI Cloud Chat" + }) - if (!response.ok) { - throw new Error("Network response was not ok " + response.statusText); + return data + } catch (e) { + throw new Error(e.message) } - - console.log("response", response); - - const responseData = await response.json(); - - return responseData; - } catch (error) { - console.error("There was a problem with the fetch operation:", error); - throw error; - } } export function isValidWorkflow(workflow) { diff --git a/js/resource/endpoints.js b/js/resource/endpoints.js index c638217..2cf2d3f 100644 --- a/js/resource/endpoints.js +++ b/js/resource/endpoints.js @@ -17,14 +17,14 @@ const workflowRun = { retrieveOutput: { method: "GET", path: "/workflow/{workflow_id}/runs/{run_id}/outputs", - } -} + }, +}; const workflow = { // upload local init: { method: "POST", - path: "/workflow/init" + path: "/workflow/init", }, create: { method: "POST", @@ -40,71 +40,75 @@ const workflow = { }, error: { method: "POST", - path: "/workflow/error" - } -} + path: "/workflow/error", + }, +}; const billing = { retrieveCustomerSession: { method: "GET", - path: "/stripe/get-customer-session" + path: "/stripe/get-customer-session", }, retrieveUsage: { method: "GET", - path: "/stripe/usage" + path: "/stripe/usage", }, retrieveCredits: { method: "GET", - path: "/stripe/credits" + path: "/stripe/credits", }, -} +}; const auth = { register: { method: "POST", - path: "/auth/register" + path: "/auth/register", }, login: { method: "POST", - path: "/auth/login" - } -} + path: "/auth/login", + }, +}; const support = { feedback: { method: "POST", - path: "/support/feedback" + path: "/support/feedback", }, support: { method: "POST", - path: "/support/support" + path: "/support/support", }, docs: { method: "POST", - path: "/support/docs" + path: "/support/docs", }, tooltipHover: { method: "POST", - path: "/support/tooltipHover" + path: "/support/tooltipHover", }, tooltipDocs: { method: "POST", - path: "/support/tooltipDocs" + path: "/support/tooltipDocs", }, assistant: { method: "POST", - path: "/support/assistant" - } -} + path: "/support/assistant", + }, +}; +const chatbot = { + sendMessage: { + method: "POST", + path: "/chat/send-message", + }, +}; export const apiEndpoints = { workflowRun, workflow, billing, auth, - support -} - - - + support, + chatbot +}; From 5c079ab1b151409bf7b5ae3bd0435ca85894b448 Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 20:55:16 -0700 Subject: [PATCH 15/19] refactor: using async to call send message instead of then() --- js/chatbot/ext.js | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index fc8b94a..1c9f407 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -27,7 +27,7 @@ const toggleChatBox = async () => { } }; -const sendMessage = () => { +const sendMessage = async () => { if (!doesApiTokenExist) { toggleChatBox(); setButtonDefault(); @@ -50,35 +50,35 @@ const sendMessage = () => { chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom chatInput.value = ""; - // get bot response - getBotResponse(message, apiToken) - .then((response) => { - const parsedBotResponse = parseWorkflowFromBot(response); - console.log("got response", parsedBotResponse); - if (isValidWorkflow(parsedBotResponse)) { - console.log("Valid workflow that could be loaded"); + try { + const response = await getBotResponse(message); + const parsedBotResponse = parseWorkflowFromBot(response); + console.log("got response", parsedBotResponse); - const botMessageElement = document.createElement("div"); - botMessageElement.innerText = JSON.stringify(parsedBotResponse); - botMessageElement.style.marginBottom = "10px"; - botMessageElement.style.padding = "10px"; - botMessageElement.style.backgroundColor = "#c7e1ff"; - botMessageElement.style.borderRadius = "4px"; + if (isValidWorkflow(parsedBotResponse)) { + console.log("Valid workflow that could be loaded"); - chatMessages.appendChild(botMessageElement); - chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + const botMessageElement = document.createElement("div"); + botMessageElement.innerText = JSON.stringify(parsedBotResponse); + botMessageElement.style.marginBottom = "10px"; + botMessageElement.style.padding = "10px"; + botMessageElement.style.backgroundColor = "#c7e1ff"; + botMessageElement.style.borderRadius = "4px"; - loadGraphFromPrompt(parsedBotResponse); - } else { - console.log("Not a valid workflow that could be loaded"); - } - }) - .catch((error) => { - console.error(error); - }); + chatMessages.appendChild(botMessageElement); + chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + + loadGraphFromPrompt(parsedBotResponse); + } else { + console.log("Not a valid workflow that could be loaded"); + } + } catch (error) { + console.error(error); + } } }; + const createChatButton = () => { const chatButton = document.createElement("div"); chatButton.id = CHAT_BUTTON_ID; From 0725ad5452bfd64203e7ffad41e1e7a0895f4707 Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 21:36:12 -0700 Subject: [PATCH 16/19] adds handling of non workflow responses --- js/chatbot/chatbot.js | 26 ++++++++++++++++---------- js/chatbot/ext.js | 32 ++++++++++++++++++-------------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/js/chatbot/chatbot.js b/js/chatbot/chatbot.js index 1d204d7..7becfe7 100644 --- a/js/chatbot/chatbot.js +++ b/js/chatbot/chatbot.js @@ -1,16 +1,16 @@ import { nimbus } from "../resource/index.js"; export async function getBotResponse(message) { - try { - const data = await nimbus.chatbot.sendMessage({ - message: message, - origin: "ComfyUI Cloud Chat" - }) + try { + const data = await nimbus.chatbot.sendMessage({ + message: message, + origin: "ComfyUI Cloud Chat", + }); - return data - } catch (e) { - throw new Error(e.message) - } + return data; + } catch (e) { + throw new Error(e.message); + } } export function isValidWorkflow(workflow) { @@ -21,5 +21,11 @@ export function parseWorkflowFromBot(response) { const cleanedResponse = response.responses.bot .replace(/^\s*```json\s*/, "") .replace(/\s*```\s*$/, ""); - return JSON.parse(cleanedResponse); + + try { + return JSON.parse(cleanedResponse); + } catch (error) { + console.error("Failed to parse JSON:", error); + return cleanedResponse; + } } diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index 1c9f407..bceeb93 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -52,33 +52,37 @@ const sendMessage = async () => { try { const response = await getBotResponse(message); + console.log("got response", response); + let botMessage = "" + const parsedBotResponse = parseWorkflowFromBot(response); - console.log("got response", parsedBotResponse); + if (isValidWorkflow(parsedBotResponse)) { console.log("Valid workflow that could be loaded"); - - const botMessageElement = document.createElement("div"); - botMessageElement.innerText = JSON.stringify(parsedBotResponse); - botMessageElement.style.marginBottom = "10px"; - botMessageElement.style.padding = "10px"; - botMessageElement.style.backgroundColor = "#c7e1ff"; - botMessageElement.style.borderRadius = "4px"; - - chatMessages.appendChild(botMessageElement); - chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom - - loadGraphFromPrompt(parsedBotResponse); + botMessage = JSON.stringify(parsedBotResponse) } else { console.log("Not a valid workflow that could be loaded"); + botMessage = response.responses.bot } + + const botMessageElement = document.createElement("div"); + botMessageElement.innerText = botMessage; + botMessageElement.style.marginBottom = "10px"; + botMessageElement.style.padding = "10px"; + botMessageElement.style.backgroundColor = "#c7e1ff"; + botMessageElement.style.borderRadius = "4px"; + + chatMessages.appendChild(botMessageElement); + chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom + + loadGraphFromPrompt(parsedBotResponse); } catch (error) { console.error(error); } } }; - const createChatButton = () => { const chatButton = document.createElement("div"); chatButton.id = CHAT_BUTTON_ID; From 0e62c4c01cd9e3e598ed37d075b2ccfa645ad87a Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 21:56:45 -0700 Subject: [PATCH 17/19] refactor: removes test code --- js/chatbot/ext.js | 49 ++++---------------------------- js/chatbot/workflow.js | 63 ------------------------------------------ 2 files changed, 5 insertions(+), 107 deletions(-) diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index bceeb93..8c40561 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -5,7 +5,7 @@ import { isValidWorkflow, parseWorkflowFromBot, } from "./chatbot.js"; -import { runWorkflowLoader, loadGraphFromPrompt } from "./workflow.js"; +import { loadGraphFromPrompt } from "./workflow.js"; import { setButtonDefault } from "../button/ui.js"; import { authDialog } from "../auth/index.js"; import { getApiToken } from "../utils.js"; @@ -53,17 +53,16 @@ const sendMessage = async () => { try { const response = await getBotResponse(message); console.log("got response", response); - let botMessage = "" - + let botMessage = ""; + const parsedBotResponse = parseWorkflowFromBot(response); - if (isValidWorkflow(parsedBotResponse)) { console.log("Valid workflow that could be loaded"); - botMessage = JSON.stringify(parsedBotResponse) + botMessage = JSON.stringify(parsedBotResponse); } else { console.log("Not a valid workflow that could be loaded"); - botMessage = response.responses.bot + botMessage = response.responses.bot; } const botMessageElement = document.createElement("div"); @@ -160,44 +159,6 @@ const createChatBox = () => { chatInputContainer.appendChild(chatSendButton); chatBox.appendChild(chatInputContainer); - // New button section - const newButton = document.createElement("button"); - newButton.innerText = "Run test fxn"; - newButton.style.position = "absolute"; - newButton.style.top = "10px"; - newButton.style.right = "10px"; - newButton.style.padding = "10px"; - newButton.style.backgroundColor = "blue"; - newButton.style.color = "white"; - newButton.style.border = "none"; - newButton.style.borderRadius = "4px"; - newButton.style.cursor = "pointer"; - - newButton.addEventListener("click", () => { - runWorkflowLoader(); - }); - - // log button - const otherButton = document.createElement("button"); - otherButton.innerText = "Log graph"; - otherButton.style.position = "absolute"; - otherButton.style.top = "100px"; - otherButton.style.right = "10px"; - otherButton.style.padding = "10px"; - otherButton.style.backgroundColor = "blue"; - otherButton.style.color = "white"; - otherButton.style.border = "none"; - otherButton.style.borderRadius = "4px"; - otherButton.style.cursor = "pointer"; - - otherButton.addEventListener("click", () => { - console.log(app.graph); - app.graph.change(); - }); - - chatBox.appendChild(newButton); - chatBox.appendChild(otherButton); - return chatBox; }; diff --git a/js/chatbot/workflow.js b/js/chatbot/workflow.js index caafb55..d25452b 100644 --- a/js/chatbot/workflow.js +++ b/js/chatbot/workflow.js @@ -1,59 +1,6 @@ import { app } from "../comfy/comfy.js"; import { getInputType } from "./edgeTypes.js"; -const newSampleInput = { - nodes: [ - { - id: 4, - class_type: "CheckpointLoaderSimple", - widget_values: { checkpoint: "v1-5-pruned-emaonly.safetensors" }, - }, - { - id: 5, - class_type: "EmptyLatentImage", - widget_values: { width: 512, height: 512, channels: 1 }, - }, - { - id: 6, - class_type: "CLIPTextEncode", - widget_values: { - text: "beautiful scenery nature glass bottle landscape, , purple galaxy bottle,", - }, - }, - { - id: 7, - class_type: "CLIPTextEncode", - widget_values: { text: "text, watermark" }, - }, - { - id: 3, - class_type: "KSampler", - widget_values: { - seed: 156680208700286, - continuous: true, - steps: 20, - denoising: 8, - scheduler: "euler", - noise: "normal", - scale: 1, - }, - }, - { id: 8, class_type: "VAEDecode" }, - { id: 9, class_type: "SaveImage" }, - ], - edges: [ - { from: 4, to: 3, from_output: 0, to_input: "model" }, - { from: 5, to: 3, from_output: 0, to_input: "latent_image" }, - { from: 4, to: 6, from_output: 1, to_input: "clip" }, - { from: 6, to: 3, from_output: 0, to_input: "positive" }, - { from: 7, to: 3, from_output: 0, to_input: "negative" }, - { from: 4, to: 7, from_output: 1, to_input: "clip" }, - { from: 3, to: 8, from_output: 0, to_input: "samples" }, - { from: 4, to: 8, from_output: 2, to_input: "vae" }, - { from: 8, to: 9, from_output: 0, to_input: "images" }, - ], -}; - function getInputIndex(node, inputName) { return node.inputs.findIndex( (input) => input.name === getInputType(inputName) @@ -185,13 +132,3 @@ export function loadGraphFromPrompt(generatedWorkflow) { app.loadGraphData(translatedData, true); app.graph.change(); } - -export function runWorkflowLoader() { - console.log("Initial graph"); - console.log(app.graph); - - loadGraphFromPrompt(newSampleInput); - - console.log("Graph after change"); - console.log(app.graph); -} From 65cf0f6f8a17c912947390f94054c317b1b2fe62 Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 22:45:06 -0700 Subject: [PATCH 18/19] adds cleaner UI --- js/chatbot/ext.js | 256 +++++++++++++++++++++++++++++++--------------- 1 file changed, 174 insertions(+), 82 deletions(-) diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index 8c40561..90156a4 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -14,10 +14,13 @@ const CHAT_BUTTON_ID = "chat-button"; const CHAT_BOX_ID = "chat-box"; const CHAT_INPUT_ID = "chat-input"; const CHAT_MESSAGES_ID = "chat-messages"; +const CHAT_SEND_BUTTON_ID = "chat-send-button"; const apiToken = getApiToken(); const doesApiTokenExist = !!apiToken; +let canSendMessage = true; + const toggleChatBox = async () => { const chatBox = document.getElementById(CHAT_BOX_ID); if (chatBox.style.display === "none") { @@ -27,6 +30,159 @@ const toggleChatBox = async () => { } }; +const createChatButton = () => { + const chatButton = document.createElement("div"); + chatButton.id = CHAT_BUTTON_ID; + + // Set up the button styles and gradient background + chatButton.style.position = "fixed"; + chatButton.style.bottom = "0"; + chatButton.style.right = "0"; + chatButton.style.margin = "20px"; + chatButton.style.background = "linear-gradient(160deg, #8A2BE2, #1E90FF)"; // Gradient background + chatButton.style.color = "white"; + chatButton.style.borderRadius = "50%"; + chatButton.style.width = "60px"; + chatButton.style.height = "60px"; + chatButton.style.zIndex = 9999; + chatButton.style.display = "flex"; + chatButton.style.alignItems = "center"; + chatButton.style.justifyContent = "center"; + chatButton.style.cursor = "pointer"; + + // Create and append SVG chat bubble icon + const chatIcon = document.createElementNS( + "http://www.w3.org/2000/svg", + "svg" + ); + chatIcon.setAttribute("viewBox", "0 0 512 512"); + chatIcon.setAttribute("width", "20"); // Adjust size as needed + chatIcon.setAttribute("height", "20"); // Adjust size as needed + chatIcon.setAttribute("fill", "currentColor"); // Use current text color + + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute( + "d", + "M416,0H96C43.072,0,0,43.072,0,96v405.333c0,4.48,2.816,8.491,7.04,10.027c1.195,0.427,2.411,0.64,3.627,0.64c3.115,0,6.123-1.344,8.192-3.84L122.325,384H416c52.928,0,96-43.072,96-96V96C512,43.072,468.928,0,416,0z" + ); + chatIcon.appendChild(path); + + chatButton.appendChild(chatIcon); + + return chatButton; +}; + +const createChatBox = () => { + const chatBox = document.createElement("div"); + chatBox.id = CHAT_BOX_ID; + chatBox.style.position = "fixed"; + chatBox.style.bottom = "100px"; + chatBox.style.right = "0"; + chatBox.style.backgroundColor = "white"; + chatBox.style.width = "300px"; + chatBox.style.height = "500px"; + chatBox.style.zIndex = 9999; + chatBox.style.borderRadius = "10px"; + chatBox.style.display = "none"; // Initially hidden + + // Messages section + const chatMessages = document.createElement("div"); + chatMessages.id = CHAT_MESSAGES_ID; + chatMessages.style.height = "80%"; + chatMessages.style.overflowY = "auto"; + chatMessages.style.padding = "10px"; + chatMessages.style.color = "black"; + chatBox.appendChild(chatMessages); + // Input section + const chatInputContainer = document.createElement("div"); + chatInputContainer.style.position = "absolute"; + chatInputContainer.style.bottom = "0"; + chatInputContainer.style.width = "100%"; + chatInputContainer.style.padding = "10px"; + chatInputContainer.style.boxSizing = "border-box"; + chatInputContainer.style.backgroundColor = "#f1f1f1"; + chatInputContainer.style.borderBottomLeftRadius = "10px"; + chatInputContainer.style.borderBottomRightRadius = "10px"; + + // Flexbox layout for horizontal alignment + chatInputContainer.style.display = "flex"; + chatInputContainer.style.alignItems = "center"; + + // Input field + const chatInput = document.createElement("input"); + chatInput.id = CHAT_INPUT_ID; + chatInput.type = "text"; + chatInput.style.flex = "1"; // Allow the input to grow and take up available space + chatInput.style.padding = "10px"; + chatInput.style.boxSizing = "border-box"; + chatInput.style.border = "1px solid #ccc"; + chatInput.style.borderRadius = "4px"; + chatInput.placeholder = "Describe your desired workflow..."; + + // Send button + const chatSendButton = document.createElement("button"); + chatSendButton.id = CHAT_SEND_BUTTON_ID; + chatSendButton.style.width = "40px"; + chatSendButton.style.height = "40px"; + chatSendButton.style.marginLeft = "10px"; + chatSendButton.style.padding = "0"; // Remove default padding + chatSendButton.style.border = "none"; + chatSendButton.style.backgroundColor = "#007acc"; + chatSendButton.style.color = "white"; + chatSendButton.style.cursor = "pointer"; + chatSendButton.style.borderRadius = "50%"; + chatSendButton.style.display = "flex"; + chatSendButton.style.alignItems = "center"; + chatSendButton.style.justifyContent = "center"; + + // Create and style the SVG icon + const sendIcon = document.createElementNS( + "http://www.w3.org/2000/svg", + "svg" + ); + sendIcon.setAttribute("fill", "#ffffff"); // Use white color for the icon + sendIcon.setAttribute("viewBox", "0 0 52 52"); + sendIcon.setAttribute("enable-background", "new 0 0 52 52"); + sendIcon.setAttribute("xmlns", "http://www.w3.org/2000/svg"); + sendIcon.setAttribute("class", "send-icon"); + sendIcon.setAttribute("xml:space", "preserve"); + sendIcon.setAttribute("width", "18px"); // Adjust size as needed + sendIcon.setAttribute("height", "18px"); // Adjust size as needed + + // Create the path element for the SVG + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute( + "d", + "M2.1,44.5l4.4-16.3h18.6c0.5,0,1-0.5,1-1v-2c0-0.5-0.5-1-1-1H6.5l-4.3-16l0,0C2.1,8,2,7.7,2,7.4C2,6.7,2.7,6,3.5,6.1c0.2,0,0.3,0.1,0.5,0.1l0,0l0,0l0,0l0,0l45,18.5c0.6,0.2,1,0.8,1,1.4s-0.4,1.1-0.9,1.3l0,0L4,46.4l0,0c-0.2,0.1-0.4,0.1-0.6,0.1C2.6,46.4,2,45.8,2,45C2,44.8,2,44.7,2.1,44.5L2.1,44.5z" + ); + sendIcon.appendChild(path); + + // Append SVG to button + chatSendButton.appendChild(sendIcon); + + // Add event listener + chatSendButton.addEventListener("click", sendMessage); + + // Append input and button to container + chatInputContainer.appendChild(chatInput); + chatInputContainer.appendChild(chatSendButton); + chatBox.appendChild(chatInputContainer); + + return chatBox; +}; + +const updateMessageBoxStatus = (canSendMessage) => { + const button = document.getElementById(CHAT_SEND_BUTTON_ID); + if (button) { + button.style.backgroundColor = canSendMessage ? "#007acc" : "#cccccc"; // Change color based on `isActive` + } + const input = document.getElementById(CHAT_INPUT_ID); + if (input) { + input.placeholder = canSendMessage ? "Describe your desired workflow..." : "Generating workflow..."; + input.disabled = !canSendMessage; + } +}; + const sendMessage = async () => { if (!doesApiTokenExist) { toggleChatBox(); @@ -39,11 +195,18 @@ const sendMessage = async () => { const message = chatInput.value.trim(); if (message) { + if (!canSendMessage) { + return; + } else { + canSendMessage = false; + updateMessageBoxStatus(canSendMessage); + } + const messageElement = document.createElement("div"); messageElement.innerText = message; messageElement.style.marginBottom = "10px"; messageElement.style.padding = "10px"; - messageElement.style.backgroundColor = "#e1ffc7"; + messageElement.style.backgroundColor = "#A7DBFD"; messageElement.style.borderRadius = "4px"; chatMessages.appendChild(messageElement); @@ -69,8 +232,14 @@ const sendMessage = async () => { botMessageElement.innerText = botMessage; botMessageElement.style.marginBottom = "10px"; botMessageElement.style.padding = "10px"; - botMessageElement.style.backgroundColor = "#c7e1ff"; + botMessageElement.style.backgroundColor = "#DFE8EE"; botMessageElement.style.borderRadius = "4px"; + botMessageElement.style.wordWrap = "break-word"; // Wrap long words + botMessageElement.style.overflow = "hidden"; // Hide overflow text + botMessageElement.style.textOverflow = "ellipsis"; // Show ellipsis if text overflows + + // Optionally, set a max-width if you want to control the width of the message box + botMessageElement.style.maxWidth = "calc(100% - 20px)"; // Adjust max-width based on container width and padding chatMessages.appendChild(botMessageElement); chatMessages.scrollTop = chatMessages.scrollHeight; // Scroll to the bottom @@ -78,90 +247,13 @@ const sendMessage = async () => { loadGraphFromPrompt(parsedBotResponse); } catch (error) { console.error(error); + } finally { + canSendMessage = true; + updateMessageBoxStatus(canSendMessage); } } }; -const createChatButton = () => { - const chatButton = document.createElement("div"); - chatButton.id = CHAT_BUTTON_ID; - chatButton.innerHTML = `
`; - chatButton.style.position = "fixed"; - chatButton.style.bottom = "0"; - chatButton.style.right = "0"; - chatButton.style.backgroundColor = "green"; - chatButton.style.color = "white"; - chatButton.style.borderRadius = "50%"; - chatButton.style.width = "100px"; - chatButton.style.height = "100px"; - chatButton.style.zIndex = 9999; - chatButton.style.display = "flex"; - chatButton.style.alignItems = "center"; - chatButton.style.justifyContent = "center"; - chatButton.style.cursor = "pointer"; - chatButton.innerText = "Chat"; - return chatButton; -}; - -const createChatBox = () => { - const chatBox = document.createElement("div"); - chatBox.id = CHAT_BOX_ID; - chatBox.style.position = "fixed"; - chatBox.style.bottom = "100px"; - chatBox.style.right = "0"; - chatBox.style.backgroundColor = "white"; - chatBox.style.width = "300px"; - chatBox.style.height = "500px"; - chatBox.style.zIndex = 9999; - chatBox.style.display = "none"; // Initially hidden - - // Messages section - const chatMessages = document.createElement("div"); - chatMessages.id = CHAT_MESSAGES_ID; - chatMessages.style.height = "80%"; - chatMessages.style.overflowY = "auto"; - chatMessages.style.padding = "10px"; - chatMessages.style.color = "black"; - chatBox.appendChild(chatMessages); - - // Input section - const chatInputContainer = document.createElement("div"); - chatInputContainer.style.position = "absolute"; - chatInputContainer.style.bottom = "0"; - chatInputContainer.style.width = "100%"; - chatInputContainer.style.padding = "10px"; - chatInputContainer.style.boxSizing = "border-box"; - chatInputContainer.style.backgroundColor = "#f1f1f1"; - - const chatInput = document.createElement("input"); - chatInput.id = CHAT_INPUT_ID; - chatInput.type = "text"; - chatInput.style.width = "calc(100% - 60px)"; // Leave space for the send button - chatInput.style.padding = "10px"; - chatInput.style.boxSizing = "border-box"; - chatInput.style.border = "1px solid #ccc"; - chatInput.style.borderRadius = "4px"; - - const chatSendButton = document.createElement("button"); - chatSendButton.innerText = "Send"; - chatSendButton.style.width = "50px"; - chatSendButton.style.marginLeft = "10px"; - chatSendButton.style.padding = "10px"; - chatSendButton.style.border = "none"; - chatSendButton.style.backgroundColor = "green"; - chatSendButton.style.color = "white"; - chatSendButton.style.borderRadius = "4px"; - chatSendButton.style.cursor = "pointer"; - - chatSendButton.addEventListener("click", sendMessage); - - chatInputContainer.appendChild(chatInput); - chatInputContainer.appendChild(chatSendButton); - chatBox.appendChild(chatInputContainer); - - return chatBox; -}; - const registerChat = () => { // With js, append an bottom that opens up a chatbox in the bottom right hand corner const chatButton = createChatButton(); From c9974e7c9bfd4cbf9bcc3221099b93eba4a45bd7 Mon Sep 17 00:00:00 2001 From: vindennt Date: Fri, 19 Jul 2024 22:46:23 -0700 Subject: [PATCH 19/19] removes unecesary console logs --- js/chatbot/ext.js | 3 --- js/chatbot/workflow.js | 18 ------------------ 2 files changed, 21 deletions(-) diff --git a/js/chatbot/ext.js b/js/chatbot/ext.js index 90156a4..ce9010f 100644 --- a/js/chatbot/ext.js +++ b/js/chatbot/ext.js @@ -215,16 +215,13 @@ const sendMessage = async () => { try { const response = await getBotResponse(message); - console.log("got response", response); let botMessage = ""; const parsedBotResponse = parseWorkflowFromBot(response); if (isValidWorkflow(parsedBotResponse)) { - console.log("Valid workflow that could be loaded"); botMessage = JSON.stringify(parsedBotResponse); } else { - console.log("Not a valid workflow that could be loaded"); botMessage = response.responses.bot; } diff --git a/js/chatbot/workflow.js b/js/chatbot/workflow.js index d25452b..3ad7155 100644 --- a/js/chatbot/workflow.js +++ b/js/chatbot/workflow.js @@ -20,8 +20,6 @@ function transformInputToGraph(input) { version: 0.4, }; - console.log("baseGraph"); - console.log(baseGraph); const xStep = 300; // step for node positions const yStep = 300; // step for node positions @@ -60,9 +58,6 @@ function transformInputToGraph(input) { columnCounter++; }); - console.log("baseGraph after nodes insertion"); - console.log(baseGraph); - // Insert edges input.edges.forEach((edgeData, index) => { const fromNode = nodesById[edgeData.from]; @@ -82,9 +77,6 @@ function transformInputToGraph(input) { }); } - console.log("toNode"); - console.log(toNode); - // add outputs fromNode.outputs.push({ name: getInputType(edgeData.to_input), @@ -98,9 +90,6 @@ function transformInputToGraph(input) { outputIndex = fromNode.outputs.length - 1; inputIndex = fromNode.inputs.length - 1; - console.log("fromNode"); - console.log(fromNode); - // add links const link = [ index + 1, // Link id @@ -112,23 +101,16 @@ function transformInputToGraph(input) { getInputType(edgeData.to_input), ]; - console.log("link"); - console.log(link); - fromNode.outputs[outputIndex].links.push(link[0]); baseGraph.links.push(link); }); - console.log("baseGraph after edges insertion"); - console.log(baseGraph); return baseGraph; } export function loadGraphFromPrompt(generatedWorkflow) { const translatedData = transformInputToGraph(generatedWorkflow); - // console.log("Translated Graph Data: "); - // console.log(translatedData); app.loadGraphData(translatedData, true); app.graph.change(); }