diff --git a/README.md b/README.md index ea84326..8254b7c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ComfyUI Griptape Nodes -This repo creates a series of nodes that enable you to utilize the [Griptape Python Framework](https://github.com/griptape-ai/griptape/) with [ComfyUI](https://github.com/comfyanonymous/ComfyUI), integrating AI into your workflow. +This repo creates a series of nodes that enable you to utilize the [Griptape Python Framework](https://github.com/griptape-ai/griptape/) with [ComfyUI](https://github.com/comfyanonymous/ComfyUI), integrating LLMs (Large Language Models) and AI into your workflow. This example creates two [Griptape Agents](https://docs.griptape.ai/stable/griptape-framework/structures/agents/) - a Writer and an Editor. The writer has a [WebScraper](https://docs.griptape.ai/stable/griptape-tools/official-tools/web-scraper/) tool and gets a description of a website. It then generates blog topic ideas and passes them to_ the Editor Agent. The editor chooses the top 3 topics, and generates outlines and next actions for the next two. A third Agent generates thumbnail images for each blog post using a [Prompt Image Generation Task](https://docs.griptape.ai/stable/griptape-framework/structures/tasks/#prompt-image-generation-task) based on the outlines. @@ -20,6 +20,7 @@ The repo currently has a subset of Griptape nodes, with more to come soon. Curre * [DateTime](https://docs.griptape.ai/stable/griptape-tools/official-tools/date-time/) * [WebScraper](https://docs.griptape.ai/stable/griptape-tools/official-tools/web-scraper) * [FileManager](https://docs.griptape.ai/stable/griptape-tools/official-tools/file-manager) + * [AudioTranscriptionClient](https://docs.griptape.ai/stable/griptape-tools/official-tools/audio-transcription-client/) * [GriptapeCloudKnowledgeBaseClient](https://docs.griptape.ai/stable/reference/griptape/tools/#griptape.tools.GriptapeCloudKnowledgeBaseClient) * Run specific Agent Tasks: @@ -47,6 +48,7 @@ In this example, we're using three `Image Description` nodes to describe the giv ## Using the nodes - Video Tutorials 1. Part 1: Installation: https://youtu.be/L4-HnKH4BSI?si=Q7IqP-KnWug7JJ5s 2. Part 2: Griptape Agents: https://youtu.be/wpQCciNel_A?si=WF_EogiZRGy0cQIm +3. Part 3: Controlling which LLM your Agents use: https://youtu.be/JlPuyH5Ot5I?si=KMPjwN3wn4L4rUyg ## Installation diff --git a/__init__.py b/__init__.py index 40df0e8..7a3767b 100644 --- a/__init__.py +++ b/__init__.py @@ -12,6 +12,8 @@ from dotenv import load_dotenv # Load the griptape_config.json data from .nodes.agent import CreateAgent, ExpandAgent, RunAgent +from .nodes.audio_drivers import gtUIOpenAiAudioTranscriptionDriver +from .nodes.audio_nodes import gtUILoadAudio from .nodes.combine_nodes import MergeTexts, ToolList from .nodes.config_nodes import ( gtUIAmazonBedrockStructureConfig, @@ -32,6 +34,7 @@ from .nodes.image_nodes import ( ) from .nodes.rules import gtUIRule from .nodes.tasks import ( + gtUIAudioTranscriptionTask, gtUIImageQueryTask, gtUIParallelImageQueryTask, gtUIPromptImageGenerationTask, @@ -47,6 +50,7 @@ from .nodes.text_nodes import ( gtUITextToClipEncode, ) from .nodes.tools import ( + gtUIAudioTranscriptionClient, gtUICalculator, gtUIDateTime, gtUIFileManager, @@ -103,11 +107,14 @@ NODE_CLASS_MAPPINGS = { "DateTime": gtUIDateTime, "WebScraper": gtUIWebScraper, "gtUIFileManager": gtUIFileManager, + "gtAudioTranscriptionClient": gtUIAudioTranscriptionClient, "ToolList": ToolList, "MergeTexts": MergeTexts, "PromptTask": gtUIPromptTask, "ToolTask": gtUIToolTask, "ToolkitTask": gtUIToolkitTask, + "AudioTranscriptionTask": gtUIAudioTranscriptionTask, + "gtUIOpenAiAudioTranscriptionDriver": gtUIOpenAiAudioTranscriptionDriver, "gtUIKnowledgeBaseTool": gtUIKnowledgeBaseTool, "gtUIWebSearch": gtUIWebSearch, "ImageQueryTask": gtUIImageQueryTask, @@ -117,6 +124,7 @@ NODE_CLASS_MAPPINGS = { "TextSummaryTask": gtUITextSummaryTask, "Rule": gtUIRule, "EnvironmentConfig": gtUIEnv, + "gtUILoadAudio": gtUILoadAudio, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -129,7 +137,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "PromptImageGenerationTask": "Griptape Create: Image from Text", "PromptImageVariationTask": "Griptape Create: Image Variation", "Rule": "Griptape Create: Rules", - "gtUIOpenAiImageGenerationDriver": "Griptape Driver: OpenAI", + "gtUIOpenAiAudioTranscriptionDriver": "Griptape Driver: OpenAI", "gtUIAmazonBedrockStableDiffusionImageGenerationDriver": "Griptape Driver: Amazon Bedrock Stable Diffusion", "gtUIAmazonBedrockTitanImageGenerationDriver": "Griptape Driver: Amazon Bedrock Titan", "gtUILeonardoImageGenerationDriver": "Griptape Driver: Leonardo.AI", @@ -141,6 +149,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ToolTask": "Griptape Run: Tool Task", "ToolkitTask": "Griptape Run: Toolkit Task", "TextSummaryTask": "Griptape Run: Text Summary", + "AudioTranscriptionTask": "Griptape Run: Audio Transcription", "ExpandAgent": "Griptape Expand: Agent Nodes", "gtUIOpenAiStructureConfig": "Griptape Agent Config: OpenAI", "gtUIAmazonBedrockStructureConfig": "Griptape Agent Config: Amazon Bedrock", @@ -150,11 +159,14 @@ NODE_DISPLAY_NAME_MAPPINGS = { "DateTime": "Griptape Tool: DateTime", "WebScraper": "Griptape Tool: WebScraper", "gtUIFileManager": "Griptape Tool: FileManager", + "gtAudioTranscriptionClient": "Griptape Tool: Audio Transcription", "gtUIKnowledgeBaseTool": "Griptape Tool: Griptape Cloud KnowledgeBase", "gtUIWebSearch": "Griptape Tool: WebSearch", "ToolList": "Griptape Combine: Tool List", "MergeTexts": "Griptape Combine: Merge Texts", "EnvironmentConfig": "Griptape Config: Environment Variables", + "gtUIOpenAiImageGenerationDriver": "Griptape Audio Driver: OpenAI", + "gtUILoadAudio": "Griptape Load: Audio", } __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] diff --git a/js/gtUINodes.js b/js/gtUINodes.js index b1c7a4a..1202e1a 100644 --- a/js/gtUINodes.js +++ b/js/gtUINodes.js @@ -1,9 +1,58 @@ import { app } from "../../../scripts/app.js"; +import { api } from "../../../scripts/api.js"; import { $el } from "../../../scripts/ui.js"; import { ComfyWidgets } from "../../../scripts/widgets.js"; import { ComfyDialog } from "../../../scripts/ui/dialog.js"; import { GriptapeConfigDialog } from "./gtUIConfigDialog.js"; +function chainCallback(object, property, callback) { + if (object == undefined) { + //This should not happen. + console.error("Tried to add callback to non-existant object") + return; + } + if (property in object) { + const callback_orig = object[property] + object[property] = function () { + const r = callback_orig.apply(this, arguments); + callback.apply(this, arguments); + return r + }; + } else { + object[property] = callback; + } +} + +async function uploadFile(file) { + //TODO: Add uploaded file to cache with Cache.put()? + try { + // Wrap file in formdata so it includes filename + const body = new FormData(); + const i = file.webkitRelativePath.lastIndexOf('/'); + const subfolder = file.webkitRelativePath.slice(0,i+1) + const new_file = new File([file], file.name, { + type: file.type, + lastModified: file.lastModified, + }); + body.append("image", new_file); + if (i > 0) { + body.append("subfolder", subfolder); + } + const resp = await api.fetchApi("/upload/image", { + method: "POST", + body, + }); + + if (resp.status === 200) { + return resp.status + } else { + alert(resp.status + " - " + resp.statusText); + } + } catch (error) { + alert(error); + } +} + class GriptapeNodes extends EventTarget { constructor() { super(); @@ -119,7 +168,104 @@ function getColor(category) { return "#171717"; } } +function gtUIAddUploadWidget(nodeType, nodeData, widgetName, type="audio") { + chainCallback(nodeType.prototype, "onNodeCreated", function() { + const pathWidget = this.widgets.find((w) => w.name === widgetName); + const fileInput = document.createElement("input"); + chainCallback(this, "onRemoved", () => { + fileInput?.remove(); + }); + if (type == "folder") { + Object.assign(fileInput, { + type: "file", + style: "display: none", + webkitdirectory: true, + onchange: async () => { + const directory = fileInput.files[0].webkitRelativePath; + const i = directory.lastIndexOf('/'); + if (i <= 0) { + throw "No directory found"; + } + const path = directory.slice(0,directory.lastIndexOf('/')) + if (pathWidget.options.values.includes(path)) { + alert("A folder of the same name already exists"); + return; + } + let successes = 0; + for(const file of fileInput.files) { + if (await uploadFile(file) == 200) { + successes++; + } else { + //Upload failed, but some prior uploads may have succeeded + //Stop future uploads to prevent cascading failures + //and only add to list if an upload has succeeded + if (successes > 0) { + break + } else { + return; + } + } + } + pathWidget.options.values.push(path); + pathWidget.value = path; + if (pathWidget.callback) { + pathWidget.callback(path) + } + }, + }); + } else if (type == "video") { + Object.assign(fileInput, { + type: "file", + accept: "video/webm,video/mp4,video/mkv,image/gif", + style: "display: none", + onchange: async () => { + if (fileInput.files.length) { + if (await uploadFile(fileInput.files[0]) != 200) { + //upload failed and file can not be added to options + return; + } + const filename = fileInput.files[0].name; + pathWidget.options.values.push(filename); + pathWidget.value = filename; + if (pathWidget.callback) { + pathWidget.callback(filename) + } + } + }, + }); + } else if (type == "audio") { + Object.assign(fileInput, { + type: "file", + accept: "audio/mpeg,audio/wav,audio/x-wav,audio/ogg", + style: "display: none", + onchange: async () => { + if (fileInput.files.length) { + if (await uploadFile(fileInput.files[0]) != 200) { + //upload failed and file can not be added to options + return; + } + const filename = fileInput.files[0].name; + pathWidget.options.values.push(filename); + pathWidget.value = filename; + if (pathWidget.callback) { + pathWidget.callback(filename) + } + } + }, + }); + }else { + throw "Unknown upload type" + } + document.body.append(fileInput); + let uploadWidget = this.addWidget("button", "choose " + type + " to upload", "image", () => { + //clear the active click event + app.canvas.node_widget = null + fileInput.click(); + }); + uploadWidget.options.serialize = false; + }); +} app.registerExtension({ name: "comfy.gtUI", @@ -141,6 +287,10 @@ app.registerExtension({ // return r; // }; // } + if (nodeData.name === "gtUILoadAudio") { + gtUIAddUploadWidget(nodeType, nodeData, "audio", "audio") + console.log("I found it!") + } if (nodeData.name === "gtUIOutputStringNode") { const onNodeCreated = nodeType.prototype.onNodeCreated; nodeType.prototype.onNodeCreated = function () { diff --git a/nodes/audio_drivers.py b/nodes/audio_drivers.py new file mode 100644 index 0000000..c982010 --- /dev/null +++ b/nodes/audio_drivers.py @@ -0,0 +1,29 @@ +from griptape.drivers import OpenAiAudioTranscriptionDriver + +from .base_driver import gtUIBaseAudioTranscriptionDriver + +openAiAudioTranscriptionModels = ["whisper-1"] + + +class gtUIOpenAiAudioTranscriptionDriver(gtUIBaseAudioTranscriptionDriver): + @classmethod + def INPUT_TYPES(s): + models = openAiAudioTranscriptionModels + + inputs = super().INPUT_TYPES() + inputs["required"].update( + { + "model": (models, {"default": models[0]}), + } + ) + return inputs + + RETURN_NAMES = ("DRIVER",) + RETURN_TYPES = ("DRIVER",) + + def create( + self, + model, + ): + driver = OpenAiAudioTranscriptionDriver(model=model) + return (driver,) diff --git a/nodes/audio_nodes.py b/nodes/audio_nodes.py new file mode 100644 index 0000000..fb36276 --- /dev/null +++ b/nodes/audio_nodes.py @@ -0,0 +1,28 @@ +import os + +import folder_paths + + +class gtUILoadAudio: + @classmethod + def INPUT_TYPES(s): + input_dir = folder_paths.get_input_directory() + files = [ + f + for f in os.listdir(input_dir) + if os.path.isfile(os.path.join(input_dir, f)) + ] + return { + "required": {"audio": (sorted(files), {"audio_upload": True})}, + } + + CATEGORY = "Griptape/Audio" + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("AUDIO PATH",) + FUNCTION = "load_audio" + + def load_audio(self, audio): + audio_path = folder_paths.get_annotated_filepath(audio) + + return (audio_path,) diff --git a/nodes/base_audio_task.py b/nodes/base_audio_task.py new file mode 100644 index 0000000..2c098b0 --- /dev/null +++ b/nodes/base_audio_task.py @@ -0,0 +1,35 @@ +from griptape.drivers import OpenAiAudioTranscriptionDriver + +from .base_task import gtUIBaseTask + + +class gtUIBaseAudioTask(gtUIBaseTask): + @classmethod + def INPUT_TYPES(cls): + inputs = super().INPUT_TYPES() + + inputs["optional"].update( + { + "audio": ( + "STRING", + { + "forceInput": True, + }, + ), + }, + ) + inputs["optional"].update({"driver": ("DRIVER",)}) + del inputs["optional"]["input_string"] + del inputs["optional"]["agent"] + del inputs["required"]["STRING"] + return inputs + + CATEGORY = "Griptape/Audio" + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("OUTPUT",) + + def run(self, audio, driver=None): + if not driver: + driver = OpenAiAudioTranscriptionDriver(model="whisper-1") + output = "Output" + return (output,) diff --git a/nodes/base_driver.py b/nodes/base_driver.py index 9d8b6d0..dce8b7d 100644 --- a/nodes/base_driver.py +++ b/nodes/base_driver.py @@ -1,6 +1,26 @@ -from griptape.drivers import ( - OpenAiImageGenerationDriver, -) +from griptape.drivers import OpenAiAudioTranscriptionDriver, OpenAiImageGenerationDriver + + +class gtUIBaseAudioTranscriptionDriver: + @classmethod + def INPUT_TYPES(s): + return { + "required": {}, + "optional": {}, + } + + RETURN_TYPES = ("DRIVER",) + RETURN_NAMES = ("DRIVER",) + + FUNCTION = "create" + + CATEGORY = "Griptape/Audio Drivers" + + def create( + self, + ): + driver = OpenAiAudioTranscriptionDriver(model="whisper-1") + return (driver,) class gtUIBaseImageGenerationDriver: diff --git a/nodes/tasks.py b/nodes/tasks.py index 86e7fac..38b9c48 100644 --- a/nodes/tasks.py +++ b/nodes/tasks.py @@ -2,22 +2,25 @@ import base64 import os from textwrap import dedent -import folder_paths from griptape.artifacts import BaseArtifact, TextArtifact from griptape.drivers import ( AmazonBedrockImageQueryDriver, AnthropicImageQueryDriver, + DummyAudioTranscriptionDriver, DummyImageQueryDriver, + OpenAiAudioTranscriptionDriver, OpenAiImageGenerationDriver, ) from griptape.engines import ( + AudioTranscriptionEngine, ImageQueryEngine, PromptImageGenerationEngine, VariationImageGenerationEngine, ) -from griptape.loaders import ImageLoader -from griptape.structures import Agent, Workflow +from griptape.loaders import AudioLoader, ImageLoader +from griptape.structures import Agent, Pipeline, Workflow from griptape.tasks import ( + AudioTranscriptionTask, CodeExecutionTask, CsvExtractionTask, ImageQueryTask, @@ -29,9 +32,13 @@ from griptape.tasks import ( ToolTask, VariationImageGenerationTask, ) +from griptape.utils import load_file from schema import Schema +import folder_paths + from ..py.griptape_config import get_config +from .base_audio_task import gtUIBaseAudioTask from .base_image_task import gtUIBaseImageTask from .base_task import gtUIBaseTask from .utilities import convert_tensor_to_base_64, image_path_to_output @@ -253,6 +260,55 @@ class gtUIPromptImageVariationTask(gtUIBaseImageTask): return (output_image, image_path) +class gtUIAudioTranscriptionTask(gtUIBaseAudioTask): + CATEGORY = "Griptape/Audio" + + def run(self, audio, driver=None): + try: + audio_artifact = AudioLoader().load(load_file(audio)) + except Exception as e: + print(f"Error loading audio file: {e}") + if audio_artifact: + if not driver: + audio_transcription_driver = OpenAiAudioTranscriptionDriver( + model="whisper-1" + ) + else: + audio_transcription_driver = driver + + # If the driver is a DummyAudioTranscriptionDriver we'll return a nice error message + if isinstance(audio_transcription_driver, DummyAudioTranscriptionDriver): + return ( + dedent( + """ + I'm sorry, this agent doesn't have access to a valid AudioTranscriptionDriver. + You might want to try using a different Agent Configuration. + + Reach out for help on Discord (https://discord.gg/gnWRz88eym) if you would like some help. + """, + ), + ) + engine = AudioTranscriptionEngine( + audio_transcription_driver=audio_transcription_driver + ) + # prompt_text = self.get_prompt_text(STRING, input_string) + + task = AudioTranscriptionTask( + input=lambda _: audio_artifact, + audio_transcription_engine=engine, + ) + pipeline = Pipeline() + try: + pipeline.add_task(task) + result = pipeline.run() + except Exception as e: + print(e) + output = result.output_task.output.value + else: + output = "No audio provided" + return (output,) + + class gtUIImageQueryTask(gtUIBaseImageTask): CATEGORY = "Griptape/Images" @@ -315,20 +371,6 @@ def do_start_task(task: CodeExecutionTask) -> BaseArtifact: class gtUIParallelImageQueryTask(gtUIBaseImageTask): CATEGORY = "Griptape/Images" - # @classmethod - # def INPUT_TYPES(cls): - # inputs = super().INPUT_TYPES() - # inputs["optional"].update( - # { - # "rulesets": ("RULESET", {"forceInput": True}), - # } - # ) - # # del inputs["optional"]["agent"] - # return inputs - - # RETURN_TYPES = ("STRING", "AGENT") - # RETURN_NAMES = ("OUTPUT", "agent") - def run( self, STRING, diff --git a/nodes/tools.py b/nodes/tools.py index 2c63e33..808916b 100644 --- a/nodes/tools.py +++ b/nodes/tools.py @@ -1,6 +1,6 @@ -import os - import requests +from griptape.drivers import OpenAiAudioTranscriptionDriver +from griptape.engines import AudioTranscriptionEngine from griptape.tools import ( Calculator, DateTime, @@ -8,12 +8,43 @@ from griptape.tools import ( GriptapeCloudKnowledgeBaseClient, WebScraper, ) +from griptape.tools.audio_transcription_client.tool import AudioTranscriptionClient from ..py.griptape_config import get_config from .base_tool import gtUIBaseTool from .duckduckgo_client import DuckDuckGoTool +class gtUIAudioTranscriptionClient(gtUIBaseTool): + """ + Griptape AudioTranscriptionClient Tool + """ + + @classmethod + def INPUT_TYPES(s): + # inputs = super().INPUT_TYPES() + # inputs["optional"].update({"driver": ("DRIVER",)}) + # return inputs + + return { + "required": {"off_prompt": ("BOOLEAN", {"default": True})}, + "optional": {"driver": ("DRIVER", {"default": None})}, + } + + def create(self, off_prompt, driver=None): + if not driver: + driver = OpenAiAudioTranscriptionDriver(model="whisper-1") + + tool = AudioTranscriptionClient( + off_prompt=off_prompt, + engine=AudioTranscriptionEngine( + audio_transcription_driver=driver, + ), + ) + + return ([tool],) + + class gtUIFileManager(gtUIBaseTool): """ The Griptape File Manager Tool diff --git a/requirements.txt b/requirements.txt index 7289e2b..c062ec8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -griptape[all] +griptape[all]==v0.26.0 python-dotenv duckduckgo_search \ No newline at end of file