nodes/griptape-ai-ComfyUI-Griptape
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #23 from griptape-ai:dev

Browse Source 
Dev
This commit is contained in:
Jason Schleifer
2024-06-18 18:55:36 -07:00
committed by GitHub
parent a05f976b9f 0b65218327
commit aff40742da
10 changed files with 374 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3 -1
View File
@@ -1,6 +1,6 @@
# ComfyUI Griptape Nodes
This repo creates a series of nodes that enable you to utilize the [Griptape Python Framework](https://github.com/griptape-ai/griptape/) with [ComfyUI](https://github.com/comfyanonymous/ComfyUI), integrating AI into your workflow.
This repo creates a series of nodes that enable you to utilize the [Griptape Python Framework](https://github.com/griptape-ai/griptape/) with [ComfyUI](https://github.com/comfyanonymous/ComfyUI), integrating LLMs (Large Language Models) and AI into your workflow.
This example creates two [Griptape Agents](https://docs.griptape.ai/stable/griptape-framework/structures/agents/) - a Writer and an Editor. The writer has a [WebScraper](https://docs.griptape.ai/stable/griptape-tools/official-tools/web-scraper/) tool and gets a description of a website. It then generates blog topic ideas and passes them to_ the Editor Agent. The editor chooses the top 3 topics, and generates outlines and next actions for the next two. A third Agent generates thumbnail images for each blog post using a [Prompt Image Generation Task](https://docs.griptape.ai/stable/griptape-framework/structures/tasks/#prompt-image-generation-task) based on the outlines.
@@ -20,6 +20,7 @@ The repo currently has a subset of Griptape nodes, with more to come soon. Curre
* [DateTime](https://docs.griptape.ai/stable/griptape-tools/official-tools/date-time/)
* [WebScraper](https://docs.griptape.ai/stable/griptape-tools/official-tools/web-scraper)
* [FileManager](https://docs.griptape.ai/stable/griptape-tools/official-tools/file-manager)
* [AudioTranscriptionClient](https://docs.griptape.ai/stable/griptape-tools/official-tools/audio-transcription-client/)
* [GriptapeCloudKnowledgeBaseClient](https://docs.griptape.ai/stable/reference/griptape/tools/#griptape.tools.GriptapeCloudKnowledgeBaseClient)
* Run specific Agent Tasks:
@@ -47,6 +48,7 @@ In this example, we're using three `Image Description` nodes to describe the giv
## Using the nodes - Video Tutorials
1. Part 1: Installation: https://youtu.be/L4-HnKH4BSI?si=Q7IqP-KnWug7JJ5s
2. Part 2: Griptape Agents: https://youtu.be/wpQCciNel_A?si=WF_EogiZRGy0cQIm
3. Part 3: Controlling which LLM your Agents use: https://youtu.be/JlPuyH5Ot5I?si=KMPjwN3wn4L4rUyg
## Installation
__init__.py
+13 -1
View File
@@ -12,6 +12,8 @@ from dotenv import load_dotenv
# Load the griptape_config.json data
from .nodes.agent import CreateAgent, ExpandAgent, RunAgent
from .nodes.audio_drivers import gtUIOpenAiAudioTranscriptionDriver
from .nodes.audio_nodes import gtUILoadAudio
from .nodes.combine_nodes import MergeTexts, ToolList
from .nodes.config_nodes import (
gtUIAmazonBedrockStructureConfig,
@@ -32,6 +34,7 @@ from .nodes.image_nodes import (
)
from .nodes.rules import gtUIRule
from .nodes.tasks import (
gtUIAudioTranscriptionTask,
gtUIImageQueryTask,
gtUIParallelImageQueryTask,
gtUIPromptImageGenerationTask,
@@ -47,6 +50,7 @@ from .nodes.text_nodes import (
gtUITextToClipEncode,
)
from .nodes.tools import (
gtUIAudioTranscriptionClient,
gtUICalculator,
gtUIDateTime,
gtUIFileManager,
@@ -103,11 +107,14 @@ NODE_CLASS_MAPPINGS = {
"DateTime": gtUIDateTime,
"WebScraper": gtUIWebScraper,
"gtUIFileManager": gtUIFileManager,
"gtAudioTranscriptionClient": gtUIAudioTranscriptionClient,
"ToolList": ToolList,
"MergeTexts": MergeTexts,
"PromptTask": gtUIPromptTask,
"ToolTask": gtUIToolTask,
"ToolkitTask": gtUIToolkitTask,
"AudioTranscriptionTask": gtUIAudioTranscriptionTask,
"gtUIOpenAiAudioTranscriptionDriver": gtUIOpenAiAudioTranscriptionDriver,
"gtUIKnowledgeBaseTool": gtUIKnowledgeBaseTool,
"gtUIWebSearch": gtUIWebSearch,
"ImageQueryTask": gtUIImageQueryTask,
@@ -117,6 +124,7 @@ NODE_CLASS_MAPPINGS = {
"TextSummaryTask": gtUITextSummaryTask,
"Rule": gtUIRule,
"EnvironmentConfig": gtUIEnv,
"gtUILoadAudio": gtUILoadAudio,
}
NODE_DISPLAY_NAME_MAPPINGS = {
@@ -129,7 +137,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"PromptImageGenerationTask": "Griptape Create: Image from Text",
"PromptImageVariationTask": "Griptape Create: Image Variation",
"Rule": "Griptape Create: Rules",
"gtUIOpenAiImageGenerationDriver": "Griptape Driver: OpenAI",
"gtUIOpenAiAudioTranscriptionDriver": "Griptape Driver: OpenAI",
"gtUIAmazonBedrockStableDiffusionImageGenerationDriver": "Griptape Driver: Amazon Bedrock Stable Diffusion",
"gtUIAmazonBedrockTitanImageGenerationDriver": "Griptape Driver: Amazon Bedrock Titan",
"gtUILeonardoImageGenerationDriver": "Griptape Driver: Leonardo.AI",
@@ -141,6 +149,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"ToolTask": "Griptape Run: Tool Task",
"ToolkitTask": "Griptape Run: Toolkit Task",
"TextSummaryTask": "Griptape Run: Text Summary",
"AudioTranscriptionTask": "Griptape Run: Audio Transcription",
"ExpandAgent": "Griptape Expand: Agent Nodes",
"gtUIOpenAiStructureConfig": "Griptape Agent Config: OpenAI",
"gtUIAmazonBedrockStructureConfig": "Griptape Agent Config: Amazon Bedrock",
@@ -150,11 +159,14 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"DateTime": "Griptape Tool: DateTime",
"WebScraper": "Griptape Tool: WebScraper",
"gtUIFileManager": "Griptape Tool: FileManager",
"gtAudioTranscriptionClient": "Griptape Tool: Audio Transcription",
"gtUIKnowledgeBaseTool": "Griptape Tool: Griptape Cloud KnowledgeBase",
"gtUIWebSearch": "Griptape Tool: WebSearch",
"ToolList": "Griptape Combine: Tool List",
"MergeTexts": "Griptape Combine: Merge Texts",
"EnvironmentConfig": "Griptape Config: Environment Variables",
"gtUIOpenAiImageGenerationDriver": "Griptape Audio Driver: OpenAI",
"gtUILoadAudio": "Griptape Load: Audio",
}
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
js/gtUINodes.js
+150
View File
@@ -1,9 +1,58 @@
import { app } from "../../../scripts/app.js";
import { api } from "../../../scripts/api.js";
import { $el } from "../../../scripts/ui.js";
import { ComfyWidgets } from "../../../scripts/widgets.js";
import { ComfyDialog } from "../../../scripts/ui/dialog.js";
import { GriptapeConfigDialog } from "./gtUIConfigDialog.js";
function chainCallback(object, property, callback) {
if (object == undefined) {
//This should not happen.
console.error("Tried to add callback to non-existant object")
return;
}
if (property in object) {
const callback_orig = object[property]
object[property] = function () {
const r = callback_orig.apply(this, arguments);
callback.apply(this, arguments);
return r
};
} else {
object[property] = callback;
}
}
async function uploadFile(file) {
//TODO: Add uploaded file to cache with Cache.put()?
try {
// Wrap file in formdata so it includes filename
const body = new FormData();
const i = file.webkitRelativePath.lastIndexOf('/');
const subfolder = file.webkitRelativePath.slice(0,i+1)
const new_file = new File([file], file.name, {
type: file.type,
lastModified: file.lastModified,
});
body.append("image", new_file);
if (i > 0) {
body.append("subfolder", subfolder);
}
const resp = await api.fetchApi("/upload/image", {
method: "POST",
body,
});
if (resp.status === 200) {
return resp.status
} else {
alert(resp.status + " - " + resp.statusText);
}
} catch (error) {
alert(error);
}
}
class GriptapeNodes extends EventTarget {
constructor() {
super();
@@ -119,7 +168,104 @@ function getColor(category) {
return "#171717";
}
}
function gtUIAddUploadWidget(nodeType, nodeData, widgetName, type="audio") {
chainCallback(nodeType.prototype, "onNodeCreated", function() {
const pathWidget = this.widgets.find((w) => w.name === widgetName);
const fileInput = document.createElement("input");
chainCallback(this, "onRemoved", () => {
fileInput?.remove();
});
if (type == "folder") {
Object.assign(fileInput, {
type: "file",
style: "display: none",
webkitdirectory: true,
onchange: async () => {
const directory = fileInput.files[0].webkitRelativePath;
const i = directory.lastIndexOf('/');
if (i <= 0) {
throw "No directory found";
}
const path = directory.slice(0,directory.lastIndexOf('/'))
if (pathWidget.options.values.includes(path)) {
alert("A folder of the same name already exists");
return;
}
let successes = 0;
for(const file of fileInput.files) {
if (await uploadFile(file) == 200) {
successes++;
} else {
//Upload failed, but some prior uploads may have succeeded
//Stop future uploads to prevent cascading failures
//and only add to list if an upload has succeeded
if (successes > 0) {
break
} else {
return;
}
}
}
pathWidget.options.values.push(path);
pathWidget.value = path;
if (pathWidget.callback) {
pathWidget.callback(path)
}
},
});
} else if (type == "video") {
Object.assign(fileInput, {
type: "file",
accept: "video/webm,video/mp4,video/mkv,image/gif",
style: "display: none",
onchange: async () => {
if (fileInput.files.length) {
if (await uploadFile(fileInput.files[0]) != 200) {
//upload failed and file can not be added to options
return;
}
const filename = fileInput.files[0].name;
pathWidget.options.values.push(filename);
pathWidget.value = filename;
if (pathWidget.callback) {
pathWidget.callback(filename)
}
}
},
});
} else if (type == "audio") {
Object.assign(fileInput, {
type: "file",
accept: "audio/mpeg,audio/wav,audio/x-wav,audio/ogg",
style: "display: none",
onchange: async () => {
if (fileInput.files.length) {
if (await uploadFile(fileInput.files[0]) != 200) {
//upload failed and file can not be added to options
return;
}
const filename = fileInput.files[0].name;
pathWidget.options.values.push(filename);
pathWidget.value = filename;
if (pathWidget.callback) {
pathWidget.callback(filename)
}
}
},
});
}else {
throw "Unknown upload type"
}
document.body.append(fileInput);
let uploadWidget = this.addWidget("button", "choose " + type + " to upload", "image", () => {
//clear the active click event
app.canvas.node_widget = null
fileInput.click();
});
uploadWidget.options.serialize = false;
});
}
app.registerExtension({
name: "comfy.gtUI",
@@ -141,6 +287,10 @@ app.registerExtension({
// return r;
// };
// }
if (nodeData.name === "gtUILoadAudio") {
gtUIAddUploadWidget(nodeType, nodeData, "audio", "audio")
console.log("I found it!")
}
if (nodeData.name === "gtUIOutputStringNode") {
const onNodeCreated = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function () {
nodes/audio_drivers.py
+29
View File
@@ -0,0 +1,29 @@
from griptape.drivers import OpenAiAudioTranscriptionDriver
from .base_driver import gtUIBaseAudioTranscriptionDriver
openAiAudioTranscriptionModels = ["whisper-1"]
class gtUIOpenAiAudioTranscriptionDriver(gtUIBaseAudioTranscriptionDriver):
@classmethod
def INPUT_TYPES(s):
models = openAiAudioTranscriptionModels
inputs = super().INPUT_TYPES()
inputs["required"].update(
{
"model": (models, {"default": models[0]}),
}
)
return inputs
RETURN_NAMES = ("DRIVER",)
RETURN_TYPES = ("DRIVER",)
def create(
self,
model,
):
driver = OpenAiAudioTranscriptionDriver(model=model)
return (driver,)
nodes/audio_nodes.py
+28
View File
@@ -0,0 +1,28 @@
import os
import folder_paths
class gtUILoadAudio:
@classmethod
def INPUT_TYPES(s):
input_dir = folder_paths.get_input_directory()
files = [
f
for f in os.listdir(input_dir)
if os.path.isfile(os.path.join(input_dir, f))
]
return {
"required": {"audio": (sorted(files), {"audio_upload": True})},
}
CATEGORY = "Griptape/Audio"
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("AUDIO PATH",)
FUNCTION = "load_audio"
def load_audio(self, audio):
audio_path = folder_paths.get_annotated_filepath(audio)
return (audio_path,)
nodes/base_audio_task.py
+35
View File
@@ -0,0 +1,35 @@
from griptape.drivers import OpenAiAudioTranscriptionDriver
from .base_task import gtUIBaseTask
class gtUIBaseAudioTask(gtUIBaseTask):
@classmethod
def INPUT_TYPES(cls):
inputs = super().INPUT_TYPES()
inputs["optional"].update(
{
"audio": (
"STRING",
{
"forceInput": True,
},
),
},
)
inputs["optional"].update({"driver": ("DRIVER",)})
del inputs["optional"]["input_string"]
del inputs["optional"]["agent"]
del inputs["required"]["STRING"]
return inputs
CATEGORY = "Griptape/Audio"
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("OUTPUT",)
def run(self, audio, driver=None):
if not driver:
driver = OpenAiAudioTranscriptionDriver(model="whisper-1")
output = "Output"
return (output,)
nodes/base_driver.py
+23 -3
View File
@@ -1,6 +1,26 @@
from griptape.drivers import (
OpenAiImageGenerationDriver,
)
from griptape.drivers import OpenAiAudioTranscriptionDriver, OpenAiImageGenerationDriver
class gtUIBaseAudioTranscriptionDriver:
@classmethod
def INPUT_TYPES(s):
return {
"required": {},
"optional": {},
}
RETURN_TYPES = ("DRIVER",)
RETURN_NAMES = ("DRIVER",)
FUNCTION = "create"
CATEGORY = "Griptape/Audio Drivers"
def create(
self,
):
driver = OpenAiAudioTranscriptionDriver(model="whisper-1")
return (driver,)
class gtUIBaseImageGenerationDriver:
nodes/tasks.py
+59 -17
View File
@@ -2,22 +2,25 @@ import base64
import os
from textwrap import dedent
import folder_paths
from griptape.artifacts import BaseArtifact, TextArtifact
from griptape.drivers import (
AmazonBedrockImageQueryDriver,
AnthropicImageQueryDriver,
DummyAudioTranscriptionDriver,
DummyImageQueryDriver,
OpenAiAudioTranscriptionDriver,
OpenAiImageGenerationDriver,
)
from griptape.engines import (
AudioTranscriptionEngine,
ImageQueryEngine,
PromptImageGenerationEngine,
VariationImageGenerationEngine,
)
from griptape.loaders import ImageLoader
from griptape.structures import Agent, Workflow
from griptape.loaders import AudioLoader, ImageLoader
from griptape.structures import Agent, Pipeline, Workflow
from griptape.tasks import (
AudioTranscriptionTask,
CodeExecutionTask,
CsvExtractionTask,
ImageQueryTask,
@@ -29,9 +32,13 @@ from griptape.tasks import (
ToolTask,
VariationImageGenerationTask,
)
from griptape.utils import load_file
from schema import Schema
import folder_paths
from ..py.griptape_config import get_config
from .base_audio_task import gtUIBaseAudioTask
from .base_image_task import gtUIBaseImageTask
from .base_task import gtUIBaseTask
from .utilities import convert_tensor_to_base_64, image_path_to_output
@@ -253,6 +260,55 @@ class gtUIPromptImageVariationTask(gtUIBaseImageTask):
return (output_image, image_path)
class gtUIAudioTranscriptionTask(gtUIBaseAudioTask):
CATEGORY = "Griptape/Audio"
def run(self, audio, driver=None):
try:
audio_artifact = AudioLoader().load(load_file(audio))
except Exception as e:
print(f"Error loading audio file: {e}")
if audio_artifact:
if not driver:
audio_transcription_driver = OpenAiAudioTranscriptionDriver(
model="whisper-1"
)
else:
audio_transcription_driver = driver
# If the driver is a DummyAudioTranscriptionDriver we'll return a nice error message
if isinstance(audio_transcription_driver, DummyAudioTranscriptionDriver):
return (
dedent(
"""
I'm sorry, this agent doesn't have access to a valid AudioTranscriptionDriver.
You might want to try using a different Agent Configuration.
Reach out for help on Discord (https://discord.gg/gnWRz88eym) if you would like some help.
""",
),
)
engine = AudioTranscriptionEngine(
audio_transcription_driver=audio_transcription_driver
)
# prompt_text = self.get_prompt_text(STRING, input_string)
task = AudioTranscriptionTask(
input=lambda _: audio_artifact,
audio_transcription_engine=engine,
)
pipeline = Pipeline()
try:
pipeline.add_task(task)
result = pipeline.run()
except Exception as e:
print(e)
output = result.output_task.output.value
else:
output = "No audio provided"
return (output,)
class gtUIImageQueryTask(gtUIBaseImageTask):
CATEGORY = "Griptape/Images"
@@ -315,20 +371,6 @@ def do_start_task(task: CodeExecutionTask) -> BaseArtifact:
class gtUIParallelImageQueryTask(gtUIBaseImageTask):
CATEGORY = "Griptape/Images"
# @classmethod
# def INPUT_TYPES(cls):
# inputs = super().INPUT_TYPES()
# inputs["optional"].update(
# {
# "rulesets": ("RULESET", {"forceInput": True}),
# }
# )
# # del inputs["optional"]["agent"]
# return inputs
# RETURN_TYPES = ("STRING", "AGENT")
# RETURN_NAMES = ("OUTPUT", "agent")
def run(
self,
STRING,
nodes/tools.py
+33 -2
View File
@@ -1,6 +1,6 @@
import os
import requests
from griptape.drivers import OpenAiAudioTranscriptionDriver
from griptape.engines import AudioTranscriptionEngine
from griptape.tools import (
Calculator,
DateTime,
@@ -8,12 +8,43 @@ from griptape.tools import (
GriptapeCloudKnowledgeBaseClient,
WebScraper,
)
from griptape.tools.audio_transcription_client.tool import AudioTranscriptionClient
from ..py.griptape_config import get_config
from .base_tool import gtUIBaseTool
from .duckduckgo_client import DuckDuckGoTool
class gtUIAudioTranscriptionClient(gtUIBaseTool):
"""
Griptape AudioTranscriptionClient Tool
"""
@classmethod
def INPUT_TYPES(s):
# inputs = super().INPUT_TYPES()
# inputs["optional"].update({"driver": ("DRIVER",)})
# return inputs
return {
"required": {"off_prompt": ("BOOLEAN", {"default": True})},
"optional": {"driver": ("DRIVER", {"default": None})},
}
def create(self, off_prompt, driver=None):
if not driver:
driver = OpenAiAudioTranscriptionDriver(model="whisper-1")
tool = AudioTranscriptionClient(
off_prompt=off_prompt,
engine=AudioTranscriptionEngine(
audio_transcription_driver=driver,
),
)
return ([tool],)
class gtUIFileManager(gtUIBaseTool):
"""
The Griptape File Manager Tool
requirements.txt
+1 -1
View File
@@ -1,3 +1,3 @@
griptape[all]
griptape[all]==v0.26.0
python-dotenv
duckduckgo_search