From 8f894d7ac297f5bf2d0e978288115cd6de2b3514 Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Sat, 29 Jun 2024 14:33:29 +1200 Subject: [PATCH 01/10] first pass of storing default config --- __init__.py | 3 +- nodes/agent.py | 153 +++++++++++--------------------------------- nodes/base_agent.py | 98 ++++++++++++++++++++++------ 3 files changed, 119 insertions(+), 135 deletions(-) diff --git a/__init__.py b/__init__.py index b591886..3bbaa28 100644 --- a/__init__.py +++ b/__init__.py @@ -11,7 +11,7 @@ import os from dotenv import load_dotenv # Load the griptape_config.json data -from .nodes.agent import CreateAgent, ExpandAgent, RunAgent +from .nodes.agent import CreateAgent, ExpandAgent, RunAgent, gtUISetDefaultAgent from .nodes.audio_drivers import gtUIOpenAiAudioTranscriptionDriver from .nodes.audio_nodes import gtUILoadAudio from .nodes.combine_nodes import MergeTexts, RulesList, ToolList, gtUIMergeInputs @@ -97,6 +97,7 @@ NODE_CLASS_MAPPINGS = { "Griptape Create: Agent": CreateAgent, "Griptape Run: Agent": RunAgent, "Griptape Expand: Agent Nodes": ExpandAgent, + "Griptape Set: Default Agent": gtUISetDefaultAgent, # AGENT CONFIG "Griptape Agent Config: OpenAI": gtUIOpenAiStructureConfig, "Griptape Agent Config: Amazon Bedrock": gtUIAmazonBedrockStructureConfig, diff --git a/nodes/agent.py b/nodes/agent.py index 00c3b57..9a7048a 100644 --- a/nodes/agent.py +++ b/nodes/agent.py @@ -1,11 +1,4 @@ -from textwrap import dedent - -from griptape.config import ( - OpenAiStructureConfig, -) -from griptape.structures import Agent as gtAgent -from griptape.tools import TaskMemoryClient - +from ..py.griptape_config import update_config_with_dict from .base_agent import BaseAgent default_prompt = "{{ input_string }}" @@ -33,15 +26,31 @@ def model_check(agent): class RunAgent(BaseAgent): DESCRIPTION = "Run a simple Griptape Agent" + @classmethod + def INPUT_TYPES(s): + inputs = super().INPUT_TYPES() + del inputs["optional"]["config"] + del inputs["optional"]["tools"] + del inputs["optional"]["rulesets"] + + return inputs + def run( self, STRING, - # input_artifact=None, agent=None, input_string=None, ): - if not agent: - agent = gtAgent() + if agent: + self.agent = agent + else: + # make sure we update the config if it's changed + self.set_default_config() + + # Warn for models + model, simple_model = self.model_check() + if simple_model: + return (self.model_response(model), self.agent) # Get the prompt text if not input_string: @@ -49,125 +58,39 @@ class RunAgent(BaseAgent): else: prompt_text = STRING + "\n\n" + input_string - # There are certain models that can't handle Tools well. - model, simple_model = model_check(agent) - if simple_model: - if model == "": - return ( - "You have provided a blank model for the Agent Configuration.\n\nPlease specify a model configuration, or disconnect it from the agent.", - agent, - ) - else: - return ( - dedent( - f"""This Agent Configuration Model: **{ agent.config.prompt_driver.model }** may run into issues using tools.\n\nPlease consider using a different configuration, a different model, or removing tools from the agent and use the **Griptape Run: Tool Task** node for specific tool use.""" - ), - agent, - ) - result = agent.run(prompt_text) - artifact = result.output_task.output + result = self.agent.run(prompt_text) output_string = result.output_task.output.value return ( output_string, - # artifact, - agent, + self.agent, ) -class CreateAgent(BaseAgent): - """ - Create a Griptape Agent - """ +class CreateAgent(BaseAgent): ... - DESCRIPTION = ( - "Create a Griptape Agent with inputs for configuration, tools, and rulesets." - ) + +class gtUISetDefaultAgent(BaseAgent): + DESCRIPTION = "Set the default agent." @classmethod def INPUT_TYPES(s): inputs = super().INPUT_TYPES() - # Update optional inputs to include 'tool' and adjust others as necessary - inputs["optional"].update( - { - "config": ( - "CONFIG", - { - "forceInput": True, - }, - ), - "tools": ("TOOL_LIST", {"forceInput": True, "INPUT_IS_LIST": True}), - "rulesets": ("RULESET", {"forceInput": True}), - }, - ) + del inputs["optional"]["tools"] + del inputs["optional"]["rulesets"] + del inputs["optional"]["agent"] + del inputs["optional"]["input_string"] + del inputs["optional"]["STRING"] return inputs - RETURN_TYPES = ("STRING", "AGENT") - RETURN_NAMES = ("OUTPUT", "AGENT") - FUNCTION = "run" + RETURN_TYPES = ("CONFIG",) + OUTPUT_NODE = True - CATEGORY = "Griptape/Agent" + def run(self, config=None): + if config: + self.agent.config = config - def run( - self, - STRING, - agent=None, - config=None, - input_string=None, - tools=[], - rulesets=[], - ): - if not config: - config = OpenAiStructureConfig() - - task_memory_client = [TaskMemoryClient(off_prompt=False)] - agent_tools = [] - if len(tools) > 0: - agent_tools += tools - - agent_tools += task_memory_client if len(agent_tools) > 0 else [] - - agent_rulesets = [] - for ruleset in rulesets: - agent_rulesets.append(ruleset) - if not agent: - agent = gtAgent(config=config, tools=agent_tools, rulesets=agent_rulesets) - else: - if config: - agent.config = config - if len(tools) > 0: - agent.tools = agent_tools - if len(rulesets) > 0: - agent.rulesets = agent_rulesets - - # There are certain models that can't handle Tools well. - # If this agent is using one of those models AND they have tools supplied, we'll - # warn the user. - model, simple_model = model_check(agent) - if simple_model: - if model == "": - return ( - "You have provided a blank model for the Agent Configuration.\n\nPlease specify a model configuration, or disconnect it from the agent.", - agent, - ) - else: - return ( - dedent( - f"""This Agent Configuration Model: **{ agent.config.prompt_driver.model }** may run into issues using tools.\n\nPlease consider using a different configuration, a different model, or removing tools from the agent and use the **Griptape Run: Tool Task** node for specific tool use.""" - ), - agent, - ) - # Run the agent if there's a prompt - if input_string or STRING not in [default_prompt, ""]: - if not input_string: - prompt_text = STRING - else: - prompt_text = STRING + "\n" + input_string - - result = agent.run(prompt_text) - output_string = result.output_task.output.value - else: - output_string = "Agent Created" - return (output_string, agent) + update_config_with_dict(self.agent.config.to_dict()) + return (config,) class ExpandAgent: diff --git a/nodes/base_agent.py b/nodes/base_agent.py index d639da8..5659eae 100644 --- a/nodes/base_agent.py +++ b/nodes/base_agent.py @@ -1,8 +1,15 @@ +from griptape.config import BaseStructureConfig from griptape.structures import Agent as gtAgent +from ..py.griptape_config import get_config + default_prompt = "{{ input_string }}" +def get_default_config(): + return get_config("agent_config") + + class BaseAgent: """ Create a Griptape Agent @@ -10,6 +17,10 @@ class BaseAgent: def __init__(self): self.default_prompt = default_prompt + self.agent = gtAgent() + config = get_default_config() + if config: + self.agent.config = BaseStructureConfig.from_dict(config) pass @classmethod @@ -23,18 +34,20 @@ class BaseAgent: "forceInput": True, }, ), + "config": ( + "CONFIG", + { + "forceInput": True, + }, + ), + "tools": ("TOOL_LIST", {"forceInput": True, "INPUT_IS_LIST": True}), + "rulesets": ("RULESET", {"forceInput": True}), "input_string": ( "STRING", { "forceInput": True, }, ), - # "inputs": ( - # "ARTIFACT", - # { - # "forceInput": True, - # }, - # ), "STRING": ( "STRING", { @@ -46,12 +59,10 @@ class BaseAgent: RETURN_TYPES = ( "STRING", - # "ARTIFACT", "AGENT", ) RETURN_NAMES = ( "OUTPUT", - # "ARTIFACT", "AGENT", ) FUNCTION = "run" @@ -60,26 +71,75 @@ class BaseAgent: CATEGORY = "Griptape/Agent" + def set_default_config(self): + agent_config = get_config("agent_config") + if agent_config: + self.agent.config = BaseStructureConfig.from_dict(agent_config) + + def model_check(self): + # There are certain models that can't handle Tools well. + # If this agent is using one of those models AND they have tools supplied, we'll + # warn the user. + simple_models = ["llama3", "mistral", "LLama-3"] + drivers = ["OllamaPromptDriver", "LMStudioPromptDriver"] + agent_prompt_driver_name = self.agent.config.prompt_driver.__class__.__name__ + model = self.agent.config.prompt_driver.model + if agent_prompt_driver_name in drivers: + if model == "": + return (model, True) + for simple in simple_models: + if simple in model: + if len(self.agent.tools) > 0: + return (model, True) + return (model, False) + + def model_response(self, model): + if model == "": + return "You have provided a blank model for the Agent Configuration.\n\nPlease specify a model configuration, or disconnect it from the agent." + else: + return f"This Agent Configuration Model: **{ self.agent.config.prompt_driver.model }** may run into issues using tools.\n\nPlease consider using a different configuration, a different model, or removing tools from the agent and use the **Griptape Run: Tool Task** node for specific tool use." + def run( self, STRING, - # inputs=None, + config=None, + tools=[], + rulesets=[], agent=None, input_string=None, ): - if not agent: - agent = gtAgent() + if agent: + self.agent = agent - # Get the prompt text - if not input_string: - prompt_text = STRING + # Replace bits of the agent based off the inputs + if config: + self.agent.config = config else: - prompt_text = STRING + "\n\n" + input_string + self.set_default_config() - result = agent.run(prompt_text) - output_string = result.output_task.output.value + if len(tools) > 0: + self.agent.tools = tools + if len(rulesets) > 0: + self.agent.rulesets = rulesets + + # Warn for models + model, simple_model = self.model_check() + if simple_model: + return (self.model_response(model), self.agent) + + # Check for inputs. If none, then just create the agent + if not input_string and STRING == "": + output_string = "Agent created." + else: + # Get the prompt text + if not input_string: + prompt_text = STRING + else: + prompt_text = STRING + "\n\n" + input_string + + result = self.agent.run(prompt_text) + output_string = result.output_task.output.value return ( output_string, - # result.output_task.output, - agent, + self.agent, ) From c5e21127d9461a532edbf00dce798bc4fd7c5f97 Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Sat, 29 Jun 2024 14:41:08 +1200 Subject: [PATCH 02/10] added update_config_with_dict --- py/griptape_config.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/py/griptape_config.py b/py/griptape_config.py index be98efe..4f956d9 100644 --- a/py/griptape_config.py +++ b/py/griptape_config.py @@ -103,3 +103,16 @@ def get_config(key, default=None): else: return default return config + + +def update_config_with_dict(config_dict={}): + with open(USER_CONFIG_FILE, "r") as file: + existing_data = json.load(file) + + agent_config = {"agent_config": config_dict} + existing_data.update(agent_config) + + with open(USER_CONFIG_FILE, "w") as file: + json.dump( + existing_data, file, indent=4 + ) # Use indent=4 for pretty formatting, optional From f88cbb017e922c6f6cac8ab022032a81c1e1268b Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Mon, 1 Jul 2024 11:56:59 +1200 Subject: [PATCH 03/10] splitting agents to their own files --- __init__.py | 5 +- nodes/agent.py | 147 ------------------------------- nodes/agent/agent.py | 55 ++++++++++++ nodes/{ => agent}/base_agent.py | 8 +- nodes/agent/create_agent.py | 6 ++ nodes/agent/expand_agent.py | 51 +++++++++++ nodes/agent/run_agent.py | 44 +++++++++ nodes/agent/set_default_agent.py | 28 ++++++ nodes/tasks.py | 2 +- 9 files changed, 194 insertions(+), 152 deletions(-) delete mode 100644 nodes/agent.py create mode 100644 nodes/agent/agent.py rename nodes/{ => agent}/base_agent.py (96%) create mode 100644 nodes/agent/create_agent.py create mode 100644 nodes/agent/expand_agent.py create mode 100644 nodes/agent/run_agent.py create mode 100644 nodes/agent/set_default_agent.py diff --git a/__init__.py b/__init__.py index 3bbaa28..80c8ee7 100644 --- a/__init__.py +++ b/__init__.py @@ -11,7 +11,10 @@ import os from dotenv import load_dotenv # Load the griptape_config.json data -from .nodes.agent import CreateAgent, ExpandAgent, RunAgent, gtUISetDefaultAgent +from .nodes.agent.create_agent import CreateAgent +from .nodes.agent.expand_agent import ExpandAgent +from .nodes.agent.run_agent import RunAgent +from .nodes.agent.set_default_agent import gtUISetDefaultAgent from .nodes.audio_drivers import gtUIOpenAiAudioTranscriptionDriver from .nodes.audio_nodes import gtUILoadAudio from .nodes.combine_nodes import MergeTexts, RulesList, ToolList, gtUIMergeInputs diff --git a/nodes/agent.py b/nodes/agent.py deleted file mode 100644 index 9a7048a..0000000 --- a/nodes/agent.py +++ /dev/null @@ -1,147 +0,0 @@ -from ..py.griptape_config import update_config_with_dict -from .base_agent import BaseAgent - -default_prompt = "{{ input_string }}" - - -def model_check(agent): - # There are certain models that can't handle Tools well. - # If this agent is using one of those models AND they have tools supplied, we'll - # warn the user. - simple_models = ["llama3", "mistral", "LLama-3"] - drivers = ["OllamaPromptDriver", "LMStudioPromptDriver"] - agent_prompt_driver_name = agent.config.prompt_driver.__class__.__name__ - model = agent.config.prompt_driver.model - print(f"Model: {model}") - if agent_prompt_driver_name in drivers: - if model == "": - return (model, True) - for simple in simple_models: - if simple in model: - if len(agent.tools) > 0: - return (model, True) - return (model, False) - - -class RunAgent(BaseAgent): - DESCRIPTION = "Run a simple Griptape Agent" - - @classmethod - def INPUT_TYPES(s): - inputs = super().INPUT_TYPES() - del inputs["optional"]["config"] - del inputs["optional"]["tools"] - del inputs["optional"]["rulesets"] - - return inputs - - def run( - self, - STRING, - agent=None, - input_string=None, - ): - if agent: - self.agent = agent - else: - # make sure we update the config if it's changed - self.set_default_config() - - # Warn for models - model, simple_model = self.model_check() - if simple_model: - return (self.model_response(model), self.agent) - - # Get the prompt text - if not input_string: - prompt_text = STRING - else: - prompt_text = STRING + "\n\n" + input_string - - result = self.agent.run(prompt_text) - output_string = result.output_task.output.value - return ( - output_string, - self.agent, - ) - - -class CreateAgent(BaseAgent): ... - - -class gtUISetDefaultAgent(BaseAgent): - DESCRIPTION = "Set the default agent." - - @classmethod - def INPUT_TYPES(s): - inputs = super().INPUT_TYPES() - del inputs["optional"]["tools"] - del inputs["optional"]["rulesets"] - del inputs["optional"]["agent"] - del inputs["optional"]["input_string"] - del inputs["optional"]["STRING"] - return inputs - - RETURN_TYPES = ("CONFIG",) - OUTPUT_NODE = True - - def run(self, config=None): - if config: - self.agent.config = config - - update_config_with_dict(self.agent.config.to_dict()) - return (config,) - - -class ExpandAgent: - DESCRIPTION = "Expand the components of a Griptape Agent." - - def __init__(self): - pass - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "agent": ( - "AGENT", - { - "forceInput": True, - }, - ), - }, - } - - RETURN_TYPES = ( - "AGENT", - "CONFIG", - "RULESET", - "TOOL_LIST", - "MEMORY", - ) - RETURN_NAMES = ( - "AGENT", - "CONFIG", - "RULESETS", - "TOOLS", - "MEMORY", - ) - - FUNCTION = "expand" - - CATEGORY = "Griptape/Agent" - OUTPUT_NODE = True - - def expand(self, agent): - rulesets = agent.rulesets - tools = agent.tools - conversation_memory = agent.conversation_memory - config = agent.config - # Run the agent - return ( - agent, - config, - rulesets, - tools, - conversation_memory, - ) diff --git a/nodes/agent/agent.py b/nodes/agent/agent.py new file mode 100644 index 0000000..f1736c7 --- /dev/null +++ b/nodes/agent/agent.py @@ -0,0 +1,55 @@ +from griptape.config import BaseStructureConfig +from griptape.structures import Agent + +from ...py.griptape_config import get_config + +default_prompt = "{{ input_string }}" + + +class gtComfyAgent(Agent): + def __init__(self, *args, **kwargs): + # Initialize the parent class + super().__init__(*args, **kwargs) + + # Add any additional initialization here + + def set_default_config(self): + agent_config = get_config("agent_config") + if agent_config: + self.agent.config = BaseStructureConfig.from_dict(agent_config) + + def model_check(self): + # There are certain models that can't handle Tools well. + # If this agent is using one of those models AND they have tools supplied, we'll + # warn the user. + simple_models = ["llama3", "mistral", "LLama-3"] + drivers = ["OllamaPromptDriver", "LMStudioPromptDriver"] + agent_prompt_driver_name = self.agent.config.prompt_driver.__class__.__name__ + model = self.agent.config.prompt_driver.model + if agent_prompt_driver_name in drivers: + if model == "": + return (model, True) + for simple in simple_models: + if simple in model: + if len(self.agent.tools) > 0: + return (model, True) + return (model, False) + + +def model_check(agent): + # There are certain models that can't handle Tools well. + # If this agent is using one of those models AND they have tools supplied, we'll + # warn the user. + simple_models = ["llama3", "mistral", "LLama-3"] + drivers = ["OllamaPromptDriver", "LMStudioPromptDriver"] + agent_prompt_driver_name = agent.config.prompt_driver.__class__.__name__ + model = agent.config.prompt_driver.model + print(f"Model: {model}") + if agent_prompt_driver_name in drivers: + if model == "": + return (model, True) + for simple in simple_models: + if simple in model: + if len(agent.tools) > 0: + return (model, True) + return (model, False) diff --git a/nodes/base_agent.py b/nodes/agent/base_agent.py similarity index 96% rename from nodes/base_agent.py rename to nodes/agent/base_agent.py index 5659eae..3a5b38f 100644 --- a/nodes/base_agent.py +++ b/nodes/agent/base_agent.py @@ -1,7 +1,7 @@ from griptape.config import BaseStructureConfig -from griptape.structures import Agent as gtAgent -from ..py.griptape_config import get_config +from ...py.griptape_config import get_config +from .agent import gtComfyAgent default_prompt = "{{ input_string }}" @@ -17,7 +17,9 @@ class BaseAgent: def __init__(self): self.default_prompt = default_prompt - self.agent = gtAgent() + self.agent = gtComfyAgent() + self.agent.set_default_config() + config = get_default_config() if config: self.agent.config = BaseStructureConfig.from_dict(config) diff --git a/nodes/agent/create_agent.py b/nodes/agent/create_agent.py new file mode 100644 index 0000000..148a7b3 --- /dev/null +++ b/nodes/agent/create_agent.py @@ -0,0 +1,6 @@ +from .base_agent import BaseAgent + +default_prompt = "{{ input_string }}" + + +class CreateAgent(BaseAgent): ... diff --git a/nodes/agent/expand_agent.py b/nodes/agent/expand_agent.py new file mode 100644 index 0000000..818cd55 --- /dev/null +++ b/nodes/agent/expand_agent.py @@ -0,0 +1,51 @@ +class ExpandAgent: + DESCRIPTION = "Expand the components of a Griptape Agent." + + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "agent": ( + "AGENT", + { + "forceInput": True, + }, + ), + }, + } + + RETURN_TYPES = ( + "AGENT", + "CONFIG", + "RULESET", + "TOOL_LIST", + "MEMORY", + ) + RETURN_NAMES = ( + "AGENT", + "CONFIG", + "RULESETS", + "TOOLS", + "MEMORY", + ) + + FUNCTION = "expand" + + CATEGORY = "Griptape/Agent" + OUTPUT_NODE = True + + def expand(self, agent): + rulesets = agent.rulesets + tools = agent.tools + conversation_memory = agent.conversation_memory + config = agent.config + return ( + agent, + config, + rulesets, + tools, + conversation_memory, + ) diff --git a/nodes/agent/run_agent.py b/nodes/agent/run_agent.py new file mode 100644 index 0000000..1461c24 --- /dev/null +++ b/nodes/agent/run_agent.py @@ -0,0 +1,44 @@ +from .base_agent import BaseAgent + + +class RunAgent(BaseAgent): + DESCRIPTION = "Run a simple Griptape Agent" + + @classmethod + def INPUT_TYPES(s): + inputs = super().INPUT_TYPES() + del inputs["optional"]["config"] + del inputs["optional"]["tools"] + del inputs["optional"]["rulesets"] + + return inputs + + def run( + self, + STRING, + agent=None, + input_string=None, + ): + if agent: + self.agent = agent + else: + # make sure we update the config if it's changed + self.set_default_config() + + # Warn for models + model, simple_model = self.model_check() + if simple_model: + return (self.model_response(model), self.agent) + + # Get the prompt text + if not input_string: + prompt_text = STRING + else: + prompt_text = STRING + "\n\n" + input_string + + result = self.agent.run(prompt_text) + output_string = result.output_task.output.value + return ( + output_string, + self.agent, + ) diff --git a/nodes/agent/set_default_agent.py b/nodes/agent/set_default_agent.py new file mode 100644 index 0000000..58d0c0c --- /dev/null +++ b/nodes/agent/set_default_agent.py @@ -0,0 +1,28 @@ +from ...py.griptape_config import update_config_with_dict +from .base_agent import BaseAgent + +default_prompt = "{{ input_string }}" + + +class gtUISetDefaultAgent(BaseAgent): + DESCRIPTION = "Set the default agent." + + @classmethod + def INPUT_TYPES(s): + inputs = super().INPUT_TYPES() + del inputs["optional"]["tools"] + del inputs["optional"]["rulesets"] + del inputs["optional"]["agent"] + del inputs["optional"]["input_string"] + del inputs["optional"]["STRING"] + return inputs + + RETURN_TYPES = ("CONFIG",) + OUTPUT_NODE = True + + def run(self, config=None): + if config: + self.agent.config = config + + update_config_with_dict(self.agent.config.to_dict()) + return (config,) diff --git a/nodes/tasks.py b/nodes/tasks.py index 599681d..de9f3f7 100644 --- a/nodes/tasks.py +++ b/nodes/tasks.py @@ -37,7 +37,7 @@ from griptape.utils import load_file from schema import Schema from ..py.griptape_config import get_config -from .agent import model_check +from .agent.agent import model_check from .base_audio_task import gtUIBaseAudioTask from .base_image_task import gtUIBaseImageTask from .base_task import gtUIBaseTask From 56f4938291fe83e8dc0ad478b04b8d0f289aeb29 Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Mon, 1 Jul 2024 13:13:38 +1200 Subject: [PATCH 04/10] updating audio --- js/gtUINodes.js | 87 ++++++++++++++------------------------- nodes/agent/agent.py | 14 +++++-- nodes/agent/base_agent.py | 66 ++++++++++++++--------------- nodes/agent/run_agent.py | 6 +-- nodes/base_audio_task.py | 30 +++++++++++++- nodes/tasks.py | 10 ++++- 6 files changed, 113 insertions(+), 100 deletions(-) diff --git a/js/gtUINodes.js b/js/gtUINodes.js index a0c632d..7d5d4cc 100644 --- a/js/gtUINodes.js +++ b/js/gtUINodes.js @@ -300,24 +300,6 @@ function gtUIAddUploadWidget(nodeType, nodeData, widgetName, type="audio") { }, }); } else if (type == "audio") { - // Create an audio element if not already created - // let audioWidget = document.getElementById('audio-widget'); - // let audioWidget = document.createElement('audio'); - // audioWidget.controls = true; // Add play controls - // document.body.append(audioWidget); // Append the audio element to the body or a specific container - // let audioButton = this.addWidget("button", "Play Audio ▶️", "audio", () => { - // audioWidget.play(); - // }); - // document.body.append(audioWidget); - // if (!audioWidget) { - // audioWidget = document.createElement('audio'); - // audioWidget.id = 'audio-widget'; - // audioWidget.controls = true; // Add play controls - // document.body.append(audioWidget); // Append the audio element to the body or a specific container - // let audioButton = this.addWidget("button", "Play Audio ▶️", "audio", () => { - // audioWidget.play(); - // }); - // } Object.assign(fileInput, { type: "file", accept: "audio/mpeg,audio/wav,audio/x-wav,audio/ogg", @@ -343,41 +325,34 @@ function gtUIAddUploadWidget(nodeType, nodeData, widgetName, type="audio") { }); - this.onDragOver = function( e) { - if (e.dataTransfer && e.dataTransfer.items) { - const audio = [...e.dataTransfer.items].find((f) => f.kind === "file" && f.type.startsWith("audio/")); - // console.log("dragover: " + audio); - // console.log(audio); - return !!audio; - } - return false; - } - this.onDragDrop = function (e) { - let handled = false; - for (const file of e.dataTransfer.files) { - if (file.type.startsWith("audio/")) { - handled = true; - console.log(file); - uploadFile(file, !handled); - handled = true; - const filename = file.name; - pathWidget.options.values.push(filename); - pathWidget.value = filename; - if (pathWidget.callback) { - pathWidget.callback(filename) - } - - // Create a URL for the audio file and set it as the source of the audio element - // const audioURL = URL.createObjectURL(file); - // audioWidget.src = audioURL; - // audioWidget.play(); // Optionally play the audio automatically - - - } - } - return handled; + // this.onDragOver = function( e) { + // if (e.dataTransfer && e.dataTransfer.items) { + // const audio = [...e.dataTransfer.items].find((f) => f.kind === "file" && f.type.startsWith("audio/")); + // // console.log("dragover: " + audio); + // // console.log(audio); + // return !!audio; + // } + // return false; + // } + // this.onDragDrop = function (e) { + // let handled = false; + // for (const file of e.dataTransfer.files) { + // if (file.type.startsWith("audio/")) { + // handled = true; + // console.log(file); + // uploadFile(file, !handled); + // handled = true; + // const filename = file.name; + // pathWidget.options.values.push(filename); + // pathWidget.value = filename; + // if (pathWidget.callback) { + // pathWidget.callback(filename) + // } + // } + // } + // return handled; - } + // } }else { throw "Unknown upload type" } @@ -418,10 +393,10 @@ app.registerExtension({ } } - // Create Audio Node - if (nodeData.name === "Griptape Load: Audio") { - gtUIAddUploadWidget(nodeType, nodeData, "audio", "audio") - } + // // Create Audio Node + // if (nodeData.name === "Griptape Load: Audio") { + // gtUIAddUploadWidget(nodeType, nodeData, "audio", "audio") + // } // Display Artifact Node if (nodeData.name === "Griptape Display: Artifact") { diff --git a/nodes/agent/agent.py b/nodes/agent/agent.py index f1736c7..a96229c 100644 --- a/nodes/agent/agent.py +++ b/nodes/agent/agent.py @@ -16,7 +16,7 @@ class gtComfyAgent(Agent): def set_default_config(self): agent_config = get_config("agent_config") if agent_config: - self.agent.config = BaseStructureConfig.from_dict(agent_config) + self.config = BaseStructureConfig.from_dict(agent_config) def model_check(self): # There are certain models that can't handle Tools well. @@ -24,17 +24,23 @@ class gtComfyAgent(Agent): # warn the user. simple_models = ["llama3", "mistral", "LLama-3"] drivers = ["OllamaPromptDriver", "LMStudioPromptDriver"] - agent_prompt_driver_name = self.agent.config.prompt_driver.__class__.__name__ - model = self.agent.config.prompt_driver.model + agent_prompt_driver_name = self.config.prompt_driver.__class__.__name__ + model = self.config.prompt_driver.model if agent_prompt_driver_name in drivers: if model == "": return (model, True) for simple in simple_models: if simple in model: - if len(self.agent.tools) > 0: + if len(self.tools) > 0: return (model, True) return (model, False) + def model_response(self, model): + if model == "": + return "You have provided a blank model for the Agent Configuration.\n\nPlease specify a model configuration, or disconnect it from the agent." + else: + return f"This Agent Configuration Model: **{ self.config.prompt_driver.model }** may run into issues using tools.\n\nPlease consider using a different configuration, a different model, or removing tools from the agent and use the **Griptape Run: Tool Task** node for specific tool use." + def model_check(agent): # There are certain models that can't handle Tools well. diff --git a/nodes/agent/base_agent.py b/nodes/agent/base_agent.py index 3a5b38f..0fea98c 100644 --- a/nodes/agent/base_agent.py +++ b/nodes/agent/base_agent.py @@ -1,5 +1,3 @@ -from griptape.config import BaseStructureConfig - from ...py.griptape_config import get_config from .agent import gtComfyAgent @@ -20,10 +18,10 @@ class BaseAgent: self.agent = gtComfyAgent() self.agent.set_default_config() - config = get_default_config() - if config: - self.agent.config = BaseStructureConfig.from_dict(config) - pass + # config = get_default_config() + # if config: + # self.agent.config = BaseStructureConfig.from_dict(config) + # pass @classmethod def INPUT_TYPES(s): @@ -73,33 +71,33 @@ class BaseAgent: CATEGORY = "Griptape/Agent" - def set_default_config(self): - agent_config = get_config("agent_config") - if agent_config: - self.agent.config = BaseStructureConfig.from_dict(agent_config) + # def set_default_config(self): + # agent_config = get_config("agent_config") + # if agent_config: + # self.agent.config = BaseStructureConfig.from_dict(agent_config) - def model_check(self): - # There are certain models that can't handle Tools well. - # If this agent is using one of those models AND they have tools supplied, we'll - # warn the user. - simple_models = ["llama3", "mistral", "LLama-3"] - drivers = ["OllamaPromptDriver", "LMStudioPromptDriver"] - agent_prompt_driver_name = self.agent.config.prompt_driver.__class__.__name__ - model = self.agent.config.prompt_driver.model - if agent_prompt_driver_name in drivers: - if model == "": - return (model, True) - for simple in simple_models: - if simple in model: - if len(self.agent.tools) > 0: - return (model, True) - return (model, False) + # def model_check(self): + # # There are certain models that can't handle Tools well. + # # If this agent is using one of those models AND they have tools supplied, we'll + # # warn the user. + # simple_models = ["llama3", "mistral", "LLama-3"] + # drivers = ["OllamaPromptDriver", "LMStudioPromptDriver"] + # agent_prompt_driver_name = self.agent.config.prompt_driver.__class__.__name__ + # model = self.agent.config.prompt_driver.model + # if agent_prompt_driver_name in drivers: + # if model == "": + # return (model, True) + # for simple in simple_models: + # if simple in model: + # if len(self.agent.tools) > 0: + # return (model, True) + # return (model, False) - def model_response(self, model): - if model == "": - return "You have provided a blank model for the Agent Configuration.\n\nPlease specify a model configuration, or disconnect it from the agent." - else: - return f"This Agent Configuration Model: **{ self.agent.config.prompt_driver.model }** may run into issues using tools.\n\nPlease consider using a different configuration, a different model, or removing tools from the agent and use the **Griptape Run: Tool Task** node for specific tool use." + # def model_response(self, model): + # if model == "": + # return "You have provided a blank model for the Agent Configuration.\n\nPlease specify a model configuration, or disconnect it from the agent." + # else: + # return f"This Agent Configuration Model: **{ self.agent.config.prompt_driver.model }** may run into issues using tools.\n\nPlease consider using a different configuration, a different model, or removing tools from the agent and use the **Griptape Run: Tool Task** node for specific tool use." def run( self, @@ -117,7 +115,7 @@ class BaseAgent: if config: self.agent.config = config else: - self.set_default_config() + self.agent.set_default_config() if len(tools) > 0: self.agent.tools = tools @@ -125,9 +123,9 @@ class BaseAgent: self.agent.rulesets = rulesets # Warn for models - model, simple_model = self.model_check() + model, simple_model = self.agent.model_check() if simple_model: - return (self.model_response(model), self.agent) + return (self.agent.model_response(model), self.agent) # Check for inputs. If none, then just create the agent if not input_string and STRING == "": diff --git a/nodes/agent/run_agent.py b/nodes/agent/run_agent.py index 1461c24..37c058d 100644 --- a/nodes/agent/run_agent.py +++ b/nodes/agent/run_agent.py @@ -23,12 +23,12 @@ class RunAgent(BaseAgent): self.agent = agent else: # make sure we update the config if it's changed - self.set_default_config() + self.agent.set_default_config() # Warn for models - model, simple_model = self.model_check() + model, simple_model = self.agent.model_check() if simple_model: - return (self.model_response(model), self.agent) + return (self.agent.model_response(model), self.agent) # Get the prompt text if not input_string: diff --git a/nodes/base_audio_task.py b/nodes/base_audio_task.py index 2c098b0..729b672 100644 --- a/nodes/base_audio_task.py +++ b/nodes/base_audio_task.py @@ -1,3 +1,7 @@ +import io +import tempfile + +import torchaudio from griptape.drivers import OpenAiAudioTranscriptionDriver from .base_task import gtUIBaseTask @@ -10,12 +14,13 @@ class gtUIBaseAudioTask(gtUIBaseTask): inputs["optional"].update( { - "audio": ( + "audio_filepath": ( "STRING", { "forceInput": True, }, ), + "audio": ("AUDIO",), }, ) inputs["optional"].update({"driver": ("DRIVER",)}) @@ -28,7 +33,28 @@ class gtUIBaseAudioTask(gtUIBaseTask): RETURN_TYPES = ("STRING",) RETURN_NAMES = ("OUTPUT",) - def run(self, audio, driver=None): + def save_audio_tempfile(self, audio_data): + temp_files = [] + + for waveform in audio_data["waveform"]: + with tempfile.NamedTemporaryFile(suffix=".flac", delete=False) as temp_file: + temp_filename = temp_file.name + + # Save the audio to the buffer + buff = io.BytesIO() + torchaudio.save( + buff, waveform, audio_data["sample_rate"], format="FLAC" + ) + + # Write the buffer's contents to the temporary file + with open(temp_filename, "wb") as f: + f.write(buff.getbuffer()) + + temp_files.append(temp_filename) + + return temp_files + + def run(self, audio=None, audio_filepath=None, driver=None): if not driver: driver = OpenAiAudioTranscriptionDriver(model="whisper-1") output = "Output" diff --git a/nodes/tasks.py b/nodes/tasks.py index de9f3f7..b807a86 100644 --- a/nodes/tasks.py +++ b/nodes/tasks.py @@ -272,7 +272,15 @@ class gtUIAudioTranscriptionTask(gtUIBaseAudioTask): DESCRIPTION = "Transcribe an audio file." CATEGORY = "Griptape/Audio" - def run(self, audio, driver=None): + def run(self, audio=None, audio_filepath=None, driver=None): + audio_artifact = None + if audio: + audio = self.save_audio_tempfile(audio)[0] + elif audio_filepath: + audio = audio_filepath + else: + return ("There is no audio file.",) + try: audio_artifact = AudioLoader().load(load_file(audio)) except Exception as e: From ca2583587cab18557985dd7180ae91e969e88ec5 Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Mon, 1 Jul 2024 15:15:18 +1200 Subject: [PATCH 05/10] audio tasks working now --- js/gtUINodes.js | 61 ++++++++++++++++++++-------------------- nodes/audio_nodes.py | 29 +++++++++++-------- nodes/base_audio_task.py | 5 ++-- 3 files changed, 51 insertions(+), 44 deletions(-) diff --git a/js/gtUINodes.js b/js/gtUINodes.js index 7d5d4cc..3e1e5dd 100644 --- a/js/gtUINodes.js +++ b/js/gtUINodes.js @@ -325,34 +325,33 @@ function gtUIAddUploadWidget(nodeType, nodeData, widgetName, type="audio") { }); - // this.onDragOver = function( e) { - // if (e.dataTransfer && e.dataTransfer.items) { - // const audio = [...e.dataTransfer.items].find((f) => f.kind === "file" && f.type.startsWith("audio/")); - // // console.log("dragover: " + audio); - // // console.log(audio); - // return !!audio; - // } - // return false; - // } - // this.onDragDrop = function (e) { - // let handled = false; - // for (const file of e.dataTransfer.files) { - // if (file.type.startsWith("audio/")) { - // handled = true; - // console.log(file); - // uploadFile(file, !handled); - // handled = true; - // const filename = file.name; - // pathWidget.options.values.push(filename); - // pathWidget.value = filename; - // if (pathWidget.callback) { - // pathWidget.callback(filename) - // } - // } - // } - // return handled; + this.onDragOver = function( e) { + if (e.dataTransfer && e.dataTransfer.items) { + const audio = [...e.dataTransfer.items].find((f) => f.kind === "file" && f.type.startsWith("audio/")); + // console.log("dragover: " + audio); + // console.log(audio); + return !!audio; + } + return false; + } + this.onDragDrop = function (e) { + let handled = false; + for (const file of e.dataTransfer.files) { + if (file.type.startsWith("audio/")) { + handled = true; + uploadFile(file, !handled); + handled = true; + const filename = file.name; + pathWidget.options.values.push(filename); + pathWidget.value = filename; + if (pathWidget.callback) { + pathWidget.callback(filename) + } + } + } + return handled; - // } + } }else { throw "Unknown upload type" } @@ -393,10 +392,10 @@ app.registerExtension({ } } - // // Create Audio Node - // if (nodeData.name === "Griptape Load: Audio") { - // gtUIAddUploadWidget(nodeType, nodeData, "audio", "audio") - // } + // Create Audio Node + if (nodeData.name === "Griptape Load: Audio") { + gtUIAddUploadWidget(nodeType, nodeData, "audio", "audio") + } // Display Artifact Node if (nodeData.name === "Griptape Display: Artifact") { diff --git a/nodes/audio_nodes.py b/nodes/audio_nodes.py index 35bfabf..27ff9f8 100644 --- a/nodes/audio_nodes.py +++ b/nodes/audio_nodes.py @@ -2,6 +2,7 @@ import mimetypes import os import folder_paths +import torchaudio def is_audio_file(filepath): @@ -11,6 +12,7 @@ def is_audio_file(filepath): class gtUILoadAudio: DESCRIPTION = "Load an audio file for processing." + SUPPORTED_FORMATS = (".wav", ".mp3", ".ogg", ".flac", ".aiff", ".aif") @classmethod def INPUT_TYPES(s): @@ -18,20 +20,25 @@ class gtUILoadAudio: files = [ f for f in os.listdir(input_dir) - if os.path.isfile(os.path.join(input_dir, f)) - and is_audio_file(os.path.join(input_dir, f)) + if ( + os.path.isfile(os.path.join(input_dir, f)) + and f.endswith(gtUILoadAudio.SUPPORTED_FORMATS) + ) ] - return { - "required": {"audio": (sorted(files), {"audio_upload": True})}, - } + return {"required": {"audio": (sorted(files), {"audio_upload": False})}} CATEGORY = "Griptape/Audio" - RETURN_TYPES = ("STRING",) - RETURN_NAMES = ("AUDIO PATH",) - FUNCTION = "load_audio" + RETURN_TYPES = ("STRING", "AUDIO") + RETURN_NAMES = ("AUDIO_PATH", "AUDIO") + FUNCTION = "gt_load_audio" - def load_audio(self, audio): + def gt_load_audio(self, audio): audio_path = folder_paths.get_annotated_filepath(audio) - - return (audio_path,) + waveform, sample_rate = torchaudio.load(audio_path) + multiplier = 1.0 + audio = {"waveform": waveform.unsqueeze(0), "sample_rate": sample_rate} + return ( + audio_path, + audio, + ) diff --git a/nodes/base_audio_task.py b/nodes/base_audio_task.py index 729b672..3bc1db4 100644 --- a/nodes/base_audio_task.py +++ b/nodes/base_audio_task.py @@ -20,10 +20,11 @@ class gtUIBaseAudioTask(gtUIBaseTask): "forceInput": True, }, ), - "audio": ("AUDIO",), + "audio": ("AUDIO", {}), + "driver": ("DRIVER", {}), }, ) - inputs["optional"].update({"driver": ("DRIVER",)}) + # inputs["optional"].update({"driver": ("DRIVER",)}) del inputs["optional"]["input_string"] del inputs["optional"]["agent"] del inputs["required"]["STRING"] From 5a17e6b27e9a25071d7b2a6bb33a74fffb0e1c03 Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Mon, 1 Jul 2024 15:53:24 +1200 Subject: [PATCH 06/10] ensuring agent tasks use default agent configs --- nodes/agent/agent.py | 3 +++ nodes/agent/base_agent.py | 5 ----- nodes/agent/run_agent.py | 3 --- nodes/base_task.py | 3 ++- nodes/tasks.py | 32 +++++++++++++++++++------------- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/nodes/agent/agent.py b/nodes/agent/agent.py index a96229c..35bf0d4 100644 --- a/nodes/agent/agent.py +++ b/nodes/agent/agent.py @@ -12,6 +12,7 @@ class gtComfyAgent(Agent): super().__init__(*args, **kwargs) # Add any additional initialization here + self.set_default_config() def set_default_config(self): agent_config = get_config("agent_config") @@ -25,6 +26,8 @@ class gtComfyAgent(Agent): simple_models = ["llama3", "mistral", "LLama-3"] drivers = ["OllamaPromptDriver", "LMStudioPromptDriver"] agent_prompt_driver_name = self.config.prompt_driver.__class__.__name__ + print(agent_prompt_driver_name) + model = self.config.prompt_driver.model if agent_prompt_driver_name in drivers: if model == "": diff --git a/nodes/agent/base_agent.py b/nodes/agent/base_agent.py index 0fea98c..e0232a2 100644 --- a/nodes/agent/base_agent.py +++ b/nodes/agent/base_agent.py @@ -18,11 +18,6 @@ class BaseAgent: self.agent = gtComfyAgent() self.agent.set_default_config() - # config = get_default_config() - # if config: - # self.agent.config = BaseStructureConfig.from_dict(config) - # pass - @classmethod def INPUT_TYPES(s): return { diff --git a/nodes/agent/run_agent.py b/nodes/agent/run_agent.py index 37c058d..2bd2e49 100644 --- a/nodes/agent/run_agent.py +++ b/nodes/agent/run_agent.py @@ -21,9 +21,6 @@ class RunAgent(BaseAgent): ): if agent: self.agent = agent - else: - # make sure we update the config if it's changed - self.agent.set_default_config() # Warn for models model, simple_model = self.agent.model_check() diff --git a/nodes/base_task.py b/nodes/base_task.py index bf987d3..3e6e140 100644 --- a/nodes/base_task.py +++ b/nodes/base_task.py @@ -1,6 +1,7 @@ -from griptape.structures import Agent from griptape.tasks import PromptTask +from .agent.agent import gtComfyAgent as Agent + class gtUIBaseTask: def __init__(self): diff --git a/nodes/tasks.py b/nodes/tasks.py index b807a86..5728bbc 100644 --- a/nodes/tasks.py +++ b/nodes/tasks.py @@ -19,7 +19,7 @@ from griptape.engines import ( VariationImageGenerationEngine, ) from griptape.loaders import AudioLoader, ImageLoader -from griptape.structures import Agent, Pipeline, Workflow +from griptape.structures import Pipeline, Workflow from griptape.tasks import ( AudioTranscriptionTask, CodeExecutionTask, @@ -37,7 +37,7 @@ from griptape.utils import load_file from schema import Schema from ..py.griptape_config import get_config -from .agent.agent import model_check +from .agent.agent import gtComfyAgent as Agent from .base_audio_task import gtUIBaseAudioTask from .base_image_task import gtUIBaseImageTask from .base_task import gtUIBaseTask @@ -555,26 +555,32 @@ class gtUIToolkitTask(gtUIBaseTask): if len(tools) == 0: return super().run(STRING, input_string, agent) + if prompt_text.strip() == "": + return ("No prompt provided", agent) # if the tool is provided, keep going if not agent: agent = Agent() - if model_check(agent): - return ( - dedent( - """ - I'm sorry, this agent uses a simple model that can't handle ToolkitTasks. - You might want to try using a different Agent Configuration, or use a simple ToolTask instead. + model, simple_model = agent.model_check() + if simple_model: + response = agent.model_response(model) + return (response, agent) + # return ( + # dedent( + # """ + # I'm sorry, this agent uses a simple model that can't handle ToolkitTasks. + # You might want to try using a different Agent Configuration, or use a simple ToolTask instead. - Reach out for help on Discord (https://discord.gg/gnWRz88eym) if you would like some help. - """ - ), - agent, - ) + # Reach out for help on Discord (https://discord.gg/gnWRz88eym) if you would like some help. + # """ + # ), + # agent, + # ) task = ToolkitTask(prompt_text, tools=tools) try: agent.add_task(task) except Exception as e: print(e) + result = agent.run() return (result.output_task.output.value, agent) From b34726f833ad9bde9372c173b251e4b6d53c2b29 Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Mon, 1 Jul 2024 18:11:02 +1200 Subject: [PATCH 07/10] added image_generation_driver input to configs --- nodes/base_config.py | 6 ++- nodes/config_nodes.py | 93 +++++++++++++++++++++++++++++++++---------- nodes/tasks.py | 21 ++++++++-- 3 files changed, 95 insertions(+), 25 deletions(-) diff --git a/nodes/base_config.py b/nodes/base_config.py index c55ffb5..561f27d 100644 --- a/nodes/base_config.py +++ b/nodes/base_config.py @@ -21,6 +21,10 @@ class gtUIBaseConfig: {"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.01}, ), "seed": ("INT", {"default": 10342349342}), + "image_generation_driver": ( + "DRIVER", + {}, + ), }, } @@ -32,5 +36,5 @@ class gtUIBaseConfig: CATEGORY = "Griptape/Agent Configs" - def create(self, temperature, seed): + def create(self, temperature, seed, image_generation_driver=None): return (OpenAiStructureConfig(),) diff --git a/nodes/config_nodes.py b/nodes/config_nodes.py index a6eabbd..08bd2e4 100644 --- a/nodes/config_nodes.py +++ b/nodes/config_nodes.py @@ -4,6 +4,7 @@ from griptape.config import ( AmazonBedrockStructureConfig, AnthropicStructureConfig, GoogleStructureConfig, + OpenAiStructureConfig, StructureConfig, ) @@ -14,6 +15,7 @@ from griptape.drivers import ( AnthropicImageQueryDriver, AnthropicPromptDriver, BedrockClaudeImageQueryModelDriver, + DummyImageGenerationDriver, GooglePromptDriver, OllamaPromptDriver, OpenAiChatPromptDriver, @@ -113,7 +115,14 @@ class gtUILMStudioStructureConfig(gtUIBaseConfig): ) return inputs - def create(self, prompt_model, port, temperature, seed): + def create( + self, + prompt_model, + port, + temperature, + seed, + image_generation_driver=DummyImageGenerationDriver(), + ): custom_config = StructureConfig( prompt_driver=LMStudioPromptDriver( model=prompt_model, @@ -126,6 +135,7 @@ class gtUILMStudioStructureConfig(gtUIBaseConfig): max_output_tokens=1024, ), ), + image_generation_driver=image_generation_driver, ) return (custom_config,) @@ -151,11 +161,18 @@ class gtUIOllamaStructureConfig(gtUIBaseConfig): ) return inputs - def create(self, prompt_model, temperature, seed): + def create( + self, + prompt_model, + temperature, + seed, + image_generation_driver=DummyImageGenerationDriver(), + ): custom_config = StructureConfig( prompt_driver=OllamaPromptDriver( model=prompt_model, options={"temperature": temperature} ), + image_generation_driver=image_generation_driver, ) return (custom_config,) @@ -202,7 +219,14 @@ class gtUIAmazonBedrockStructureConfig(gtUIBaseConfig): ) return inputs - def create(self, prompt_model, image_query_model, temperature, seed): + def create( + self, + prompt_model, + image_query_model, + temperature, + seed, + image_generation_driver=None, + ): custom_config = AmazonBedrockStructureConfig() custom_config.prompt_driver = AmazonBedrockPromptDriver( model=prompt_model, temperature=temperature @@ -211,7 +235,8 @@ class gtUIAmazonBedrockStructureConfig(gtUIBaseConfig): image_query_model_driver=BedrockClaudeImageQueryModelDriver(), model=image_query_model, ) - + if image_generation_driver: + custom_config.image_generation_driver = image_generation_driver return (custom_config,) @@ -229,11 +254,14 @@ class gtUIGoogleStructureConfig(gtUIBaseConfig): "Google Structure Config. Use Google's models for prompt and image query." ) - def create(self, temperature, seed): + def create( + self, temperature, seed, image_generation_driver=DummyImageGenerationDriver() + ): custom_config = GoogleStructureConfig( prompt_driver=GooglePromptDriver( model="gemini-pro", temperature=temperature - ) + ), + image_generation_driver=image_generation_driver, ) return (custom_config,) @@ -282,7 +310,14 @@ class gtUIAnthropicStructureConfig(gtUIBaseConfig): ) return inputs - def create(self, prompt_model, image_query_model, temperature, seed): + def create( + self, + prompt_model, + image_query_model, + temperature, + seed, + image_generation_driver=DummyImageGenerationDriver(), + ): custom_config = AnthropicStructureConfig() custom_config.prompt_driver = AnthropicPromptDriver( model=prompt_model, temperature=temperature @@ -290,6 +325,7 @@ class gtUIAnthropicStructureConfig(gtUIBaseConfig): custom_config.image_query_driver = AnthropicImageQueryDriver( model=image_query_model ) + custom_config.image_generation_driver = image_generation_driver return (custom_config,) @@ -318,23 +354,38 @@ class gtUIOpenAiStructureConfig(gtUIBaseConfig): ) return inputs - def create(self, prompt_model, image_query_model, temperature, seed): + def create( + self, + prompt_model, + image_query_model, + temperature, + seed, + image_generation_driver=None, + ): OPENAI_API_KEY = get_config("env.OPENAI_API_KEY") - custom_config = StructureConfig( - prompt_driver=OpenAiChatPromptDriver( - model=prompt_model, - api_key=OPENAI_API_KEY, - temperature=temperature, - seed=seed, - ), - embedding_driver=OpenAiEmbeddingDriver(api_key=OPENAI_API_KEY), - image_generation_driver=OpenAiImageGenerationDriver( + prompt_driver = OpenAiChatPromptDriver( + model=prompt_model, + api_key=OPENAI_API_KEY, + temperature=temperature, + seed=seed, + ) + embedding_driver = OpenAiEmbeddingDriver(api_key=OPENAI_API_KEY) + if not image_generation_driver: + image_generation_driver = OpenAiImageGenerationDriver( api_key=OPENAI_API_KEY, model="dalle-e-3", - ), - image_query_driver=OpenAiImageQueryDriver( - api_key=OPENAI_API_KEY, model=image_query_model - ), + ) + + image_query_driver = OpenAiImageQueryDriver( + api_key=OPENAI_API_KEY, model=image_query_model + ) + + # OpenAiStructureConfig() + custom_config = OpenAiStructureConfig( + prompt_driver=prompt_driver, + embedding_driver=embedding_driver, + image_generation_driver=image_generation_driver, + image_query_driver=image_query_driver, ) return (custom_config,) diff --git a/nodes/tasks.py b/nodes/tasks.py index 5728bbc..4684d12 100644 --- a/nodes/tasks.py +++ b/nodes/tasks.py @@ -8,6 +8,7 @@ from griptape.drivers import ( AmazonBedrockImageQueryDriver, AnthropicImageQueryDriver, DummyAudioTranscriptionDriver, + DummyImageGenerationDriver, DummyImageQueryDriver, OpenAiAudioTranscriptionDriver, OpenAiImageGenerationDriver, @@ -176,10 +177,24 @@ class gtUIPromptImageGenerationTask(gtUIBaseTask): agent = Agent() prompt_text = self.get_prompt_text(STRING, input_string) + if not driver: - driver = OpenAiImageGenerationDriver( - model="dall-e-3", quality="hd", style="natural", api_key=OPENAI_API_KEY - ) + # Check and see if the agent.config has an image_generation_driver + if isinstance( + agent.config.image_generation_driver, DummyImageGenerationDriver + ): + # create a default driver + driver = OpenAiImageGenerationDriver( + model="dall-e-3", + quality="hd", + style="natural", + api_key=OPENAI_API_KEY, + ) + print( + "Current driver doesn't have an image_generation - using OpenAI by default." + ) + else: + driver = agent.config.image_generation_driver # Create an engine configured to use the driver. engine = PromptImageGenerationEngine( image_generation_driver=driver, From a0dbd70be0041c0a449c252d21d75adc71ec77c8 Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Mon, 1 Jul 2024 18:23:18 +1200 Subject: [PATCH 08/10] replaced ollama and lmstudio configs with base_urls and ports - but models are strings instead of being pulled from install --- nodes/config_nodes.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/nodes/config_nodes.py b/nodes/config_nodes.py index 08bd2e4..2271906 100644 --- a/nodes/config_nodes.py +++ b/nodes/config_nodes.py @@ -67,9 +67,13 @@ class gtUIEnv: ollama_models = get_ollama_models() ollama_models.append("") +ollama_port = "11434" +ollama_base_url = "http://localhost" lmstudio_models = get_lmstudio_models(port="1234") lmstudio_models.append("") +lmstudio_port = "1234" +lmstudio_base_url = "http://localhost" class LMStudioPromptDriver(OpenAiChatPromptDriver): @@ -107,9 +111,10 @@ class gtUILMStudioStructureConfig(gtUIBaseConfig): # {"default": lmstudio_models[0]}, # ), "prompt_model": ("STRING", {"default": ""}), + "base_url": ("STRING", {"default": lmstudio_base_url}), "port": ( "STRING", - {"default": "1234"}, + {"default": lmstudio_port}, ), }, ) @@ -118,6 +123,7 @@ class gtUILMStudioStructureConfig(gtUIBaseConfig): def create( self, prompt_model, + base_url, port, temperature, seed, @@ -126,7 +132,7 @@ class gtUILMStudioStructureConfig(gtUIBaseConfig): custom_config = StructureConfig( prompt_driver=LMStudioPromptDriver( model=prompt_model, - base_url=f"http://localhost:{port}/v1", + base_url=f"{base_url}:{port}/v1", api_key="lm_studio", temperature=temperature, tokenizer=SimpleTokenizer( @@ -154,9 +160,11 @@ class gtUIOllamaStructureConfig(gtUIBaseConfig): inputs["required"].update( { "prompt_model": ( - ollama_models, + "STRING", {"default": ollama_models[0]}, ), + "base_url": ("STRING", {"default": ollama_base_url}), + "port": ("STRING", {"default": ollama_port}), }, ) return inputs @@ -165,12 +173,16 @@ class gtUIOllamaStructureConfig(gtUIBaseConfig): self, prompt_model, temperature, + base_url, + port, seed, image_generation_driver=DummyImageGenerationDriver(), ): custom_config = StructureConfig( prompt_driver=OllamaPromptDriver( - model=prompt_model, options={"temperature": temperature} + model=prompt_model, + options={"temperature": temperature}, + host=f"{base_url}:{port}", ), image_generation_driver=image_generation_driver, ) From c706089b2f73763da96ecf30f16520d6d5c855a0 Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Tue, 2 Jul 2024 04:17:01 +1200 Subject: [PATCH 09/10] added changes to ollama and lm studio configs to provide ports and urls --- js/gtUINodes.js | 97 +++++++++++++++++++++++++++++++++++++++++-- nodes/config_nodes.py | 14 +++---- 2 files changed, 101 insertions(+), 10 deletions(-) diff --git a/js/gtUINodes.js b/js/gtUINodes.js index 3e1e5dd..46d9ab3 100644 --- a/js/gtUINodes.js +++ b/js/gtUINodes.js @@ -225,6 +225,68 @@ class GriptapeNodes extends EventTarget { document.head.appendChild(link); } } +async function getOllamaModels(baseUrl, port) { + const url = `${baseUrl}:${port}/api/tags`; + + try { + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Failed to fetch models: ${response.status}`); + } + + const modelsInfo = await response.json(); + + // Extract the model names + const models = modelsInfo.models.map(model => model.name); + + return models; + } catch (error) { + console.error("Error fetching Ollama models:", error); + return []; + } +} +async function getLMStudioModels(baseUrl, port) { + const url = `${baseUrl}:${port}/v1/models`; + + try { + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Failed to fetch models: ${response.status}`); + } + + const modelsInfo = await response.json(); + + // Extract the model names + const models = modelsInfo.data.map(model => model.id); + + return models; + } catch (error) { + console.error("Error fetching LM Studio models:", error); + return []; + } +} +async function updatePromptModelList(node, models) { + const modelWidget = node.widgets.find((w) => w.name === "prompt_model"); + const selectedItem = modelWidget.value; + + modelWidget.options.values = models; + console.log("Models" + models); + // if (models.length > 0) { + // modelWidget.value = models[0]; + // } + const warning = "No models returned from server" + if (models.length == 0){ + modelWidget.value = warning + } + if ((selectedItem === "" || selectedItem === null || selectedItem == warning) && models.length > 0 && !models.includes(selectedItem)) { + modelWidget.value = models[0]; + } + + +} + function gtUIAddUploadWidget(nodeType, nodeData, widgetName, type="audio") { chainCallback(nodeType.prototype, "onNodeCreated", function() { const pathWidget = this.widgets.find((w) => w.name === widgetName); @@ -380,18 +442,47 @@ app.registerExtension({ // this.color = getColor(nodeData.category); // }; - // Set Config node randomization to Fixed + + // Configuration Nodes if (nodeData.name.includes("Griptape Agent Config")) { const onNodeCreated = nodeType.prototype.onNodeCreated; nodeType.prototype.onNodeCreated = async function () { + + // Ollama Config Node + if (nodeData.name.includes("Ollama")) { + + // get the base_url + const base_url = this.widgets.find((w) => w.name === "base_url"); + const port = this.widgets.find((w) => w.name === "port"); + getOllamaModels(base_url.value, port.value).then((models) => { + const model = this.widgets.find((w) => w.name === "prompt_model"); + model.options.values = models; + updatePromptModelList(this, models); + }) + } + + // LMStudio Config Node + if (nodeData.name.includes("LM Studio")) { + + // get the base_url + const base_url = this.widgets.find((w) => w.name === "base_url"); + const port = this.widgets.find((w) => w.name === "port"); + getLMStudioModels(base_url.value, port.value).then((models) => { + const model = this.widgets.find((w) => w.name === "prompt_model"); + model.options.values = models; + updatePromptModelList(this, models); + }) + } + + + // Set Config node randomization to Fixed for (const widget of this.widgets) { if (widget.name === "control_after_generate") { widget.value = "fixed"; } - } } + } } - // Create Audio Node if (nodeData.name === "Griptape Load: Audio") { gtUIAddUploadWidget(nodeType, nodeData, "audio", "audio") diff --git a/nodes/config_nodes.py b/nodes/config_nodes.py index 2271906..2a33d36 100644 --- a/nodes/config_nodes.py +++ b/nodes/config_nodes.py @@ -106,11 +106,11 @@ class gtUILMStudioStructureConfig(gtUIBaseConfig): inputs = super().INPUT_TYPES() inputs["required"].update( { - # "prompt_model": ( - # lmstudio_models, - # {"default": lmstudio_models[0]}, - # ), - "prompt_model": ("STRING", {"default": ""}), + "prompt_model": ( + [], + {"default": ""}, + ), + # "prompt_model": ("STRING", {"default": ""}), "base_url": ("STRING", {"default": lmstudio_base_url}), "port": ( "STRING", @@ -160,8 +160,8 @@ class gtUIOllamaStructureConfig(gtUIBaseConfig): inputs["required"].update( { "prompt_model": ( - "STRING", - {"default": ollama_models[0]}, + [""], + {"default": ""}, ), "base_url": ("STRING", {"default": ollama_base_url}), "port": ("STRING", {"default": ollama_port}), From 7ebd1ddb780502834e05509c880722a5e76c7363 Mon Sep 17 00:00:00 2001 From: Jason Schleifer Date: Tue, 2 Jul 2024 04:48:53 +1200 Subject: [PATCH 10/10] fixed comment --- js/gtUINodes.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/js/gtUINodes.js b/js/gtUINodes.js index 46d9ab3..0e25482 100644 --- a/js/gtUINodes.js +++ b/js/gtUINodes.js @@ -162,7 +162,7 @@ class GriptapeNodes extends EventTarget { getGriptapeContextMenuItems() { const that = this; return [ - // TODO: Figure out how to pull and set the environment variables + // // TODO: Figure out how to pull and set the environment variables // { // content: "⚙️ Environment Variables", // callback: (...args) => { @@ -463,7 +463,8 @@ app.registerExtension({ // LMStudio Config Node if (nodeData.name.includes("LM Studio")) { - + chainCallback(nodeType.prototype, "onNodeUpdated", function() { + }); // get the base_url const base_url = this.widgets.find((w) => w.name === "base_url"); const port = this.widgets.find((w) => w.name === "port");