commit b85f77f685f1f6ee467df25c2232aded38b95419 Author: Sean Dunn Date: Mon Nov 4 21:31:48 2024 -0600 first commit! diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..61881b8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +__pycache__/ +*.py[cod] +/output/ +/input/ +!/input/example.png +/models/ +/temp/ +/custom_nodes/ +!custom_nodes/example_node.py.example +extra_model_paths.yaml +/.vs +.vscode/ +.idea/ +venv/ +.venv/ +/web/extensions/* +!/web/extensions/logging.js.example +!/web/extensions/core/ +/tests-ui/data/object_info.json +/user/ +*.log +web_custom_versions/ +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..de34606 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Sean E. Dunn + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..228644f --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# ComfyUI-DataBeast +This repository offers various nodes for data-driven processing in Comfy-UI. + +## Notice: +* V1.0 First release + +## Nodes + * TBD diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..25e8006 --- /dev/null +++ b/__init__.py @@ -0,0 +1,40 @@ +""" +@author: hanoixan +@title: DataBeast +@nickname: DataBeast +@description: This extension provides nodes for controlling data-driven processing in Comfy-UI +""" + +import importlib + +version_code = [1, 0] +version_str = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '') +print(f"### Loading: ComfyUI-DataBeast ({version_str})") + +node_list = [ + "conversion", + "general", + "math", + "string" +] + +NODE_CLASS_MAPPINGS = {} +NODE_DISPLAY_NAME_MAPPINGS = {} + +for module_name in node_list: + imported_module = importlib.import_module(".nodes.{}".format(module_name), __name__) + + NODE_CLASS_MAPPINGS = {**NODE_CLASS_MAPPINGS, **imported_module.NODE_CLASS_MAPPINGS} + NODE_DISPLAY_NAME_MAPPINGS = {**NODE_DISPLAY_NAME_MAPPINGS, **imported_module.NODE_DISPLAY_NAME_MAPPINGS} + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] + +try: + import cm_global + cm_global.register_extension('ComfyUI-DataBeast', + {'version': version_code, + 'name': 'DataBeast', + 'nodes': set(NODE_CLASS_MAPPINGS.keys()), + 'description': 'This extension provides nodes for data-driven processing in Comfy-UI', }) +except: + pass diff --git a/nodes/__init__.py b/nodes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nodes/conversion.py b/nodes/conversion.py new file mode 100644 index 0000000..1668b9f --- /dev/null +++ b/nodes/conversion.py @@ -0,0 +1,135 @@ +import json +import re +import hashlib +from .core import any_typ, DBItem, copy_visit_all_items + +class DBConvertToInt: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "input": (any_typ, {"defaultInput": None, "tooltip": "Any input type."}), + }, + } + + RETURN_TYPES = ("INT",) + RETURN_NAMES = ("output",) + + FUNCTION = "exec" + CATEGORY = "DataBeast" + + def exec(self, input): + output = 0 + + try: + if isinstance (input, DBItem): + output = int(input.item) + else: + output = int(input) + except Exception as ex: + pass + + return (output,) + +class DBConvertToFloat: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "input": (any_typ, {"defaultInput": None, "tooltip": "Any input type."}), + }, + } + + RETURN_TYPES = ("FLOAT",) + RETURN_NAMES = ("output",) + + FUNCTION = "exec" + CATEGORY = "DataBeast" + + def exec(self, input): + output = 0.0 + + try: + if isinstance (input, DBItem): + output = float(input.item) + else: + output = float(input) + except Exception as ex: + pass + + return (output,) + +class DBConvertToString: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "input": (any_typ, {"defaultInput": None, "tooltip": "Any input type."}), + "filesafe": ('BOOLEAN', {"defaultInput": False, "tooltip": "If True, safe for use in file paths."}), + }, + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("output",) + + FUNCTION = "exec" + CATEGORY = "DataBeast" + + def exec(self, input, filesafe): + output = "" + + def shorten_string_visitor(item, max_len): + if isinstance(item, str): + if len(item) > max_len: + hash4 = hashlib.sha256(item.encode('utf-8')).hexdigest()[:4].upper() + return f"{item[:max_len]}{hash4}" + else: + return item + else: + return item + + try: + # unwrap + if isinstance (input, DBItem): + input = input.item + + if isinstance(input, dict) or isinstance(input, list): + if filesafe: + def shortenize_string(s): + s = s.lower() + s = re.sub(r'[^a-zA-Z \t]', '', s) + s = re.sub(r'[aeiouAEIOU]', '', s) + s = ''.join(word.capitalize() for word in s.split()) + s = re.sub(r'[ \t]+', '', s) + return s + + def small_string_visitor(item): + if isinstance(item, str): + item = shortenize_string(item) + return shorten_string_visitor(item, 16) + + input = copy_visit_all_items(input, small_string_visitor) + + output = json.dumps(input) + else: + output = str(input) + except Exception as ex: + pass + + if filesafe: + # replace sequences of non-alphnum chars with _, and abbreviate all sequences of alpha-num to 6 max + output = re.sub(r'[^\w\-]', '_', output) + output = re.sub(r'_+', '_', output) + output = output.strip('_') + output = shorten_string_visitor(output, 64) + return (output,) + + +NODE_CLASS_MAPPINGS = { + "DBConvertToInt //DataBeast": DBConvertToInt, + "DBConvertToFloat //DataBeast": DBConvertToFloat, + "DBConvertToString //DataBeast": DBConvertToString, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "DBConvertToInt //DataBeast": "DB Convert to Int", + "DBConvertToFloat //DataBeast": "DB Convert to Float", + "DBConvertToString //DataBeast": "DB Convert to String", +} diff --git a/nodes/core.py b/nodes/core.py new file mode 100644 index 0000000..e878978 --- /dev/null +++ b/nodes/core.py @@ -0,0 +1,190 @@ +import re +import itertools + +from RestrictedPython import compile_restricted, safe_globals + +# inspired any-type solution from Inspire-Pack + +class AnyType(str): + def __ne__(self, __value: object) -> bool: + return False + +any_typ = AnyType("*") + +# + +class DBItem: + def __init__(self): + self.item=None + +# + +def safe_eval(expression, local_vars, title): + def safe_eval_getitem(ob, index): + return ob[index] + + safe_eval_globals = dict(safe_globals) + safe_eval_globals['_getitem_'] = safe_eval_getitem + + byte_code = compile_restricted(expression, title, "eval") + evaluated_result = eval(byte_code, safe_eval_globals, local_vars) + return evaluated_result + +# + +def resolve_all_string_expressions_r(obj, expr_pattern, ref_source, evaluated): + + def eval_expression(expression, ref_source): + # avoid infinite recursion + if expression in evaluated: + return evaluated[expression] + + result = '' + + try: + full_expression = f"ref_source{expression}" + local_vars = {"ref_source": ref_source} + evaluated_result = str(safe_eval(full_expression, local_vars, "")) + result = evaluated_result + except Exception as ex: + pass + + evaluated[expression] = result + return result + + if isinstance(obj, str): + return expr_pattern.sub(lambda m: eval_expression(m.group(2), ref_source), obj) + elif isinstance(obj, dict): + return {k: resolve_all_string_expressions_r(v, expr_pattern, ref_source, evaluated) for k,v in obj.items()} + elif isinstance(obj, list): + return [resolve_all_string_expressions_r(item, expr_pattern, ref_source, evaluated) for item in obj] + else: + return obj + +def resolve_all_string_expressions(obj, expr_pattern, ref_source): + # recursively find all strings in obj and replace ${item expression} with result. avoid infinite recursion. + evaluated = {} + return resolve_all_string_expressions_r(obj, expr_pattern, ref_source, evaluated) + +# + +def copy_visit_all_items(obj, visitor): + if isinstance(obj, dict): + return {copy_visit_all_items(k, visitor): copy_visit_all_items(v, visitor) for k,v in obj.items()} + elif isinstance(obj, list): + return [copy_visit_all_items(v, visitor) for v in obj] + else: + return visitor(obj) + +# + +def generate_permutation_list(desc): + try: + vars_expr_pattern = re.compile(r"(\$vars\{([^}]+)\})") + + vars = desc['vars'] + template = desc['template'] + + limit = -1 + if 'limit' in desc: + limit = desc['limit'] + + pvars = {} + + # generate vars lists + for vkey,vvalue in vars.items(): + if isinstance(vvalue, dict): + min = vvalue['min'] + max = vvalue['max'] + steps = vvalue['steps'] + pvalues = [] + if steps < 1: + pvalues = [] + elif steps == 1: + pvalues = [min] + else: + delta = max - min + dd = delta / (steps - 1) + for s in range(0,steps-1): + pvalues.append(min + dd * s) + pvalues.append(max) + elif isinstance(vvalue, list): + pvalues = vvalue + else: + pvalues = [vvalue] + + pvars[vkey] = pvalues + + def permute_lists(pvars): + pkeys = list(pvars.keys()) + values = list(pvars.values()) + + # Generate all possible permutations using itertools.product + permutations = list(itertools.product(*values)) + + # Create a list of dictionaries for each permutation + pdicts = [dict(zip(pkeys, perm)) for perm in permutations] + + return pdicts + + pdicts = permute_lists(pvars) + + out_list = [] + + # evaluate template using each purmutations dict + for pd in pdicts: + ret_pd = resolve_all_string_expressions(template, vars_expr_pattern, pd) + out_list.append(ret_pd) + + if limit > -1 and limit < len(out_list): + out_list = out_list[:limit] + + return out_list + except Exception as ex: + return [] + +def to_float(desc): + try: + return float(desc['expression']) + except Exception as ex: + return 0.0 + +valid_db_exec_functions = { + 'generate_permutation_list': generate_permutation_list, + 'to_float': to_float +} + + + +#### REMOVE + +# def resolve_references_r(obj, root_obj, evaluated): + +# def eval_expression(expression, root_obj): +# # avoid infinite recursion +# if expression in evaluated: +# return '' + +# evaluated[expression] = True + +# try: +# full_expression = f"root_obj{expression}" +# local_vars = {"root_obj": root_obj} +# evaluated_result = safe_eval(full_expression, local_vars, "") +# return evaluated_result +# except Exception as ex: +# return '' + +# if isinstance(obj, str): +# return REPLACE_EXPR_PATTERN.sub(lambda m: eval_expression(m.group(2), root_obj), obj) +# elif isinstance(obj, dict): +# return {k: resolve_references_r(v, root_obj, evaluated) for k,v in obj.items()} +# elif isinstance(obj, list): +# return [resolve_references_r(item, root_obj, evaluated) for item in obj] +# else: +# return obj + +# def resolve_references(obj): +# # recursively find all strings in obj and replace ${item expression} with result. avoid infinite recursion. +# evaluated = {} +# return resolve_references_r(obj, obj, evaluated) diff --git a/nodes/general.py b/nodes/general.py new file mode 100644 index 0000000..fc73f17 --- /dev/null +++ b/nodes/general.py @@ -0,0 +1,200 @@ +import csv +import json +import yaml +import re +import os.path + +from .core import DBItem, safe_eval, valid_db_exec_functions, resolve_all_string_expressions + +REPLACE_EXPR_PATTERN = re.compile(r"(\$\{([^}]+)\})") + +def resolve_db_exec_r(obj): + + def handle_db_exec(obj): + func_name = obj['db_exec'] + if func_name in valid_db_exec_functions: + ret_obj = valid_db_exec_functions[func_name](obj) + return ret_obj + else: + ret_obj = dict(obj) + ret_obj['failed_db_exec'] = func_name + return ret_obj + + if isinstance(obj, dict): + if 'db_exec' in obj: + # must handle anything created by a handler + handled = handle_db_exec(obj) + resolved = resolve_db_exec_r(handled) + return resolved + else: + new_dict = {k: resolve_db_exec_r(v) for k,v in obj.items()} + return new_dict + elif isinstance(obj, list): + new_list = [resolve_db_exec_r(item) for item in obj] + return new_list + else: + return obj + +def resolve_db_exec(obj): + # recursively find all values that are a db_exec command dictionary + # replace value in parent with whatever is returned + return resolve_db_exec_r(obj) + + +def filter_string(filter, exclusive, string): + filter = filter.strip() + if isinstance(filter, str) and len(filter) > 0: + ret_lines = [] + lines = string.splitlines() + for line in lines: + match = re.match(filter, line) + if match != None and not exclusive: + groups = match.groups() + if not (groups is None): + ret_lines.append("".join(groups)) + else: + ret_lines.append(line) + elif match == None and exclusive: + ret_lines.append(line) + string = "\n".join(ret_lines) + return string + +def parse_csv_list(csv_string): + csv_stream = StringIO(csv_string) + reader = csv.DictReader(csv_stream, skipinitialspace=True) + return list(reader) + +def parse_json_root(json_string): + out_list = [] + return json.loads(json_string) + +def parse_yaml_root(yaml_string): + out_list = [] + return yaml.safe_load(yaml_string) + +# + +class DBLoadData: + @classmethod + def IS_CHANGED(cls, *values): + return float("NaN") + + @classmethod + def INPUT_TYPES(s): + return {"required": { + "path": ("STRING", {"defaultInput": None, "tooltip": "Path to file to load. \n.json and .csv are treated as their formats, \nanything else is interepreted as YAML."}), + "filter": ("STRING", {"defaultInput": None, "tooltip": "Pattern filter. \nIf exclusive is true, only include lines \nthat DO NOT match this regex filter. If not exclusive, \ninclude ONLY lines that match this filter. If using \ncapture groups and not exclusive, all capture groups \nwill be concatenated into the final text for that line."}), + "exclusive": ("BOOLEAN", {"defaultInput": False, "tooltip": "Exclude with pattern. \nIf false, include only lines with the filter regex \npattern, if there is one."}) + }, + } + + RETURN_TYPES = ("DB_ITEM",) + RETURN_NAMES = ("data",) + + FUNCTION = "exec" + CATEGORY = "DataBeast" + + def exec(self, path, filter, exclusive): + # load path into list of dictionaries + out_root = {} + + try: + with open(path, "r") as loaded_file: + file_string = loaded_file.read() + file_string = filter_string(filter, exclusive, file_string) + ext = os.path.splitext(os.path.basename(path))[1].lower() + expression = "" + if ext == '.csv': + batches = parse_csv_list(file_string) + out_root = {'items': batches} + elif ext == '.json': + out_root = parse_json_root(file_string) + else: + out_root = parse_yaml_root(file_string) + + out_root = resolve_all_string_expressions(out_root, REPLACE_EXPR_PATTERN, out_root) + + out_root = resolve_db_exec(out_root) + + except Exception as ex: + pass + # TODO: popup an error + + out_root_item = DBItem() + out_root_item.item = out_root + + return (out_root_item,) + +class DBGetValue: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "source": ("DB_ITEM", {"defaultInput": None, "tooltip": "Source item to query."}), + "expression": ("STRING", {"defaultInput": None, "tooltip": "The expression to apply to the source, if necessary. \nEx: To get the 2nd index of the key \"foo\" assuming \nthe source is a table, the expression is: [\"foo\"][1]"}), + "default": ("STRING", {"defaultInput": None, "tooltip": "The value to use if the key doesn't exist."}), + }, + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("value",) + FUNCTION = "exec" + CATEGORY = "DataBeast" + + def exec(self, source, expression, default): + value = default + value = "" + try: + source_item = source.item + full_expression = "source_item{}".format(expression) + local_vars = {"source_item": source_item} + value = str(safe_eval(full_expression, local_vars, "")) + except Exception as ex: + # TODO: popup an error + pass + return (value,) + +class DBGetBatchList: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "source": ("DB_ITEM", {"defaultInput": None, "tooltip": "Source item to get data from."}), + "expression": ("STRING", {"defaultInput": None, "tooltip": "Expression to get batch list. \nA Python expression that when appended onto the \ninput data leads to a list in the loaded table \nto generate the batch items from.\n\nEx: If we assume inputs data is a table with \nan \"items\" key and list value, we would write: [\"items\"]"}) + }, + } + + RETURN_TYPES = ("DB_ITEM",) + RETURN_NAMES = ("item",) + OUTPUT_IS_LIST = (True,) + + FUNCTION = "exec" + CATEGORY = "DataBeast" + + def exec(self, source, expression): + try: + source_item = source.item + full_expression = "source_item{}".format(expression) + local_vars = {"source_item": source_item} + + out_batches = safe_eval(full_expression, local_vars, "") + except Exception as ex: + out_batches = [] + + out_list = [] + for d in out_batches: + item = DBItem() + item.item = d + out_list.append(item) + + return (out_list,) + +NODE_CLASS_MAPPINGS = { + "DBLoadData //DataBeast": DBLoadData, + "DBGetValue //DataBeast": DBGetValue, + "DBGetBatchList //DataBeast": DBGetBatchList +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "DBLoadData //DataBeast": "DB Load Data", + "DBGetValue //DataBeast": "DB Get Value", + "DBGetBatchList //DataBeast": "DB Get Batch List", +} diff --git a/nodes/math.py b/nodes/math.py new file mode 100644 index 0000000..effeedf --- /dev/null +++ b/nodes/math.py @@ -0,0 +1,37 @@ +from .core import any_typ, DBItem, safe_eval + +class DBFloatExpression: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "expression": ("STRING", {"defaultInput": None, "tooltip": "Python math expression taking any of inputs a,b,c,d."}), + "a": ("FLOAT", {"defaultInput": None, "tooltip": "Input a."}), + "b": ("FLOAT", {"defaultInput": None, "tooltip": "Input b."}), + "c": ("FLOAT", {"defaultInput": None, "tooltip": "Input c."}), + "d": ("FLOAT", {"defaultInput": None, "tooltip": "Input d."}), + }, + } + + RETURN_TYPES = ("FLOAT",) + RETURN_NAMES = ("output",) + + FUNCTION = "exec" + CATEGORY = "DataBeast" + + def exec(self, expression, a, b, c, d): + output = 0 + try: + local_vars = {"a": a, "b": b, "c": c, "d": d} + output = float(safe_eval(expression, local_vars, "")) + except Exception as ex: + pass + + return (output,) + +NODE_CLASS_MAPPINGS = { + "DBFloatExpression //DataBeast": DBFloatExpression, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "DBFloatExpression //DataBeast": "DB Float Expression", +} diff --git a/nodes/string.py b/nodes/string.py new file mode 100644 index 0000000..03957c4 --- /dev/null +++ b/nodes/string.py @@ -0,0 +1,37 @@ +from .core import any_typ, DBItem, safe_eval + +class DBStringExpression: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "expression": ("STRING", {"defaultInput": None, "tooltip": "Python string expression taking any of inputs a,b,c,d."}), + "a": ("STRING", {"defaultInput": None, "tooltip": "Input a."}), + "b": ("STRING", {"defaultInput": None, "tooltip": "Input b."}), + "c": ("STRING", {"defaultInput": None, "tooltip": "Input c."}), + "d": ("STRING", {"defaultInput": None, "tooltip": "Input d."}), + }, + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("output",) + + FUNCTION = "exec" + CATEGORY = "DataBeast" + + def exec(self, expression, a, b, c, d): + output = 0 + try: + local_vars = {"a": a, "b": b, "c": c, "d": d} + output = str(safe_eval(expression, local_vars, "")) + except Exception as ex: + pass + + return (output,) + +NODE_CLASS_MAPPINGS = { + "DBStringExpression //DataBeast": DBStringExpression, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "DBStringExpression //DataBeast": "DB String Expression", +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3ba41ea --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "comfyui-databeast" +description = "This extension provides convenience nodes for batch processing." +version = "1.0" +license = { file = "LICENSE" } +dependencies = ["pyyaml", "RestrictedPython"] + +[project.urls] +Repository = "https://github.com/hanoixan/ComfyUI-DataBeast" + +[tool.comfy] +PublisherId = "hanoixan" +DisplayName = "ComfyUI DataBeast" +Icon = "" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0d82dac --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pyyaml +RestrictedPython \ No newline at end of file