From 71ce3e3266758c3bbe7c11a5ac58e006b94f2fda Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 12:22:19 -0800 Subject: [PATCH 01/26] WIP --- .gitignore | 4 ++++ README.md | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 68bc17f..e8e5aa5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +_venv/ + +############### + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index 672ab67..860d4dc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # ComfyUI-Keyframed -[Work In Progress] ComfyUI nodes to facilitate value keyframing + +🚧 Work In Progress 🚧 - ComfyUI nodes to facilitate value keyframing \ No newline at end of file From 9987ea1fd5ce69c6cb27d1bf27563bd82836085f Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 13:03:45 -0800 Subject: [PATCH 02/26] hmm --- README.md | 5 ++++- __init__.py | 0 nodes.py | 4 ++++ requirements.txt | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 __init__.py create mode 100644 nodes.py create mode 100644 requirements.txt diff --git a/README.md b/README.md index 860d4dc..5c6e14c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ # ComfyUI-Keyframed -🚧 Work In Progress 🚧 - ComfyUI nodes to facilitate value keyframing \ No newline at end of file +🚧 Work In Progress 🚧 - ComfyUI nodes to facilitate value keyframing by providing an interface for using [keyframed](https://github.com/dmarx/keyframed) in ComfyUI workflows. + + +...Open question: if I make this, what will differentiate it from https://github.com/FizzleDorf/ComfyUI_FizzNodes ? \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nodes.py b/nodes.py new file mode 100644 index 0000000..f181243 --- /dev/null +++ b/nodes.py @@ -0,0 +1,4 @@ +import keyframed as kf + +CATEGORY = "keyframed" + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b020240 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +keyframed \ No newline at end of file From 6e8f892f0ac9e97b69a916bef205121b39a9b50c Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 13:57:55 -0800 Subject: [PATCH 03/26] fleshing it out --- README.md | 9 ++++- __init__.py | 3 ++ example.py | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++ nodes.py | 73 +++++++++++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 example.py diff --git a/README.md b/README.md index 5c6e14c..c04c515 100644 --- a/README.md +++ b/README.md @@ -3,4 +3,11 @@ 🚧 Work In Progress 🚧 - ComfyUI nodes to facilitate value keyframing by providing an interface for using [keyframed](https://github.com/dmarx/keyframed) in ComfyUI workflows. -...Open question: if I make this, what will differentiate it from https://github.com/FizzleDorf/ComfyUI_FizzNodes ? \ No newline at end of file +...Open question: if I make this, what will differentiate it from https://github.com/FizzleDorf/ComfyUI_FizzNodes ? + +* easier curve composition +* easier to change interpolators/easing functions + +# Philosophy + +Curves, interpolators, and keyframes are objects that can be passed around, plugged and unplugged, and interchanged. \ No newline at end of file diff --git a/__init__.py b/__init__.py index e69de29..186760a 100644 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,3 @@ +from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +__all__ =["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/example.py b/example.py new file mode 100644 index 0000000..0b61415 --- /dev/null +++ b/example.py @@ -0,0 +1,102 @@ +class Example: + """ + A example node + + Class methods + ------------- + INPUT_TYPES (dict): + Tell the main program input parameters of nodes. + + Attributes + ---------- + RETURN_TYPES (`tuple`): + The type of each element in the output tulple. + RETURN_NAMES (`tuple`): + Optional: The name of each output in the output tulple. + FUNCTION (`str`): + The name of the entry-point method. For example, if `FUNCTION = "execute"` then it will run Example().execute() + OUTPUT_NODE ([`bool`]): + If this node is an output node that outputs a result/image from the graph. The SaveImage node is an example. + The backend iterates on these output nodes and tries to execute all their parents if their parent graph is properly connected. + Assumed to be False if not present. + CATEGORY (`str`): + The category the node should appear in the UI. + execute(s) -> tuple || None: + The entry point method. The name of this method must be the same as the value of property `FUNCTION`. + For example, if `FUNCTION = "execute"` then this method's name must be `execute`, if `FUNCTION = "foo"` then it must be `foo`. + """ + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(s): + """ + Return a dictionary which contains config for all input fields. + Some types (string): "MODEL", "VAE", "CLIP", "CONDITIONING", "LATENT", "IMAGE", "INT", "STRING", "FLOAT". + Input types "INT", "STRING" or "FLOAT" are special values for fields on the node. + The type can be a list for selection. + + Returns: `dict`: + - Key input_fields_group (`string`): Can be either required, hidden or optional. A node class must have property `required` + - Value input_fields (`dict`): Contains input fields config: + * Key field_name (`string`): Name of a entry-point method's argument + * Value field_config (`tuple`): + + First value is a string indicate the type of field or a list for selection. + + Secound value is a config for type "INT", "STRING" or "FLOAT". + """ + return { + "required": { + "image": ("IMAGE",), + "int_field": ("INT", { + "default": 0, + "min": 0, #Minimum value + "max": 4096, #Maximum value + "step": 64, #Slider's step + "display": "number" # Cosmetic only: display as "number" or "slider" + }), + "float_field": ("FLOAT", { + "default": 1.0, + "min": 0.0, + "max": 10.0, + "step": 0.01, + "round": 0.001, #The value represeting the precision to round to, will be set to the step value by default. Can be set to False to disable rounding. + "display": "number"}), + "print_to_screen": (["enable", "disable"],), + "string_field": ("STRING", { + "multiline": False, #True if you want the field to look like the one on the ClipTextEncode node + "default": "Hello World!" + }), + }, + } + + RETURN_TYPES = ("IMAGE",) + #RETURN_NAMES = ("image_output_name",) + + FUNCTION = "test" + + #OUTPUT_NODE = False + + CATEGORY = "Example" + + def test(self, image, string_field, int_field, float_field, print_to_screen): + if print_to_screen == "enable": + print(f"""Your input contains: + string_field aka input text: {string_field} + int_field: {int_field} + float_field: {float_field} + """) + #do some processing on the image, in this example I just invert it + image = 1.0 - image + return (image,) + + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "Example": Example +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "Example": "Example Node" +} \ No newline at end of file diff --git a/nodes.py b/nodes.py index f181243..8193bfb 100644 --- a/nodes.py +++ b/nodes.py @@ -1,4 +1,77 @@ import keyframed as kf +from keyframed.dsl import curve_from_cn_string CATEGORY = "keyframed" +class _BASE: + CATEGORY=CATEGORY + FUNCTION = 'main' + +class KfCurveFromString(_BASE): + RETURN_TYPES = ("KEYFRAMED_CURVE",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": {"chigozie_string": ("STRING", { + "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node + "default": "0: 1" + }), + }, + } + + def main(self, chigozie_string): + return curve_from_cn_string(chigozie_string) + +class KfCurveFromYAML(_BASE): + RETURN_TYPES = ("KEYFRAMED_CURVE",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": {"yaml": ("STRING", { + "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node + "default": """curve: +- - 0 + - 0 + - linear +- - 1 + - 1 +loop: false +bounce: false +duration: 1 +label: foo""" + }), + }, + } + + def main(self, yaml): + return kf.serialization.from_yaml(yaml) + +class KfEvaluateCurveAtT(_BASE): + RETURN_TYPES = ("FLOAT","INT") + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "curve": ("KEYFRAMED_CURVE",), + "t": ("INT",) + }, + } + + def main(self, curve, t): + return curve[t] + +NODE_CLASS_MAPPINGS = { + "KfCurveFromString": KfCurveFromString, + "KfCurveFromYAML": KfCurveFromYAML, + "KfEvaluateCurveAtT": KfEvaluateCurveAtT, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "KfCurveFromString": "Curve From String", + "KfCurveFromYAML": "Curve From YAML", + "KfEvaluateCurveAtT": "Evaluate Curve At T", +} \ No newline at end of file From 8aeedc64caa02b5467831a1398dde49a822a7605 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 14:03:32 -0800 Subject: [PATCH 04/26] install... --- __init__.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/__init__.py b/__init__.py index 186760a..b6df696 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,24 @@ from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS __all__ =["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] + +# scavenged install sequence from https://github.com/FizzleDorf/ComfyUI_FizzNodes/blob/main/__init__.py +def is_installed(package, package_overwrite=None): + try: + spec = importlib.util.find_spec(package) + except ModuleNotFoundError: + pass + + package = package_overwrite or package + + if spec is None: + print(f"Installing {package}...") + command = f'"{python}" -m pip install {package}' + + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ) + + if result.returncode != 0: + print(f"Couldn't install\nCommand: {command}\nError code: {result.returncode}") + +# to do: read from requirements.txt +is_installed("keyframed") \ No newline at end of file From 7ba4a9031af9c50d0f9c604d928c5c06f4b1266f Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 14:04:50 -0800 Subject: [PATCH 05/26] changed order --- __init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/__init__.py b/__init__.py index b6df696..1fc3084 100644 --- a/__init__.py +++ b/__init__.py @@ -1,7 +1,3 @@ -from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS - -__all__ =["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] - # scavenged install sequence from https://github.com/FizzleDorf/ComfyUI_FizzNodes/blob/main/__init__.py def is_installed(package, package_overwrite=None): try: @@ -21,4 +17,8 @@ def is_installed(package, package_overwrite=None): print(f"Couldn't install\nCommand: {command}\nError code: {result.returncode}") # to do: read from requirements.txt -is_installed("keyframed") \ No newline at end of file +is_installed("keyframed") + +from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +__all__ =["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] From 74b611ffd6d959a7a6c5f6cfd08031a0432d868e Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 14:06:27 -0800 Subject: [PATCH 06/26] missing imports --- __init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/__init__.py b/__init__.py index 1fc3084..cb6a4dc 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,7 @@ +import os +import subprocess +import importlib.util + # scavenged install sequence from https://github.com/FizzleDorf/ComfyUI_FizzNodes/blob/main/__init__.py def is_installed(package, package_overwrite=None): try: From 890c2d55ef3710c2b07486cee898edc3ff409b25 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 14:07:45 -0800 Subject: [PATCH 07/26] executable --- __init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/__init__.py b/__init__.py index cb6a4dc..06c0f77 100644 --- a/__init__.py +++ b/__init__.py @@ -1,6 +1,7 @@ import os import subprocess import importlib.util +import sys # scavenged install sequence from https://github.com/FizzleDorf/ComfyUI_FizzNodes/blob/main/__init__.py def is_installed(package, package_overwrite=None): @@ -11,6 +12,7 @@ def is_installed(package, package_overwrite=None): package = package_overwrite or package + python = sys.executable if spec is None: print(f"Installing {package}...") command = f'"{python}" -m pip install {package}' From 798cc0acb6cfedb669474f836d807afe6e4f7196 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 14:22:46 -0800 Subject: [PATCH 08/26] IT'S ALIVE!!!! --- nodes.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/nodes.py b/nodes.py index 8193bfb..e006cfd 100644 --- a/nodes.py +++ b/nodes.py @@ -3,11 +3,9 @@ from keyframed.dsl import curve_from_cn_string CATEGORY = "keyframed" -class _BASE: +class KfCurveFromString: CATEGORY=CATEGORY FUNCTION = 'main' - -class KfCurveFromString(_BASE): RETURN_TYPES = ("KEYFRAMED_CURVE",) @classmethod @@ -23,7 +21,9 @@ class KfCurveFromString(_BASE): def main(self, chigozie_string): return curve_from_cn_string(chigozie_string) -class KfCurveFromYAML(_BASE): +class KfCurveFromYAML: + CATEGORY=CATEGORY + FUNCTION = 'main' RETURN_TYPES = ("KEYFRAMED_CURVE",) @classmethod @@ -48,20 +48,22 @@ label: foo""" def main(self, yaml): return kf.serialization.from_yaml(yaml) -class KfEvaluateCurveAtT(_BASE): +class KfEvaluateCurveAtT: + CATEGORY=CATEGORY + FUNCTION = 'main' RETURN_TYPES = ("FLOAT","INT") @classmethod def INPUT_TYPES(s): return { "required": { - "curve": ("KEYFRAMED_CURVE",), - "t": ("INT",) + "curve": ("KEYFRAMED_CURVE",{"forceInput": True,}), + "t": ("INT",{"default":0}) }, } def main(self, curve, t): - return curve[t] + return curve[t], int(curve[t]) NODE_CLASS_MAPPINGS = { "KfCurveFromString": KfCurveFromString, @@ -73,5 +75,5 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { "KfCurveFromString": "Curve From String", "KfCurveFromYAML": "Curve From YAML", - "KfEvaluateCurveAtT": "Evaluate Curve At T", + "KfEvaluateCurveAtT": "EvaluateCurveAtT", } \ No newline at end of file From 99cd12e1505f59d231a8dfdc24c23f8a253279bf Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 17:57:11 -0800 Subject: [PATCH 09/26] skeleton ACN compatibility --- nodes.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index e006cfd..324bb09 100644 --- a/nodes.py +++ b/nodes.py @@ -1,8 +1,10 @@ import keyframed as kf from keyframed.dsl import curve_from_cn_string +import warnings CATEGORY = "keyframed" + class KfCurveFromString: CATEGORY=CATEGORY FUNCTION = 'main' @@ -21,6 +23,7 @@ class KfCurveFromString: def main(self, chigozie_string): return curve_from_cn_string(chigozie_string) + class KfCurveFromYAML: CATEGORY=CATEGORY FUNCTION = 'main' @@ -47,7 +50,8 @@ label: foo""" def main(self, yaml): return kf.serialization.from_yaml(yaml) - + + class KfEvaluateCurveAtT: CATEGORY=CATEGORY FUNCTION = 'main' @@ -65,10 +69,30 @@ class KfEvaluateCurveAtT: def main(self, curve, t): return curve[t], int(curve[t]) + +class KfCurveToAcnLatentKeyframe: + CATEGORY=CATEGORY + FUNCTION = 'main' + RETURN_NAMES = ("LATENT_KF", ) + RETURN_TYPES = ("LATENT_KEYFRAME",) + """Compatibility with Kosinkadink "Advanced Controlnet" AnimateDiff""" + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "curve": ("KEYFRAMED_CURVE",{"forceInput": True,}), + }, + } + def main(self, curve): + warnings.warn("KfCurveToAcnLatentKeyframe not implemented") + return curve + + NODE_CLASS_MAPPINGS = { "KfCurveFromString": KfCurveFromString, "KfCurveFromYAML": KfCurveFromYAML, "KfEvaluateCurveAtT": KfEvaluateCurveAtT, + "KfCurveToAcnLatentKeyframe": KfCurveToAcnLatentKeyframe, } # A dictionary that contains the friendly/humanly readable titles for the nodes @@ -76,4 +100,5 @@ NODE_DISPLAY_NAME_MAPPINGS = { "KfCurveFromString": "Curve From String", "KfCurveFromYAML": "Curve From YAML", "KfEvaluateCurveAtT": "EvaluateCurveAtT", + "KfCurveToAcnLatentKeyframe": "Curve to ACN Latent Keyframe", } \ No newline at end of file From f815a2a818a70d2ea553695970529ada8dfafdbb Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 20:27:12 -0800 Subject: [PATCH 10/26] debug pass through --- __init__.py | 9 +++++++++ debug.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 debug.py diff --git a/__init__.py b/__init__.py index 06c0f77..af13844 100644 --- a/__init__.py +++ b/__init__.py @@ -27,4 +27,13 @@ is_installed("keyframed") from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS +print(os.environ.get('COMFYUI_DEBUG_MODE')) + +from .debug import NODE_CLASS_MAPPINGS as ncm0, NODE_DISPLAY_NAME_MAPPINGS as ndnm0 + +# there's probably a cleaner, more-dummy-proof way to do this. +# feels like an accident waiting to happen. low risk though. +NODE_CLASS_MAPPINGS.update(ncm0) +NODE_DISPLAY_NAME_MAPPINGS.update(ndnm0) + __all__ =["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/debug.py b/debug.py new file mode 100644 index 0000000..dda3fcf --- /dev/null +++ b/debug.py @@ -0,0 +1,47 @@ +import logging + +logging.basicConfig(level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +CATEGORY="keyframed/debug" + +class KfDebug_Passthrough: + CATEGORY=CATEGORY + FUNCTION = 'main' + # RETURN_TYPES = ("FLOAT","INT") + + # @classmethod + # def INPUT_TYPES(s): + # return { + # "required": { + # "curve": ("KEYFRAMED_CURVE",{"forceInput": True,}), + # "t": ("INT",{"default":0}) + # }, + # } + + def main(self, item): + logger.info(f"type: {type(item)}") + logger.info(f"item: {item}") + return (item,) # pretty sure it's gotta be a tuple? + +class KfDebug_Float(KfDebug_Passthrough): + CATEGORY=CATEGORY + FUNCTION = 'main' + RETURN_TYPES = ("FLOAT",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "item": ("FLOAT",{"forceInput": True,}), + }, + } + +NODE_CLASS_MAPPINGS = { + #"KfDebug_Passthrough": KfDebug_Passthrough, + "KfDebug_Float": KfDebug_Float, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = {k:k for k in NODE_CLASS_MAPPINGS} \ No newline at end of file From edea3f4be1a4beef8c41fff518f1556f401ead55 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 4 Dec 2023 21:45:52 -0800 Subject: [PATCH 11/26] improved debugging messages --- debug.py | 71 +++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/debug.py b/debug.py index dda3fcf..d7d6625 100644 --- a/debug.py +++ b/debug.py @@ -1,4 +1,7 @@ import logging +import torch +import numpy as np + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -6,25 +9,51 @@ logger = logging.getLogger(__name__) CATEGORY="keyframed/debug" +# i could probably use icecream or something like that for this +def _inspect(item, depth=0): + pad="\t"*depth + if depth > 0: + pad +="- " + # NB: Linter says using f-strings in log statements can hinder performance + logger.info(f"{pad}type: {type(item)}") + log_item=True + + if hasattr(item, "shape"): + logger.info(f"{pad}item.shape: {item.shape}") + log_item=False + elif hasattr(item, "size"): + logger.info(f"{pad}item.shape: {item.size}") + #logger.info(f"{pad}item.shape: {item.size()}") + log_item=False + + # to do: be fancy and change to a match statement + #if isinstance(item, dict): + if hasattr(item, 'keys'): + logger.info(f"{pad}item.keys(): {item.keys()}") + if hasattr(item, 'items'): + for k,v in item.items(): + logger.info(f"{pad}key: {k}") + logger.info(f"{pad}value: {_inspect(v, depth=depth+1)}") + log_item=False + + if isinstance(item, list): + logger.info(f"{pad}len(item): {len(item)}") + for entry in item: + _inspect(entry, depth=depth+1) + log_item=False + + if log_item: + logger.info(f"{pad}item: {item}") + + class KfDebug_Passthrough: CATEGORY=CATEGORY FUNCTION = 'main' - # RETURN_TYPES = ("FLOAT","INT") - - # @classmethod - # def INPUT_TYPES(s): - # return { - # "required": { - # "curve": ("KEYFRAMED_CURVE",{"forceInput": True,}), - # "t": ("INT",{"default":0}) - # }, - # } - def main(self, item): - logger.info(f"type: {type(item)}") - logger.info(f"item: {item}") + _inspect(item) return (item,) # pretty sure it's gotta be a tuple? + class KfDebug_Float(KfDebug_Passthrough): CATEGORY=CATEGORY FUNCTION = 'main' @@ -38,9 +67,25 @@ class KfDebug_Float(KfDebug_Passthrough): }, } + +class KfDebug_Cond(KfDebug_Passthrough): + CATEGORY=CATEGORY + FUNCTION = 'main' + RETURN_TYPES = ("CONDITIONING",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "item": ("CONDITIONING",{"forceInput": True,}), + }, + } + + NODE_CLASS_MAPPINGS = { #"KfDebug_Passthrough": KfDebug_Passthrough, "KfDebug_Float": KfDebug_Float, + "KfDebug_Cond": KfDebug_Cond, } # A dictionary that contains the friendly/humanly readable titles for the nodes From 60f2f42b0f12fce2e2e90cb77fe2526b21b023cc Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 01:06:07 -0800 Subject: [PATCH 12/26] added custom labeling for debug points --- debug.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/debug.py b/debug.py index d7d6625..94b6a23 100644 --- a/debug.py +++ b/debug.py @@ -49,23 +49,36 @@ def _inspect(item, depth=0): class KfDebug_Passthrough: CATEGORY=CATEGORY FUNCTION = 'main' - def main(self, item): + + _FORCED_INPUT = {"label": ("STRING", { + "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node + "default": ""})} + + def main(self, item, label): + #if label: + logger.info(f"label: {label}") _inspect(item) return (item,) # pretty sure it's gotta be a tuple? +########################### + + class KfDebug_Float(KfDebug_Passthrough): CATEGORY=CATEGORY FUNCTION = 'main' RETURN_TYPES = ("FLOAT",) @classmethod - def INPUT_TYPES(s): - return { + def INPUT_TYPES(cls): + outv= { "required": { "item": ("FLOAT",{"forceInput": True,}), }, } + outv["required"].update(cls._FORCED_INPUT) + #logger.debug(outv) + return outv class KfDebug_Cond(KfDebug_Passthrough): @@ -74,12 +87,42 @@ class KfDebug_Cond(KfDebug_Passthrough): RETURN_TYPES = ("CONDITIONING",) @classmethod - def INPUT_TYPES(s): - return { + def INPUT_TYPES(cls): + outv = { "required": { "item": ("CONDITIONING",{"forceInput": True,}), }, } + outv["required"].update(cls._FORCED_INPUT) + return outv + + +# class KfDebug_(KfDebug_Passthrough): +# CATEGORY=CATEGORY +# FUNCTION = 'main' +# RETURN_TYPES = ("CONDITIONING",) + +# @classmethod +# def INPUT_TYPES(s): +# return { +# "required": { +# "item": ("CONDITIONING",{"forceInput": True,}), +# }, +# } + + +# class KfDebug_(KfDebug_Passthrough): +# CATEGORY=CATEGORY +# FUNCTION = 'main' +# RETURN_TYPES = ("CONDITIONING",) + +# @classmethod +# def INPUT_TYPES(s): +# return { +# "required": { +# "item": ("CONDITIONING",{"forceInput": True,}), +# }, +# } NODE_CLASS_MAPPINGS = { From 8bd1a39767f4a940afa10259f81be6ee785104dd Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 02:03:16 -0800 Subject: [PATCH 13/26] refactored, added dummy output --- debug.py | 80 +++++++++++++++++++++++--------------------------------- nodes.py | 8 +++--- 2 files changed, 37 insertions(+), 51 deletions(-) diff --git a/debug.py b/debug.py index 94b6a23..2bc2fbc 100644 --- a/debug.py +++ b/debug.py @@ -52,7 +52,17 @@ class KfDebug_Passthrough: _FORCED_INPUT = {"label": ("STRING", { "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node - "default": ""})} + "default": "debugging passthrough"})} + + @classmethod + def INPUT_TYPES(cls): + outv = { + "required": { + "item": (cls.RETURN_TYPES[0],{"forceInput": True,}), + }, + } + outv["required"].update(cls._FORCED_INPUT) + return outv def main(self, item, label): #if label: @@ -61,74 +71,48 @@ class KfDebug_Passthrough: return (item,) # pretty sure it's gotta be a tuple? +class KfDebug_DummyOutput(KfDebug_Passthrough): + OUTPUT_NODE=True + _FORCED_INPUT = {"label": ("STRING", { + "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node + "default": "dummy output"})} + ########################### +# there should be a way to create a type-agnostic passthrough node class KfDebug_Float(KfDebug_Passthrough): - CATEGORY=CATEGORY - FUNCTION = 'main' RETURN_TYPES = ("FLOAT",) - @classmethod - def INPUT_TYPES(cls): - outv= { - "required": { - "item": ("FLOAT",{"forceInput": True,}), - }, - } - outv["required"].update(cls._FORCED_INPUT) - #logger.debug(outv) - return outv - class KfDebug_Cond(KfDebug_Passthrough): - CATEGORY=CATEGORY - FUNCTION = 'main' RETURN_TYPES = ("CONDITIONING",) - @classmethod - def INPUT_TYPES(cls): - outv = { - "required": { - "item": ("CONDITIONING",{"forceInput": True,}), - }, - } - outv["required"].update(cls._FORCED_INPUT) - return outv + +class KfDebug_Curve(KfDebug_Passthrough): + RETURN_TYPES = ("KEYFRAMED_CURVE",) -# class KfDebug_(KfDebug_Passthrough): -# CATEGORY=CATEGORY -# FUNCTION = 'main' -# RETURN_TYPES = ("CONDITIONING",) - -# @classmethod -# def INPUT_TYPES(s): -# return { -# "required": { -# "item": ("CONDITIONING",{"forceInput": True,}), -# }, -# } +class KfDebug_Latent(KfDebug_Passthrough): + RETURN_TYPES = ("LATENT",) -# class KfDebug_(KfDebug_Passthrough): -# CATEGORY=CATEGORY -# FUNCTION = 'main' -# RETURN_TYPES = ("CONDITIONING",) +########################### -# @classmethod -# def INPUT_TYPES(s): -# return { -# "required": { -# "item": ("CONDITIONING",{"forceInput": True,}), -# }, -# } +class KfDebugDummy_Curve(KfDebug_DummyOutput): + RETURN_TYPES = ("KEYFRAMED_CURVE",) + + +########################### NODE_CLASS_MAPPINGS = { #"KfDebug_Passthrough": KfDebug_Passthrough, "KfDebug_Float": KfDebug_Float, "KfDebug_Cond": KfDebug_Cond, + "KfDebug_Curve": KfDebug_Curve, + "KfDebug_Latent": KfDebug_Latent, + "KfDebugDummy_Curve": KfDebugDummy_Curve, } # A dictionary that contains the friendly/humanly readable titles for the nodes diff --git a/nodes.py b/nodes.py index 324bb09..ea5fbd6 100644 --- a/nodes.py +++ b/nodes.py @@ -21,7 +21,8 @@ class KfCurveFromString: } def main(self, chigozie_string): - return curve_from_cn_string(chigozie_string) + curve = curve_from_cn_string(chigozie_string) + return (curve,) class KfCurveFromYAML: @@ -49,7 +50,8 @@ label: foo""" } def main(self, yaml): - return kf.serialization.from_yaml(yaml) + curve = kf.serialization.from_yaml(yaml) + return (curve,) class KfEvaluateCurveAtT: @@ -85,7 +87,7 @@ class KfCurveToAcnLatentKeyframe: } def main(self, curve): warnings.warn("KfCurveToAcnLatentKeyframe not implemented") - return curve + return (curve,) NODE_CLASS_MAPPINGS = { From 115cd1371f3268accb19df78e98e0c6da6ba3082 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 10:48:25 -0800 Subject: [PATCH 14/26] added a bunch of debug nodes --- debug.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- nodes.py | 1 + 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/debug.py b/debug.py index 2bc2fbc..5b8f5c4 100644 --- a/debug.py +++ b/debug.py @@ -1,7 +1,7 @@ import logging import torch import numpy as np - +from PIL.Image import Image logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -9,7 +9,8 @@ logger = logging.getLogger(__name__) CATEGORY="keyframed/debug" -# i could probably use icecream or something like that for this +# maybe use icecream here instead? +# https://github.com/gruns/icecream def _inspect(item, depth=0): pad="\t"*depth if depth > 0: @@ -18,13 +19,20 @@ def _inspect(item, depth=0): logger.info(f"{pad}type: {type(item)}") log_item=True + # maybe a bit overengineered. whatever. if hasattr(item, "shape"): logger.info(f"{pad}item.shape: {item.shape}") log_item=False elif hasattr(item, "size"): - logger.info(f"{pad}item.shape: {item.size}") - #logger.info(f"{pad}item.shape: {item.size()}") + try: + logger.info(f"{pad}item.shape: {item.size()}") + except TypeError: + logger.info(f"{pad}item.shape: {item.size}") log_item=False + + if isinstance(item, Image): + logger.info(f"{pad}item.mode: {item.mode}") + # to do: be fancy and change to a match statement #if isinstance(item, dict): @@ -97,12 +105,46 @@ class KfDebug_Latent(KfDebug_Passthrough): RETURN_TYPES = ("LATENT",) +# Some types (string): "MODEL", "VAE", "CLIP", "IMAGE", "INT", "STRING" +class KfDebug_Model(KfDebug_Passthrough): + RETURN_TYPES = ("MODEL",) + + +class KfDebug_Vae(KfDebug_Passthrough): + RETURN_TYPES = ("VAE",) + + +class KfDebug_Clip(KfDebug_Passthrough): + RETURN_TYPES = ("Clip",) + + +class KfDebug_Image(KfDebug_Passthrough): + RETURN_TYPES = ("Image",) + + +class KfDebug_Int(KfDebug_Passthrough): + RETURN_TYPES = ("INT",) + + +class KfDebug_String(KfDebug_Passthrough): + RETURN_TYPES = ("STRING",) + + +class KfDebug_Segs(KfDebug_Passthrough): + RETURN_TYPES = ("SEGS",) + + ########################### class KfDebugDummy_Curve(KfDebug_DummyOutput): RETURN_TYPES = ("KEYFRAMED_CURVE",) +class KfDebugDummy_Segs(KfDebug_DummyOutput): + RETURN_TYPES = ("SEGS",) + + + ########################### @@ -113,6 +155,13 @@ NODE_CLASS_MAPPINGS = { "KfDebug_Curve": KfDebug_Curve, "KfDebug_Latent": KfDebug_Latent, "KfDebugDummy_Curve": KfDebugDummy_Curve, + "KfDebug_Model": KfDebug_Model, + "KfDebug_Vae": KfDebug_Vae, + "KfDebug_Clip": KfDebug_Clip, + "KfDebug_Image": KfDebug_Image, + "KfDebug_Int": KfDebug_Int, + "KfDebug_String": KfDebug_String, + "KfDebug_Segs": KfDebug_Segs, } # A dictionary that contains the friendly/humanly readable titles for the nodes diff --git a/nodes.py b/nodes.py index ea5fbd6..6565412 100644 --- a/nodes.py +++ b/nodes.py @@ -35,6 +35,7 @@ class KfCurveFromYAML: return { "required": {"yaml": ("STRING", { "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node + # TO DO: replace this with kf.serializaiton.to_dict() # or whatever "default": """curve: - - 0 - 0 From 030431ff5728b051507d97e885720a155c0c8d61 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 11:00:15 -0800 Subject: [PATCH 15/26] all passthrough nodes output nodes. no dummy reqd --- debug.py | 70 ++++++++++++++++++++++++++------------------------------ 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/debug.py b/debug.py index 5b8f5c4..5626d2f 100644 --- a/debug.py +++ b/debug.py @@ -57,6 +57,7 @@ def _inspect(item, depth=0): class KfDebug_Passthrough: CATEGORY=CATEGORY FUNCTION = 'main' + OUTPUT_NODE=True _FORCED_INPUT = {"label": ("STRING", { "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node @@ -79,73 +80,69 @@ class KfDebug_Passthrough: return (item,) # pretty sure it's gotta be a tuple? -class KfDebug_DummyOutput(KfDebug_Passthrough): - OUTPUT_NODE=True - _FORCED_INPUT = {"label": ("STRING", { - "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node - "default": "dummy output"})} +# class KfDebug_DummyOutput(KfDebug_Passthrough): +# OUTPUT_NODE=True +# _FORCED_INPUT = {"label": ("STRING", { +# "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node +# "default": "dummy output"})} ########################### +### Built-in Types + + # there should be a way to create a type-agnostic passthrough node -class KfDebug_Float(KfDebug_Passthrough): - RETURN_TYPES = ("FLOAT",) +class KfDebug_Clip(KfDebug_Passthrough): + RETURN_TYPES = ("CLIP",) class KfDebug_Cond(KfDebug_Passthrough): RETURN_TYPES = ("CONDITIONING",) -class KfDebug_Curve(KfDebug_Passthrough): - RETURN_TYPES = ("KEYFRAMED_CURVE",) - - -class KfDebug_Latent(KfDebug_Passthrough): - RETURN_TYPES = ("LATENT",) - - -# Some types (string): "MODEL", "VAE", "CLIP", "IMAGE", "INT", "STRING" -class KfDebug_Model(KfDebug_Passthrough): - RETURN_TYPES = ("MODEL",) - - -class KfDebug_Vae(KfDebug_Passthrough): - RETURN_TYPES = ("VAE",) - - -class KfDebug_Clip(KfDebug_Passthrough): - RETURN_TYPES = ("Clip",) +class KfDebug_Float(KfDebug_Passthrough): + RETURN_TYPES = ("FLOAT",) class KfDebug_Image(KfDebug_Passthrough): - RETURN_TYPES = ("Image",) + RETURN_TYPES = ("IMAGE",) class KfDebug_Int(KfDebug_Passthrough): RETURN_TYPES = ("INT",) +class KfDebug_Latent(KfDebug_Passthrough): + RETURN_TYPES = ("LATENT",) + + +class KfDebug_Model(KfDebug_Passthrough): + RETURN_TYPES = ("MODEL",) + + class KfDebug_String(KfDebug_Passthrough): RETURN_TYPES = ("STRING",) +class KfDebug_Vae(KfDebug_Passthrough): + RETURN_TYPES = ("VAE",) + + +############################################## + +### Custom Node Types + + class KfDebug_Segs(KfDebug_Passthrough): RETURN_TYPES = ("SEGS",) -########################### - -class KfDebugDummy_Curve(KfDebug_DummyOutput): +class KfDebug_Curve(KfDebug_Passthrough): RETURN_TYPES = ("KEYFRAMED_CURVE",) -class KfDebugDummy_Segs(KfDebug_DummyOutput): - RETURN_TYPES = ("SEGS",) - - - -########################### +# ########################### NODE_CLASS_MAPPINGS = { @@ -154,7 +151,6 @@ NODE_CLASS_MAPPINGS = { "KfDebug_Cond": KfDebug_Cond, "KfDebug_Curve": KfDebug_Curve, "KfDebug_Latent": KfDebug_Latent, - "KfDebugDummy_Curve": KfDebugDummy_Curve, "KfDebug_Model": KfDebug_Model, "KfDebug_Vae": KfDebug_Vae, "KfDebug_Clip": KfDebug_Clip, From 136b633f84b7b5e636f149aa24ab9eebf9146765 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 11:03:14 -0800 Subject: [PATCH 16/26] alphabetical --- debug.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/debug.py b/debug.py index 5626d2f..35fadbc 100644 --- a/debug.py +++ b/debug.py @@ -147,17 +147,17 @@ class KfDebug_Curve(KfDebug_Passthrough): NODE_CLASS_MAPPINGS = { #"KfDebug_Passthrough": KfDebug_Passthrough, - "KfDebug_Float": KfDebug_Float, + "KfDebug_Clip": KfDebug_Clip, "KfDebug_Cond": KfDebug_Cond, "KfDebug_Curve": KfDebug_Curve, - "KfDebug_Latent": KfDebug_Latent, - "KfDebug_Model": KfDebug_Model, - "KfDebug_Vae": KfDebug_Vae, - "KfDebug_Clip": KfDebug_Clip, + "KfDebug_Float": KfDebug_Float, "KfDebug_Image": KfDebug_Image, "KfDebug_Int": KfDebug_Int, - "KfDebug_String": KfDebug_String, + "KfDebug_Latent": KfDebug_Latent, + "KfDebug_Model": KfDebug_Model, "KfDebug_Segs": KfDebug_Segs, + "KfDebug_String": KfDebug_String, + "KfDebug_Vae": KfDebug_Vae, } # A dictionary that contains the friendly/humanly readable titles for the nodes From d2fc17a76da4078a5d3f0a0f61988fc7290cd06f Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 13:38:28 -0800 Subject: [PATCH 17/26] improved debug message --- debug.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug.py b/debug.py index 35fadbc..aa900eb 100644 --- a/debug.py +++ b/debug.py @@ -41,7 +41,8 @@ def _inspect(item, depth=0): if hasattr(item, 'items'): for k,v in item.items(): logger.info(f"{pad}key: {k}") - logger.info(f"{pad}value: {_inspect(v, depth=depth+1)}") + #logger.info(f"{pad}value: {_inspect(v, depth=depth+1)}") + _inspect(v, depth=depth+1) log_item=False if isinstance(item, list): From 44b24cdb50db92d790cf701606db4e649e5a8aee Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 13:39:09 -0800 Subject: [PATCH 18/26] added KfApplyCurveToCond --- nodes.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/nodes.py b/nodes.py index 6565412..a913ec3 100644 --- a/nodes.py +++ b/nodes.py @@ -1,6 +1,14 @@ import keyframed as kf from keyframed.dsl import curve_from_cn_string -import warnings +import logging +import torch +#import warnings + + +logging.basicConfig(level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + CATEGORY = "keyframed" @@ -73,35 +81,93 @@ class KfEvaluateCurveAtT: return curve[t], int(curve[t]) -class KfCurveToAcnLatentKeyframe: +# class KfCurveToAcnLatentKeyframe: +# CATEGORY=CATEGORY +# FUNCTION = 'main' +# RETURN_NAMES = ("LATENT_KF", ) +# RETURN_TYPES = ("LATENT_KEYFRAME",) +# """Compatibility with Kosinkadink "Advanced Controlnet" AnimateDiff""" +# @classmethod +# def INPUT_TYPES(s): +# return { +# "required": { +# "curve": ("KEYFRAMED_CURVE",{"forceInput": True,}), +# }, +# } +# def main(self, curve): +# warnings.warn("KfCurveToAcnLatentKeyframe not implemented") +# return (curve,) + + +class KfApplyCurveToCond: CATEGORY=CATEGORY FUNCTION = 'main' - RETURN_NAMES = ("LATENT_KF", ) - RETURN_TYPES = ("LATENT_KEYFRAME",) - """Compatibility with Kosinkadink "Advanced Controlnet" AnimateDiff""" + #RETURN_TYPES = ("CONDITIONING","LATENT_KEYFRAME",) + RETURN_TYPES = ("CONDITIONING",) + @classmethod def INPUT_TYPES(s): return { "required": { - "curve": ("KEYFRAMED_CURVE",{"forceInput": True,}), + "curve": ("KEYFRAMED_CURVE", {"forceInput": True,}), + "cond": ("CONDITIONING", {"forceInput": True,}), + }, + "optional":{ + "latents": ("LATENT", {}), + "start_t": ("INT", {"default":0, }), + "n": ("INT", {}), }, } - def main(self, curve): - warnings.warn("KfCurveToAcnLatentKeyframe not implemented") - return (curve,) + def main(self, curve, cond, latents=None, start_t=0, n=0): + logger.info(f"latents: {latents}") + #if latents is not None: + device = 'cpu' # probably should be handling this some other way + if isinstance(latents, torch.Tensor): + n = latents.shape[0] # batch dimension + device = latents.device + weights = [curve[start_t+i] for i in range(n)] + weights = torch.tensor(weights, device=device) + cond_out = [] + for c_tensor, c_dict in cond: + weights.to(c_tensor.device) + if c_tensor.shape[0] == 1: + c_tensor = c_tensor.repeat(n, 1, 1) # batch, n_tokens, embeding_dim + + logger.info(f"c_tensor.shape:{c_tensor.shape}") + logger.info(f"weights.shape:{weights.shape}") + logger.info(f"weights.shape:{weights.view(n,1,1).shape}") + #c_tensor.mul_(weights) + c_tensor.mul_(weights.view(n,1,1)) + #c_tensor = c_tensor * weights + #c_tensor = c_tensor + if "pooled_output" in c_dict: + pooled = c_dict['pooled_output'] + if pooled.shape[0] == 1: + pooled = pooled.repeat(n, 1) # batch, embeding_dim + #pooled.mul_(weights) + c_dict['pooled_output'] = pooled * weights.view(n,1) + cond_out.append((c_tensor, c_dict)) + return (cond_out,) + #outv = torch.ones_like(latents) * torch.tensor(weights, device=latents.device) + #return (cond, outv) + + +################################################################## NODE_CLASS_MAPPINGS = { "KfCurveFromString": KfCurveFromString, "KfCurveFromYAML": KfCurveFromYAML, "KfEvaluateCurveAtT": KfEvaluateCurveAtT, - "KfCurveToAcnLatentKeyframe": KfCurveToAcnLatentKeyframe, + "KfApplyCurveToCond": KfApplyCurveToCond, + #"KfCurveToAcnLatentKeyframe": KfCurveToAcnLatentKeyframe, } # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "KfCurveFromString": "Curve From String", "KfCurveFromYAML": "Curve From YAML", - "KfEvaluateCurveAtT": "EvaluateCurveAtT", - "KfCurveToAcnLatentKeyframe": "Curve to ACN Latent Keyframe", + "KfEvaluateCurveAtT": "Evaluate Curve At T", + "KfApplyCurveToCond": "Apply Curve to Conditioning" + #"KfCurveToAcnLatentKeyframe": "Curve to ACN Latent Keyframe", } \ No newline at end of file From cb652039048c8827f7e4cd90c8c53fab273b00d2 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 14:34:26 -0800 Subject: [PATCH 19/26] latents -> latents['samples'] --- nodes.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nodes.py b/nodes.py index a913ec3..90480ae 100644 --- a/nodes.py +++ b/nodes.py @@ -23,7 +23,7 @@ class KfCurveFromString: return { "required": {"chigozie_string": ("STRING", { "multiline": True, #True if you want the field to look like the one on the ClipTextEncode node - "default": "0: 1" + "default": "0:(1)" }), }, } @@ -119,12 +119,14 @@ class KfApplyCurveToCond: }, } def main(self, curve, cond, latents=None, start_t=0, n=0): - logger.info(f"latents: {latents}") - #if latents is not None: + #logger.info(f"latents: {latents}") + logger.info(f"type(latents): {type(latents)}") # Latent is a dict that (presently) has one key, `samples` device = 'cpu' # probably should be handling this some other way - if isinstance(latents, torch.Tensor): - n = latents.shape[0] # batch dimension - device = latents.device + #if latents is not None: + if isinstance(latents, dict): + if 'samples' in latents: + n = latents['samples'].shape[0] # batch dimension + device = latents['samples'].device weights = [curve[start_t+i] for i in range(n)] weights = torch.tensor(weights, device=device) cond_out = [] From 37beb6015359fdddba197b1ca2e2491fca477409 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 16:29:18 -0800 Subject: [PATCH 20/26] simple conditioning add --- debug.py | 2 +- nodes.py | 47 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/debug.py b/debug.py index aa900eb..439cfba 100644 --- a/debug.py +++ b/debug.py @@ -45,7 +45,7 @@ def _inspect(item, depth=0): _inspect(v, depth=depth+1) log_item=False - if isinstance(item, list): + if isinstance(item, list) or isinstance(item, tuple): logger.info(f"{pad}len(item): {len(item)}") for entry in item: _inspect(entry, depth=depth+1) diff --git a/nodes.py b/nodes.py index 90480ae..47fecb6 100644 --- a/nodes.py +++ b/nodes.py @@ -132,22 +132,23 @@ class KfApplyCurveToCond: cond_out = [] for c_tensor, c_dict in cond: weights.to(c_tensor.device) + m=c_tensor.shape[0] if c_tensor.shape[0] == 1: c_tensor = c_tensor.repeat(n, 1, 1) # batch, n_tokens, embeding_dim - - logger.info(f"c_tensor.shape:{c_tensor.shape}") - logger.info(f"weights.shape:{weights.shape}") - logger.info(f"weights.shape:{weights.view(n,1,1).shape}") + m=n + #logger.info(f"c_tensor.shape:{c_tensor.shape}") + #logger.info(f"weights.shape:{weights.shape}") + #logger.info(f"weights.shape:{weights.view(n,1,1).shape}") #c_tensor.mul_(weights) - c_tensor.mul_(weights.view(n,1,1)) + c_tensor.mul_(weights.view(m,1,1)) #c_tensor = c_tensor * weights #c_tensor = c_tensor if "pooled_output" in c_dict: pooled = c_dict['pooled_output'] if pooled.shape[0] == 1: - pooled = pooled.repeat(n, 1) # batch, embeding_dim + pooled = pooled.repeat(m, 1) # batch, embeding_dim #pooled.mul_(weights) - c_dict['pooled_output'] = pooled * weights.view(n,1) + c_dict['pooled_output'] = pooled * weights.view(m,1) cond_out.append((c_tensor, c_dict)) return (cond_out,) @@ -155,6 +156,34 @@ class KfApplyCurveToCond: #return (cond, outv) +# TODO: Add Conds +#class ConditioningAverage: +class KfConditioningAdd: + @classmethod + def INPUT_TYPES(s): + return {"required": {"conditioning_1": ("CONDITIONING", ), + "conditioning_2": ("CONDITIONING", ), + }} + RETURN_TYPES = ("CONDITIONING",) + FUNCTION = "main" + + CATEGORY = "conditioning" + + def main(self, conditioning_1, conditioning_2): + assert len(conditioning_1) == len(conditioning_2) + + outv = [] + for i, ((c1_tensor, c1_dict), (c2_tensor, c2_dict) ) in enumerate(zip(conditioning_1, conditioning_2)): + c1_tensor += c2_tensor + if ('pooled_output' in c1_dict) and ('pooled_output' in c2_dict): + c1_dict['pooled_output'] += c2_dict['pooled_output'] + outv.append((c1_tensor, c1_dict)) + return (outv, ) + +# TODO: Add Curves (to compute normalization) + +# TODO: Divide Cond By Curve --> add "" + ################################################################## NODE_CLASS_MAPPINGS = { @@ -162,6 +191,7 @@ NODE_CLASS_MAPPINGS = { "KfCurveFromYAML": KfCurveFromYAML, "KfEvaluateCurveAtT": KfEvaluateCurveAtT, "KfApplyCurveToCond": KfApplyCurveToCond, + "KfConditioningAdd": KfConditioningAdd, #"KfCurveToAcnLatentKeyframe": KfCurveToAcnLatentKeyframe, } @@ -170,6 +200,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "KfCurveFromString": "Curve From String", "KfCurveFromYAML": "Curve From YAML", "KfEvaluateCurveAtT": "Evaluate Curve At T", - "KfApplyCurveToCond": "Apply Curve to Conditioning" + "KfApplyCurveToCond": "Apply Curve to Conditioning", + "KfConditioningAdd": "Add Conditions" #"KfCurveToAcnLatentKeyframe": "Curve to ACN Latent Keyframe", } \ No newline at end of file From 377d19725cb6dfc624dcdfc19cf02454f6ff8cd2 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 20:23:10 -0800 Subject: [PATCH 21/26] experimenting with manual normalization --- nodes.py | 69 +++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/nodes.py b/nodes.py index 47fecb6..7359cf1 100644 --- a/nodes.py +++ b/nodes.py @@ -2,6 +2,7 @@ import keyframed as kf from keyframed.dsl import curve_from_cn_string import logging import torch +from copy import deepcopy #import warnings @@ -121,29 +122,32 @@ class KfApplyCurveToCond: def main(self, curve, cond, latents=None, start_t=0, n=0): #logger.info(f"latents: {latents}") logger.info(f"type(latents): {type(latents)}") # Latent is a dict that (presently) has one key, `samples` - device = 'cpu' # probably should be handling this some other way + #device = 'cpu' # probably should be handling this some other way #if latents is not None: if isinstance(latents, dict): if 'samples' in latents: n = latents['samples'].shape[0] # batch dimension - device = latents['samples'].device - weights = [curve[start_t+i] for i in range(n)] - weights = torch.tensor(weights, device=device) + #device = latents['samples'].device + #weights = [curve[start_t+i] for i in range(n)] + #weights = torch.tensor(weights, device=device) cond_out = [] for c_tensor, c_dict in cond: - weights.to(c_tensor.device) + #weights.to(c_tensor.device) m=c_tensor.shape[0] if c_tensor.shape[0] == 1: c_tensor = c_tensor.repeat(n, 1, 1) # batch, n_tokens, embeding_dim m=n + weights = [curve[start_t+i] for i in range(m)] + weights = torch.tensor(weights, device=c_tensor.device) #logger.info(f"c_tensor.shape:{c_tensor.shape}") #logger.info(f"weights.shape:{weights.shape}") #logger.info(f"weights.shape:{weights.view(n,1,1).shape}") #c_tensor.mul_(weights) - c_tensor.mul_(weights.view(m,1,1)) - #c_tensor = c_tensor * weights + #c_tensor.mul_(weights.view(m,1,1)) # I think these in-place/mutating operations are messing things up + c_tensor = c_tensor * weights.view(m,1,1) #c_tensor = c_tensor if "pooled_output" in c_dict: + c_dict = deepcopy(c_dict) # hate this. pooled = c_dict['pooled_output'] if pooled.shape[0] == 1: pooled = pooled.repeat(m, 1) # batch, embeding_dim @@ -159,15 +163,15 @@ class KfApplyCurveToCond: # TODO: Add Conds #class ConditioningAverage: class KfConditioningAdd: + CATEGORY = CATEGORY + FUNCTION = "main" + RETURN_TYPES = ("CONDITIONING",) + @classmethod def INPUT_TYPES(s): return {"required": {"conditioning_1": ("CONDITIONING", ), "conditioning_2": ("CONDITIONING", ), }} - RETURN_TYPES = ("CONDITIONING",) - FUNCTION = "main" - - CATEGORY = "conditioning" def main(self, conditioning_1, conditioning_2): assert len(conditioning_1) == len(conditioning_2) @@ -180,9 +184,46 @@ class KfConditioningAdd: outv.append((c1_tensor, c1_dict)) return (outv, ) -# TODO: Add Curves (to compute normalization) -# TODO: Divide Cond By Curve --> add "" +# TODO: Add Curves (to compute normalization) +class KfCurvesAdd: + CATEGORY = CATEGORY + FUNCTION = "main" + RETURN_TYPES = ("KEYFRAMED_CURVE",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "curve_1": ("KEYFRAMED_CURVE",{"forceInput": True,}), + "curve_2": ("KEYFRAMED_CURVE",{"forceInput": True,}), + }, + } + + def main(self, curve_1, curve_2): + return (curve_1 + curve_2, ) + +class KfCurveInverse: + CATEGORY = CATEGORY + FUNCTION = "main" + RETURN_TYPES = ("KEYFRAMED_CURVE",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "curve": ("KEYFRAMED_CURVE",{"forceInput": True,}), + }, + "hidden": { + "a": ("FLOAT", {"default": 0.0001}), + }, + } + + def main(self, curve, a=0.0001): + curve = curve + a + curve = 1/curve + return (curve,) + ################################################################## @@ -192,6 +233,8 @@ NODE_CLASS_MAPPINGS = { "KfEvaluateCurveAtT": KfEvaluateCurveAtT, "KfApplyCurveToCond": KfApplyCurveToCond, "KfConditioningAdd": KfConditioningAdd, + "KfCurvesAdd": KfCurvesAdd, + "KfCurveInverse": KfCurveInverse, #"KfCurveToAcnLatentKeyframe": KfCurveToAcnLatentKeyframe, } From 8c687ba4426d9485cc8109197d3e001c70a3558c Mon Sep 17 00:00:00 2001 From: David Date: Tue, 5 Dec 2023 23:13:44 -0800 Subject: [PATCH 22/26] rough 'draw' node --- debug.py | 4 ++++ nodes.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/debug.py b/debug.py index 439cfba..8082700 100644 --- a/debug.py +++ b/debug.py @@ -19,6 +19,10 @@ def _inspect(item, depth=0): logger.info(f"{pad}type: {type(item)}") log_item=True + if hasattr(item, "dtype"): + logger.info(f"{pad}item.dtype: {item.dtype}") + log_item=False + # maybe a bit overengineered. whatever. if hasattr(item, "shape"): logger.info(f"{pad}item.shape: {item.shape}") diff --git a/nodes.py b/nodes.py index 7359cf1..a5c681e 100644 --- a/nodes.py +++ b/nodes.py @@ -4,6 +4,10 @@ import logging import torch from copy import deepcopy #import warnings +import matplotlib.pyplot as plt +import numpy as np +import io +from PIL import Image logging.basicConfig(level=logging.INFO, @@ -203,6 +207,7 @@ class KfCurvesAdd: def main(self, curve_1, curve_2): return (curve_1 + curve_2, ) + class KfCurveInverse: CATEGORY = CATEGORY FUNCTION = "main" @@ -225,6 +230,50 @@ class KfCurveInverse: return (curve,) +class KfCurveDraw: + CATEGORY = CATEGORY + FUNCTION = "main" + RETURN_TYPES = ("IMAGE",) + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "curve": ("KEYFRAMED_CURVE",) + } + } + + def main(self, curve): + """ + + """ + # Create a figure and axes object + fig, ax = plt.subplots() + + # Build the plot using the provided function + #build_plot(ax) + #curve.plot(ax=ax) + curve.plot() + width, height = 10, 5 #inches + plt.figure(figsize=(width, height)) + + # Save the plot to a BytesIO object + buf = io.BytesIO() + plt.savefig(buf, format='png', bbox_inches='tight') + buf.seek(0) + + # Read the image into a numpy array, converting it to RGB mode + pil_image = Image.open(buf).convert('RGB') + plot_array = np.array(pil_image) #.astype(np.uint8) + + # Convert the array to the desired shape [batch, channels, width, height] + #plot_array = np.transpose(plot_array, (2, 0, 1)) # Reorder to [channels, width, height] + #plot_array = np.expand_dims(plot_array, axis=0) # Add the batch dimension + #plot_array = torch.tensor(plot_array) #.float() + plot_array = torch.from_numpy(plot_array) + return (plot_array,) + + ################################################################## NODE_CLASS_MAPPINGS = { @@ -235,6 +284,7 @@ NODE_CLASS_MAPPINGS = { "KfConditioningAdd": KfConditioningAdd, "KfCurvesAdd": KfCurvesAdd, "KfCurveInverse": KfCurveInverse, + "KfCurveDraw": KfCurveDraw, #"KfCurveToAcnLatentKeyframe": KfCurveToAcnLatentKeyframe, } From c6559e02f74ecf306646ea8bdefe5911beaa13ba Mon Sep 17 00:00:00 2001 From: David Date: Wed, 6 Dec 2023 00:04:31 -0800 Subject: [PATCH 23/26] nodes for curve arithmetic --- nodes.py | 144 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 104 insertions(+), 40 deletions(-) diff --git a/nodes.py b/nodes.py index a5c681e..5742b8e 100644 --- a/nodes.py +++ b/nodes.py @@ -189,49 +189,30 @@ class KfConditioningAdd: return (outv, ) -# TODO: Add Curves (to compute normalization) -class KfCurvesAdd: - CATEGORY = CATEGORY - FUNCTION = "main" - RETURN_TYPES = ("KEYFRAMED_CURVE",) +# class KfCurveInverse: +# CATEGORY = CATEGORY +# FUNCTION = "main" +# RETURN_TYPES = ("KEYFRAMED_CURVE",) - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "curve_1": ("KEYFRAMED_CURVE",{"forceInput": True,}), - "curve_2": ("KEYFRAMED_CURVE",{"forceInput": True,}), - }, - } - - def main(self, curve_1, curve_2): - return (curve_1 + curve_2, ) - - -class KfCurveInverse: - CATEGORY = CATEGORY - FUNCTION = "main" - RETURN_TYPES = ("KEYFRAMED_CURVE",) - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "curve": ("KEYFRAMED_CURVE",{"forceInput": True,}), - }, - "hidden": { - "a": ("FLOAT", {"default": 0.0001}), - }, - } +# @classmethod +# def INPUT_TYPES(s): +# return { +# "required": { +# "curve": ("KEYFRAMED_CURVE",{"forceInput": True,}), +# }, +# "hidden": { +# "a": ("FLOAT", {"default": 0.0001}), +# }, +# } - def main(self, curve, a=0.0001): - curve = curve + a - curve = 1/curve - return (curve,) +# def main(self, curve, a=0.0001): +# curve = curve + a +# curve = 1/curve +# return (curve,) class KfCurveDraw: - CATEGORY = CATEGORY + CATEGORY = f"{CATEGORY}/experimental" FUNCTION = "main" RETURN_TYPES = ("IMAGE",) @@ -273,6 +254,82 @@ class KfCurveDraw: plot_array = torch.from_numpy(plot_array) return (plot_array,) +########################################### + +# curve arithmetic + +# TODO: Add Curves (to compute normalization) +class KfCurvesAdd: + CATEGORY = CATEGORY + FUNCTION = "main" + RETURN_TYPES = ("KEYFRAMED_CURVE",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "curve_1": ("KEYFRAMED_CURVE",{"forceInput": True,}), + "curve_2": ("KEYFRAMED_CURVE",{"forceInput": True,}), + }, + } + + def main(self, curve_1, curve_2): + return (curve_1 + curve_2, ) + + +class KfCurvesSubtract: + CATEGORY = CATEGORY + FUNCTION = "main" + RETURN_TYPES = ("KEYFRAMED_CURVE",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "curve_1": ("KEYFRAMED_CURVE",{"forceInput": True,}), + "curve_2": ("KEYFRAMED_CURVE",{"forceInput": True,}), + }, + } + + def main(self, curve_1, curve_2): + return (curve_1 - curve_2, ) + + +class KfCurvesMultiply: + CATEGORY = CATEGORY + FUNCTION = "main" + RETURN_TYPES = ("KEYFRAMED_CURVE",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "curve_1": ("KEYFRAMED_CURVE",{"forceInput": True,}), + "curve_2": ("KEYFRAMED_CURVE",{"forceInput": True,}), + }, + } + + def main(self, curve_1, curve_2): + return (curve_1 * curve_2, ) + + +class KfCurvesDivide: + CATEGORY = CATEGORY + FUNCTION = "main" + RETURN_TYPES = ("KEYFRAMED_CURVE",) + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "curve_1": ("KEYFRAMED_CURVE",{"forceInput": True,}), + "curve_2": ("KEYFRAMED_CURVE",{"forceInput": True,}), + }, + } + + def main(self, curve_1, curve_2): + return (curve_1 / curve_2, ) + ################################################################## @@ -282,9 +339,12 @@ NODE_CLASS_MAPPINGS = { "KfEvaluateCurveAtT": KfEvaluateCurveAtT, "KfApplyCurveToCond": KfApplyCurveToCond, "KfConditioningAdd": KfConditioningAdd, - "KfCurvesAdd": KfCurvesAdd, "KfCurveInverse": KfCurveInverse, "KfCurveDraw": KfCurveDraw, + "KfCurvesAdd": KfCurvesAdd, + "KfCurvesSubtract": KfCurvesSubtract, + "KfCurvesMultiply": KfCurvesMultiply, + "KfCurvesDivide": KfCurvesDivide, #"KfCurveToAcnLatentKeyframe": KfCurveToAcnLatentKeyframe, } @@ -294,6 +354,10 @@ NODE_DISPLAY_NAME_MAPPINGS = { "KfCurveFromYAML": "Curve From YAML", "KfEvaluateCurveAtT": "Evaluate Curve At T", "KfApplyCurveToCond": "Apply Curve to Conditioning", - "KfConditioningAdd": "Add Conditions" + "KfConditioningAdd": "Add Conditions", #"KfCurveToAcnLatentKeyframe": "Curve to ACN Latent Keyframe", + "KfCurvesAdd": "Curve_1 + Curve_2", + "KfCurvesSubtract": "Curve_1 - Curve_2", + "KfCurvesMultiply": "Curve_1 * Curve_2", + "KfCurvesDivide": "Curve_1 / Curve_2", } \ No newline at end of file From cdfebee77cefe5f7e1d5904750571f6901219fb9 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 6 Dec 2023 00:10:49 -0800 Subject: [PATCH 24/26] notes --- nodes.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nodes.py b/nodes.py index 5742b8e..5427af4 100644 --- a/nodes.py +++ b/nodes.py @@ -331,6 +331,18 @@ class KfCurvesDivide: return (curve_1 / curve_2, ) +################################################################## + +# Working with parameter groupd + +# Create parameter group + +# add curve(s) to parameter group + +# get curve from parameter group + + + ################################################################## NODE_CLASS_MAPPINGS = { From 9788f239f1505e1a4bbe42658beaefb5fd15fde1 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 6 Dec 2023 00:13:02 -0800 Subject: [PATCH 25/26] suppress deprecated node --- nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index 5427af4..122c497 100644 --- a/nodes.py +++ b/nodes.py @@ -351,7 +351,7 @@ NODE_CLASS_MAPPINGS = { "KfEvaluateCurveAtT": KfEvaluateCurveAtT, "KfApplyCurveToCond": KfApplyCurveToCond, "KfConditioningAdd": KfConditioningAdd, - "KfCurveInverse": KfCurveInverse, + #"KfCurveInverse": KfCurveInverse, "KfCurveDraw": KfCurveDraw, "KfCurvesAdd": KfCurvesAdd, "KfCurvesSubtract": KfCurvesSubtract, From 4d44f6d7c37276c32ffd48b977576f1aea19366e Mon Sep 17 00:00:00 2001 From: David Date: Wed, 6 Dec 2023 10:26:49 -0800 Subject: [PATCH 26/26] notes --- nodes.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nodes.py b/nodes.py index 122c497..42bc7fb 100644 --- a/nodes.py +++ b/nodes.py @@ -333,14 +333,22 @@ class KfCurvesDivide: ################################################################## -# Working with parameter groupd +#### Working with parameter groups -# Create parameter group + +# Label curve +## inputs: curve, label (text widget) # add curve(s) to parameter group +## inputs: pgroup, curve +## returns pgroup +## if pgroup not provided, new one created # get curve from parameter group +## inputs: pgroup, label +## returns curve +# extract a time slice from the parameter group ##################################################################