From 2a7fac69c5ce753886ffb584896f8d2bd1b63edd Mon Sep 17 00:00:00 2001 From: RyanOnTheInside <7623207+ryanontheinside@users.noreply.github.com> Date: Sun, 24 Nov 2024 07:58:12 -0500 Subject: [PATCH] reactive advancedliveportrait --- __init__.py | 20 +- ...EX_Advanced_Audio_Reactive_expression.json | 2330 +++++++++++ ...audioreactive_live_portrait_depthflow.json | 3567 +++++++++++++++++ nodes/flex/flex_advanced_live_portrait.py | 394 ++ 4 files changed, 6307 insertions(+), 4 deletions(-) create mode 100644 examples/FLEX_Advanced_Audio_Reactive_expression.json create mode 100644 examples/audioreactive_live_portrait_depthflow.json create mode 100644 nodes/flex/flex_advanced_live_portrait.py diff --git a/__init__.py b/__init__.py index a552b3d..d782bd3 100644 --- a/__init__.py +++ b/__init__.py @@ -1,7 +1,9 @@ from .node_configs.node_configs import CombinedMeta from collections import OrderedDict +import os +import folder_paths -#allows for central management and inheritance of class variables for help documentation +#NOTE: allows for central management and inheritance of class variables for help documentation class RyanOnTheInside(metaclass=CombinedMeta): @classmethod def get_description(cls): @@ -275,8 +277,16 @@ from .nodes.preprocessors.pose import PoseInterpolator from .nodes.doom.doom import Doom -import os -import folder_paths + +HAS_ADVANCED_LIVE_PORTRAIT = os.path.exists(os.path.join(os.path.dirname(os.path.dirname(__file__)), "ComfyUI-AdvancedLivePortrait")) + +if HAS_ADVANCED_LIVE_PORTRAIT: + from .nodes.flex.flex_advanced_live_portrait import FlexExpressionEditor +else: + print("ComfyUI-AdvancedLivePortrait not found. FlexExpressionEditor will not be available. Install ComfyUI-AdvancedLivePortrait and restart ComfyUI.") + + + # Get the directory of the current file current_dir = os.path.dirname(os.path.realpath(__file__)) @@ -291,6 +301,7 @@ os.makedirs(midi_path, exist_ok=True) NODE_CLASS_MAPPINGS = { #NOTE: PoseInterpolator is not working yet #"PoseInterpolator": PoseInterpolator, + "Doom": Doom, "ManualFeaturePipe": ManualFeaturePipe, @@ -548,7 +559,8 @@ NODE_DISPLAY_NAME_MAPPINGS = { } - +if HAS_ADVANCED_LIVE_PORTRAIT: + NODE_CLASS_MAPPINGS["FlexExpressionEditor"] = FlexExpressionEditor import re diff --git a/examples/FLEX_Advanced_Audio_Reactive_expression.json b/examples/FLEX_Advanced_Audio_Reactive_expression.json new file mode 100644 index 0000000..861cabb --- /dev/null +++ b/examples/FLEX_Advanced_Audio_Reactive_expression.json @@ -0,0 +1,2330 @@ +{ + "last_node_id": 88, + "last_link_id": 76, + "nodes": [ + { + "id": 42, + "type": "BrightnessFeatureNode", + "pos": [ + -1937.0077758142834, + 948.0409576835841 + ], + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 0, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BrightnessFeatureNode" + }, + "widgets_values": [ + "mean_brightness", + 30 + ] + }, + { + "id": 43, + "type": "ColorFeatureNode", + "pos": [ + -1607.0077758142834, + 948.0409576835841 + ], + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 1, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ColorFeatureNode" + }, + "widgets_values": [ + "dominant_color", + 30 + ] + }, + { + "id": 47, + "type": "ManualFeatureFromPipe", + "pos": [ + -1237.0077758142834, + 538.0409576835838 + ], + "size": { + "0": 352.79998779296875, + "1": 150 + }, + "flags": {}, + "order": 2, + "mode": 4, + "inputs": [ + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ManualFeatureFromPipe" + }, + "widgets_values": [ + "0,10,20", + "0.0,0.5,1.0", + 1, + "none" + ] + }, + { + "id": 59, + "type": "PreviewFeature", + "pos": [ + 227, + 750 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": { + "collapsed": true + }, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 47 + } + ], + "outputs": [ + { + "name": "FEATURE_PREVIEW", + "type": "IMAGE", + "links": [ + 46 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewFeature" + }, + "widgets_values": [ + false + ] + }, + { + "id": 40, + "type": "TimeFeatureNode", + "pos": [ + -1597.0077758142834, + 538.0409576835838 + ], + "size": { + "0": 262.2439880371094, + "1": 150 + }, + "flags": {}, + "order": 3, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "TimeFeatureNode" + }, + "widgets_values": [ + "smooth", + 30, + 1, + 0 + ] + }, + { + "id": 48, + "type": "AreaFeatureNode", + "pos": [ + -1207.0077758142834, + 738.0409576835838 + ], + "size": { + "0": 317.4000244140625, + "1": 126 + }, + "flags": {}, + "order": 4, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "masks", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AreaFeatureNode" + }, + "widgets_values": [ + "total_area", + 30, + 0.5 + ] + }, + { + "id": 46, + "type": "DepthFeatureNode", + "pos": [ + -1267.0077758142834, + 938.0409576835841 + ], + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 5, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "depth_maps", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DepthFeatureNode" + }, + "widgets_values": [ + "mean_depth", + 30 + ] + }, + { + "id": 67, + "type": "PreviewFeature", + "pos": [ + 227, + 420 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": { + "collapsed": true + }, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 59 + } + ], + "outputs": [ + { + "name": "FEATURE_PREVIEW", + "type": "IMAGE", + "links": [ + 60 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewFeature" + }, + "widgets_values": [ + false + ] + }, + { + "id": 24, + "type": "AdvancedLivePortrait", + "pos": [ + 2680, + 360 + ], + "size": { + "0": 235.1999969482422, + "1": 474.2106018066406 + }, + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "name": "src_images", + "type": "IMAGE", + "link": null + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": 71 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": null + }, + { + "name": "command", + "type": "STRING", + "link": 70, + "widget": { + "name": "command" + } + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 16 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AdvancedLivePortrait" + }, + "widgets_values": [ + 0, + 0, + 1.7000000000000002, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", + false, + true, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0" + ] + }, + { + "id": 44, + "type": "RhythmFeatureExtractor", + "pos": [ + -1610.415877376783, + 744.9609491386618 + ], + "size": { + "0": 352.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 6, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "RhythmFeatureExtractor" + }, + "widgets_values": [ + "beat_locations", + 30, + 4 + ] + }, + { + "id": 45, + "type": "PitchFeatureExtractor", + "pos": [ + -1917.415877376783, + 1112.9609491386618 + ], + "size": { + "0": 344.3999938964844, + "1": 122 + }, + "flags": {}, + "order": 7, + "mode": 4, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": null + }, + { + "name": "opt_pitch_range_collections", + "type": "PITCH_RANGE_COLLECTION", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "PitchFeatureExtractor" + }, + "widgets_values": [ + "frequency", + "medium" + ] + }, + { + "id": 68, + "type": "PreviewImage", + "pos": [ + 746, + 420 + ], + "size": { + "0": 198.45782470703125, + "1": 246 + }, + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 60 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 31, + "type": "DownloadOpenUnmixModel", + "pos": [ + -164, + 619 + ], + "size": { + "0": 361.20001220703125, + "1": 58 + }, + "flags": {}, + "order": 8, + "mode": 0, + "outputs": [ + { + "name": "OPEN_UNMIX_MODEL", + "type": "OPEN_UNMIX_MODEL", + "links": [ + 23 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DownloadOpenUnmixModel" + }, + "widgets_values": [ + "umxl" + ] + }, + { + "id": 32, + "type": "AudioSeparatorSimple", + "pos": [ + -164, + 729 + ], + "size": { + "0": 336, + "1": 106 + }, + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "OPEN_UNMIX_MODEL", + "link": 23 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 24 + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "drums_audio", + "type": "AUDIO", + "links": [ + 54, + 76 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "vocals_audio", + "type": "AUDIO", + "links": [], + "shape": 3, + "slot_index": 2 + }, + { + "name": "bass_audio", + "type": "AUDIO", + "links": [], + "shape": 3, + "slot_index": 3 + }, + { + "name": "other_audio", + "type": "AUDIO", + "links": [ + 51, + 52 + ], + "shape": 3, + "slot_index": 4 + } + ], + "properties": { + "Node name for S&R": "AudioSeparatorSimple" + } + }, + { + "id": 64, + "type": "PreviewAudio", + "pos": [ + -164, + 1088 + ], + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 51 + } + ], + "properties": { + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [ + null + ] + }, + { + "id": 51, + "type": "PreviewImage", + "pos": [ + 738.1399993896484, + 729 + ], + "size": { + "0": 206.3178253173828, + "1": 246 + }, + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 46 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 41, + "type": "ProximityFeatureNode", + "pos": [ + -1973.415877376783, + 781.9609491386618 + ], + "size": { + "0": 336, + "1": 122 + }, + "flags": {}, + "order": 9, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "anchor_locations", + "type": "LOCATION", + "link": null + }, + { + "name": "query_locations", + "type": "LOCATION", + "link": null + } + ], + "outputs": [ + { + "name": "proximity_feature", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ProximityFeatureNode" + }, + "widgets_values": [ + 30, + "frame" + ] + }, + { + "id": 38, + "type": "MIDILoadAndExtract", + "pos": [ + -1928.415877376783, + 144.96094913866202 + ], + "size": { + "0": 1020, + "1": 346 + }, + "flags": {}, + "order": 10, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "selectedNotes": [] + }, + "widgets_values": [ + "Velocity", + "prom_single_track.mid", + "all", + 30, + false, + "", + "upload", + "refresh" + ] + }, + { + "id": 58, + "type": "ManualFeatureFromPipe", + "pos": [ + 227, + 729 + ], + "size": { + "0": 352.79998779296875, + "1": 150 + }, + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 44 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 45, + 47 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "ManualFeatureFromPipe" + }, + "widgets_values": [ + "0", + "-1", + 1, + "ease_out" + ] + }, + { + "id": 21, + "type": "AudioFeatureExtractorFirst", + "pos": [ + 227, + 980 + ], + "size": { + "0": 394.79998779296875, + "1": 170 + }, + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 52 + } + ], + "outputs": [ + { + "name": "feature", + "type": "FEATURE", + "links": [ + 73 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": [ + 44 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractorFirst" + }, + "widgets_values": [ + "amplitude_envelope", + 512, + 512, + 15 + ] + }, + { + "id": 73, + "type": "FeatureRenormalize", + "pos": [ + 227, + 1211 + ], + "size": { + "0": 310.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 73 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 74 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": [ + 72 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "FeatureRenormalize" + }, + "widgets_values": [ + -1, + 1, + false + ] + }, + { + "id": 27, + "type": "VHS_VideoCombine", + "pos": [ + 2945, + 280 + ], + "size": [ + 253.4889678955078, + 557.4889678955078 + ], + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 16 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 30 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AdvancedLivePortrait", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AdvancedLivePortrait_00306-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + } + } + } + }, + { + "id": 53, + "type": "Note", + "pos": [ + 227, + 1387 + ], + "size": { + "0": 234.10646057128906, + "1": 70.32601165771484 + }, + "flags": {}, + "order": 11, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "audio feature from drums\nmanual feature for fun\nanother audio feature for the synth" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 39, + "type": "MotionFeatureNode", + "pos": [ + -1915, + 542 + ], + "size": { + "0": 268.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 12, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "MotionFeatureNode" + }, + "widgets_values": [ + "mean_motion", + 30, + "Farneback", + 0, + 0 + ] + }, + { + "id": 81, + "type": "FeatureMixer", + "pos": [ + -2048.801412019285, + 1448.3726434811094 + ], + "size": { + "0": 367.79998779296875, + "1": 342 + }, + "flags": {}, + "order": 13, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureMixer" + }, + "widgets_values": [ + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 0.5, + false + ] + }, + { + "id": 82, + "type": "FeatureCombine", + "pos": [ + -1645.801412019285, + 1451.3726434811094 + ], + "size": { + "0": 380.4000244140625, + "1": 150 + }, + "flags": {}, + "order": 14, + "mode": 4, + "inputs": [ + { + "name": "feature1", + "type": "FEATURE", + "link": null + }, + { + "name": "feature2", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureCombine" + }, + "widgets_values": [ + "add", + 1, + 1, + false + ] + }, + { + "id": 83, + "type": "FeatureOscillator", + "pos": [ + -1232.801412019285, + 1461.3726434811094 + ], + "size": { + "0": 367.79998779296875, + "1": 198 + }, + "flags": {}, + "order": 15, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureOscillator" + }, + "widgets_values": [ + "sine", + 1, + 0.5, + 0, + 0.5, + false + ] + }, + { + "id": 86, + "type": "FeatureMath", + "pos": [ + -1639.801412019285, + 1655.3726434811094 + ], + "size": { + "0": 367.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 16, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureMath" + }, + "widgets_values": [ + 0, + "add", + false + ] + }, + { + "id": 84, + "type": "FeatureScaler", + "pos": [ + -2049.801412019285, + 1850.3726434811097 + ], + "size": { + "0": 367.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 17, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureScaler" + }, + "widgets_values": [ + "linear", + 0, + 1, + 2, + false + ] + }, + { + "id": 85, + "type": "FeatureSmoothing", + "pos": [ + -1633.801412019285, + 1852.3726434811097 + ], + "size": { + "0": 367.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 18, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureSmoothing" + }, + "widgets_values": [ + "moving_average", + 5, + 0.3, + 1, + false + ] + }, + { + "id": 87, + "type": "FeatureAccumulate", + "pos": [ + -1229.801412019285, + 1731.3726434811094 + ], + "size": { + "0": 367.79998779296875, + "1": 222 + }, + "flags": {}, + "order": 19, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureAccumulate" + }, + "widgets_values": [ + 0, + 1, + 0, + false, + false, + 0, + false + ] + }, + { + "id": 88, + "type": "Note", + "pos": [ + -675, + 1660 + ], + "size": { + "0": 343.6423645019531, + "1": 137.95391845703125 + }, + "flags": {}, + "order": 20, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "\n<--------here's some ways to manipulate features" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 63, + "type": "AudioFeatureExtractorFirst", + "pos": [ + 227, + 420 + ], + "size": { + "0": 394.79998779296875, + "1": 170 + }, + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 54 + } + ], + "outputs": [ + { + "name": "feature", + "type": "FEATURE", + "links": [ + 55, + 59 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": [], + "shape": 3, + "slot_index": 1 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractorFirst" + }, + "widgets_values": [ + "amplitude_envelope", + 512, + 512, + 15 + ] + }, + { + "id": 56, + "type": "PreviewAudio", + "pos": [ + -152, + 897 + ], + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 76 + } + ], + "properties": { + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [ + null + ] + }, + { + "id": 69, + "type": "PreviewImage", + "pos": [ + 746.2026214599609, + 1067 + ], + "size": { + "0": 198.2552032470703, + "1": 246 + }, + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 72 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 35, + "type": "Note", + "pos": [ + 1500, + 1290 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 21, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "modulate pupils with synth" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 36, + "type": "Note", + "pos": [ + 1870, + 1290 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": { + "collapsed": false + }, + "order": 22, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "modulate roll with manual feature" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 65, + "type": "Note", + "pos": [ + 2250, + 1290 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": { + "collapsed": false + }, + "order": 23, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "modulate pitch with drums" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 61, + "type": "PreviewAudio", + "pos": [ + -163, + 285 + ], + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 48 + } + ], + "properties": { + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [ + null + ] + }, + { + "id": 16, + "type": "VHS_LoadAudioUpload", + "pos": [ + -164, + 420 + ], + "size": { + "0": 243.818359375, + "1": 130 + }, + "flags": {}, + "order": 24, + "mode": 0, + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 24, + 30, + 48 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadAudioUpload" + }, + "widgets_values": { + "audio": "royalty_free.mp3", + "start_time": 16.01, + "duration": 7, + "choose audio to upload": "image" + } + }, + { + "id": 22, + "type": "LoadImage", + "pos": [ + 1079, + 420 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 25, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "slot_index": 1, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 15, + "type": "FlexExpressionEditor", + "pos": [ + 1430, + 420 + ], + "size": { + "0": 338.77764892578125, + "1": 830 + }, + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 17 + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": null + }, + { + "name": "feature", + "type": "FEATURE", + "link": 74 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 18 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3, + "slot_index": 2 + }, + { + "name": "command", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 0, + -14.5, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 1, + 0, + "pupil_x", + "absolute" + ] + }, + { + "id": 28, + "type": "FlexExpressionEditor", + "pos": [ + 1830, + 420 + ], + "size": { + "0": 336, + "1": 830 + }, + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": 18 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 45 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [], + "shape": 3, + "slot_index": 0 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 49 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + }, + { + "name": "command", + "type": "STRING", + "links": [ + 70 + ], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 15, + 0, + 15, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 1, + 0, + "rotate_roll", + "absolute" + ] + }, + { + "id": 62, + "type": "FlexExpressionEditor", + "pos": [ + 2210, + 420 + ], + "size": { + "0": 336, + "1": 830 + }, + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": 49 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 55 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [], + "shape": 3, + "slot_index": 0 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 71 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + }, + { + "name": "command", + "type": "STRING", + "links": [], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor" + }, + "widgets_values": [ + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 1, + 0, + "rotate_pitch", + "absolute" + ] + }, + { + "id": 79, + "type": "Note", + "pos": [ + 2688, + 885 + ], + "size": { + "0": 486.4787902832031, + "1": 230.9814453125 + }, + "flags": {}, + "order": 26, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Flex Parameters: \n constrain_min_max: ensures modulated value is kept within bounds of the \n min and max of the parameter\n strength: a strength multiplier\n feature_threshold: minimum feature value to consider\n feature_param: the expression parameter to modulate\n feature_mode:\n -relative: modulation occurs relative to the parameter value\n -absolute: modulation occurs relative to 0\n\nNotes:\n The value for a parameter in a previous editor changes the starting position of \n that \n parameter in the next editor.\n\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 60, + "type": "Note", + "pos": [ + -630, + 616 + ], + "size": { + "0": 343.6423645019531, + "1": 137.95391845703125 + }, + "flags": {}, + "order": 27, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Audio is one of many feature sources, and they are interchangeable.\n\n<--------here's some other feature sources. \ntutorial: https://youtu.be/QmWk2xse7pI" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 16, + 24, + 0, + 27, + 0, + "IMAGE" + ], + [ + 17, + 22, + 0, + 15, + 0, + "IMAGE" + ], + [ + 18, + 15, + 1, + 28, + 3, + "EDITOR_LINK" + ], + [ + 23, + 31, + 0, + 32, + 0, + "OPEN_UNMIX_MODEL" + ], + [ + 24, + 16, + 0, + 32, + 1, + "AUDIO" + ], + [ + 30, + 16, + 0, + 27, + 1, + "AUDIO" + ], + [ + 44, + 21, + 1, + 58, + 0, + "FEATURE_PIPE" + ], + [ + 45, + 58, + 0, + 28, + 4, + "FEATURE" + ], + [ + 46, + 59, + 0, + 51, + 0, + "IMAGE" + ], + [ + 47, + 58, + 0, + 59, + 0, + "FEATURE" + ], + [ + 48, + 16, + 0, + 61, + 0, + "AUDIO" + ], + [ + 49, + 28, + 1, + 62, + 3, + "EDITOR_LINK" + ], + [ + 51, + 32, + 4, + 64, + 0, + "AUDIO" + ], + [ + 52, + 32, + 4, + 21, + 0, + "AUDIO" + ], + [ + 54, + 32, + 1, + 63, + 0, + "AUDIO" + ], + [ + 55, + 63, + 0, + 62, + 4, + "FEATURE" + ], + [ + 59, + 63, + 0, + 67, + 0, + "FEATURE" + ], + [ + 60, + 67, + 0, + 68, + 0, + "IMAGE" + ], + [ + 70, + 28, + 3, + 24, + 3, + "STRING" + ], + [ + 71, + 62, + 1, + 24, + 1, + "EDITOR_LINK" + ], + [ + 72, + 73, + 1, + 69, + 0, + "IMAGE" + ], + [ + 73, + 21, + 0, + 73, + 0, + "FEATURE" + ], + [ + 74, + 73, + 0, + 15, + 4, + "FEATURE" + ], + [ + 76, + 32, + 1, + 56, + 0, + "AUDIO" + ] + ], + "groups": [ + { + "title": ":)", + "bounding": [ + -2768, + -53, + 1989, + 1400 + ], + "color": "#3f789e", + "font_size": 1000 + }, + { + "title": "Feature Manipulation", + "bounding": [ + -2092, + 1352, + 1315, + 699 + ], + "color": "#3f789e", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 1.2100000000000002, + "offset": [ + 1166.858494860238, + -303.268542001848 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/examples/audioreactive_live_portrait_depthflow.json b/examples/audioreactive_live_portrait_depthflow.json new file mode 100644 index 0000000..ca24791 --- /dev/null +++ b/examples/audioreactive_live_portrait_depthflow.json @@ -0,0 +1,3567 @@ +{ + "last_node_id": 106, + "last_link_id": 103, + "nodes": [ + { + "id": 42, + "type": "BrightnessFeatureNode", + "pos": { + "0": -1937.0078125, + "1": 948.0409545898438 + }, + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 0, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BrightnessFeatureNode", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "mean_brightness", + 30 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 43, + "type": "ColorFeatureNode", + "pos": { + "0": -1607.0078125, + "1": 948.0409545898438 + }, + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 1, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ColorFeatureNode", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "dominant_color", + 30 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 47, + "type": "ManualFeatureFromPipe", + "pos": { + "0": -1237.0078125, + "1": 538.0409545898438 + }, + "size": { + "0": 352.79998779296875, + "1": 150 + }, + "flags": {}, + "order": 2, + "mode": 4, + "inputs": [ + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ManualFeatureFromPipe", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "0,10,20", + "0.0,0.5,1.0", + 1, + "none" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 40, + "type": "TimeFeatureNode", + "pos": { + "0": -1597.0078125, + "1": 538.0409545898438 + }, + "size": { + "0": 262.2439880371094, + "1": 150 + }, + "flags": {}, + "order": 3, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "TimeFeatureNode", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "smooth", + 30, + 1, + 0 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 48, + "type": "AreaFeatureNode", + "pos": { + "0": -1207.0078125, + "1": 738.0409545898438 + }, + "size": { + "0": 317.4000244140625, + "1": 126 + }, + "flags": {}, + "order": 4, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "masks", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AreaFeatureNode", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "total_area", + 30, + 0.5 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 46, + "type": "DepthFeatureNode", + "pos": { + "0": -1267.0078125, + "1": 938.0409545898438 + }, + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 5, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "depth_maps", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DepthFeatureNode", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "mean_depth", + 30 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 67, + "type": "PreviewFeature", + "pos": { + "0": 227, + "1": 420 + }, + "size": { + "0": 315, + "1": 58 + }, + "flags": { + "collapsed": true + }, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 59 + } + ], + "outputs": [ + { + "name": "FEATURE_PREVIEW", + "type": "IMAGE", + "links": [ + 60 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewFeature", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 44, + "type": "RhythmFeatureExtractor", + "pos": { + "0": -1610.4158935546875, + "1": 744.9609375 + }, + "size": { + "0": 352.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 6, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "RhythmFeatureExtractor", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "beat_locations", + 30, + 4 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 45, + "type": "PitchFeatureExtractor", + "pos": { + "0": -1917.4158935546875, + "1": 1112.9609375 + }, + "size": { + "0": 344.3999938964844, + "1": 122 + }, + "flags": {}, + "order": 7, + "mode": 4, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": null + }, + { + "name": "opt_pitch_range_collections", + "type": "PITCH_RANGE_COLLECTION", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "PitchFeatureExtractor", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "frequency", + "medium" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 68, + "type": "PreviewImage", + "pos": { + "0": 746, + "1": 420 + }, + "size": { + "0": 198.45782470703125, + "1": 246 + }, + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 60 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 31, + "type": "DownloadOpenUnmixModel", + "pos": { + "0": -164, + "1": 619 + }, + "size": { + "0": 361.20001220703125, + "1": 58 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "OPEN_UNMIX_MODEL", + "type": "OPEN_UNMIX_MODEL", + "links": [ + 23 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DownloadOpenUnmixModel", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "umxl" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 32, + "type": "AudioSeparatorSimple", + "pos": { + "0": -164, + "1": 729 + }, + "size": { + "0": 336, + "1": 106 + }, + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "OPEN_UNMIX_MODEL", + "link": 23 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 24 + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "drums_audio", + "type": "AUDIO", + "links": [ + 54, + 76 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "vocals_audio", + "type": "AUDIO", + "links": [], + "shape": 3, + "slot_index": 2 + }, + { + "name": "bass_audio", + "type": "AUDIO", + "links": [], + "shape": 3, + "slot_index": 3 + }, + { + "name": "other_audio", + "type": "AUDIO", + "links": [ + 51, + 52 + ], + "shape": 3, + "slot_index": 4 + } + ], + "properties": { + "Node name for S&R": "AudioSeparatorSimple", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 64, + "type": "PreviewAudio", + "pos": { + "0": -164, + "1": 1088 + }, + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 51 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAudio", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + null + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 51, + "type": "PreviewImage", + "pos": { + "0": 738.1400146484375, + "1": 729 + }, + "size": { + "0": 206.3178253173828, + "1": 246 + }, + "flags": {}, + "order": 47, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 46 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 41, + "type": "ProximityFeatureNode", + "pos": { + "0": -1973.4158935546875, + "1": 781.9609375 + }, + "size": { + "0": 336, + "1": 122 + }, + "flags": {}, + "order": 9, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "anchor_locations", + "type": "LOCATION", + "link": null + }, + { + "name": "query_locations", + "type": "LOCATION", + "link": null + } + ], + "outputs": [ + { + "name": "proximity_feature", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ProximityFeatureNode", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 30, + "frame" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 38, + "type": "MIDILoadAndExtract", + "pos": { + "0": -1928.4158935546875, + "1": 144.96095275878906 + }, + "size": { + "0": 1020, + "1": 346 + }, + "flags": {}, + "order": 10, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "selectedNotes": [], + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "Velocity", + "prom_single_track.mid", + "all", + 30, + false, + "", + "upload", + "refresh" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 21, + "type": "AudioFeatureExtractorFirst", + "pos": { + "0": 227, + "1": 980 + }, + "size": { + "0": 394.79998779296875, + "1": 170 + }, + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 52 + } + ], + "outputs": [ + { + "name": "feature", + "type": "FEATURE", + "links": [ + 73 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": [ + 44 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractorFirst", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "amplitude_envelope", + 512, + 512, + 15 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 73, + "type": "FeatureRenormalize", + "pos": { + "0": 227, + "1": 1211 + }, + "size": { + "0": 310.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 73 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 74 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": [ + 72 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "FeatureRenormalize", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + -1, + 1, + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 53, + "type": "Note", + "pos": { + "0": 227, + "1": 1387 + }, + "size": { + "0": 234.10646057128906, + "1": 70.32601165771484 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "audio feature from drums\nmanual feature for fun\nanother audio feature for the synth" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 39, + "type": "MotionFeatureNode", + "pos": { + "0": -1915, + "1": 542 + }, + "size": { + "0": 268.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 12, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "MotionFeatureNode", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "mean_motion", + 30, + "Farneback", + 0, + 0 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 81, + "type": "FeatureMixer", + "pos": { + "0": -2048.801513671875, + "1": 1448.3726806640625 + }, + "size": { + "0": 367.79998779296875, + "1": 342 + }, + "flags": {}, + "order": 13, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureMixer", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 0.5, + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 82, + "type": "FeatureCombine", + "pos": { + "0": -1645.8013916015625, + "1": 1451.3726806640625 + }, + "size": { + "0": 380.4000244140625, + "1": 150 + }, + "flags": {}, + "order": 14, + "mode": 4, + "inputs": [ + { + "name": "feature1", + "type": "FEATURE", + "link": null + }, + { + "name": "feature2", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureCombine", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "add", + 1, + 1, + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 83, + "type": "FeatureOscillator", + "pos": { + "0": -1232.8013916015625, + "1": 1461.3726806640625 + }, + "size": { + "0": 367.79998779296875, + "1": 198 + }, + "flags": {}, + "order": 15, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureOscillator", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "sine", + 1, + 0.5, + 0, + 0.5, + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 86, + "type": "FeatureMath", + "pos": { + "0": -1639.8013916015625, + "1": 1655.3726806640625 + }, + "size": { + "0": 367.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 16, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureMath", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 0, + "add", + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 84, + "type": "FeatureScaler", + "pos": { + "0": -2049.801513671875, + "1": 1850.3726806640625 + }, + "size": { + "0": 367.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 17, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureScaler", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "linear", + 0, + 1, + 2, + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 85, + "type": "FeatureSmoothing", + "pos": { + "0": -1633.8013916015625, + "1": 1852.3726806640625 + }, + "size": { + "0": 367.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 18, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureSmoothing", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "moving_average", + 5, + 0.3, + 1, + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 87, + "type": "FeatureAccumulate", + "pos": { + "0": -1229.8013916015625, + "1": 1731.3726806640625 + }, + "size": { + "0": 367.79998779296875, + "1": 222 + }, + "flags": {}, + "order": 19, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureAccumulate", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 0, + 1, + 0, + false, + false, + 0, + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -675, + "1": 1660 + }, + "size": { + "0": 343.6423645019531, + "1": 137.95391845703125 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "\n<--------here's some ways to manipulate features" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 56, + "type": "PreviewAudio", + "pos": { + "0": -152, + "1": 897 + }, + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 76 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAudio", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + null + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 69, + "type": "PreviewImage", + "pos": { + "0": 746.20263671875, + "1": 1067 + }, + "size": { + "0": 198.2552032470703, + "1": 246 + }, + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 72 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 35, + "type": "Note", + "pos": { + "0": 1500, + "1": 1290 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "modulate pupils with synth" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 36, + "type": "Note", + "pos": { + "0": 1870, + "1": 1290 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": { + "collapsed": false + }, + "order": 22, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "modulate roll with manual feature" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 65, + "type": "Note", + "pos": { + "0": 2250, + "1": 1290 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": { + "collapsed": false + }, + "order": 23, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "modulate pitch with drums" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": 2688, + "1": 885 + }, + "size": { + "0": 393.7351989746094, + "1": 118.73271179199219 + }, + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "-relative: relative to the parameter value\n-absolute: relative to 0\n\n-the value for a parameter in a previous editor changes the starting position of that part of the expression\n\n-this means a parameter can be modulated multiple times\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 60, + "type": "Note", + "pos": { + "0": -630, + "1": 616 + }, + "size": { + "0": 343.6423645019531, + "1": 137.95391845703125 + }, + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Audio is one of many feature sources, and they are interchangeable.\n\n<--------here's some other feature sources. \n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 59, + "type": "PreviewFeature", + "pos": { + "0": 244, + "1": 699 + }, + "size": { + "0": 315, + "1": 58 + }, + "flags": { + "collapsed": true + }, + "order": 45, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 47 + } + ], + "outputs": [ + { + "name": "FEATURE_PREVIEW", + "type": "IMAGE", + "links": [ + 46 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewFeature", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 58, + "type": "ManualFeatureFromPipe", + "pos": { + "0": 234, + "1": 768 + }, + "size": { + "0": 352.79998779296875, + "1": 150 + }, + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 44 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 45, + 47 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "ManualFeatureFromPipe", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "0", + "-1", + 1, + "ease_out" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 16, + "type": "VHS_LoadAudioUpload", + "pos": { + "0": -164, + "1": 420 + }, + "size": { + "0": 243.818359375, + "1": 130 + }, + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 24, + 30, + 48, + 84 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadAudioUpload", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": { + "audio": "Spooky Scary Skeletons (Sped Up).mp3", + "start_time": 16.01, + "duration": 7, + "choose audio to upload": "image" + }, + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 22, + "type": "LoadImage", + "pos": { + "0": 1079, + "1": 420 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "slot_index": 1, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "exp_image.png", + "image" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 61, + "type": "PreviewAudio", + "pos": { + "0": -163, + "1": 285 + }, + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 48 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewAudio", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + null + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 15, + "type": "FlexExpressionEditor", + "pos": { + "0": 1430, + "1": 420 + }, + "size": { + "0": 338.77764892578125, + "1": 816.7449340820312 + }, + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 17, + "shape": 7 + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null, + "shape": 7 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": null, + "shape": 7 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 74, + "shape": 7 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 18 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3, + "slot_index": 2 + }, + { + "name": "command", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 0, + -14.5, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 0, + "pupil_x", + "relative" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 28, + "type": "FlexExpressionEditor", + "pos": { + "0": 1830, + "1": 420 + }, + "size": { + "0": 336, + "1": 806 + }, + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null, + "shape": 7 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": 18, + "shape": 7 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 45, + "shape": 7 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [], + "shape": 3, + "slot_index": 0 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 49 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + }, + { + "name": "command", + "type": "STRING", + "links": [ + 77 + ], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 0, + 0, + 15, + 0, + 15, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 0, + "rotate_roll", + "relative" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 62, + "type": "FlexExpressionEditor", + "pos": { + "0": 2210, + "1": 419 + }, + "size": { + "0": 336, + "1": 806 + }, + "flags": {}, + "order": 48, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null, + "shape": 7 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": 49, + "shape": 7 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 55, + "shape": 7 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [], + "shape": 3, + "slot_index": 0 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 71 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3, + "slot_index": 2 + }, + { + "name": "command", + "type": "STRING", + "links": [], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 0, + "rotate_pitch", + "relative" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 90, + "type": "DownloadAndLoadDepthCrafterModel", + "pos": { + "0": 3357.42626953125, + "1": 330.4696044921875 + }, + "size": { + "0": 308.93145751953125, + "1": 82 + }, + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "depthcrafter_model", + "type": "DEPTHCRAFTER_MODEL", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "DownloadAndLoadDepthCrafterModel", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + true, + false + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 27, + "type": "VHS_VideoCombine", + "pos": { + "0": 2945, + "1": 280 + }, + "size": [ + 253.4889678955078, + 553.4889678955078 + ], + "flags": {}, + "order": 50, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 16, + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 30, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + }, + { + "name": "filename_prefix", + "type": "STRING", + "link": 85, + "widget": { + "name": "filename_prefix" + } + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AdvancedLivePortrait", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "241106144130_00001-audio.mp4", + "subfolder": "241106/AdvancedLivePortrait", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + } + } + }, + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 89, + "type": "DepthCrafter", + "pos": { + "0": 3327.42626953125, + "1": 466.4696044921875 + }, + "size": { + "0": 367.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 51, + "mode": 0, + "inputs": [ + { + "name": "depthcrafter_model", + "type": "DEPTHCRAFTER_MODEL", + "link": 78 + }, + { + "name": "images", + "type": "IMAGE", + "link": 79 + } + ], + "outputs": [ + { + "name": "depth_maps", + "type": "IMAGE", + "links": [ + 87, + 95 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DepthCrafter", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 768, + 10, + 1.2, + 110, + 25 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 97, + "type": "Image To Mask", + "pos": { + "0": 3838.6787109375, + "1": 498.88232421875 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 54, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 95 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 96 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Image To Mask", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "intensity" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 24, + "type": "AdvancedLivePortrait", + "pos": { + "0": 2680, + "1": 360 + }, + "size": { + "0": 235.1999969482422, + "1": 474.2106018066406 + }, + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "name": "src_images", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": 71, + "shape": 7 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "command", + "type": "STRING", + "link": 77, + "widget": { + "name": "command" + } + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 16, + 79, + 81, + 98 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AdvancedLivePortrait", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 0, + 0, + 1.7000000000000002, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", + false, + true, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 63, + "type": "AudioFeatureExtractorFirst", + "pos": { + "0": 227, + "1": 420 + }, + "size": { + "0": 394.79998779296875, + "1": 170 + }, + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 54 + } + ], + "outputs": [ + { + "name": "feature", + "type": "FEATURE", + "links": [ + 55, + 59, + 100, + 103 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": [], + "shape": 3, + "slot_index": 1 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractorFirst", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "amplitude_envelope", + 512, + 512, + 15 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 104, + "type": "DepthflowMotionSetTarget", + "pos": { + "0": 4325.931640625, + "1": 744.1055908203125 + }, + "size": [ + 264.1174546221373, + 198 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null, + "shape": 7 + }, + { + "name": "motion", + "type": "DEPTHFLOW_MOTION", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "DEPTHFLOW_MOTION", + "type": "DEPTHFLOW_MOTION", + "links": [ + 101 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DepthflowMotionSetTarget", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1, + 0, + "value", + "relative", + "Focus", + 0.5 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 105, + "type": "DepthflowMotionSetTarget", + "pos": { + "0": 4322.931640625, + "1": 996.1051025390625 + }, + "size": { + "0": 264.1174621582031, + "1": 198 + }, + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 103, + "shape": 7 + }, + { + "name": "motion", + "type": "DEPTHFLOW_MOTION", + "link": 101, + "shape": 7 + } + ], + "outputs": [ + { + "name": "DEPTHFLOW_MOTION", + "type": "DEPTHFLOW_MOTION", + "links": [ + 102 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DepthflowMotionSetTarget", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 2.5, + 0, + "value", + "relative", + "Dolly", + 1 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 102, + "type": "DilateErodeMask", + "pos": { + "0": 3792.6787109375, + "1": 624.88232421875 + }, + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 55, + "mode": 0, + "inputs": [ + { + "name": "masks", + "type": "MASK", + "link": 96 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 97 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DilateErodeMask", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 5, + "circle" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 99, + "type": "MaskToImage", + "pos": { + "0": 3855.6787109375, + "1": 771.88232421875 + }, + "size": { + "0": 176.39999389648438, + "1": 26 + }, + "flags": {}, + "order": 56, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 97 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 91, + 92 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "MaskToImage", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 100, + "type": "PreviewImage", + "pos": { + "0": 3845.6787109375, + "1": 870.88232421875 + }, + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 57, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 91 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 95, + "type": "PreviewImage", + "pos": { + "0": 3397, + "1": 748 + }, + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 53, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 87 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 103, + "type": "BatchCount+", + "pos": { + "0": 3396, + "1": 142 + }, + "size": { + "0": 210, + "1": 26 + }, + "flags": {}, + "order": 52, + "mode": 0, + "inputs": [ + { + "name": "batch", + "type": "*", + "link": 98 + } + ], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 99 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "BatchCount+", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 94, + "type": "FileNamePrefixDateDirFirst", + "pos": { + "0": 2880, + "1": -18 + }, + "size": { + "0": 327.6000061035156, + "1": 130 + }, + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "filename_prefix", + "type": "STRING", + "links": [ + 85, + 86 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FileNamePrefixDateDirFirst", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + "true", + "true", + "AdvancedLivePortrait", + "" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 91, + "type": "Depthflow", + "pos": { + "0": 4286, + "1": 316 + }, + "size": [ + 315, + 286 + ], + "flags": {}, + "order": 58, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 81 + }, + { + "name": "depth_map", + "type": "IMAGE", + "link": 92 + }, + { + "name": "motion", + "type": "DEPTHFLOW_MOTION", + "link": 102 + }, + { + "name": "effects", + "type": "DEPTHFLOW_EFFECTS", + "link": null, + "shape": 7 + }, + { + "name": "num_frames", + "type": "INT", + "link": 99, + "widget": { + "name": "num_frames" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 83 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Depthflow", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1, + 15, + 15, + 30, + 50, + 2, + 0, + "mirror" + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 93, + "type": "VHS_VideoCombine", + "pos": { + "0": 4667, + "1": 319 + }, + "size": [ + 450.2064208984375, + 750.2064208984375 + ], + "flags": {}, + "order": 59, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 83, + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 84, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + }, + { + "name": "filename_prefix", + "type": "STRING", + "link": 86, + "widget": { + "name": "filename_prefix" + } + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AdvancedLivePortrait", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "241106144130_00012-audio.mp4", + "subfolder": "241106/AdvancedLivePortrait", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + } + } + }, + "color": "#2a363b", + "bgcolor": "#3f5159" + }, + { + "id": 92, + "type": "DepthflowMotionPresetDolly", + "pos": { + "0": 4324, + "1": 1350 + }, + "size": [ + 270.4984918509772, + 250 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 100, + "shape": 7 + } + ], + "outputs": [ + { + "name": "DEPTHFLOW_MOTION", + "type": "DEPTHFLOW_MOTION", + "links": [] + } + ], + "properties": { + "Node name for S&R": "DepthflowMotionPresetDolly", + "ttNbgOverride": { + "color": "#2a363b", + "bgcolor": "#3f5159", + "groupcolor": "#3f789e" + } + }, + "widgets_values": [ + 1, + 0, + "intensity", + "relative", + 2.05, + false, + false, + false, + 0.5 + ], + "color": "#2a363b", + "bgcolor": "#3f5159" + } + ], + "links": [ + [ + 16, + 24, + 0, + 27, + 0, + "IMAGE" + ], + [ + 17, + 22, + 0, + 15, + 0, + "IMAGE" + ], + [ + 18, + 15, + 1, + 28, + 3, + "EDITOR_LINK" + ], + [ + 23, + 31, + 0, + 32, + 0, + "OPEN_UNMIX_MODEL" + ], + [ + 24, + 16, + 0, + 32, + 1, + "AUDIO" + ], + [ + 30, + 16, + 0, + 27, + 1, + "AUDIO" + ], + [ + 44, + 21, + 1, + 58, + 0, + "FEATURE_PIPE" + ], + [ + 45, + 58, + 0, + 28, + 4, + "FEATURE" + ], + [ + 46, + 59, + 0, + 51, + 0, + "IMAGE" + ], + [ + 47, + 58, + 0, + 59, + 0, + "FEATURE" + ], + [ + 48, + 16, + 0, + 61, + 0, + "AUDIO" + ], + [ + 49, + 28, + 1, + 62, + 3, + "EDITOR_LINK" + ], + [ + 51, + 32, + 4, + 64, + 0, + "AUDIO" + ], + [ + 52, + 32, + 4, + 21, + 0, + "AUDIO" + ], + [ + 54, + 32, + 1, + 63, + 0, + "AUDIO" + ], + [ + 55, + 63, + 0, + 62, + 4, + "FEATURE" + ], + [ + 59, + 63, + 0, + 67, + 0, + "FEATURE" + ], + [ + 60, + 67, + 0, + 68, + 0, + "IMAGE" + ], + [ + 71, + 62, + 1, + 24, + 1, + "EDITOR_LINK" + ], + [ + 72, + 73, + 1, + 69, + 0, + "IMAGE" + ], + [ + 73, + 21, + 0, + 73, + 0, + "FEATURE" + ], + [ + 74, + 73, + 0, + 15, + 4, + "FEATURE" + ], + [ + 76, + 32, + 1, + 56, + 0, + "AUDIO" + ], + [ + 77, + 28, + 3, + 24, + 3, + "STRING" + ], + [ + 78, + 90, + 0, + 89, + 0, + "DEPTHCRAFTER_MODEL" + ], + [ + 79, + 24, + 0, + 89, + 1, + "IMAGE" + ], + [ + 81, + 24, + 0, + 91, + 0, + "IMAGE" + ], + [ + 83, + 91, + 0, + 93, + 0, + "IMAGE" + ], + [ + 84, + 16, + 0, + 93, + 1, + "AUDIO" + ], + [ + 85, + 94, + 0, + 27, + 4, + "STRING" + ], + [ + 86, + 94, + 0, + 93, + 4, + "STRING" + ], + [ + 87, + 89, + 0, + 95, + 0, + "IMAGE" + ], + [ + 91, + 99, + 0, + 100, + 0, + "IMAGE" + ], + [ + 92, + 99, + 0, + 91, + 1, + "IMAGE" + ], + [ + 95, + 89, + 0, + 97, + 0, + "IMAGE" + ], + [ + 96, + 97, + 0, + 102, + 0, + "MASK" + ], + [ + 97, + 102, + 0, + 99, + 0, + "MASK" + ], + [ + 98, + 24, + 0, + 103, + 0, + "*" + ], + [ + 99, + 103, + 0, + 91, + 4, + "INT" + ], + [ + 100, + 63, + 0, + 92, + 0, + "FEATURE" + ], + [ + 101, + 104, + 0, + 105, + 1, + "DEPTHFLOW_MOTION" + ], + [ + 102, + 105, + 0, + 91, + 2, + "DEPTHFLOW_MOTION" + ], + [ + 103, + 63, + 0, + 105, + 0, + "FEATURE" + ] + ], + "groups": [ + { + "title": "Create Depthmap", + "bounding": [ + 3307, + 246, + 408, + 414 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Dilate Mask (fix edges)", + "bounding": [ + 3773, + 415, + 355, + 722 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": ":)", + "bounding": [ + -2768, + -53, + 1989, + 1400 + ], + "color": "#3f789e", + "font_size": 1000, + "flags": {} + }, + { + "title": "Feature Manipulation", + "bounding": [ + -2092, + 1352, + 1315, + 699 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Dolly Zoom Feature", + "bounding": [ + 4302, + 660, + 307, + 554 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.32349184307614853, + "offset": [ + -656.71486786166, + 590.7996280570777 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes/flex/flex_advanced_live_portrait.py b/nodes/flex/flex_advanced_live_portrait.py new file mode 100644 index 0000000..456af35 --- /dev/null +++ b/nodes/flex/flex_advanced_live_portrait.py @@ -0,0 +1,394 @@ +import os +import sys +import numpy as np +import torch +import cv2 +from PIL import Image +import folder_paths +import comfy.utils +import time +import copy +import dill +import yaml +from ultralytics import YOLO +import importlib.util + +#NOTE: attempt to import advanced live portrait +def import_module_from_path(module_name, module_path): + package_dir = os.path.dirname(module_path) + original_sys_path = sys.path.copy() + + try: + # Add the parent directory to sys.path + parent_dir = os.path.dirname(package_dir) + if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) + + # Add LivePortrait directory to sys.path + liveportrait_path = os.path.join(package_dir, 'LivePortrait') + if os.path.exists(liveportrait_path) and liveportrait_path not in sys.path: + sys.path.insert(0, liveportrait_path) + + # Create module spec + spec = importlib.util.spec_from_file_location(module_name, module_path) + if spec is None: + raise ImportError(f"Cannot create a module spec for {module_path}") + + module = importlib.util.module_from_spec(spec) + + # Add the module to sys.modules + sys.modules[module_name] = module + + # Set up package structure + package_name = os.path.basename(package_dir).replace('-', '_') + module.__package__ = package_name + + # Create package spec + package_spec = importlib.util.spec_from_file_location( + package_name, + os.path.join(package_dir, '__init__.py') + ) + + # Create LivePortrait package spec + liveportrait_spec = importlib.util.spec_from_file_location( + f"{package_name}.LivePortrait", + os.path.join(liveportrait_path, '__init__.py') + ) + + # Add package to sys.modules with proper specs + if package_name not in sys.modules: + package = type( + 'Package', + (), + { + '__path__': [package_dir], + '__package__': package_name, + '__spec__': package_spec, + 'LivePortrait': type( + 'LivePortrait', + (), + { + '__path__': [liveportrait_path], + '__package__': f"{package_name}.LivePortrait", + '__spec__': liveportrait_spec + } + ) + } + ) + sys.modules[package_name] = package + sys.modules[f"{package_name}.LivePortrait"] = package.LivePortrait + + # Execute the module + spec.loader.exec_module(module) + + return module + + finally: + # Restore original sys.path + sys.path = original_sys_path + +#NOTE: need init files in AdvancedLivePortrait + +current_file_path = os.path.abspath(__file__) +current_directory = os.path.dirname(current_file_path) +custom_nodes_dir = os.path.abspath(os.path.join(current_directory, '..', '..', '..')) + +# Path to AdvancedLivePortrait's nodes.py +advanced_live_portrait_dir = os.path.join(custom_nodes_dir, 'ComfyUI-AdvancedLivePortrait') + +# Import the nodes.py module +advanced_nodes_path = os.path.join(advanced_live_portrait_dir, 'nodes.py') +advanced_nodes_module = import_module_from_path('advanced_live_portrait_nodes', advanced_nodes_path) + +# Import required classes and variables from advanced_nodes_module +ExpressionEditor = advanced_nodes_module.ExpressionEditor +g_engine = advanced_nodes_module.g_engine +ExpressionSet = advanced_nodes_module.ExpressionSet +retargeting = advanced_nodes_module.retargeting + +# Import other necessary components similarly +get_rotation_matrix = advanced_nodes_module.get_rotation_matrix +get_rgb_size = advanced_nodes_module.get_rgb_size +pil2tensor = advanced_nodes_module.pil2tensor + +class FlexExpressionEditor(ExpressionEditor): + """ + Enhanced version of ExpressionEditor that adds support for feature-based parameter modulation. + Allows for dynamic modification of facial expressions based on input features (e.g. audio, motion, etc.). + + TODO: Consider refactoring these methods into the base ExpressionEditor class: + - generate_preview_image + - process_sample_image + - create_expression_set + + These methods have been implemented here to avoid modifying the original ExpressionEditor + until the author can review the changes. + """ + + @classmethod + def INPUT_TYPES(cls): + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "constrain_min_max": ("BOOLEAN", {"default": True}) + }) + # Rename motion_link to flex_motion_link in optional inputs + base_inputs["optional"]["flex_motion_link"] = base_inputs["optional"].pop("motion_link") + base_inputs["optional"].update({ + "feature": ("FEATURE",), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01}), + "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0]}), + "feature_mode": (["relative", "absolute"], {"default": "absolute"}), + + }) + + return base_inputs + + RETURN_TYPES = ("IMAGE", "EDITOR_LINK", "EXP_DATA", "STRING") + RETURN_NAMES = ("image", "flex_motion_link", "save_exp", "command") + FUNCTION = "run" + OUTPUT_NODE = True + CATEGORY = "AdvancedLivePortrait" + + @classmethod + def get_modifiable_params(cls): + return ["rotate_pitch", "rotate_yaw", "rotate_roll", "blink", "eyebrow", "wink", "pupil_x", "pupil_y", + "aaa", "eee", "woo", "smile", "None"] + + def modulate_param(self, base_value, feature_value, strength, mode="relative", param_name=None, constrain_min_max=True): + """ + Helper method to modulate a parameter based on feature value. + + Args: + base_value: Original parameter value + feature_value: Feature value to modulate with + strength: Modulation strength + mode: Either "relative" or "absolute" + param_name: Name of the parameter being modulated (for min/max constraints) + constrain_min_max: Whether to constrain the output to the parameter's min/max values + + Returns: + Modulated parameter value + """ + if mode == "relative": + modulated_value = base_value * (1 + feature_value * strength) + else: # absolute + modulated_value = base_value * feature_value * strength + + if constrain_min_max and param_name: + # Get parameter constraints from INPUT_TYPES + param_info = self.INPUT_TYPES()["required"].get(param_name) + if param_info and isinstance(param_info[1], dict): + param_min = param_info[1].get("min") + param_max = param_info[1].get("max") + if param_min is not None: + modulated_value = max(param_min, modulated_value) + if param_max is not None: + modulated_value = min(param_max, modulated_value) + + return modulated_value + + #TODO: base class + def generate_preview_image(self, psi, preview_es): + """ + Generate preview image from the expression set and PSI data. + + Args: + psi: PSI data containing source image information + preview_es: Expression set to apply for preview + + Returns: + tuple: (output_image_tensor, results_dict) + """ + s_info = psi.x_s_info + pipeline = g_engine.get_pipeline() + + new_rotate = get_rotation_matrix(s_info['pitch'] + preview_es.r[0], + s_info['yaw'] + preview_es.r[1], + s_info['roll'] + preview_es.r[2]) + x_d_new = (s_info['scale'] * (1 + preview_es.s)) * ((s_info['kp'] + preview_es.e) @ new_rotate) + s_info['t'] + + x_d_new = pipeline.stitching(psi.x_s_user, x_d_new) + crop_out = pipeline.warp_decode(psi.f_s_user, psi.x_s_user, x_d_new) + crop_out = pipeline.parse_output(crop_out['out'])[0] + crop_with_fullsize = cv2.warpAffine(crop_out, psi.crop_trans_m, get_rgb_size(psi.src_rgb), cv2.INTER_LINEAR) + out = np.clip(psi.mask_ori * crop_with_fullsize + (1 - psi.mask_ori) * psi.src_rgb, 0, 255).astype(np.uint8) + out_img = pil2tensor(out) + + filename = g_engine.get_temp_img_name() + folder_paths.get_save_image_path(filename, folder_paths.get_temp_directory()) + img = Image.fromarray(crop_out) + img.save(os.path.join(folder_paths.get_temp_directory(), filename), compress_level=1) + results = [{"filename": filename, "type": "temp"}] + + return out_img, results + + #TODO: base class + def initialize_psi(self, src_image, motion_link, crop_factor): + """Initialize PSI data from source image or motion link""" + new_editor_link = [] + + if motion_link is not None: + self.psi = motion_link[0] + new_editor_link.append(self.psi) + elif src_image is not None: + if id(src_image) != id(self.src_image) or self.crop_factor != crop_factor: + self.crop_factor = crop_factor + self.psi = g_engine.prepare_source(src_image, crop_factor) + self.src_image = src_image + new_editor_link.append(self.psi) + else: + return None + + return new_editor_link + + #TODO: base class + def process_sample_image(self, sample_image, sample_parts, sample_ratio, es, rotate_pitch, rotate_yaw, rotate_roll): + """Process sample image and apply transformations""" + pipeline = g_engine.get_pipeline() + + if id(self.sample_image) != id(sample_image): + self.sample_image = sample_image + d_image_np = (sample_image * 255).byte().numpy() + d_face = g_engine.crop_face(d_image_np[0], 1.7) + i_d = g_engine.prepare_src_image(d_face) + self.d_info = pipeline.get_kp_info(i_d) + self.d_info['exp'][0, 5, 0] = 0 + self.d_info['exp'][0, 5, 1] = 0 + + if sample_parts == "OnlyExpression" or sample_parts == "All": + es.e += self.d_info['exp'] * sample_ratio + if sample_parts == "OnlyRotation" or sample_parts == "All": + rotate_pitch += self.d_info['pitch'] * sample_ratio + rotate_yaw += self.d_info['yaw'] * sample_ratio + rotate_roll += self.d_info['roll'] * sample_ratio + elif sample_parts == "OnlyMouth": + retargeting(es.e, self.d_info['exp'], sample_ratio, (14, 17, 19, 20)) + elif sample_parts == "OnlyEyes": + retargeting(es.e, self.d_info['exp'], sample_ratio, (1, 2, 11, 13, 15, 16)) + + return rotate_pitch, rotate_yaw, rotate_roll + + def run(self, rotate_pitch, rotate_yaw, rotate_roll, blink, eyebrow, wink, pupil_x, pupil_y, aaa, eee, woo, smile, + src_ratio, sample_ratio, sample_parts, crop_factor, constrain_min_max, src_image=None, sample_image=None, + flex_motion_link=None, add_exp=None, feature=None, strength=1.0, feature_threshold=0.0, + feature_param="None", feature_mode="relative"): + rotate_yaw = -rotate_yaw + + # Initialize PSI data + new_editor_link = self.initialize_psi(src_image, flex_motion_link, crop_factor) + if new_editor_link is None: + return (None, None, None, None) + + pipeline = g_engine.get_pipeline() + psi = self.psi + s_info = psi.x_s_info + + es = ExpressionSet() + + # Process sample image if provided + if sample_image is not None: + rotate_pitch, rotate_yaw, rotate_roll = self.process_sample_image( + sample_image, sample_parts, sample_ratio, es, rotate_pitch, rotate_yaw, rotate_roll + ) + + # Add any additional expression data + if add_exp is not None: + es.add(add_exp) + + # Prepare parameters dictionary + params = { + 'blink': blink, + 'eyebrow': eyebrow, + 'wink': wink, + 'pupil_x': pupil_x, + 'pupil_y': pupil_y, + 'aaa': aaa, + 'eee': eee, + 'woo': woo, + 'smile': smile, + 'rotate_pitch': rotate_pitch, + 'rotate_yaw': rotate_yaw, + 'rotate_roll': rotate_roll, + } + + # When calling calc_fe, unpack the parameters in the correct order + def apply_params(es, params): + return g_engine.calc_fe( + es, + params['blink'], + params['eyebrow'], + params['wink'], + params['pupil_x'], + params['pupil_y'], + params['aaa'], + params['eee'], + params['woo'], + params['smile'], + params['rotate_pitch'], + params['rotate_yaw'], + params['rotate_roll'] + ) + + # Check if feature is provided for modulation + if (feature is not None) and feature_param != "None" and feature_param in params: + total_frames = feature.frame_count + feature_values = [feature.get_value_at_frame(i) for i in range(total_frames)] + else: + # No modulation + total_frames = 1 + if flex_motion_link is not None and len(flex_motion_link) > 1: + total_frames = len(flex_motion_link) - 1 # Subtract 1 because first element is psi + feature_values = [1.0] * total_frames + + # Generate 'command' string expected by AdvancedLivePortrait node + command_lines = [] + for idx in range(total_frames): + # Start with base expression set or previous modifications + if flex_motion_link is not None and idx + 1 < len(flex_motion_link): + es_frame = ExpressionSet(es=flex_motion_link[idx + 1]) + if add_exp is not None: + es_frame.add(add_exp) + else: + es_frame = ExpressionSet(es=es) + + # Apply feature modulation if available + frame_params = params.copy() + if (feature is not None) and feature_param != "None" and feature_param in params: + feature_value = feature_values[idx] + if abs(feature_value) >= feature_threshold: + frame_params[feature_param] = self.modulate_param( + params[feature_param], + feature_value, + strength, + feature_mode, + feature_param, + constrain_min_max + ) + + # Calculate the new rotations and add them to existing ones + new_rotations = apply_params(es_frame.e, frame_params) + if flex_motion_link is not None and idx + 1 < len(flex_motion_link): + es_frame.r += new_rotations # Add to existing rotations + else: + es_frame.r = new_rotations # Set new rotations + + new_editor_link.append(es_frame) + + # Create command line + command_lines.append(f"{idx}=1:0") + + command = '\n'.join(command_lines) + + # Apply the expressions for preview image (using first frame) + preview_es = new_editor_link[1] if len(new_editor_link) > 1 else es + out_img, results = self.generate_preview_image(psi, preview_es) + + # Find the frame with maximum feature value and use its expression + if feature is not None and feature_param != "None" and len(new_editor_link) > 1: + max_idx = feature_values.index(max(feature_values)) + save_exp = new_editor_link[max_idx + 1] # +1 because first element is psi + else: + save_exp = es + + return {"ui": {"images": results}, "result": (out_img, new_editor_link, save_exp, command)} \ No newline at end of file