From 7dd4c79f04d369cc1eb7d18c1da5b50abbf42692 Mon Sep 17 00:00:00 2001 From: peteromallet Date: Wed, 7 Feb 2024 12:07:24 +0100 Subject: [PATCH] Push new workflow --- demo/creative_interpolation_example.json | 2942 +++++++++++++--------- 1 file changed, 1691 insertions(+), 1251 deletions(-) diff --git a/demo/creative_interpolation_example.json b/demo/creative_interpolation_example.json index 78c300b..4fbc2a4 100644 --- a/demo/creative_interpolation_example.json +++ b/demo/creative_interpolation_example.json @@ -1,6 +1,6 @@ { - "last_node_id": 535, - "last_link_id": 983, + "last_node_id": 556, + "last_link_id": 1007, "nodes": [ { "id": 482, @@ -63,8 +63,8 @@ "id": 370, "type": "CLIPVisionLoader", "pos": [ - -318.36356795651943, - -946.5696654619941 + -311.1125192367565, + -925.790646319921 ], "size": { "0": 210, @@ -95,8 +95,8 @@ "id": 369, "type": "IPAdapterModelLoader", "pos": [ - -58.363567956519574, - -949.5696654619941 + -51.11251923675685, + -928.790646319921 ], "size": { "0": 210, @@ -124,103 +124,11 @@ ] }, { - "id": 464, - "type": "ADE_EmptyLatentImageLarge", - "pos": [ - -238.64675798342108, - -781.0622036741253 - ], - "size": { - "0": 310.79998779296875, - "1": 74 - }, - "flags": { - "collapsed": false - }, - "order": 27, - "mode": 0, - "inputs": [ - { - "name": "width", - "type": "INT", - "link": 976, - "widget": { - "name": "width" - }, - "slot_index": 0 - }, - { - "name": "height", - "type": "INT", - "link": 977, - "widget": { - "name": "height" - } - }, - { - "name": "batch_size", - "type": "INT", - "link": 906, - "widget": { - "name": "batch_size" - }, - "slot_index": 2 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 876, - 883 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ADE_EmptyLatentImageLarge" - }, - "widgets_values": [ - 512, - 512, - 52 - ], - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 436, - "type": "PreviewImage", - "pos": [ - 447.353242016579, - -498.06220367412527 - ], - "size": { - "0": 820.0277099609375, - "1": 511.5301818847656 - }, - "flags": {}, - "order": 24, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 901 - } - ], - "properties": { - "Node name for S&R": "PreviewImage" - } - }, - { - "id": 456, + "id": 454, "type": "Note Plus (mtb)", "pos": { - "0": 183.35324096679688, - "1": -952.0617065429688, + "0": 181.60427856445312, + "1": -512.2832641601562, "2": 0, "3": 0, "4": 0, @@ -231,8 +139,8 @@ "9": 0 }, "size": { - "0": 210, - "1": 270 + "0": 230, + "1": 260 }, "flags": {}, "order": 3, @@ -242,7 +150,7 @@ "title": "Note+ (mtb)", "properties": {}, "widgets_values": [ - "## ⬅ Download the models from Comfy Manager\n
\nSearch the names and download the relevant models that are associated with the IPAdapter_plus node.", + "## Observe the graph to understand the the motion ⮕\n
\n As you tweak the settings, observe the impact on the graph. You can disconnect the positive, negative and model connections to just observe the graph without creating videos.", "markdown", "" ], @@ -251,113 +159,82 @@ "shape": 1 }, { - "id": 389, - "type": "PreviewImage", + "id": 281, + "type": "VHS_VideoCombine", "pos": [ - 419.353242016579, - -840.062203674125 + 1552, + -1054 + ], + "size": [ + 810, + 1094 ], - "size": { - "0": 887.6273803710938, - "1": 246 - }, "flags": {}, - "order": 19, + "order": 42, "mode": 0, "inputs": [ { "name": "images", "type": "IMAGE", - "link": 737 + "link": 938 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null + }, + { + "name": "batch_manager", + "type": "VHS_BatchManager", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 } ], "properties": { - "Node name for S&R": "PreviewImage" + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 3, + "filename_prefix": "steerable-motion/AD_", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 20, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AD__00602.mp4", + "subfolder": "steerable-motion", + "type": "output", + "format": "video/h264-mp4" + } + } } }, - { - "id": 445, - "type": "Note Plus (mtb)", - "pos": { - "0": -1589, - "1": -1463, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 650, - "1": 1040 - }, - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "# A brief guide\n\n
\n\n## Philosophy for getting the most from this:\n\n
\n\nThis isn’t a tool like text to video that will perform well out of the box - it’s more like a paint brush, an artistic tool that you need to figure out how to get the best from. \n\n
\n\nThrough trial and error, you'll need to build an understanding of how the motion and settings work, what its limitations are, which inputs work best with it, etc.\n\n
\n\nIf you can figure out how to wield it, this approach can provide enough control for you to make beautiful things that match your imagination precisely.\n\n
\n\n## How to use this:\n
\n1.Select a folder with the images in them above. The most important thing that will influence the quality of your creation is the input images.\n\n
\n\n2.Try the basic settings to start. Based on the resulting generation, try to understand what happened - you can learn about the settings and observe the graphs to the right.\n\n
\n\n3.Tweak the settings until you achieve your desired effect. \n\n
\n\n## What this will & won't be suitable for:\n\nI believe that this will be good for a wide range of storytelling-driven and abstract motion.\n\nHowever, it probably won't be good for complex motion - other approaches like vid2vid are far more suitable for this.\n\n## If you're having problems:\n\n
\n\nFirst, download the latest version of this workflow from [here](https://raw.githubusercontent.com/peteromallet/steerable-motion/main/demo/creative_interpolation_example.json) and get the latest version of the node from Comfy Manager.\n\n
\n\nIf that doesn't work, drop into [our Discord](https://discord.gg/V72vk8T67n) and share your problem and I'll try to help ASAP.\n\n
\n\n\n## Want to join a community of people who are pushing open source models to their technical and artistic limits?\n\n
\n\nYou're welcome to join [our Discord](https://discord.gg/UJf6aum7WZ).\n\n
\n\nPlease share anything you generate in the steerable-motion channel.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 446, - "type": "Note Plus (mtb)", - "pos": { - "0": 2385, - "1": -2182, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 210, - "1": 270 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## ⬅ Tweak motion_scale to change amount of movement\n
\nYou can tweak motion_scale on the left to increase or decrease the amount of motion in the generation.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, { "id": 461, "type": "CheckpointLoaderSimple", "pos": [ - 383, - -1774 + 335.9805321743486, + -1386.285167406442 ], "size": { - "0": 350.5392761230469, - "1": 98 + "0": 210, + "1": 117.29389953613281 }, "flags": {}, - "order": 6, + "order": 4, "mode": 0, "outputs": [ { @@ -392,337 +269,19 @@ "Realistic_Vision_V5.0.safetensors" ] }, - { - "id": 462, - "type": "CLIPTextEncode", - "pos": [ - 377, - -1593 - ], - "size": { - "0": 370.0784606933594, - "1": 96.74236297607422 - }, - "flags": {}, - "order": 21, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 894 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 913 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "" - ], - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 463, - "type": "CLIPTextEncode", - "pos": [ - 382, - -1413 - ], - "size": { - "0": 362.1751708984375, - "1": 84.13288116455078 - }, - "flags": {}, - "order": 22, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 895 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 914 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "smoke" - ], - "color": "#322", - "bgcolor": "#533" - }, - { - "id": 467, - "type": "ACN_SparseCtrlLoaderAdvanced", - "pos": [ - -709, - -1671 - ], - "size": { - "0": 314.50390625, - "1": 150 - }, - "flags": {}, - "order": 26, - "mode": 0, - "inputs": [ - { - "name": "sparse_method", - "type": "SPARSE_METHOD", - "link": 907, - "slot_index": 0 - }, - { - "name": "tk_optional", - "type": "TIMESTEP_KEYFRAME", - "link": null - } - ], - "outputs": [ - { - "name": "CONTROL_NET", - "type": "CONTROL_NET", - "links": [ - 879, - 978 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ACN_SparseCtrlLoaderAdvanced" - }, - "widgets_values": [ - "v3_sd15_sparsectrl_rgb.ckpt", - true, - 1, - 1 - ] - }, - { - "id": 469, - "type": "ACN_SparseCtrlRGBPreprocessor", - "pos": [ - -712, - -1830 - ], - "size": { - "0": 312.9380187988281, - "1": 66 - }, - "flags": {}, - "order": 28, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 881 - }, - { - "name": "vae", - "type": "VAE", - "link": 882 - }, - { - "name": "latent_size", - "type": "LATENT", - "link": 883 - } - ], - "outputs": [ - { - "name": "proc_IMAGE", - "type": "IMAGE", - "links": [ - 880, - 963 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ACN_SparseCtrlRGBPreprocessor" - } - }, - { - "id": 451, - "type": "Note Plus (mtb)", - "pos": { - "0": -806.6468505859375, - "1": -376.062255859375, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 459.86431884765625, - "1": 127.09552764892578 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Set duration of frame influence ⮕\n
\n1.0 equals around the distance between this and the next frames, 2.0 equals twice this. You can set these values for each frame individually in the field below.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 452, - "type": "Note Plus (mtb)", - "pos": { - "0": -784.6468505859375, - "1": -194.06219482421875, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 429.66748046875, - "1": 138.77713012695312 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Set high and low point of influence strength ⮕\n
\n\nThe number are the range the strength runs from and to - the first value is the low point, the last is the high.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 450, - "type": "Note Plus (mtb)", - "pos": { - "0": -801.6468505859375, - "1": -557.062255859375, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 441.21502685546875, - "1": 147.5499725341797 - }, - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Setting frame position ⮕\n\n
\n\nSetting linear frame distribution makes the frames spaced out by the linear value - for example, if it's set to 16, the frames will be at positions 0, 16, 32, etc. If you set type to dynamic, you'll need to enter the values in the text box below.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 455, - "type": "Note Plus (mtb)", - "pos": { - "0": -822.6468505859375, - "1": -1001.0614624023438, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 450, - "1": 390 - }, - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Understanding the settings\n\n
\nThere are 3 main settings:\n\n
\n\n- Key frame position\n- Length of influence\n- Strength of influence\n\n
\n\nThe type_of fields decide whether each of settings are the same for each frame or different.\n\n
\n\nWhen the type has been set to 'linear', you can adjust the value for every in with the _value fields.\n\n
\n\nWhen the type is set to 'dynamic', the text box below each setting are for the individual values - follow the format provided to set values for each frame.\n", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, { "id": 458, "type": "VAELoader", "pos": [ - 876, - -1487 + 669.9805321743487, + -1370.285167406442 ], "size": { - "0": 280.35577392578125, - "1": 95.60185241699219 + "0": 210, + "1": 99.132568359375 }, "flags": {}, - "order": 11, + "order": 5, "mode": 0, "outputs": [ { @@ -747,15 +306,15 @@ "id": 508, "type": "GetImageSize+", "pos": [ - 906, - -1706 + 995.9805321743487, + -1367.285167406442 ], "size": { - "0": 210, - "1": 46 + "0": 142.8000030517578, + "1": 51.235694885253906 }, "flags": {}, - "order": 20, + "order": 24, "mode": 0, "inputs": [ { @@ -788,487 +347,6 @@ "Node name for S&R": "GetImageSize+" } }, - { - "id": 187, - "type": "ADE_AnimateDiffLoaderWithContext", - "pos": [ - 2025, - -2177 - ], - "size": { - "0": 312.07550048828125, - "1": 252.39730834960938 - }, - "flags": { - "pinned": false - }, - "order": 25, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 900, - "slot_index": 0 - }, - { - "name": "context_options", - "type": "CONTEXT_OPTIONS", - "link": 980, - "slot_index": 1 - }, - { - "name": "motion_lora", - "type": "MOTION_LORA", - "link": null - }, - { - "name": "motion_model_settings", - "type": "MOTION_MODEL_SETTINGS", - "link": null, - "slot_index": 3 - }, - { - "name": "sample_settings", - "type": "sample_settings", - "link": null - }, - { - "name": "ad_keyframes", - "type": "AD_KEYFRAMES", - "link": null - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 935 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ADE_AnimateDiffLoaderWithContext" - }, - "widgets_values": [ - "v3_sd15_mm.ckpt", - "sqrt_linear (AnimateDiff)", - 1.3, - false - ], - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 454, - "type": "Note Plus (mtb)", - "pos": { - "0": 174.35324096679688, - "1": -533.062255859375, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 230, - "1": 260 - }, - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## Observe the graph to understand the the motion ⮕\n
\n As you tweak the settings, observe the impact on the graph. You can disconnect the positive, negative and model connections to just observe the graph without creating videos.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 472, - "type": "Note Plus (mtb)", - "pos": { - "0": -1517, - "1": -359, - "2": 0, - "3": 0, - "4": 0, - "5": 0, - "6": 0, - "7": 0, - "8": 0, - "9": 0 - }, - "size": { - "0": 591.9019165039062, - "1": 347.8489685058594 - }, - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Note+ (mtb)", - "properties": {}, - "widgets_values": [ - "## To download these models from Comfy Manager\n
\n\nOn the bar on the right, click \"Manager\", then click \"Install Models\" and search up the below - the models that show for those queries should be the right ones:\n
\n
\n\n\n- For Animatediff, search: v3\\_sd15\\_mm\n- For SparseCtrl, search: v3\\_sd15\\_sparsectrl\n- For s3\\_15 adapter, search: v3\\_sd15\\_adapter\n- For Clip Vision, search: CLIP-ViT-H-14\n- For ControlNet Tile, search: tile\n- For IP-Adapter-Plus, search: plus\\_sd15.safetensors\n\n
\n
\n\nFor the VAE and styling model, *make sure to use SD 1.5 models*.", - "markdown", - "" - ], - "color": "#223", - "bgcolor": "#335", - "shape": 1 - }, - { - "id": 521, - "type": "ADE_NoiseLayerAdd", - "pos": [ - 1522, - -2588 - ], - "size": { - "0": 380.4000244140625, - "1": 198 - }, - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [ - { - "name": "prev_noise_layers", - "type": "NOISE_LAYERS", - "link": null - }, - { - "name": "mask_optional", - "type": "MASK", - "link": null - }, - { - "name": "seed_override", - "type": "INT", - "link": null, - "widget": { - "name": "seed_override" - } - } - ], - "outputs": [ - { - "name": "NOISE_LAYERS", - "type": "NOISE_LAYERS", - "links": null, - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ADE_NoiseLayerAdd" - }, - "widgets_values": [ - 140, - "FreeNoise", - "use existing", - -78, - 0.5, - 0 - ] - }, - { - "id": 342, - "type": "ADE_AnimateDiffUniformContextOptions", - "pos": [ - 2074, - -1854 - ], - "size": { - "0": 310.79998779296875, - "1": 365.04229736328125 - }, - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [ - { - "name": "prev_context", - "type": "CONTEXT_OPTIONS", - "link": null - }, - { - "name": "view_opts", - "type": "VIEW_OPTS", - "link": null - } - ], - "outputs": [ - { - "name": "CONTEXT_OPTIONS", - "type": "CONTEXT_OPTIONS", - "links": [ - 980 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ADE_AnimateDiffUniformContextOptions" - }, - "widgets_values": [ - 16, - 2, - 4, - "uniform", - false, - "flat", - false, - 0, - 1 - ] - }, - { - "id": 470, - "type": "LoraLoader", - "pos": [ - -692, - -1420 - ], - "size": { - "0": 256.37469482421875, - "1": 126.03836059570312 - }, - "flags": {}, - "order": 18, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 896 - }, - { - "name": "clip", - "type": "CLIP", - "link": 893 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 909 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 894, - 895 - ], - "shape": 3, - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "LoraLoader" - }, - "widgets_values": [ - "v3_sd15_adapter.ckpt", - 0.01, - 0.25 - ] - }, - { - "id": 505, - "type": "ACN_AdvancedControlNetApply", - "pos": [ - -204, - -1492 - ], - "size": { - "0": 285.6000061035156, - "1": 246 - }, - "flags": {}, - "order": 30, - "mode": 0, - "inputs": [ - { - "name": "positive", - "type": "CONDITIONING", - "link": 967, - "slot_index": 0 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 968, - "slot_index": 1 - }, - { - "name": "control_net", - "type": "CONTROL_NET", - "link": 978 - }, - { - "name": "image", - "type": "IMAGE", - "link": 963 - }, - { - "name": "mask_optional", - "type": "MASK", - "link": null - }, - { - "name": "timestep_kf", - "type": "TIMESTEP_KEYFRAME", - "link": null - }, - { - "name": "latent_kf_override", - "type": "LATENT_KEYFRAME", - "link": null - }, - { - "name": "weights_override", - "type": "CONTROL_NET_WEIGHTS", - "link": null - } - ], - "outputs": [ - { - "name": "positive", - "type": "CONDITIONING", - "links": [ - 971 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "negative", - "type": "CONDITIONING", - "links": [ - 972 - ], - "shape": 3, - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "ACN_AdvancedControlNetApply" - }, - "widgets_values": [ - 0.5, - 0.6, - 0.675 - ] - }, - { - "id": 468, - "type": "ACN_AdvancedControlNetApply", - "pos": [ - -239, - -1880.1191788018743 - ], - "size": { - "0": 314.1339111328125, - "1": 246 - }, - "flags": {}, - "order": 29, - "mode": 0, - "inputs": [ - { - "name": "positive", - "type": "CONDITIONING", - "link": 917, - "slot_index": 0 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 918, - "slot_index": 1 - }, - { - "name": "control_net", - "type": "CONTROL_NET", - "link": 879 - }, - { - "name": "image", - "type": "IMAGE", - "link": 880 - }, - { - "name": "mask_optional", - "type": "MASK", - "link": null - }, - { - "name": "timestep_kf", - "type": "TIMESTEP_KEYFRAME", - "link": null - }, - { - "name": "latent_kf_override", - "type": "LATENT_KEYFRAME", - "link": null - }, - { - "name": "weights_override", - "type": "CONTROL_NET_WEIGHTS", - "link": null - } - ], - "outputs": [ - { - "name": "positive", - "type": "CONDITIONING", - "links": [ - 967 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "negative", - "type": "CONDITIONING", - "links": [ - 968 - ], - "shape": 3, - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "ACN_AdvancedControlNetApply" - }, - "widgets_values": [ - 0.6, - 0, - 0.05 - ] - }, { "id": 401, "type": "VHS_LoadImagesPath", @@ -1281,7 +359,7 @@ 194 ], "flags": {}, - "order": 16, + "order": 6, "mode": 0, "outputs": [ { @@ -1314,7 +392,7 @@ "Node name for S&R": "VHS_LoadImagesPath" }, "widgets_values": { - "directory": "/workspace/ComfyUI/input/3images/", + "directory": "/workspace/ComfyUI/input//Dance into shadow/", "image_load_cap": 0, "skip_first_images": 0, "select_every_nth": 1, @@ -1325,7 +403,7 @@ "params": { "frame_load_cap": 0, "skip_first_images": 0, - "filename": "/workspace/ComfyUI/input/3images/", + "filename": "/workspace/ComfyUI/input//Dance into shadow/", "type": "path", "format": "folder", "select_every_nth": 1 @@ -1335,12 +413,475 @@ "color": "#332922", "bgcolor": "#593930" }, + { + "id": 389, + "type": "PreviewImage", + "pos": [ + 426, + -819 + ], + "size": { + "0": 887.6273803710938, + "1": 246 + }, + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 737 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 541, + "type": "PromptSchedule", + "pos": [ + 340, + -1826 + ], + "size": { + "0": 389.9056396484375, + "1": 364 + }, + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 994 + } + ], + "outputs": [ + { + "name": "POS", + "type": "CONDITIONING", + "links": [ + 995 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "NEG", + "type": "CONDITIONING", + "links": null, + "shape": 3 + } + ], + "title": "Positive Prompt", + "properties": { + "Node name for S&R": "PromptSchedule" + }, + "widgets_values": [ + "\"4\": \"\", \"36\": \"\", \"68\": \"\"", + 120, + 0, + false, + "", + "", + 0, + 0, + 0, + 0 + ] + }, + { + "id": 543, + "type": "PromptSchedule", + "pos": [ + 826, + -1829 + ], + "size": { + "0": 356.6048278808594, + "1": 364 + }, + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 996 + } + ], + "outputs": [ + { + "name": "POS", + "type": "CONDITIONING", + "links": [], + "shape": 3, + "slot_index": 0 + }, + { + "name": "NEG", + "type": "CONDITIONING", + "links": [ + 997 + ], + "shape": 3, + "slot_index": 1 + } + ], + "title": "Negative Prompt", + "properties": { + "Node name for S&R": "PromptSchedule" + }, + "widgets_values": [ + "\"4\": \"\", \"36\": \"\", \"68\": \"\"", + 120, + 0, + false, + "", + "", + 0, + 0, + 0, + 0 + ] + }, + { + "id": 473, + "type": "BatchCreativeInterpolation", + "pos": [ + -271, + -608 + ], + "size": { + "0": 378.3703308105469, + "1": 680 + }, + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 995 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 997 + }, + { + "name": "images", + "type": "IMAGE", + "link": 908 + }, + { + "name": "model", + "type": "MODEL", + "link": 909 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 910 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 911 + } + ], + "outputs": [ + { + "name": "GRAPH", + "type": "IMAGE", + "links": [ + 901 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "POSITIVE", + "type": "CONDITIONING", + "links": [ + 917 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "NEGATIVE", + "type": "CONDITIONING", + "links": [ + 918 + ], + "shape": 3, + "slot_index": 2 + }, + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 1002 + ], + "shape": 3, + "slot_index": 3 + }, + { + "name": "KEYFRAME_POSITIONS", + "type": "SPARSE_METHOD", + "links": [ + 907 + ], + "shape": 3, + "slot_index": 4 + }, + { + "name": "BATCH_SIZE", + "type": "INT", + "links": [ + 906 + ], + "shape": 3, + "slot_index": 5 + }, + { + "name": "COLOR_MATCH_FRAMES", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BatchCreativeInterpolation" + }, + "widgets_values": [ + "control_v11f1e_sd15_tile_fp16.safetensors", + "dynamic", + 16, + "0, 24, 48, 72", + "linear", + "0.85", + "(None, 0.96), (0.96, 0.94), (0.94, 0.92), (0.92, None)", + "dynamic", + "(0.4,0.5,0.4)", + "(None, 0.5, 0.34), (0.34, 0.5, 0.33), (0.33, 0.5, 0.34), (0.34, 0.5, None)", + 0.9, + 4, + 0, + 1, + 0.2, + 0, + 0.6, + 0, + 0.65 + ] + }, + { + "id": 464, + "type": "ADE_EmptyLatentImageLarge", + "pos": [ + -231.39570926365826, + -760.2831845320522 + ], + "size": { + "0": 310.79998779296875, + "1": 74 + }, + "flags": { + "collapsed": false + }, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "link": 976, + "widget": { + "name": "width" + }, + "slot_index": 0 + }, + { + "name": "height", + "type": "INT", + "link": 977, + "widget": { + "name": "height" + } + }, + { + "name": "batch_size", + "type": "INT", + "link": 906, + "widget": { + "name": "batch_size" + }, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 876, + 883, + 1006 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_EmptyLatentImageLarge" + }, + "widgets_values": [ + 512, + 512, + 52 + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 546, + "type": "ADE_LoadAnimateDiffModel", + "pos": [ + 2615.5055483441884, + -889.6864292533057 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "ad_settings", + "type": "AD_SETTINGS", + "link": null + } + ], + "outputs": [ + { + "name": "MOTION_MODEL", + "type": "MOTION_MODEL_ADE", + "links": [ + 999 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_LoadAnimateDiffModel" + }, + "widgets_values": [ + "v3_sd15_mm.ckpt" + ] + }, + { + "id": 342, + "type": "ADE_AnimateDiffUniformContextOptions", + "pos": [ + 2057, + -1885 + ], + "size": { + "0": 310.79998779296875, + "1": 365.04229736328125 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "prev_context", + "type": "CONTEXT_OPTIONS", + "link": null + }, + { + "name": "view_opts", + "type": "VIEW_OPTS", + "link": null + } + ], + "outputs": [ + { + "name": "CONTEXT_OPTIONS", + "type": "CONTEXT_OPTIONS", + "links": [ + 1005 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffUniformContextOptions" + }, + "widgets_values": [ + 16, + 2, + 4, + "uniform", + false, + "flat", + false, + 0, + 1 + ] + }, + { + "id": 446, + "type": "Note Plus (mtb)", + "pos": { + "0": 2500, + "1": -2165, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 311.85103613281126, + 300 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## ⬅ Tweak motion_scale to change amount of movement\n
\nYou can tweak motion_scale on the left to increase or decrease the amount of motion in the generation - just change the value in brackets. \n
\n
\nYou can also schedule motion scales with this format:\n
\n
\n0:(1.3),16:(1.6)", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, { "id": 292, "type": "STMFNet VFI", "pos": [ - 2049.2307111212854, - -1235.158321614905 + 2116.531416015626, + -1265.8558007812508 ], "size": { "0": 295.6000061035156, @@ -1349,7 +890,7 @@ "flags": { "collapsed": true }, - "order": 33, + "order": 41, "mode": 0, "inputs": [ { @@ -1389,8 +930,8 @@ "id": 354, "type": "VHS_SplitImages", "pos": [ - 1669.2307111212854, - -1234.158321614905 + 1664, + -1270 ], "size": { "0": 231.97152709960938, @@ -1399,7 +940,7 @@ "flags": { "collapsed": true }, - "order": 32, + "order": 40, "mode": 0, "inputs": [ { @@ -1447,70 +988,86 @@ } }, { - "id": 281, - "type": "VHS_VideoCombine", + "id": 552, + "type": "ADE_AnimateDiffLoRALoader", "pos": [ - 1552, - -1054 + 2603, + -1261 ], "size": [ - 810, - 1074 + 316.6560767949222, + 82 ], "flags": {}, - "order": 34, - "mode": 0, + "order": 10, + "mode": 4, "inputs": [ { - "name": "images", - "type": "IMAGE", - "link": 938 - }, - { - "name": "audio", - "type": "VHS_AUDIO", + "name": "prev_motion_lora", + "type": "MOTION_LORA", "link": null } ], "outputs": [ { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": null, - "shape": 3 + "name": "MOTION_LORA", + "type": "MOTION_LORA", + "links": [ + 1007 + ], + "shape": 3, + "slot_index": 0 } ], "properties": { - "Node name for S&R": "VHS_VideoCombine" + "Node name for S&R": "ADE_AnimateDiffLoRALoader" }, - "widgets_values": { - "frame_rate": 15, - "loop_count": 3, - "filename_prefix": "steerable-motion/AD_", - "format": "video/h264-mp4", - "pix_fmt": "yuv420p", - "crf": 20, - "save_metadata": true, - "pingpong": false, - "save_output": true, - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "AD__00485.mp4", - "subfolder": "steerable-motion", - "type": "output", - "format": "video/h264-mp4" - } - } - } + "widgets_values": [ + "400_woman_dancing_temporal_unet.safetensors", + 0.65 + ] }, { - "id": 513, + "id": 551, "type": "Note Plus (mtb)", "pos": { - "0": 2423, - "1": -1877, + "0": 3022, + "1": -1134, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 234.37235839843515, + 148.41668457031153 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## You can add Motion LoRAs above\n
\nJust un-bypass the nodes. You can learn more about training them [here](https://github.com/ExponentialML/AnimateDiff-MotionDirector).", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 472, + "type": "Note Plus (mtb)", + "pos": { + "0": -1496, + "1": -329, "2": 0, "3": 0, "4": 0, @@ -1521,8 +1078,208 @@ "9": 0 }, "size": { - "0": 254.81239318847656, - "1": 404.44708251953125 + "0": 591.9019165039062, + "1": 347.8489685058594 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## To download these models from Comfy Manager\n
\n\nOn the bar on the right, click \"Manager\", then click \"Install Models\" and search up the below - the models that show for those queries should be the right ones:\n
\n
\n\n\n- For Animatediff, search: v3\\_sd15\\_mm\n- For SparseCtrl, search: v3\\_sd15\\_sparsectrl\n- For s3\\_15 adapter, search: v3\\_sd15\\_adapter\n- For Clip Vision, search: CLIP-ViT-H-14\n- For ControlNet Tile, search: tile\n- For IP-Adapter-Plus, search: plus\\_sd15.safetensors\n\n
\n
\n\nFor the VAE and styling model, *make sure to use SD 1.5 models*.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 436, + "type": "PreviewImage", + "pos": [ + 493, + -490 + ], + "size": { + "0": 820.0277099609375, + "1": 511.5301818847656 + }, + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 901 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 553, + "type": "Note Plus (mtb)", + "pos": { + "0": 134, + "1": -126, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 329.7846012594214, + 207.47863415191 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## ⬅ TIP-Adapter vs CN Tile \n\n
\n\nSeeing relative_ipa_strength to >0.0 turns on IP-Adapter while setting relative_cn_strength to >0.0 turns on ControlNet Tile. I recommend leaving IP-Adapter at 1.0, and ControlNet Tile at 0.0 but you're welcome to experment. For example, 0.5 leaves them at half strength.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 451, + "type": "Note Plus (mtb)", + "pos": { + "0": -797, + "1": -367, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 468.78867611215117, + 147.40195837099685 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## Set duration of frame influence ⮕\n
\n1.0 equals around the distance between this and the next frames, 2.0 equals twice this. You can set these values for each frame individually in the field below. The first value is the influence on the frame before, the second is on the frame after.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 452, + "type": "Note Plus (mtb)", + "pos": { + "0": -804, + "1": -184, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 493.8266298828121, + 163.10225195312478 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## Set high and low point of influence strength ⮕\n
\n\nThe number are the range the strength runs from and to - the first value is the starting point, the middle is the peak, while the ending is the ending point.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 445, + "type": "Note Plus (mtb)", + "pos": { + "0": -1572, + "1": -1426, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 650, + "1": 1040 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "# A brief guide\n\n
\n\n## Philosophy for getting the most from this:\n\n
\n\nThis isn’t a tool like text to video that will perform well out of the box - it’s more like a paint brush, an artistic tool that you need to figure out how to get the best from. \n\n
\n\nThrough trial and error, you'll need to build an understanding of how the motion and settings work, what its limitations are, which inputs work best with it, etc.\n\n
\n\nIf you can figure out how to wield it, this approach can provide enough control for you to make beautiful things that match your imagination precisely.\n\n
\n\n## How to use this:\n
\n1.Select a folder with the images in them above. The most important thing that will influence the quality of your creation is the input images.\n\n
\n\n2.Try the basic settings to start. Based on the resulting generation, try to understand what happened - you can learn about the settings and observe the graphs to the right.\n\n
\n\n3.Tweak the settings until you achieve your desired effect. \n\n
\n\n## What this will & won't be suitable for:\n\nI believe that this will be good for a wide range of storytelling-driven and abstract motion.\n\nHowever, it probably won't be good for complex motion - other approaches like vid2vid are far more suitable for this.\n\n## If you're having problems:\n\n
\n\nFirst, download the latest version of this workflow from [here](https://raw.githubusercontent.com/peteromallet/steerable-motion/main/demo/creative_interpolation_example.json) and get the latest version of the node from Comfy Manager.\n\n
\n\nIf that doesn't work, drop into [our Discord](https://discord.gg/V72vk8T67n) and share your problem and I'll try to help ASAP.\n\n
\n\n\n## Want to join a community of people who are pushing open source models to their technical and artistic limits?\n\n
\n\nYou're welcome to join [our Discord](https://discord.gg/UJf6aum7WZ).\n\n
\n\nPlease share anything you generate in the steerable-motion channel.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 455, + "type": "Note Plus (mtb)", + "pos": { + "0": -815, + "1": -980, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 450, + "1": 390 }, "flags": {}, "order": 17, @@ -1531,6 +1288,494 @@ "outputs": [], "title": "Note+ (mtb)", "properties": {}, + "widgets_values": [ + "## Understanding the settings\n\n
\nThere are 3 main settings:\n\n
\n\n- Key frame position\n- Length of influence\n- Strength of influence\n\n
\n\nThe type_of fields decide whether each of settings are the same for each frame or different.\n\n
\n\nWhen the type has been set to 'linear', you can adjust the value for every in with the _value fields.\n\n
\n\nWhen the type is set to 'dynamic', the text box below each setting are for the individual values - follow the format provided to set values for each frame.\n", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 450, + "type": "Note Plus (mtb)", + "pos": { + "0": -787, + "1": -544, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 441.21502685546875, + "1": 147.5499725341797 + }, + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## Setting frame position ⮕\n\n
\n\nSetting linear frame distribution makes the frames spaced out by the linear value - for example, if it's set to 16, the frames will be at positions 0, 16, 32, etc. If you set type to dynamic, you'll need to enter the values in the text box below.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 456, + "type": "Note Plus (mtb)", + "pos": { + "0": 188, + "1": -950, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 210, + 186.0922504882799 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## ⬅ Download the models from Comfy Manager\n
\nSee details on where to find them on the left.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 556, + "type": "Note Plus (mtb)", + "pos": { + "0": -724, + "1": -1952, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 334.258056640625, + 188.175537109375 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, + "widgets_values": [ + "## Tweak Sparse Ctrl Strength is improve adherence ⮕\n
\nI'd recommend that you increase the end_percent to the right. Higher values tend to cause flickering. However, increase the adapter strength below along with it below to fix this to an extend.", + "markdown", + "" + ], + "color": "#223", + "bgcolor": "#335", + "shape": 1 + }, + { + "id": 467, + "type": "ACN_SparseCtrlLoaderAdvanced", + "pos": [ + -714, + -1569 + ], + "size": { + "0": 314.50390625, + "1": 150 + }, + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "sparse_method", + "type": "SPARSE_METHOD", + "link": 907, + "slot_index": 0 + }, + { + "name": "tk_optional", + "type": "TIMESTEP_KEYFRAME", + "link": null + } + ], + "outputs": [ + { + "name": "CONTROL_NET", + "type": "CONTROL_NET", + "links": [ + 879, + 978 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ACN_SparseCtrlLoaderAdvanced" + }, + "widgets_values": [ + "v3_sd15_sparsectrl_rgb.ckpt", + true, + 1, + 1 + ] + }, + { + "id": 470, + "type": "LoraLoader", + "pos": [ + -719, + -1368 + ], + "size": [ + 314.343994140625, + 126 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 896 + }, + { + "name": "clip", + "type": "CLIP", + "link": 893 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 909 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 994, + 996 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoraLoader" + }, + "widgets_values": [ + "v3_sd15_adapter.ckpt", + 0.4, + 0.25 + ] + }, + { + "id": 469, + "type": "ACN_SparseCtrlRGBPreprocessor", + "pos": [ + -763, + -1705 + ], + "size": { + "0": 312.9380187988281, + "1": 66 + }, + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 881 + }, + { + "name": "vae", + "type": "VAE", + "link": 882 + }, + { + "name": "latent_size", + "type": "LATENT", + "link": 883 + } + ], + "outputs": [ + { + "name": "proc_IMAGE", + "type": "IMAGE", + "links": [ + 880, + 963 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ACN_SparseCtrlRGBPreprocessor" + } + }, + { + "id": 505, + "type": "ACN_AdvancedControlNetApply", + "pos": [ + -276, + -1510 + ], + "size": { + "0": 285.6000061035156, + "1": 266 + }, + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 967, + "slot_index": 0 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 968, + "slot_index": 1 + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 978 + }, + { + "name": "image", + "type": "IMAGE", + "link": 963 + }, + { + "name": "mask_optional", + "type": "MASK", + "link": null + }, + { + "name": "timestep_kf", + "type": "TIMESTEP_KEYFRAME", + "link": null + }, + { + "name": "latent_kf_override", + "type": "LATENT_KEYFRAME", + "link": null + }, + { + "name": "weights_override", + "type": "CONTROL_NET_WEIGHTS", + "link": null + }, + { + "name": "model_optional", + "type": "MODEL", + "link": null + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 971 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 972 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "model_opt", + "type": "MODEL", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ACN_AdvancedControlNetApply" + }, + "widgets_values": [ + 0.5, + 0.6, + 0.675 + ] + }, + { + "id": 468, + "type": "ACN_AdvancedControlNetApply", + "pos": [ + -294, + -1903 + ], + "size": { + "0": 314.1339111328125, + "1": 266 + }, + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 917, + "slot_index": 0 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 918, + "slot_index": 1 + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": 879 + }, + { + "name": "image", + "type": "IMAGE", + "link": 880 + }, + { + "name": "mask_optional", + "type": "MASK", + "link": null + }, + { + "name": "timestep_kf", + "type": "TIMESTEP_KEYFRAME", + "link": null + }, + { + "name": "latent_kf_override", + "type": "LATENT_KEYFRAME", + "link": null + }, + { + "name": "weights_override", + "type": "CONTROL_NET_WEIGHTS", + "link": null + }, + { + "name": "model_optional", + "type": "MODEL", + "link": null + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 967 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 968 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "model_opt", + "type": "MODEL", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ACN_AdvancedControlNetApply" + }, + "widgets_values": [ + 0.6, + 0, + 0.3 + ] + }, + { + "id": 513, + "type": "Note Plus (mtb)", + "pos": { + "0": 2465, + "1": -1797, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": [ + 316.796583007811, + 265.0110820312493 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Note+ (mtb)", + "properties": {}, "widgets_values": [ "## ⬅ Tweak the context to change how the motion is influences by previous and forthcoming motion\n
\n\n- context_stride + context_overlap = 2 seems to provide the smoothest motion but leaves some artifacts.\n- context_stride 1, context_overlap = 4 feels like the best combination of smooothness and artifact avoidance.", "markdown", @@ -1540,12 +1785,273 @@ "bgcolor": "#335", "shape": 1 }, + { + "id": 548, + "type": "BatchValueScheduleLatentInput", + "pos": [ + 2057, + -2119 + ], + "size": { + "0": 352.79998779296875, + "1": 140 + }, + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "num_latents", + "type": "LATENT", + "link": 1006 + } + ], + "outputs": [ + { + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 998 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "INT", + "type": "INT", + "links": null, + "shape": 3, + "slot_index": 1 + }, + { + "name": "LATENT", + "type": "LATENT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BatchValueScheduleLatentInput" + }, + "widgets_values": [ + "0:(1.3)", + true + ] + }, + { + "id": 547, + "type": "ADE_UseEvolvedSampling", + "pos": [ + 2615, + -1099 + ], + "size": { + "0": 315, + "1": 118 + }, + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 1002 + }, + { + "name": "m_models", + "type": "M_MODELS", + "link": 1001 + }, + { + "name": "context_options", + "type": "CONTEXT_OPTIONS", + "link": 1005 + }, + { + "name": "sample_settings", + "type": "SAMPLE_SETTINGS", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 1003 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_UseEvolvedSampling" + }, + "widgets_values": [ + "sqrt_linear (AnimateDiff)" + ] + }, + { + "id": 545, + "type": "ADE_ApplyAnimateDiffModel", + "pos": [ + 3014, + -906 + ], + "size": { + "0": 319.20001220703125, + "1": 182 + }, + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "motion_model", + "type": "MOTION_MODEL_ADE", + "link": 999 + }, + { + "name": "motion_lora", + "type": "MOTION_LORA", + "link": 1004 + }, + { + "name": "scale_multival", + "type": "MULTIVAL", + "link": 1000 + }, + { + "name": "effect_multival", + "type": "MULTIVAL", + "link": null + }, + { + "name": "ad_keyframes", + "type": "AD_KEYFRAMES", + "link": null + }, + { + "name": "prev_m_models", + "type": "M_MODELS", + "link": null + } + ], + "outputs": [ + { + "name": "M_MODELS", + "type": "M_MODELS", + "links": [ + 1001 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_ApplyAnimateDiffModel" + }, + "widgets_values": [ + 0, + 1 + ] + }, + { + "id": 544, + "type": "ADE_MultivalDynamic", + "pos": [ + 2634, + -767 + ], + "size": { + "0": 308.1750793457031, + "1": 58 + }, + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "mask_optional", + "type": "MASK", + "link": null, + "slot_index": 0 + }, + { + "name": "float_val", + "type": "FLOAT", + "link": 998, + "widget": { + "name": "float_val" + } + } + ], + "outputs": [ + { + "name": "MULTIVAL", + "type": "MULTIVAL", + "links": [ + 1000 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_MultivalDynamic" + }, + "widgets_values": [ + 2 + ], + "shape": 2 + }, + { + "id": 536, + "type": "ADE_AnimateDiffLoRALoader", + "pos": [ + 3002, + -1263 + ], + "size": [ + 308.66559124414107, + 82 + ], + "flags": {}, + "order": 25, + "mode": 4, + "inputs": [ + { + "name": "prev_motion_lora", + "type": "MOTION_LORA", + "link": 1007 + } + ], + "outputs": [ + { + "name": "MOTION_LORA", + "type": "MOTION_LORA", + "links": [ + 1004 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffLoRALoader" + }, + "widgets_values": [ + "400_woman_dancing_temporal_unet.safetensors", + 0.4 + ] + }, { "id": 207, "type": "KSampler Adv. (Efficient)", "pos": [ - 1486, - -2172 + 1459, + -2161 ], "size": { "0": 496.3030090332031, @@ -1554,13 +2060,13 @@ "flags": { "pinned": false }, - "order": 31, + "order": 39, "mode": 0, "inputs": [ { "name": "model", "type": "MODEL", - "link": 935 + "link": 1003 }, { "name": "positive", @@ -1656,139 +2162,6 @@ "color": "#222233", "bgcolor": "#333355", "shape": 1 - }, - { - "id": 473, - "type": "BatchCreativeInterpolation", - "pos": [ - -278, - -629 - ], - "size": [ - 378.3703423237811, - 680 - ], - "flags": {}, - "order": 23, - "mode": 0, - "inputs": [ - { - "name": "positive", - "type": "CONDITIONING", - "link": 913 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 914 - }, - { - "name": "images", - "type": "IMAGE", - "link": 908 - }, - { - "name": "model", - "type": "MODEL", - "link": 909 - }, - { - "name": "ipadapter", - "type": "IPADAPTER", - "link": 910 - }, - { - "name": "clip_vision", - "type": "CLIP_VISION", - "link": 911 - } - ], - "outputs": [ - { - "name": "GRAPH", - "type": "IMAGE", - "links": [ - 901 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "POSITIVE", - "type": "CONDITIONING", - "links": [ - 917 - ], - "shape": 3, - "slot_index": 1 - }, - { - "name": "NEGATIVE", - "type": "CONDITIONING", - "links": [ - 918 - ], - "shape": 3, - "slot_index": 2 - }, - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 900 - ], - "shape": 3, - "slot_index": 3 - }, - { - "name": "KEYFRAME_POSITIONS", - "type": "SPARSE_METHOD", - "links": [ - 907 - ], - "shape": 3, - "slot_index": 4 - }, - { - "name": "BATCH_SIZE", - "type": "INT", - "links": [ - 906 - ], - "shape": 3, - "slot_index": 5 - }, - { - "name": "COLOR_MATCH_FRAMES", - "type": "STRING", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "BatchCreativeInterpolation" - }, - "widgets_values": [ - "control_v11f1e_sd15_tile_fp16.safetensors", - "linear", - 16, - "0,16,32,48", - "linear", - "0.75", - "", - "linear", - "(0.4,0.5,0.4)", - "(0.0,1.0),(0.0,1.0),(0.0,1.0),(0.0,1.0)", - 0.85, - 4, - 0, - 1, - 0.2, - 0, - 0.6, - 0, - 0.65 - ] } ], "links": [ @@ -1864,22 +2237,6 @@ 1, "CLIP" ], - [ - 894, - 470, - 1, - 462, - 0, - "CLIP" - ], - [ - 895, - 470, - 1, - 463, - 0, - "CLIP" - ], [ 896, 461, @@ -1888,14 +2245,6 @@ 0, "MODEL" ], - [ - 900, - 473, - 3, - 187, - 0, - "MODEL" - ], [ 901, 473, @@ -1952,22 +2301,6 @@ 5, "CLIP_VISION" ], - [ - 913, - 462, - 0, - 473, - 0, - "CONDITIONING" - ], - [ - 914, - 463, - 0, - 473, - 1, - "CONDITIONING" - ], [ 917, 473, @@ -2000,14 +2333,6 @@ 0, "IMAGE" ], - [ - 935, - 187, - 0, - 207, - 0, - "MODEL" - ], [ 938, 292, @@ -2089,12 +2414,116 @@ "CONTROL_NET" ], [ - 980, + 994, + 470, + 1, + 541, + 0, + "CLIP" + ], + [ + 995, + 541, + 0, + 473, + 0, + "CONDITIONING" + ], + [ + 996, + 470, + 1, + 543, + 0, + "CLIP" + ], + [ + 997, + 543, + 1, + 473, + 1, + "CONDITIONING" + ], + [ + 998, + 548, + 0, + 544, + 1, + "FLOAT" + ], + [ + 999, + 546, + 0, + 545, + 0, + "MOTION_MODEL_ADE" + ], + [ + 1000, + 544, + 0, + 545, + 2, + "MULTIVAL" + ], + [ + 1001, + 545, + 0, + 547, + 1, + "M_MODELS" + ], + [ + 1002, + 473, + 3, + 547, + 0, + "MODEL" + ], + [ + 1003, + 547, + 0, + 207, + 0, + "MODEL" + ], + [ + 1004, + 536, + 0, + 545, + 1, + "MOTION_LORA" + ], + [ + 1005, 342, 0, - 187, - 1, + 547, + 2, "CONTEXT_OPTIONS" + ], + [ + 1006, + 464, + 0, + 548, + 0, + "LATENT" + ], + [ + 1007, + 552, + 0, + 536, + 0, + "MOTION_LORA" ] ], "groups": [ @@ -2103,8 +2532,8 @@ "bounding": [ 1351, -2293, - 1409, - 868 + 1563, + 865 ], "color": "#8A8", "font_size": 24 @@ -2112,10 +2541,10 @@ { "title": "Increase Framerate", "bounding": [ - 1359, - -1327, - 1048, - 129 + 1507, + -1363, + 868, + 133 ], "color": "#8A8", "font_size": 24 @@ -2134,8 +2563,8 @@ { "title": "Group", "bounding": [ - -865, - -1101, + -858, + -1080, 2194, 1190 ], @@ -2157,7 +2586,7 @@ "title": "Basic Node", "bounding": [ 268, - -1905, + -1906, 996, 680 ], @@ -2174,6 +2603,17 @@ ], "color": "#3f789e", "font_size": 24 + }, + { + "title": "Advanced AD Stuff", + "bounding": [ + 2557, + -1370, + 821, + 707 + ], + "color": "#3f789e", + "font_size": 24 } ], "config": {},