Files
banodoco-steerable-motion/demo/creative_interpolation_example.json
T
2023-11-14 11:50:07 +01:00

1274 lines
23 KiB
JSON

{
"last_node_id": 364,
"last_link_id": 662,
"nodes": [
{
"id": 294,
"type": "IPAdapterModelLoader",
"pos": [
433.73271339338095,
-9.957814472915118
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "IPADAPTER",
"type": "IPADAPTER",
"links": [
564
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "IPAdapterModelLoader"
},
"widgets_values": [
"ip-adapter_sd15.bin"
]
},
{
"id": 342,
"type": "ADE_AnimateDiffUniformContextOptions",
"pos": [
1487,
-1400
],
"size": {
"0": 315,
"1": 154
},
"flags": {},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "CONTEXT_OPTIONS",
"type": "CONTEXT_OPTIONS",
"links": [
627
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ADE_AnimateDiffUniformContextOptions"
},
"widgets_values": [
16,
2,
2,
"uniform",
false
]
},
{
"id": 354,
"type": "VHS_SplitImages",
"pos": [
2439,
-913
],
"size": {
"0": 315,
"1": 118
},
"flags": {},
"order": 19,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 653
}
],
"outputs": [
{
"name": "IMAGE_A",
"type": "IMAGE",
"links": null,
"shape": 3
},
{
"name": "A_count",
"type": "INT",
"links": null,
"shape": 3
},
{
"name": "IMAGE_B",
"type": "IMAGE",
"links": [
654
],
"shape": 3,
"slot_index": 2
},
{
"name": "B_count",
"type": "INT",
"links": null,
"shape": 3,
"slot_index": 3
}
],
"properties": {
"Node name for S&R": "VHS_SplitImages"
},
"widgets_values": {
"split_index": 4
}
},
{
"id": 352,
"type": "CLIPTextEncode",
"pos": [
203,
-1012
],
"size": {
"0": 483.6921691894531,
"1": 182.32534790039062
},
"flags": {
"pinned": false
},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 644
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
660
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"hands, hand, (worst quality, low quality:1.2)"
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 189,
"type": "Efficient Loader",
"pos": [
-410,
-1497
],
"size": {
"0": 448.8115539550781,
"1": 605.9432983398438
},
"flags": {
"pinned": false
},
"order": 2,
"mode": 0,
"inputs": [
{
"name": "lora_stack",
"type": "LORA_STACK",
"link": null
},
{
"name": "cnet_stack",
"type": "CONTROL_NET_STACK",
"link": null,
"slot_index": 1
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
571
],
"shape": 3,
"slot_index": 0
},
{
"name": "CONDITIONING+",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 1
},
{
"name": "CONDITIONING-",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 2
},
{
"name": "LATENT",
"type": "LATENT",
"links": [
633
],
"shape": 3,
"slot_index": 3
},
{
"name": "VAE",
"type": "VAE",
"links": [
632
],
"shape": 3,
"slot_index": 4
},
{
"name": "CLIP",
"type": "CLIP",
"links": [
634,
644
],
"shape": 3,
"slot_index": 5
},
{
"name": "DEPENDENCIES",
"type": "DEPENDENCIES",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "Efficient Loader"
},
"widgets_values": [
"Realistic_Vision_V5.0.safetensors",
"vae-ft-mse-840000-ema-pruned.safetensors",
-2,
"None",
1,
1,
"",
"",
"none",
"comfy",
512,
512,
100
],
"color": "#223333",
"bgcolor": "#335555",
"shape": 1
},
{
"id": 292,
"type": "STMFNet VFI",
"pos": [
2478,
-703
],
"size": {
"0": 443.4000244140625,
"1": 150
},
"flags": {},
"order": 20,
"mode": 0,
"inputs": [
{
"name": "frames",
"type": "IMAGE",
"link": 654
},
{
"name": "optional_interpolation_states",
"type": "INTERPOLATION_STATES",
"link": null
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
656
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "STMFNet VFI"
},
"widgets_values": [
"stmfnet.pth",
14,
2,
true
]
},
{
"id": 332,
"type": "PreviewImage",
"pos": [
-1460,
-1290
],
"size": {
"0": 380,
"1": 250
},
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 619
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 297,
"type": "CLIPVisionLoader",
"pos": [
-511,
-11
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 3,
"mode": 0,
"outputs": [
{
"name": "CLIP_VISION",
"type": "CLIP_VISION",
"links": [
566
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "CLIPVisionLoader"
},
"widgets_values": [
"SD1.5/pytorch_model.bin"
]
},
{
"id": 281,
"type": "VHS_VideoCombine",
"pos": [
3351.7515134107794,
-1039.9403187565451
],
"size": [
320,
496
],
"flags": {},
"order": 21,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 656
}
],
"outputs": [],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 14,
"loop_count": 0,
"filename_prefix": "creative_interpolation_results/AD_",
"format": "video/h264-mp4",
"pingpong": false,
"save_image": true,
"crf": 20,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "AD__00016.mp4",
"subfolder": "creative_interpolation_results",
"type": "output",
"format": "video/h264-mp4"
}
}
}
},
{
"id": 300,
"type": "IPAdapterEncoder",
"pos": [
-0.7293116041451541,
-239.86153266843684
],
"size": {
"0": 315,
"1": 258
},
"flags": {},
"order": 14,
"mode": 0,
"inputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 566,
"slot_index": 0
},
{
"name": "image_1",
"type": "IMAGE",
"link": 657
},
{
"name": "image_2",
"type": "IMAGE",
"link": null
},
{
"name": "image_3",
"type": "IMAGE",
"link": null
},
{
"name": "image_4",
"type": "IMAGE",
"link": null
}
],
"outputs": [
{
"name": "EMBEDS",
"type": "EMBEDS",
"links": [
563
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "IPAdapterEncoder"
},
"widgets_values": [
false,
0,
1,
1,
1,
1
]
},
{
"id": 187,
"type": "ADE_AnimateDiffLoaderWithContext",
"pos": [
1911,
-1414
],
"size": {
"0": 315,
"1": 190
},
"flags": {
"pinned": false
},
"order": 17,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 570,
"slot_index": 0
},
{
"name": "context_options",
"type": "CONTEXT_OPTIONS",
"link": 627,
"slot_index": 1
},
{
"name": "motion_lora",
"type": "MOTION_LORA",
"link": null
},
{
"name": "motion_model_settings",
"type": "MOTION_MODEL_SETTINGS",
"link": null,
"slot_index": 3
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
543
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ADE_AnimateDiffLoaderWithContext"
},
"widgets_values": [
"mm_sd_v15_v2.ckpt",
"sqrt_linear (AnimateDiff)",
0.8,
false
],
"color": "#232",
"bgcolor": "#353"
},
{
"id": 301,
"type": "IPAdapterApplyEncoded",
"pos": [
483.27068839585485,
-251.86153266843684
],
"size": {
"0": 315,
"1": 98
},
"flags": {},
"order": 16,
"mode": 0,
"inputs": [
{
"name": "ipadapter",
"type": "IPADAPTER",
"link": 564,
"slot_index": 0
},
{
"name": "embeds",
"type": "EMBEDS",
"link": 563,
"slot_index": 1
},
{
"name": "model",
"type": "MODEL",
"link": 571
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
570
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "IPAdapterApplyEncoded"
},
"widgets_values": [
0.6900000000000001
]
},
{
"id": 207,
"type": "KSampler Adv. (Efficient)",
"pos": [
1780,
-1040
],
"size": {
"0": 325,
"1": 658
},
"flags": {
"pinned": false
},
"order": 18,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 543
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 661
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 662
},
{
"name": "latent_image",
"type": "LATENT",
"link": 633
},
{
"name": "optional_vae",
"type": "VAE",
"link": 632
},
{
"name": "script",
"type": "SCRIPT",
"link": null,
"slot_index": 5
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [],
"shape": 3,
"slot_index": 0
},
{
"name": "CONDITIONING+",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 1
},
{
"name": "CONDITIONING-",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 2
},
{
"name": "LATENT",
"type": "LATENT",
"links": [],
"shape": 3,
"slot_index": 3
},
{
"name": "VAE",
"type": "VAE",
"links": [],
"shape": 3,
"slot_index": 4
},
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
653
],
"shape": 3,
"slot_index": 5
}
],
"title": "KSampler Adv. (Efficient), CN sampler",
"properties": {
"Node name for S&R": "KSampler Adv. (Efficient)"
},
"widgets_values": [
"enable",
6,
null,
20,
10,
"ddpm",
"karras",
5,
20,
"disable",
"auto",
"true"
],
"color": "#222233",
"bgcolor": "#333355",
"shape": 1
},
{
"id": 323,
"type": "VHS_LoadImagesPath",
"pos": [
-1792,
-811
],
"size": {
"0": 315,
"1": 170
},
"flags": {},
"order": 4,
"mode": 0,
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
619,
657,
658
],
"shape": 3,
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3
},
{
"name": "INT",
"type": "INT",
"links": [],
"shape": 3,
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "VHS_LoadImagesPath"
},
"widgets_values": {
"directory": "creative_interpolation",
"image_load_cap": 0,
"skip_first_images": 0,
"select_every_nth": 1
},
"color": "#332922",
"bgcolor": "#593930"
},
{
"id": 359,
"type": "Note",
"pos": [
-2088,
-675
],
"size": [
261.6278922750405,
118.72221974820172
],
"flags": {},
"order": 5,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"## You'll need to create this folder in ComfyUI/inputs and place your images inside it in numerical order - 1.png, 2.png, etc."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 347,
"type": "BatchPromptSchedule",
"pos": [
232,
-1553
],
"size": {
"0": 418.2182922363281,
"1": 459.75811767578125
},
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 634
}
],
"outputs": [
{
"name": "POS",
"type": "CONDITIONING",
"links": [
659
],
"shape": 3,
"slot_index": 0
},
{
"name": "NEG",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "BatchPromptSchedule"
},
"widgets_values": [
"\"0\" :\"\",\n\"16\" :\"\",\n\"32\" :\"\",\n\"48\" :\"\",\n\"64\" :\"\",\n\"80\" :\"\",\n\"96\" :\"\"\n",
120,
false,
"PRE",
"APP",
0,
0,
0,
0
]
},
{
"id": 358,
"type": "LinearBatchCreativeInterpolation",
"pos": [
770,
-1421
],
"size": {
"0": 315,
"1": 218
},
"flags": {},
"order": 15,
"mode": 0,
"inputs": [
{
"name": "positive",
"type": "CONDITIONING",
"link": 659
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 660
},
{
"name": "images",
"type": "IMAGE",
"link": 658
}
],
"outputs": [
{
"name": "positive",
"type": "CONDITIONING",
"links": [
661
],
"shape": 3,
"slot_index": 0
},
{
"name": "negative",
"type": "CONDITIONING",
"links": [
662
],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "LinearBatchCreativeInterpolation"
},
"widgets_values": [
"control_v11f1e_sd15_tile_fp16.safetensors",
16,
1.1,
0.5,
0.87,
"ease-in-out"
]
},
{
"id": 361,
"type": "Note",
"pos": [
824,
-185
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 6,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Ipadapter weight also has a huge impact on the outcome - more = more adherence to the original images. Sometimes this can be too much control."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 362,
"type": "Note",
"pos": [
1946,
-1729
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 7,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"motion_scale can also be really useful - tuning it down can remove jolty motion."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 363,
"type": "Note",
"pos": [
2672,
-423
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 8,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Increasing the multipler makes the video longer by creating new frames - it can result in some weird effects >2 but worth experimenting with for additional motion/fluidity."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 360,
"type": "Note",
"pos": [
810,
-1753
],
"size": [
312.1289306640617,
245.28931274414026
],
"flags": {},
"order": 9,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Please experiment with the different settings to achieve your desired effect:\n\n- frames_per_key_frame: How many frames to generate between each main key frame you provide.\n- length_of_key_frame_influence: How many frames to apply the ControlNet for after each key frame - the larger the number, the the wider range the input images will influence.\n- cn_strength: How strong the control of the ControlNet should overall.\n- soft_scaled_cn_weights_multiplier: kinda similar to cn_strength but also different - please experiment!\n- interpolation: the type of interpolation from one image to the next."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 364,
"type": "Note",
"pos": [
-389,
-837
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 10,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Please set batch_size at frames_per_key_frame * number_of_key_frames + 4 - the 4 is to give additional buffer to solve issues with the uniform context scheduling. A hacky solution for now."
],
"color": "#432",
"bgcolor": "#653"
}
],
"links": [
[
543,
187,
0,
207,
0,
"MODEL"
],
[
563,
300,
0,
301,
1,
"EMBEDS"
],
[
564,
294,
0,
301,
0,
"IPADAPTER"
],
[
566,
297,
0,
300,
0,
"CLIP_VISION"
],
[
570,
301,
0,
187,
0,
"MODEL"
],
[
571,
189,
0,
301,
2,
"MODEL"
],
[
619,
323,
0,
332,
0,
"IMAGE"
],
[
627,
342,
0,
187,
1,
"CONTEXT_OPTIONS"
],
[
632,
189,
4,
207,
4,
"VAE"
],
[
633,
189,
3,
207,
3,
"LATENT"
],
[
634,
189,
5,
347,
0,
"CLIP"
],
[
644,
189,
5,
352,
0,
"CLIP"
],
[
653,
207,
5,
354,
0,
"IMAGE"
],
[
654,
354,
2,
292,
0,
"IMAGE"
],
[
656,
292,
0,
281,
0,
"IMAGE"
],
[
657,
323,
0,
300,
1,
"IMAGE"
],
[
658,
323,
0,
358,
2,
"IMAGE"
],
[
659,
347,
0,
358,
0,
"CONDITIONING"
],
[
660,
352,
0,
358,
1,
"CONDITIONING"
],
[
661,
358,
0,
207,
1,
"CONDITIONING"
],
[
662,
358,
1,
207,
2,
"CONDITIONING"
]
],
"groups": [
{
"title": "IPAdapter",
"bounding": [
-663,
-400,
1688,
543
],
"color": "#3f789e",
"font_size": 24
},
{
"title": "First sampler, interpolation",
"bounding": [
1418,
-1603,
862,
1341
],
"color": "#8A8",
"font_size": 24
},
{
"title": "Increase Framerate",
"bounding": [
2368,
-1029,
624,
518
],
"color": "#8A8",
"font_size": 24
},
{
"title": "Saving",
"bounding": [
3325,
-1153,
375,
657
],
"color": "#a1309b",
"font_size": 24
},
{
"title": "Group",
"bounding": [
-602,
-1770,
1783,
1088
],
"color": "#3f789e",
"font_size": 24
},
{
"title": "Group",
"bounding": [
-1849,
-1462,
914,
1124
],
"color": "#3f789e",
"font_size": 24
}
],
"config": {},
"extra": {},
"version": 0.4
}