Files
banodoco-steerable-motion/demo/creative_interpolation_example.json
T
2023-11-17 11:44:00 +01:00

1279 lines
24 KiB
JSON

{
"last_node_id": 365,
"last_link_id": 667,
"nodes": [
{
"id": 354,
"type": "VHS_SplitImages",
"pos": [
2529.475905889869,
-1142.7707149068153
],
"size": {
"0": 315,
"1": 118
},
"flags": {},
"order": 19,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 653
}
],
"outputs": [
{
"name": "IMAGE_A",
"type": "IMAGE",
"links": null,
"shape": 3
},
{
"name": "A_count",
"type": "INT",
"links": null,
"shape": 3
},
{
"name": "IMAGE_B",
"type": "IMAGE",
"links": [
654
],
"shape": 3,
"slot_index": 2
},
{
"name": "B_count",
"type": "INT",
"links": null,
"shape": 3,
"slot_index": 3
}
],
"properties": {
"Node name for S&R": "VHS_SplitImages"
},
"widgets_values": {
"split_index": 4
}
},
{
"id": 292,
"type": "STMFNet VFI",
"pos": [
2568.475905889869,
-932.7707149068144
],
"size": {
"0": 443.4000244140625,
"1": 150
},
"flags": {},
"order": 20,
"mode": 0,
"inputs": [
{
"name": "frames",
"type": "IMAGE",
"link": 654
},
{
"name": "optional_interpolation_states",
"type": "INTERPOLATION_STATES",
"link": null
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
656
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "STMFNet VFI"
},
"widgets_values": [
"stmfnet.pth",
14,
2,
true
]
},
{
"id": 332,
"type": "PreviewImage",
"pos": [
-1460,
-1290
],
"size": {
"0": 380,
"1": 250
},
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 619
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 281,
"type": "VHS_VideoCombine",
"pos": [
3351.7515134107794,
-1039.9403187565451
],
"size": [
320,
496
],
"flags": {},
"order": 21,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 656
}
],
"outputs": [],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 14,
"loop_count": 0,
"filename_prefix": "creative_interpolation_results/AD_",
"format": "video/h264-mp4",
"pingpong": false,
"save_image": true,
"crf": 20,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "AD__00049.mp4",
"subfolder": "creative_interpolation_results",
"type": "output",
"format": "video/h264-mp4"
}
}
}
},
{
"id": 364,
"type": "Note",
"pos": [
-416,
-964
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 0,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Please set batch_size at frames_per_key_frame * number_of_key_frames + 4 - the 4 is to give additional buffer to solve issues with the uniform context scheduling. A hacky solution for now."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 362,
"type": "Note",
"pos": [
1884,
-1504
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 1,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"motion_scale can also be really useful - tuning it down can remove jolty motion."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 187,
"type": "ADE_AnimateDiffLoaderWithContext",
"pos": [
1910,
-1122
],
"size": {
"0": 315,
"1": 190
},
"flags": {
"pinned": false
},
"order": 17,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 570,
"slot_index": 0
},
{
"name": "context_options",
"type": "CONTEXT_OPTIONS",
"link": 627,
"slot_index": 1
},
{
"name": "motion_lora",
"type": "MOTION_LORA",
"link": null
},
{
"name": "motion_model_settings",
"type": "MOTION_MODEL_SETTINGS",
"link": null,
"slot_index": 3
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
543
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ADE_AnimateDiffLoaderWithContext"
},
"widgets_values": [
"mm_sd_v15_v2.ckpt",
"sqrt_linear (AnimateDiff)",
0.8,
false
],
"color": "#232",
"bgcolor": "#353"
},
{
"id": 363,
"type": "Note",
"pos": [
2651.475905889869,
-676.7707149068145
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 2,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Increasing the multipler makes the video longer by creating new frames - it can result in some weird effects >2 but worth experimenting with for additional motion/fluidity."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 359,
"type": "Note",
"pos": [
-1737,
-576
],
"size": {
"0": 261.6278991699219,
"1": 118.72222137451172
},
"flags": {},
"order": 3,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"## You'll need to create this folder in ComfyUI/inputs and place your images inside it in numerical order - 1.png, 2.png, etc."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 294,
"type": "IPAdapterModelLoader",
"pos": [
-171.81098593750033,
11.26162191162112
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 4,
"mode": 0,
"outputs": [
{
"name": "IPADAPTER",
"type": "IPADAPTER",
"links": [
564
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "IPAdapterModelLoader"
},
"widgets_values": [
"ip-adapter_sd15.bin"
]
},
{
"id": 301,
"type": "IPAdapterApplyEncoded",
"pos": [
-53.81098593749994,
-264.7383780883789
],
"size": {
"0": 315,
"1": 98
},
"flags": {},
"order": 16,
"mode": 0,
"inputs": [
{
"name": "ipadapter",
"type": "IPADAPTER",
"link": 564,
"slot_index": 0
},
{
"name": "embeds",
"type": "EMBEDS",
"link": 563,
"slot_index": 1
},
{
"name": "model",
"type": "MODEL",
"link": 571
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
570
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "IPAdapterApplyEncoded"
},
"widgets_values": [
0.6900000000000001
]
},
{
"id": 300,
"type": "IPAdapterEncoder",
"pos": [
-509.8109859375002,
-311.73837808837897
],
"size": {
"0": 315,
"1": 258
},
"flags": {},
"order": 14,
"mode": 0,
"inputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 566,
"slot_index": 0
},
{
"name": "image_1",
"type": "IMAGE",
"link": 657
},
{
"name": "image_2",
"type": "IMAGE",
"link": null
},
{
"name": "image_3",
"type": "IMAGE",
"link": null
},
{
"name": "image_4",
"type": "IMAGE",
"link": null
}
],
"outputs": [
{
"name": "EMBEDS",
"type": "EMBEDS",
"links": [
563
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "IPAdapterEncoder"
},
"widgets_values": [
false,
0,
1,
1,
1,
1
]
},
{
"id": 297,
"type": "CLIPVisionLoader",
"pos": [
-846.8109859375003,
43.26162191162103
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 5,
"mode": 0,
"outputs": [
{
"name": "CLIP_VISION",
"type": "CLIP_VISION",
"links": [
566
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "CLIPVisionLoader"
},
"widgets_values": [
"SD1.5/pytorch_model.bin"
]
},
{
"id": 207,
"type": "KSampler Adv. (Efficient)",
"pos": [
1510,
-1040
],
"size": {
"0": 325,
"1": 658
},
"flags": {
"pinned": false
},
"order": 18,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 543
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 666
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 667
},
{
"name": "latent_image",
"type": "LATENT",
"link": 633
},
{
"name": "optional_vae",
"type": "VAE",
"link": 632
},
{
"name": "script",
"type": "SCRIPT",
"link": null,
"slot_index": 5
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [],
"shape": 3,
"slot_index": 0
},
{
"name": "CONDITIONING+",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 1
},
{
"name": "CONDITIONING-",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 2
},
{
"name": "LATENT",
"type": "LATENT",
"links": [],
"shape": 3,
"slot_index": 3
},
{
"name": "VAE",
"type": "VAE",
"links": [],
"shape": 3,
"slot_index": 4
},
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
653
],
"shape": 3,
"slot_index": 5
}
],
"title": "KSampler Adv. (Efficient), CN sampler",
"properties": {
"Node name for S&R": "KSampler Adv. (Efficient)"
},
"widgets_values": [
"enable",
6,
null,
20,
10,
"ddpm",
"karras",
5,
20,
"disable",
"auto",
"true"
],
"color": "#222233",
"bgcolor": "#333355",
"shape": 1
},
{
"id": 360,
"type": "Note",
"pos": [
631,
-1677
],
"size": {
"0": 508.23260498046875,
"1": 385.82281494140625
},
"flags": {},
"order": 6,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Key frames are **distributed either linearly of dynamically**. \n\nIf you set type_of_frame_distribution to linear, you need to set linear_frame_distribution_value to the gap you wish to have between each key frame - e.g. 16 would mean the frames are at 0, 16, 32, 48, etc.\n\nAlternatively, if you set it to dynamic, you can input the positions of the key frames in the text box below this.\n\nSimilarly, type_of_key_frame_influence determines how the frames influence the next ones. If you set it to linear, linear_key_frame_influence_value will determine how long every frame influences - 1.0 will make it normal, higher values will make it longer. If you set type_of_key_frame_influence to dynamic, you can input distinct values in linear_key_frame_influence_value for each frame.\n\nOther than this, please experiment with the different settings to achieve your desired effect:\n\n- frames_per_key_frame: How many frames to generate between each main key frame you provide.\n- cn_strength: How strong the control of the ControlNet should overall.\n- soft_scaled_cn_weights_multiplier: kinda similar to cn_strength but also different - please experiment!\n- interpolation: the type of interpolation from one image to the next.\n- buffer is the number of buffer frames places before your video - this is to prevent the end of the generation influencing the beginning due to how the context scheduler works."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 352,
"type": "CLIPTextEncode",
"pos": [
10,
-962
],
"size": {
"0": 483.6921691894531,
"1": 182.32534790039062
},
"flags": {
"pinned": false
},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 644
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
664
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"hands, hand, (worst quality, low quality:1.2)"
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 189,
"type": "Efficient Loader",
"pos": [
-510,
-1656
],
"size": {
"0": 448.8115539550781,
"1": 605.9432983398438
},
"flags": {
"pinned": false
},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "lora_stack",
"type": "LORA_STACK",
"link": null
},
{
"name": "cnet_stack",
"type": "CONTROL_NET_STACK",
"link": null,
"slot_index": 1
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
571
],
"shape": 3,
"slot_index": 0
},
{
"name": "CONDITIONING+",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 1
},
{
"name": "CONDITIONING-",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 2
},
{
"name": "LATENT",
"type": "LATENT",
"links": [
633
],
"shape": 3,
"slot_index": 3
},
{
"name": "VAE",
"type": "VAE",
"links": [
632
],
"shape": 3,
"slot_index": 4
},
{
"name": "CLIP",
"type": "CLIP",
"links": [
634,
644
],
"shape": 3,
"slot_index": 5
},
{
"name": "DEPENDENCIES",
"type": "DEPENDENCIES",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "Efficient Loader"
},
"widgets_values": [
"Realistic_Vision_V5.0.safetensors",
"vae-ft-mse-840000-ema-pruned.safetensors",
-2,
"None",
1,
1,
"",
"",
"none",
"comfy",
512,
512,
100
],
"color": "#223333",
"bgcolor": "#335555",
"shape": 1
},
{
"id": 347,
"type": "BatchPromptSchedule",
"pos": [
100,
-1600
],
"size": {
"0": 418.2182922363281,
"1": 459.75811767578125
},
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 634
}
],
"outputs": [
{
"name": "POS",
"type": "CONDITIONING",
"links": [
663
],
"shape": 3,
"slot_index": 0
},
{
"name": "NEG",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "BatchPromptSchedule"
},
"widgets_values": [
"\"0\" :\"\",\n\"16\" :\"\",\n\"32\" :\"\",\n\"48\" :\"\",\n\"64\" :\"\",\n\"80\" :\"\",\n\"96\" :\"\"\n",
324,
false,
"PRE",
"APP",
0,
0,
0,
0
]
},
{
"id": 361,
"type": "Note",
"pos": [
1620,
-177
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 8,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Ipadapter weight also has a huge impact on the outcome - more = more adherence to the original images. Sometimes this can be too much control."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 323,
"type": "VHS_LoadImagesPath",
"pos": [
-1775,
-829
],
"size": {
"0": 315,
"1": 170
},
"flags": {},
"order": 9,
"mode": 0,
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
619,
657,
665
],
"shape": 3,
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3
},
{
"name": "INT",
"type": "INT",
"links": [],
"shape": 3,
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "VHS_LoadImagesPath"
},
"widgets_values": {
"directory": "creative_interpolation_input",
"image_load_cap": 0,
"skip_first_images": 0,
"select_every_nth": 1
},
"color": "#332922",
"bgcolor": "#593930"
},
{
"id": 342,
"type": "ADE_AnimateDiffUniformContextOptions",
"pos": [
1465,
-1439
],
"size": {
"0": 315,
"1": 154
},
"flags": {},
"order": 10,
"mode": 0,
"outputs": [
{
"name": "CONTEXT_OPTIONS",
"type": "CONTEXT_OPTIONS",
"links": [
627
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ADE_AnimateDiffUniformContextOptions"
},
"widgets_values": [
16,
2,
2,
"uniform",
false
]
},
{
"id": 365,
"type": "BatchCreativeInterpolation",
"pos": [
600,
-1160
],
"size": {
"0": 488.5699157714844,
"1": 382
},
"flags": {},
"order": 15,
"mode": 0,
"inputs": [
{
"name": "positive",
"type": "CONDITIONING",
"link": 663
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 664
},
{
"name": "images",
"type": "IMAGE",
"link": 665
}
],
"outputs": [
{
"name": "positive",
"type": "CONDITIONING",
"links": [
666
],
"shape": 3,
"slot_index": 0
},
{
"name": "negative",
"type": "CONDITIONING",
"links": [
667
],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "BatchCreativeInterpolation"
},
"widgets_values": [
"control_v11f1e_sd15_tile_fp16.safetensors",
"linear",
16,
"0,40,26,40,46,90",
"linear",
1.1,
"2.0,0.5,2.0,0.5,0.5,0.5",
0.5,
0.85,
"ease-in-out",
4
]
}
],
"links": [
[
543,
187,
0,
207,
0,
"MODEL"
],
[
563,
300,
0,
301,
1,
"EMBEDS"
],
[
564,
294,
0,
301,
0,
"IPADAPTER"
],
[
566,
297,
0,
300,
0,
"CLIP_VISION"
],
[
570,
301,
0,
187,
0,
"MODEL"
],
[
571,
189,
0,
301,
2,
"MODEL"
],
[
619,
323,
0,
332,
0,
"IMAGE"
],
[
627,
342,
0,
187,
1,
"CONTEXT_OPTIONS"
],
[
632,
189,
4,
207,
4,
"VAE"
],
[
633,
189,
3,
207,
3,
"LATENT"
],
[
634,
189,
5,
347,
0,
"CLIP"
],
[
644,
189,
5,
352,
0,
"CLIP"
],
[
653,
207,
5,
354,
0,
"IMAGE"
],
[
654,
354,
2,
292,
0,
"IMAGE"
],
[
656,
292,
0,
281,
0,
"IMAGE"
],
[
657,
323,
0,
300,
1,
"IMAGE"
],
[
663,
347,
0,
365,
0,
"CONDITIONING"
],
[
664,
352,
0,
365,
1,
"CONDITIONING"
],
[
665,
323,
0,
365,
2,
"IMAGE"
],
[
666,
365,
0,
207,
1,
"CONDITIONING"
],
[
667,
365,
1,
207,
2,
"CONDITIONING"
]
],
"groups": [
{
"title": "IPAdapter",
"bounding": [
-912,
-413,
1674,
582
],
"color": "#3f789e",
"font_size": 24
},
{
"title": "First sampler, interpolation",
"bounding": [
1418,
-1603,
862,
1341
],
"color": "#8A8",
"font_size": 24
},
{
"title": "Increase Framerate",
"bounding": [
2458,
-1259,
657,
898
],
"color": "#8A8",
"font_size": 24
},
{
"title": "Saving",
"bounding": [
3325,
-1153,
375,
657
],
"color": "#a1309b",
"font_size": 24
},
{
"title": "Group",
"bounding": [
-602,
-1770,
1783,
1088
],
"color": "#3f789e",
"font_size": 24
},
{
"title": "Group",
"bounding": [
-1849,
-1462,
914,
1124
],
"color": "#3f789e",
"font_size": 24
}
],
"config": {},
"extra": {},
"version": 0.4
}