Files
banodoco-steerable-motion/demo/creative_interpolation_example.json
T
2023-11-16 19:34:17 +01:00

1277 lines
24 KiB
JSON

{
"last_node_id": 365,
"last_link_id": 667,
"nodes": [
{
"id": 354,
"type": "VHS_SplitImages",
"pos": [
2529.475905889869,
-1142.7707149068153
],
"size": {
"0": 315,
"1": 118
},
"flags": {},
"order": 19,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 653
}
],
"outputs": [
{
"name": "IMAGE_A",
"type": "IMAGE",
"links": null,
"shape": 3
},
{
"name": "A_count",
"type": "INT",
"links": null,
"shape": 3
},
{
"name": "IMAGE_B",
"type": "IMAGE",
"links": [
654
],
"shape": 3,
"slot_index": 2
},
{
"name": "B_count",
"type": "INT",
"links": null,
"shape": 3,
"slot_index": 3
}
],
"properties": {
"Node name for S&R": "VHS_SplitImages"
},
"widgets_values": {
"split_index": 4
}
},
{
"id": 292,
"type": "STMFNet VFI",
"pos": [
2568.475905889869,
-932.7707149068144
],
"size": {
"0": 443.4000244140625,
"1": 150
},
"flags": {},
"order": 20,
"mode": 0,
"inputs": [
{
"name": "frames",
"type": "IMAGE",
"link": 654
},
{
"name": "optional_interpolation_states",
"type": "INTERPOLATION_STATES",
"link": null
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
656
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "STMFNet VFI"
},
"widgets_values": [
"stmfnet.pth",
14,
2,
true
]
},
{
"id": 332,
"type": "PreviewImage",
"pos": [
-1460,
-1290
],
"size": {
"0": 380,
"1": 250
},
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 619
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 281,
"type": "VHS_VideoCombine",
"pos": [
3351.7515134107794,
-1039.9403187565451
],
"size": [
320,
496
],
"flags": {},
"order": 21,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 656
}
],
"outputs": [],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 14,
"loop_count": 0,
"filename_prefix": "creative_interpolation_results/AD_",
"format": "video/h264-mp4",
"pingpong": false,
"save_image": true,
"crf": 20,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "AD__00041.mp4",
"subfolder": "creative_interpolation_results",
"type": "output",
"format": "video/h264-mp4"
}
}
}
},
{
"id": 347,
"type": "BatchPromptSchedule",
"pos": [
95,
-1565
],
"size": {
"0": 418.2182922363281,
"1": 459.75811767578125
},
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 634
}
],
"outputs": [
{
"name": "POS",
"type": "CONDITIONING",
"links": [
663
],
"shape": 3,
"slot_index": 0
},
{
"name": "NEG",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "BatchPromptSchedule"
},
"widgets_values": [
"\"0\" :\"\",\n\"16\" :\"\",\n\"32\" :\"\",\n\"48\" :\"\",\n\"64\" :\"\",\n\"80\" :\"\",\n\"96\" :\"\"\n",
324,
false,
"PRE",
"APP",
0,
0,
0,
0
]
},
{
"id": 364,
"type": "Note",
"pos": [
-416,
-964
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 0,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Please set batch_size at frames_per_key_frame * number_of_key_frames + 4 - the 4 is to give additional buffer to solve issues with the uniform context scheduling. A hacky solution for now."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 361,
"type": "Note",
"pos": [
682.4829725972701,
-283.2324170759726
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 1,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Ipadapter weight also has a huge impact on the outcome - more = more adherence to the original images. Sometimes this can be too much control."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 342,
"type": "ADE_AnimateDiffUniformContextOptions",
"pos": [
1465,
-1439
],
"size": {
"0": 315,
"1": 154
},
"flags": {},
"order": 2,
"mode": 0,
"outputs": [
{
"name": "CONTEXT_OPTIONS",
"type": "CONTEXT_OPTIONS",
"links": [
627
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ADE_AnimateDiffUniformContextOptions"
},
"widgets_values": [
16,
2,
2,
"uniform",
false
]
},
{
"id": 362,
"type": "Note",
"pos": [
1884,
-1504
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 3,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"motion_scale can also be really useful - tuning it down can remove jolty motion."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 187,
"type": "ADE_AnimateDiffLoaderWithContext",
"pos": [
1910,
-1122
],
"size": {
"0": 315,
"1": 190
},
"flags": {
"pinned": false
},
"order": 17,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 570,
"slot_index": 0
},
{
"name": "context_options",
"type": "CONTEXT_OPTIONS",
"link": 627,
"slot_index": 1
},
{
"name": "motion_lora",
"type": "MOTION_LORA",
"link": null
},
{
"name": "motion_model_settings",
"type": "MOTION_MODEL_SETTINGS",
"link": null,
"slot_index": 3
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
543
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "ADE_AnimateDiffLoaderWithContext"
},
"widgets_values": [
"mm_sd_v15_v2.ckpt",
"sqrt_linear (AnimateDiff)",
0.8,
false
],
"color": "#232",
"bgcolor": "#353"
},
{
"id": 363,
"type": "Note",
"pos": [
2651.475905889869,
-676.7707149068145
],
"size": {
"0": 312.1289367675781,
"1": 245.289306640625
},
"flags": {},
"order": 4,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Increasing the multipler makes the video longer by creating new frames - it can result in some weird effects >2 but worth experimenting with for additional motion/fluidity."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 207,
"type": "KSampler Adv. (Efficient)",
"pos": [
1510,
-1040
],
"size": {
"0": 325,
"1": 658
},
"flags": {
"pinned": false
},
"order": 18,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "MODEL",
"link": 543
},
{
"name": "positive",
"type": "CONDITIONING",
"link": 666
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 667
},
{
"name": "latent_image",
"type": "LATENT",
"link": 633
},
{
"name": "optional_vae",
"type": "VAE",
"link": 632
},
{
"name": "script",
"type": "SCRIPT",
"link": null,
"slot_index": 5
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [],
"shape": 3,
"slot_index": 0
},
{
"name": "CONDITIONING+",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 1
},
{
"name": "CONDITIONING-",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 2
},
{
"name": "LATENT",
"type": "LATENT",
"links": [],
"shape": 3,
"slot_index": 3
},
{
"name": "VAE",
"type": "VAE",
"links": [],
"shape": 3,
"slot_index": 4
},
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
653
],
"shape": 3,
"slot_index": 5
}
],
"title": "KSampler Adv. (Efficient), CN sampler",
"properties": {
"Node name for S&R": "KSampler Adv. (Efficient)"
},
"widgets_values": [
"enable",
6,
null,
20,
10,
"ddpm",
"karras",
5,
20,
"disable",
"auto",
"true"
],
"color": "#222233",
"bgcolor": "#333355",
"shape": 1
},
{
"id": 352,
"type": "CLIPTextEncode",
"pos": [
95,
-962
],
"size": {
"0": 483.6921691894531,
"1": 182.32534790039062
},
"flags": {
"pinned": false
},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 644
}
],
"outputs": [
{
"name": "CONDITIONING",
"type": "CONDITIONING",
"links": [
664
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPTextEncode"
},
"widgets_values": [
"hands, hand, (worst quality, low quality:1.2)"
],
"color": "#322",
"bgcolor": "#533"
},
{
"id": 365,
"type": "BatchCreativeInterpolation",
"pos": [
636,
-1269
],
"size": [
488.56990545482176,
347.0449828807655
],
"flags": {},
"order": 15,
"mode": 0,
"inputs": [
{
"name": "positive",
"type": "CONDITIONING",
"link": 663
},
{
"name": "negative",
"type": "CONDITIONING",
"link": 664
},
{
"name": "images",
"type": "IMAGE",
"link": 665
}
],
"outputs": [
{
"name": "positive",
"type": "CONDITIONING",
"links": [
666
],
"shape": 3,
"slot_index": 0
},
{
"name": "negative",
"type": "CONDITIONING",
"links": [
667
],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "BatchCreativeInterpolation"
},
"widgets_values": [
"control_v11f1e_sd15_tile_fp16.safetensors",
"linear",
16,
"0,10,26,40,46",
1.1,
0.5,
0.85,
"ease-in-out",
4
]
},
{
"id": 189,
"type": "Efficient Loader",
"pos": [
-510,
-1656
],
"size": {
"0": 448.8115539550781,
"1": 605.9432983398438
},
"flags": {
"pinned": false
},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "lora_stack",
"type": "LORA_STACK",
"link": null
},
{
"name": "cnet_stack",
"type": "CONTROL_NET_STACK",
"link": null,
"slot_index": 1
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
571
],
"shape": 3,
"slot_index": 0
},
{
"name": "CONDITIONING+",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 1
},
{
"name": "CONDITIONING-",
"type": "CONDITIONING",
"links": [],
"shape": 3,
"slot_index": 2
},
{
"name": "LATENT",
"type": "LATENT",
"links": [
633
],
"shape": 3,
"slot_index": 3
},
{
"name": "VAE",
"type": "VAE",
"links": [
632
],
"shape": 3,
"slot_index": 4
},
{
"name": "CLIP",
"type": "CLIP",
"links": [
634,
644
],
"shape": 3,
"slot_index": 5
},
{
"name": "DEPENDENCIES",
"type": "DEPENDENCIES",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "Efficient Loader"
},
"widgets_values": [
"Realistic_Vision_V5.0.safetensors",
"vae-ft-mse-840000-ema-pruned.safetensors",
-2,
"None",
1,
1,
"",
"",
"none",
"comfy",
512,
512,
52
],
"color": "#223333",
"bgcolor": "#335555",
"shape": 1
},
{
"id": 323,
"type": "VHS_LoadImagesPath",
"pos": [
-1775,
-829
],
"size": {
"0": 315,
"1": 170
},
"flags": {},
"order": 6,
"mode": 0,
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
619,
657,
665
],
"shape": 3,
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null,
"shape": 3
},
{
"name": "INT",
"type": "INT",
"links": [],
"shape": 3,
"slot_index": 2
}
],
"properties": {
"Node name for S&R": "VHS_LoadImagesPath"
},
"widgets_values": {
"directory": "creative_interpolation_input",
"image_load_cap": 0,
"skip_first_images": 0,
"select_every_nth": 1
},
"color": "#332922",
"bgcolor": "#593930"
},
{
"id": 359,
"type": "Note",
"pos": [
-1737,
-576
],
"size": {
"0": 261.6278991699219,
"1": 118.72222137451172
},
"flags": {},
"order": 7,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"## You'll need to create this folder in ComfyUI/inputs and place your images inside it in numerical order - 1.png, 2.png, etc."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 360,
"type": "Note",
"pos": [
614,
-1687
],
"size": [
506.9915747996947,
319.2813254950913
],
"flags": {},
"order": 8,
"mode": 0,
"properties": {
"text": ""
},
"widgets_values": [
"Key frames are **distributed either linearly of dynamically**. \n\nIf you set type_of_frame_distribution to linear, you need to set linear_frames_per_keyframe to the gap you wish to have between each key frame - e.g. 16 would mean the frames are at 0, 16, 32, 48, etc.\n\nAlternatively, if you set it to dynamic, you can input the positions of the key frames in the text box below this.\n\nOther than this, please experiment with the different settings to achieve your desired effect:\n\n- frames_per_key_frame: How many frames to generate between each main key frame you provide.\n- length_of_key_frame_influence: How many frames to apply the ControlNet for after each key frame - the larger the number, the the wider range the input images will influence.\n- cn_strength: How strong the control of the ControlNet should overall.\n- soft_scaled_cn_weights_multiplier: kinda similar to cn_strength but also different - please experiment!\n- interpolation: the type of interpolation from one image to the next.\n- buffer is the number of buffer frames places before your video - this is to prevent the end of the generation influencing the beginning due to how the context scheduler works."
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 294,
"type": "IPAdapterModelLoader",
"pos": [
170,
-61
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 9,
"mode": 0,
"outputs": [
{
"name": "IPADAPTER",
"type": "IPADAPTER",
"links": [
564
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "IPAdapterModelLoader"
},
"widgets_values": [
"ip-adapter_sd15.bin"
]
},
{
"id": 301,
"type": "IPAdapterApplyEncoded",
"pos": [
288,
-337
],
"size": {
"0": 315,
"1": 98
},
"flags": {},
"order": 16,
"mode": 0,
"inputs": [
{
"name": "ipadapter",
"type": "IPADAPTER",
"link": 564,
"slot_index": 0
},
{
"name": "embeds",
"type": "EMBEDS",
"link": 563,
"slot_index": 1
},
{
"name": "model",
"type": "MODEL",
"link": 571
}
],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
570
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "IPAdapterApplyEncoded"
},
"widgets_values": [
0.6900000000000001
]
},
{
"id": 300,
"type": "IPAdapterEncoder",
"pos": [
-168,
-384
],
"size": {
"0": 315,
"1": 258
},
"flags": {},
"order": 14,
"mode": 0,
"inputs": [
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 566,
"slot_index": 0
},
{
"name": "image_1",
"type": "IMAGE",
"link": 657
},
{
"name": "image_2",
"type": "IMAGE",
"link": null
},
{
"name": "image_3",
"type": "IMAGE",
"link": null
},
{
"name": "image_4",
"type": "IMAGE",
"link": null
}
],
"outputs": [
{
"name": "EMBEDS",
"type": "EMBEDS",
"links": [
563
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "IPAdapterEncoder"
},
"widgets_values": [
false,
0,
1,
1,
1,
1
]
},
{
"id": 297,
"type": "CLIPVisionLoader",
"pos": [
-505,
-29
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 10,
"mode": 0,
"outputs": [
{
"name": "CLIP_VISION",
"type": "CLIP_VISION",
"links": [
566
],
"shape": 3
}
],
"properties": {
"Node name for S&R": "CLIPVisionLoader"
},
"widgets_values": [
"SD1.5/pytorch_model.bin"
]
}
],
"links": [
[
543,
187,
0,
207,
0,
"MODEL"
],
[
563,
300,
0,
301,
1,
"EMBEDS"
],
[
564,
294,
0,
301,
0,
"IPADAPTER"
],
[
566,
297,
0,
300,
0,
"CLIP_VISION"
],
[
570,
301,
0,
187,
0,
"MODEL"
],
[
571,
189,
0,
301,
2,
"MODEL"
],
[
619,
323,
0,
332,
0,
"IMAGE"
],
[
627,
342,
0,
187,
1,
"CONTEXT_OPTIONS"
],
[
632,
189,
4,
207,
4,
"VAE"
],
[
633,
189,
3,
207,
3,
"LATENT"
],
[
634,
189,
5,
347,
0,
"CLIP"
],
[
644,
189,
5,
352,
0,
"CLIP"
],
[
653,
207,
5,
354,
0,
"IMAGE"
],
[
654,
354,
2,
292,
0,
"IMAGE"
],
[
656,
292,
0,
281,
0,
"IMAGE"
],
[
657,
323,
0,
300,
1,
"IMAGE"
],
[
663,
347,
0,
365,
0,
"CONDITIONING"
],
[
664,
352,
0,
365,
1,
"CONDITIONING"
],
[
665,
323,
0,
365,
2,
"IMAGE"
],
[
666,
365,
0,
207,
1,
"CONDITIONING"
],
[
667,
365,
1,
207,
2,
"CONDITIONING"
]
],
"groups": [
{
"title": "IPAdapter",
"bounding": [
-570,
-485,
1674,
582
],
"color": "#3f789e",
"font_size": 24
},
{
"title": "First sampler, interpolation",
"bounding": [
1418,
-1603,
862,
1341
],
"color": "#8A8",
"font_size": 24
},
{
"title": "Increase Framerate",
"bounding": [
2458,
-1259,
657,
898
],
"color": "#8A8",
"font_size": 24
},
{
"title": "Saving",
"bounding": [
3325,
-1153,
375,
657
],
"color": "#a1309b",
"font_size": 24
},
{
"title": "Group",
"bounding": [
-602,
-1770,
1783,
1088
],
"color": "#3f789e",
"font_size": 24
},
{
"title": "Group",
"bounding": [
-1849,
-1462,
914,
1124
],
"color": "#3f789e",
"font_size": 24
}
],
"config": {},
"extra": {},
"version": 0.4
}