From f7e831d65251c0ea81210f0e27c53903cf483ed4 Mon Sep 17 00:00:00 2001 From: kijai <40791699+kijai@users.noreply.github.com> Date: Tue, 2 Jul 2024 19:27:19 +0300 Subject: [PATCH] expose context options --- examples/mimic_motion_example_02.json | 536 +++++++++++++------------- nodes.py | 8 +- 2 files changed, 274 insertions(+), 270 deletions(-) diff --git a/examples/mimic_motion_example_02.json b/examples/mimic_motion_example_02.json index 6bf55ce..7ec64f8 100644 --- a/examples/mimic_motion_example_02.json +++ b/examples/mimic_motion_example_02.json @@ -58,84 +58,6 @@ true ] }, - { - "id": 5, - "type": "VHS_LoadVideo", - "pos": [ - -402, - 787 - ], - "size": [ - 235.1999969482422, - 658.5777723524305 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [ - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 86 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "frame_count", - "type": "INT", - "links": null, - "shape": 3 - }, - { - "name": "audio", - "type": "VHS_AUDIO", - "links": null, - "shape": 3 - }, - { - "name": "video_info", - "type": "VHS_VIDEOINFO", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VHS_LoadVideo" - }, - "widgets_values": { - "video": "pose1.mp4", - "force_rate": 0, - "force_size": "Disabled", - "custom_width": 512, - "custom_height": 512, - "frame_load_cap": 16, - "skip_first_frames": 0, - "select_every_nth": 3, - "choose video to upload": "image", - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "frame_load_cap": 16, - "skip_first_frames": 0, - "force_rate": 0, - "filename": "pose1.mp4", - "type": "input", - "format": "video/mp4", - "select_every_nth": 3 - } - } - } - }, { "id": 35, "type": "ImageResizeKJ", @@ -258,7 +180,7 @@ "shape": 3 }, { - "name": "17 count", + "name": "129 count", "type": "INT", "links": null, "shape": 3 @@ -275,10 +197,10 @@ 723, 819 ], - "size": [ - 440, - 978.6666666666666 - ], + "size": { + "0": 440, + "1": 218 + }, "flags": {}, "order": 11, "mode": 0, @@ -421,10 +343,10 @@ 290, 251 ], - "size": [ - 210, - 246 - ], + "size": { + "0": 210, + "1": 246 + }, "flags": {}, "order": 8, "mode": 0, @@ -451,7 +373,7 @@ "1": 82 }, "flags": {}, - "order": 1, + "order": 0, "mode": 0, "outputs": [ { @@ -478,12 +400,12 @@ 281, 104 ], - "size": [ - 293.73750640869093, - 101.2688590393065 - ], + "size": { + "0": 293.7375183105469, + "1": 101.26885986328125 + }, "flags": {}, - "order": 2, + "order": 1, "mode": 0, "properties": { "text": "" @@ -501,12 +423,12 @@ 779, 105 ], - "size": [ - 290.12339111328083, - 69.71562744140618 - ], + "size": { + "0": 290.1233825683594, + "1": 69.71562957763672 + }, "flags": {}, - "order": 3, + "order": 2, "mode": 0, "properties": { "text": "" @@ -567,168 +489,6 @@ null ] }, - { - "id": 16, - "type": "VHS_VideoCombine", - "pos": [ - 1452, - 196 - ], - "size": [ - 1530.494967759278, - 1199.1081290425352 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 93 - }, - { - "name": "audio", - "type": "VHS_AUDIO", - "link": null - }, - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null - } - ], - "outputs": [ - { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": { - "frame_rate": 8, - "loop_count": 0, - "filename_prefix": "MimicMotion", - "format": "video/h264-mp4", - "pix_fmt": "yuv420p", - "crf": 19, - "save_metadata": true, - "pingpong": false, - "save_output": false, - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "MimicMotion_00001.mp4", - "subfolder": "", - "type": "temp", - "format": "video/h264-mp4" - } - } - } - }, - { - "id": 1, - "type": "MimicMotionSampler", - "pos": [ - 1101, - 419 - ], - "size": { - "0": 307.6666259765625, - "1": 290 - }, - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "mimic_pipeline", - "type": "MIMICPIPE", - "link": 1, - "slot_index": 0 - }, - { - "name": "ref_image", - "type": "IMAGE", - "link": 128, - "slot_index": 1 - }, - { - "name": "pose_images", - "type": "IMAGE", - "link": 92 - } - ], - "outputs": [ - { - "name": "images", - "type": "IMAGE", - "links": [ - 99 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "MimicMotionSampler" - }, - "widgets_values": [ - 25, - 2, - 2, - 123, - "fixed", - 15, - 0.02, - true - ] - }, - { - "id": 3, - "type": "LoadImage", - "pos": [ - -393, - 311 - ], - "size": { - "0": 213.0849151611328, - "1": 410.70074462890625 - }, - "flags": {}, - "order": 4, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 61, - 124 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "demo1.jpg", - "image" - ] - }, { "id": 48, "type": "ImageResizeKJ", @@ -736,10 +496,10 @@ -57, 170 ], - "size": [ - 315, - 242 - ], + "size": { + "0": 315, + "1": 242 + }, "flags": {}, "order": 6, "mode": 0, @@ -809,6 +569,248 @@ 0, 0 ] + }, + { + "id": 16, + "type": "VHS_VideoCombine", + "pos": [ + 1482, + 192 + ], + "size": [ + 1530.4949951171875, + 1199.1081452546296 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 93 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "MimicMotion", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "MimicMotion_00005.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4" + } + } + } + }, + { + "id": 3, + "type": "LoadImage", + "pos": [ + -393, + 311 + ], + "size": { + "0": 213.0849151611328, + "1": 410.70074462890625 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 61, + 124 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2bc567e9f448176b4e2416ceebf9beab.png", + "image" + ] + }, + { + "id": 5, + "type": "VHS_LoadVideo", + "pos": [ + -402, + 787 + ], + "size": [ + 235.1999969482422, + 658.5777723524305 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 86 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose1.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 16, + "skip_first_frames": 0, + "select_every_nth": 3, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 16, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose1.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 3 + } + } + } + }, + { + "id": 1, + "type": "MimicMotionSampler", + "pos": [ + 1101, + 419 + ], + "size": { + "0": 307.6666259765625, + "1": 314 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "mimic_pipeline", + "type": "MIMICPIPE", + "link": 1, + "slot_index": 0 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 128, + "slot_index": 1 + }, + { + "name": "pose_images", + "type": "IMAGE", + "link": 92 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 99 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "MimicMotionSampler" + }, + "widgets_values": [ + 25, + 2, + 2, + 123, + "fixed", + 15, + 0.02, + 16, + 6, + true + ] } ], "links": [ @@ -953,10 +955,10 @@ "config": {}, "extra": { "ds": { - "scale": 0.5644739300537774, + "scale": 0.6830134553650705, "offset": { - "0": 738.7613525390625, - "1": 31.221477508544922 + "0": 839.62744140625, + "1": 131.28245544433594 } } }, diff --git a/nodes.py b/nodes.py index f3c9a1e..6d7c634 100644 --- a/nodes.py +++ b/nodes.py @@ -135,6 +135,8 @@ class MimicMotionSampler: "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), "fps": ("INT", {"default": 15, "min": 2, "max": 100, "step": 1}), "noise_aug_strength": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.01}), + "context_size": ("INT", {"default": 16, "min": 1, "max": 128, "step": 1}), + "context_overlap": ("INT", {"default": 6, "min": 1, "max": 128, "step": 1}), "keep_model_loaded": ("BOOLEAN", {"default": True}), }, } @@ -144,7 +146,7 @@ class MimicMotionSampler: FUNCTION = "process" CATEGORY = "MimicMotionWrapper" - def process(self, mimic_pipeline, ref_image, pose_images, cfg_min, cfg_max, steps, seed, noise_aug_strength, fps, keep_model_loaded): + def process(self, mimic_pipeline, ref_image, pose_images, cfg_min, cfg_max, steps, seed, noise_aug_strength, fps, keep_model_loaded, context_size, context_overlap): device = mm.get_torch_device() offload_device = mm.unet_offload_device() mm.unload_all_models() @@ -175,8 +177,8 @@ class MimicMotionSampler: ref_img, image_pose=pose_images, num_frames=B, - tile_size = 16, - tile_overlap= 6, + tile_size = context_size, + tile_overlap= context_overlap, height=H, width=W, fps=fps,