From 824e61c5bab6060869d0edfe1ed33c231e2e2000 Mon Sep 17 00:00:00 2001 From: PowerHouseMan Date: Wed, 21 Aug 2024 15:07:54 +0900 Subject: [PATCH] main 1. New workflow. 2. Create animations without video. 3. Tracking source video. --- nodes.py | 131 +++-- ...t.png => workflow_advanced_screenshot.png} | Bin ...shot.png => workflow_basic_screenshot.png} | Bin .../Advanced_Animate_without_vid.json | 359 ++++++++++++ ...anced_Inserting_expressions_into_vid.json} | 540 +++++++++--------- ...acis_Extracting_expression_from_photo.json | 241 ++++++++ ...Liveportrait_with_expression_editing.json} | 424 ++------------ .../Bacis_Simple_expression_editing.json | 194 +++++++ 8 files changed, 1200 insertions(+), 689 deletions(-) rename sample/{workflow2_advanced_screenshot.png => workflow_advanced_screenshot.png} (100%) rename sample/{workflow1_basic_screenshot.png => workflow_basic_screenshot.png} (100%) create mode 100644 sample/workflows/Advanced_Animate_without_vid.json rename sample/{workflow2_advanced.json => workflows/Advanced_Inserting_expressions_into_vid.json} (87%) create mode 100644 sample/workflows/Bacis_Extracting_expression_from_photo.json rename sample/{workflow1_basic.json => workflows/Bacis_Liveportrait_with_expression_editing.json} (50%) create mode 100644 sample/workflows/Bacis_Simple_expression_editing.json diff --git a/nodes.py b/nodes.py index b44cd2c..974df17 100644 --- a/nodes.py +++ b/nodes.py @@ -97,6 +97,11 @@ class LP_Engine: pipeline = None detect_model = None mask_img = None + temp_img_idx = 0 + + def get_temp_img_name(self): + self.temp_img_idx += 1 + return "expression_edit_preview" + str(self.temp_img_idx) + ".png" def download_model(_, file_path, model_url): print('AdvancedLivePortrait: Downloading model...') @@ -221,60 +226,76 @@ class LP_Engine: bboxes = self.get_face_bboxes(image_rgb) w, h = get_rgb_size(image_rgb) + print(f"w, h:{w, h}") + + cx = w / 2 + min_diff = w + best_box = None for x1, y1, x2, y2 in bboxes: bbox_w = x2 - x1 - bbox_h = y2 - y1 - if bbox_w < 30: continue + diff = abs(cx - (x1 + bbox_w / 2)) + if diff < min_diff: + best_box = [x1, y1, x2, y2] + print(f"diff, min_diff, best_box:{diff, min_diff, best_box}") + min_diff = diff - crop_w = bbox_w * crop_factor - crop_h = bbox_h * crop_factor + if best_box == None: + print("Failed to detect face!!") + return [0, 0, w, h] - crop_w = max(crop_h, crop_w) - crop_h = crop_w + x1, y1, x2, y2 = best_box - kernel_x = x1 + bbox_w / 2 - kernel_y = y1 + bbox_h / 2 + #for x1, y1, x2, y2 in bboxes: + bbox_w = x2 - x1 + bbox_h = y2 - y1 - new_x1 = kernel_x - crop_w / 2 - new_x2 = kernel_x + crop_w / 2 - new_y1 = kernel_y - crop_h / 2 - new_y2 = kernel_y + crop_h / 2 + crop_w = bbox_w * crop_factor + crop_h = bbox_h * crop_factor - if not sort: - return [int(new_x1), int(new_y1), int(new_x2), int(new_y2)] + crop_w = max(crop_h, crop_w) + crop_h = crop_w + kernel_x = int(x1 + bbox_w / 2) + kernel_y = int(y1 + bbox_h / 2) + + new_x1 = int(kernel_x - crop_w / 2) + new_x2 = int(kernel_x + crop_w / 2) + new_y1 = int(kernel_y - crop_h / 2) + new_y2 = int(kernel_y + crop_h / 2) + + if not sort: + return [int(new_x1), int(new_y1), int(new_x2), int(new_y2)] + + if new_x1 < 0: + new_x2 -= new_x1 + new_x1 = 0 + elif w < new_x2: + new_x1 -= (new_x2 - w) + new_x2 = w if new_x1 < 0: new_x2 -= new_x1 new_x1 = 0 - elif w < new_x2: - new_x1 -= (new_x2 - w) - new_x2 = w - if new_x1 < 0: - new_x2 -= new_x1 - new_x1 = 0 + if new_y1 < 0: + new_y2 -= new_y1 + new_y1 = 0 + elif h < new_y2: + new_y1 -= (new_y2 - h) + new_y2 = h if new_y1 < 0: new_y2 -= new_y1 new_y1 = 0 - elif h < new_y2: - new_y1 -= (new_y2 - h) - new_y2 = h - if new_y1 < 0: - new_y2 -= new_y1 - new_y1 = 0 - if w < new_x2 and h < new_y2: - over_x = new_x2 - w - over_y = new_y2 - h - over_min = min(over_x, over_y) - new_x2 -= over_min - new_y2 -= over_min + if w < new_x2 and h < new_y2: + over_x = new_x2 - w + over_y = new_y2 - h + over_min = min(over_x, over_y) + new_x2 -= over_min + new_y2 -= over_min - return [int(new_x1), int(new_y1), int(new_x2), int(new_y2)] + return [int(new_x1), int(new_y1), int(new_x2), int(new_y2)] - print("Failed to detect face!!") - return [0, 0, w, h] def calc_face_region(self, square, dsize): region = copy.deepcopy(square) @@ -336,26 +357,28 @@ class LP_Engine: if is_changed: face_img = self.expand_img(face_img, crop_region) return face_img - def prepare_source(self, source_image, crop_factor, is_video = False): + def prepare_source(self, source_image, crop_factor, is_video = False, tracking = False): print("Prepare source...") engine = self.get_pipeline() source_image_np = (source_image * 255).byte().numpy() img_rgb = source_image_np[0] - crop_region = self.detect_face(img_rgb, crop_factor) - face_region, is_changed = self.calc_face_region(crop_region, get_rgb_size(img_rgb)) - - s_x = (face_region[2] - face_region[0]) / 512. - s_y = (face_region[3] - face_region[1]) / 512. - crop_trans_m = create_transform_matrix(crop_region[0], crop_region[1], s_x, s_y) - mask_ori = cv2.warpAffine(self.GetMaskImg(), crop_trans_m, get_rgb_size(img_rgb), cv2.INTER_LINEAR) - mask_ori = mask_ori.astype(np.float32) / 255. - - if is_changed: - s = (crop_region[2] - crop_region[0]) / 512. - crop_trans_m = create_transform_matrix(crop_region[0], crop_region[1], s, s) psi_list = [] for img_rgb in source_image_np: + if tracking or len(psi_list) == 0: + crop_region = self.detect_face(img_rgb, crop_factor) + face_region, is_changed = self.calc_face_region(crop_region, get_rgb_size(img_rgb)) + + s_x = (face_region[2] - face_region[0]) / 512. + s_y = (face_region[3] - face_region[1]) / 512. + crop_trans_m = create_transform_matrix(crop_region[0], crop_region[1], s_x, s_y) + mask_ori = cv2.warpAffine(self.GetMaskImg(), crop_trans_m, get_rgb_size(img_rgb), cv2.INTER_LINEAR) + mask_ori = mask_ori.astype(np.float32) / 255. + + if is_changed: + s = (crop_region[2] - crop_region[0]) / 512. + crop_trans_m = create_transform_matrix(crop_region[0], crop_region[1], s, s) + face_img = rgb_crop(img_rgb, face_region) if is_changed: face_img = self.expand_img(face_img, crop_region) i_s = self.prepare_src_image(face_img) @@ -645,6 +668,8 @@ class AdvancedLivePortrait: "crop_factor": ("FLOAT", {"default": crop_factor_default, "min": crop_factor_min, "max": crop_factor_max, "step": 0.1}), "turn_on": ("BOOLEAN", {"default": True}), + "tracking_src_vid": ("BOOLEAN", {"default": False}), + "animate_without_vid": ("BOOLEAN", {"default": False}), "command": ("STRING", {"multiline": True, "default": ""}), }, "optional": { @@ -698,7 +723,7 @@ class AdvancedLivePortrait: return cmd_list, total_length - def run(self, retargeting_eyes, retargeting_mouth, turn_on, command, crop_factor, + def run(self, retargeting_eyes, retargeting_mouth, turn_on, tracking_src_vid, animate_without_vid, command, crop_factor, src_images=None, driving_images=None, motion_link=None): if turn_on == False: return (None,None) src_length = 1 @@ -714,7 +739,7 @@ class AdvancedLivePortrait: self.crop_factor = crop_factor self.src_images = src_images if 1 < src_length: - self.psi_list = g_engine.prepare_source(src_images, crop_factor, True) + self.psi_list = g_engine.prepare_source(src_images, crop_factor, True, tracking_src_vid) else: self.psi_list = [g_engine.prepare_source(src_images, crop_factor)] @@ -730,9 +755,11 @@ class AdvancedLivePortrait: self.driving_values = g_engine.prepare_driving_video(driving_images) driving_length = len(self.driving_values) - #total_length = max(driving_length, cmd_length, src_length) total_length = max(driving_length, src_length) + if animate_without_vid: + total_length = max(total_length, cmd_length) + c_i_es = ExpressionSet() c_o_es = ExpressionSet() d_0_es = None @@ -925,7 +952,7 @@ class ExpressionEditor: out_img = pil2tensor(out) - filename = "fe_edit_preview.png" + filename = g_engine.get_temp_img_name() #"fe_edit_preview.png" folder_paths.get_save_image_path(filename, folder_paths.get_temp_directory()) img = Image.fromarray(crop_out) img.save(os.path.join(folder_paths.get_temp_directory(), filename), compress_level=1) diff --git a/sample/workflow2_advanced_screenshot.png b/sample/workflow_advanced_screenshot.png similarity index 100% rename from sample/workflow2_advanced_screenshot.png rename to sample/workflow_advanced_screenshot.png diff --git a/sample/workflow1_basic_screenshot.png b/sample/workflow_basic_screenshot.png similarity index 100% rename from sample/workflow1_basic_screenshot.png rename to sample/workflow_basic_screenshot.png diff --git a/sample/workflows/Advanced_Animate_without_vid.json b/sample/workflows/Advanced_Animate_without_vid.json new file mode 100644 index 0000000..e0ff6cb --- /dev/null +++ b/sample/workflows/Advanced_Animate_without_vid.json @@ -0,0 +1,359 @@ +{ + "last_node_id": 18, + "last_link_id": 11, + "nodes": [ + { + "id": 3, + "type": "LoadImage", + "pos": [ + 1385, + 278 + ], + "size": [ + 223.20395091844193, + 308.6326788490376 + ], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 11 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "slot_index": 1, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 7, + "type": "AdvancedLivePortrait", + "pos": [ + 1889, + 270 + ], + "size": { + "0": 235.1999969482422, + "1": 523.0364379882812 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "src_images", + "type": "IMAGE", + "link": null + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": 10 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 8 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AdvancedLivePortrait" + }, + "widgets_values": [ + 0, + 0, + 1.7000000000000002, + true, + false, + true, + "1 = 5:5\n0 = 3:5" + ] + }, + { + "id": 10, + "type": "VHS_VideoCombine", + "pos": [ + 2163, + 203 + ], + "size": [ + 261.79998779296875, + 565.7999877929688 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 8 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AdvancedLivePortrait", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AdvancedLivePortrait_00021.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + } + } + } + }, + { + "id": 15, + "type": "ExpressionEditor", + "pos": [ + 1643, + 271 + ], + "size": { + "0": 210, + "1": 690 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 11 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "links": [ + 10 + ], + "slot_index": 1, + "shape": 3 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 25, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7 + ] + }, + { + "id": 16, + "type": "Note", + "pos": [ + 1892, + 831 + ], + "size": [ + 483.8180073265835, + 161.2032413725933 + ], + "flags": {}, + "order": 3, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Command Description\n1 = 5:5\n0 = 3:5\n\n[Motion index] = [Changing frame length] : [Length of frames waiting for next motion]\n\nfor example, first line -> 1 = 5:5\nChange over 5 frames with motion 1, and wait for the next motion for 5 frames" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "Note", + "pos": [ + 1629, + 996 + ], + "size": [ + 227.12395091844178, + 58 + ], + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "motion 1" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 18, + "type": "Note", + "pos": [ + 1373, + 625 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "orginal src : motion 0\n" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 8, + 7, + 0, + 10, + 0, + "IMAGE" + ], + [ + 10, + 15, + 1, + 7, + 1, + "EDITOR_LINK" + ], + [ + 11, + 3, + 0, + 15, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.1000000000000005, + "offset": [ + -841.7371241966999, + 104.19308875020204 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/sample/workflow2_advanced.json b/sample/workflows/Advanced_Inserting_expressions_into_vid.json similarity index 87% rename from sample/workflow2_advanced.json rename to sample/workflows/Advanced_Inserting_expressions_into_vid.json index b1032b2..7810b6f 100644 --- a/sample/workflow2_advanced.json +++ b/sample/workflows/Advanced_Inserting_expressions_into_vid.json @@ -3,96 +3,56 @@ "last_link_id": 8, "nodes": [ { - "id": 8, - "type": "ExpressionEditor", + "id": 3, + "type": "LoadImage", "pos": [ - 1300, - 288 - ], - "size": [ - 210, - 666.122442592595 + 736, + 299 ], + "size": { + "0": 315, + "1": 314 + }, "flags": {}, - "order": 7, + "order": 0, "mode": 0, - "inputs": [ - { - "name": "src_image", - "type": "IMAGE", - "link": null - }, - { - "name": "motion_link", - "type": "EDITOR_LINK", - "link": 5 - }, - { - "name": "sample_image", - "type": "IMAGE", - "link": null - }, - { - "name": "add_exp", - "type": "EXP_DATA", - "link": null - } - ], "outputs": [ { - "name": "image", + "name": "IMAGE", "type": "IMAGE", - "links": null, + "links": [ + 3 + ], + "slot_index": 0, "shape": 3 }, { - "name": "motion_link", - "type": "EDITOR_LINK", - "links": [ - 6 - ], - "shape": 3, - "slot_index": 1 - }, - { - "name": "save_exp", - "type": "EXP_DATA", + "name": "MASK", + "type": "MASK", "links": null, + "slot_index": 1, "shape": 3 } ], "properties": { - "Node name for S&R": "ExpressionEditor" + "Node name for S&R": "LoadImage" }, "widgets_values": [ - 0, - 0, - 0, - 0, - 0, - 0, - 15, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1.7000000000000002 + "source_image.png", + "image" ] }, { "id": 6, "type": "ExpressionEditor", "pos": [ - 1049, - 286 - ], - "size": [ - 210, - 669.4464425925951 + 1085, + 294 ], + "size": { + "0": 210, + "1": 690 + }, "flags": {}, "order": 6, "mode": 0, @@ -131,8 +91,8 @@ "links": [ 5 ], - "shape": 3, - "slot_index": 1 + "slot_index": 1, + "shape": 3 }, { "name": "save_exp", @@ -159,22 +119,159 @@ 0, 1, 1, + "OnlyExpression", 1.7000000000000002 ] }, + { + "id": 7, + "type": "AdvancedLivePortrait", + "pos": [ + 1845, + 304 + ], + "size": { + "0": 235.1999969482422, + "1": 523.0364379882812 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "src_images", + "type": "IMAGE", + "link": null + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": 6 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 8 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AdvancedLivePortrait" + }, + "widgets_values": [ + 0, + 0, + 1.7000000000000002, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", + false, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0" + ] + }, + { + "id": 8, + "type": "ExpressionEditor", + "pos": [ + 1342, + 304 + ], + "size": { + "0": 210, + "1": 690 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": null + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": 5 + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "links": [ + 6 + ], + "slot_index": 1, + "shape": 3 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 0, + 15, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7 + ] + }, { "id": 9, "type": "VHS_LoadVideo", "pos": [ - 1548, - 378 + 1570, + 399 ], "size": [ 235.1999969482422, 491.1999969482422 ], "flags": {}, - "order": 0, + "order": 1, "mode": 0, "inputs": [ { @@ -195,8 +292,8 @@ "links": [ 7 ], - "shape": 3, - "slot_index": 0 + "slot_index": 0, + "shape": 3 }, { "name": "frame_count", @@ -245,197 +342,12 @@ } } }, - { - "id": 3, - "type": "LoadImage", - "pos": [ - 670, - 283 - ], - "size": { - "0": 315, - "1": 314 - }, - "flags": {}, - "order": 1, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 3 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3, - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "source_image.png", - "image" - ] - }, - { - "id": 14, - "type": "Note", - "pos": [ - 667, - 633 - ], - "size": { - "0": 210, - "1": 58 - }, - "flags": {}, - "order": 2, - "mode": 0, - "properties": { - "text": "" - }, - "widgets_values": [ - "orginal src : motion0\n" - ], - "color": "#432", - "bgcolor": "#653" - }, - { - "id": 12, - "type": "Note", - "pos": [ - 1014, - 973 - ], - "size": [ - 210, - 58 - ], - "flags": {}, - "order": 3, - "mode": 0, - "properties": { - "text": "" - }, - "widgets_values": [ - "motion1\n" - ], - "color": "#432", - "bgcolor": "#653" - }, - { - "id": 13, - "type": "Note", - "pos": [ - 1301, - 974 - ], - "size": { - "0": 210, - "1": 58 - }, - "flags": {}, - "order": 4, - "mode": 0, - "properties": { - "text": "" - }, - "widgets_values": [ - "motion2\n" - ], - "color": "#432", - "bgcolor": "#653" - }, - { - "id": 7, - "type": "AdvancedLivePortrait", - "pos": [ - 1824, - 340 - ], - "size": [ - 235.1999969482422, - 523.0364425925951 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "src_images", - "type": "IMAGE", - "link": null - }, - { - "name": "motion_link", - "type": "EDITOR_LINK", - "link": 6 - }, - { - "name": "driving_images", - "type": "IMAGE", - "link": 7 - } - ], - "outputs": [ - { - "name": "images", - "type": "IMAGE", - "links": [ - 8 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "AdvancedLivePortrait" - }, - "widgets_values": [ - 0, - 0, - 1.7000000000000002, - "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", - "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0" - ] - }, - { - "id": 11, - "type": "Note", - "pos": [ - 1823, - 892 - ], - "size": [ - 539.0599993763458, - 184.96444259259488 - ], - "flags": {}, - "order": 5, - "mode": 0, - "properties": { - "text": "" - }, - "widgets_values": [ - "Command Description\n1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0\n\n[Motion index] = [Changing frame length] : [Length of frames waiting for next motion]\n\nfor example, second line -> 2 = 5:10\nChange over 5 frames with motion 2, and wait for the next motion for 10 frames\n\n" - ], - "color": "#432", - "bgcolor": "#653" - }, { "id": 10, "type": "VHS_VideoCombine", "pos": [ - 2122, - 244 + 2117, + 305 ], "size": [ 261.79998779296875, @@ -480,25 +392,119 @@ "widgets_values": { "frame_rate": 15, "loop_count": 0, - "filename_prefix": "AnimateDiff", + "filename_prefix": "AdvancedLivePortrait", "format": "video/h264-mp4", "pix_fmt": "yuv420p", "crf": 19, "save_metadata": true, "pingpong": false, - "save_output": false, + "save_output": true, "videopreview": { "hidden": false, "paused": false, "params": { - "filename": "AnimateDiff_00003.mp4", + "filename": "AdvancedLivePortrait_00023.mp4", "subfolder": "", - "type": "temp", + "type": "output", "format": "video/h264-mp4", "frame_rate": 15 } } } + }, + { + "id": 11, + "type": "Note", + "pos": [ + 1834, + 935 + ], + "size": { + "0": 539.0599975585938, + "1": 184.96444702148438 + }, + "flags": {}, + "order": 5, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Command Description\n1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0\n\n[Motion index] = [Changing frame length] : [Length of frames waiting for next motion]\n\nfor example, second line -> 2 = 5:10\nChange over 5 frames with motion 2, and wait for the next motion for 10 frames\n\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 12, + "type": "Note", + "pos": [ + 1089, + 1038 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "motion1\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 13, + "type": "Note", + "pos": [ + 1341, + 1041 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": { + "collapsed": false + }, + "order": 3, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "motion2\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 14, + "type": "Note", + "pos": [ + 776, + 684 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 4, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "orginal src : motion0\n" + ], + "color": "#432", + "bgcolor": "#653" } ], "links": [ @@ -547,10 +553,10 @@ "config": {}, "extra": { "ds": { - "scale": 1, + "scale": 1.1000000000000005, "offset": [ - -410.5455448308912, - -80.52844259259504 + -678.8740830330757, + -35.741089089667945 ] } }, diff --git a/sample/workflows/Bacis_Extracting_expression_from_photo.json b/sample/workflows/Bacis_Extracting_expression_from_photo.json new file mode 100644 index 0000000..e22f5c4 --- /dev/null +++ b/sample/workflows/Bacis_Extracting_expression_from_photo.json @@ -0,0 +1,241 @@ +{ + "last_node_id": 33, + "last_link_id": 39, + "nodes": [ + { + "id": 19, + "type": "LoadImage", + "pos": [ + 1022, + 702 + ], + "size": { + "0": 220.3717041015625, + "1": 314 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 25 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "exp_image.png", + "image" + ] + }, + { + "id": 20, + "type": "LoadImage", + "pos": [ + 1027.712857031252, + 322.70146904296865 + ], + "size": { + "0": 210.9286651611328, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 26 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 21, + "type": "ExpressionEditor", + "pos": [ + 1305, + 338 + ], + "size": [ + 252.62867736816406, + 690 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 26 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": 25 + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 39 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "links": null, + "shape": 3 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7 + ] + }, + { + "id": 33, + "type": "SaveImage", + "pos": [ + 1617, + 390 + ], + "size": [ + 263.6363636363635, + 278.18181818181813 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 39 + } + ], + "properties": {}, + "widgets_values": [ + "ExpressionEditing" + ] + } + ], + "links": [ + [ + 25, + 19, + 0, + 21, + 2, + "IMAGE" + ], + [ + 26, + 20, + 0, + 21, + 0, + "IMAGE" + ], + [ + 39, + 21, + 0, + 33, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Extracting facial expressions from photos", + "bounding": [ + 970, + 233, + 939, + 823 + ], + "color": "#a1309b", + "font_size": 30 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 1, + "offset": [ + -256.9678555487616, + -36.006961063159906 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/sample/workflow1_basic.json b/sample/workflows/Bacis_Liveportrait_with_expression_editing.json similarity index 50% rename from sample/workflow1_basic.json rename to sample/workflows/Bacis_Liveportrait_with_expression_editing.json index 277a70b..f3eaebf 100644 --- a/sample/workflow1_basic.json +++ b/sample/workflows/Bacis_Liveportrait_with_expression_editing.json @@ -3,11 +3,11 @@ "last_link_id": 38, "nodes": [ { - "id": 15, + "id": 26, "type": "LoadImage", "pos": [ - 654.9384793707275, - 32.435132185058634 + 574.0879504882818, + 354.13510488281264 ], "size": { "0": 210, @@ -16,84 +16,6 @@ "flags": {}, "order": 0, "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 19 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "source_image.png", - "image" - ] - }, - { - "id": 19, - "type": "LoadImage", - "pos": [ - 1308.9368865429728, - 310.8936660083008 - ], - "size": { - "0": 220.3717041015625, - "1": 314 - }, - "flags": {}, - "order": 1, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 25 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "exp_image (1).png", - "image" - ] - }, - { - "id": 26, - "type": "LoadImage", - "pos": [ - 620.239467871092, - 782.9509859374981 - ], - "size": { - "0": 210, - "1": 314 - }, - "flags": {}, - "order": 2, - "mode": 0, "outputs": [ { "name": "IMAGE", @@ -101,8 +23,8 @@ "links": [ 30 ], - "shape": 3, - "slot_index": 0 + "slot_index": 0, + "shape": 3 }, { "name": "MASK", @@ -123,15 +45,15 @@ "id": 27, "type": "VHS_LoadVideo", "pos": [ - 1127.239467871095, - 907.9509859374981 + 1113, + 443 ], "size": [ 235.1999969482422, 491.1999969482422 ], "flags": {}, - "order": 3, + "order": 1, "mode": 0, "inputs": [ { @@ -152,8 +74,8 @@ "links": [ 31 ], - "shape": 3, - "slot_index": 0 + "slot_index": 0, + "shape": 3 }, { "name": "frame_count", @@ -203,63 +125,24 @@ } }, { - "id": 20, - "type": "LoadImage", + "id": 28, + "type": "AdvancedLivePortrait", "pos": [ - 1318.2737525390667, - -22.338175012206996 + 1386, + 353 + ], + "size": [ + 235.1999969482422, + 674.0390319824219 ], - "size": { - "0": 210.9286651611328, - "1": 314 - }, "flags": {}, - "order": 4, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 26 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "source_image.png", - "image" - ] - }, - { - "id": 14, - "type": "ExpressionEditor", - "pos": [ - 931.634000524902, - -3.1228280700683335 - ], - "size": { - "0": 210, - "1": 666 - }, - "flags": {}, - "order": 5, + "order": 3, "mode": 0, "inputs": [ { - "name": "src_image", + "name": "src_images", "type": "IMAGE", - "link": 19 + "link": 38 }, { "name": "motion_link", @@ -267,71 +150,48 @@ "link": null }, { - "name": "sample_image", + "name": "driving_images", "type": "IMAGE", - "link": null - }, - { - "name": "add_exp", - "type": "EXP_DATA", - "link": null + "link": 31 } ], "outputs": [ { - "name": "image", + "name": "images", "type": "IMAGE", - "links": null, - "shape": 3, - "slot_index": 0 - }, - { - "name": "motion_link", - "type": "EDITOR_LINK", - "links": null, - "shape": 3 - }, - { - "name": "save_exp", - "type": "EXP_DATA", - "links": null, + "links": [ + 35 + ], + "slot_index": 0, "shape": 3 } ], "properties": { - "Node name for S&R": "ExpressionEditor" + "Node name for S&R": "AdvancedLivePortrait" }, "widgets_values": [ 0, 0, - 0, - 0, - 0, - 23.5, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1.7000000000000002 + 1.5, + true, + "", + false, + "" ] }, { "id": 29, "type": "VHS_VideoCombine", "pos": [ - 1706.7501000000009, - 796.943396582031 + 1664, + 357 ], "size": [ 272.9300842285156, 576.9300842285156 ], "flags": {}, - "order": 9, + "order": 4, "mode": 0, "inputs": [ { @@ -375,12 +235,12 @@ "crf": 19, "save_metadata": true, "pingpong": false, - "save_output": false, + "save_output": true, "videopreview": { "hidden": false, "paused": false, "params": { - "filename": "AdvancedLivePortrait_00009.mp4", + "filename": "AdvancedLivePortrait_00001.mp4", "subfolder": "", "type": "temp", "format": "video/h264-mp4", @@ -389,150 +249,19 @@ } } }, - { - "id": 28, - "type": "AdvancedLivePortrait", - "pos": [ - 1401.2394678711005, - 798.9509859374981 - ], - "size": { - "0": 235.1999969482422, - "1": 628 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "src_images", - "type": "IMAGE", - "link": 38 - }, - { - "name": "motion_link", - "type": "EDITOR_LINK", - "link": null - }, - { - "name": "driving_images", - "type": "IMAGE", - "link": 31 - } - ], - "outputs": [ - { - "name": "images", - "type": "IMAGE", - "links": [ - 35 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "AdvancedLivePortrait" - }, - "widgets_values": [ - 0, - 0, - 1.7000000000000002, - "", - "" - ] - }, - { - "id": 21, - "type": "ExpressionEditor", - "pos": [ - 1601.2737525390667, - -32.33817501220699 - ], - "size": { - "0": 210, - "1": 666 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "src_image", - "type": "IMAGE", - "link": 26 - }, - { - "name": "motion_link", - "type": "EDITOR_LINK", - "link": null - }, - { - "name": "sample_image", - "type": "IMAGE", - "link": 25 - }, - { - "name": "add_exp", - "type": "EXP_DATA", - "link": null - } - ], - "outputs": [ - { - "name": "image", - "type": "IMAGE", - "links": null, - "shape": 3, - "slot_index": 0 - }, - { - "name": "motion_link", - "type": "EDITOR_LINK", - "links": null, - "shape": 3 - }, - { - "name": "save_exp", - "type": "EXP_DATA", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "ExpressionEditor" - }, - "widgets_values": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1.7000000000000002 - ] - }, { "id": 30, "type": "ExpressionEditor", "pos": [ - 864.2394678710932, - 788.9509859374982 + 837.0879504882818, + 349.1351048828126 ], "size": { - "0": 210, - "1": 666 + "0": 256.8715515136719, + "1": 690 }, "flags": {}, - "order": 6, + "order": 2, "mode": 0, "inputs": [ { @@ -563,8 +292,8 @@ "links": [ 38 ], - "shape": 3, - "slot_index": 0 + "slot_index": 0, + "shape": 3 }, { "name": "motion_link", @@ -597,35 +326,12 @@ 1.3, 1, 1, + "OnlyExpression", 1.7000000000000002 ] } ], "links": [ - [ - 19, - 15, - 0, - 14, - 0, - "IMAGE" - ], - [ - 25, - 19, - 0, - 21, - 2, - "IMAGE" - ], - [ - 26, - 20, - 0, - 21, - 0, - "IMAGE" - ], [ 30, 26, @@ -660,35 +366,13 @@ ] ], "groups": [ - { - "title": "Simple facial expression editing", - "bounding": [ - 608, - -102, - 571, - 768 - ], - "color": "#88A", - "font_size": 30 - }, - { - "title": "Extracting facial expressions from photos", - "bounding": [ - 1249, - -100, - 605, - 753 - ], - "color": "#a1309b", - "font_size": 30 - }, { "title": "Live Portrait with facial expression editing", "bounding": [ - 598, - 701, - 1412, - 767 + 555, + 266, + 1427, + 789 ], "color": "#3f789e", "font_size": 30 @@ -697,10 +381,10 @@ "config": {}, "extra": { "ds": { - "scale": 0.9090909090909091, + "scale": 0.8264462809917354, "offset": [ - 213.02298586006117, - -49.46441845818874 + 185.1062862025528, + 195.77375741392547 ] } }, diff --git a/sample/workflows/Bacis_Simple_expression_editing.json b/sample/workflows/Bacis_Simple_expression_editing.json new file mode 100644 index 0000000..408833d --- /dev/null +++ b/sample/workflows/Bacis_Simple_expression_editing.json @@ -0,0 +1,194 @@ +{ + "last_node_id": 33, + "last_link_id": 40, + "nodes": [ + { + "id": 14, + "type": "ExpressionEditor", + "pos": [ + 1044.0000000000002, + 362.78992797851555 + ], + "size": [ + 260.81048583984375, + 690 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 19 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 40 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "links": null, + "shape": 3 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 23.5, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7000000000000002 + ] + }, + { + "id": 15, + "type": "LoadImage", + "pos": [ + 776, + 366.78992797851555 + ], + "size": { + "0": 210, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 19 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 33, + "type": "SaveImage", + "pos": [ + 1351.0000000000005, + 373.78992797851555 + ], + "size": [ + 257.83631037110194, + 270 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 40 + } + ], + "properties": {}, + "widgets_values": [ + "ExpressionEditing" + ] + } + ], + "links": [ + [ + 19, + 15, + 0, + 14, + 0, + "IMAGE" + ], + [ + 40, + 14, + 0, + 33, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Simple facial expression editing", + "bounding": [ + 745, + 261, + 896, + 835 + ], + "color": "#88A", + "font_size": 30 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015777, + "offset": [ + 561.7325573974937, + 267.43204960931945 + ] + } + }, + "version": 0.4 +} \ No newline at end of file