diff --git a/examples/liveportrait_image_example_01.json b/examples/liveportrait_image_example_01.json new file mode 100644 index 0000000..71c3815 --- /dev/null +++ b/examples/liveportrait_image_example_01.json @@ -0,0 +1,1424 @@ +{ + "last_node_id": 200, + "last_link_id": 478, + "nodes": [ + { + "id": 129, + "type": "LivePortraitLoadCropper", + "pos": [ + -1253, + -757 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "cropper", + "type": "LPCROPPER", + "links": [], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LivePortraitLoadCropper" + }, + "widgets_values": [ + "CPU", + true + ] + }, + { + "id": 134, + "type": "GetImageSizeAndCount", + "pos": [ + -200, + 360 + ], + "size": { + "0": 210, + "1": 86 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 471 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [], + "shape": 3, + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "height", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "count", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 188, + "type": "Note", + "pos": [ + 30, + 310 + ], + "size": { + "0": 229.1644287109375, + "1": 95.92726135253906 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "If you want to crop your driving video, use this as driving_images instead" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "GetImageSizeAndCount", + "pos": [ + -522, + -572 + ], + "size": { + "0": 210, + "1": 86 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 434 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 445, + 455 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "512 width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "512 height", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "1 count", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 8, + "type": "VHS_LoadVideo", + "pos": [ + -1253, + 28 + ], + "size": [ + 340.1555480957031, + 572.1555480957031 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "frame_load_cap", + "type": "INT", + "link": 302, + "widget": { + "name": "frame_load_cap" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 436 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "d6.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 128, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 128, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "d6.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 165, + "type": "ImageResizeKJ", + "pos": [ + -873, + -573 + ], + "size": { + "0": 315, + "1": 242 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 466 + }, + { + "name": "get_image_size", + "type": "IMAGE", + "link": null + }, + { + "name": "width_input", + "type": "INT", + "link": null, + "widget": { + "name": "width_input" + } + }, + { + "name": "height_input", + "type": "INT", + "link": null, + "widget": { + "name": "height_input" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 434 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "height", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ImageResizeKJ" + }, + "widgets_values": [ + 512, + 512, + "lanczos", + true, + 2, + 0, + 0 + ] + }, + { + "id": 197, + "type": "LivePortraitCropper", + "pos": [ + -640, + 360 + ], + "size": { + "0": 330, + "1": 242 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "LIVEPORTRAITPIPE", + "link": 472, + "slot_index": 0 + }, + { + "name": "cropper", + "type": "LPCROPPER", + "link": 476 + }, + { + "name": "source_image", + "type": "IMAGE", + "link": 469 + } + ], + "outputs": [ + { + "name": "cropped_image", + "type": "IMAGE", + "links": [ + 471 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "crop_info", + "type": "CROPINFO", + "links": [ + 470 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LivePortraitCropper" + }, + "widgets_values": [ + 512, + 2.3, + 0, + -0.125, + 0, + "large-small", + true + ] + }, + { + "id": 198, + "type": "LivePortraitLoadMediaPipeCropper", + "pos": [ + -1256, + -893 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "cropper", + "type": "LPCROPPER", + "links": [ + 474 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LivePortraitLoadMediaPipeCropper" + }, + "widgets_values": [ + "CPU", + true + ] + }, + { + "id": 1, + "type": "DownloadAndLoadLivePortraitModels", + "pos": [ + -934, + -1067 + ], + "size": { + "0": 302.43463134765625, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "live_portrait_pipe", + "type": "LIVEPORTRAITPIPE", + "links": [ + 446, + 448, + 472 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DownloadAndLoadLivePortraitModels" + }, + "widgets_values": [ + "fp16" + ] + }, + { + "id": 200, + "type": "Note", + "pos": [ + -1542, + -877 + ], + "size": [ + 265.68303695794293, + 168.53929752859244 + ], + "flags": {}, + "order": 4, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Choose your face detector here, MediaPipe doesn't run on GPU and isn't as good at detecting, but is faster on CPU and has Apache 2.0 license.\n\nInsightface can be run on GPU and is better at detecting, but it's model license is NON-COMMERCIAL" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 199, + "type": "Reroute", + "pos": [ + -855, + -812 + ], + "size": [ + 115.6, + 26 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 474 + } + ], + "outputs": [ + { + "name": "LPCROPPER", + "type": "LPCROPPER", + "links": [ + 475, + 476 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": true, + "horizontal": false + } + }, + { + "id": 133, + "type": "PrimitiveNode", + "pos": [ + -1533, + -164 + ], + "size": { + "0": 210, + "1": 82 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 302 + ], + "widget": { + "name": "frame_load_cap" + }, + "slot_index": 0 + } + ], + "title": "frame_load_cap", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 128, + "fixed" + ] + }, + { + "id": 187, + "type": "Note", + "pos": [ + 80, + 140 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 6, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Connect this if you want to use the eye/lip retargeting mode" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 182, + "type": "GetImageSizeAndCount", + "pos": [ + -850, + -195 + ], + "size": { + "0": 210, + "1": 86 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 436 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 451, + 469 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "480 width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "480 height", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "128 count", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 180, + "type": "LivePortraitRetargeting", + "pos": [ + -390, + 120 + ], + "size": { + "0": 430.8000183105469, + "1": 130 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "driving_crop_info", + "type": "CROPINFO", + "link": 470, + "slot_index": 0 + } + ], + "outputs": [ + { + "name": "retargeting_info", + "type": "RETARGETINGINFO", + "links": [], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LivePortraitRetargeting" + }, + "widgets_values": [ + true, + 1, + true, + 1 + ] + }, + { + "id": 190, + "type": "LivePortraitProcess", + "pos": [ + 166, + -422 + ], + "size": { + "0": 430.8000183105469, + "1": 282 + }, + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "LIVEPORTRAITPIPE", + "link": 448 + }, + { + "name": "crop_info", + "type": "CROPINFO", + "link": 449 + }, + { + "name": "source_image", + "type": "IMAGE", + "link": 456 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": 451 + }, + { + "name": "opt_retargeting_info", + "type": "RETARGETINGINFO", + "link": null + } + ], + "outputs": [ + { + "name": "cropped_image", + "type": "IMAGE", + "links": [ + 454 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "output", + "type": "LP_OUT", + "links": [ + 452 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LivePortraitProcess" + }, + "widgets_values": [ + false, + 0.03, + true, + 1, + "constant", + "relative", + 0.000003 + ] + }, + { + "id": 83, + "type": "CreateShapeMask", + "pos": [ + 407, + -876 + ], + "size": { + "0": 315, + "1": 270 + }, + "flags": {}, + "order": 7, + "mode": 0, + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 192 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "mask_inverted", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CreateShapeMask" + }, + "widgets_values": [ + "square", + 1, + 256, + 256, + 0, + 512, + 512, + 480, + 480 + ] + }, + { + "id": 81, + "type": "GrowMaskWithBlur", + "pos": [ + 751, + -873 + ], + "size": { + "0": 315, + "1": 246 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 192, + "slot_index": 0 + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 187, + 463 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "mask_inverted", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GrowMaskWithBlur" + }, + "widgets_values": [ + 0, + 0, + true, + false, + 13.9, + 1, + 1, + false + ] + }, + { + "id": 82, + "type": "MaskPreview+", + "pos": [ + 1091, + -873 + ], + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 187 + } + ], + "properties": { + "Node name for S&R": "MaskPreview+" + } + }, + { + "id": 181, + "type": "GetImageSizeAndCount", + "pos": [ + 1083, + -443 + ], + "size": { + "0": 210, + "1": 86 + }, + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 458 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 478 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "512 width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "512 height", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "128 count", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 168, + "type": "VHS_VideoCombine", + "pos": [ + 630, + -302 + ], + "size": [ + 669.8613793918787, + 973.8613793918787 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 478 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "LivePortrait", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "LivePortrait_00005.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 196, + "type": "LoadImage", + "pos": [ + -1267, + -582 + ], + "size": [ + 330.4995504373579, + 384.24166509694487 + ], + "flags": {}, + "order": 8, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 466 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ComfyUI_temp_gjesb_00023_.png", + "image" + ] + }, + { + "id": 192, + "type": "Reroute", + "pos": [ + 73, + -519 + ], + "size": [ + 82, + 26 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 455 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 456, + 457 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": true, + "horizontal": false + } + }, + { + "id": 191, + "type": "LivePortraitComposite", + "pos": [ + 673, + -507 + ], + "size": { + "0": 355.20001220703125, + "1": 86 + }, + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "source_image", + "type": "IMAGE", + "link": 457 + }, + { + "name": "cropped_image", + "type": "IMAGE", + "link": 454 + }, + { + "name": "liveportrait_out", + "type": "LP_OUT", + "link": 452 + }, + { + "name": "mask", + "type": "MASK", + "link": 463 + } + ], + "outputs": [ + { + "name": "full_images", + "type": "IMAGE", + "links": [ + 458 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "mask", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LivePortraitComposite" + } + }, + { + "id": 189, + "type": "LivePortraitCropper", + "pos": [ + -203, + -830 + ], + "size": { + "0": 330, + "1": 242 + }, + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "LIVEPORTRAITPIPE", + "link": 446, + "slot_index": 0 + }, + { + "name": "cropper", + "type": "LPCROPPER", + "link": 475, + "slot_index": 1 + }, + { + "name": "source_image", + "type": "IMAGE", + "link": 445 + } + ], + "outputs": [ + { + "name": "cropped_image", + "type": "IMAGE", + "links": null, + "shape": 3, + "slot_index": 0 + }, + { + "name": "crop_info", + "type": "CROPINFO", + "links": [ + 449 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LivePortraitCropper" + }, + "widgets_values": [ + 512, + 2.3, + 0, + -0.125, + 0, + "large-small", + true + ] + } + ], + "links": [ + [ + 187, + 81, + 0, + 82, + 0, + "MASK" + ], + [ + 192, + 83, + 0, + 81, + 0, + "MASK" + ], + [ + 302, + 133, + 0, + 8, + 2, + "INT" + ], + [ + 434, + 165, + 0, + 78, + 0, + "IMAGE" + ], + [ + 436, + 8, + 0, + 182, + 0, + "IMAGE" + ], + [ + 445, + 78, + 0, + 189, + 2, + "IMAGE" + ], + [ + 446, + 1, + 0, + 189, + 0, + "LIVEPORTRAITPIPE" + ], + [ + 448, + 1, + 0, + 190, + 0, + "LIVEPORTRAITPIPE" + ], + [ + 449, + 189, + 1, + 190, + 1, + "CROPINFO" + ], + [ + 451, + 182, + 0, + 190, + 3, + "IMAGE" + ], + [ + 452, + 190, + 1, + 191, + 2, + "LP_OUT" + ], + [ + 454, + 190, + 0, + 191, + 1, + "IMAGE" + ], + [ + 455, + 78, + 0, + 192, + 0, + "*" + ], + [ + 456, + 192, + 0, + 190, + 2, + "IMAGE" + ], + [ + 457, + 192, + 0, + 191, + 0, + "IMAGE" + ], + [ + 458, + 191, + 0, + 181, + 0, + "IMAGE" + ], + [ + 463, + 81, + 0, + 191, + 3, + "MASK" + ], + [ + 466, + 196, + 0, + 165, + 0, + "IMAGE" + ], + [ + 469, + 182, + 0, + 197, + 2, + "IMAGE" + ], + [ + 470, + 197, + 1, + 180, + 0, + "CROPINFO" + ], + [ + 471, + 197, + 0, + 134, + 0, + "IMAGE" + ], + [ + 472, + 1, + 0, + 197, + 0, + "LIVEPORTRAITPIPE" + ], + [ + 474, + 198, + 0, + 199, + 0, + "*" + ], + [ + 475, + 199, + 0, + 189, + 1, + "LPCROPPER" + ], + [ + 476, + 199, + 0, + 197, + 1, + "LPCROPPER" + ], + [ + 478, + 181, + 0, + 168, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.6209213230591553, + "offset": { + "0": 1699.882189562642, + "1": 1132.3910371015413 + } + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/examples/liveportrait_example_01.json b/examples/liveportrait_realtime_example_01.json similarity index 52% rename from examples/liveportrait_example_01.json rename to examples/liveportrait_realtime_example_01.json index 40e3239..827aa48 100644 --- a/examples/liveportrait_example_01.json +++ b/examples/liveportrait_realtime_example_01.json @@ -1,70 +1,96 @@ { - "last_node_id": 31, - "last_link_id": 68, + "last_node_id": 208, + "last_link_id": 480, "nodes": [ { - "id": 4, - "type": "LoadImage", + "id": 204, + "type": "LivePortraitLoadMediaPipeCropper", "pos": [ - 138, - 323 + -1059, + -767 ], "size": { - "0": 272.85791015625, - "1": 331.60894775390625 + "0": 315, + "1": 82 }, "flags": {}, "order": 0, "mode": 0, "outputs": [ { - "name": "IMAGE", - "type": "IMAGE", + "name": "cropper", + "type": "LPCROPPER", "links": [ - 59 + 478 ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, "shape": 3 } ], "properties": { - "Node name for S&R": "LoadImage" + "Node name for S&R": "LivePortraitLoadMediaPipeCropper" }, "widgets_values": [ - "oldman.jpg", - "image" + "CPU", + true ] }, { - "id": 19, + "id": 1, + "type": "DownloadAndLoadLivePortraitModels", + "pos": [ + -1046, + -904 + ], + "size": { + "0": 302.43463134765625, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "live_portrait_pipe", + "type": "LIVEPORTRAITPIPE", + "links": [ + 446, + 448 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DownloadAndLoadLivePortraitModels" + }, + "widgets_values": [ + "fp16" + ] + }, + { + "id": 165, "type": "ImageResizeKJ", "pos": [ - 507, - 675 + -715, + -617 ], "size": { "0": 315, "1": 242 }, "flags": {}, - "order": 4, + "order": 6, "mode": 0, "inputs": [ { "name": "image", "type": "IMAGE", - "link": 30 + "link": 466 }, { "name": "get_image_size", "type": "IMAGE", - "link": 68 + "link": null }, { "name": "width_input", @@ -88,7 +114,7 @@ "name": "IMAGE", "type": "IMAGE", "links": [ - 32 + 434 ], "shape": 3, "slot_index": 0 @@ -112,290 +138,282 @@ "widgets_values": [ 512, 512, - "nearest-exact", - false, + "lanczos", + true, 2, 0, 0 ] }, { - "id": 18, - "type": "ImageConcatMulti", + "id": 78, + "type": "GetImageSizeAndCount", "pos": [ - 860, - 679 + -364, + -619 ], "size": { "0": 210, - "1": 150 + "1": 86 }, "flags": {}, - "order": 5, + "order": 7, "mode": 0, "inputs": [ { - "name": "image_1", + "name": "image", "type": "IMAGE", - "link": 32 - }, - { - "name": "image_2", - "type": "IMAGE", - "link": 67 + "link": 434 } ], "outputs": [ { - "name": "images", + "name": "image", "type": "IMAGE", "links": [ - 64 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": {}, - "widgets_values": [ - 2, - "right", - false, - null - ] - }, - { - "id": 23, - "type": "VHS_VideoCombine", - "pos": [ - 1098, - 240 - ], - "size": [ - 1253.234130859375, - 940.6170654296875 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 64 - }, - { - "name": "audio", - "type": "VHS_AUDIO", - "link": null - }, - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null - }, - { - "name": "vae", - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": { - "frame_rate": 24, - "loop_count": 0, - "filename_prefix": "LivePortrait", - "format": "video/h264-mp4", - "pix_fmt": "yuv420p", - "crf": 19, - "save_metadata": true, - "pingpong": false, - "save_output": false, - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "filename": "LivePortrait_00001.mp4", - "subfolder": "", - "type": "temp", - "format": "video/h264-mp4", - "frame_rate": 24 - } - } - } - }, - { - "id": 1, - "type": "DownloadAndLoadLivePortraitModels", - "pos": [ - 142, - 205 - ], - "size": { - "0": 252, - "1": 58 - }, - "flags": {}, - "order": 1, - "mode": 0, - "outputs": [ - { - "name": "live_portrait_pipe", - "type": "LIVEPORTRAITPIPE", - "links": [ - 58 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DownloadAndLoadLivePortraitModels" - }, - "widgets_values": [ - "auto" - ] - }, - { - "id": 8, - "type": "VHS_LoadVideo", - "pos": [ - 161, - 714 - ], - "size": [ - 235.1999969482422, - 491.1999969482422 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null - }, - { - "name": "vae", - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 30, - 60 + 445, + 475 ], "shape": 3, "slot_index": 0 }, { - "name": "frame_count", + "name": "512 width", "type": "INT", "links": null, "shape": 3 }, { - "name": "audio", - "type": "VHS_AUDIO", + "name": "384 height", + "type": "INT", "links": null, "shape": 3 }, { - "name": "video_info", - "type": "VHS_VIDEOINFO", + "name": "1 count", + "type": "INT", "links": null, "shape": 3 } ], "properties": { - "Node name for S&R": "VHS_LoadVideo" - }, - "widgets_values": { - "video": "d3.mp4", - "force_rate": 0, - "force_size": "Disabled", - "custom_width": 512, - "custom_height": 512, - "frame_load_cap": 0, - "skip_first_frames": 0, - "select_every_nth": 1, - "choose video to upload": "image", - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "frame_load_cap": 0, - "skip_first_frames": 0, - "force_rate": 0, - "filename": "d3.mp4", - "type": "input", - "format": "video/mp4", - "select_every_nth": 1 - } - } + "Node name for S&R": "GetImageSizeAndCount" } }, { - "id": 30, - "type": "LivePortraitProcess", + "id": 207, + "type": "Note", "pos": [ - 500, - 249 + -850, + -202 + ], + "size": [ + 230.3331222759989, + 105.34622315856666 + ], + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Example live inputs, direct webcam capture using cv2 or screencapture using mss. Both are about same speed." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 206, + "type": "Screencap_mss", + "pos": [ + -568, + -34 ], "size": { - "0": 367.79998779296875, - "1": 362 + "0": 315, + "1": 178 }, "flags": {}, "order": 3, "mode": 0, + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "Screencap_mss" + }, + "widgets_values": [ + 0, + 0, + 512, + 512, + 1, + 0.1 + ] + }, + { + "id": 198, + "type": "PreviewImage", + "pos": [ + 413, + -826 + ], + "size": { + "0": 521.2196044921875, + "1": 566.1187133789062 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 470 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 196, + "type": "LoadImage", + "pos": [ + -1058, + -623 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 466 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "Mona-Lisa-oil-wood-panel-Leonardo-da.webp", + "image" + ] + }, + { + "id": 205, + "type": "WebcamCaptureCV2", + "pos": [ + -577, + -283 + ], + "size": { + "0": 315, + "1": 178 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 479 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WebcamCaptureCV2" + }, + "widgets_values": [ + 0, + 0, + 512, + 512, + 0, + false + ] + }, + { + "id": 190, + "type": "LivePortraitProcess", + "pos": [ + -79, + -552 + ], + "size": { + "0": 430.8000183105469, + "1": 282 + }, + "flags": {}, + "order": 9, + "mode": 0, "inputs": [ { "name": "pipeline", "type": "LIVEPORTRAITPIPE", - "link": 58 + "link": 448 + }, + { + "name": "crop_info", + "type": "CROPINFO", + "link": 449 }, { "name": "source_image", "type": "IMAGE", - "link": 59 + "link": 475 }, { "name": "driving_images", "type": "IMAGE", - "link": 60 + "link": 479, + "slot_index": 3 + }, + { + "name": "opt_retargeting_info", + "type": "RETARGETINGINFO", + "link": null } ], "outputs": [ { - "name": "cropped_images", + "name": "cropped_image", "type": "IMAGE", - "links": [], + "links": [ + 470 + ], "shape": 3, "slot_index": 0 }, { - "name": "full_images", - "type": "IMAGE", - "links": [ - 67, - 68 - ], + "name": "output", + "type": "LP_OUT", + "links": [], "shape": 3, "slot_index": 1 } @@ -403,85 +421,160 @@ "properties": { "Node name for S&R": "LivePortraitProcess" }, + "widgets_values": [ + false, + 0.03, + true, + 1, + "constant", + "single_frame", + 0.000003 + ] + }, + { + "id": 189, + "type": "LivePortraitCropper", + "pos": [ + -73, + -876 + ], + "size": { + "0": 330, + "1": 242 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "LIVEPORTRAITPIPE", + "link": 446, + "slot_index": 0 + }, + { + "name": "cropper", + "type": "LPCROPPER", + "link": 478, + "slot_index": 1 + }, + { + "name": "source_image", + "type": "IMAGE", + "link": 445 + } + ], + "outputs": [ + { + "name": "cropped_image", + "type": "IMAGE", + "links": null, + "shape": 3, + "slot_index": 0 + }, + { + "name": "crop_info", + "type": "CROPINFO", + "links": [ + 449 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LivePortraitCropper" + }, "widgets_values": [ 512, - 2.3, + 2.34, + 0.099, + 0.148, 0, - -0.11, - true, - false, - 1, - false, - 1, - true, - true, - "CPU" + "large-small", + false ] } ], "links": [ [ - 30, - 8, + 434, + 165, 0, - 19, + 78, 0, "IMAGE" ], [ - 32, - 19, + 445, + 78, 0, - 18, - 0, - "IMAGE" - ], - [ - 58, - 1, - 0, - 30, - 0, - "LIVEPORTRAITPIPE" - ], - [ - 59, - 4, - 0, - 30, - 1, - "IMAGE" - ], - [ - 60, - 8, - 0, - 30, + 189, 2, "IMAGE" ], [ - 64, - 18, + 446, + 1, 0, - 23, + 189, + 0, + "LIVEPORTRAITPIPE" + ], + [ + 448, + 1, + 0, + 190, + 0, + "LIVEPORTRAITPIPE" + ], + [ + 449, + 189, + 1, + 190, + 1, + "CROPINFO" + ], + [ + 466, + 196, + 0, + 165, 0, "IMAGE" ], [ - 67, - 30, - 1, - 18, - 1, + 470, + 190, + 0, + 198, + 0, "IMAGE" ], [ - 68, - 30, - 1, - 19, + 475, + 78, + 0, + 190, + 2, + "IMAGE" + ], + [ + 478, + 204, + 0, + 189, 1, + "LPCROPPER" + ], + [ + 479, + 205, + 0, + 190, + 3, "IMAGE" ] ], @@ -489,10 +582,10 @@ "config": {}, "extra": { "ds": { - "scale": 0.8264462809917354, + "scale": 0.7513148009015781, "offset": { - "0": 173.40487670898438, - "1": -0.9636010527610779 + "0": 1468.4081568988054, + "1": 1224.8414164288351 } } }, diff --git a/examples/liveportrait_video_example_02.json b/examples/liveportrait_video_example_02.json new file mode 100644 index 0000000..50c94e1 --- /dev/null +++ b/examples/liveportrait_video_example_02.json @@ -0,0 +1,1662 @@ +{ + "last_node_id": 198, + "last_link_id": 477, + "nodes": [ + { + "id": 165, + "type": "ImageResizeKJ", + "pos": [ + -873, + -573 + ], + "size": { + "0": 315, + "1": 242 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 433 + }, + { + "name": "get_image_size", + "type": "IMAGE", + "link": null + }, + { + "name": "width_input", + "type": "INT", + "link": null, + "widget": { + "name": "width_input" + } + }, + { + "name": "height_input", + "type": "INT", + "link": null, + "widget": { + "name": "height_input" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 434 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "height", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ImageResizeKJ" + }, + "widgets_values": [ + 1024, + 1024, + "bilinear", + true, + 2, + 0, + 0 + ] + }, + { + "id": 82, + "type": "MaskPreview+", + "pos": [ + 1410, + -810 + ], + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 187 + } + ], + "properties": { + "Node name for S&R": "MaskPreview+" + } + }, + { + "id": 83, + "type": "CreateShapeMask", + "pos": [ + 690, + -800 + ], + "size": { + "0": 315, + "1": 270 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 192 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "mask_inverted", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CreateShapeMask" + }, + "widgets_values": [ + "circle", + 1, + 256, + 256, + 0, + 512, + 512, + 472, + 472 + ] + }, + { + "id": 133, + "type": "PrimitiveNode", + "pos": [ + -1576, + -82 + ], + "size": { + "0": 210, + "1": 82 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 302, + 324 + ], + "widget": { + "name": "frame_load_cap" + }, + "slot_index": 0 + } + ], + "title": "frame_load_cap", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 128, + "fixed" + ] + }, + { + "id": 182, + "type": "GetImageSizeAndCount", + "pos": [ + -831, + 37 + ], + "size": { + "0": 210, + "1": 86 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 436 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 451, + 465 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "480 width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "480 height", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "128 count", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 78, + "type": "GetImageSizeAndCount", + "pos": [ + -522, + -572 + ], + "size": { + "0": 210, + "1": 86 + }, + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 434 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 445, + 455 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "1024 width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "580 height", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "128 count", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 192, + "type": "Reroute", + "pos": [ + 423, + -466 + ], + "size": [ + 82, + 26 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 455 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 456, + 457 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": true, + "horizontal": false + } + }, + { + "id": 8, + "type": "VHS_LoadVideo", + "pos": [ + -1253, + 28 + ], + "size": [ + 340.1555480957031, + 572.1555480957031 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "frame_load_cap", + "type": "INT", + "link": 302, + "widget": { + "name": "frame_load_cap" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 436 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "d6.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 128, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 128, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "d6.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 193, + "type": "VHS_VideoInfo", + "pos": [ + -1064, + -48 + ], + "size": { + "0": 393, + "1": 206 + }, + "flags": { + "collapsed": true + }, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "link": 459 + } + ], + "outputs": [ + { + "name": "source_fps🟨", + "type": "FLOAT", + "links": [ + 460 + ], + "shape": 3 + }, + { + "name": "source_frame_count🟨", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "source_duration🟨", + "type": "FLOAT", + "links": null, + "shape": 3 + }, + { + "name": "source_width🟨", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "source_height🟨", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "loaded_fps🟦", + "type": "FLOAT", + "links": null, + "shape": 3 + }, + { + "name": "loaded_frame_count🟦", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "loaded_duration🟦", + "type": "FLOAT", + "links": null, + "shape": 3 + }, + { + "name": "loaded_width🟦", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "loaded_height🟦", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoInfo" + }, + "widgets_values": {} + }, + { + "id": 194, + "type": "Reroute", + "pos": [ + -1054, + -114 + ], + "size": [ + 115.6, + 26 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 461 + } + ], + "outputs": [ + { + "name": "VHS_AUDIO", + "type": "VHS_AUDIO", + "links": [ + 462 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": true, + "horizontal": false + } + }, + { + "id": 187, + "type": "Note", + "pos": [ + 118, + -243 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Connect this if you want to use the eye/lip retargeting mode" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 190, + "type": "LivePortraitProcess", + "pos": [ + 551, + -383 + ], + "size": { + "0": 430.8000183105469, + "1": 282 + }, + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "LIVEPORTRAITPIPE", + "link": 448 + }, + { + "name": "crop_info", + "type": "CROPINFO", + "link": 449 + }, + { + "name": "source_image", + "type": "IMAGE", + "link": 456 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": 451 + }, + { + "name": "opt_retargeting_info", + "type": "RETARGETINGINFO", + "link": null + } + ], + "outputs": [ + { + "name": "cropped_image", + "type": "IMAGE", + "links": [ + 454 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "output", + "type": "LP_OUT", + "links": [ + 452 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LivePortraitProcess" + }, + "widgets_values": [ + false, + 0.03, + true, + 1, + "constant", + "source_video_smoothed", + 0.000003 + ] + }, + { + "id": 191, + "type": "LivePortraitComposite", + "pos": [ + 1013, + -440 + ], + "size": { + "0": 355.20001220703125, + "1": 86 + }, + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "source_image", + "type": "IMAGE", + "link": 457 + }, + { + "name": "cropped_image", + "type": "IMAGE", + "link": 454 + }, + { + "name": "liveportrait_out", + "type": "LP_OUT", + "link": 452 + }, + { + "name": "mask", + "type": "MASK", + "link": 471 + } + ], + "outputs": [ + { + "name": "full_images", + "type": "IMAGE", + "links": [ + 458 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "mask", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LivePortraitComposite" + } + }, + { + "id": 181, + "type": "GetImageSizeAndCount", + "pos": [ + 1400, + -432 + ], + "size": { + "0": 210, + "1": 86 + }, + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 458 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 469 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "1024 width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "580 height", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "128 count", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 168, + "type": "VHS_VideoCombine", + "pos": [ + 1012, + -244 + ], + "size": [ + 678.661175161676, + 695.6427014593934 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 469 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": 462 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "frame_rate", + "type": "FLOAT", + "link": 460, + "widget": { + "name": "frame_rate" + }, + "slot_index": 4 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "LivePortrait", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "LivePortrait_00006-audio.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4", + "frame_rate": 24.087404244631422 + } + } + } + }, + { + "id": 146, + "type": "VHS_LoadVideo", + "pos": [ + -1271, + -572 + ], + "size": [ + 340.1555480957031, + 433.2201215636055 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "frame_load_cap", + "type": "INT", + "link": 324, + "widget": { + "name": "frame_load_cap" + }, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 433 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "links": [ + 461 + ], + "shape": 3, + "slot_index": 2 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": [ + 459 + ], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "Kung_Fu_Neo_vs_Morpheus_Trim_1.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 128, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 128, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "Kung_Fu_Neo_vs_Morpheus_Trim_1.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 195, + "type": "LivePortraitCropper", + "pos": [ + -636, + 187 + ], + "size": { + "0": 330, + "1": 242 + }, + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "LIVEPORTRAITPIPE", + "link": 466, + "slot_index": 0 + }, + { + "name": "cropper", + "type": "LPCROPPER", + "link": 476 + }, + { + "name": "source_image", + "type": "IMAGE", + "link": 465 + } + ], + "outputs": [ + { + "name": "cropped_image", + "type": "IMAGE", + "links": [ + 470 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "crop_info", + "type": "CROPINFO", + "links": [ + 467 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LivePortraitCropper" + }, + "widgets_values": [ + 512, + 2.3, + 0, + -0.125, + 0, + "large-small", + true + ] + }, + { + "id": 180, + "type": "LivePortraitRetargeting", + "pos": [ + -265, + 3 + ], + "size": { + "0": 430.8000183105469, + "1": 130 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "driving_crop_info", + "type": "CROPINFO", + "link": 467, + "slot_index": 0 + } + ], + "outputs": [ + { + "name": "retargeting_info", + "type": "RETARGETINGINFO", + "links": [], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LivePortraitRetargeting" + }, + "widgets_values": [ + true, + 1, + true, + 1 + ] + }, + { + "id": 134, + "type": "GetImageSizeAndCount", + "pos": [ + -241, + 247 + ], + "size": { + "0": 210, + "1": 86 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 470 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [], + "shape": 3, + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "height", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "count", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GetImageSizeAndCount" + } + }, + { + "id": 188, + "type": "Note", + "pos": [ + 13, + 222 + ], + "size": { + "0": 229.1644287109375, + "1": 95.92726135253906 + }, + "flags": {}, + "order": 3, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "If you want to crop your driving video, use this as driving_images instead" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 81, + "type": "GrowMaskWithBlur", + "pos": [ + 1050, + -800 + ], + "size": { + "0": 315, + "1": 246 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 192, + "slot_index": 0 + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 187, + 471 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "mask_inverted", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "GrowMaskWithBlur" + }, + "widgets_values": [ + 0, + 0, + true, + false, + 13.9, + 1, + 1, + false + ] + }, + { + "id": 189, + "type": "LivePortraitCropper", + "pos": [ + -3, + -755 + ], + "size": { + "0": 330, + "1": 242 + }, + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "LIVEPORTRAITPIPE", + "link": 446, + "slot_index": 0 + }, + { + "name": "cropper", + "type": "LPCROPPER", + "link": 477, + "slot_index": 1 + }, + { + "name": "source_image", + "type": "IMAGE", + "link": 445 + } + ], + "outputs": [ + { + "name": "cropped_image", + "type": "IMAGE", + "links": null, + "shape": 3, + "slot_index": 0 + }, + { + "name": "crop_info", + "type": "CROPINFO", + "links": [ + 449 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LivePortraitCropper" + }, + "widgets_values": [ + 512, + 2.3, + 0, + -0.125, + 0, + "large-small", + true + ] + }, + { + "id": 1, + "type": "DownloadAndLoadLivePortraitModels", + "pos": [ + -1247, + -1022 + ], + "size": { + "0": 302.43463134765625, + "1": 58 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "live_portrait_pipe", + "type": "LIVEPORTRAITPIPE", + "links": [ + 446, + 448, + 466 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DownloadAndLoadLivePortraitModels" + }, + "widgets_values": [ + "fp16" + ] + }, + { + "id": 129, + "type": "LivePortraitLoadCropper", + "pos": [ + -1253, + -757 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "cropper", + "type": "LPCROPPER", + "links": [], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LivePortraitLoadCropper" + }, + "widgets_values": [ + "CPU", + true + ] + }, + { + "id": 196, + "type": "LivePortraitLoadMediaPipeCropper", + "pos": [ + -1253, + -896 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "cropper", + "type": "LPCROPPER", + "links": [ + 475 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LivePortraitLoadMediaPipeCropper" + }, + "widgets_values": [ + "CPU", + true + ] + }, + { + "id": 197, + "type": "Reroute", + "pos": [ + -776, + -854 + ], + "size": [ + 115.6, + 26 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 475 + } + ], + "outputs": [ + { + "name": "LPCROPPER", + "type": "LPCROPPER", + "links": [ + 476, + 477 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": true, + "horizontal": false + } + }, + { + "id": 198, + "type": "Note", + "pos": [ + -1537, + -865 + ], + "size": { + "0": 265.68304443359375, + "1": 168.53929138183594 + }, + "flags": {}, + "order": 7, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Choose your face detector here, MediaPipe doesn't run on GPU and isn't as good at detecting, but is faster on CPU and has Apache 2.0 license.\n\nInsightface can be run on GPU and is better at detecting, but it's model license is NON-COMMERCIAL" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 187, + 81, + 0, + 82, + 0, + "MASK" + ], + [ + 192, + 83, + 0, + 81, + 0, + "MASK" + ], + [ + 302, + 133, + 0, + 8, + 2, + "INT" + ], + [ + 324, + 133, + 0, + 146, + 2, + "INT" + ], + [ + 433, + 146, + 0, + 165, + 0, + "IMAGE" + ], + [ + 434, + 165, + 0, + 78, + 0, + "IMAGE" + ], + [ + 436, + 8, + 0, + 182, + 0, + "IMAGE" + ], + [ + 445, + 78, + 0, + 189, + 2, + "IMAGE" + ], + [ + 446, + 1, + 0, + 189, + 0, + "LIVEPORTRAITPIPE" + ], + [ + 448, + 1, + 0, + 190, + 0, + "LIVEPORTRAITPIPE" + ], + [ + 449, + 189, + 1, + 190, + 1, + "CROPINFO" + ], + [ + 451, + 182, + 0, + 190, + 3, + "IMAGE" + ], + [ + 452, + 190, + 1, + 191, + 2, + "LP_OUT" + ], + [ + 454, + 190, + 0, + 191, + 1, + "IMAGE" + ], + [ + 455, + 78, + 0, + 192, + 0, + "*" + ], + [ + 456, + 192, + 0, + 190, + 2, + "IMAGE" + ], + [ + 457, + 192, + 0, + 191, + 0, + "IMAGE" + ], + [ + 458, + 191, + 0, + 181, + 0, + "IMAGE" + ], + [ + 459, + 146, + 3, + 193, + 0, + "VHS_VIDEOINFO" + ], + [ + 460, + 193, + 0, + 168, + 4, + "FLOAT" + ], + [ + 461, + 146, + 2, + 194, + 0, + "*" + ], + [ + 462, + 194, + 0, + 168, + 1, + "VHS_AUDIO" + ], + [ + 465, + 182, + 0, + 195, + 2, + "IMAGE" + ], + [ + 466, + 1, + 0, + 195, + 0, + "LIVEPORTRAITPIPE" + ], + [ + 467, + 195, + 1, + 180, + 0, + "CROPINFO" + ], + [ + 469, + 181, + 0, + 168, + 0, + "IMAGE" + ], + [ + 470, + 195, + 0, + 134, + 0, + "IMAGE" + ], + [ + 471, + 81, + 0, + 191, + 3, + "MASK" + ], + [ + 475, + 196, + 0, + 197, + 0, + "*" + ], + [ + 476, + 197, + 0, + 195, + 1, + "LPCROPPER" + ], + [ + 477, + 197, + 0, + 189, + 1, + "LPCROPPER" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.6209213230591552, + "offset": { + "0": 1804.8014223358243, + "1": 1129.1173424169717 + } + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/liveportrait/config/argument_config.py b/liveportrait/config/argument_config.py deleted file mode 100644 index 8a91c3e..0000000 --- a/liveportrait/config/argument_config.py +++ /dev/null @@ -1,44 +0,0 @@ -# coding: utf-8 - -""" -config for user -""" - -import os.path as osp -from dataclasses import dataclass -#import tyro -from typing_extensions import Annotated -from .base_config import PrintableConfig, make_abs_path - - -@dataclass(repr=False) # use repr from PrintableConfig -class ArgumentConfig(PrintableConfig): - ########## input arguments ########## - #source_image: Annotated[str, tyro.conf.arg(aliases=["-s"])] = make_abs_path('../../assets/examples/source/s6.jpg') # path to the reference portrait - #driving_info: Annotated[str, tyro.conf.arg(aliases=["-d"])] = make_abs_path('../../assets/examples/driving/d0.mp4') # path to driving video or template (.pkl format) - #output_dir: Annotated[str, tyro.conf.arg(aliases=["-o"])] = 'animations/' # directory to save output video - ##################################### - - ########## inference arguments ########## - device_id: int = 0 - flag_lip_zero : bool = True # whether let the lip to close state before animation, only take effect when flag_eye_retargeting and flag_lip_retargeting is False - flag_eye_retargeting: bool = False - flag_lip_retargeting: bool = False - flag_stitching: bool = True # we recommend setting it to True! - flag_relative: bool = True # whether to use relative pose - flag_pasteback: bool = True # whether to paste-back/stitch the animated face cropping from the face-cropping space to the original image space - flag_do_crop: bool = True # whether to crop the reference portrait to the face-cropping space - flag_do_rot: bool = True # whether to conduct the rotation when flag_do_crop is True - ######################################### - - ########## crop arguments ########## - dsize: int = 512 - scale: float = 2.3 - vx_ratio: float = 0 # vx ratio - vy_ratio: float = -0.125 # vy ratio +up, -down - #################################### - - ########## gradio arguments ########## - #server_port: Annotated[int, tyro.conf.arg(aliases=["-p"])] = 8890 - #share: bool = False - #server_name: str = "0.0.0.0" diff --git a/liveportrait/config/crop_config.py b/liveportrait/config/crop_config.py deleted file mode 100644 index d3c79be..0000000 --- a/liveportrait/config/crop_config.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding: utf-8 - -""" -parameters used for crop faces -""" - -import os.path as osp -from dataclasses import dataclass -from typing import Union, List -from .base_config import PrintableConfig - - -@dataclass(repr=False) # use repr from PrintableConfig -class CropConfig(PrintableConfig): - dsize: int = 512 # crop size - scale: float = 2.3 # scale factor - vx_ratio: float = 0 # vx ratio - vy_ratio: float = -0.125 # vy ratio +up, -down diff --git a/liveportrait/config/inference_config.py b/liveportrait/config/inference_config.py index 0da3e3c..50cbf10 100644 --- a/liveportrait/config/inference_config.py +++ b/liveportrait/config/inference_config.py @@ -29,21 +29,13 @@ class InferenceConfig(PrintableConfig): flag_stitching: bool = True # we recommend setting it to True! flag_relative: bool = True # whether to use relative pose - anchor_frame: int = 0 # set this value if find_best_frame is True input_shape: Tuple[int, int] = (256, 256) # input shape output_format: Literal['mp4', 'gif'] = 'mp4' # output video format output_fps: int = 30 # fps for output video crf: int = 15 # crf for output video - flag_write_result: bool = True # whether to write output video - flag_pasteback: bool = True # whether to paste-back/stitch the animated face cropping from the face-cropping space to the original image space - mask_crop = None flag_write_gif: bool = False - size_gif: int = 256 - ref_max_shape: int = 1280 - ref_shape_n: int = 2 device_id: int = 0 - flag_do_crop: bool = False # whether to crop the reference portrait to the face-cropping space flag_do_rot: bool = True # whether to conduct the rotation when flag_do_crop is True diff --git a/liveportrait/live_portrait_pipeline.py b/liveportrait/live_portrait_pipeline.py index 06b5e71..7747465 100644 --- a/liveportrait/live_portrait_pipeline.py +++ b/liveportrait/live_portrait_pipeline.py @@ -4,184 +4,276 @@ Pipeline of LivePortrait """ -import cv2 -import numpy as np -import os.path as osp -from tqdm import tqdm - -from .config.inference_config import InferenceConfig - -#from .utils.cropper import Cropper -from .utils.camera import get_rotation_matrix -#from .utils.video import images2video, concat_frames -from .utils.crop import _transform_img -#from .utils.retargeting_utils import calc_lip_close_ratio -#from .utils.io import load_image_rgb, load_driving_info -#from .utils.helper import mkdir, basename, dct2cuda, is_video, is_template, resize_to_limit -from .utils.helper import resize_to_limit -#from .utils.rprint import rlog as log -from .live_portrait_wrapper import LivePortraitWrapper - import comfy.utils +import comfy.model_management as mm +import gc +from tqdm import tqdm +import numpy as np +from .config.inference_config import InferenceConfig +from .utils.camera import get_rotation_matrix +from .live_portrait_wrapper import LivePortraitWrapper +from .utils.retargeting_utils import calc_eye_close_ratio, calc_lip_close_ratio +from .utils.filter import smooth -def make_abs_path(fn): - return osp.join(osp.dirname(osp.realpath(__file__)), fn) - +import os +script_directory = os.path.dirname(os.path.abspath(__file__)) class LivePortraitPipeline(object): - - def __init__(self, appearance_feature_extractor, motion_extractor, warping_module, - spade_generator, stitching_retargeting_module, inference_cfg: InferenceConfig): - + def __init__( + self, + appearance_feature_extractor, + motion_extractor, + warping_module, + spade_generator, + stitching_retargeting_module, + inference_cfg: InferenceConfig, + ): self.live_portrait_wrapper: LivePortraitWrapper = LivePortraitWrapper( - appearance_feature_extractor, motion_extractor, warping_module, - spade_generator, stitching_retargeting_module, cfg=inference_cfg) + appearance_feature_extractor, + motion_extractor, + warping_module, + spade_generator, + stitching_retargeting_module, + cfg=inference_cfg, + ) - def execute(self, img_rgb, driving_images_np): - inference_cfg = self.live_portrait_wrapper.cfg # for convenience - ######## process reference portrait ######## - #img_rgb = load_image_rgb(args.source_image) - img_rgb = resize_to_limit(img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n) - #log(f"Load source image from {args.source_image}") - crop_info = self.cropper.crop_single_image(img_rgb) - source_lmk = crop_info['lmk_crop'] - _, img_crop_256x256 = crop_info['img_crop'], crop_info['img_crop_256x256'] - if inference_cfg.flag_do_crop: - I_s = self.live_portrait_wrapper.prepare_source(img_crop_256x256) - else: - I_s = self.live_portrait_wrapper.prepare_source(img_rgb) - x_s_info = self.live_portrait_wrapper.get_kp_info(I_s) - x_c_s = x_s_info['kp'] - R_s = get_rotation_matrix(x_s_info['pitch'], x_s_info['yaw'], x_s_info['roll']) - f_s = self.live_portrait_wrapper.extract_feature_3d(I_s) - x_s = self.live_portrait_wrapper.transform_keypoint(x_s_info) + def execute( + self, driving_images, crop_info, driving_landmarks, delta_multiplier, relative_motion_mode, driving_smooth_observation_variance, mismatch_method="constant", + ): + inference_cfg = self.live_portrait_wrapper.cfg + device = inference_cfg.device_id - if inference_cfg.flag_lip_zero: - # let lip-open scalar to be 0 at first - c_d_lip_before_animation = [0.] - combined_lip_ratio_tensor_before_animation = self.live_portrait_wrapper.calc_combined_lip_ratio(c_d_lip_before_animation, source_lmk) - if combined_lip_ratio_tensor_before_animation[0][0] < inference_cfg.lip_zero_threshold: - inference_cfg.flag_lip_zero = False - else: - lip_delta_before_animation = self.live_portrait_wrapper.retarget_lip(x_s, combined_lip_ratio_tensor_before_animation) - ############################################ - - ######## process driving info ######## - #if is_video(args.driving_info): - #log(f"Load from video file (mp4 mov avi etc...): {args.driving_info}") - # TODO: 这里track一下驱动视频 -> 构建模板 - #driving_rgb_lst = load_driving_info(args.driving_info) - - driving_rgb_lst = driving_images_np - - driving_rgb_lst_256 = [cv2.resize(_, (256, 256)) for _ in driving_rgb_lst] - I_d_lst = self.live_portrait_wrapper.prepare_driving_videos(driving_rgb_lst_256) - n_frames = I_d_lst.shape[0] - if inference_cfg.flag_eye_retargeting or inference_cfg.flag_lip_retargeting: - driving_lmk_lst = self.cropper.get_retargeting_lmk_info(driving_rgb_lst) - input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst) - - # elif is_template(args.driving_info): - # log(f"Load from video templates {args.driving_info}") - # with open(args.driving_info, 'rb') as f: - # template_lst, driving_lmk_lst = pickle.load(f) - # n_frames = template_lst[0]['n_frames'] - # input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst) - # else: - # raise Exception("Unsupported driving types!") - ######################################### - - ######## prepare for pasteback ######## - if inference_cfg.flag_pasteback: - if inference_cfg.mask_crop is None: - inference_cfg.mask_crop = cv2.imread(make_abs_path('./utils/resources/mask_template.png'), cv2.IMREAD_COLOR) - mask_ori = _transform_img(inference_cfg.mask_crop, crop_info['M_c2o'], dsize=(img_rgb.shape[1], img_rgb.shape[0])) - mask_ori = mask_ori.astype(np.float32) / 255. - I_p_paste_lst = [] - ######################################### - - I_p_lst = [] + out_list = [] R_d_0, x_d_0_info = None, None - pbar = comfy.utils.ProgressBar(n_frames) - for i in tqdm(range(n_frames), desc='Animating...', total=n_frames): - #if is_video(args.driving_info): - # extract kp info by M - I_d_i = I_d_lst[i] - x_d_i_info = self.live_portrait_wrapper.get_kp_info(I_d_i) - R_d_i = get_rotation_matrix(x_d_i_info['pitch'], x_d_i_info['yaw'], x_d_i_info['roll']) - # else: - # # from template - # x_d_i_info = template_lst[i] - # x_d_i_info = dct2cuda(x_d_i_info, inference_cfg.device_id) - # R_d_i = x_d_i_info['R_d'] + source_images_num = len(crop_info["crop_info_list"]) + + if mismatch_method == "cut" or relative_motion_mode == "source_video_smoothed": + total_frames = source_images_num + else: + total_frames = driving_images.shape[0] + + + + disable_progress_bar = True if relative_motion_mode == "single_frame" else False + + source_info = crop_info["source_info"] + source_rot_list = crop_info["source_rot_list"] + f_s_list = crop_info["f_s_list"] + x_s_list = crop_info["x_s_list"] + + driving_info = [] + driving_exp_list = [] + driving_rot_list = [] + + for i in tqdm(range(driving_images.shape[0]), desc='Processing driving images...', total=driving_images.shape[0], disable=disable_progress_bar): + #get driving keypoints info + safe_index = min(i, source_images_num - 1) + if crop_info["crop_info_list"][safe_index] is None: + driving_info.append(None) + driving_rot_list.append(None) + driving_exp_list.append(None) + continue + x_d_info = self.live_portrait_wrapper.get_kp_info(driving_images[i].unsqueeze(0).to(device)) + if i == 0: - R_d_0 = R_d_i - x_d_0_info = x_d_i_info + first = x_d_info - if inference_cfg.flag_relative: - R_new = (R_d_i @ R_d_0.permute(0, 2, 1)) @ R_s - delta_new = x_s_info['exp'] + (x_d_i_info['exp'] - x_d_0_info['exp']) - scale_new = x_s_info['scale'] * (x_d_i_info['scale'] / x_d_0_info['scale']) - t_new = x_s_info['t'] + (x_d_i_info['t'] - x_d_0_info['t']) + driving_info.append(x_d_info) + + driving_exp = source_info[safe_index]["exp"] + x_d_info["exp"] - first["exp"] + driving_exp_list.append(driving_exp.cpu()) + + R_d = get_rotation_matrix( + x_d_info["pitch"], x_d_info["yaw"], x_d_info["roll"] + ) + driving_rot_list.append(R_d) + + if relative_motion_mode == "source_video_smoothed": + x_d_r_lst = [] + first_driving_rot = driving_rot_list[0].cpu().numpy().astype(np.float32).transpose(0, 2, 1) + for i in tqdm(range(source_images_num), desc='Smoothing...', total=source_images_num): + if driving_rot_list[i] is None: + x_d_r_lst.append(None) + continue + driving_rot = driving_rot_list[i].cpu().numpy().astype(np.float32) + source_rot = source_rot_list[i].cpu().numpy().astype(np.float32) + dot = np.dot(driving_rot, first_driving_rot) @ source_rot + x_d_r_lst.append(dot) + + driving_exp_list_smooth = smooth(driving_exp_list, source_info[0]["exp"].shape, device, observation_variance=driving_smooth_observation_variance) + driving_rot_list_smooth = smooth(x_d_r_lst, source_rot_list[0].shape, device, observation_variance=driving_smooth_observation_variance) + + pbar = comfy.utils.ProgressBar(total_frames) + + for i in tqdm(range(total_frames), desc='Animating...', total=total_frames, disable=disable_progress_bar): + + safe_index = min(i, len(crop_info["crop_info_list"]) - 1) + + # skip and return empty frames if no crop due to no face detected + if crop_info["crop_info_list"][safe_index] is None: + out_list.append({}) + pbar.update(1) + continue + + source_lmk = crop_info["crop_info_list"][safe_index]["lmk_crop"] + + x_d_info = driving_info[i] + R_d = driving_rot_list[i] + + x_s_info = source_info[safe_index] + R_s = source_rot_list[safe_index] + f_s = f_s_list[safe_index] + x_s = x_s_list[safe_index] + + x_c_s = x_s_info["kp"] + + #lip zero + if inference_cfg.flag_lip_zero: + c_d_lip_before_animation = [0.0] + combined_lip_ratio_tensor_before_animation = (self.live_portrait_wrapper.calc_combined_lip_ratio(c_d_lip_before_animation, source_lmk)) + + if (combined_lip_ratio_tensor_before_animation[0][0] < inference_cfg.lip_zero_threshold): + inference_cfg.flag_lip_zero = False + else: + lip_delta_before_animation = (self.live_portrait_wrapper.retarget_lip(x_s, combined_lip_ratio_tensor_before_animation)) + + if relative_motion_mode == "relative": + if i == 0: + R_d_0 = R_d + x_d_0_info = x_d_info + R_new = (R_d @ R_d_0.permute(0, 2, 1)) @ R_s + delta_new = x_s_info["exp"] + (x_d_info["exp"] - x_d_0_info["exp"]) + scale_new = x_s_info["scale"] * (x_d_info["scale"] / x_d_0_info["scale"]) + t_new = x_s_info["t"] + (x_d_info["t"] - x_d_0_info["t"]) + elif relative_motion_mode == "source_video_smoothed": + R_new = driving_rot_list_smooth[i] + delta_new = driving_exp_list_smooth[i] + scale_new = x_s_info["scale"] + t_new = x_d_info["t"] + elif relative_motion_mode == "relative_rotation_only": + R_new = R_s + delta_new = x_s_info['exp'] + scale_new = x_s_info["scale"] + t_new = x_d_info["t"] + elif relative_motion_mode == "single_frame": + R_new = R_d + delta_new = x_d_info['exp'] + scale_new = x_s_info["scale"] + t_new = x_d_info["t"] else: - R_new = R_d_i - delta_new = x_d_i_info['exp'] - scale_new = x_s_info['scale'] - t_new = x_d_i_info['t'] + R_new = R_d + delta_new = x_s_info['exp'] + scale_new = x_s_info["scale"] + t_new = x_d_info["t"] - t_new[..., 2].fill_(0) # zero tz + t_new[..., 2].fill_(0) # zero tz + + delta_new = delta_new * delta_multiplier + x_d_i_new = scale_new * (x_c_s @ R_new + delta_new) + t_new - - # Algorithm 1: - if not inference_cfg.flag_stitching and not inference_cfg.flag_eye_retargeting and not inference_cfg.flag_lip_retargeting: + if ( + not inference_cfg.flag_stitching + and not inference_cfg.flag_eye_retargeting + and not inference_cfg.flag_lip_retargeting + ): # without stitching or retargeting if inference_cfg.flag_lip_zero: x_d_i_new += lip_delta_before_animation.reshape(-1, x_s.shape[1], 3) else: pass - elif inference_cfg.flag_stitching and not inference_cfg.flag_eye_retargeting and not inference_cfg.flag_lip_retargeting: + elif ( + inference_cfg.flag_stitching + and not inference_cfg.flag_eye_retargeting + and not inference_cfg.flag_lip_retargeting + ): # with stitching and without retargeting if inference_cfg.flag_lip_zero: - x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new) + lip_delta_before_animation.reshape(-1, x_s.shape[1], 3) + x_d_i_new = self.live_portrait_wrapper.stitching( + x_s, x_d_i_new + ) + lip_delta_before_animation.reshape(-1, x_s.shape[1], 3) else: x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new) + + #with eye/lip retargeting else: eyes_delta, lip_delta = None, None if inference_cfg.flag_eye_retargeting: - c_d_eyes_i = input_eye_ratio_lst[i] - combined_eye_ratio_tensor = self.live_portrait_wrapper.calc_combined_eye_ratio(c_d_eyes_i, source_lmk) - combined_eye_ratio_tensor = combined_eye_ratio_tensor * inference_cfg.eyes_retargeting_multiplier + c_d_eyes_i = calc_eye_close_ratio(driving_landmarks[i][None]) + combined_eye_ratio_tensor = ( + self.live_portrait_wrapper.calc_combined_eye_ratio( + c_d_eyes_i, source_lmk + ) + ) + combined_eye_ratio_tensor = ( + combined_eye_ratio_tensor + * inference_cfg.eyes_retargeting_multiplier + ) # ∆_eyes,i = R_eyes(x_s; c_s,eyes, c_d,eyes,i) - eyes_delta = self.live_portrait_wrapper.retarget_eye(x_s, combined_eye_ratio_tensor) + eyes_delta = self.live_portrait_wrapper.retarget_eye( + x_s, combined_eye_ratio_tensor + ) if inference_cfg.flag_lip_retargeting: - c_d_lip_i = input_lip_ratio_lst[i] - combined_lip_ratio_tensor = self.live_portrait_wrapper.calc_combined_lip_ratio(c_d_lip_i, source_lmk) - combined_lip_ratio_tensor = combined_lip_ratio_tensor * inference_cfg.lip_retargeting_multiplier + c_d_lip_i = calc_lip_close_ratio(driving_landmarks[i][None]) + combined_lip_ratio_tensor = ( + self.live_portrait_wrapper.calc_combined_lip_ratio( + c_d_lip_i, source_lmk + ) + ) + combined_lip_ratio_tensor = ( + combined_lip_ratio_tensor + * inference_cfg.lip_retargeting_multiplier + ) # ∆_lip,i = R_lip(x_s; c_s,lip, c_d,lip,i) - lip_delta = self.live_portrait_wrapper.retarget_lip(x_s, combined_lip_ratio_tensor) + lip_delta = self.live_portrait_wrapper.retarget_lip( + x_s, combined_lip_ratio_tensor + ) - if inference_cfg.flag_relative: # use x_s - x_d_i_new = x_s + \ - (eyes_delta.reshape(-1, x_s.shape[1], 3) if eyes_delta is not None else 0) + \ - (lip_delta.reshape(-1, x_s.shape[1], 3) if lip_delta is not None else 0) + if relative_motion_mode != "off": # use x_s + x_d_i_new = ( + x_s + + ( + eyes_delta.reshape(-1, x_s.shape[1], 3) + if eyes_delta is not None + else 0 + ) + + ( + lip_delta.reshape(-1, x_s.shape[1], 3) + if lip_delta is not None + else 0 + ) + ) else: # use x_d,i - x_d_i_new = x_d_i_new + \ - (eyes_delta.reshape(-1, x_s.shape[1], 3) if eyes_delta is not None else 0) + \ - (lip_delta.reshape(-1, x_s.shape[1], 3) if lip_delta is not None else 0) + x_d_i_new = ( + x_d_i_new + + ( + eyes_delta.reshape(-1, x_s.shape[1], 3) + if eyes_delta is not None + else 0 + ) + + ( + lip_delta.reshape(-1, x_s.shape[1], 3) + if lip_delta is not None + else 0 + ) + ) if inference_cfg.flag_stitching: x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new) + if inference_cfg.flag_stitching: + x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new) + out = self.live_portrait_wrapper.warp_decode(f_s, x_s, x_d_i_new) - I_p_i = self.live_portrait_wrapper.parse_output(out['out'])[0] - I_p_lst.append(I_p_i) + + out_list.append(out) + pbar.update(1) - #if inference_cfg.flag_pasteback: - I_p_i_to_ori = _transform_img(I_p_i, crop_info['M_c2o'], dsize=(img_rgb.shape[1], img_rgb.shape[0])) - I_p_i_to_ori_blend = np.clip(mask_ori * I_p_i_to_ori + (1 - mask_ori) * img_rgb, 0, 255).astype(np.uint8) - out = np.hstack([I_p_i_to_ori, I_p_i_to_ori_blend]) - I_p_paste_lst.append(I_p_i_to_ori_blend) + out_dict = { + "out_list": out_list, + "crop_info": crop_info, + "mismatch_method": mismatch_method, + } - return I_p_lst, I_p_paste_lst + return out_dict diff --git a/liveportrait/live_portrait_wrapper.py b/liveportrait/live_portrait_wrapper.py index 90cf47a..0f8dc4c 100644 --- a/liveportrait/live_portrait_wrapper.py +++ b/liveportrait/live_portrait_wrapper.py @@ -32,11 +32,6 @@ class LivePortraitWrapper(object): self.device_id = cfg.device_id self.timer = Timer() - def update_config(self, user_args): - for k, v in user_args.items(): - if hasattr(self.cfg, k): - setattr(self.cfg, k, v) - def prepare_source(self, img: np.ndarray) -> torch.Tensor: """ construct the input as standard img: HxWx3, uint8, 256x256 @@ -58,24 +53,6 @@ class LivePortraitWrapper(object): x = x.to(self.device_id) return x - def prepare_driving_videos(self, imgs) -> torch.Tensor: - """ construct the input as standard - imgs: NxBxHxWx3, uint8 - """ - if isinstance(imgs, list): - _imgs = np.array(imgs)[..., np.newaxis] # TxHxWx3x1 - elif isinstance(imgs, np.ndarray): - _imgs = imgs - else: - raise ValueError(f'imgs type error: {type(imgs)}') - - y = _imgs.astype(np.float32) / 255. - y = np.clip(y, 0, 1) # clip to 0~1 - y = torch.from_numpy(y).permute(0, 4, 3, 1, 2) # TxHxWx3x1 -> Tx1x3xHxW - y = y.to(self.device_id) - - return y - def extract_feature_3d(self, x: torch.Tensor) -> torch.Tensor: """ get the appearance feature of the image by F x: Bx3xHxW, normalized to 0~1 @@ -193,35 +170,6 @@ class LivePortraitWrapper(object): return delta - def retarget_keypoints(self, frame_idx, num_keypoints, input_eye_ratios, input_lip_ratios, source_landmarks, portrait_wrapper, kp_source, driving_transformed_kp): - # TODO: GPT style, refactor it... - if self.cfg.flag_eye_retargeting: - print("Retargeting eye...") - # ∆_eyes,i = R_eyes(x_s; c_s,eyes, c_d,eyes,i) - eye_delta = compute_eye_delta(frame_idx, input_eye_ratios, source_landmarks, portrait_wrapper, kp_source) - else: - # α_eyes = 0 - eye_delta = None - - if self.cfg.flag_lip_retargeting: - print("Retargeting lip...") - # ∆_lip,i = R_lip(x_s; c_s,lip, c_d,lip,i) - lip_delta = compute_lip_delta(frame_idx, input_lip_ratios, source_landmarks, portrait_wrapper, kp_source) - else: - # α_lip = 0 - lip_delta = None - - if self.cfg.flag_relative: # use x_s - new_driving_kp = kp_source + \ - (eye_delta.reshape(-1, num_keypoints, 3) if eye_delta is not None else 0) + \ - (lip_delta.reshape(-1, num_keypoints, 3) if lip_delta is not None else 0) - else: # use x_d,i - new_driving_kp = driving_transformed_kp + \ - (eye_delta.reshape(-1, num_keypoints, 3) if eye_delta is not None else 0) + \ - (lip_delta.reshape(-1, num_keypoints, 3) if lip_delta is not None else 0) - - return new_driving_kp - def stitch(self, kp_source: torch.Tensor, kp_driving: torch.Tensor) -> torch.Tensor: """ kp_source: BxNx3 @@ -264,7 +212,7 @@ class LivePortraitWrapper(object): kp_source: BxNx3 kp_driving: BxNx3 """ - # The line 18 in Algorithm 1: D(W(f_s; x_s, x′_d,i)） + # The line 18 in Algorithm 1: D(W(f_s; x_s, x′_d,i) with torch.autocast(get_autocast_device(self.device_id), dtype=torch.float16) if self.cfg.flag_use_half_precision else nullcontext(): # get decoder input ret_dct = self.warping_module(feature_3d, kp_source=kp_source, kp_driving=kp_driving) @@ -272,23 +220,14 @@ class LivePortraitWrapper(object): ret_dct['out'] = self.spade_generator(feature=ret_dct['out']) # float the dict - if self.cfg.flag_use_half_precision: - for k, v in ret_dct.items(): - if isinstance(v, torch.Tensor): - ret_dct[k] = v.float() + for k, v in ret_dct.items(): + if isinstance(v, torch.Tensor): + ret_dct[k] = v.cpu() + if self.cfg.flag_use_half_precision: + ret_dct[k] = ret_dct[k].float() return ret_dct - def parse_output(self, out: torch.Tensor) -> np.ndarray: - """ construct the output as standard - return: 1xHxWx3, uint8 - """ - out = np.transpose(out.data.cpu().numpy(), [0, 2, 3, 1]) # 1x3xHxW -> 1xHxWx3 - out = np.clip(out, 0, 1) # clip to 0~1 - out = np.clip(out * 255, 0, 255).astype(np.uint8) # 0~1 -> 0~255 - - return out - def calc_retargeting_ratio(self, source_lmk, driving_lmk_lst): input_eye_ratio_lst = [] input_lip_ratio_lst = [] @@ -301,17 +240,19 @@ class LivePortraitWrapper(object): def calc_combined_eye_ratio(self, input_eye_ratio, source_lmk): eye_close_ratio = calc_eye_close_ratio(source_lmk[None]) - eye_close_ratio_tensor = torch.from_numpy(eye_close_ratio).float().to(self.device_id) - input_eye_ratio_tensor = torch.Tensor([input_eye_ratio[0][0]]).reshape(1, 1).to(self.device_id) + eye_close_ratios_tensor = torch.from_numpy(eye_close_ratio).float().to(self.device_id) + input_eye_ratio_array = np.array(input_eye_ratio[0][0]).reshape(1, 1) + input_eye_ratio_tensor = torch.from_numpy(input_eye_ratio_array).float().to(self.device_id) # [c_s,eyes, c_d,eyes,i] - combined_eye_ratio_tensor = torch.cat([eye_close_ratio_tensor, input_eye_ratio_tensor], dim=1) - return combined_eye_ratio_tensor + combined_eye_ratios_tensor = torch.cat([eye_close_ratios_tensor, input_eye_ratio_tensor], dim=1) + return combined_eye_ratios_tensor def calc_combined_lip_ratio(self, input_lip_ratio, source_lmk): lip_close_ratio = calc_lip_close_ratio(source_lmk[None]) lip_close_ratio_tensor = torch.from_numpy(lip_close_ratio).float().to(self.device_id) # [c_s,lip, c_d,lip,i] - input_lip_ratio_tensor = torch.Tensor([input_lip_ratio[0]]).to(self.device_id) + input_lip_ratio_array = np.array([input_lip_ratio[0]]) + input_lip_ratio_tensor = torch.from_numpy(input_lip_ratio_array).float().to(self.device_id) if input_lip_ratio_tensor.shape != [1, 1]: input_lip_ratio_tensor = input_lip_ratio_tensor.reshape(1, 1) combined_lip_ratio_tensor = torch.cat([lip_close_ratio_tensor, input_lip_ratio_tensor], dim=1) diff --git a/liveportrait/modules/dense_motion.py b/liveportrait/modules/dense_motion.py index 429ecdc..502243f 100644 --- a/liveportrait/modules/dense_motion.py +++ b/liveportrait/modules/dense_motion.py @@ -47,7 +47,13 @@ class DenseMotionNetwork(nn.Module): feature_repeat = feature.unsqueeze(1).unsqueeze(1).repeat(1, self.num_kp+1, 1, 1, 1, 1, 1) # (bs, num_kp+1, 1, c, d, h, w) feature_repeat = feature_repeat.view(bs * (self.num_kp+1), -1, d, h, w) # (bs*(num_kp+1), c, d, h, w) sparse_motions = sparse_motions.view((bs * (self.num_kp+1), d, h, w, -1)) # (bs*(num_kp+1), d, h, w, 3) - sparse_deformed = F.grid_sample(feature_repeat, sparse_motions, align_corners=False) + try: + sparse_deformed = F.grid_sample(feature_repeat, sparse_motions, align_corners=False) + except NotImplementedError: #MPS fallback + out_device = feature_repeat.device # Store input device + feature_repeat = feature_repeat.to('cpu') + sparse_motions = sparse_motions.to('cpu') + sparse_deformed = F.grid_sample(feature_repeat, sparse_motions, align_corners=False).to(out_device) sparse_deformed = sparse_deformed.view((bs, self.num_kp+1, -1, d, h, w)) # (bs, num_kp+1, c, d, h, w) return sparse_deformed @@ -61,7 +67,7 @@ class DenseMotionNetwork(nn.Module): # adding background feature try: zeros = torch.zeros(heatmap.shape[0], 1, spatial_size[0], spatial_size[1], spatial_size[2]).type(heatmap.type()).to(heatmap.device) - except: + except ValueError: zeros = torch.zeros(heatmap.shape[0], 1, spatial_size[0], spatial_size[1], spatial_size[2]).to(heatmap.device) heatmap = torch.cat([zeros, heatmap], dim=1) heatmap = heatmap.unsqueeze(2) # (bs, 1+num_kp, 1, d, h, w) diff --git a/liveportrait/modules/util.py b/liveportrait/modules/util.py index f83980b..050a6e8 100644 --- a/liveportrait/modules/util.py +++ b/liveportrait/modules/util.py @@ -158,7 +158,11 @@ class DownBlock3d(nn.Module): out = self.conv(x) out = self.norm(out) out = F.relu(out) - out = self.pool(out) + try: + out = self.pool(out) + except NotImplementedError: + out_device = out.device # Store input device + out = self.pool(out.to('cpu')).to(out_device) return out diff --git a/liveportrait/modules/warping_network.py b/liveportrait/modules/warping_network.py index 9191a19..4b3419a 100644 --- a/liveportrait/modules/warping_network.py +++ b/liveportrait/modules/warping_network.py @@ -44,7 +44,11 @@ class WarpingNetwork(nn.Module): self.estimate_occlusion_map = estimate_occlusion_map def deform_input(self, inp, deformation): - return F.grid_sample(inp, deformation, align_corners=False) + try: + return F.grid_sample(inp, deformation, align_corners=False) + except NotImplementedError: + out_device = inp.device # Store input device + return F.grid_sample(inp.to('cpu'), deformation.to('cpu'), align_corners=False).to(out_device) def forward(self, feature_3d, kp_driving, kp_source): if self.dense_motion_network is not None: diff --git a/liveportrait/template_maker.py b/liveportrait/template_maker.py deleted file mode 100644 index 8d21bf5..0000000 --- a/liveportrait/template_maker.py +++ /dev/null @@ -1,65 +0,0 @@ -# coding: utf-8 - -""" -Make video template -""" - -import os -import cv2 -import numpy as np -import pickle -from tqdm import tqdm -from .utils.cropper import Cropper - -from .utils.io import load_driving_info -from .utils.camera import get_rotation_matrix -from .utils.helper import mkdir, basename -from .utils.rprint import rlog as log -from .config.crop_config import CropConfig -from .config.inference_config import InferenceConfig -from .live_portrait_wrapper import LivePortraitWrapper - -class TemplateMaker: - - def __init__(self, inference_cfg: InferenceConfig, crop_cfg: CropConfig): - self.live_portrait_wrapper: LivePortraitWrapper = LivePortraitWrapper(cfg=inference_cfg) - self.cropper = Cropper(crop_cfg=crop_cfg) - - def make_motion_template(self, video_fp: str, output_path: str, **kwargs): - """ make video template (.pkl format) - video_fp: driving video file path - output_path: where to save the pickle file - """ - - driving_rgb_lst = load_driving_info(video_fp) - driving_rgb_lst = [cv2.resize(_, (256, 256)) for _ in driving_rgb_lst] - driving_lmk_lst = self.cropper.get_retargeting_lmk_info(driving_rgb_lst) - I_d_lst = self.live_portrait_wrapper.prepare_driving_videos(driving_rgb_lst) - - n_frames = I_d_lst.shape[0] - - templates = [] - - - for i in tqdm(range(n_frames), desc='Making templates...', total=n_frames): - I_d_i = I_d_lst[i] - x_d_i_info = self.live_portrait_wrapper.get_kp_info(I_d_i) - R_d_i = get_rotation_matrix(x_d_i_info['pitch'], x_d_i_info['yaw'], x_d_i_info['roll']) - # collect s_d, R_d, δ_d and t_d for inference - template_dct = { - 'n_frames': n_frames, - 'frames_index': i, - } - template_dct['scale'] = x_d_i_info['scale'].cpu().numpy().astype(np.float32) - template_dct['R_d'] = R_d_i.cpu().numpy().astype(np.float32) - template_dct['exp'] = x_d_i_info['exp'].cpu().numpy().astype(np.float32) - template_dct['t'] = x_d_i_info['t'].cpu().numpy().astype(np.float32) - - templates.append(template_dct) - - mkdir(output_path) - # Save the dictionary as a pickle file - pickle_fp = os.path.join(output_path, f'{basename(video_fp)}.pkl') - with open(pickle_fp, 'wb') as f: - pickle.dump([templates, driving_lmk_lst], f) - log(f"Template saved at {pickle_fp}") diff --git a/liveportrait/utils/crop.py b/liveportrait/utils/crop.py index c061ef4..b617e00 100644 --- a/liveportrait/utils/crop.py +++ b/liveportrait/utils/crop.py @@ -4,14 +4,12 @@ cropping function and the related preprocess functions for cropping """ -import cv2; cv2.setNumThreads(0); cv2.ocl.setUseOpenCL(False) # NOTE: enforce single thread +import cv2#; cv2.setNumThreads(0); cv2.ocl.setUseOpenCL(False) # NOTE: enforce single thread import numpy as np -from .rprint import rprint as print from math import sin, cos, acos, degrees - DTYPE = np.float32 CV2_INTERP = cv2.INTER_LINEAR - +import comfy.model_management as mm def _transform_img(img, M, dsize, flags=CV2_INTERP, borderMode=None): """ conduct similarity or affine transformation to the image, do not do border operation! @@ -29,6 +27,43 @@ def _transform_img(img, M, dsize, flags=CV2_INTERP, borderMode=None): else: return cv2.warpAffine(img, M[:2, :], dsize=_dsize, flags=flags) +import torch +import kornia.geometry.transform as KGT + +def _transform_img_kornia(img, M, dsize, device, flags='bilinear', borderMode='zeros'): + """Conduct similarity or affine transformation to the image using Kornia. + + img: Input image as a PyTorch tensor of shape (C, H, W). + M: 2x3 transformation matrix as a PyTorch tensor. + dsize: Target shape (width, height). + """ + + # Convert dsize to tensor shape (H, W) + _dsize = torch.tensor([dsize[1], dsize[0]]) # Kornia expects (H, W) + + # Convert M from numpy.ndarray to PyTorch tensor + M = torch.from_numpy(M).float().to(device) + if M.shape == (3, 3): + M = M[:2, :].unsqueeze(0) # Adjust M to the expected shape Bx2x3 + elif M.shape == (2, 3): + M = M.unsqueeze(0) # Add batch dimension if not present + + # Reshape M for Kornia (1, 2, 3) and upscale to 3D affine matrix if not already + if M.shape == (2, 3): + M = M.unsqueeze(0) # Add batch dimension + + # Convert image to floating point tensor if not already + if img.dtype != torch.float32: + img = img.float() + img = img.to(device) + + # Reshape img for Kornia (B, C, H, W) + img = img.permute(0, 3, 1, 2) + + # Apply the affine transformation + img_warped = KGT.warp_affine(img, M, _dsize, mode=flags, padding_mode=borderMode) + + return img_warped def _transform_pts(pts, M): """ conduct similarity or affine transformation to the pts @@ -39,6 +74,23 @@ def _transform_pts(pts, M): return pts @ M[:2, :2].T + M[:2, 2] +def parse_pt2_from_pt478(pt478, use_lip=True): + """ + parsing the 2 points according to the 101 points, which cancels the roll + """ + # the former version use the eye center, but it is not robust, now use interpolation + pt_left_eye = pt478[468] # left eye center + pt_right_eye = pt478[473] # right eye center + + if use_lip: + # use lip + pt_center_eye = (pt_left_eye + pt_right_eye) / 2 + pt_center_lip = pt478[14] + pt2 = np.stack([pt_center_eye, pt_center_lip], axis=0) + else: + pt2 = np.stack([pt_left_eye, pt_right_eye], axis=0) + return pt2 + def parse_pt2_from_pt101(pt101, use_lip=True): """ parsing the 2 points according to the 101 points, which cancels the roll @@ -89,29 +141,60 @@ def parse_pt2_from_pt203(pt203, use_lip=True): pt2 = np.stack([pt_left_eye, pt_right_eye], axis=0) return pt2 +def parse_pt2_from_pt9(pt9, use_lip=True): + ''' + animal_face = {"keypoints": ['right eye right', 'right eye left', 'left eye right', 'left eye left', 'nose tip', 'lip right', 'lip left', 'upper lip', 'lower lip'], "skeleton": []} -def parse_pt2_from_pt68(pt68, use_lip=True): - """ - parsing the 2 points according to the 68 points, which cancels the roll - """ - lm_idx = np.array([31, 37, 40, 43, 46, 49, 55], dtype=np.int32) - 1 + + ''' if use_lip: - pt5 = np.stack([ - np.mean(pt68[lm_idx[[1, 2]], :], 0), # left eye - np.mean(pt68[lm_idx[[3, 4]], :], 0), # right eye - pt68[lm_idx[0], :], # nose - pt68[lm_idx[5], :], # lip - pt68[lm_idx[6], :] # lip + pt9 = np.stack([ + (pt9[2]+pt9[3])/2, # left eye + (pt9[0]+pt9[1])/2, # right eye + pt9[4], + # (pt9[5]+pt9[6]+pt9[7]+pt9[8])/4 # lip + (pt9[5] + pt9[6] ) / 2 # lip ], axis=0) - pt2 = np.stack([ - (pt5[0] + pt5[1]) / 2, - (pt5[3] + pt5[4]) / 2 + (pt9[0] + pt9[1]) / 2, # eye + pt9[3] # lip ], axis=0) else: pt2 = np.stack([ - np.mean(pt68[lm_idx[[1, 2]], :], 0), # left eye - np.mean(pt68[lm_idx[[3, 4]], :], 0), # right eye + (pt9[2] + pt9[3]) / 2, + (pt9[0] + pt9[1]) / 2, + ], axis=0) + + return pt2 + +def parse_pt2_from_pt68(pt68, use_lip=True): + ''' +face = {"keypoints": ['right cheekbone 1', 'right cheekbone 2', 'right cheek 1', 'right cheek 2', 'right cheek 3', 'right cheek 4', 'right cheek 5', 'right chin', 'chin center', +'left chin', 'left cheek 5', 'left cheek 4', 'left cheek 3', 'left cheek 2', 'left cheek 1', 'left cheekbone 2', 'left cheekbone 1', 'right eyebrow 1', 'right eyebrow 2', 'right eyebrow 3', +'right eyebrow 4', 'right eyebrow 5', 'left eyebrow 1', 'left eyebrow 2', 'left eyebrow 3', 'left eyebrow 4', 'left eyebrow 5', 'nasal bridge 1', 'nasal bridge 2', 'nasal bridge 3', 'nasal bridge 4', +'right nasal wing 1', 'right nasal wing 2', 'nasal wing center', 'left nasal wing 1', 'left nasal wing 2', 'right eye eye corner 1', 'right eye upper eyelid 1', 'right eye upper eyelid 2', +'right eye eye corner 2', 'right eye lower eyelid 2', 'right eye lower eyelid 1', 'left eye eye corner 1', 'left eye upper eyelid 1', 'left eye upper eyelid 2', 'left eye eye corner 2', 'left eye lower eyelid 2', +'left eye lower eyelid 1', 'right mouth corner', 'upper lip outer edge 1', 'upper lip outer edge 2', 'upper lip outer edge 3', 'upper lip outer edge 4', 'upper lip outer edge 5', 'left mouth corner', +'lower lip outer edge 5', 'lower lip outer edge 4', 'lower lip outer edge 3', 'lower lip outer edge 2', 'lower lip outer edge 1', 'upper lip inter edge 1', 'upper lip inter edge 2', 'upper lip inter edge 3', +'upper lip inter edge 4', 'upper lip inter edge 5', 'lower lip inter edge 3', 'lower lip inter edge 2', 'lower lip inter edge 1'], "skeleton": []} + + + ''' + if use_lip: + pt68 = np.stack([ + (pt68[42] + pt68[43] + pt68[44] + pt68[45] + pt68[46]+ pt68[47])/6, # left eye + (pt68[36] + pt68[37] + pt68[38] + pt68[39] + pt68[40] + pt68[41]) / 6, # right eye + (pt68[48] + pt68[54])/2 + + ], axis=0) + pt2 = np.stack([ + (pt68[0] + pt68[1]) / 2, + pt68[2] + ], axis=0) + else: + pt2 = np.stack([ + (pt68[42] + pt68[43] + pt68[44] + pt68[45] + pt68[46] + pt68[47]) / 6, # left eye + (pt68[36] + pt68[37] + pt68[38] + pt68[39] + pt68[40] + pt68[41]) / 6, # right eye ], axis=0) return pt2 @@ -145,9 +228,13 @@ def parse_pt2_from_pt_x(pts, use_lip=True): pt2 = parse_pt2_from_pt5(pts, use_lip=use_lip) elif pts.shape[0] == 203: pt2 = parse_pt2_from_pt203(pts, use_lip=use_lip) + elif pts.shape[0] == 478: + pt2 = parse_pt2_from_pt478(pts, use_lip=use_lip) elif pts.shape[0] > 101: # take the first 101 points pt2 = parse_pt2_from_pt101(pts[:101], use_lip=use_lip) + elif pts.shape[0] == 9: + pt2 = parse_pt2_from_pt9(pts, use_lip=use_lip) else: raise Exception(f'Unknow shape: {pts.shape}') @@ -350,13 +437,15 @@ def crop_image(img, pts: np.ndarray, **kwargs): dsize = kwargs.get('dsize', 224) scale = kwargs.get('scale', 1.5) # 1.5 | 1.6 vy_ratio = kwargs.get('vy_ratio', -0.1) # -0.0625 | -0.1 + vx_ratio = kwargs.get('vx_ratio', 0) M_INV, _ = _estimate_similar_transform_from_pts( pts, dsize=dsize, scale=scale, vy_ratio=vy_ratio, - flag_do_rot=kwargs.get('flag_do_rot', True), + vx_ratio=vx_ratio, + flag_do_rot=kwargs.get('rotate', True), ) if img is None: @@ -379,15 +468,13 @@ def crop_image(img, pts: np.ndarray, **kwargs): ret_dct = { 'M_o2c': M_o2c, # from the original image to the cropped image 3x3 'M_c2o': M_c2o, # from the cropped image to the original image 3x3 - 'img_crop': img_crop, # the cropped image 'pt_crop': pt_crop, # the landmarks of the cropped image } - return ret_dct + return ret_dct, img_crop def average_bbox_lst(bbox_lst): if len(bbox_lst) == 0: return None bbox_arr = np.array(bbox_lst) return np.mean(bbox_arr, axis=0).tolist() - diff --git a/liveportrait/utils/cropper.py b/liveportrait/utils/cropper.py index 024d9b7..ae82a79 100644 --- a/liveportrait/utils/cropper.py +++ b/liveportrait/utils/cropper.py @@ -1,49 +1,39 @@ # coding: utf-8 import numpy as np -import os.path as osp from typing import List, Union, Tuple from dataclasses import dataclass, field -import cv2; cv2.setNumThreads(0); cv2.ocl.setUseOpenCL(False) +import cv2#; cv2.setNumThreads(0); cv2.ocl.setUseOpenCL(False) from .landmark_runner import LandmarkRunner -from .face_analysis_diy import FaceAnalysisDIY -#from .helper import prefix -from .crop import crop_image, crop_image_by_bbox, parse_bbox_from_landmark, average_bbox_lst -#from .timer import Timer -from .rprint import rlog as log -from .io import load_image_rgb -#from .video import VideoWriter, get_fps, change_video_fps + +from .crop import crop_image import folder_paths import os script_directory = os.path.dirname(os.path.abspath(__file__)) -def make_abs_path(fn): - return osp.join(osp.dirname(osp.realpath(__file__)), fn) - - @dataclass class Trajectory: - start: int = -1 # 起始帧 闭区间 - end: int = -1 # 结束帧 闭区间 + start: int = -1 + end: int = -1 lmk_lst: Union[Tuple, List, np.ndarray] = field(default_factory=list) # lmk list bbox_lst: Union[Tuple, List, np.ndarray] = field(default_factory=list) # bbox list frame_rgb_lst: Union[Tuple, List, np.ndarray] = field(default_factory=list) # frame list frame_rgb_crop_lst: Union[Tuple, List, np.ndarray] = field(default_factory=list) # frame crop list - -class Cropper(object): - def __init__(self, provider, **kwargs) -> None: +class CropperInsightFace(object): + def __init__(self, **kwargs) -> None: device_id = kwargs.get('device_id', 0) + provider = kwargs.get('onnx_device', 'CPU') self.landmark_runner = LandmarkRunner( - #ckpt_path=make_abs_path('../../pretrained_weights/liveportrait/landmark.onnx'), ckpt_path=os.path.join(folder_paths.models_dir, 'liveportrait', 'landmark.onnx'), onnx_provider=provider, device_id=device_id ) self.landmark_runner.warmup() + from .face_analysis_diy import FaceAnalysisDIY self.face_analysis_wrapper = FaceAnalysisDIY( name='buffalo_l', root=os.path.join(folder_paths.models_dir, 'insightface'), @@ -52,21 +42,8 @@ class Cropper(object): self.face_analysis_wrapper.prepare(ctx_id=device_id, det_size=(512, 512)) self.face_analysis_wrapper.warmup() - self.crop_cfg = kwargs.get('crop_cfg', None) - - def update_config(self, user_args): - for k, v in user_args.items(): - if hasattr(self.crop_cfg, k): - setattr(self.crop_cfg, k, v) - - def crop_single_image(self, obj, **kwargs): - direction = kwargs.get('direction', 'large-small') - - # crop and align a single image - if isinstance(obj, str): - img_rgb = load_image_rgb(obj) - elif isinstance(obj, np.ndarray): - img_rgb = obj + def crop_single_image(self, img_rgb, dsize, scale, vy_ratio, vx_ratio, face_index, face_index_order, rotate): + direction = face_index_order src_face = self.face_analysis_wrapper.get( img_rgb, @@ -75,73 +52,85 @@ class Cropper(object): ) if len(src_face) == 0: - log('No face detected in the source image.') - raise Exception("No face detected in the source image!") - elif len(src_face) > 1: - log(f'More than one face detected in the image, only pick one face by rule {direction}.') + ret_dct = {} + return ret_dct - src_face = src_face[0] + src_face = src_face[face_index] # choose the index if multiple faces detected pts = src_face.landmark_2d_106 - + # crop the face - ret_dct = crop_image( + ret_dct, image_crop = crop_image( img_rgb, # ndarray pts, # 106x2 or Nx2 - dsize=kwargs.get('dsize', 512), - scale=kwargs.get('scale', 2.3), - vy_ratio=kwargs.get('vy_ratio', -0.15), + dsize=dsize, + scale=scale, + vy_ratio=vy_ratio, + vx_ratio=vx_ratio, + rotate=rotate ) # update a 256x256 version for network input or else - ret_dct['img_crop_256x256'] = cv2.resize(ret_dct['img_crop'], (256, 256), interpolation=cv2.INTER_AREA) - ret_dct['pt_crop_256x256'] = ret_dct['pt_crop'] * 256 / kwargs.get('dsize', 512) + cropped_image_256 = cv2.resize(image_crop, (256, 256), interpolation=cv2.INTER_AREA) + ret_dct['pt_crop_256x256'] = ret_dct['pt_crop'] * 256 / dsize + input_image_size = img_rgb.shape[:2] + ret_dct['input_image_size'] = input_image_size + recon_ret = self.landmark_runner.run(img_rgb, pts) lmk = recon_ret['pts'] ret_dct['lmk_crop'] = lmk - return ret_dct + return ret_dct, cropped_image_256 + +class CropperMediaPipe(object): + def __init__(self, **kwargs) -> None: + device_id = kwargs.get('device_id', 0) + provider = kwargs.get('onnx_device', 'CPU') + self.landmark_runner = LandmarkRunner( + ckpt_path=os.path.join(folder_paths.models_dir, 'liveportrait', 'landmark.onnx'), + onnx_provider=provider, + device_id=device_id + ) + self.landmark_runner.warmup() + from ...media_pipe.mp_utils import LMKExtractor + self.lmk_extractor = LMKExtractor() - def get_retargeting_lmk_info(self, driving_rgb_lst): - # TODO: implement a tracking-based version - driving_lmk_lst = [] - for driving_image in driving_rgb_lst: - ret_dct = self.crop_single_image(driving_image) - driving_lmk_lst.append(ret_dct['lmk_crop']) - return driving_lmk_lst + def crop_single_image(self, img_rgb, dsize, scale, vy_ratio, vx_ratio, face_index, face_index_order, rotate): + + face_result = self.lmk_extractor(img_rgb) - def make_video_clip(self, driving_rgb_lst, output_path, output_fps=30, **kwargs): - trajectory = Trajectory() - direction = kwargs.get('direction', 'large-small') - for idx, driving_image in enumerate(driving_rgb_lst): - if idx == 0 or trajectory.start == -1: - src_face = self.face_analysis_wrapper.get( - driving_image, - flag_do_landmark_2d_106=True, - direction=direction - ) - if len(src_face) == 0: - # No face detected in the driving_image - continue - elif len(src_face) > 1: - log(f'More than one face detected in the driving frame_{idx}, only pick one face by rule {direction}.') - src_face = src_face[0] - pts = src_face.landmark_2d_106 - lmk_203 = self.landmark_runner(driving_image, pts)['pts'] - trajectory.start, trajectory.end = idx, idx - else: - lmk_203 = self.face_recon_wrapper(driving_image, trajectory.lmk_lst[-1])['pts'] - trajectory.end = idx + if face_result is None: + ret_dct = {} + cropped_image_256 = None + return ret_dct, cropped_image_256 - trajectory.lmk_lst.append(lmk_203) - ret_bbox = parse_bbox_from_landmark(lmk_203, scale=self.crop_cfg.globalscale, vy_ratio=elf.crop_cfg.vy_ratio)['bbox'] - bbox = [ret_bbox[0, 0], ret_bbox[0, 1], ret_bbox[2, 0], ret_bbox[2, 1]] # 4, - trajectory.bbox_lst.append(bbox) # bbox - trajectory.frame_rgb_lst.append(driving_image) + face_landmarks = face_result[face_index] - global_bbox = average_bbox_lst(trajectory.bbox_lst) - for idx, (frame_rgb, lmk) in enumerate(zip(trajectory.frame_rgb_lst, trajectory.lmk_lst)): - ret_dct = crop_image_by_bbox( - frame_rgb, global_bbox, lmk=lmk, - dsize=self.video_crop_cfg.dsize, flag_rot=self.video_crop_cfg.flag_rot, borderValue=self.video_crop_cfg.borderValue - ) - frame_rgb_crop = ret_dct['img_crop'] + lmks = [] + for index in range(len(face_landmarks)): + x = face_landmarks[index].x * img_rgb.shape[1] + y = face_landmarks[index].y * img_rgb.shape[0] + lmks.append([x, y]) + pts = np.array(lmks) + + # crop the face + ret_dct, image_crop = crop_image( + img_rgb, # ndarray + pts, # 106x2 or Nx2 + dsize=dsize, + scale=scale, + vy_ratio=vy_ratio, + vx_ratio=vx_ratio, + rotate=rotate + ) + # update a 256x256 version for network input or else + cropped_image_256 = cv2.resize(image_crop, (256, 256), interpolation=cv2.INTER_AREA) + ret_dct['pt_crop_256x256'] = ret_dct['pt_crop'] * 256 / dsize + + input_image_size = img_rgb.shape[:2] + ret_dct['input_image_size'] = input_image_size + + recon_ret = self.landmark_runner.run(img_rgb, pts) + lmk = recon_ret['pts'] + ret_dct['lmk_crop'] = lmk + + return ret_dct, cropped_image_256 \ No newline at end of file diff --git a/liveportrait/utils/face_analysis_diy.py b/liveportrait/utils/face_analysis_diy.py index 376334f..ac29ace 100644 --- a/liveportrait/utils/face_analysis_diy.py +++ b/liveportrait/utils/face_analysis_diy.py @@ -1,16 +1,30 @@ # coding: utf-8 """ -face detectoin and alignment using InsightFace +face detection and alignment using InsightFace """ +from insightface.utils import transform + +#patch Insightface function to get rid of the annoying warnings +def patched_estimate_affine_matrix_3d23d(X, Y): + ''' Using least-squares solution + Args: + X: [n, 3]. 3d points(fixed) + Y: [n, 3]. corresponding 3d points(moving). Y = PX + Returns: + P_Affine: (3, 4). Affine camera matrix (the third row is [0, 0, 0, 1]). + ''' + X_homo = np.hstack((X, np.ones([X.shape[0],1]))) # n x 4 + P = np.linalg.lstsq(X_homo, Y, rcond=None)[0].T # Affine matrix. 3 x 4 + return P + +transform.estimate_affine_matrix_3d23d = patched_estimate_affine_matrix_3d23d import numpy as np -from .rprint import rlog as log from insightface.app import FaceAnalysis from insightface.app.common import Face from .timer import Timer - def sort_by_direction(faces, direction: str = 'large-small', face_center=None): if len(faces) <= 0: return faces @@ -76,4 +90,4 @@ class FaceAnalysisDIY(FaceAnalysis): self.get(img_bgr) elapse = self.timer.toc() - log(f'FaceAnalysisDIY warmup time: {elapse:.3f}s') + print(f'FaceAnalysisDIY warmup time: {elapse:.3f}s') diff --git a/liveportrait/utils/filter.py b/liveportrait/utils/filter.py new file mode 100644 index 0000000..a7393ea --- /dev/null +++ b/liveportrait/utils/filter.py @@ -0,0 +1,30 @@ +import torch +import numpy as np +from pykalman import KalmanFilter + + +def smooth(x_d_lst, shape, device, observation_variance=3e-6, process_variance=1e-5): + # Reshape x_d_lst, skipping None values + x_d_lst_reshape = [x.reshape(-1) for x in x_d_lst if x is not None] + + if not x_d_lst_reshape: # Check if x_d_lst_reshape is empty after filtering + return [None] * len(x_d_lst) # Return a list of Nones with the same length as x_d_lst + + x_d_stacked = np.vstack(x_d_lst_reshape) + + kf = KalmanFilter( + initial_state_mean=x_d_stacked[0], + n_dim_obs=x_d_stacked.shape[1], + transition_covariance=process_variance * np.eye(x_d_stacked.shape[1]), + observation_covariance=observation_variance * np.eye(x_d_stacked.shape[1]) + ) + + smoothed_state_means, _ = kf.smooth(x_d_stacked) + + # Initialize an iterator for smoothed_state_means + smoothed_states_iter = iter(smoothed_state_means) + + # Create x_d_lst_smooth, inserting None for each None encountered in the original list + x_d_lst_smooth = [torch.tensor(next(smoothed_states_iter).reshape(shape[-2:]), dtype=torch.float32, device=device) if x is not None else None for x in x_d_lst] + + return x_d_lst_smooth \ No newline at end of file diff --git a/liveportrait/utils/helper.py b/liveportrait/utils/helper.py index fe1df7a..98b2cc5 100644 --- a/liveportrait/utils/helper.py +++ b/liveportrait/utils/helper.py @@ -4,58 +4,15 @@ utility functions and classes to handle feature extraction and model loading """ -import os import os.path as osp import cv2 import torch from collections import OrderedDict -def suffix(filename): - """a.jpg -> jpg""" - pos = filename.rfind(".") - if pos == -1: - return "" - return filename[pos + 1:] - - -def prefix(filename): - """a.jpg -> a""" - pos = filename.rfind(".") - if pos == -1: - return filename - return filename[:pos] - - -def basename(filename): - """a/b/c.jpg -> c""" - return prefix(osp.basename(filename)) - - -def is_video(file_path): - if file_path.lower().endswith((".mp4", ".mov", ".avi", ".webm")) or osp.isdir(file_path): - return True - return False - -def is_template(file_path): - if file_path.endswith(".pkl"): - return True - return False - - -def mkdir(d, log=False): - # return self-assined `d`, for one line code - if not osp.exists(d): - os.makedirs(d, exist_ok=True) - if log: - print(f"Make dir: {d}") - return d - - def squeeze_tensor_to_numpy(tensor): out = tensor.data.squeeze(0).cpu().numpy() return out - def dct2cuda(dct: dict, device_id: int): for key in dct: dct[key] = torch.tensor(dct[key]).to(device_id) @@ -95,11 +52,6 @@ def calculate_transformation(config, s_kp_info, t_0_kp_info, t_i_kp_info, R_s, R new_scale = s_kp_info['scale'] * (t_i_kp_info['scale'] / t_0_kp_info['scale']) return new_rotation, new_expression, new_translation, new_scale -def load_description(fp): - with open(fp, 'r', encoding='utf-8') as f: - content = f.read() - return content - def resize_to_limit(img, max_dim=1280, n=2): h, w = img.shape[:2] diff --git a/liveportrait/utils/io.py b/liveportrait/utils/io.py deleted file mode 100644 index f930c48..0000000 --- a/liveportrait/utils/io.py +++ /dev/null @@ -1,97 +0,0 @@ -# coding: utf-8 - -import os -from glob import glob -import os.path as osp -import imageio -import numpy as np -import cv2; cv2.setNumThreads(0); cv2.ocl.setUseOpenCL(False) - - -def load_image_rgb(image_path: str): - if not osp.exists(image_path): - raise FileNotFoundError(f"Image not found: {image_path}") - img = cv2.imread(image_path, cv2.IMREAD_COLOR) - return cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - - -def load_driving_info(driving_info): - driving_video_ori = [] - - def load_images_from_directory(directory): - image_paths = sorted(glob(osp.join(directory, '*.png')) + glob(osp.join(directory, '*.jpg'))) - return [load_image_rgb(im_path) for im_path in image_paths] - - def load_images_from_video(file_path): - reader = imageio.get_reader(file_path) - return [image for idx, image in enumerate(reader)] - - if osp.isdir(driving_info): - driving_video_ori = load_images_from_directory(driving_info) - elif osp.isfile(driving_info): - driving_video_ori = load_images_from_video(driving_info) - - return driving_video_ori - - -def contiguous(obj): - if not obj.flags.c_contiguous: - obj = obj.copy(order="C") - return obj - - -def _resize_to_limit(img: np.ndarray, max_dim=1920, n=2): - """ - ajust the size of the image so that the maximum dimension does not exceed max_dim, and the width and the height of the image are multiples of n. - :param img: the image to be processed. - :param max_dim: the maximum dimension constraint. - :param n: the number that needs to be multiples of. - :return: the adjusted image. - """ - h, w = img.shape[:2] - - # ajust the size of the image according to the maximum dimension - if max_dim > 0 and max(h, w) > max_dim: - if h > w: - new_h = max_dim - new_w = int(w * (max_dim / h)) - else: - new_w = max_dim - new_h = int(h * (max_dim / w)) - img = cv2.resize(img, (new_w, new_h)) - - # ensure that the image dimensions are multiples of n - n = max(n, 1) - new_h = img.shape[0] - (img.shape[0] % n) - new_w = img.shape[1] - (img.shape[1] % n) - - if new_h == 0 or new_w == 0: - # when the width or height is less than n, no need to process - return img - - if new_h != img.shape[0] or new_w != img.shape[1]: - img = img[:new_h, :new_w] - - return img - - -def load_img_online(obj, mode="bgr", **kwargs): - max_dim = kwargs.get("max_dim", 1920) - n = kwargs.get("n", 2) - if isinstance(obj, str): - if mode.lower() == "gray": - img = cv2.imread(obj, cv2.IMREAD_GRAYSCALE) - else: - img = cv2.imread(obj, cv2.IMREAD_COLOR) - else: - img = obj - - # Resize image to satisfy constraints - img = _resize_to_limit(img, max_dim=max_dim, n=n) - - if mode.lower() == "bgr": - return contiguous(img) - elif mode.lower() == "rgb": - return contiguous(img[..., ::-1]) - else: - raise Exception(f"Unknown mode {mode}") diff --git a/liveportrait/utils/landmark_runner.py b/liveportrait/utils/landmark_runner.py index 7b0dcbe..00f348b 100644 --- a/liveportrait/utils/landmark_runner.py +++ b/liveportrait/utils/landmark_runner.py @@ -1,19 +1,12 @@ # coding: utf-8 -import os.path as osp -import cv2; cv2.setNumThreads(0); cv2.ocl.setUseOpenCL(False) +import cv2#; cv2.setNumThreads(0); cv2.ocl.setUseOpenCL(False) import torch import numpy as np import onnxruntime from .timer import Timer -from .rprint import rlog from .crop import crop_image, _transform_pts - -def make_abs_path(fn): - return osp.join(osp.dirname(osp.realpath(__file__)), fn) - - def to_ndarray(obj): if isinstance(obj, torch.Tensor): return obj.cpu().numpy() @@ -22,12 +15,11 @@ def to_ndarray(obj): else: return np.array(obj) - class LandmarkRunner(object): """landmark runner""" def __init__(self, **kwargs): ckpt_path = kwargs.get('ckpt_path') - onnx_provider = kwargs.get('onnx_provider', 'cuda') # 默认用cuda + onnx_provider = kwargs.get('onnx_provider', 'cuda') device_id = kwargs.get('device_id', 0) self.dsize = kwargs.get('dsize', 224) self.timer = Timer() @@ -40,7 +32,7 @@ class LandmarkRunner(object): ) else: opts = onnxruntime.SessionOptions() - opts.intra_op_num_threads = 4 # 默认线程数为 4 + opts.intra_op_num_threads = 4 self.session = onnxruntime.InferenceSession( ckpt_path, providers=['CPUExecutionProvider'], sess_options=opts @@ -52,8 +44,7 @@ class LandmarkRunner(object): def run(self, img_rgb: np.ndarray, lmk=None): if lmk is not None: - crop_dct = crop_image(img_rgb, lmk, dsize=self.dsize, scale=1.5, vy_ratio=-0.1) - img_crop_rgb = crop_dct['img_crop'] + crop_dct, img_crop_rgb = crop_image(img_rgb, lmk, dsize=self.dsize, scale=1.5, vy_ratio=-0.1) else: img_crop_rgb = cv2.resize(img_rgb, (self.dsize, self.dsize)) scale = max(img_rgb.shape[:2]) / self.dsize @@ -72,13 +63,12 @@ class LandmarkRunner(object): pts = to_ndarray(out_pts[0]).reshape(-1, 2) * self.dsize # scale to 0-224 pts = _transform_pts(pts, M=crop_dct['M_c2o']) - + del crop_dct, img_crop_rgb return { 'pts': pts, # 2d landmarks 203 points } def warmup(self): - # 构造dummy image进行warmup self.timer.tic() dummy_image = np.zeros((1, 3, self.dsize, self.dsize), dtype=np.float32) @@ -86,4 +76,4 @@ class LandmarkRunner(object): _ = self._run(dummy_image) elapse = self.timer.toc() - rlog(f'LandmarkRunner warmup time: {elapse:.3f}s') + print(f'LandmarkRunner warmup time: {elapse:.3f}s') diff --git a/liveportrait/utils/rprint.py b/liveportrait/utils/rprint.py deleted file mode 100644 index c43a42f..0000000 --- a/liveportrait/utils/rprint.py +++ /dev/null @@ -1,16 +0,0 @@ -# coding: utf-8 - -""" -custom print and log functions -""" - -__all__ = ['rprint', 'rlog'] - -try: - from rich.console import Console - console = Console() - rprint = console.print - rlog = console.log -except: - rprint = print - rlog = print diff --git a/liveportrait/utils/video.py b/liveportrait/utils/video.py deleted file mode 100644 index 2df7aee..0000000 --- a/liveportrait/utils/video.py +++ /dev/null @@ -1,139 +0,0 @@ -# coding: utf-8 - -""" -functions for processing video -""" - -import os.path as osp -import numpy as np -import subprocess -import imageio -import cv2 - -from tqdm import tqdm -from .helper import prefix -from .rprint import rprint as print - - -def exec_cmd(cmd): - subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - -def images2video(images, wfp, **kwargs): - fps = kwargs.get('fps', 30) - video_format = kwargs.get('format', 'mp4') # default is mp4 format - codec = kwargs.get('codec', 'libx264') # default is libx264 encoding - quality = kwargs.get('quality') # video quality - pixelformat = kwargs.get('pixelformat', 'yuv420p') # video pixel format - image_mode = kwargs.get('image_mode', 'rgb') - macro_block_size = kwargs.get('macro_block_size', 2) - ffmpeg_params = ['-crf', str(kwargs.get('crf', 18))] - - writer = imageio.get_writer( - wfp, fps=fps, format=video_format, - codec=codec, quality=quality, ffmpeg_params=ffmpeg_params, pixelformat=pixelformat, macro_block_size=macro_block_size - ) - - n = len(images) - for i in tqdm(range(n), desc='writing', transient=True): - if image_mode.lower() == 'bgr': - writer.append_data(images[i][..., ::-1]) - else: - writer.append_data(images[i]) - - writer.close() - - # print(f':smiley: Dump to {wfp}\n', style="bold green") - print(f'Dump to {wfp}\n') - - -def video2gif(video_fp, fps=30, size=256): - if osp.exists(video_fp): - d = osp.split(video_fp)[0] - fn = prefix(osp.basename(video_fp)) - palette_wfp = osp.join(d, 'palette.png') - gif_wfp = osp.join(d, f'{fn}.gif') - # generate the palette - cmd = f'ffmpeg -i {video_fp} -vf "fps={fps},scale={size}:-1:flags=lanczos,palettegen" {palette_wfp} -y' - exec_cmd(cmd) - # use the palette to generate the gif - cmd = f'ffmpeg -i {video_fp} -i {palette_wfp} -filter_complex "fps={fps},scale={size}:-1:flags=lanczos[x];[x][1:v]paletteuse" {gif_wfp} -y' - exec_cmd(cmd) - else: - print(f'video_fp: {video_fp} not exists!') - - -def merge_audio_video(video_fp, audio_fp, wfp): - if osp.exists(video_fp) and osp.exists(audio_fp): - cmd = f'ffmpeg -i {video_fp} -i {audio_fp} -c:v copy -c:a aac {wfp} -y' - exec_cmd(cmd) - print(f'merge {video_fp} and {audio_fp} to {wfp}') - else: - print(f'video_fp: {video_fp} or audio_fp: {audio_fp} not exists!') - - -def blend(img: np.ndarray, mask: np.ndarray, background_color=(255, 255, 255)): - mask_float = mask.astype(np.float32) / 255. - background_color = np.array(background_color).reshape([1, 1, 3]) - bg = np.ones_like(img) * background_color - img = np.clip(mask_float * img + (1 - mask_float) * bg, 0, 255).astype(np.uint8) - return img - - -def concat_frames(I_p_lst, driving_rgb_lst, img_rgb): - # TODO: add more concat style, e.g., left-down corner driving - out_lst = [] - for idx, _ in tqdm(enumerate(I_p_lst), total=len(I_p_lst), desc='Concatenating result...'): - source_image_drived = I_p_lst[idx] - image_drive = driving_rgb_lst[idx] - - # resize images to match source_image_drived shape - h, w, _ = source_image_drived.shape - image_drive_resized = cv2.resize(image_drive, (w, h)) - img_rgb_resized = cv2.resize(img_rgb, (w, h)) - - # concatenate images horizontally - frame = np.concatenate((image_drive_resized, img_rgb_resized, source_image_drived), axis=1) - out_lst.append(frame) - return out_lst - - -class VideoWriter: - def __init__(self, **kwargs): - self.fps = kwargs.get('fps', 30) - self.wfp = kwargs.get('wfp', 'video.mp4') - self.video_format = kwargs.get('format', 'mp4') - self.codec = kwargs.get('codec', 'libx264') - self.quality = kwargs.get('quality') - self.pixelformat = kwargs.get('pixelformat', 'yuv420p') - self.image_mode = kwargs.get('image_mode', 'rgb') - self.ffmpeg_params = kwargs.get('ffmpeg_params') - - self.writer = imageio.get_writer( - self.wfp, fps=self.fps, format=self.video_format, - codec=self.codec, quality=self.quality, - ffmpeg_params=self.ffmpeg_params, pixelformat=self.pixelformat - ) - - def write(self, image): - if self.image_mode.lower() == 'bgr': - self.writer.append_data(image[..., ::-1]) - else: - self.writer.append_data(image) - - def close(self): - if self.writer is not None: - self.writer.close() - - -def change_video_fps(input_file, output_file, fps=20, codec='libx264', crf=5): - cmd = f"ffmpeg -i {input_file} -c:v {codec} -crf {crf} -r {fps} {output_file} -y" - exec_cmd(cmd) - - -def get_fps(filepath): - import ffmpeg - probe = ffmpeg.probe(filepath) - video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None) - fps = eval(video_stream['avg_frame_rate']) - return fps diff --git a/media_pipe/__init__.py b/media_pipe/__init__.py new file mode 100644 index 0000000..877471d --- /dev/null +++ b/media_pipe/__init__.py @@ -0,0 +1 @@ +from .mp_utils import LMKExtractor \ No newline at end of file diff --git a/media_pipe/face_landmark.py b/media_pipe/face_landmark.py new file mode 100644 index 0000000..b6580cb --- /dev/null +++ b/media_pipe/face_landmark.py @@ -0,0 +1,3305 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe face landmarker task.""" + +import dataclasses +import enum +from typing import Callable, Mapping, Optional, List + +import numpy as np + +from mediapipe.framework.formats import classification_pb2 +from mediapipe.framework.formats import landmark_pb2 +from mediapipe.framework.formats import matrix_data_pb2 +from mediapipe.python import packet_creator +from mediapipe.python import packet_getter +from mediapipe.python._framework_bindings import image as image_module +from mediapipe.python._framework_bindings import packet as packet_module +# pylint: disable=unused-import +from mediapipe.tasks.cc.vision.face_geometry.proto import face_geometry_pb2 +# pylint: enable=unused-import +from mediapipe.tasks.cc.vision.face_landmarker.proto import face_landmarker_graph_options_pb2 +from mediapipe.tasks.python.components.containers import category as category_module +from mediapipe.tasks.python.components.containers import landmark as landmark_module +from mediapipe.tasks.python.core import base_options as base_options_module +from mediapipe.tasks.python.core import task_info as task_info_module +from mediapipe.tasks.python.core.optional_dependencies import doc_controls +from mediapipe.tasks.python.vision.core import base_vision_task_api +from mediapipe.tasks.python.vision.core import image_processing_options as image_processing_options_module +from mediapipe.tasks.python.vision.core import vision_task_running_mode as running_mode_module + +_BaseOptions = base_options_module.BaseOptions +_FaceLandmarkerGraphOptionsProto = ( + face_landmarker_graph_options_pb2.FaceLandmarkerGraphOptions +) +_LayoutEnum = matrix_data_pb2.MatrixData.Layout +_RunningMode = running_mode_module.VisionTaskRunningMode +_ImageProcessingOptions = image_processing_options_module.ImageProcessingOptions +_TaskInfo = task_info_module.TaskInfo + +_IMAGE_IN_STREAM_NAME = 'image_in' +_IMAGE_OUT_STREAM_NAME = 'image_out' +_IMAGE_TAG = 'IMAGE' +_NORM_RECT_STREAM_NAME = 'norm_rect_in' +_NORM_RECT_TAG = 'NORM_RECT' +_NORM_LANDMARKS_STREAM_NAME = 'norm_landmarks' +_NORM_LANDMARKS_TAG = 'NORM_LANDMARKS' +_BLENDSHAPES_STREAM_NAME = 'blendshapes' +_BLENDSHAPES_TAG = 'BLENDSHAPES' +_FACE_GEOMETRY_STREAM_NAME = 'face_geometry' +_FACE_GEOMETRY_TAG = 'FACE_GEOMETRY' +_TASK_GRAPH_NAME = 'mediapipe.tasks.vision.face_landmarker.FaceLandmarkerGraph' +_MICRO_SECONDS_PER_MILLISECOND = 1000 + + +class Blendshapes(enum.IntEnum): + """The 52 blendshape coefficients.""" + + NEUTRAL = 0 + BROW_DOWN_LEFT = 1 + BROW_DOWN_RIGHT = 2 + BROW_INNER_UP = 3 + BROW_OUTER_UP_LEFT = 4 + BROW_OUTER_UP_RIGHT = 5 + CHEEK_PUFF = 6 + CHEEK_SQUINT_LEFT = 7 + CHEEK_SQUINT_RIGHT = 8 + EYE_BLINK_LEFT = 9 + EYE_BLINK_RIGHT = 10 + EYE_LOOK_DOWN_LEFT = 11 + EYE_LOOK_DOWN_RIGHT = 12 + EYE_LOOK_IN_LEFT = 13 + EYE_LOOK_IN_RIGHT = 14 + EYE_LOOK_OUT_LEFT = 15 + EYE_LOOK_OUT_RIGHT = 16 + EYE_LOOK_UP_LEFT = 17 + EYE_LOOK_UP_RIGHT = 18 + EYE_SQUINT_LEFT = 19 + EYE_SQUINT_RIGHT = 20 + EYE_WIDE_LEFT = 21 + EYE_WIDE_RIGHT = 22 + JAW_FORWARD = 23 + JAW_LEFT = 24 + JAW_OPEN = 25 + JAW_RIGHT = 26 + MOUTH_CLOSE = 27 + MOUTH_DIMPLE_LEFT = 28 + MOUTH_DIMPLE_RIGHT = 29 + MOUTH_FROWN_LEFT = 30 + MOUTH_FROWN_RIGHT = 31 + MOUTH_FUNNEL = 32 + MOUTH_LEFT = 33 + MOUTH_LOWER_DOWN_LEFT = 34 + MOUTH_LOWER_DOWN_RIGHT = 35 + MOUTH_PRESS_LEFT = 36 + MOUTH_PRESS_RIGHT = 37 + MOUTH_PUCKER = 38 + MOUTH_RIGHT = 39 + MOUTH_ROLL_LOWER = 40 + MOUTH_ROLL_UPPER = 41 + MOUTH_SHRUG_LOWER = 42 + MOUTH_SHRUG_UPPER = 43 + MOUTH_SMILE_LEFT = 44 + MOUTH_SMILE_RIGHT = 45 + MOUTH_STRETCH_LEFT = 46 + MOUTH_STRETCH_RIGHT = 47 + MOUTH_UPPER_UP_LEFT = 48 + MOUTH_UPPER_UP_RIGHT = 49 + NOSE_SNEER_LEFT = 50 + NOSE_SNEER_RIGHT = 51 + + +class FaceLandmarksConnections: + """The connections between face landmarks.""" + + @dataclasses.dataclass + class Connection: + """The connection class for face landmarks.""" + + start: int + end: int + + FACE_LANDMARKS_LIPS: List[Connection] = [ + Connection(61, 146), + Connection(146, 91), + Connection(91, 181), + Connection(181, 84), + Connection(84, 17), + Connection(17, 314), + Connection(314, 405), + Connection(405, 321), + Connection(321, 375), + Connection(375, 291), + Connection(61, 185), + Connection(185, 40), + Connection(40, 39), + Connection(39, 37), + Connection(37, 0), + Connection(0, 267), + Connection(267, 269), + Connection(269, 270), + Connection(270, 409), + Connection(409, 291), + Connection(78, 95), + Connection(95, 88), + Connection(88, 178), + Connection(178, 87), + Connection(87, 14), + Connection(14, 317), + Connection(317, 402), + Connection(402, 318), + Connection(318, 324), + Connection(324, 308), + Connection(78, 191), + Connection(191, 80), + Connection(80, 81), + Connection(81, 82), + Connection(82, 13), + Connection(13, 312), + Connection(312, 311), + Connection(311, 310), + Connection(310, 415), + Connection(415, 308), + ] + + FACE_LANDMARKS_LEFT_EYE: List[Connection] = [ + Connection(263, 249), + Connection(249, 390), + Connection(390, 373), + Connection(373, 374), + Connection(374, 380), + Connection(380, 381), + Connection(381, 382), + Connection(382, 362), + Connection(263, 466), + Connection(466, 388), + Connection(388, 387), + Connection(387, 386), + Connection(386, 385), + Connection(385, 384), + Connection(384, 398), + Connection(398, 362), + ] + + FACE_LANDMARKS_LEFT_EYEBROW: List[Connection] = [ + Connection(276, 283), + Connection(283, 282), + Connection(282, 295), + Connection(295, 285), + Connection(300, 293), + Connection(293, 334), + Connection(334, 296), + Connection(296, 336), + ] + + FACE_LANDMARKS_LEFT_IRIS: List[Connection] = [ + Connection(474, 475), + Connection(475, 476), + Connection(476, 477), + Connection(477, 474), + ] + + FACE_LANDMARKS_RIGHT_EYE: List[Connection] = [ + Connection(33, 7), + Connection(7, 163), + Connection(163, 144), + Connection(144, 145), + Connection(145, 153), + Connection(153, 154), + Connection(154, 155), + Connection(155, 133), + Connection(33, 246), + Connection(246, 161), + Connection(161, 160), + Connection(160, 159), + Connection(159, 158), + Connection(158, 157), + Connection(157, 173), + Connection(173, 133), + ] + + FACE_LANDMARKS_RIGHT_EYEBROW: List[Connection] = [ + Connection(46, 53), + Connection(53, 52), + Connection(52, 65), + Connection(65, 55), + Connection(70, 63), + Connection(63, 105), + Connection(105, 66), + Connection(66, 107), + ] + + FACE_LANDMARKS_RIGHT_IRIS: List[Connection] = [ + Connection(469, 470), + Connection(470, 471), + Connection(471, 472), + Connection(472, 469), + ] + + FACE_LANDMARKS_FACE_OVAL: List[Connection] = [ + Connection(10, 338), + Connection(338, 297), + Connection(297, 332), + Connection(332, 284), + Connection(284, 251), + Connection(251, 389), + Connection(389, 356), + Connection(356, 454), + Connection(454, 323), + Connection(323, 361), + Connection(361, 288), + Connection(288, 397), + Connection(397, 365), + Connection(365, 379), + Connection(379, 378), + Connection(378, 400), + Connection(400, 377), + Connection(377, 152), + Connection(152, 148), + Connection(148, 176), + Connection(176, 149), + Connection(149, 150), + Connection(150, 136), + Connection(136, 172), + Connection(172, 58), + Connection(58, 132), + Connection(132, 93), + Connection(93, 234), + Connection(234, 127), + Connection(127, 162), + Connection(162, 21), + Connection(21, 54), + Connection(54, 103), + Connection(103, 67), + Connection(67, 109), + Connection(109, 10), + ] + + FACE_LANDMARKS_CONTOURS: List[Connection] = ( + FACE_LANDMARKS_LIPS + + FACE_LANDMARKS_LEFT_EYE + + FACE_LANDMARKS_LEFT_EYEBROW + + FACE_LANDMARKS_RIGHT_EYE + + FACE_LANDMARKS_RIGHT_EYEBROW + + FACE_LANDMARKS_FACE_OVAL + ) + + FACE_LANDMARKS_TESSELATION: List[Connection] = [ + Connection(127, 34), + Connection(34, 139), + Connection(139, 127), + Connection(11, 0), + Connection(0, 37), + Connection(37, 11), + Connection(232, 231), + Connection(231, 120), + Connection(120, 232), + Connection(72, 37), + Connection(37, 39), + Connection(39, 72), + Connection(128, 121), + Connection(121, 47), + Connection(47, 128), + Connection(232, 121), + Connection(121, 128), + Connection(128, 232), + Connection(104, 69), + Connection(69, 67), + Connection(67, 104), + Connection(175, 171), + Connection(171, 148), + Connection(148, 175), + Connection(118, 50), + Connection(50, 101), + Connection(101, 118), + Connection(73, 39), + Connection(39, 40), + Connection(40, 73), + Connection(9, 151), + Connection(151, 108), + Connection(108, 9), + Connection(48, 115), + Connection(115, 131), + Connection(131, 48), + Connection(194, 204), + Connection(204, 211), + Connection(211, 194), + Connection(74, 40), + Connection(40, 185), + Connection(185, 74), + Connection(80, 42), + Connection(42, 183), + Connection(183, 80), + Connection(40, 92), + Connection(92, 186), + Connection(186, 40), + Connection(230, 229), + Connection(229, 118), + Connection(118, 230), + Connection(202, 212), + Connection(212, 214), + Connection(214, 202), + Connection(83, 18), + Connection(18, 17), + Connection(17, 83), + Connection(76, 61), + Connection(61, 146), + Connection(146, 76), + Connection(160, 29), + Connection(29, 30), + Connection(30, 160), + Connection(56, 157), + Connection(157, 173), + Connection(173, 56), + Connection(106, 204), + Connection(204, 194), + Connection(194, 106), + Connection(135, 214), + Connection(214, 192), + Connection(192, 135), + Connection(203, 165), + Connection(165, 98), + Connection(98, 203), + Connection(21, 71), + Connection(71, 68), + Connection(68, 21), + Connection(51, 45), + Connection(45, 4), + Connection(4, 51), + Connection(144, 24), + Connection(24, 23), + Connection(23, 144), + Connection(77, 146), + Connection(146, 91), + Connection(91, 77), + Connection(205, 50), + Connection(50, 187), + Connection(187, 205), + Connection(201, 200), + Connection(200, 18), + Connection(18, 201), + Connection(91, 106), + Connection(106, 182), + Connection(182, 91), + Connection(90, 91), + Connection(91, 181), + Connection(181, 90), + Connection(85, 84), + Connection(84, 17), + Connection(17, 85), + Connection(206, 203), + Connection(203, 36), + Connection(36, 206), + Connection(148, 171), + Connection(171, 140), + Connection(140, 148), + Connection(92, 40), + Connection(40, 39), + Connection(39, 92), + Connection(193, 189), + Connection(189, 244), + Connection(244, 193), + Connection(159, 158), + Connection(158, 28), + Connection(28, 159), + Connection(247, 246), + Connection(246, 161), + Connection(161, 247), + Connection(236, 3), + Connection(3, 196), + Connection(196, 236), + Connection(54, 68), + Connection(68, 104), + Connection(104, 54), + Connection(193, 168), + Connection(168, 8), + Connection(8, 193), + Connection(117, 228), + Connection(228, 31), + Connection(31, 117), + Connection(189, 193), + Connection(193, 55), + Connection(55, 189), + Connection(98, 97), + Connection(97, 99), + Connection(99, 98), + Connection(126, 47), + Connection(47, 100), + Connection(100, 126), + Connection(166, 79), + Connection(79, 218), + Connection(218, 166), + Connection(155, 154), + Connection(154, 26), + Connection(26, 155), + Connection(209, 49), + Connection(49, 131), + Connection(131, 209), + Connection(135, 136), + Connection(136, 150), + Connection(150, 135), + Connection(47, 126), + Connection(126, 217), + Connection(217, 47), + Connection(223, 52), + Connection(52, 53), + Connection(53, 223), + Connection(45, 51), + Connection(51, 134), + Connection(134, 45), + Connection(211, 170), + Connection(170, 140), + Connection(140, 211), + Connection(67, 69), + Connection(69, 108), + Connection(108, 67), + Connection(43, 106), + Connection(106, 91), + Connection(91, 43), + Connection(230, 119), + Connection(119, 120), + Connection(120, 230), + Connection(226, 130), + Connection(130, 247), + Connection(247, 226), + Connection(63, 53), + Connection(53, 52), + Connection(52, 63), + Connection(238, 20), + Connection(20, 242), + Connection(242, 238), + Connection(46, 70), + Connection(70, 156), + Connection(156, 46), + Connection(78, 62), + Connection(62, 96), + Connection(96, 78), + Connection(46, 53), + Connection(53, 63), + Connection(63, 46), + Connection(143, 34), + Connection(34, 227), + Connection(227, 143), + Connection(123, 117), + Connection(117, 111), + Connection(111, 123), + Connection(44, 125), + Connection(125, 19), + Connection(19, 44), + Connection(236, 134), + Connection(134, 51), + Connection(51, 236), + Connection(216, 206), + Connection(206, 205), + Connection(205, 216), + Connection(154, 153), + Connection(153, 22), + Connection(22, 154), + Connection(39, 37), + Connection(37, 167), + Connection(167, 39), + Connection(200, 201), + Connection(201, 208), + Connection(208, 200), + Connection(36, 142), + Connection(142, 100), + Connection(100, 36), + Connection(57, 212), + Connection(212, 202), + Connection(202, 57), + Connection(20, 60), + Connection(60, 99), + Connection(99, 20), + Connection(28, 158), + Connection(158, 157), + Connection(157, 28), + Connection(35, 226), + Connection(226, 113), + Connection(113, 35), + Connection(160, 159), + Connection(159, 27), + Connection(27, 160), + Connection(204, 202), + Connection(202, 210), + Connection(210, 204), + Connection(113, 225), + Connection(225, 46), + Connection(46, 113), + Connection(43, 202), + Connection(202, 204), + Connection(204, 43), + Connection(62, 76), + Connection(76, 77), + Connection(77, 62), + Connection(137, 123), + Connection(123, 116), + Connection(116, 137), + Connection(41, 38), + Connection(38, 72), + Connection(72, 41), + Connection(203, 129), + Connection(129, 142), + Connection(142, 203), + Connection(64, 98), + Connection(98, 240), + Connection(240, 64), + Connection(49, 102), + Connection(102, 64), + Connection(64, 49), + Connection(41, 73), + Connection(73, 74), + Connection(74, 41), + Connection(212, 216), + Connection(216, 207), + Connection(207, 212), + Connection(42, 74), + Connection(74, 184), + Connection(184, 42), + Connection(169, 170), + Connection(170, 211), + Connection(211, 169), + Connection(170, 149), + Connection(149, 176), + Connection(176, 170), + Connection(105, 66), + Connection(66, 69), + Connection(69, 105), + Connection(122, 6), + Connection(6, 168), + Connection(168, 122), + Connection(123, 147), + Connection(147, 187), + Connection(187, 123), + Connection(96, 77), + Connection(77, 90), + Connection(90, 96), + Connection(65, 55), + Connection(55, 107), + Connection(107, 65), + Connection(89, 90), + Connection(90, 180), + Connection(180, 89), + Connection(101, 100), + Connection(100, 120), + Connection(120, 101), + Connection(63, 105), + Connection(105, 104), + Connection(104, 63), + Connection(93, 137), + Connection(137, 227), + Connection(227, 93), + Connection(15, 86), + Connection(86, 85), + Connection(85, 15), + Connection(129, 102), + Connection(102, 49), + Connection(49, 129), + Connection(14, 87), + Connection(87, 86), + Connection(86, 14), + Connection(55, 8), + Connection(8, 9), + Connection(9, 55), + Connection(100, 47), + Connection(47, 121), + Connection(121, 100), + Connection(145, 23), + Connection(23, 22), + Connection(22, 145), + Connection(88, 89), + Connection(89, 179), + Connection(179, 88), + Connection(6, 122), + Connection(122, 196), + Connection(196, 6), + Connection(88, 95), + Connection(95, 96), + Connection(96, 88), + Connection(138, 172), + Connection(172, 136), + Connection(136, 138), + Connection(215, 58), + Connection(58, 172), + Connection(172, 215), + Connection(115, 48), + Connection(48, 219), + Connection(219, 115), + Connection(42, 80), + Connection(80, 81), + Connection(81, 42), + Connection(195, 3), + Connection(3, 51), + Connection(51, 195), + Connection(43, 146), + Connection(146, 61), + Connection(61, 43), + Connection(171, 175), + Connection(175, 199), + Connection(199, 171), + Connection(81, 82), + Connection(82, 38), + Connection(38, 81), + Connection(53, 46), + Connection(46, 225), + Connection(225, 53), + Connection(144, 163), + Connection(163, 110), + Connection(110, 144), + Connection(52, 65), + Connection(65, 66), + Connection(66, 52), + Connection(229, 228), + Connection(228, 117), + Connection(117, 229), + Connection(34, 127), + Connection(127, 234), + Connection(234, 34), + Connection(107, 108), + Connection(108, 69), + Connection(69, 107), + Connection(109, 108), + Connection(108, 151), + Connection(151, 109), + Connection(48, 64), + Connection(64, 235), + Connection(235, 48), + Connection(62, 78), + Connection(78, 191), + Connection(191, 62), + Connection(129, 209), + Connection(209, 126), + Connection(126, 129), + Connection(111, 35), + Connection(35, 143), + Connection(143, 111), + Connection(117, 123), + Connection(123, 50), + Connection(50, 117), + Connection(222, 65), + Connection(65, 52), + Connection(52, 222), + Connection(19, 125), + Connection(125, 141), + Connection(141, 19), + Connection(221, 55), + Connection(55, 65), + Connection(65, 221), + Connection(3, 195), + Connection(195, 197), + Connection(197, 3), + Connection(25, 7), + Connection(7, 33), + Connection(33, 25), + Connection(220, 237), + Connection(237, 44), + Connection(44, 220), + Connection(70, 71), + Connection(71, 139), + Connection(139, 70), + Connection(122, 193), + Connection(193, 245), + Connection(245, 122), + Connection(247, 130), + Connection(130, 33), + Connection(33, 247), + Connection(71, 21), + Connection(21, 162), + Connection(162, 71), + Connection(170, 169), + Connection(169, 150), + Connection(150, 170), + Connection(188, 174), + Connection(174, 196), + Connection(196, 188), + Connection(216, 186), + Connection(186, 92), + Connection(92, 216), + Connection(2, 97), + Connection(97, 167), + Connection(167, 2), + Connection(141, 125), + Connection(125, 241), + Connection(241, 141), + Connection(164, 167), + Connection(167, 37), + Connection(37, 164), + Connection(72, 38), + Connection(38, 12), + Connection(12, 72), + Connection(38, 82), + Connection(82, 13), + Connection(13, 38), + Connection(63, 68), + Connection(68, 71), + Connection(71, 63), + Connection(226, 35), + Connection(35, 111), + Connection(111, 226), + Connection(101, 50), + Connection(50, 205), + Connection(205, 101), + Connection(206, 92), + Connection(92, 165), + Connection(165, 206), + Connection(209, 198), + Connection(198, 217), + Connection(217, 209), + Connection(165, 167), + Connection(167, 97), + Connection(97, 165), + Connection(220, 115), + Connection(115, 218), + Connection(218, 220), + Connection(133, 112), + Connection(112, 243), + Connection(243, 133), + Connection(239, 238), + Connection(238, 241), + Connection(241, 239), + Connection(214, 135), + Connection(135, 169), + Connection(169, 214), + Connection(190, 173), + Connection(173, 133), + Connection(133, 190), + Connection(171, 208), + Connection(208, 32), + Connection(32, 171), + Connection(125, 44), + Connection(44, 237), + Connection(237, 125), + Connection(86, 87), + Connection(87, 178), + Connection(178, 86), + Connection(85, 86), + Connection(86, 179), + Connection(179, 85), + Connection(84, 85), + Connection(85, 180), + Connection(180, 84), + Connection(83, 84), + Connection(84, 181), + Connection(181, 83), + Connection(201, 83), + Connection(83, 182), + Connection(182, 201), + Connection(137, 93), + Connection(93, 132), + Connection(132, 137), + Connection(76, 62), + Connection(62, 183), + Connection(183, 76), + Connection(61, 76), + Connection(76, 184), + Connection(184, 61), + Connection(57, 61), + Connection(61, 185), + Connection(185, 57), + Connection(212, 57), + Connection(57, 186), + Connection(186, 212), + Connection(214, 207), + Connection(207, 187), + Connection(187, 214), + Connection(34, 143), + Connection(143, 156), + Connection(156, 34), + Connection(79, 239), + Connection(239, 237), + Connection(237, 79), + Connection(123, 137), + Connection(137, 177), + Connection(177, 123), + Connection(44, 1), + Connection(1, 4), + Connection(4, 44), + Connection(201, 194), + Connection(194, 32), + Connection(32, 201), + Connection(64, 102), + Connection(102, 129), + Connection(129, 64), + Connection(213, 215), + Connection(215, 138), + Connection(138, 213), + Connection(59, 166), + Connection(166, 219), + Connection(219, 59), + Connection(242, 99), + Connection(99, 97), + Connection(97, 242), + Connection(2, 94), + Connection(94, 141), + Connection(141, 2), + Connection(75, 59), + Connection(59, 235), + Connection(235, 75), + Connection(24, 110), + Connection(110, 228), + Connection(228, 24), + Connection(25, 130), + Connection(130, 226), + Connection(226, 25), + Connection(23, 24), + Connection(24, 229), + Connection(229, 23), + Connection(22, 23), + Connection(23, 230), + Connection(230, 22), + Connection(26, 22), + Connection(22, 231), + Connection(231, 26), + Connection(112, 26), + Connection(26, 232), + Connection(232, 112), + Connection(189, 190), + Connection(190, 243), + Connection(243, 189), + Connection(221, 56), + Connection(56, 190), + Connection(190, 221), + Connection(28, 56), + Connection(56, 221), + Connection(221, 28), + Connection(27, 28), + Connection(28, 222), + Connection(222, 27), + Connection(29, 27), + Connection(27, 223), + Connection(223, 29), + Connection(30, 29), + Connection(29, 224), + Connection(224, 30), + Connection(247, 30), + Connection(30, 225), + Connection(225, 247), + Connection(238, 79), + Connection(79, 20), + Connection(20, 238), + Connection(166, 59), + Connection(59, 75), + Connection(75, 166), + Connection(60, 75), + Connection(75, 240), + Connection(240, 60), + Connection(147, 177), + Connection(177, 215), + Connection(215, 147), + Connection(20, 79), + Connection(79, 166), + Connection(166, 20), + Connection(187, 147), + Connection(147, 213), + Connection(213, 187), + Connection(112, 233), + Connection(233, 244), + Connection(244, 112), + Connection(233, 128), + Connection(128, 245), + Connection(245, 233), + Connection(128, 114), + Connection(114, 188), + Connection(188, 128), + Connection(114, 217), + Connection(217, 174), + Connection(174, 114), + Connection(131, 115), + Connection(115, 220), + Connection(220, 131), + Connection(217, 198), + Connection(198, 236), + Connection(236, 217), + Connection(198, 131), + Connection(131, 134), + Connection(134, 198), + Connection(177, 132), + Connection(132, 58), + Connection(58, 177), + Connection(143, 35), + Connection(35, 124), + Connection(124, 143), + Connection(110, 163), + Connection(163, 7), + Connection(7, 110), + Connection(228, 110), + Connection(110, 25), + Connection(25, 228), + Connection(356, 389), + Connection(389, 368), + Connection(368, 356), + Connection(11, 302), + Connection(302, 267), + Connection(267, 11), + Connection(452, 350), + Connection(350, 349), + Connection(349, 452), + Connection(302, 303), + Connection(303, 269), + Connection(269, 302), + Connection(357, 343), + Connection(343, 277), + Connection(277, 357), + Connection(452, 453), + Connection(453, 357), + Connection(357, 452), + Connection(333, 332), + Connection(332, 297), + Connection(297, 333), + Connection(175, 152), + Connection(152, 377), + Connection(377, 175), + Connection(347, 348), + Connection(348, 330), + Connection(330, 347), + Connection(303, 304), + Connection(304, 270), + Connection(270, 303), + Connection(9, 336), + Connection(336, 337), + Connection(337, 9), + Connection(278, 279), + Connection(279, 360), + Connection(360, 278), + Connection(418, 262), + Connection(262, 431), + Connection(431, 418), + Connection(304, 408), + Connection(408, 409), + Connection(409, 304), + Connection(310, 415), + Connection(415, 407), + Connection(407, 310), + Connection(270, 409), + Connection(409, 410), + Connection(410, 270), + Connection(450, 348), + Connection(348, 347), + Connection(347, 450), + Connection(422, 430), + Connection(430, 434), + Connection(434, 422), + Connection(313, 314), + Connection(314, 17), + Connection(17, 313), + Connection(306, 307), + Connection(307, 375), + Connection(375, 306), + Connection(387, 388), + Connection(388, 260), + Connection(260, 387), + Connection(286, 414), + Connection(414, 398), + Connection(398, 286), + Connection(335, 406), + Connection(406, 418), + Connection(418, 335), + Connection(364, 367), + Connection(367, 416), + Connection(416, 364), + Connection(423, 358), + Connection(358, 327), + Connection(327, 423), + Connection(251, 284), + Connection(284, 298), + Connection(298, 251), + Connection(281, 5), + Connection(5, 4), + Connection(4, 281), + Connection(373, 374), + Connection(374, 253), + Connection(253, 373), + Connection(307, 320), + Connection(320, 321), + Connection(321, 307), + Connection(425, 427), + Connection(427, 411), + Connection(411, 425), + Connection(421, 313), + Connection(313, 18), + Connection(18, 421), + Connection(321, 405), + Connection(405, 406), + Connection(406, 321), + Connection(320, 404), + Connection(404, 405), + Connection(405, 320), + Connection(315, 16), + Connection(16, 17), + Connection(17, 315), + Connection(426, 425), + Connection(425, 266), + Connection(266, 426), + Connection(377, 400), + Connection(400, 369), + Connection(369, 377), + Connection(322, 391), + Connection(391, 269), + Connection(269, 322), + Connection(417, 465), + Connection(465, 464), + Connection(464, 417), + Connection(386, 257), + Connection(257, 258), + Connection(258, 386), + Connection(466, 260), + Connection(260, 388), + Connection(388, 466), + Connection(456, 399), + Connection(399, 419), + Connection(419, 456), + Connection(284, 332), + Connection(332, 333), + Connection(333, 284), + Connection(417, 285), + Connection(285, 8), + Connection(8, 417), + Connection(346, 340), + Connection(340, 261), + Connection(261, 346), + Connection(413, 441), + Connection(441, 285), + Connection(285, 413), + Connection(327, 460), + Connection(460, 328), + Connection(328, 327), + Connection(355, 371), + Connection(371, 329), + Connection(329, 355), + Connection(392, 439), + Connection(439, 438), + Connection(438, 392), + Connection(382, 341), + Connection(341, 256), + Connection(256, 382), + Connection(429, 420), + Connection(420, 360), + Connection(360, 429), + Connection(364, 394), + Connection(394, 379), + Connection(379, 364), + Connection(277, 343), + Connection(343, 437), + Connection(437, 277), + Connection(443, 444), + Connection(444, 283), + Connection(283, 443), + Connection(275, 440), + Connection(440, 363), + Connection(363, 275), + Connection(431, 262), + Connection(262, 369), + Connection(369, 431), + Connection(297, 338), + Connection(338, 337), + Connection(337, 297), + Connection(273, 375), + Connection(375, 321), + Connection(321, 273), + Connection(450, 451), + Connection(451, 349), + Connection(349, 450), + Connection(446, 342), + Connection(342, 467), + Connection(467, 446), + Connection(293, 334), + Connection(334, 282), + Connection(282, 293), + Connection(458, 461), + Connection(461, 462), + Connection(462, 458), + Connection(276, 353), + Connection(353, 383), + Connection(383, 276), + Connection(308, 324), + Connection(324, 325), + Connection(325, 308), + Connection(276, 300), + Connection(300, 293), + Connection(293, 276), + Connection(372, 345), + Connection(345, 447), + Connection(447, 372), + Connection(352, 345), + Connection(345, 340), + Connection(340, 352), + Connection(274, 1), + Connection(1, 19), + Connection(19, 274), + Connection(456, 248), + Connection(248, 281), + Connection(281, 456), + Connection(436, 427), + Connection(427, 425), + Connection(425, 436), + Connection(381, 256), + Connection(256, 252), + Connection(252, 381), + Connection(269, 391), + Connection(391, 393), + Connection(393, 269), + Connection(200, 199), + Connection(199, 428), + Connection(428, 200), + Connection(266, 330), + Connection(330, 329), + Connection(329, 266), + Connection(287, 273), + Connection(273, 422), + Connection(422, 287), + Connection(250, 462), + Connection(462, 328), + Connection(328, 250), + Connection(258, 286), + Connection(286, 384), + Connection(384, 258), + Connection(265, 353), + Connection(353, 342), + Connection(342, 265), + Connection(387, 259), + Connection(259, 257), + Connection(257, 387), + Connection(424, 431), + Connection(431, 430), + Connection(430, 424), + Connection(342, 353), + Connection(353, 276), + Connection(276, 342), + Connection(273, 335), + Connection(335, 424), + Connection(424, 273), + Connection(292, 325), + Connection(325, 307), + Connection(307, 292), + Connection(366, 447), + Connection(447, 345), + Connection(345, 366), + Connection(271, 303), + Connection(303, 302), + Connection(302, 271), + Connection(423, 266), + Connection(266, 371), + Connection(371, 423), + Connection(294, 455), + Connection(455, 460), + Connection(460, 294), + Connection(279, 278), + Connection(278, 294), + Connection(294, 279), + Connection(271, 272), + Connection(272, 304), + Connection(304, 271), + Connection(432, 434), + Connection(434, 427), + Connection(427, 432), + Connection(272, 407), + Connection(407, 408), + Connection(408, 272), + Connection(394, 430), + Connection(430, 431), + Connection(431, 394), + Connection(395, 369), + Connection(369, 400), + Connection(400, 395), + Connection(334, 333), + Connection(333, 299), + Connection(299, 334), + Connection(351, 417), + Connection(417, 168), + Connection(168, 351), + Connection(352, 280), + Connection(280, 411), + Connection(411, 352), + Connection(325, 319), + Connection(319, 320), + Connection(320, 325), + Connection(295, 296), + Connection(296, 336), + Connection(336, 295), + Connection(319, 403), + Connection(403, 404), + Connection(404, 319), + Connection(330, 348), + Connection(348, 349), + Connection(349, 330), + Connection(293, 298), + Connection(298, 333), + Connection(333, 293), + Connection(323, 454), + Connection(454, 447), + Connection(447, 323), + Connection(15, 16), + Connection(16, 315), + Connection(315, 15), + Connection(358, 429), + Connection(429, 279), + Connection(279, 358), + Connection(14, 15), + Connection(15, 316), + Connection(316, 14), + Connection(285, 336), + Connection(336, 9), + Connection(9, 285), + Connection(329, 349), + Connection(349, 350), + Connection(350, 329), + Connection(374, 380), + Connection(380, 252), + Connection(252, 374), + Connection(318, 402), + Connection(402, 403), + Connection(403, 318), + Connection(6, 197), + Connection(197, 419), + Connection(419, 6), + Connection(318, 319), + Connection(319, 325), + Connection(325, 318), + Connection(367, 364), + Connection(364, 365), + Connection(365, 367), + Connection(435, 367), + Connection(367, 397), + Connection(397, 435), + Connection(344, 438), + Connection(438, 439), + Connection(439, 344), + Connection(272, 271), + Connection(271, 311), + Connection(311, 272), + Connection(195, 5), + Connection(5, 281), + Connection(281, 195), + Connection(273, 287), + Connection(287, 291), + Connection(291, 273), + Connection(396, 428), + Connection(428, 199), + Connection(199, 396), + Connection(311, 271), + Connection(271, 268), + Connection(268, 311), + Connection(283, 444), + Connection(444, 445), + Connection(445, 283), + Connection(373, 254), + Connection(254, 339), + Connection(339, 373), + Connection(282, 334), + Connection(334, 296), + Connection(296, 282), + Connection(449, 347), + Connection(347, 346), + Connection(346, 449), + Connection(264, 447), + Connection(447, 454), + Connection(454, 264), + Connection(336, 296), + Connection(296, 299), + Connection(299, 336), + Connection(338, 10), + Connection(10, 151), + Connection(151, 338), + Connection(278, 439), + Connection(439, 455), + Connection(455, 278), + Connection(292, 407), + Connection(407, 415), + Connection(415, 292), + Connection(358, 371), + Connection(371, 355), + Connection(355, 358), + Connection(340, 345), + Connection(345, 372), + Connection(372, 340), + Connection(346, 347), + Connection(347, 280), + Connection(280, 346), + Connection(442, 443), + Connection(443, 282), + Connection(282, 442), + Connection(19, 94), + Connection(94, 370), + Connection(370, 19), + Connection(441, 442), + Connection(442, 295), + Connection(295, 441), + Connection(248, 419), + Connection(419, 197), + Connection(197, 248), + Connection(263, 255), + Connection(255, 359), + Connection(359, 263), + Connection(440, 275), + Connection(275, 274), + Connection(274, 440), + Connection(300, 383), + Connection(383, 368), + Connection(368, 300), + Connection(351, 412), + Connection(412, 465), + Connection(465, 351), + Connection(263, 467), + Connection(467, 466), + Connection(466, 263), + Connection(301, 368), + Connection(368, 389), + Connection(389, 301), + Connection(395, 378), + Connection(378, 379), + Connection(379, 395), + Connection(412, 351), + Connection(351, 419), + Connection(419, 412), + Connection(436, 426), + Connection(426, 322), + Connection(322, 436), + Connection(2, 164), + Connection(164, 393), + Connection(393, 2), + Connection(370, 462), + Connection(462, 461), + Connection(461, 370), + Connection(164, 0), + Connection(0, 267), + Connection(267, 164), + Connection(302, 11), + Connection(11, 12), + Connection(12, 302), + Connection(268, 12), + Connection(12, 13), + Connection(13, 268), + Connection(293, 300), + Connection(300, 301), + Connection(301, 293), + Connection(446, 261), + Connection(261, 340), + Connection(340, 446), + Connection(330, 266), + Connection(266, 425), + Connection(425, 330), + Connection(426, 423), + Connection(423, 391), + Connection(391, 426), + Connection(429, 355), + Connection(355, 437), + Connection(437, 429), + Connection(391, 327), + Connection(327, 326), + Connection(326, 391), + Connection(440, 457), + Connection(457, 438), + Connection(438, 440), + Connection(341, 382), + Connection(382, 362), + Connection(362, 341), + Connection(459, 457), + Connection(457, 461), + Connection(461, 459), + Connection(434, 430), + Connection(430, 394), + Connection(394, 434), + Connection(414, 463), + Connection(463, 362), + Connection(362, 414), + Connection(396, 369), + Connection(369, 262), + Connection(262, 396), + Connection(354, 461), + Connection(461, 457), + Connection(457, 354), + Connection(316, 403), + Connection(403, 402), + Connection(402, 316), + Connection(315, 404), + Connection(404, 403), + Connection(403, 315), + Connection(314, 405), + Connection(405, 404), + Connection(404, 314), + Connection(313, 406), + Connection(406, 405), + Connection(405, 313), + Connection(421, 418), + Connection(418, 406), + Connection(406, 421), + Connection(366, 401), + Connection(401, 361), + Connection(361, 366), + Connection(306, 408), + Connection(408, 407), + Connection(407, 306), + Connection(291, 409), + Connection(409, 408), + Connection(408, 291), + Connection(287, 410), + Connection(410, 409), + Connection(409, 287), + Connection(432, 436), + Connection(436, 410), + Connection(410, 432), + Connection(434, 416), + Connection(416, 411), + Connection(411, 434), + Connection(264, 368), + Connection(368, 383), + Connection(383, 264), + Connection(309, 438), + Connection(438, 457), + Connection(457, 309), + Connection(352, 376), + Connection(376, 401), + Connection(401, 352), + Connection(274, 275), + Connection(275, 4), + Connection(4, 274), + Connection(421, 428), + Connection(428, 262), + Connection(262, 421), + Connection(294, 327), + Connection(327, 358), + Connection(358, 294), + Connection(433, 416), + Connection(416, 367), + Connection(367, 433), + Connection(289, 455), + Connection(455, 439), + Connection(439, 289), + Connection(462, 370), + Connection(370, 326), + Connection(326, 462), + Connection(2, 326), + Connection(326, 370), + Connection(370, 2), + Connection(305, 460), + Connection(460, 455), + Connection(455, 305), + Connection(254, 449), + Connection(449, 448), + Connection(448, 254), + Connection(255, 261), + Connection(261, 446), + Connection(446, 255), + Connection(253, 450), + Connection(450, 449), + Connection(449, 253), + Connection(252, 451), + Connection(451, 450), + Connection(450, 252), + Connection(256, 452), + Connection(452, 451), + Connection(451, 256), + Connection(341, 453), + Connection(453, 452), + Connection(452, 341), + Connection(413, 464), + Connection(464, 463), + Connection(463, 413), + Connection(441, 413), + Connection(413, 414), + Connection(414, 441), + Connection(258, 442), + Connection(442, 441), + Connection(441, 258), + Connection(257, 443), + Connection(443, 442), + Connection(442, 257), + Connection(259, 444), + Connection(444, 443), + Connection(443, 259), + Connection(260, 445), + Connection(445, 444), + Connection(444, 260), + Connection(467, 342), + Connection(342, 445), + Connection(445, 467), + Connection(459, 458), + Connection(458, 250), + Connection(250, 459), + Connection(289, 392), + Connection(392, 290), + Connection(290, 289), + Connection(290, 328), + Connection(328, 460), + Connection(460, 290), + Connection(376, 433), + Connection(433, 435), + Connection(435, 376), + Connection(250, 290), + Connection(290, 392), + Connection(392, 250), + Connection(411, 416), + Connection(416, 433), + Connection(433, 411), + Connection(341, 463), + Connection(463, 464), + Connection(464, 341), + Connection(453, 464), + Connection(464, 465), + Connection(465, 453), + Connection(357, 465), + Connection(465, 412), + Connection(412, 357), + Connection(343, 412), + Connection(412, 399), + Connection(399, 343), + Connection(360, 363), + Connection(363, 440), + Connection(440, 360), + Connection(437, 399), + Connection(399, 456), + Connection(456, 437), + Connection(420, 456), + Connection(456, 363), + Connection(363, 420), + Connection(401, 435), + Connection(435, 288), + Connection(288, 401), + Connection(372, 383), + Connection(383, 353), + Connection(353, 372), + Connection(339, 255), + Connection(255, 249), + Connection(249, 339), + Connection(448, 261), + Connection(261, 255), + Connection(255, 448), + Connection(133, 243), + Connection(243, 190), + Connection(190, 133), + Connection(133, 155), + Connection(155, 112), + Connection(112, 133), + Connection(33, 246), + Connection(246, 247), + Connection(247, 33), + Connection(33, 130), + Connection(130, 25), + Connection(25, 33), + Connection(398, 384), + Connection(384, 286), + Connection(286, 398), + Connection(362, 398), + Connection(398, 414), + Connection(414, 362), + Connection(362, 463), + Connection(463, 341), + Connection(341, 362), + Connection(263, 359), + Connection(359, 467), + Connection(467, 263), + Connection(263, 249), + Connection(249, 255), + Connection(255, 263), + Connection(466, 467), + Connection(467, 260), + Connection(260, 466), + Connection(75, 60), + Connection(60, 166), + Connection(166, 75), + Connection(238, 239), + Connection(239, 79), + Connection(79, 238), + Connection(162, 127), + Connection(127, 139), + Connection(139, 162), + Connection(72, 11), + Connection(11, 37), + Connection(37, 72), + Connection(121, 232), + Connection(232, 120), + Connection(120, 121), + Connection(73, 72), + Connection(72, 39), + Connection(39, 73), + Connection(114, 128), + Connection(128, 47), + Connection(47, 114), + Connection(233, 232), + Connection(232, 128), + Connection(128, 233), + Connection(103, 104), + Connection(104, 67), + Connection(67, 103), + Connection(152, 175), + Connection(175, 148), + Connection(148, 152), + Connection(119, 118), + Connection(118, 101), + Connection(101, 119), + Connection(74, 73), + Connection(73, 40), + Connection(40, 74), + Connection(107, 9), + Connection(9, 108), + Connection(108, 107), + Connection(49, 48), + Connection(48, 131), + Connection(131, 49), + Connection(32, 194), + Connection(194, 211), + Connection(211, 32), + Connection(184, 74), + Connection(74, 185), + Connection(185, 184), + Connection(191, 80), + Connection(80, 183), + Connection(183, 191), + Connection(185, 40), + Connection(40, 186), + Connection(186, 185), + Connection(119, 230), + Connection(230, 118), + Connection(118, 119), + Connection(210, 202), + Connection(202, 214), + Connection(214, 210), + Connection(84, 83), + Connection(83, 17), + Connection(17, 84), + Connection(77, 76), + Connection(76, 146), + Connection(146, 77), + Connection(161, 160), + Connection(160, 30), + Connection(30, 161), + Connection(190, 56), + Connection(56, 173), + Connection(173, 190), + Connection(182, 106), + Connection(106, 194), + Connection(194, 182), + Connection(138, 135), + Connection(135, 192), + Connection(192, 138), + Connection(129, 203), + Connection(203, 98), + Connection(98, 129), + Connection(54, 21), + Connection(21, 68), + Connection(68, 54), + Connection(5, 51), + Connection(51, 4), + Connection(4, 5), + Connection(145, 144), + Connection(144, 23), + Connection(23, 145), + Connection(90, 77), + Connection(77, 91), + Connection(91, 90), + Connection(207, 205), + Connection(205, 187), + Connection(187, 207), + Connection(83, 201), + Connection(201, 18), + Connection(18, 83), + Connection(181, 91), + Connection(91, 182), + Connection(182, 181), + Connection(180, 90), + Connection(90, 181), + Connection(181, 180), + Connection(16, 85), + Connection(85, 17), + Connection(17, 16), + Connection(205, 206), + Connection(206, 36), + Connection(36, 205), + Connection(176, 148), + Connection(148, 140), + Connection(140, 176), + Connection(165, 92), + Connection(92, 39), + Connection(39, 165), + Connection(245, 193), + Connection(193, 244), + Connection(244, 245), + Connection(27, 159), + Connection(159, 28), + Connection(28, 27), + Connection(30, 247), + Connection(247, 161), + Connection(161, 30), + Connection(174, 236), + Connection(236, 196), + Connection(196, 174), + Connection(103, 54), + Connection(54, 104), + Connection(104, 103), + Connection(55, 193), + Connection(193, 8), + Connection(8, 55), + Connection(111, 117), + Connection(117, 31), + Connection(31, 111), + Connection(221, 189), + Connection(189, 55), + Connection(55, 221), + Connection(240, 98), + Connection(98, 99), + Connection(99, 240), + Connection(142, 126), + Connection(126, 100), + Connection(100, 142), + Connection(219, 166), + Connection(166, 218), + Connection(218, 219), + Connection(112, 155), + Connection(155, 26), + Connection(26, 112), + Connection(198, 209), + Connection(209, 131), + Connection(131, 198), + Connection(169, 135), + Connection(135, 150), + Connection(150, 169), + Connection(114, 47), + Connection(47, 217), + Connection(217, 114), + Connection(224, 223), + Connection(223, 53), + Connection(53, 224), + Connection(220, 45), + Connection(45, 134), + Connection(134, 220), + Connection(32, 211), + Connection(211, 140), + Connection(140, 32), + Connection(109, 67), + Connection(67, 108), + Connection(108, 109), + Connection(146, 43), + Connection(43, 91), + Connection(91, 146), + Connection(231, 230), + Connection(230, 120), + Connection(120, 231), + Connection(113, 226), + Connection(226, 247), + Connection(247, 113), + Connection(105, 63), + Connection(63, 52), + Connection(52, 105), + Connection(241, 238), + Connection(238, 242), + Connection(242, 241), + Connection(124, 46), + Connection(46, 156), + Connection(156, 124), + Connection(95, 78), + Connection(78, 96), + Connection(96, 95), + Connection(70, 46), + Connection(46, 63), + Connection(63, 70), + Connection(116, 143), + Connection(143, 227), + Connection(227, 116), + Connection(116, 123), + Connection(123, 111), + Connection(111, 116), + Connection(1, 44), + Connection(44, 19), + Connection(19, 1), + Connection(3, 236), + Connection(236, 51), + Connection(51, 3), + Connection(207, 216), + Connection(216, 205), + Connection(205, 207), + Connection(26, 154), + Connection(154, 22), + Connection(22, 26), + Connection(165, 39), + Connection(39, 167), + Connection(167, 165), + Connection(199, 200), + Connection(200, 208), + Connection(208, 199), + Connection(101, 36), + Connection(36, 100), + Connection(100, 101), + Connection(43, 57), + Connection(57, 202), + Connection(202, 43), + Connection(242, 20), + Connection(20, 99), + Connection(99, 242), + Connection(56, 28), + Connection(28, 157), + Connection(157, 56), + Connection(124, 35), + Connection(35, 113), + Connection(113, 124), + Connection(29, 160), + Connection(160, 27), + Connection(27, 29), + Connection(211, 204), + Connection(204, 210), + Connection(210, 211), + Connection(124, 113), + Connection(113, 46), + Connection(46, 124), + Connection(106, 43), + Connection(43, 204), + Connection(204, 106), + Connection(96, 62), + Connection(62, 77), + Connection(77, 96), + Connection(227, 137), + Connection(137, 116), + Connection(116, 227), + Connection(73, 41), + Connection(41, 72), + Connection(72, 73), + Connection(36, 203), + Connection(203, 142), + Connection(142, 36), + Connection(235, 64), + Connection(64, 240), + Connection(240, 235), + Connection(48, 49), + Connection(49, 64), + Connection(64, 48), + Connection(42, 41), + Connection(41, 74), + Connection(74, 42), + Connection(214, 212), + Connection(212, 207), + Connection(207, 214), + Connection(183, 42), + Connection(42, 184), + Connection(184, 183), + Connection(210, 169), + Connection(169, 211), + Connection(211, 210), + Connection(140, 170), + Connection(170, 176), + Connection(176, 140), + Connection(104, 105), + Connection(105, 69), + Connection(69, 104), + Connection(193, 122), + Connection(122, 168), + Connection(168, 193), + Connection(50, 123), + Connection(123, 187), + Connection(187, 50), + Connection(89, 96), + Connection(96, 90), + Connection(90, 89), + Connection(66, 65), + Connection(65, 107), + Connection(107, 66), + Connection(179, 89), + Connection(89, 180), + Connection(180, 179), + Connection(119, 101), + Connection(101, 120), + Connection(120, 119), + Connection(68, 63), + Connection(63, 104), + Connection(104, 68), + Connection(234, 93), + Connection(93, 227), + Connection(227, 234), + Connection(16, 15), + Connection(15, 85), + Connection(85, 16), + Connection(209, 129), + Connection(129, 49), + Connection(49, 209), + Connection(15, 14), + Connection(14, 86), + Connection(86, 15), + Connection(107, 55), + Connection(55, 9), + Connection(9, 107), + Connection(120, 100), + Connection(100, 121), + Connection(121, 120), + Connection(153, 145), + Connection(145, 22), + Connection(22, 153), + Connection(178, 88), + Connection(88, 179), + Connection(179, 178), + Connection(197, 6), + Connection(6, 196), + Connection(196, 197), + Connection(89, 88), + Connection(88, 96), + Connection(96, 89), + Connection(135, 138), + Connection(138, 136), + Connection(136, 135), + Connection(138, 215), + Connection(215, 172), + Connection(172, 138), + Connection(218, 115), + Connection(115, 219), + Connection(219, 218), + Connection(41, 42), + Connection(42, 81), + Connection(81, 41), + Connection(5, 195), + Connection(195, 51), + Connection(51, 5), + Connection(57, 43), + Connection(43, 61), + Connection(61, 57), + Connection(208, 171), + Connection(171, 199), + Connection(199, 208), + Connection(41, 81), + Connection(81, 38), + Connection(38, 41), + Connection(224, 53), + Connection(53, 225), + Connection(225, 224), + Connection(24, 144), + Connection(144, 110), + Connection(110, 24), + Connection(105, 52), + Connection(52, 66), + Connection(66, 105), + Connection(118, 229), + Connection(229, 117), + Connection(117, 118), + Connection(227, 34), + Connection(34, 234), + Connection(234, 227), + Connection(66, 107), + Connection(107, 69), + Connection(69, 66), + Connection(10, 109), + Connection(109, 151), + Connection(151, 10), + Connection(219, 48), + Connection(48, 235), + Connection(235, 219), + Connection(183, 62), + Connection(62, 191), + Connection(191, 183), + Connection(142, 129), + Connection(129, 126), + Connection(126, 142), + Connection(116, 111), + Connection(111, 143), + Connection(143, 116), + Connection(118, 117), + Connection(117, 50), + Connection(50, 118), + Connection(223, 222), + Connection(222, 52), + Connection(52, 223), + Connection(94, 19), + Connection(19, 141), + Connection(141, 94), + Connection(222, 221), + Connection(221, 65), + Connection(65, 222), + Connection(196, 3), + Connection(3, 197), + Connection(197, 196), + Connection(45, 220), + Connection(220, 44), + Connection(44, 45), + Connection(156, 70), + Connection(70, 139), + Connection(139, 156), + Connection(188, 122), + Connection(122, 245), + Connection(245, 188), + Connection(139, 71), + Connection(71, 162), + Connection(162, 139), + Connection(149, 170), + Connection(170, 150), + Connection(150, 149), + Connection(122, 188), + Connection(188, 196), + Connection(196, 122), + Connection(206, 216), + Connection(216, 92), + Connection(92, 206), + Connection(164, 2), + Connection(2, 167), + Connection(167, 164), + Connection(242, 141), + Connection(141, 241), + Connection(241, 242), + Connection(0, 164), + Connection(164, 37), + Connection(37, 0), + Connection(11, 72), + Connection(72, 12), + Connection(12, 11), + Connection(12, 38), + Connection(38, 13), + Connection(13, 12), + Connection(70, 63), + Connection(63, 71), + Connection(71, 70), + Connection(31, 226), + Connection(226, 111), + Connection(111, 31), + Connection(36, 101), + Connection(101, 205), + Connection(205, 36), + Connection(203, 206), + Connection(206, 165), + Connection(165, 203), + Connection(126, 209), + Connection(209, 217), + Connection(217, 126), + Connection(98, 165), + Connection(165, 97), + Connection(97, 98), + Connection(237, 220), + Connection(220, 218), + Connection(218, 237), + Connection(237, 239), + Connection(239, 241), + Connection(241, 237), + Connection(210, 214), + Connection(214, 169), + Connection(169, 210), + Connection(140, 171), + Connection(171, 32), + Connection(32, 140), + Connection(241, 125), + Connection(125, 237), + Connection(237, 241), + Connection(179, 86), + Connection(86, 178), + Connection(178, 179), + Connection(180, 85), + Connection(85, 179), + Connection(179, 180), + Connection(181, 84), + Connection(84, 180), + Connection(180, 181), + Connection(182, 83), + Connection(83, 181), + Connection(181, 182), + Connection(194, 201), + Connection(201, 182), + Connection(182, 194), + Connection(177, 137), + Connection(137, 132), + Connection(132, 177), + Connection(184, 76), + Connection(76, 183), + Connection(183, 184), + Connection(185, 61), + Connection(61, 184), + Connection(184, 185), + Connection(186, 57), + Connection(57, 185), + Connection(185, 186), + Connection(216, 212), + Connection(212, 186), + Connection(186, 216), + Connection(192, 214), + Connection(214, 187), + Connection(187, 192), + Connection(139, 34), + Connection(34, 156), + Connection(156, 139), + Connection(218, 79), + Connection(79, 237), + Connection(237, 218), + Connection(147, 123), + Connection(123, 177), + Connection(177, 147), + Connection(45, 44), + Connection(44, 4), + Connection(4, 45), + Connection(208, 201), + Connection(201, 32), + Connection(32, 208), + Connection(98, 64), + Connection(64, 129), + Connection(129, 98), + Connection(192, 213), + Connection(213, 138), + Connection(138, 192), + Connection(235, 59), + Connection(59, 219), + Connection(219, 235), + Connection(141, 242), + Connection(242, 97), + Connection(97, 141), + Connection(97, 2), + Connection(2, 141), + Connection(141, 97), + Connection(240, 75), + Connection(75, 235), + Connection(235, 240), + Connection(229, 24), + Connection(24, 228), + Connection(228, 229), + Connection(31, 25), + Connection(25, 226), + Connection(226, 31), + Connection(230, 23), + Connection(23, 229), + Connection(229, 230), + Connection(231, 22), + Connection(22, 230), + Connection(230, 231), + Connection(232, 26), + Connection(26, 231), + Connection(231, 232), + Connection(233, 112), + Connection(112, 232), + Connection(232, 233), + Connection(244, 189), + Connection(189, 243), + Connection(243, 244), + Connection(189, 221), + Connection(221, 190), + Connection(190, 189), + Connection(222, 28), + Connection(28, 221), + Connection(221, 222), + Connection(223, 27), + Connection(27, 222), + Connection(222, 223), + Connection(224, 29), + Connection(29, 223), + Connection(223, 224), + Connection(225, 30), + Connection(30, 224), + Connection(224, 225), + Connection(113, 247), + Connection(247, 225), + Connection(225, 113), + Connection(99, 60), + Connection(60, 240), + Connection(240, 99), + Connection(213, 147), + Connection(147, 215), + Connection(215, 213), + Connection(60, 20), + Connection(20, 166), + Connection(166, 60), + Connection(192, 187), + Connection(187, 213), + Connection(213, 192), + Connection(243, 112), + Connection(112, 244), + Connection(244, 243), + Connection(244, 233), + Connection(233, 245), + Connection(245, 244), + Connection(245, 128), + Connection(128, 188), + Connection(188, 245), + Connection(188, 114), + Connection(114, 174), + Connection(174, 188), + Connection(134, 131), + Connection(131, 220), + Connection(220, 134), + Connection(174, 217), + Connection(217, 236), + Connection(236, 174), + Connection(236, 198), + Connection(198, 134), + Connection(134, 236), + Connection(215, 177), + Connection(177, 58), + Connection(58, 215), + Connection(156, 143), + Connection(143, 124), + Connection(124, 156), + Connection(25, 110), + Connection(110, 7), + Connection(7, 25), + Connection(31, 228), + Connection(228, 25), + Connection(25, 31), + Connection(264, 356), + Connection(356, 368), + Connection(368, 264), + Connection(0, 11), + Connection(11, 267), + Connection(267, 0), + Connection(451, 452), + Connection(452, 349), + Connection(349, 451), + Connection(267, 302), + Connection(302, 269), + Connection(269, 267), + Connection(350, 357), + Connection(357, 277), + Connection(277, 350), + Connection(350, 452), + Connection(452, 357), + Connection(357, 350), + Connection(299, 333), + Connection(333, 297), + Connection(297, 299), + Connection(396, 175), + Connection(175, 377), + Connection(377, 396), + Connection(280, 347), + Connection(347, 330), + Connection(330, 280), + Connection(269, 303), + Connection(303, 270), + Connection(270, 269), + Connection(151, 9), + Connection(9, 337), + Connection(337, 151), + Connection(344, 278), + Connection(278, 360), + Connection(360, 344), + Connection(424, 418), + Connection(418, 431), + Connection(431, 424), + Connection(270, 304), + Connection(304, 409), + Connection(409, 270), + Connection(272, 310), + Connection(310, 407), + Connection(407, 272), + Connection(322, 270), + Connection(270, 410), + Connection(410, 322), + Connection(449, 450), + Connection(450, 347), + Connection(347, 449), + Connection(432, 422), + Connection(422, 434), + Connection(434, 432), + Connection(18, 313), + Connection(313, 17), + Connection(17, 18), + Connection(291, 306), + Connection(306, 375), + Connection(375, 291), + Connection(259, 387), + Connection(387, 260), + Connection(260, 259), + Connection(424, 335), + Connection(335, 418), + Connection(418, 424), + Connection(434, 364), + Connection(364, 416), + Connection(416, 434), + Connection(391, 423), + Connection(423, 327), + Connection(327, 391), + Connection(301, 251), + Connection(251, 298), + Connection(298, 301), + Connection(275, 281), + Connection(281, 4), + Connection(4, 275), + Connection(254, 373), + Connection(373, 253), + Connection(253, 254), + Connection(375, 307), + Connection(307, 321), + Connection(321, 375), + Connection(280, 425), + Connection(425, 411), + Connection(411, 280), + Connection(200, 421), + Connection(421, 18), + Connection(18, 200), + Connection(335, 321), + Connection(321, 406), + Connection(406, 335), + Connection(321, 320), + Connection(320, 405), + Connection(405, 321), + Connection(314, 315), + Connection(315, 17), + Connection(17, 314), + Connection(423, 426), + Connection(426, 266), + Connection(266, 423), + Connection(396, 377), + Connection(377, 369), + Connection(369, 396), + Connection(270, 322), + Connection(322, 269), + Connection(269, 270), + Connection(413, 417), + Connection(417, 464), + Connection(464, 413), + Connection(385, 386), + Connection(386, 258), + Connection(258, 385), + Connection(248, 456), + Connection(456, 419), + Connection(419, 248), + Connection(298, 284), + Connection(284, 333), + Connection(333, 298), + Connection(168, 417), + Connection(417, 8), + Connection(8, 168), + Connection(448, 346), + Connection(346, 261), + Connection(261, 448), + Connection(417, 413), + Connection(413, 285), + Connection(285, 417), + Connection(326, 327), + Connection(327, 328), + Connection(328, 326), + Connection(277, 355), + Connection(355, 329), + Connection(329, 277), + Connection(309, 392), + Connection(392, 438), + Connection(438, 309), + Connection(381, 382), + Connection(382, 256), + Connection(256, 381), + Connection(279, 429), + Connection(429, 360), + Connection(360, 279), + Connection(365, 364), + Connection(364, 379), + Connection(379, 365), + Connection(355, 277), + Connection(277, 437), + Connection(437, 355), + Connection(282, 443), + Connection(443, 283), + Connection(283, 282), + Connection(281, 275), + Connection(275, 363), + Connection(363, 281), + Connection(395, 431), + Connection(431, 369), + Connection(369, 395), + Connection(299, 297), + Connection(297, 337), + Connection(337, 299), + Connection(335, 273), + Connection(273, 321), + Connection(321, 335), + Connection(348, 450), + Connection(450, 349), + Connection(349, 348), + Connection(359, 446), + Connection(446, 467), + Connection(467, 359), + Connection(283, 293), + Connection(293, 282), + Connection(282, 283), + Connection(250, 458), + Connection(458, 462), + Connection(462, 250), + Connection(300, 276), + Connection(276, 383), + Connection(383, 300), + Connection(292, 308), + Connection(308, 325), + Connection(325, 292), + Connection(283, 276), + Connection(276, 293), + Connection(293, 283), + Connection(264, 372), + Connection(372, 447), + Connection(447, 264), + Connection(346, 352), + Connection(352, 340), + Connection(340, 346), + Connection(354, 274), + Connection(274, 19), + Connection(19, 354), + Connection(363, 456), + Connection(456, 281), + Connection(281, 363), + Connection(426, 436), + Connection(436, 425), + Connection(425, 426), + Connection(380, 381), + Connection(381, 252), + Connection(252, 380), + Connection(267, 269), + Connection(269, 393), + Connection(393, 267), + Connection(421, 200), + Connection(200, 428), + Connection(428, 421), + Connection(371, 266), + Connection(266, 329), + Connection(329, 371), + Connection(432, 287), + Connection(287, 422), + Connection(422, 432), + Connection(290, 250), + Connection(250, 328), + Connection(328, 290), + Connection(385, 258), + Connection(258, 384), + Connection(384, 385), + Connection(446, 265), + Connection(265, 342), + Connection(342, 446), + Connection(386, 387), + Connection(387, 257), + Connection(257, 386), + Connection(422, 424), + Connection(424, 430), + Connection(430, 422), + Connection(445, 342), + Connection(342, 276), + Connection(276, 445), + Connection(422, 273), + Connection(273, 424), + Connection(424, 422), + Connection(306, 292), + Connection(292, 307), + Connection(307, 306), + Connection(352, 366), + Connection(366, 345), + Connection(345, 352), + Connection(268, 271), + Connection(271, 302), + Connection(302, 268), + Connection(358, 423), + Connection(423, 371), + Connection(371, 358), + Connection(327, 294), + Connection(294, 460), + Connection(460, 327), + Connection(331, 279), + Connection(279, 294), + Connection(294, 331), + Connection(303, 271), + Connection(271, 304), + Connection(304, 303), + Connection(436, 432), + Connection(432, 427), + Connection(427, 436), + Connection(304, 272), + Connection(272, 408), + Connection(408, 304), + Connection(395, 394), + Connection(394, 431), + Connection(431, 395), + Connection(378, 395), + Connection(395, 400), + Connection(400, 378), + Connection(296, 334), + Connection(334, 299), + Connection(299, 296), + Connection(6, 351), + Connection(351, 168), + Connection(168, 6), + Connection(376, 352), + Connection(352, 411), + Connection(411, 376), + Connection(307, 325), + Connection(325, 320), + Connection(320, 307), + Connection(285, 295), + Connection(295, 336), + Connection(336, 285), + Connection(320, 319), + Connection(319, 404), + Connection(404, 320), + Connection(329, 330), + Connection(330, 349), + Connection(349, 329), + Connection(334, 293), + Connection(293, 333), + Connection(333, 334), + Connection(366, 323), + Connection(323, 447), + Connection(447, 366), + Connection(316, 15), + Connection(15, 315), + Connection(315, 316), + Connection(331, 358), + Connection(358, 279), + Connection(279, 331), + Connection(317, 14), + Connection(14, 316), + Connection(316, 317), + Connection(8, 285), + Connection(285, 9), + Connection(9, 8), + Connection(277, 329), + Connection(329, 350), + Connection(350, 277), + Connection(253, 374), + Connection(374, 252), + Connection(252, 253), + Connection(319, 318), + Connection(318, 403), + Connection(403, 319), + Connection(351, 6), + Connection(6, 419), + Connection(419, 351), + Connection(324, 318), + Connection(318, 325), + Connection(325, 324), + Connection(397, 367), + Connection(367, 365), + Connection(365, 397), + Connection(288, 435), + Connection(435, 397), + Connection(397, 288), + Connection(278, 344), + Connection(344, 439), + Connection(439, 278), + Connection(310, 272), + Connection(272, 311), + Connection(311, 310), + Connection(248, 195), + Connection(195, 281), + Connection(281, 248), + Connection(375, 273), + Connection(273, 291), + Connection(291, 375), + Connection(175, 396), + Connection(396, 199), + Connection(199, 175), + Connection(312, 311), + Connection(311, 268), + Connection(268, 312), + Connection(276, 283), + Connection(283, 445), + Connection(445, 276), + Connection(390, 373), + Connection(373, 339), + Connection(339, 390), + Connection(295, 282), + Connection(282, 296), + Connection(296, 295), + Connection(448, 449), + Connection(449, 346), + Connection(346, 448), + Connection(356, 264), + Connection(264, 454), + Connection(454, 356), + Connection(337, 336), + Connection(336, 299), + Connection(299, 337), + Connection(337, 338), + Connection(338, 151), + Connection(151, 337), + Connection(294, 278), + Connection(278, 455), + Connection(455, 294), + Connection(308, 292), + Connection(292, 415), + Connection(415, 308), + Connection(429, 358), + Connection(358, 355), + Connection(355, 429), + Connection(265, 340), + Connection(340, 372), + Connection(372, 265), + Connection(352, 346), + Connection(346, 280), + Connection(280, 352), + Connection(295, 442), + Connection(442, 282), + Connection(282, 295), + Connection(354, 19), + Connection(19, 370), + Connection(370, 354), + Connection(285, 441), + Connection(441, 295), + Connection(295, 285), + Connection(195, 248), + Connection(248, 197), + Connection(197, 195), + Connection(457, 440), + Connection(440, 274), + Connection(274, 457), + Connection(301, 300), + Connection(300, 368), + Connection(368, 301), + Connection(417, 351), + Connection(351, 465), + Connection(465, 417), + Connection(251, 301), + Connection(301, 389), + Connection(389, 251), + Connection(394, 395), + Connection(395, 379), + Connection(379, 394), + Connection(399, 412), + Connection(412, 419), + Connection(419, 399), + Connection(410, 436), + Connection(436, 322), + Connection(322, 410), + Connection(326, 2), + Connection(2, 393), + Connection(393, 326), + Connection(354, 370), + Connection(370, 461), + Connection(461, 354), + Connection(393, 164), + Connection(164, 267), + Connection(267, 393), + Connection(268, 302), + Connection(302, 12), + Connection(12, 268), + Connection(312, 268), + Connection(268, 13), + Connection(13, 312), + Connection(298, 293), + Connection(293, 301), + Connection(301, 298), + Connection(265, 446), + Connection(446, 340), + Connection(340, 265), + Connection(280, 330), + Connection(330, 425), + Connection(425, 280), + Connection(322, 426), + Connection(426, 391), + Connection(391, 322), + Connection(420, 429), + Connection(429, 437), + Connection(437, 420), + Connection(393, 391), + Connection(391, 326), + Connection(326, 393), + Connection(344, 440), + Connection(440, 438), + Connection(438, 344), + Connection(458, 459), + Connection(459, 461), + Connection(461, 458), + Connection(364, 434), + Connection(434, 394), + Connection(394, 364), + Connection(428, 396), + Connection(396, 262), + Connection(262, 428), + Connection(274, 354), + Connection(354, 457), + Connection(457, 274), + Connection(317, 316), + Connection(316, 402), + Connection(402, 317), + Connection(316, 315), + Connection(315, 403), + Connection(403, 316), + Connection(315, 314), + Connection(314, 404), + Connection(404, 315), + Connection(314, 313), + Connection(313, 405), + Connection(405, 314), + Connection(313, 421), + Connection(421, 406), + Connection(406, 313), + Connection(323, 366), + Connection(366, 361), + Connection(361, 323), + Connection(292, 306), + Connection(306, 407), + Connection(407, 292), + Connection(306, 291), + Connection(291, 408), + Connection(408, 306), + Connection(291, 287), + Connection(287, 409), + Connection(409, 291), + Connection(287, 432), + Connection(432, 410), + Connection(410, 287), + Connection(427, 434), + Connection(434, 411), + Connection(411, 427), + Connection(372, 264), + Connection(264, 383), + Connection(383, 372), + Connection(459, 309), + Connection(309, 457), + Connection(457, 459), + Connection(366, 352), + Connection(352, 401), + Connection(401, 366), + Connection(1, 274), + Connection(274, 4), + Connection(4, 1), + Connection(418, 421), + Connection(421, 262), + Connection(262, 418), + Connection(331, 294), + Connection(294, 358), + Connection(358, 331), + Connection(435, 433), + Connection(433, 367), + Connection(367, 435), + Connection(392, 289), + Connection(289, 439), + Connection(439, 392), + Connection(328, 462), + Connection(462, 326), + Connection(326, 328), + Connection(94, 2), + Connection(2, 370), + Connection(370, 94), + Connection(289, 305), + Connection(305, 455), + Connection(455, 289), + Connection(339, 254), + Connection(254, 448), + Connection(448, 339), + Connection(359, 255), + Connection(255, 446), + Connection(446, 359), + Connection(254, 253), + Connection(253, 449), + Connection(449, 254), + Connection(253, 252), + Connection(252, 450), + Connection(450, 253), + Connection(252, 256), + Connection(256, 451), + Connection(451, 252), + Connection(256, 341), + Connection(341, 452), + Connection(452, 256), + Connection(414, 413), + Connection(413, 463), + Connection(463, 414), + Connection(286, 441), + Connection(441, 414), + Connection(414, 286), + Connection(286, 258), + Connection(258, 441), + Connection(441, 286), + Connection(258, 257), + Connection(257, 442), + Connection(442, 258), + Connection(257, 259), + Connection(259, 443), + Connection(443, 257), + Connection(259, 260), + Connection(260, 444), + Connection(444, 259), + Connection(260, 467), + Connection(467, 445), + Connection(445, 260), + Connection(309, 459), + Connection(459, 250), + Connection(250, 309), + Connection(305, 289), + Connection(289, 290), + Connection(290, 305), + Connection(305, 290), + Connection(290, 460), + Connection(460, 305), + Connection(401, 376), + Connection(376, 435), + Connection(435, 401), + Connection(309, 250), + Connection(250, 392), + Connection(392, 309), + Connection(376, 411), + Connection(411, 433), + Connection(433, 376), + Connection(453, 341), + Connection(341, 464), + Connection(464, 453), + Connection(357, 453), + Connection(453, 465), + Connection(465, 357), + Connection(343, 357), + Connection(357, 412), + Connection(412, 343), + Connection(437, 343), + Connection(343, 399), + Connection(399, 437), + Connection(344, 360), + Connection(360, 440), + Connection(440, 344), + Connection(420, 437), + Connection(437, 456), + Connection(456, 420), + Connection(360, 420), + Connection(420, 363), + Connection(363, 360), + Connection(361, 401), + Connection(401, 288), + Connection(288, 361), + Connection(265, 372), + Connection(372, 353), + Connection(353, 265), + Connection(390, 339), + Connection(339, 249), + Connection(249, 390), + Connection(339, 448), + Connection(448, 255), + Connection(255, 339), + ] + + +@dataclasses.dataclass +class FaceLandmarkerResult: + """The face landmarks detection result from FaceLandmarker, where each vector element represents a single face detected in the image. + + Attributes: + face_landmarks: Detected face landmarks in normalized image coordinates. + face_blendshapes: Optional face blendshapes results. + facial_transformation_matrixes: Optional facial transformation matrix. + """ + + face_landmarks: List[List[landmark_module.NormalizedLandmark]] + face_blendshapes: List[List[category_module.Category]] + facial_transformation_matrixes: List[np.ndarray] + + +def _build_landmarker_result( + output_packets: Mapping[str, packet_module.Packet] +) -> FaceLandmarkerResult: + """Constructs a `FaceLandmarkerResult` from output packets.""" + face_landmarks_proto_list = packet_getter.get_proto_list( + output_packets[_NORM_LANDMARKS_STREAM_NAME] + ) + + face_landmarks_results = [] + for proto in face_landmarks_proto_list: + face_landmarks = landmark_pb2.NormalizedLandmarkList() + face_landmarks.MergeFrom(proto) + face_landmarks_list = [] + for face_landmark in face_landmarks.landmark: + face_landmarks_list.append( + landmark_module.NormalizedLandmark.create_from_pb2(face_landmark) + ) + face_landmarks_results.append(face_landmarks_list) + + face_blendshapes_results = [] + if _BLENDSHAPES_STREAM_NAME in output_packets: + face_blendshapes_proto_list = packet_getter.get_proto_list( + output_packets[_BLENDSHAPES_STREAM_NAME] + ) + for proto in face_blendshapes_proto_list: + face_blendshapes_categories = [] + face_blendshapes_classifications = classification_pb2.ClassificationList() + face_blendshapes_classifications.MergeFrom(proto) + for face_blendshapes in face_blendshapes_classifications.classification: + face_blendshapes_categories.append( + category_module.Category( + index=face_blendshapes.index, + score=face_blendshapes.score, + display_name=face_blendshapes.display_name, + category_name=face_blendshapes.label, + ) + ) + face_blendshapes_results.append(face_blendshapes_categories) + + facial_transformation_matrixes_results = [] + if _FACE_GEOMETRY_STREAM_NAME in output_packets: + facial_transformation_matrixes_proto_list = packet_getter.get_proto_list( + output_packets[_FACE_GEOMETRY_STREAM_NAME] + ) + for proto in facial_transformation_matrixes_proto_list: + if hasattr(proto, 'pose_transform_matrix'): + matrix_data = matrix_data_pb2.MatrixData() + matrix_data.MergeFrom(proto.pose_transform_matrix) + matrix = np.array(matrix_data.packed_data) + matrix = matrix.reshape((matrix_data.rows, matrix_data.cols)) + matrix = ( + matrix if matrix_data.layout == _LayoutEnum.ROW_MAJOR else matrix.T + ) + facial_transformation_matrixes_results.append(matrix) + + return FaceLandmarkerResult( + face_landmarks_results, + face_blendshapes_results, + facial_transformation_matrixes_results, + ) + +def _build_landmarker_result2( + output_packets: Mapping[str, packet_module.Packet] +) -> FaceLandmarkerResult: + """Constructs a `FaceLandmarkerResult` from output packets.""" + face_landmarks_proto_list = packet_getter.get_proto_list( + output_packets[_NORM_LANDMARKS_STREAM_NAME] + ) + + face_landmarks_results = [] + for proto in face_landmarks_proto_list: + face_landmarks = landmark_pb2.NormalizedLandmarkList() + face_landmarks.MergeFrom(proto) + face_landmarks_list = [] + for face_landmark in face_landmarks.landmark: + face_landmarks_list.append( + landmark_module.NormalizedLandmark.create_from_pb2(face_landmark) + ) + face_landmarks_results.append(face_landmarks_list) + + face_blendshapes_results = [] + if _BLENDSHAPES_STREAM_NAME in output_packets: + face_blendshapes_proto_list = packet_getter.get_proto_list( + output_packets[_BLENDSHAPES_STREAM_NAME] + ) + for proto in face_blendshapes_proto_list: + face_blendshapes_categories = [] + face_blendshapes_classifications = classification_pb2.ClassificationList() + face_blendshapes_classifications.MergeFrom(proto) + for face_blendshapes in face_blendshapes_classifications.classification: + face_blendshapes_categories.append( + category_module.Category( + index=face_blendshapes.index, + score=face_blendshapes.score, + display_name=face_blendshapes.display_name, + category_name=face_blendshapes.label, + ) + ) + face_blendshapes_results.append(face_blendshapes_categories) + + facial_transformation_matrixes_results = [] + if _FACE_GEOMETRY_STREAM_NAME in output_packets: + facial_transformation_matrixes_proto_list = packet_getter.get_proto_list( + output_packets[_FACE_GEOMETRY_STREAM_NAME] + ) + for proto in facial_transformation_matrixes_proto_list: + if hasattr(proto, 'pose_transform_matrix'): + matrix_data = matrix_data_pb2.MatrixData() + matrix_data.MergeFrom(proto.pose_transform_matrix) + matrix = np.array(matrix_data.packed_data) + matrix = matrix.reshape((matrix_data.rows, matrix_data.cols)) + matrix = ( + matrix if matrix_data.layout == _LayoutEnum.ROW_MAJOR else matrix.T + ) + facial_transformation_matrixes_results.append(matrix) + + return FaceLandmarkerResult( + face_landmarks_results, + face_blendshapes_results, + facial_transformation_matrixes_results, + ), facial_transformation_matrixes_proto_list[0].mesh + +@dataclasses.dataclass +class FaceLandmarkerOptions: + """Options for the face landmarker task. + + Attributes: + base_options: Base options for the face landmarker task. + running_mode: The running mode of the task. Default to the image mode. + FaceLandmarker has three running modes: 1) The image mode for detecting + face landmarks on single image inputs. 2) The video mode for detecting + face landmarks on the decoded frames of a video. 3) The live stream mode + for detecting face landmarks on the live stream of input data, such as + from camera. In this mode, the "result_callback" below must be specified + to receive the detection results asynchronously. + num_faces: The maximum number of faces that can be detected by the + FaceLandmarker. + min_face_detection_confidence: The minimum confidence score for the face + detection to be considered successful. + min_face_presence_confidence: The minimum confidence score of face presence + score in the face landmark detection. + min_tracking_confidence: The minimum confidence score for the face tracking + to be considered successful. + output_face_blendshapes: Whether FaceLandmarker outputs face blendshapes + classification. Face blendshapes are used for rendering the 3D face model. + output_facial_transformation_matrixes: Whether FaceLandmarker outputs facial + transformation_matrix. Facial transformation matrix is used to transform + the face landmarks in canonical face to the detected face, so that users + can apply face effects on the detected landmarks. + result_callback: The user-defined result callback for processing live stream + data. The result callback should only be specified when the running mode + is set to the live stream mode. + """ + + base_options: _BaseOptions + running_mode: _RunningMode = _RunningMode.IMAGE + num_faces: int = 1 + min_face_detection_confidence: float = 0.5 + min_face_presence_confidence: float = 0.5 + min_tracking_confidence: float = 0.5 + output_face_blendshapes: bool = False + output_facial_transformation_matrixes: bool = False + result_callback: Optional[ + Callable[[FaceLandmarkerResult, image_module.Image, int], None] + ] = None + + @doc_controls.do_not_generate_docs + def to_pb2(self) -> _FaceLandmarkerGraphOptionsProto: + """Generates an FaceLandmarkerGraphOptions protobuf object.""" + base_options_proto = self.base_options.to_pb2() + base_options_proto.use_stream_mode = ( + False if self.running_mode == _RunningMode.IMAGE else True + ) + + # Initialize the face landmarker options from base options. + face_landmarker_options_proto = _FaceLandmarkerGraphOptionsProto( + base_options=base_options_proto + ) + + # Configure face detector options. + face_landmarker_options_proto.face_detector_graph_options.num_faces = ( + self.num_faces + ) + face_landmarker_options_proto.face_detector_graph_options.min_detection_confidence = ( + self.min_face_detection_confidence + ) + + # Configure face landmark detector options. + face_landmarker_options_proto.min_tracking_confidence = ( + self.min_tracking_confidence + ) + face_landmarker_options_proto.face_landmarks_detector_graph_options.min_detection_confidence = ( + self.min_face_detection_confidence + ) + return face_landmarker_options_proto + + +class FaceLandmarker(base_vision_task_api.BaseVisionTaskApi): + """Class that performs face landmarks detection on images.""" + + @classmethod + def create_from_model_path(cls, model_path: str) -> 'FaceLandmarker': + """Creates an `FaceLandmarker` object from a TensorFlow Lite model and the default `FaceLandmarkerOptions`. + + Note that the created `FaceLandmarker` instance is in image mode, for + detecting face landmarks on single image inputs. + + Args: + model_path: Path to the model. + + Returns: + `FaceLandmarker` object that's created from the model file and the + default `FaceLandmarkerOptions`. + + Raises: + ValueError: If failed to create `FaceLandmarker` object from the + provided file such as invalid file path. + RuntimeError: If other types of error occurred. + """ + base_options = _BaseOptions(model_asset_path=model_path) + options = FaceLandmarkerOptions( + base_options=base_options, running_mode=_RunningMode.IMAGE + ) + return cls.create_from_options(options) + + @classmethod + def create_from_options( + cls, options: FaceLandmarkerOptions + ) -> 'FaceLandmarker': + """Creates the `FaceLandmarker` object from face landmarker options. + + Args: + options: Options for the face landmarker task. + + Returns: + `FaceLandmarker` object that's created from `options`. + + Raises: + ValueError: If failed to create `FaceLandmarker` object from + `FaceLandmarkerOptions` such as missing the model. + RuntimeError: If other types of error occurred. + """ + + def packets_callback(output_packets: Mapping[str, packet_module.Packet]): + if output_packets[_IMAGE_OUT_STREAM_NAME].is_empty(): + return + + image = packet_getter.get_image(output_packets[_IMAGE_OUT_STREAM_NAME]) + if output_packets[_IMAGE_OUT_STREAM_NAME].is_empty(): + return + + if output_packets[_NORM_LANDMARKS_STREAM_NAME].is_empty(): + empty_packet = output_packets[_NORM_LANDMARKS_STREAM_NAME] + options.result_callback( + FaceLandmarkerResult([], [], []), + image, + empty_packet.timestamp.value // _MICRO_SECONDS_PER_MILLISECOND, + ) + return + + face_landmarks_result = _build_landmarker_result(output_packets) + timestamp = output_packets[_NORM_LANDMARKS_STREAM_NAME].timestamp + options.result_callback( + face_landmarks_result, + image, + timestamp.value // _MICRO_SECONDS_PER_MILLISECOND, + ) + + output_streams = [ + ':'.join([_NORM_LANDMARKS_TAG, _NORM_LANDMARKS_STREAM_NAME]), + ':'.join([_IMAGE_TAG, _IMAGE_OUT_STREAM_NAME]), + ] + + if options.output_face_blendshapes: + output_streams.append( + ':'.join([_BLENDSHAPES_TAG, _BLENDSHAPES_STREAM_NAME]) + ) + if options.output_facial_transformation_matrixes: + output_streams.append( + ':'.join([_FACE_GEOMETRY_TAG, _FACE_GEOMETRY_STREAM_NAME]) + ) + + task_info = _TaskInfo( + task_graph=_TASK_GRAPH_NAME, + input_streams=[ + ':'.join([_IMAGE_TAG, _IMAGE_IN_STREAM_NAME]), + ':'.join([_NORM_RECT_TAG, _NORM_RECT_STREAM_NAME]), + ], + output_streams=output_streams, + task_options=options, + ) + return cls( + task_info.generate_graph_config( + enable_flow_limiting=options.running_mode + == _RunningMode.LIVE_STREAM + ), + options.running_mode, + packets_callback if options.result_callback else None, + ) + + def detect( + self, + image: image_module.Image, + image_processing_options: Optional[_ImageProcessingOptions] = None, + ) -> FaceLandmarkerResult: + """Performs face landmarks detection on the given image. + + Only use this method when the FaceLandmarker is created with the image + running mode. + + The image can be of any size with format RGB or RGBA. + TODO: Describes how the input image will be preprocessed after the yuv + support is implemented. + + Args: + image: MediaPipe Image. + image_processing_options: Options for image processing. + + Returns: + The face landmarks detection results. + + Raises: + ValueError: If any of the input arguments is invalid. + RuntimeError: If face landmarker detection failed to run. + """ + + normalized_rect = self.convert_to_normalized_rect( + image_processing_options, image, roi_allowed=False + ) + output_packets = self._process_image_data({ + _IMAGE_IN_STREAM_NAME: packet_creator.create_image(image), + _NORM_RECT_STREAM_NAME: packet_creator.create_proto( + normalized_rect.to_pb2() + ), + }) + + if output_packets[_NORM_LANDMARKS_STREAM_NAME].is_empty(): + return FaceLandmarkerResult([], [], []) + + return _build_landmarker_result2(output_packets) + + def detect_for_video( + self, + image: image_module.Image, + timestamp_ms: int, + image_processing_options: Optional[_ImageProcessingOptions] = None, + ): + """Performs face landmarks detection on the provided video frame. + + Only use this method when the FaceLandmarker is created with the video + running mode. + + Only use this method when the FaceLandmarker is created with the video + running mode. It's required to provide the video frame's timestamp (in + milliseconds) along with the video frame. The input timestamps should be + monotonically increasing for adjacent calls of this method. + + Args: + image: MediaPipe Image. + timestamp_ms: The timestamp of the input video frame in milliseconds. + image_processing_options: Options for image processing. + + Returns: + The face landmarks detection results. + + Raises: + ValueError: If any of the input arguments is invalid. + RuntimeError: If face landmarker detection failed to run. + """ + normalized_rect = self.convert_to_normalized_rect( + image_processing_options, image, roi_allowed=False + ) + output_packets = self._process_video_data({ + _IMAGE_IN_STREAM_NAME: packet_creator.create_image(image).at( + timestamp_ms * _MICRO_SECONDS_PER_MILLISECOND + ), + _NORM_RECT_STREAM_NAME: packet_creator.create_proto( + normalized_rect.to_pb2() + ).at(timestamp_ms * _MICRO_SECONDS_PER_MILLISECOND), + }) + + if output_packets[_NORM_LANDMARKS_STREAM_NAME].is_empty(): + return FaceLandmarkerResult([], [], []) + + return _build_landmarker_result2(output_packets) + + def detect_async( + self, + image: image_module.Image, + timestamp_ms: int, + image_processing_options: Optional[_ImageProcessingOptions] = None, + ) -> None: + """Sends live image data to perform face landmarks detection. + + The results will be available via the "result_callback" provided in the + FaceLandmarkerOptions. Only use this method when the FaceLandmarker is + created with the live stream running mode. + + Only use this method when the FaceLandmarker is created with the live + stream running mode. The input timestamps should be monotonically increasing + for adjacent calls of this method. This method will return immediately after + the input image is accepted. The results will be available via the + `result_callback` provided in the `FaceLandmarkerOptions`. The + `detect_async` method is designed to process live stream data such as + camera input. To lower the overall latency, face landmarker may drop the + input images if needed. In other words, it's not guaranteed to have output + per input image. + + The `result_callback` provides: + - The face landmarks detection results. + - The input image that the face landmarker runs on. + - The input timestamp in milliseconds. + + Args: + image: MediaPipe Image. + timestamp_ms: The timestamp of the input image in milliseconds. + image_processing_options: Options for image processing. + + Raises: + ValueError: If the current input timestamp is smaller than what the + face landmarker has already processed. + """ + normalized_rect = self.convert_to_normalized_rect( + image_processing_options, image, roi_allowed=False + ) + self._send_live_stream_data({ + _IMAGE_IN_STREAM_NAME: packet_creator.create_image(image).at( + timestamp_ms * _MICRO_SECONDS_PER_MILLISECOND + ), + _NORM_RECT_STREAM_NAME: packet_creator.create_proto( + normalized_rect.to_pb2() + ).at(timestamp_ms * _MICRO_SECONDS_PER_MILLISECOND), + }) \ No newline at end of file diff --git a/media_pipe/mp_utils.py b/media_pipe/mp_utils.py new file mode 100644 index 0000000..aaae45d --- /dev/null +++ b/media_pipe/mp_utils.py @@ -0,0 +1,38 @@ +import os +import mediapipe as mp + +from mediapipe.tasks import python +from mediapipe.tasks.python import vision +from . import face_landmark + +CUR_DIR = os.path.dirname(__file__) + +class LMKExtractor(): + def __init__(self): + # Create an FaceLandmarker object. + self.mode = mp.tasks.vision.FaceDetectorOptions.running_mode.IMAGE + base_options = python.BaseOptions(model_asset_path=os.path.join(CUR_DIR, 'mp_models','face_landmarker_v2_with_blendshapes.task')) + base_options.delegate = mp.tasks.BaseOptions.Delegate.CPU + options = vision.FaceLandmarkerOptions(base_options=base_options, + running_mode=self.mode, + output_face_blendshapes=False, + output_facial_transformation_matrixes=True, + num_faces=1, + min_face_detection_confidence=0.5, + min_face_presence_confidence=0.5, + min_tracking_confidence=0.5) + self.detector = face_landmark.FaceLandmarker.create_from_options(options) + + det_base_options = python.BaseOptions(model_asset_path=os.path.join(CUR_DIR, 'mp_models','blaze_face_short_range.tflite')) + det_options = vision.FaceDetectorOptions(base_options=det_base_options) + self.det_detector = vision.FaceDetector.create_from_options(det_options) + + def __call__(self, img): + image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img) + try: + detection_result, _ = self.detector.detect(image) + except: + return None + + return detection_result.face_landmarks + \ No newline at end of file diff --git a/nodes.py b/nodes.py index 29d718e..8c3a03e 100644 --- a/nodes.py +++ b/nodes.py @@ -4,106 +4,77 @@ import yaml import folder_paths import comfy.model_management as mm import comfy.utils +import numpy as np +import cv2 +from tqdm import tqdm +import gc script_directory = os.path.dirname(os.path.abspath(__file__)) from .liveportrait.live_portrait_pipeline import LivePortraitPipeline -from .liveportrait.utils.cropper import Cropper +try: + from .liveportrait.utils.cropper import CropperMediaPipe +except: + raise ModuleNotFoundError("Can't load MediaPipe, MediaPipeCropper not available") +try: + from .liveportrait.utils.cropper import CropperInsightFace +except: + raise ModuleNotFoundError("Can't load InsightFace, InsightFaceCropper not available") + from .liveportrait.modules.spade_generator import SPADEDecoder from .liveportrait.modules.warping_network import WarpingNetwork from .liveportrait.modules.motion_extractor import MotionExtractor -from .liveportrait.modules.appearance_feature_extractor import AppearanceFeatureExtractor -from .liveportrait.modules.stitching_retargeting_network import StitchingRetargetingNetwork +from .liveportrait.modules.appearance_feature_extractor import ( + AppearanceFeatureExtractor, +) +from .liveportrait.modules.stitching_retargeting_network import ( + StitchingRetargetingNetwork, +) +from .liveportrait.utils.camera import get_rotation_matrix +from .liveportrait.utils.crop import _transform_img_kornia + +import logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +log = logging.getLogger(__name__) class InferenceConfig: - def __init__(self, - mask_crop = None, - flag_use_half_precision=True, - flag_lip_zero=True, - lip_zero_threshold=0.03, - flag_eye_retargeting=False, - flag_lip_retargeting=False, - flag_stitching=True, - flag_relative=True, - anchor_frame=0, - input_shape=(256, 256), - flag_write_result=True, - flag_pasteback=True, - ref_max_shape=1280, - ref_shape_n=2, - device_id=0, - flag_do_crop=True, - flag_do_rot=True): + def __init__( + self, + flag_use_half_precision=True, + flag_lip_zero=True, + lip_zero_threshold=0.03, + flag_eye_retargeting=False, + flag_lip_retargeting=False, + flag_stitching=True, + input_shape=(256, 256), + device_id=0, + flag_do_rot=True, + ): self.flag_use_half_precision = flag_use_half_precision self.flag_lip_zero = flag_lip_zero self.lip_zero_threshold = lip_zero_threshold self.flag_eye_retargeting = flag_eye_retargeting self.flag_lip_retargeting = flag_lip_retargeting self.flag_stitching = flag_stitching - self.flag_relative = flag_relative - self.anchor_frame = anchor_frame self.input_shape = input_shape - self.flag_write_result = flag_write_result - self.flag_pasteback = flag_pasteback - self.ref_max_shape = ref_max_shape - self.ref_shape_n = ref_shape_n self.device_id = device_id - self.flag_do_crop = flag_do_crop self.flag_do_rot = flag_do_rot - self.mask_crop=mask_crop - -class CropConfig: - def __init__(self, dsize=512, scale=2.3, vx_ratio=0, vy_ratio=-0.125): - self.dsize = dsize - self.scale = scale - self.vx_ratio = vx_ratio - self.vy_ratio = vy_ratio - -class ArgumentConfig: - def __init__(self, - device_id=0, - flag_lip_zero=True, - flag_eye_retargeting=False, - flag_lip_retargeting=False, - flag_stitching=True, - flag_relative=True, - flag_pasteback=True, - flag_do_crop=True, - flag_do_rot=True, - dsize=512, - scale=2.3, - vx_ratio=0, - vy_ratio=-0.125, - ): - self.device_id = device_id - self.flag_lip_zero = flag_lip_zero - self.flag_eye_retargeting = flag_eye_retargeting - self.flag_lip_retargeting = flag_lip_retargeting - self.flag_stitching = flag_stitching - self.flag_relative = flag_relative - self.flag_pasteback = flag_pasteback - self.flag_do_crop = flag_do_crop - self.flag_do_rot = flag_do_rot - self.dsize = dsize - self.scale = scale - self.vx_ratio = vx_ratio - self.vy_ratio = vy_ratio - + class DownloadAndLoadLivePortraitModels: @classmethod def INPUT_TYPES(s): - return {"required": { - }, + return { + "required": {}, "optional": { - "precision": ( + "precision": ( [ - 'auto', - 'fp16', - 'fp32', - ], { - "default": 'auto' - }), - } + "fp16", + "fp32", + "auto", + ], + {"default": "auto"}, + ), + }, } RETURN_TYPES = ("LIVEPORTRAITPIPE",) @@ -111,26 +82,26 @@ class DownloadAndLoadLivePortraitModels: FUNCTION = "loadmodel" CATEGORY = "LivePortrait" - def loadmodel(self, precision='auto'): + def loadmodel(self, precision="fp16"): device = mm.get_torch_device() mm.soft_empty_cache() if precision == 'auto': try: if mm.is_device_mps(device): - print("LivePortrait using fp32 for MPS") + log.info("LivePortrait using fp32 for MPS") dtype = 'fp32' elif mm.should_use_fp16(): - print("LivePortrait using fp16") + log.info("LivePortrait using fp16") dtype = 'fp16' else: - print("LivePortrait using fp32") + log.info("LivePortrait using fp32") dtype = 'fp32' except: raise AttributeError("ComfyUI version too old, can't autodetect properly. Set your dtypes manually.") else: dtype = precision - print(f"LivePortrait using {dtype}") + log.info(f"LivePortrait using {dtype}") pbar = comfy.utils.ProgressBar(3) @@ -138,86 +109,108 @@ class DownloadAndLoadLivePortraitModels: model_path = os.path.join(download_path) if not os.path.exists(model_path): - print(f"Downloading model to: {model_path}") + log.info(f"Downloading model to: {model_path}") from huggingface_hub import snapshot_download - snapshot_download(repo_id="Kijai/LivePortrait_safetensors", - local_dir=download_path, - local_dir_use_symlinks=False) - model_config_path = os.path.join(script_directory, 'liveportrait', 'config', 'models.yaml') - with open(model_config_path, 'r') as file: + snapshot_download( + repo_id="Kijai/LivePortrait_safetensors", + local_dir=download_path, + local_dir_use_symlinks=False, + ) + + model_config_path = os.path.join( + script_directory, "liveportrait", "config", "models.yaml" + ) + with open(model_config_path, "r") as file: model_config = yaml.safe_load(file) - - feature_extractor_path = os.path.join(model_path, 'appearance_feature_extractor.safetensors') - motion_extractor_path = os.path.join(model_path, 'motion_extractor.safetensors') - warping_module_path = os.path.join(model_path, 'warping_module.safetensors') - spade_generator_path = os.path.join(model_path, 'spade_generator.safetensors') - stitching_retargeting_path = os.path.join(model_path, 'stitching_retargeting_module.safetensors') + + feature_extractor_path = os.path.join( + model_path, "appearance_feature_extractor.safetensors" + ) + motion_extractor_path = os.path.join(model_path, "motion_extractor.safetensors") + warping_module_path = os.path.join(model_path, "warping_module.safetensors") + spade_generator_path = os.path.join(model_path, "spade_generator.safetensors") + stitching_retargeting_path = os.path.join( + model_path, "stitching_retargeting_module.safetensors" + ) # init F - model_params = model_config['model_params']['appearance_feature_extractor_params'] - self.appearance_feature_extractor = AppearanceFeatureExtractor(**model_params).to(device) - self.appearance_feature_extractor.load_state_dict(comfy.utils.load_torch_file(feature_extractor_path)) + model_params = model_config["model_params"][ + "appearance_feature_extractor_params" + ] + self.appearance_feature_extractor = AppearanceFeatureExtractor( + **model_params + ).to(device) + self.appearance_feature_extractor.load_state_dict( + comfy.utils.load_torch_file(feature_extractor_path) + ) self.appearance_feature_extractor.eval() - print('Load appearance_feature_extractor done.') + log.info("Load appearance_feature_extractor done.") pbar.update(1) # init M - model_params = model_config['model_params']['motion_extractor_params'] + model_params = model_config["model_params"]["motion_extractor_params"] self.motion_extractor = MotionExtractor(**model_params).to(device) - self.motion_extractor.load_state_dict(comfy.utils.load_torch_file(motion_extractor_path)) + self.motion_extractor.load_state_dict( + comfy.utils.load_torch_file(motion_extractor_path) + ) self.motion_extractor.eval() - print('Load motion_extractor done.') + log.info("Load motion_extractor done.") pbar.update(1) # init W - model_params = model_config['model_params']['warping_module_params'] + model_params = model_config["model_params"]["warping_module_params"] self.warping_module = WarpingNetwork(**model_params).to(device) - self.warping_module.load_state_dict(comfy.utils.load_torch_file(warping_module_path)) + self.warping_module.load_state_dict( + comfy.utils.load_torch_file(warping_module_path) + ) self.warping_module.eval() - print('Load warping_module done.') + log.info("Load warping_module done.") pbar.update(1) # init G - model_params = model_config['model_params']['spade_generator_params'] + model_params = model_config["model_params"]["spade_generator_params"] self.spade_generator = SPADEDecoder(**model_params).to(device) - self.spade_generator.load_state_dict(comfy.utils.load_torch_file(spade_generator_path)) + self.spade_generator.load_state_dict( + comfy.utils.load_torch_file(spade_generator_path) + ) self.spade_generator.eval() - print('Load spade_generator done.') + log.info("Load spade_generator done.") pbar.update(1) def filter_checkpoint_for_model(checkpoint, prefix): """Filter and adjust the checkpoint dictionary for a specific model based on the prefix.""" # Create a new dictionary where keys are adjusted by removing the prefix and the model name - filtered_checkpoint = {key.replace(prefix + "_module.", ""): value for key, value in checkpoint.items() if key.startswith(prefix)} + filtered_checkpoint = { + key.replace(prefix + "_module.", ""): value + for key, value in checkpoint.items() + if key.startswith(prefix) + } return filtered_checkpoint - config = model_config['model_params']['stitching_retargeting_module_params'] + config = model_config["model_params"]["stitching_retargeting_module_params"] checkpoint = comfy.utils.load_torch_file(stitching_retargeting_path) - stitcher_prefix = 'retarget_shoulder' + stitcher_prefix = "retarget_shoulder" stitcher_checkpoint = filter_checkpoint_for_model(checkpoint, stitcher_prefix) - stitcher = StitchingRetargetingNetwork(**config.get('stitching')) + stitcher = StitchingRetargetingNetwork(**config.get("stitching")) stitcher.load_state_dict(stitcher_checkpoint) - stitcher = stitcher.to(device) - stitcher.eval() + stitcher = stitcher.to(device).eval() - lip_prefix = 'retarget_mouth' + lip_prefix = "retarget_mouth" lip_checkpoint = filter_checkpoint_for_model(checkpoint, lip_prefix) - retargetor_lip = StitchingRetargetingNetwork(**config.get('lip')) + retargetor_lip = StitchingRetargetingNetwork(**config.get("lip")) retargetor_lip.load_state_dict(lip_checkpoint) - retargetor_lip = retargetor_lip.to(device) - retargetor_lip.eval() + retargetor_lip = retargetor_lip.to(device).eval() - eye_prefix = 'retarget_eye' + eye_prefix = "retarget_eye" eye_checkpoint = filter_checkpoint_for_model(checkpoint, eye_prefix) - retargetor_eye = StitchingRetargetingNetwork(**config.get('eye')) + retargetor_eye = StitchingRetargetingNetwork(**config.get("eye")) retargetor_eye.load_state_dict(eye_checkpoint) - retargetor_eye = retargetor_eye.to(device) - retargetor_eye.eval() - print('Load stitching_retargeting_module done.') + retargetor_eye = retargetor_eye.to(device).eval() + log.info("Load stitching_retargeting_module done.") self.stich_retargeting_module = { - 'stitching': stitcher, - 'lip': retargetor_lip, - 'eye': retargetor_eye + "stitching": stitcher, + "lip": retargetor_lip, + "eye": retargetor_eye, } pipeline = LivePortraitPipeline( @@ -227,98 +220,553 @@ class DownloadAndLoadLivePortraitModels: self.spade_generator, self.stich_retargeting_module, InferenceConfig( - device_id=device, - flag_use_half_precision = True if dtype == 'fp16' else False - ) + device_id=device, + flag_use_half_precision=True if precision == "fp16" else False, + ), ) return (pipeline,) + class LivePortraitProcess: @classmethod def INPUT_TYPES(s): return {"required": { "pipeline": ("LIVEPORTRAITPIPE",), + "crop_info": ("CROPINFO", {"default": {}}), "source_image": ("IMAGE",), "driving_images": ("IMAGE",), + "lip_zero": ("BOOLEAN", {"default": False}), + "lip_zero_threshold": ("FLOAT", {"default": 0.03, "min": 0.001, "max": 4.0, "step": 0.001}), + "stitching": ("BOOLEAN", {"default": True}), + "delta_multiplier": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.001}), + "mismatch_method": ( + [ + "constant", + "cycle", + "mirror", + "cut" + ], + {"default": "constant"}, + ), + + "relative_motion_mode": ( + [ + "relative", + "source_video_smoothed", + "relative_rotation_only", + "single_frame", + "off" + ], + ), + "driving_smooth_observation_variance": ("FLOAT", {"default": 3e-6, "min": 1e-11, "max": 1e-2, "step": 1e-11}), + }, + + "optional": { + "opt_retargeting_info": ("RETARGETINGINFO", {"default": None}), + } + } + + RETURN_TYPES = ( + "IMAGE", + "LP_OUT", + ) + RETURN_NAMES = ( + "cropped_image", + "output", + ) + FUNCTION = "process" + CATEGORY = "LivePortrait" + + def process( + self, + source_image: torch.Tensor, + driving_images: torch.Tensor, + crop_info: dict, + pipeline: LivePortraitPipeline, + lip_zero: bool, + lip_zero_threshold: float, + stitching: bool, + relative_motion_mode: str, + driving_smooth_observation_variance: float, + delta_multiplier: float = 1.0, + mismatch_method: str = "constant", + opt_retargeting_info: dict = None, + ): + if driving_images.shape[0] < source_image.shape[0]: + raise ValueError("The number of driving images should be larger than the number of source images.") + + if opt_retargeting_info is not None: + pipeline.live_portrait_wrapper.cfg.flag_eye_retargeting = opt_retargeting_info["eye_retargeting"] + pipeline.live_portrait_wrapper.cfg.eyes_retargeting_multiplier = (opt_retargeting_info["eyes_retargeting_multiplier"]) + pipeline.live_portrait_wrapper.cfg.flag_lip_retargeting = opt_retargeting_info["lip_retargeting"] + pipeline.live_portrait_wrapper.cfg.lip_retargeting_multiplier = (opt_retargeting_info["lip_retargeting_multiplier"]) + driving_landmarks = opt_retargeting_info["driving_landmarks"] + else: + pipeline.live_portrait_wrapper.cfg.flag_eye_retargeting = False + pipeline.live_portrait_wrapper.cfg.eyes_retargeting_multiplier = 1.0 + pipeline.live_portrait_wrapper.cfg.flag_lip_retargeting = False + pipeline.live_portrait_wrapper.cfg.lip_retargeting_multiplier = 1.0 + driving_landmarks = None + + pipeline.live_portrait_wrapper.cfg.flag_stitching = stitching + pipeline.live_portrait_wrapper.cfg.flag_lip_zero = lip_zero + pipeline.live_portrait_wrapper.cfg.lip_zero_threshold = lip_zero_threshold + + if lip_zero and opt_retargeting_info is not None: + log.warning("Warning: lip_zero only has an effect with lip or eye retargeting") + + if driving_images.shape[1] != 256 or driving_images.shape[2] != 256: + driving_images_256 = comfy.utils.common_upscale(driving_images.permute(0, 3, 1, 2), 256, 256, "lanczos", "disabled") + else: + driving_images_256 = driving_images.permute(0, 3, 1, 2) + + if pipeline.live_portrait_wrapper.cfg.flag_use_half_precision: + driving_images_256 = driving_images_256.to(torch.float16) + + out = pipeline.execute( + driving_images_256, + crop_info, + driving_landmarks, + delta_multiplier, + relative_motion_mode, + driving_smooth_observation_variance, + mismatch_method + ) + + total_frames = len(out["out_list"]) + + if total_frames > 1: + cropped_image_list = [] + for i in (range(total_frames)): + if not out["out_list"][i]: + cropped_image_list.append(torch.zeros(1, 512, 512, 3, dtype=torch.float32, device = "cpu")) + else: + cropped_image = torch.clamp(out["out_list"][i]["out"], 0, 1).permute(0, 2, 3, 1).cpu() + cropped_image_list.append(cropped_image) + + cropped_out_tensors = torch.cat(cropped_image_list, dim=0) + else: + cropped_out_tensors = torch.clamp(out["out_list"][0]["out"], 0, 1).permute(0, 2, 3, 1) + + return (cropped_out_tensors, out,) + +class LivePortraitComposite: + @classmethod + def INPUT_TYPES(s): + return {"required": { + + "source_image": ("IMAGE",), + "cropped_image": ("IMAGE",), + "liveportrait_out": ("LP_OUT", ), + }, + "optional": { + "mask": ("MASK", {"default": None}), + } + } + + RETURN_TYPES = ( + "IMAGE", + "MASK", + ) + RETURN_NAMES = ( + "full_images", + "mask", + ) + FUNCTION = "process" + CATEGORY = "LivePortrait" + + def process(self, source_image, cropped_image, liveportrait_out, mask=None): + mm.soft_empty_cache() + gc.collect() + device = mm.get_torch_device() + if mm.is_device_mps(device): + device = torch.device('cpu') #this function returns NaNs on MPS, defaulting to CPU + + B, H, W, C = source_image.shape + source_image = source_image.permute(0, 3, 1, 2) # B,H,W,C -> B,C,H,W + cropped_image = cropped_image.permute(0, 3, 1, 2) + + if mask is not None: + crop_mask = mask.unsqueeze(-1).expand(-1, -1, -1, 3) + else: + log.info("Using default mask template") + crop_mask = cv2.imread(os.path.join(script_directory, "liveportrait", "utils", "resources", "mask_template.png"), cv2.IMREAD_COLOR) + crop_mask = torch.from_numpy(crop_mask) + crop_mask = crop_mask.unsqueeze(0).float() / 255.0 + + crop_info = liveportrait_out["crop_info"] + composited_image_list = [] + out_mask_list = [] + + total_frames = len(liveportrait_out["out_list"]) + log.info(f"Total frames: {total_frames}") + + pbar = comfy.utils.ProgressBar(total_frames) + for i in tqdm(range(total_frames), desc='Compositing..', total=total_frames): + safe_index = min(i, len(crop_info["crop_info_list"]) - 1) + + if liveportrait_out["mismatch_method"] == "cut": + source_frame = source_image[safe_index].unsqueeze(0).to(device) + else: + source_frame = _get_source_frame(source_image, i, liveportrait_out["mismatch_method"]).unsqueeze(0).to(device) + + if not liveportrait_out["out_list"][i]: + composited_image_list.append(source_frame.cpu()) + out_mask_list.append(torch.zeros((1, 3, H, W), device="cpu")) + else: + cropped_image = torch.clamp(liveportrait_out["out_list"][i]["out"], 0, 1).permute(0, 2, 3, 1) + + # Transform and blend + cropped_image_to_original = _transform_img_kornia( + cropped_image, + crop_info["crop_info_list"][safe_index]["M_c2o"], + dsize=(W, H), + device=device + ) + + mask_ori = _transform_img_kornia( + crop_mask, + crop_info["crop_info_list"][safe_index]["M_c2o"], + dsize=(W, H), + device=device + ) + + cropped_image_to_original_blend = torch.clip( + mask_ori * cropped_image_to_original + (1 - mask_ori) * source_frame, 0, 1 + ) + + composited_image_list.append(cropped_image_to_original_blend.cpu()) + out_mask_list.append(mask_ori.cpu()) + pbar.update(1) + + full_tensors_out = torch.cat(composited_image_list, dim=0) + full_tensors_out = full_tensors_out.permute(0, 2, 3, 1) + + mask_tensors_out = torch.cat(out_mask_list, dim=0) + mask_tensors_out = mask_tensors_out[:, 0, :, :] + + return ( + full_tensors_out.float(), + mask_tensors_out.float() + ) + +def _get_source_frame(source, idx, method): + if source.shape[0] == 1: + return source[0] + + if method == "constant": + return source[min(idx, source.shape[0] - 1)] + elif method == "cycle": + return source[idx % source.shape[0]] + elif method == "mirror": + cycle_length = 2 * source.shape[0] - 2 + mirror_idx = idx % cycle_length + if mirror_idx >= source.shape[0]: + mirror_idx = cycle_length - mirror_idx + return source[mirror_idx] + +class LivePortraitLoadCropper: + @classmethod + def INPUT_TYPES(s): + return {"required": { + + "onnx_device": ( + ['CPU', 'CUDA', 'ROCM', 'CoreML'], { + "default": 'CPU' + }), + "keep_model_loaded": ("BOOLEAN", {"default": True}) + }, + } + + RETURN_TYPES = ("LPCROPPER",) + RETURN_NAMES = ("cropper",) + FUNCTION = "crop" + CATEGORY = "LivePortrait" + + def crop(self, onnx_device, keep_model_loaded): + cropper_init_config = { + 'keep_model_loaded': keep_model_loaded, + 'onnx_device': onnx_device + } + + if not hasattr(self, 'cropper') or self.cropper is None or self.current_config != cropper_init_config: + self.current_config = cropper_init_config + self.cropper = CropperInsightFace(**cropper_init_config) + + return (self.cropper,) + +class LivePortraitLoadMediaPipeCropper: + @classmethod + def INPUT_TYPES(s): + return {"required": { + + "landmarkrunner_onnx_device": ( + ['CPU', 'CUDA', 'ROCM', 'CoreML'], { + "default": 'CPU' + }), + "keep_model_loaded": ("BOOLEAN", {"default": True}) + }, + } + + RETURN_TYPES = ("LPCROPPER",) + RETURN_NAMES = ("cropper",) + FUNCTION = "crop" + CATEGORY = "LivePortrait" + + def crop(self, landmarkrunner_onnx_device, keep_model_loaded): + cropper_init_config = { + 'keep_model_loaded': keep_model_loaded, + 'onnx_device': landmarkrunner_onnx_device + } + + if not hasattr(self, 'cropper') or self.cropper is None or self.current_config != cropper_init_config: + self.current_config = cropper_init_config + self.cropper = CropperMediaPipe(**cropper_init_config) + + return (self.cropper,) + +class LivePortraitCropper: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "pipeline": ("LIVEPORTRAITPIPE",), + "cropper": ("LPCROPPER",), + "source_image": ("IMAGE",), "dsize": ("INT", {"default": 512, "min": 64, "max": 2048}), "scale": ("FLOAT", {"default": 2.3, "min": 1.0, "max": 4.0, "step": 0.01}), - "vx_ratio": ("FLOAT", {"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.01}), - "vy_ratio": ("FLOAT", {"default": -0.125, "min": -1.0, "max": 1.0, "step": 0.01}), - "lip_zero": ("BOOLEAN", {"default": True}), + "vx_ratio": ("FLOAT", {"default": 0.0, "min": -1.0, "max": 1.0, "step": 0.001}), + "vy_ratio": ("FLOAT", {"default": -0.125, "min": -1.0, "max": 1.0, "step": 0.001}), + "face_index": ("INT", {"default": 0, "min": 0, "max": 100}), + "face_index_order": ( + [ + 'large-small', + 'left-right', + 'right-left', + 'top-bottom', + 'bottom-top', + 'small-large', + 'distance-from-retarget-face' + ], + ), + "rotate": ("BOOLEAN", {"default": True}), + }, + } + + RETURN_TYPES = ("IMAGE", "CROPINFO",) + RETURN_NAMES = ("cropped_image", "crop_info",) + FUNCTION = "process" + CATEGORY = "LivePortrait" + + def process(self, pipeline, cropper, source_image, dsize, scale, vx_ratio, vy_ratio, face_index, face_index_order, rotate): + source_image_np = (source_image.contiguous() * 255).byte().numpy() + + # Initialize lists + crop_info_list = [] + cropped_images_list = [] + source_info = [] + source_rot_list = [] + f_s_list = [] + x_s_list = [] + + # Initialize a progress bar for the combined operation + pbar = comfy.utils.ProgressBar(len(source_image_np)) + for i in tqdm(range(len(source_image_np)), desc='Detecting, cropping, and processing..', total=len(source_image_np)): + # Cropping operation + crop_info, cropped_image_256 = cropper.crop_single_image(source_image_np[i], dsize, scale, vy_ratio, vx_ratio, face_index, face_index_order, rotate) + + # Processing source images + if crop_info: + crop_info_list.append(crop_info) + + cropped_images_list.append(cropped_image_256) + + I_s = pipeline.live_portrait_wrapper.prepare_source(cropped_image_256) + + x_s_info = pipeline.live_portrait_wrapper.get_kp_info(I_s) + source_info.append(x_s_info) + + x_s = pipeline.live_portrait_wrapper.transform_keypoint(x_s_info) + x_s_list.append(x_s) + + R_s = get_rotation_matrix(x_s_info["pitch"], x_s_info["yaw"], x_s_info["roll"]) + source_rot_list.append(R_s) + + f_s = pipeline.live_portrait_wrapper.extract_feature_3d(I_s) + f_s_list.append(f_s) + + del I_s + + else: + log.warning(f"Warning: No face detected on frame {str(i)}, skipping") + cropped_images_list.append(np.zeros((256, 256, 3), dtype=np.uint8)) + crop_info_list.append(None) + f_s_list.append(None) + x_s_list.append(None) + source_info.append(None) + source_rot_list.append(None) + + # Update progress bar + pbar.update(1) + + cropped_tensors_out = ( + torch.stack([torch.from_numpy(np_array) for np_array in cropped_images_list]) + / 255 + ) + + crop_info_dict = { + 'crop_info_list': crop_info_list, + 'source_rot_list': source_rot_list, + 'f_s_list': f_s_list, + 'x_s_list': x_s_list, + 'source_info': source_info + } + + return (cropped_tensors_out, crop_info_dict) + +class LivePortraitRetargeting: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "driving_crop_info": ("CROPINFO", {"default": []}), "eye_retargeting": ("BOOLEAN", {"default": False}), "eyes_retargeting_multiplier": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 10.0, "step": 0.001}), "lip_retargeting": ("BOOLEAN", {"default": False}), "lip_retargeting_multiplier": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 10.0, "step": 0.001}), - "stitching": ("BOOLEAN", {"default": True}), - "relative": ("BOOLEAN", {"default": True}), }, - "optional": { - "onnx_device": ( - [ - 'CPU', - 'CUDA', - ], { - "default": 'CPU' - }), - } - - } - RETURN_TYPES = ("IMAGE", "IMAGE",) - RETURN_NAMES = ("cropped_images", "full_images",) + RETURN_TYPES = ("RETARGETINGINFO",) + RETURN_NAMES = ("retargeting_info",) FUNCTION = "process" CATEGORY = "LivePortrait" - def process(self, source_image, driving_images, dsize, scale, vx_ratio, vy_ratio, pipeline, - lip_zero, eye_retargeting, lip_retargeting, stitching, relative, eyes_retargeting_multiplier, lip_retargeting_multiplier, onnx_device='CUDA'): - source_image_np = (source_image * 255).byte().numpy() - driving_images_np = (driving_images * 255).byte().numpy() + def process(self, driving_crop_info, eye_retargeting, eyes_retargeting_multiplier, lip_retargeting, lip_retargeting_multiplier): - crop_cfg = CropConfig( - dsize = dsize, - scale = scale, - vx_ratio = vx_ratio, - vy_ratio = vy_ratio, - ) + driving_landmarks = [] + for crop in driving_crop_info["crop_info_list"]: + driving_landmarks.append(crop['lmk_crop']) + + retargeting_info = { + 'eye_retargeting': eye_retargeting, + 'eyes_retargeting_multiplier': eyes_retargeting_multiplier, + 'lip_retargeting': lip_retargeting, + 'lip_retargeting_multiplier': lip_retargeting_multiplier, + 'driving_landmarks': driving_landmarks + } + + return (retargeting_info,) + + +class KeypointsToImage: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "crop_info": ("CROPINFO", {"default": []}), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("keypoints_image",) + FUNCTION = "drawkeypoints" + CATEGORY = "LivePortrait" + + def drawkeypoints(self, crop_info): + height, width = crop_info["crop_info_list"][0]['input_image_size'] + keypoints_img_list = [] + pbar = comfy.utils.ProgressBar(len(crop_info)) + for crop in crop_info["crop_info_list"]: + if crop: + keypoints = crop['lmk_crop'].copy() + # Draw each landmark as a circle + blank_image = np.zeros((height, width, 3), dtype=np.uint8) * 255 + for (x, y) in keypoints: + # Ensure the coordinates are within the dimensions of the blank image + if 0 <= x < width and 0 <= y < height: + cv2.circle(blank_image, (int(x), int(y)), radius=2, color=(0, 0, 255)) + + keypoints_image = cv2.cvtColor(blank_image, cv2.COLOR_BGR2RGB) + else: + keypoints_image = np.zeros((height, width, 3), dtype=np.uint8) * 255 + keypoints_img_list.append(keypoints_image) + pbar.update(1) + + keypoints_img_tensor = ( + torch.stack([torch.from_numpy(np_array) for np_array in keypoints_img_list]) / 255).float() + + + return (keypoints_img_tensor,) + +class KeypointScaler: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "crop_info": ("CROPINFO", {"default": {}}), + "scale": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 10.0, "step": 0.001}), + "offset_x": ("INT", {"default": 0, "min": -1024, "max": 1024, "step": 1}), + "offset_y": ("INT", {"default": 0, "min": -1024, "max": 1024, "step": 1}), + + } + } + + RETURN_TYPES = ("CROPINFO", "IMAGE",) + RETURN_NAMES = ("crop_info", "keypoints_image",) + FUNCTION = "process" + CATEGORY = "LivePortrait" + + def process(self, crop_info, offset_x, offset_y, scale): + + keypoints = crop_info['crop_info']['lmk_crop'].copy() + + # Create an offset array + # Calculate the centroid of the keypoints + centroid = keypoints.mean(axis=0) + + # Translate keypoints to origin by subtracting the centroid + translated_keypoints = keypoints - centroid + + # Scale the translated keypoints + scaled_keypoints = translated_keypoints * scale + + # Translate scaled keypoints back to original position and then apply the offset + final_keypoints = scaled_keypoints + centroid + np.array([offset_x, offset_y]) + + crop_info['crop_info']['lmk_crop'] = final_keypoints #fix this + + # Draw each landmark as a circle + width, height = 512, 512 + blank_image = np.zeros((height, width, 3), dtype=np.uint8) * 255 + for (x, y) in final_keypoints: + # Ensure the coordinates are within the dimensions of the blank image + if 0 <= x < width and 0 <= y < height: + cv2.circle(blank_image, (int(x), int(y)), radius=2, color=(0, 0, 255)) + + keypoints_image = cv2.cvtColor(blank_image, cv2.COLOR_BGR2RGB) + keypoints_image_tensor = torch.from_numpy(keypoints_image) / 255 + keypoints_image_tensor = keypoints_image_tensor.unsqueeze(0).cpu().float() - cropper = Cropper(crop_cfg=crop_cfg, provider=onnx_device) - pipeline.cropper = cropper - pipeline.live_portrait_wrapper.cfg.flag_eye_retargeting = eye_retargeting - pipeline.live_portrait_wrapper.cfg.eyes_retargeting_multiplier = eyes_retargeting_multiplier - pipeline.live_portrait_wrapper.cfg.flag_lip_retargeting = lip_retargeting - pipeline.live_portrait_wrapper.cfg.lip_retargeting_multiplier = lip_retargeting_multiplier - pipeline.live_portrait_wrapper.cfg.flag_stitching = stitching - pipeline.live_portrait_wrapper.cfg.flag_relative = relative - pipeline.live_portrait_wrapper.cfg.flag_lip_zero = lip_zero - - cropped_out_list = [] - full_out_list = [] - for img in source_image_np: - cropped_frames, full_frame = pipeline.execute(img, driving_images_np) - cropped_tensors = [torch.from_numpy(np_array) for np_array in cropped_frames] - cropped_tensors_out = torch.stack(cropped_tensors) / 255 - cropped_tensors_out = cropped_tensors_out.cpu().float() - - full_tensors = [torch.from_numpy(np_array) for np_array in full_frame] - full_tensors_out = torch.stack(full_tensors) / 255 - full_tensors_out = full_tensors_out.cpu().float() - - cropped_out_list.append(cropped_tensors_out) - full_out_list.append(full_tensors_out) - - cropped_tensors_out = torch.cat(cropped_out_list, dim=0) - full_tensors_out = torch.cat(full_out_list, dim=0) - - return (cropped_tensors_out, full_tensors_out) + return (crop_info, keypoints_image_tensor,) NODE_CLASS_MAPPINGS = { "DownloadAndLoadLivePortraitModels": DownloadAndLoadLivePortraitModels, "LivePortraitProcess": LivePortraitProcess, + "LivePortraitCropper": LivePortraitCropper, + "LivePortraitRetargeting": LivePortraitRetargeting, + #"KeypointScaler": KeypointScaler, + "KeypointsToImage": KeypointsToImage, + "LivePortraitLoadCropper": LivePortraitLoadCropper, + "LivePortraitLoadMediaPipeCropper": LivePortraitLoadMediaPipeCropper, + "LivePortraitComposite": LivePortraitComposite, } NODE_DISPLAY_NAME_MAPPINGS = { "DownloadAndLoadLivePortraitModels": "(Down)Load LivePortraitModels", - "LivePortraitProcess": "LivePortraitProcess", + "LivePortraitProcess": "LivePortrait Process", + "LivePortraitCropper": "LivePortrait Cropper", + "LivePortraitRetargeting": "LivePortrait Retargeting", + #"KeypointScaler": "KeypointScaler", + "KeypointsToImage": "LivePortrait KeypointsToImage", + "LivePortraitLoadCropper": "LivePortrait Load InsightFaceCropper", + "LivePortraitLoadMediaPipeCropper": "LivePortrait Load MediaPipeCropper", + "LivePortraitComposite": "LivePortrait Composite", } \ No newline at end of file diff --git a/readme.md b/readme.md index aab8650..9dd5af2 100644 --- a/readme.md +++ b/readme.md @@ -1,15 +1,46 @@ # ComfyUI nodes to use [LivePortrait](https://github.com/KwaiVGI/LivePortrait) +## Update -https://github.com/kijai/ComfyUI-LivePortrait/assets/40791699/e55e10f6-af61-4d73-b162-af29eb847516 +Rework of almost the whole thing that's been in develop is now merged into main, this means old workflows will not work, but everything should be faster and there's lots of new features. +For legacy purposes the old main branch is moved to the legacy -branch + +Changes +- Added MediaPipe as alternative to Insightface, everything should now be covered under MIT and Apache-2.0 licenses when using it. +- Proper Vid2vid including smoothing algorhitm (thanks @melMass) +- Improved speed and efficiency, allows for near realtime view even in Comfy (~80-100ms delay) +- Restructured nodes for more options +- Auto skipping frames with no face detected +- Numerous other things I have forgotten about at this point, it's been a lot +- Better Mac support on MPS (thanks @Grant-CP + +# Examples: + +Realtime with webcam feed: + +https://github.com/user-attachments/assets/31f77c10-b757-44ae-bb26-39e45ec0b2d9 + +Image2vid: + +https://github.com/user-attachments/assets/cfec0419-d1eb-4e67-8913-890eeb155eef + +Vid2Vid: + +https://github.com/user-attachments/assets/28438fcb-fbb0-4e4e-baf4-00fe06c455de I have converted all the pickle files to safetensors: https://huggingface.co/Kijai/LivePortrait_safetensors/tree/main They go here (and are automatically downloaded if the folder is not present) `ComfyUI/models/liveportrait` +# Face detectors -Insightface is also required. +You can either use the original default Insightface, or Google's MediaPipe. + +Biggest difference is the license: Insightface is strictly for NON-COMMERCIAL use. +MediaPipe is a bit worse at detection, and can't run on GPU in Windows, though it's much faster on CPU compared to Insightface + +Insightface is not automatically installed, if you wish to use it follow these instructions: If you have a working compile environment, installing it can be as easy as: `pip install insightface` diff --git a/requirements.txt b/requirements.txt index 6e0484b..f894946 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ pyyaml numpy opencv-python -rich -onnxruntime-gpu \ No newline at end of file +onnxruntime-gpu +pykalman +mediapipe \ No newline at end of file