Merge branch 'main' into develop

Merge branch 'develop' of https://github.com/kijai/ComfyUI-LivePortraitKJ into develop
update workflows
2024-08-02 20:01:52 +03:00 · 2024-07-24 17:56:47 +03:00 · 2024-07-24 17:56:46 +03:00 · 2024-07-24 17:54:47 +03:00 · 2024-07-24 16:09:26 +03:00 · 2024-07-24 03:57:59 +03:00
4 changed files with 174 additions and 208 deletions
@@ -1,5 +1,5 @@
 {
-  "last_node_id": 201,
+  "last_node_id": 200,
  "last_link_id": 478,
  "nodes": [
    {
@@ -11,7 +11,7 @@
      ],
      "size": {
        "0": 315,
-        "1": 106
+        "1": 82
      },
      "flags": {},
      "order": 0,
@@ -30,8 +30,7 @@
      },
      "widgets_values": [
        "CPU",
-        true,
-        0.5
+        true
      ]
    },
    {
@@ -46,7 +45,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 20,
+      "order": 19,
      "mode": 0,
      "inputs": [
        {
@@ -121,7 +120,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 16,
+      "order": 15,
      "mode": 0,
      "inputs": [
        {
@@ -345,7 +344,7 @@
        "1": 242
      },
      "flags": {},
-      "order": 17,
+      "order": 16,
      "mode": 0,
      "inputs": [
        {
@@ -398,6 +397,38 @@
        true
      ]
    },
+    {
+      "id": 198,
+      "type": "LivePortraitLoadMediaPipeCropper",
+      "pos": [
+        -1256,
+        -893
+      ],
+      "size": {
+        "0": 315,
+        "1": 82
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "cropper",
+          "type": "LPCROPPER",
+          "links": [
+            474
+          ],
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LivePortraitLoadMediaPipeCropper"
+      },
+      "widgets_values": [
+        "CPU",
+        true
+      ]
+    },
    {
      "id": 1,
      "type": "DownloadAndLoadLivePortraitModels",
@@ -407,10 +438,10 @@
      ],
      "size": {
        "0": 302.43463134765625,
-        "1": 82
+        "1": 58
      },
      "flags": {},
-      "order": 2,
+      "order": 3,
      "mode": 0,
      "outputs": [
        {
@@ -429,10 +460,32 @@
        "Node name for S&R": "DownloadAndLoadLivePortraitModels"
      },
      "widgets_values": [
-        "auto",
-        "human"
+        "auto"
      ]
    },
+    {
+      "id": 200,
+      "type": "Note",
+      "pos": [
+        -1542,
+        -877
+      ],
+      "size": [
+        265.68303695794293,
+        168.53929752859244
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "properties": {
+        "text": ""
+      },
+      "widgets_values": [
+        "Choose your face detector here, MediaPipe doesn't run on GPU and isn't as good at detecting, but is faster on CPU and has Apache 2.0 license.\n\nInsightface can be run on GPU and is better at detecting, but it's model license is NON-COMMERCIAL"
+      ],
+      "color": "#432",
+      "bgcolor": "#653"
+    },
    {
      "id": 199,
      "type": "Reroute",
@@ -445,7 +498,7 @@
        26
      ],
      "flags": {},
-      "order": 13,
+      "order": 9,
      "mode": 0,
      "inputs": [
        {
@@ -482,7 +535,7 @@
        "1": 82
      },
      "flags": {},
-      "order": 3,
+      "order": 5,
      "mode": 0,
      "outputs": [
        {
@@ -518,7 +571,7 @@
        "1": 58
      },
      "flags": {},
-      "order": 4,
+      "order": 6,
      "mode": 0,
      "properties": {
        "text": ""
@@ -541,7 +594,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 14,
+      "order": 13,
      "mode": 0,
      "inputs": [
        {
@@ -596,7 +649,7 @@
        "1": 130
      },
      "flags": {},
-      "order": 21,
+      "order": 20,
      "mode": 0,
      "inputs": [
        {
@@ -625,6 +678,80 @@
        1
      ]
    },
+    {
+      "id": 190,
+      "type": "LivePortraitProcess",
+      "pos": [
+        166,
+        -422
+      ],
+      "size": {
+        "0": 430.8000183105469,
+        "1": 282
+      },
+      "flags": {},
+      "order": 21,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "pipeline",
+          "type": "LIVEPORTRAITPIPE",
+          "link": 448
+        },
+        {
+          "name": "crop_info",
+          "type": "CROPINFO",
+          "link": 449
+        },
+        {
+          "name": "source_image",
+          "type": "IMAGE",
+          "link": 456
+        },
+        {
+          "name": "driving_images",
+          "type": "IMAGE",
+          "link": 451
+        },
+        {
+          "name": "opt_retargeting_info",
+          "type": "RETARGETINGINFO",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "cropped_image",
+          "type": "IMAGE",
+          "links": [
+            454
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "output",
+          "type": "LP_OUT",
+          "links": [
+            452
+          ],
+          "shape": 3,
+          "slot_index": 1
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LivePortraitProcess"
+      },
+      "widgets_values": [
+        false,
+        0.03,
+        true,
+        1,
+        "constant",
+        "relative",
+        0.000003
+      ]
+    },
    {
      "id": 83,
      "type": "CreateShapeMask",
@@ -637,7 +764,7 @@
        "1": 270
      },
      "flags": {},
-      "order": 5,
+      "order": 7,
      "mode": 0,
      "outputs": [
        {
@@ -737,7 +864,7 @@
        "1": 246
      },
      "flags": {},
-      "order": 15,
+      "order": 14,
      "mode": 0,
      "inputs": [
        {
@@ -762,7 +889,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 24,
+      "order": 23,
      "mode": 0,
      "inputs": [
        {
@@ -812,11 +939,11 @@
        -302
      ],
      "size": [
-        669.8613891601562,
-        973.8613891601562
+        669.8613793918787,
+        973.8613793918787
      ],
      "flags": {},
-      "order": 25,
+      "order": 24,
      "mode": 0,
      "inputs": [
        {
@@ -865,7 +992,7 @@
          "hidden": false,
          "paused": false,
          "params": {
-            "filename": "LivePortrait_00002.mp4",
+            "filename": "LivePortrait_00005.mp4",
            "subfolder": "",
            "type": "temp",
            "format": "video/h264-mp4",
@@ -881,12 +1008,12 @@
        -1267,
        -582
      ],
-      "size": {
-        "0": 330.4995422363281,
-        "1": 384.2416687011719
-      },
+      "size": [
+        330.4995504373579,
+        384.24166509694487
+      ],
      "flags": {},
-      "order": 6,
+      "order": 8,
      "mode": 0,
      "outputs": [
        {
@@ -925,7 +1052,7 @@
        26
      ],
      "flags": {},
-      "order": 19,
+      "order": 18,
      "mode": 0,
      "inputs": [
        {
@@ -962,7 +1089,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 23,
+      "order": 22,
      "mode": 0,
      "inputs": [
        {
@@ -1019,7 +1146,7 @@
        "1": 242
      },
      "flags": {},
-      "order": 18,
+      "order": 17,
      "mode": 0,
      "inputs": [
        {
@@ -1070,170 +1197,6 @@
        "large-small",
        true
      ]
-    },
-    {
-      "id": 201,
-      "type": "LivePortraitLoadFaceAlignmentCropper",
-      "pos": [
-        -1267,
-        -1107
-      ],
-      "size": {
-        "0": 319.20001220703125,
-        "1": 154
-      },
-      "flags": {},
-      "order": 7,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "cropper",
-          "type": "LPCROPPER",
-          "links": null,
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LivePortraitLoadFaceAlignmentCropper"
-      },
-      "widgets_values": [
-        "blazeface_back_camera",
-        "torch_gpu",
-        "cuda",
-        "fp16",
-        true
-      ]
-    },
-    {
-      "id": 200,
-      "type": "Note",
-      "pos": [
-        -1598,
-        -968
-      ],
-      "size": {
-        "0": 273.2294921875,
-        "1": 223.99388122558594
-      },
-      "flags": {},
-      "order": 8,
-      "mode": 0,
-      "properties": {
-        "text": ""
-      },
-      "widgets_values": [
-        "Choose your face detector here, MediaPipe doesn't run on GPU and isn't as good at detecting, but is faster on CPU and has Apache 2.0 license.\n\nFaceAlignment is also uses blazeface like MediaPipe, but can also use the back_camera version that's much better at detecting smaller faces. BSD3 License\n\nInsightface can be run on GPU and is better at detecting, but it's model license is NON-COMMERCIAL"
-      ],
-      "color": "#432",
-      "bgcolor": "#653"
-    },
-    {
-      "id": 198,
-      "type": "LivePortraitLoadMediaPipeCropper",
-      "pos": [
-        -1260,
-        -893
-      ],
-      "size": {
-        "0": 315,
-        "1": 82
-      },
-      "flags": {},
-      "order": 9,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "cropper",
-          "type": "LPCROPPER",
-          "links": [
-            474
-          ],
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LivePortraitLoadMediaPipeCropper"
-      },
-      "widgets_values": [
-        "CPU",
-        true
-      ]
-    },
-    {
-      "id": 190,
-      "type": "LivePortraitProcess",
-      "pos": [
-        166,
-        -422
-      ],
-      "size": {
-        "0": 430.8000183105469,
-        "1": 330
-      },
-      "flags": {},
-      "order": 22,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "pipeline",
-          "type": "LIVEPORTRAITPIPE",
-          "link": 448
-        },
-        {
-          "name": "crop_info",
-          "type": "CROPINFO",
-          "link": 449
-        },
-        {
-          "name": "source_image",
-          "type": "IMAGE",
-          "link": 456
-        },
-        {
-          "name": "driving_images",
-          "type": "IMAGE",
-          "link": 451
-        },
-        {
-          "name": "opt_retargeting_info",
-          "type": "RETARGETINGINFO",
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "name": "cropped_image",
-          "type": "IMAGE",
-          "links": [
-            454
-          ],
-          "shape": 3,
-          "slot_index": 0
-        },
-        {
-          "name": "output",
-          "type": "LP_OUT",
-          "links": [
-            452
-          ],
-          "shape": 3,
-          "slot_index": 1
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LivePortraitProcess"
-      },
-      "widgets_values": [
-        false,
-        0.03,
-        true,
-        1,
-        "constant",
-        "relative",
-        0.000003,
-        true,
-        1
-      ]
    }
  ],
  "links": [
@@ -1450,10 +1413,10 @@
  "config": {},
  "extra": {
    "ds": {
-      "scale": 0.620921323059155,
+      "scale": 0.6209213230591553,
      "offset": {
-        "0": 1355.4192699221378,
-        "1": 1004.737376044247
+        "0": 1699.882189562642,
+        "1": 1132.3910371015413
      }
    }
  },
@@ -54,6 +54,7 @@ class LivePortraitPipeline(object):
        else:
            total_frames = driving_images.shape[0]

+        

        disable_progress_bar = True if relative_motion_mode == "single_frame" else False

@@ -43,17 +43,23 @@ def _transform_img_kornia(img, M, dsize, device, flags='bilinear', borderMode='z

    # Convert M from numpy.ndarray to PyTorch tensor
    M = torch.from_numpy(M).float().to(device)
-    
    if M.shape == (3, 3):
-        M = M[:2, :]
+        M = M[:2, :].unsqueeze(0)  # Adjust M to the expected shape Bx2x3
+    elif M.shape == (2, 3):
+        M = M.unsqueeze(0)  # Add batch dimension if not present
+
+    # Reshape M for Kornia (1, 2, 3) and upscale to 3D affine matrix if not already
    if M.shape == (2, 3):
-        M = M.unsqueeze(0)
+        M = M.unsqueeze(0)  # Add batch dimension

    # Convert image to floating point tensor if not already
    if img.dtype != torch.float32:
        img = img.float()
-    img = img.permute(0, 3, 1, 2).to(device) # Reshape img for Kornia (B, C, H, W)
-   
+    img = img.to(device)
+
+    # Reshape img for Kornia (B, C, H, W)
+    img = img.permute(0, 3, 1, 2)
+
    # Apply the affine transformation
    img_warped = KGT.warp_affine(img, M, _dsize, mode=flags, padding_mode=borderMode)

@@ -54,8 +54,7 @@ class CropperInsightFace(object):

        if len(src_face) == 0:
            ret_dct = {}
-            cropped_image_256 = None
-            return ret_dct, cropped_image_256
+            return ret_dct

        src_face = src_face[face_index] # choose the index if multiple faces detected
        pts = src_face.landmark_2d_106
@@ -72,7 +71,6 @@ class CropperInsightFace(object):
        )
        # update a 256x256 version for network input or else
        cropped_image_256 = cv2.resize(image_crop, (256, 256), interpolation=cv2.INTER_AREA)
-        del image_crop
        ret_dct['pt_crop_256x256'] = ret_dct['pt_crop'] * 256 / dsize

        input_image_size = img_rgb.shape[:2]
@@ -136,7 +134,6 @@ class CropperMediaPipe(object):
        )
        # update a 256x256 version for network input or else
        cropped_image_256 = cv2.resize(image_crop, (256, 256), interpolation=cv2.INTER_AREA)
-        del image_crop
        ret_dct['pt_crop_256x256'] = ret_dct['pt_crop'] * 256 / dsize

        input_image_size = img_rgb.shape[:2]
@@ -202,7 +199,6 @@ class CropperFaceAlignment(object):
        )
        # update a 256x256 version for network input or else
        cropped_image_256 = cv2.resize(image_crop, (256, 256), interpolation=cv2.INTER_AREA)
-        del image_crop
        ret_dct['pt_crop_256x256'] = ret_dct['pt_crop'] * 256 / dsize

        input_image_size = img_rgb.shape[:2]
Author	SHA1	Message	Date
kijai	d3738609c5	Merge branch 'main' into develop	2024-08-02 20:01:52 +03:00
kijai	b608558b9e	Merge branch 'develop' of https://github.com/kijai/ComfyUI-LivePortraitKJ into develop	2024-07-24 17:56:47 +03:00
kijai	ad29b02bc1	update workflows	2024-07-24 17:56:46 +03:00
Jukka Seppänen	dd205ab4a4	Update readme.md	2024-07-24 17:54:47 +03:00
kijai	ba0886a905	fix running without insightface installed	2024-07-24 16:09:26 +03:00
kijai	068ab2c280	Add MediaPipe as alternative face detector	2024-07-24 03:57:59 +03:00
kijai	6261f4e474	cleanup, memory fixes	2024-07-23 22:58:33 +03:00
kijai	46675b2016	update workflows, cleanup	2024-07-23 21:13:15 +03:00
kijai	806263dd25	cleanup, fixes	2024-07-22 20:39:43 +03:00
kijai	ac89dc1e2f	fix no face frame skip	2024-07-22 19:19:46 +03:00
kijai	27d745b53e	add other examples	2024-07-22 17:52:34 +03:00
kijai	052762578c	Update readme.md	2024-07-22 17:45:49 +03:00
kijai	e825c51c87	separate composition to it's own node	2024-07-22 16:21:35 +03:00
kijai	177b324fcd	Update live_portrait_pipeline.py	2024-07-22 01:02:56 +03:00
kijai	5c03bd8439	MPS fallbacks	2024-07-22 00:57:44 +03:00
kijai	ef5ff7075f	Update requirements.txt	2024-07-21 20:35:33 +03:00
kijai	92fad03ee5	restructure a bit for more caching	2024-07-21 20:20:46 +03:00
kijai	4cefac79b8	Add single_frame mode for webcam	2024-07-21 19:42:22 +03:00
kijai	5e3c92d55c	restructuring, video smoothing	2024-07-21 19:20:52 +03:00
kijai	cc0501a2db	flag_relative_rotation_only	2024-07-21 13:21:26 +03:00
kijai	3dc822fd2f	to use GPU for pasteback	2024-07-20 20:24:15 +03:00
kijai	697b9a78e6	Restructure nodes, skip frames with no face detect	2024-07-20 17:39:44 +03:00
kijai	2a7bd6116f	Update nodes.py	2024-07-10 17:14:43 +03:00
kijai	a7d09f5d49	example workflow	2024-07-10 16:33:31 +03:00
kijai	8e85d5b96d	some optimizations	2024-07-10 16:06:01 +03:00
kijai	30989a9d37	Update nodes.py	2024-07-10 01:17:10 +03:00
kijai	eecf645603	rotate option for cropper	2024-07-09 23:02:47 +03:00
kijai	1b080706df	Update nodes.py	2024-07-09 22:46:51 +03:00
kijai	336f3f7c23	add cut method	2024-07-09 22:19:56 +03:00
kijai	f27e1cca13	remove nearest option	2024-07-09 22:11:19 +03:00
kijai	86e91a6e9d	better error for retargeting	2024-07-09 22:06:56 +03:00
kijai	92529f7ca8	cleanup	2024-07-09 21:59:32 +03:00
kijai	c0959056ae	eye/lip retargeting fixes	2024-07-09 21:49:54 +03:00
kijai	2e40fe3820	Update live_portrait_pipeline.py	2024-07-09 21:23:10 +03:00
kijai	0a5e187637	keep Cropper in memory	2024-07-09 21:15:17 +03:00
kijai	4e19dbd6d1	Do video cropping on the cropped node too	2024-07-09 20:19:53 +03:00
kijai	9c190804a7	big cleanup	2024-07-09 19:02:27 +03:00
kijai	d9ca40e1d6	logging	2024-07-09 15:08:17 +03:00
kijai	b68cf8788c	fix warning	2024-07-09 14:35:51 +03:00
kijai	e702b26895	Update cropper.py	2024-07-09 14:31:22 +03:00
kijai	c21705edb5	Don't draw keypoints for every frame by default	2024-07-09 14:25:39 +03:00
kijai	a284bb52b2	Bring back mismatch_method selection	2024-07-09 14:04:56 +03:00
kijai	9884aac18a	Fix eye/lip retargeting	2024-07-09 14:00:43 +03:00
kijai	f8aada81db	face_index selection	2024-07-09 13:00:03 +03:00
kijai	6735771664	tqdm progress bars	2024-07-09 11:51:38 +03:00
kijai	857ddbc6d7	skip autocast if not needed for mps	2024-07-09 02:54:27 +03:00
kijai	a6edcda97d	output masks	2024-07-09 01:37:05 +03:00
kijai	ca01d706d0	custom mask support	2024-07-09 00:57:13 +03:00
kijai	0dc9a8a695	Merge branch 'add_video_source' into develop	2024-07-09 00:13:07 +03:00
Mel Massadian	ba6b3f5f68	bring KJ edits	2024-07-08 23:09:56 +02:00
kijai	ee7d5b4241	revert this for compatibility	2024-07-09 00:08:10 +03:00
kijai	03df9f35cd	Merge branch 'add_video_source' into develop	2024-07-08 23:55:22 +03:00
kijai	ec6b5c8c85	calc_combined_eye_ratio	2024-07-08 23:52:58 +03:00
Mel Massadian	8509d9a551	remove unused imports	2024-07-08 22:52:48 +02:00
Mel Massadian	e724da1161	fix relative mode use R_d_0 instead of source	2024-07-08 21:38:27 +02:00
Mel Massadian	68d0ddf72a	remove reference frame attempt also use batches for driving when either retargetting is enabled	2024-07-08 21:22:50 +02:00
kijai	6f9dba7777	fixes	2024-07-08 20:50:44 +03:00
kijai	811ca557fb	more	2024-07-08 20:31:00 +03:00
kijai	6d790bdcc3	Merge branch 'add_video_source' into develop	2024-07-08 20:30:35 +03:00
kijai	ef8b4263b4	separating functions to nodes	2024-07-08 20:21:45 +03:00
Mel Massadian	eb5fddf4de	fix issues from merge	2024-07-08 19:10:53 +02:00
Mel Massadian	9c7db3c59a	Merge branch 'main' into add_video_source	2024-07-08 19:09:02 +02:00
Mel Massadian	bf3410cd0d	trying reference frame	2024-07-08 19:04:42 +02:00
Mel Massadian	24c65627db	local updates before merging main	2024-07-08 19:03:28 +02:00
Mel Massadian	72bb6910e9	initial too much diff due to formatting	2024-07-08 16:48:15 +02:00