cleanup

fix insightface cropper frame skipping
Update liveportrait_image_example_01.json
2024-08-05 02:39:29 +03:00 · 2024-08-05 02:24:11 +03:00 · 2024-08-02 20:11:28 +03:00
4 changed files with 208 additions and 174 deletions
@@ -1,5 +1,5 @@
 {
-  "last_node_id": 200,
+  "last_node_id": 201,
  "last_link_id": 478,
  "nodes": [
    {
@@ -11,7 +11,7 @@
      ],
      "size": {
        "0": 315,
-        "1": 82
+        "1": 106
      },
      "flags": {},
      "order": 0,
@@ -30,7 +30,8 @@
      },
      "widgets_values": [
        "CPU",
-        true
+        true,
+        0.5
      ]
    },
    {
@@ -45,7 +46,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 19,
+      "order": 20,
      "mode": 0,
      "inputs": [
        {
@@ -120,7 +121,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 15,
+      "order": 16,
      "mode": 0,
      "inputs": [
        {
@@ -344,7 +345,7 @@
        "1": 242
      },
      "flags": {},
-      "order": 16,
+      "order": 17,
      "mode": 0,
      "inputs": [
        {
@@ -397,38 +398,6 @@
        true
      ]
    },
-    {
-      "id": 198,
-      "type": "LivePortraitLoadMediaPipeCropper",
-      "pos": [
-        -1256,
-        -893
-      ],
-      "size": {
-        "0": 315,
-        "1": 82
-      },
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "cropper",
-          "type": "LPCROPPER",
-          "links": [
-            474
-          ],
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LivePortraitLoadMediaPipeCropper"
-      },
-      "widgets_values": [
-        "CPU",
-        true
-      ]
-    },
    {
      "id": 1,
      "type": "DownloadAndLoadLivePortraitModels",
@@ -438,10 +407,10 @@
      ],
      "size": {
        "0": 302.43463134765625,
-        "1": 58
+        "1": 82
      },
      "flags": {},
-      "order": 3,
+      "order": 2,
      "mode": 0,
      "outputs": [
        {
@@ -460,32 +429,10 @@
        "Node name for S&R": "DownloadAndLoadLivePortraitModels"
      },
      "widgets_values": [
-        "auto"
+        "auto",
+        "human"
      ]
    },
-    {
-      "id": 200,
-      "type": "Note",
-      "pos": [
-        -1542,
-        -877
-      ],
-      "size": [
-        265.68303695794293,
-        168.53929752859244
-      ],
-      "flags": {},
-      "order": 4,
-      "mode": 0,
-      "properties": {
-        "text": ""
-      },
-      "widgets_values": [
-        "Choose your face detector here, MediaPipe doesn't run on GPU and isn't as good at detecting, but is faster on CPU and has Apache 2.0 license.\n\nInsightface can be run on GPU and is better at detecting, but it's model license is NON-COMMERCIAL"
-      ],
-      "color": "#432",
-      "bgcolor": "#653"
-    },
    {
      "id": 199,
      "type": "Reroute",
@@ -498,7 +445,7 @@
        26
      ],
      "flags": {},
-      "order": 9,
+      "order": 13,
      "mode": 0,
      "inputs": [
        {
@@ -535,7 +482,7 @@
        "1": 82
      },
      "flags": {},
-      "order": 5,
+      "order": 3,
      "mode": 0,
      "outputs": [
        {
@@ -571,7 +518,7 @@
        "1": 58
      },
      "flags": {},
-      "order": 6,
+      "order": 4,
      "mode": 0,
      "properties": {
        "text": ""
@@ -594,7 +541,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 13,
+      "order": 14,
      "mode": 0,
      "inputs": [
        {
@@ -649,7 +596,7 @@
        "1": 130
      },
      "flags": {},
-      "order": 20,
+      "order": 21,
      "mode": 0,
      "inputs": [
        {
@@ -678,80 +625,6 @@
        1
      ]
    },
-    {
-      "id": 190,
-      "type": "LivePortraitProcess",
-      "pos": [
-        166,
-        -422
-      ],
-      "size": {
-        "0": 430.8000183105469,
-        "1": 282
-      },
-      "flags": {},
-      "order": 21,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "pipeline",
-          "type": "LIVEPORTRAITPIPE",
-          "link": 448
-        },
-        {
-          "name": "crop_info",
-          "type": "CROPINFO",
-          "link": 449
-        },
-        {
-          "name": "source_image",
-          "type": "IMAGE",
-          "link": 456
-        },
-        {
-          "name": "driving_images",
-          "type": "IMAGE",
-          "link": 451
-        },
-        {
-          "name": "opt_retargeting_info",
-          "type": "RETARGETINGINFO",
-          "link": null
-        }
-      ],
-      "outputs": [
-        {
-          "name": "cropped_image",
-          "type": "IMAGE",
-          "links": [
-            454
-          ],
-          "shape": 3,
-          "slot_index": 0
-        },
-        {
-          "name": "output",
-          "type": "LP_OUT",
-          "links": [
-            452
-          ],
-          "shape": 3,
-          "slot_index": 1
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LivePortraitProcess"
-      },
-      "widgets_values": [
-        false,
-        0.03,
-        true,
-        1,
-        "constant",
-        "relative",
-        0.000003
-      ]
-    },
    {
      "id": 83,
      "type": "CreateShapeMask",
@@ -764,7 +637,7 @@
        "1": 270
      },
      "flags": {},
-      "order": 7,
+      "order": 5,
      "mode": 0,
      "outputs": [
        {
@@ -864,7 +737,7 @@
        "1": 246
      },
      "flags": {},
-      "order": 14,
+      "order": 15,
      "mode": 0,
      "inputs": [
        {
@@ -889,7 +762,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 23,
+      "order": 24,
      "mode": 0,
      "inputs": [
        {
@@ -939,11 +812,11 @@
        -302
      ],
      "size": [
-        669.8613793918787,
-        973.8613793918787
+        669.8613891601562,
+        973.8613891601562
      ],
      "flags": {},
-      "order": 24,
+      "order": 25,
      "mode": 0,
      "inputs": [
        {
@@ -992,7 +865,7 @@
          "hidden": false,
          "paused": false,
          "params": {
-            "filename": "LivePortrait_00005.mp4",
+            "filename": "LivePortrait_00002.mp4",
            "subfolder": "",
            "type": "temp",
            "format": "video/h264-mp4",
@@ -1008,12 +881,12 @@
        -1267,
        -582
      ],
-      "size": [
-        330.4995504373579,
-        384.24166509694487
-      ],
+      "size": {
+        "0": 330.4995422363281,
+        "1": 384.2416687011719
+      },
      "flags": {},
-      "order": 8,
+      "order": 6,
      "mode": 0,
      "outputs": [
        {
@@ -1052,7 +925,7 @@
        26
      ],
      "flags": {},
-      "order": 18,
+      "order": 19,
      "mode": 0,
      "inputs": [
        {
@@ -1089,7 +962,7 @@
        "1": 86
      },
      "flags": {},
-      "order": 22,
+      "order": 23,
      "mode": 0,
      "inputs": [
        {
@@ -1146,7 +1019,7 @@
        "1": 242
      },
      "flags": {},
-      "order": 17,
+      "order": 18,
      "mode": 0,
      "inputs": [
        {
@@ -1197,6 +1070,170 @@
        "large-small",
        true
      ]
+    },
+    {
+      "id": 201,
+      "type": "LivePortraitLoadFaceAlignmentCropper",
+      "pos": [
+        -1267,
+        -1107
+      ],
+      "size": {
+        "0": 319.20001220703125,
+        "1": 154
+      },
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "cropper",
+          "type": "LPCROPPER",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LivePortraitLoadFaceAlignmentCropper"
+      },
+      "widgets_values": [
+        "blazeface_back_camera",
+        "torch_gpu",
+        "cuda",
+        "fp16",
+        true
+      ]
+    },
+    {
+      "id": 200,
+      "type": "Note",
+      "pos": [
+        -1598,
+        -968
+      ],
+      "size": {
+        "0": 273.2294921875,
+        "1": 223.99388122558594
+      },
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "properties": {
+        "text": ""
+      },
+      "widgets_values": [
+        "Choose your face detector here, MediaPipe doesn't run on GPU and isn't as good at detecting, but is faster on CPU and has Apache 2.0 license.\n\nFaceAlignment is also uses blazeface like MediaPipe, but can also use the back_camera version that's much better at detecting smaller faces. BSD3 License\n\nInsightface can be run on GPU and is better at detecting, but it's model license is NON-COMMERCIAL"
+      ],
+      "color": "#432",
+      "bgcolor": "#653"
+    },
+    {
+      "id": 198,
+      "type": "LivePortraitLoadMediaPipeCropper",
+      "pos": [
+        -1260,
+        -893
+      ],
+      "size": {
+        "0": 315,
+        "1": 82
+      },
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "cropper",
+          "type": "LPCROPPER",
+          "links": [
+            474
+          ],
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LivePortraitLoadMediaPipeCropper"
+      },
+      "widgets_values": [
+        "CPU",
+        true
+      ]
+    },
+    {
+      "id": 190,
+      "type": "LivePortraitProcess",
+      "pos": [
+        166,
+        -422
+      ],
+      "size": {
+        "0": 430.8000183105469,
+        "1": 330
+      },
+      "flags": {},
+      "order": 22,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "pipeline",
+          "type": "LIVEPORTRAITPIPE",
+          "link": 448
+        },
+        {
+          "name": "crop_info",
+          "type": "CROPINFO",
+          "link": 449
+        },
+        {
+          "name": "source_image",
+          "type": "IMAGE",
+          "link": 456
+        },
+        {
+          "name": "driving_images",
+          "type": "IMAGE",
+          "link": 451
+        },
+        {
+          "name": "opt_retargeting_info",
+          "type": "RETARGETINGINFO",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "cropped_image",
+          "type": "IMAGE",
+          "links": [
+            454
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "output",
+          "type": "LP_OUT",
+          "links": [
+            452
+          ],
+          "shape": 3,
+          "slot_index": 1
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LivePortraitProcess"
+      },
+      "widgets_values": [
+        false,
+        0.03,
+        true,
+        1,
+        "constant",
+        "relative",
+        0.000003,
+        true,
+        1
+      ]
    }
  ],
  "links": [
@@ -1413,10 +1450,10 @@
  "config": {},
  "extra": {
    "ds": {
-      "scale": 0.6209213230591553,
+      "scale": 0.620921323059155,
      "offset": {
-        "0": 1699.882189562642,
-        "1": 1132.3910371015413
+        "0": 1355.4192699221378,
+        "1": 1004.737376044247
      }
    }
  },
@@ -54,7 +54,6 @@ class LivePortraitPipeline(object):
        else:
            total_frames = driving_images.shape[0]

-        

        disable_progress_bar = True if relative_motion_mode == "single_frame" else False

@@ -43,23 +43,17 @@ def _transform_img_kornia(img, M, dsize, device, flags='bilinear', borderMode='z

    # Convert M from numpy.ndarray to PyTorch tensor
    M = torch.from_numpy(M).float().to(device)
+    
    if M.shape == (3, 3):
-        M = M[:2, :].unsqueeze(0)  # Adjust M to the expected shape Bx2x3
-    elif M.shape == (2, 3):
-        M = M.unsqueeze(0)  # Add batch dimension if not present
-
-    # Reshape M for Kornia (1, 2, 3) and upscale to 3D affine matrix if not already
+        M = M[:2, :]
    if M.shape == (2, 3):
-        M = M.unsqueeze(0)  # Add batch dimension
+        M = M.unsqueeze(0)

    # Convert image to floating point tensor if not already
    if img.dtype != torch.float32:
        img = img.float()
-    img = img.to(device)
-
-    # Reshape img for Kornia (B, C, H, W)
-    img = img.permute(0, 3, 1, 2)
-
+    img = img.permute(0, 3, 1, 2).to(device) # Reshape img for Kornia (B, C, H, W)
+   
    # Apply the affine transformation
    img_warped = KGT.warp_affine(img, M, _dsize, mode=flags, padding_mode=borderMode)

@@ -54,7 +54,8 @@ class CropperInsightFace(object):

        if len(src_face) == 0:
            ret_dct = {}
-            return ret_dct
+            cropped_image_256 = None
+            return ret_dct, cropped_image_256

        src_face = src_face[face_index] # choose the index if multiple faces detected
        pts = src_face.landmark_2d_106
@@ -71,6 +72,7 @@ class CropperInsightFace(object):
        )
        # update a 256x256 version for network input or else
        cropped_image_256 = cv2.resize(image_crop, (256, 256), interpolation=cv2.INTER_AREA)
+        del image_crop
        ret_dct['pt_crop_256x256'] = ret_dct['pt_crop'] * 256 / dsize

        input_image_size = img_rgb.shape[:2]
@@ -134,6 +136,7 @@ class CropperMediaPipe(object):
        )
        # update a 256x256 version for network input or else
        cropped_image_256 = cv2.resize(image_crop, (256, 256), interpolation=cv2.INTER_AREA)
+        del image_crop
        ret_dct['pt_crop_256x256'] = ret_dct['pt_crop'] * 256 / dsize

        input_image_size = img_rgb.shape[:2]
@@ -199,6 +202,7 @@ class CropperFaceAlignment(object):
        )
        # update a 256x256 version for network input or else
        cropped_image_256 = cv2.resize(image_crop, (256, 256), interpolation=cv2.INTER_AREA)
+        del image_crop
        ret_dct['pt_crop_256x256'] = ret_dct['pt_crop'] * 256 / dsize

        input_image_size = img_rgb.shape[:2]
Author	SHA1	Message	Date
kijai	4d9dc6205b	cleanup	2024-08-05 02:39:29 +03:00
kijai	c74fb1d479	fix insightface cropper frame skipping	2024-08-05 02:24:11 +03:00
kijai	2d9497348e	Update liveportrait_image_example_01.json	2024-08-02 20:11:28 +03:00