169 lines
5.1 KiB
Python
169 lines
5.1 KiB
Python
import os
|
|
import torch
|
|
import numpy as np
|
|
import cv2
|
|
from insightface.utils import face_align
|
|
import mediapipe as mp
|
|
from scipy.spatial import ConvexHull
|
|
from folder_paths import models_dir
|
|
from .BiSeNet import BiSeNet
|
|
|
|
def squareimg(img):
|
|
h, w, _ = img.shape
|
|
s = max(h, w)
|
|
ph, pw = (s - h) // 2, (s - w) // 2
|
|
img = cv2.copyMakeBorder(img, ph, ph, pw, pw, cv2.BORDER_CONSTANT)
|
|
return (img, ph, pw)
|
|
|
|
def transform_back(points, rot, pw, ph):
|
|
points = np.hstack((points, np.ones((len(points),1)))) @ rot.T
|
|
points -= np.array((pw, ph))
|
|
return points
|
|
|
|
def get_candidates(img, face_analysis):
|
|
simg, ph, pw = squareimg(img)
|
|
h, w, _ = img.shape
|
|
s = max(h, w)
|
|
candidates = []
|
|
for i in range(4):
|
|
im = np.rot90(simg, i)
|
|
rot = cv2.getRotationMatrix2D((s/2, s/2), -90*i, 1)
|
|
dets = face_analysis.get(im)
|
|
for face in dets:
|
|
face.kps = transform_back(face.kps, rot, pw, ph)
|
|
face.landmark_2d_106 = transform_back(face.landmark_2d_106, rot, pw, ph)
|
|
|
|
# original bbox and size
|
|
a,b,c,d = transform_back(face.bbox.reshape((2,2)), rot, pw, ph).flatten()
|
|
a,c = min(a,c),max(a,c)
|
|
b,d = min(b,d),max(b,d)
|
|
face.w, face.h = abs(c-a), abs(d-b)
|
|
|
|
face.centroid = np.array(((a+c)/2, (b+d)/2))
|
|
face.s = max(abs(a-c), abs(b-d))
|
|
|
|
face.angle = i*90
|
|
candidates += dets
|
|
return candidates
|
|
|
|
def group_faces(candidates):
|
|
groups = []
|
|
for face in candidates:
|
|
group = None
|
|
for g in groups:
|
|
if any([np.linalg.norm(face.centroid - other.centroid) < np.sqrt(face.s)*2 for other in g]):
|
|
group = g
|
|
break
|
|
if group is None:
|
|
groups.append([face])
|
|
else:
|
|
group.append(face)
|
|
return groups
|
|
|
|
def get_best_faces(groups):
|
|
faces = [min(group, key=lambda face:abs(face.pose[2])) for group in groups]
|
|
return faces
|
|
|
|
def get_faces(img, face_analysis):
|
|
candidates = get_candidates(img, face_analysis)
|
|
groups = group_faces(candidates)
|
|
faces = get_best_faces(groups)
|
|
return faces
|
|
|
|
def crop_faces(img, face, crop_size, crop_factor):
|
|
M = face_align.estimate_norm(face.kps, crop_size, mode=None)
|
|
cx, cy = face.kps.mean(axis=0)
|
|
T1 = np.array([[1, 0, -cx], [0, 1, -cy], [0, 0, 1]])
|
|
T2 = np.array([[1, 0, cx], [0, 1, cy], [0, 0, 1]])
|
|
S = np.array([[1/crop_factor, 0, 0], [0, 1/crop_factor, 0], [0, 0, 1]])
|
|
M = M @ T2 @ S @ T1
|
|
crop = cv2.warpAffine(img, M, (crop_size,crop_size))
|
|
|
|
return M, crop
|
|
|
|
def get_face_mesh(crop):
|
|
with mp.solutions.face_mesh.FaceMesh(max_num_faces=10) as face_mesh:
|
|
mesh = face_mesh.process(crop)
|
|
h, w, _ = crop.shape
|
|
if mesh.multi_face_landmarks is not None:
|
|
all_pts = np.array([np.array([(w*l.x, h*l.y) for l in lmks.landmark]) for lmks in mesh.multi_face_landmarks], dtype=np.int32)
|
|
idx = np.argmin(np.abs(all_pts - np.array([w/2,h/2])).sum(axis=(1,2)))
|
|
points = all_pts[idx]
|
|
return points
|
|
else:
|
|
return None
|
|
|
|
def mask_simple_square(face, M, crop):
|
|
# rotated bbox and size
|
|
a,b,c,d = face.bbox
|
|
rect = np.array([
|
|
[a,b,1],
|
|
[a,d,1],
|
|
[c,b,1],
|
|
[c,d,1],
|
|
]) @ M.T
|
|
lx, ly = [int(x) for x in np.min(rect, axis=0)]
|
|
hx, hy = [int(x) for x in np.max(rect, axis=0)]
|
|
mask = np.zeros((512,512), dtype=np.int32)
|
|
mask = cv2.rectangle(mask, (lx,ly), (hx,hy), 1, -1)
|
|
mask = mask.astype(np.float32)
|
|
return mask
|
|
|
|
def mask_convex_hull(face, M, crop):
|
|
points = get_face_mesh(crop)
|
|
if points is None: return mask_simple_square(face, M, crop)
|
|
hull = ConvexHull(points)
|
|
mask = np.zeros((512,512), dtype=np.int32)
|
|
cv2.fillPoly(mask, [points[hull.vertices,:]], color=1)
|
|
mask = mask.astype(np.float32)
|
|
return mask
|
|
|
|
def mask_BiSeNet(face, M, crop):
|
|
with torch.no_grad():
|
|
bisenet = BiSeNet(n_classes=19)
|
|
bisenet.cuda()
|
|
model_path = os.path.join(models_dir, 'bisenet', '79999_iter.pth')
|
|
bisenet.load_state_dict(torch.load(model_path))
|
|
bisenet.eval()
|
|
crop_t = torch.from_numpy(crop)[None].permute(0,3,1,2).contiguous().float().div(255).cuda()
|
|
segms_t = bisenet(crop_t)[0].argmax(1).float()
|
|
# convert segmentation to mask
|
|
# 1: skin
|
|
# 2: l_brow
|
|
# 3: r_brow
|
|
# 4: l_eye
|
|
# 5: r_eye
|
|
# 6: eye_g
|
|
# 7: l_ear
|
|
# 8: r_ear
|
|
# 9: ear_r
|
|
# 10: nose
|
|
# 11: mouth
|
|
# 12: u_lip
|
|
# 13: l_lip
|
|
# 14: neck
|
|
# 15: neck_l
|
|
# 16: cloth
|
|
# 17: hair
|
|
# 18: hat
|
|
face_part_ids = torch.tensor([1,2,3,4,5,6,7,8,9,10,11,12,13]).cuda()
|
|
segms_t = torch.sum(segms_t.repeat(len(face_part_ids), 1,1,1) == face_part_ids[...,None,None,None], axis=0).float()
|
|
mask = segms_t[0].cpu()
|
|
return mask
|
|
|
|
mask_types = [
|
|
'simple_square',
|
|
'convex_hull',
|
|
'BiSeNet',
|
|
]
|
|
|
|
mask_funs = {
|
|
'simple_square': mask_simple_square,
|
|
'convex_hull': mask_convex_hull,
|
|
'BiSeNet': mask_BiSeNet,
|
|
}
|
|
|
|
def mask_crop(face, M, crop, mask_type):
|
|
mask = mask_funs[mask_type](face, M, crop)
|
|
return mask
|