import numpy as np import random import cv2 INDEX_LEFT_EYEBROW = [276, 283, 282, 295, 285, 336, 296, 334, 293, 300] INDEX_RIGHT_EYEBROW = [46, 53, 52, 65, 55, 107, 66, 105, 63, 70] INDEX_EYEBROW = INDEX_LEFT_EYEBROW + INDEX_RIGHT_EYEBROW INDEX_NOSE_EDGE = [343, 355, 358, 327, 326, 2, 97, 98, 129, 126, 114] INDEX_NOSE_MID = [6, 197,195,5,4] INDEX_NOSE = INDEX_NOSE_EDGE + INDEX_NOSE_MID INDEX_LIPS_INNER = [78, 95, 88, 178, 87, 14, 317, 402, 318, 324,308,415,310,311,312,13,82,81,80,191] INDEX_LIPS_OUTER = [61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291,409,270,269,267,0,37,39,40,185,] INDEX_LIPS = INDEX_LIPS_INNER + INDEX_LIPS_OUTER INDEX_LEFT_EYE = [263, 249, 390, 373, 374, 380, 381, 382, 362, 398, 384, 385, 386, 387, 388, 466] INDEX_RIGHT_EYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246] INDEX_EYE = INDEX_LEFT_EYE + INDEX_RIGHT_EYE # 下半边脸的轮廓 INDEX_FACE_OVAL = [ 454, 323, 361, 288, 397, 365, 379, 378, 400, 377, 152, 148, 176, 149, 150, 136, 172, 58, 132, 93, 234, # 206, 426 ] INDEX_MUSCLE = [ 371,266,425,427,434,394, 169,214,207,205,36,142 ] main_keypoints_index = INDEX_EYEBROW + INDEX_NOSE + INDEX_LIPS + INDEX_EYE + INDEX_FACE_OVAL + INDEX_MUSCLE rotate_ref_index = INDEX_EYEBROW + INDEX_NOSE + INDEX_EYE + INDEX_FACE_OVAL + INDEX_MUSCLE # print(len(main_keypoints_index)) Normalized = True if Normalized: tmp = 0 list_ = [] for i in [INDEX_LEFT_EYEBROW, INDEX_RIGHT_EYEBROW, INDEX_NOSE_EDGE, INDEX_NOSE_MID, INDEX_LIPS_INNER, INDEX_LIPS_OUTER, INDEX_LEFT_EYE, INDEX_RIGHT_EYE, INDEX_FACE_OVAL, INDEX_MUSCLE]: i = (tmp + np.arange(len(i))).tolist() list_.append(i) tmp += len(i) [INDEX_LEFT_EYEBROW, INDEX_RIGHT_EYEBROW, INDEX_NOSE_EDGE, INDEX_NOSE_MID, INDEX_LIPS_INNER, INDEX_LIPS_OUTER, INDEX_LEFT_EYE, INDEX_RIGHT_EYE, INDEX_FACE_OVAL, INDEX_MUSCLE] = list_ INDEX_EYEBROW = INDEX_LEFT_EYEBROW + INDEX_RIGHT_EYEBROW INDEX_NOSE = INDEX_NOSE_EDGE + INDEX_NOSE_MID INDEX_LIPS = INDEX_LIPS_INNER + INDEX_LIPS_OUTER INDEX_EYE = INDEX_LEFT_EYE + INDEX_RIGHT_EYE # print(INDEX_FACE_OVAL_STABLE) # print(len(main_keypoints_index)) # print(len(set(main_keypoints_index))) FACE_MASK_INDEX = INDEX_FACE_OVAL[2:-2] def crop_face(keypoints, is_train = False, size = [512, 512]): """ x_ratio: 裁剪出一个正方形，边长根据keypoints的宽度 * x_ratio决定 """ x_min, y_min, x_max, y_max = np.min(keypoints[FACE_MASK_INDEX, 0]), np.min(keypoints[FACE_MASK_INDEX, 1]), np.max(keypoints[FACE_MASK_INDEX, 0]), np.max( keypoints[FACE_MASK_INDEX, 1]) y_min = keypoints[33, 1] # 两眼间的点开始y轴裁剪 border_width_half = max(x_max - x_min, y_max - y_min) * 0.6 center_x = int((x_min + x_max) / 2.0) center_y = int((y_min + y_max) / 2.0) if is_train: w_offset = random.randint(-2, 2) h_offset = random.randint(-2, 2) center_x = center_x + w_offset center_y = center_y + h_offset x_min, y_min, x_max, y_max = int(center_x - border_width_half), int(center_y - border_width_half), int( center_x + border_width_half), int(center_y + border_width_half) x_min = max(0, x_min) y_min = max(0, y_min) x_max = min(size[1], x_max) y_max = min(size[0], y_max) return [x_min, y_min, x_max, y_max] def draw_face_feature_maps(keypoints, mode = ["mouth", "nose", "eye", "oval"], size=(256, 256), im_edges = None, mouth_width = None, mouth_height = None): w, h = size # edge map for face region from keypoints if im_edges is None: im_edges = np.zeros((h, w, 3), np.uint8) # edge map for all edges if "mouth_bias" in mode: w0, w1, h0, h1 = (int(keypoints[INDEX_NOSE_EDGE[5], 0] - mouth_width / 2), int(keypoints[INDEX_NOSE_EDGE[5], 0] + mouth_width / 2), int(keypoints[INDEX_NOSE_EDGE[5], 1] + mouth_height / 4), int(keypoints[INDEX_NOSE_EDGE[5], 1] + mouth_height / 4 + mouth_height)) w0, h0 = max(0, w0), max(h0, 0) # print(w0, w1, h0, h1) mouth_mask = np.zeros((h, w, 3), np.uint8) # edge map for all edges mouth_mask[h0:h1, w0:w1] = 255 mouth_index = np.where(mouth_mask == 255) blur = (10, 10) img_mouth = cv2.cvtColor(im_edges, cv2.COLOR_BGR2GRAY) img_mouth = cv2.blur(img_mouth, blur) # print(mouth_index) mean_ = int(np.mean(img_mouth[(mouth_index[0], mouth_index[1])])) max_, min_ = random.randint(mean_ + 40, mean_ + 70), random.randint(mean_ - 70, mean_ - 40) img_mouth = (img_mouth.astype(np.float32) - min_) / (max_ - min_) * 255. img_mouth = img_mouth.clip(0, 255).astype(np.uint8) # print(img_mouth.shape[0], img_mouth.shape[1]) img_mouth = cv2.resize(img_mouth, (100, 50)) # 定义噪声的标准差 sigma = 8 # 你可以根据需要调整这个值 # 生成与图片相同大小和类型的噪声 noise = sigma * np.random.randn(img_mouth.shape[0], img_mouth.shape[1]) # 将噪声添加到图片上 img_mouth = img_mouth + noise img_mouth = cv2.resize(img_mouth.clip(0, 255).astype(np.uint8), (im_edges.shape[0], im_edges.shape[1])) img_mouth = np.concatenate( [img_mouth[:, :, np.newaxis], img_mouth[:, :, np.newaxis], img_mouth[:, :, np.newaxis]], axis=2) # bias_x = int(min(size[0] - max(mouth_index[0]), min(mouth_index[0]))) # bias_y = int(min(size[0] - max(mouth_index[1]), min(mouth_index[1]))) # print(bias_x, bias_y) # bias_x = random.randint(-bias_x // 10, bias_x // 10) # bias_y = random.randint(-bias_y // 10, bias_y // 10) # # mouth_index2 = np.array(mouth_index) # # print(mouth_index) # mouth_index2[0] = mouth_index[0] + bias_x # mouth_index2[1] = mouth_index[1] + bias_y # mouth_index2 = (mouth_index2[0], mouth_index2[1], mouth_index2[2]) # # print(mouth_index2.T, mouth_index2.shape) # output = np.zeros((h, w, 3), np.uint8) # output[mouth_index2] = img_mouth[mouth_index] # return im_edges output[mouth_index] = img_mouth[mouth_index] im_edges = output if "nose" in mode: for ii in range(len(INDEX_NOSE_EDGE) - 1): pt1 = [int(flt) for flt in keypoints[INDEX_NOSE_EDGE[ii]]][:2] pt2 = [int(flt) for flt in keypoints[INDEX_NOSE_EDGE[ii+1]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (0, 255, 0), 2) for ii in range(len(INDEX_NOSE_MID) -1): pt1 = [int(flt) for flt in keypoints[INDEX_NOSE_MID[ii]]][:2] pt2 = [int(flt) for flt in keypoints[INDEX_NOSE_MID[ii + 1]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (0, 255, 0), 2) if "eye" in mode: for ii in range(len(INDEX_LEFT_EYE)): pt1 = [int(flt) for flt in keypoints[INDEX_LEFT_EYE[ii]]][:2] pt2 = [int(flt) for flt in keypoints[INDEX_LEFT_EYE[(ii + 1)%len(INDEX_LEFT_EYE)]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (0, 255, 0), 2) for ii in range(len(INDEX_RIGHT_EYE)): pt1 = [int(flt) for flt in keypoints[INDEX_RIGHT_EYE[ii]]][:2] pt2 = [int(flt) for flt in keypoints[INDEX_RIGHT_EYE[(ii + 1) % len(INDEX_RIGHT_EYE)]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (0, 255, 0), 2) if "oval" in mode: tmp = INDEX_FACE_OVAL[:6] for ii in range(len(tmp) -1): pt1 = [int(flt) for flt in keypoints[tmp[ii]]][:2] pt2 = [int(flt) for flt in keypoints[tmp[ii + 1]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (0, 0, 255), 2) tmp = INDEX_FACE_OVAL[-6:] for ii in range(len(tmp) - 1): pt1 = [int(flt) for flt in keypoints[tmp[ii]]][:2] pt2 = [int(flt) for flt in keypoints[tmp[ii + 1]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (0, 0, 255), 2) # if "mouth_outer" in mode: # pts = keypoints[INDEX_LIPS_OUTER] # pts = pts.reshape((-1, 1, 2)).astype(np.int32) # cv2.fillPoly(im_edges, [pts], color=(255, 0, 0)) # if "mouth" in mode: # pts = keypoints[INDEX_LIPS_OUTER][:,2] # pts = pts.reshape((-1, 1, 2)).astype(np.int32) # cv2.fillPoly(im_edges, [pts], color=(255, 0, 0)) # pts = keypoints[INDEX_LIPS_INNER][:,2] # pts = pts.reshape((-1, 1, 2)).astype(np.int32) # cv2.fillPoly(im_edges, [pts], color=(0, 0, 0)) # if "mouth_outer" in mode: # for ii in range(len(INDEX_LIPS_OUTER)): # pt1 = [int(flt) for flt in keypoints[INDEX_LIPS_OUTER[ii]]][:2] # pt2 = [int(flt) for flt in keypoints[INDEX_LIPS_OUTER[(ii + 1)%len(INDEX_LIPS_OUTER)]]][:2] # cv2.line(im_edges, tuple(pt1), tuple(pt2), (255, 0, 0), 2) if "mouth" in mode: for ii in range(len(INDEX_LIPS_OUTER)): pt1 = [int(flt) for flt in keypoints[INDEX_LIPS_OUTER[ii]]][:2] pt2 = [int(flt) for flt in keypoints[INDEX_LIPS_OUTER[(ii + 1)%len(INDEX_LIPS_OUTER)]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (255, 0, 0), 2) for ii in range(len(INDEX_LIPS_INNER)): pt1 = [int(flt) for flt in keypoints[INDEX_LIPS_INNER[ii]]][:2] pt2 = [int(flt) for flt in keypoints[INDEX_LIPS_INNER[(ii + 1)%len(INDEX_LIPS_INNER)]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (255, 0, 0), 2) if "muscle" in mode: for ii in range(len(INDEX_MUSCLE) - 1): pt1 = [int(flt) for flt in keypoints[INDEX_MUSCLE[ii]]][:2] pt2 = [int(flt) for flt in keypoints[INDEX_MUSCLE[(ii + 1) % len(INDEX_MUSCLE)]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (255, 255, 255), 2) if "oval_all" in mode: tmp = INDEX_FACE_OVAL for ii in range(len(tmp) -1): pt1 = [int(flt) for flt in keypoints[tmp[ii]]][:2] pt2 = [int(flt) for flt in keypoints[tmp[ii + 1]]][:2] cv2.line(im_edges, tuple(pt1), tuple(pt2), (0, 0, 255), 2) return im_edges def smooth_array(array, weight = [0.1,0.8,0.1], mode = "numpy"): ''' Args: array: [n_frames, n_values]， 需要转换为[n_values, 1, n_frames] weight: Conv1d.weight, 一维卷积核权重 Returns: array: [n_frames, n_values]， 光滑后的array ''' if mode == "torch": import torch import torch.nn as nn import torch.nn.functional as F input = torch.Tensor(np.transpose(array[:, np.newaxis, :], (2, 1, 0))) smooth_length = len(weight) assert smooth_length % 2 == 1, "卷积核权重个数必须使用奇数" pad = (smooth_length // 2, smooth_length // 2) # 当pad只有两个参数时，仅改变最后一个维度, 左边扩充1列，右边扩充1列 input = F.pad(input, pad, "replicate") with torch.no_grad(): conv1 = nn.Conv1d(in_channels=1, out_channels=1, kernel_size=smooth_length) # 卷积核的元素值初始化 weight = torch.tensor(weight).view(1, 1, -1) conv1.weight = torch.nn.Parameter(weight) nn.init.constant_(conv1.bias, 0) # 偏置值为0 # print(conv1.weight) out = conv1(input) return out.permute(2, 1, 0).squeeze().numpy() else: # out = np.zeros([array.shape[0] + 2, array.shape[1]]) # input = np.zeros([array.shape[0] + 2, array.shape[1]]) # input[0] = array[0] # input[-1] = array[-1] # input[1:-1] = array # for i in range(out.shape[1]): # out[:, i] = np.convolve(input[:, i], weight, mode="same") # out0 = out[1:-1] fliter = np.array([weight]).T x0 = array fliter = np.repeat(fliter, x0.shape[1], axis=1) out0 = np.zeros_like(x0) for i in range(len(x0)): if i == 0 or i == len(x0) - 1: out0[i] = x0[i] else: tmp = x0[i - 1:i + 2] * fliter out0[i] = np.sum(tmp, axis=0) return out0 def generate_face_mask(): face_mask = np.zeros([256, 256], dtype=np.uint8) for i in range(20): ii = 19 - i face_mask[ii, :] = 13 * i face_mask[255 - ii, :] = 13 * i face_mask[:, ii] = 13 * i face_mask[:, 255 - ii] = 13 * i face_mask = np.array([face_mask, face_mask, face_mask]).transpose(1, 2, 0).astype(float) / 255. print(face_mask.shape) return face_mask from math import cos,sin,radians def RotateAngle2Matrix(tmp): #tmp为xyz的旋转角,角度值 tmp = [radians(i) for i in tmp] matX = np.array([[1.0, 0, 0], [0.0, cos(tmp[0]), -sin(tmp[0])], [0.0, sin(tmp[0]), cos(tmp[0])]]) matY = np.array([[cos(tmp[1]), 0, sin(tmp[1])], [0.0, 1, 0], [-sin(tmp[1]), 0, cos(tmp[1])]]) matZ = np.array([[cos(tmp[2]), -sin(tmp[2]), 0], [sin(tmp[2]), cos(tmp[2]), 0], [0, 0, 1]]) matRotate = np.matmul(matZ, matY) matRotate = np.matmul(matRotate, matX) return matRotate