import os os.environ["KMP_DUPLICATE_LIB_OK"] = "true" current_dir = os.path.dirname(os.path.abspath(__file__)) import sys sys.path.append(os.path.join(current_dir, "..")) import uuid import tqdm import numpy as np import cv2 import glob import math import pickle from talkingface.util.smooth import smooth_array from talkingface.run_utils import calc_face_mat import tqdm from utils import rotation_matrix_to_euler_angles import mediapipe as mp from obj_utils.utils import generateRenderInfo render_verts_, render_face = generateRenderInfo() face_pts_mean = render_verts_[:478, :3] mp_face_mesh = mp.solutions.face_mesh mp_face_detection = mp.solutions.face_detection def detect_face(frame): # 剔除掉多个人脸、大角度侧脸(鼻子不在两个眼之间)、部分人脸框在画面外、人脸像素低于80*80的 with mp_face_detection.FaceDetection( model_selection=1, min_detection_confidence=0.5) as face_detection: results = face_detection.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) if not results.detections or len(results.detections) > 1: return -1, None rect = results.detections[0].location_data.relative_bounding_box out_rect = [rect.xmin, rect.xmin + rect.width, rect.ymin, rect.ymin + rect.height] nose_ = mp_face_detection.get_key_point( results.detections[0], mp_face_detection.FaceKeyPoint.NOSE_TIP) l_eye_ = mp_face_detection.get_key_point( results.detections[0], mp_face_detection.FaceKeyPoint.LEFT_EYE) r_eye_ = mp_face_detection.get_key_point( results.detections[0], mp_face_detection.FaceKeyPoint.RIGHT_EYE) # print(nose_, l_eye_, r_eye_) if nose_.x > l_eye_.x or nose_.x < r_eye_.x: return -2, out_rect h, w = frame.shape[:2] # print(frame.shape) if rect.xmin < 0 or rect.ymin < 0 or rect.xmin + rect.width > w or rect.ymin + rect.height > h: return -3, out_rect if rect.width * w < 100 or rect.height * h < 100: return -4, out_rect return 1, out_rect def calc_face_interact(face0, face1): x_min = min(face0[0], face1[0]) x_max = max(face0[1], face1[1]) y_min = min(face0[2], face1[2]) y_max = max(face0[3], face1[3]) tmp0 = ((face0[1] - face0[0]) * (face0[3] - face0[2])) / ((x_max - x_min) * (y_max - y_min)) tmp1 = ((face1[1] - face1[0]) * (face1[3] - face1[2])) / ((x_max - x_min) * (y_max - y_min)) return min(tmp0, tmp1) def detect_face_mesh(frame): with mp_face_mesh.FaceMesh( static_image_mode=True, max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.5) as face_mesh: results = face_mesh.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) pts_3d = np.zeros([478, 3]) if not results.multi_face_landmarks: print("****** WARNING! No face detected! ******") else: image_height, image_width = frame.shape[:2] for face_landmarks in results.multi_face_landmarks: for index_, i in enumerate(face_landmarks.landmark): x_px = min(math.floor(i.x * image_width), image_width - 1) y_px = min(math.floor(i.y * image_height), image_height - 1) z_px = min(math.floor(i.z * image_width), image_width - 1) pts_3d[index_] = np.array([x_px, y_px, z_px]) return pts_3d def ExtractFromVideo(video_path, circle = False): cap = cv2.VideoCapture(video_path) if not cap.isOpened(): return 0 dir_path = os.path.dirname(video_path) vid_width = cap.get(cv2.CAP_PROP_FRAME_WIDTH) # 宽度 vid_height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) # 高度 totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT) # 总帧数 totalFrames = int(totalFrames) pts_3d = np.zeros([totalFrames, 478, 3]) face_rect_list = [] # os.makedirs("../preparation/{}/image".format(model_name)) for frame_index in tqdm.tqdm(range(totalFrames)): ret, frame = cap.read() # 按帧读取视频 # #到视频结尾时终止 if ret is False: break # cv2.imwrite("../preparation/{}/image/{:0>6d}.png".format(model_name, frame_index), frame) tag_, rect = detect_face(frame) if frame_index == 0 and tag_ != 1: print("第一帧人脸检测异常,请剔除掉多个人脸、大角度侧脸(鼻子不在两个眼之间)、部分人脸框在画面外、人脸像素低于80*80") pts_3d = -1 break elif tag_ == -1: # 有时候人脸检测会失败,就用上一帧的结果替代这一帧的结果 rect = face_rect_list[-1] elif tag_ != 1: print("第{}帧人脸检测异常,请剔除掉多个人脸、大角度侧脸(鼻子不在两个眼之间)、部分人脸框在画面外、人脸像素低于80*80, tag: {}".format(frame_index, tag_)) # exit() if len(face_rect_list) > 0: face_area_inter = calc_face_interact(face_rect_list[-1], rect) # print(frame_index, face_area_inter) if face_area_inter < 0.6: print("人脸区域变化幅度太大,请复查,超出值为{}, frame_num: {}".format(face_area_inter, frame_index)) pts_3d = -2 break face_rect_list.append(rect) x_min = rect[0] * vid_width y_min = rect[2] * vid_height x_max = rect[1] * vid_width y_max = rect[3] * vid_height seq_w, seq_h = x_max - x_min, y_max - y_min x_mid, y_mid = (x_min + x_max) / 2, (y_min + y_max) / 2 # x_min = int(max(0, x_mid - seq_w * 0.65)) # y_min = int(max(0, y_mid - seq_h * 0.4)) # x_max = int(min(vid_width, x_mid + seq_w * 0.65)) # y_max = int(min(vid_height, y_mid + seq_h * 0.8)) crop_size = int(max(seq_w * 1.35, seq_h * 1.35)) x_min = int(max(0, x_mid - crop_size * 0.5)) y_min = int(max(0, y_mid - crop_size * 0.45)) x_max = int(min(vid_width, x_min + crop_size)) y_max = int(min(vid_height, y_min + crop_size)) frame_face = frame[y_min:y_max, x_min:x_max] # cv2.imshow("s", frame_face) # cv2.waitKey(20) frame_kps = detect_face_mesh(frame_face) pts_3d[frame_index] = frame_kps + np.array([x_min, y_min, 0]) cap.release() # 释放视频对象 return pts_3d def run(video_path, template_path): cap = cv2.VideoCapture(video_path) if not cap.isOpened(): return 0 vid_width = cap.get(cv2.CAP_PROP_FRAME_WIDTH) # 宽度 vid_height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) # 高度 totalFrames = cap.get(cv2.CAP_PROP_FRAME_COUNT) # 总帧数 totalFrames = int(totalFrames) cap.release() pts_3d = ExtractFromVideo(video_path) import numpy as np if type(pts_3d) is np.ndarray and len(pts_3d) == totalFrames: print("关键点已提取") else: print("error in video: {}!!!".format(video_path)) return x_min, y_min, x_max, y_max = np.min(pts_3d[:, :, 0]), np.min( pts_3d[:, :, 1]), np.max( pts_3d[:, :, 0]), np.max(pts_3d[:, :, 1]) new_w = int((x_max - x_min) * 0.55) * 2 new_h = int((y_max - y_min) * 0.6) * 2 center_x = int((x_max + x_min) / 2.) center_y = int(y_min + (y_max - y_min) * 0.6) size = max(new_h, new_w) x_min, y_min, x_max, y_max = int(center_x - size // 2), int(center_y - size // 2), int( center_x + size // 2), int(center_y + size // 2) # 确定裁剪区域上边top和左边left坐标 top = y_min left = x_min # 裁剪区域与原图的重合区域 top_coincidence = int(max(top, 0)) bottom_coincidence = int(min(y_max, vid_height)) left_coincidence = int(max(left, 0)) right_coincidence = int(min(x_max, vid_width)) print("人脸活动范围:{}:{}, {}:{}".format(top_coincidence, bottom_coincidence, left_coincidence, right_coincidence)) out_size = 512 scale = 512. / size pts_3d = (pts_3d - np.array([left_coincidence, top_coincidence, 0])) * scale pts_3d = pts_3d.reshape(len(pts_3d), -1) pts_3d = smooth_array(pts_3d).reshape(len(pts_3d), -1, 3) mat_list, pts_normalized_list, face_pts_mean_personal_primer = calc_face_mat(pts_3d, face_pts_mean) pts_normalized_list = np.array(pts_normalized_list).reshape(len(pts_normalized_list), -1) x = pts_normalized_list from sklearn import decomposition import numpy as np import matplotlib.pyplot as plt if len(pts_normalized_list) > 30: n_components = 20 else: n_components = len(pts_normalized_list)//2 pca = decomposition.PCA(n_components=n_components) pca.fit(x) print("pca.mean_: ", pca.mean_) print("pca.explained_variance_: ", pca.explained_variance_) y = pca.transform(x) x_reconstructed = pca.inverse_transform(y) # plt.figure() # plt.plot([np.sum([pca.explained_variance_ratio_[:i]]) for i in range(n_components)], 'k', linewidth=2) # plt.xlabel('n_components', fontsize=16) # plt.ylabel('explained_variance_ratio_', fontsize=16) # plt.grid() # plt.savefig('explained_variance_ratio_.png') # plt.show() pts_normalized_list = x_reconstructed.reshape(len(pts_normalized_list), 478, 3) euler_angles_list = [] trans_list = [] for i in mat_list: euler_angles = rotation_matrix_to_euler_angles(i[:3, :3]) euler_angles_list.append(euler_angles) trans_list.append(i[:3, 3]) euler_angles_list = np.array(euler_angles_list) trans_list = np.array(trans_list) trans_list = trans_list - np.mean(trans_list, axis=0) # print(trans_list) rotate_trans_param = np.concatenate([euler_angles_list, trans_list], axis=1) blendshape_verts_bias = pts_normalized_list - face_pts_mean_personal_primer os.makedirs(os.path.dirname(template_path), exist_ok=True) print(template_path, rotate_trans_param.shape, blendshape_verts_bias.shape) with open(template_path, "wb") as f: pickle.dump([rotate_trans_param, blendshape_verts_bias, [top_coincidence, bottom_coincidence, left_coincidence, right_coincidence]], f) def main(): # 检查命令行参数的数量 if len(sys.argv) != 3: print("Usage: python generate_move_template.py ") sys.exit(1) # 参数数量不正确时退出程序 video_path = sys.argv[1] template_path = sys.argv[2] print(f"Video path is set to: {video_path}, template path is set to: {template_path}") run(video_path, template_path) if __name__ == "__main__": main()