From 4fdc17c08cc02a5e8070e61e8df48b4f58d48544 Mon Sep 17 00:00:00 2001 From: yanghantao <7196390@qq.com> Date: Thu, 19 Sep 2024 14:31:39 +0800 Subject: [PATCH] add align type:JawLine --- __init__.py | 249 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 205 insertions(+), 44 deletions(-) diff --git a/__init__.py b/__init__.py index 98c363e..1198340 100644 --- a/__init__.py +++ b/__init__.py @@ -871,7 +871,7 @@ class FaceShaperMatchV2: "source_crop_info": ("CROPINFO", ), "target_crop_info": ("CROPINFO", ), "landmarkType": (["ALL","OUTLINE"], ), - "AlignType":(["Width","Height","Landmarks"], ), + "AlignType":(["Width","Height","Landmarks","JawLine"], ), }, } @@ -881,6 +881,85 @@ class FaceShaperMatchV2: CATEGORY = "FaceShaper" + def LandMark203_to_68(self,source): + #jawLine + out = [source[108]] + for i in range(0,7): + out.append((source[110+i*2]+source[111+i*2])/2) + out.append(source[126]) + for i in range(0,7): + out.append((source[128+i*2]+source[129+i*2])/2) + out.append(source[144]) + #leftEyeBow + out.append(source[145]) + out.append((source[148]+source[162])/2) + out.append((source[150]+source[160])/2) + out.append((source[152]+source[158])/2) + out.append(source[155]) + #rightEyeBow + out.append(source[165]) + out.append((source[168]+source[182])/2) + out.append((source[170]+source[180])/2) + out.append((source[172]+source[177])/2) + out.append(source[175]) + + #nose + out.append(source[199]) + out.append((source[199]+source[200])/2) + out.append(source[200]) + out.append(source[201]) + out.append(source[189]) + out.append(source[190]) + out.append(source[202]) + out.append(source[191]) + out.append(source[192]) + + #leftEye + out.append(source[0]) + out.append(source[3]) + out.append(source[8]) + out.append(source[12]) + out.append(source[16]) + out.append(source[21]) + + #rightEye + out.append(source[24]) + out.append(source[28]) + out.append(source[33]) + out.append(source[36]) + out.append(source[39]) + out.append(source[45]) + + #UpperLipUp + out.append(source[48]) + out.append(source[51]) + out.append(source[54]) + out.append(source[57]) + out.append(source[60]) + out.append(source[63]) + out.append(source[66]) + + #LowerLipDown + out.append(source[69]) + out.append(source[72]) + out.append(source[75]) + out.append(source[78]) + out.append(source[81]) + + + out.append(source[84]) + out.append(source[87]) + out.append(source[90]) + out.append(source[93]) + out.append(source[96]) + + out.append(source[99]) + out.append(source[102]) + out.append(source[105]) + + return out + + def run(self,source_image,source_crop_info, target_crop_info,landmarkType,AlignType): @@ -898,18 +977,65 @@ class FaceShaperMatchV2: #print(len(landmarks1)) #len(lendmarks1) will always be 203 #V2版本改了检测工具，因此只用203个点 - if landmarkType == "ALL" or AlignType == "Landmarks": - leftEye1=np.mean( landmarks1[0:23],axis=0) - rightEye1=np.mean( landmarks1[24:47],axis=0) - leftEye2=np.mean( landmarks2[0:23],axis=0) - rightEye2=np.mean( landmarks2[24:47],axis=0) - jaw1=landmarks1[108:144] - jaw2=landmarks2[108:144] + #203个点太多，影响液化算法的运行效率，再次转换成68个点 + use_68_points=True + if(use_68_points): + landmarks1 = self.LandMark203_to_68(landmarks1) + landmarks2 = self.LandMark203_to_68(landmarks2) + landmarks1 = landmarks1[0:65] + landmarks2 = landmarks2[0:65] + #else: + + + if(use_68_points): + leftEye1=np.mean( landmarks1[36:42],axis=0) + rightEye1=np.mean( landmarks1[42:48],axis=0) + leftEye2=np.mean( landmarks2[36:42],axis=0) + rightEye2=np.mean( landmarks2[42:48],axis=0) + jaw1=landmarks1[0:17] + jaw2=landmarks2[0:17] + centerOfJaw1=np.mean( jaw1,axis=0) + centerOfJaw2=np.mean( jaw2,axis=0) + else: + #保留这个是为了以后要是想改回来 + leftEye1=np.mean( landmarks1[0:24],axis=0) + rightEye1=np.mean( landmarks1[24:48],axis=0) + leftEye2=np.mean( landmarks2[0:24],axis=0) + rightEye2=np.mean( landmarks2[24:48],axis=0) + jaw1=landmarks1[108:145] + jaw2=landmarks2[108:145] centerOfJaw1=np.mean( jaw1,axis=0) centerOfJaw2=np.mean( jaw2,axis=0) - else: - landmarks1 = landmarks1[108:144] - landmarks2 = landmarks2[108:144] + + + # if landmarkType == "ALL": # or AlignType == "Landmarks" or AlignType == "JawLine": + # if(len(landmarks1)==203): + # #保留这个是为了以后要是想改回来 + # leftEye1=np.mean( landmarks1[0:24],axis=0) + # rightEye1=np.mean( landmarks1[24:48],axis=0) + # leftEye2=np.mean( landmarks2[0:24],axis=0) + # rightEye2=np.mean( landmarks2[24:48],axis=0) + # jaw1=landmarks1[108:145] + # jaw2=landmarks2[108:145] + # centerOfJaw1=np.mean( jaw1,axis=0) + # centerOfJaw2=np.mean( jaw2,axis=0) + # else: + # leftEye1=np.mean( landmarks1[36:42],axis=0) + # rightEye1=np.mean( landmarks1[42:48],axis=0) + # leftEye2=np.mean( landmarks2[36:42],axis=0) + # rightEye2=np.mean( landmarks2[42:48],axis=0) + # jaw1=landmarks1[0:17] + # jaw2=landmarks2[0:17] + # centerOfJaw1=np.mean( jaw1,axis=0) + # centerOfJaw2=np.mean( jaw2,axis=0) + # else: + # if(len(landmarks1)==203): + # #保留这个是为了以后要是想改回来 + # landmarks1 = landmarks1[108:145] + # landmarks2 = landmarks2[108:145] + # else: + # landmarks1 = landmarks1[0:17] + # landmarks2 = landmarks2[0:17] #画面划分成16*16个区域，然后去掉边界框以外的区域。 src_points = np.array([ @@ -919,12 +1045,10 @@ class FaceShaperMatchV2: ]) #上面这些区域同时被加入src和dst，使这些区域不被拉伸（效果是图片边缘不被拉伸） - src_points = src_points[(src_points[:, 0] <= w/8) | (src_points[:, 0] >= 7*w/8) | (src_points[:, 1] >= 7*h/8)| (src_points[:, 1] <= h/8)] - #mark_img = self.draw_landmarks(mark_img, src_points, color=(255, 0, 255)) + src_points = src_points[(src_points[:, 0] <= w/8) | (src_points[:, 0] >= 7*w/8) | (src_points[:, 1] >= 7*h/8)| (src_points[:, 1] <= h/8)] dst_points = src_points.copy() - #不知道原作者为何把这个数组叫dst，其实这是变形前的坐标，即原图的坐标 - dst_points = np.append(dst_points,landmarks1,axis=0) + #变形目标人物的landmarks，先计算边界框 landmarks2=np.array(landmarks2) @@ -934,6 +1058,8 @@ class FaceShaperMatchV2: max_y = np.max(landmarks2[:, 1]) #得到目标人物的边界框的长宽比 ratio2 = (max_x - min_x) / (max_y - min_y) + middlePoint2 = [ (max_x + min_x) / 2, (max_y + min_y) / 2] + #print("ratio2",ratio2) #变形原始人物的landmarks，边界框 landmarks1=np.array(landmarks1) @@ -945,41 +1071,80 @@ class FaceShaperMatchV2: ratio1 = (max_x - min_x) / (max_y - min_y) middlePoint = [ (max_x + min_x) / 2, (max_y + min_y) / 2] - landmarks1_cpy = landmarks1.copy() + #print("ratio1",ratio1) + if AlignType=="Width": - #保持人物脸部边界框中心点不变，垂直方向上缩放，使边界框的比例变得跟目标人物的边界框比例一致 - landmarks1_cpy[:, 1] = (landmarks1_cpy[:, 1] - middlePoint[1]) * ratio1 / ratio2 + middlePoint[1] - elif AlignType=="Height": - #保持人物脸部边界框中心点不变，水平方向上缩放，使边界框的比例变得跟目标人物的边界框比例一致 - landmarks1_cpy[:, 0] = (landmarks1_cpy[:, 0] - middlePoint[0]) * ratio2 / ratio1 + middlePoint[0] - elif AlignType=="Landmarks": - MiddleOfEyes1 = (leftEye1+rightEye1)/2 - MiddleOfEyes2 = (leftEye2+rightEye2)/2 - distance1 = ((leftEye1[0] - rightEye1[0]) ** 2 + (leftEye1[1] - rightEye1[1]) ** 2) ** 0.5 - distance2 = ((leftEye2[0] - rightEye2[0]) ** 2 + (leftEye2[1] - rightEye2[1]) ** 2) ** 0.5 - factor = distance1 / distance2 - MiddleOfEyes2 = np.array(MiddleOfEyes2) - landmarks1_cpy = (landmarks2 - MiddleOfEyes2) * factor + MiddleOfEyes1 + #保持人物脸部边界框中心点不变，垂直方向上缩放，使边界框的比例变得跟目标人物的边界框比例一致 + if(landmarkType=="ALL"): + dst_points = np.append(dst_points,landmarks1,axis=0) + target_points = landmarks1.copy() + else: + dst_points = np.append(dst_points,jaw1,axis=0) + jaw1=np.array(jaw1) + target_points = jaw1.copy() + target_points[:, 1] = (target_points[:, 1] - middlePoint[1]) * ratio1 / ratio2 + middlePoint[1] + src_points = np.append(src_points,target_points,axis=0)#不知道原作者为何把这个数组叫src，其实这是变形后的坐标 - centerOfJaw2 = np.array(centerOfJaw2) - jawLineTarget = (landmarks2[108:144] - centerOfJaw2) * factor + centerOfJaw1 - landmarks1_cpy[108:144] = jawLineTarget - #不知道原作者为何把这个数组叫src，其实这是变形后的坐标 - src_points = np.append(src_points,landmarks1_cpy,axis=0) - #print(landmarks1_cpy) + elif AlignType=="Height": + #保持人物脸部边界框中心点不变，水平方向上缩放，使边界框的比例变得跟目标人物的边界框比例一致 + if(landmarkType=="ALL"): + dst_points = np.append(dst_points,landmarks1,axis=0) #不知道原作者为何把这个数组叫dst，其实这是变形前的坐标，即原图的坐标 + target_points = landmarks1.copy() + else: + dst_points = np.append(dst_points,jaw1,axis=0)#不知道原作者为何把这个数组叫dst，其实这是变形前的坐标，即原图的坐标 + jaw1=np.array(jaw1) + target_points = jaw1.copy() + target_points[:, 0] = (target_points[:, 0] - middlePoint[0]) * ratio2 / ratio1 + middlePoint[0] + src_points = np.append(src_points,target_points,axis=0)#不知道原作者为何把这个数组叫src，其实这是变形后的坐标 + + elif AlignType=="Landmarks": + if(landmarkType=="ALL"): + #以双眼中心为基准点，按双眼距离计算缩放系数。效果是变形前后眼睛位置不变 + MiddleOfEyes1 = (leftEye1+rightEye1)/2 + MiddleOfEyes2 = (leftEye2+rightEye2)/2 + distance1 = ((leftEye1[0] - rightEye1[0]) ** 2 + (leftEye1[1] - rightEye1[1]) ** 2) ** 0.5 + distance2 = ((leftEye2[0] - rightEye2[0]) ** 2 + (leftEye2[1] - rightEye2[1]) ** 2) ** 0.5 + factor = distance1 / distance2 + MiddleOfEyes2 = np.array(MiddleOfEyes2) + target_points = (landmarks2 - MiddleOfEyes2) * factor + MiddleOfEyes1 + + #面部轮廓线则以轮廓线中心点为基准点，缩放系数还是从双眼距离计算 + centerOfJaw2 = np.array(centerOfJaw2) + jawLineTarget = (landmarks2[108:144] - centerOfJaw2) * factor + centerOfJaw1 + target_points[108:144] = jawLineTarget + + dst_points = np.append(dst_points,landmarks1,axis=0)#不知道原作者为何把这个数组叫dst，其实这是变形前的坐标，即原图的坐标 + else: + #此时只有轮廓线landMark。对齐两个landMark的中心点，然后用2替换掉1 + dst_points = np.append(dst_points,jaw1,axis=0)#不知道原作者为何把这个数组叫dst，其实这是变形前的坐标，即原图的坐标 + target_points=(jaw2-centerOfJaw2)+centerOfJaw1 + src_points = np.append(src_points,target_points,axis=0)#不知道原作者为何把这个数组叫src，其实这是变形后的坐标 + + + elif AlignType=="JawLine": + lenOfJaw=len(jaw1) + distance1= ((jaw1[0][0] - jaw1[lenOfJaw-1][0]) ** 2 + (jaw1[0][1] - jaw1[lenOfJaw-1][1]) ** 2) ** 0.5 + distance2= ((jaw2[0][0] - jaw2[lenOfJaw-1][0]) ** 2 + (jaw2[0][1] - jaw2[lenOfJaw-1][1]) ** 2) ** 0.5 + factor = distance1 / distance2 + if landmarkType == "ALL": + dst_points = np.append(dst_points,landmarks1,axis=0) + target_points=(landmarks2-jaw2[0])*factor+jaw1[0] + src_points = np.append(src_points,target_points,axis=0) + else: + dst_points = np.append(dst_points,jaw1,axis=0) + target_points=(jaw2-jaw2[0])*factor+jaw1[0] + src_points = np.append(src_points,target_points,axis=0) mark_img = draw_pointsOnImg(image1, dst_points, color=(255, 255, 0),radius=4) mark_img = draw_pointsOnImg(mark_img, src_points, color=(255, 0, 0),radius=3) mark_img = drawLineBetweenPoints(mark_img, dst_points,src_points) - #mark_img = draw_landmarks(image1, landmarks1, color=(255, 0, 0),radius=3) - #mark_img2 = draw_landmarks(image1, source_crop_info["crop_info_list"][0]['lmk_crop'], color=(255, 0, 0),radius=3) - # Create the RBF interpolator instance + + #### 开始对图片进行液化变形 #Tried many times, finally find out these array should be exchange w,h before go into RBFInterpolator src_points[:, [0, 1]] = src_points[:, [1, 0]] dst_points[:, [0, 1]] = dst_points[:, [1, 0]] - rbfy = RBFInterpolator(src_points,dst_points[:,1],kernel="thin_plate_spline") rbfx = RBFInterpolator(src_points,dst_points[:,0],kernel="thin_plate_spline") @@ -994,16 +1159,12 @@ class FaceShaperMatchV2: map_x = rbfx(flatten).reshape(height,width).astype(np.float32) # Apply the remapping to the image using OpenCV warped_image = cv2.remap(image1, map_y, map_x, cv2.INTER_LINEAR) - + ######### 液化变形结束 warped_image = torch.from_numpy(warped_image.astype(np.float32) / 255.0).unsqueeze(0) mark_img = torch.from_numpy(mark_img.astype(np.float32) / 255.0).unsqueeze(0) - #mark_img2= torch.from_numpy(mark_img2.astype(np.float32) / 255.0).unsqueeze(0) output.append(warped_image) output.append(mark_img) - #output.append(mark_img2) - #print("warped_image",warped_image.shape) - #print("mark_img",mark_img.shape) return (output)