kijai-ComfyUI-WanVideoWrapper/fantasyportrait/camer.py

import math
import os.path as osp

import numpy as np


def smoothing_factor(t_e, cutoff):
    r = 2 * math.pi * cutoff * t_e
    return r / (r + 1)


def exponential_smoothing(a, x, x_prev):
    return a * x + (1 - a) * x_prev


class OneEuroFilter:
    def __init__(self, dx0=0.0, d_cutoff=1.0):
        self.d_cutoff = float(d_cutoff)
        self.dx_prev = float(dx0)

    def __call__(self, x, x_prev, fcmin=1.0, min_cutoff=1.0, beta=0.0):
        if x_prev is None:
            return x
        # t_e = 1
        a_d = smoothing_factor(fcmin, self.d_cutoff)
        dx = (x - x_prev) / fcmin
        dx_hat = exponential_smoothing(a_d, dx, self.dx_prev)
        cutoff = min_cutoff + beta * abs(dx_hat)
        a = smoothing_factor(fcmin, cutoff)
        x_hat = exponential_smoothing(a, x, x_prev)
        self.dx_prev = dx_hat
        return x_hat


def cult_dis(old_kpts, new_kpts):
    dis = np.sqrt(
        np.square(new_kpts[:, 0] - old_kpts[:, 0])
        + np.square(new_kpts[:, 1] - old_kpts[:, 1])
    )
    return dis


class Smoother222(object):
    def __init__(self):
        # face config
        self.face_idx = list(range(0, 33))
        self.face_down_idx = list(range(9, 24))
        self.filter_face = OneEuroFilter()
        # nose config
        self.nose_idx = list(range(33, 48))
        self.filter_nose = OneEuroFilter()
        # eyebrow config
        self.eyebrow_idx = list(range(48, 74))
        self.filter_eyebrow = OneEuroFilter()
        # eye config
        self.left_eye_idx = list(range(74, 96))
        self.filter_left_eye = OneEuroFilter()
        self.right_eye_idx = list(range(96, 118))
        self.filter_right_eye = OneEuroFilter()
        # mouth config
        self.mouth_idx = list(range(118, 182))
        self.filter_mouth = OneEuroFilter()
        # pupil config
        self.left_pupil_idx = list(range(182, 202))
        self.filter_left_pupil = OneEuroFilter()
        self.right_pupil_idx = list(range(202, 222))
        self.filter_right_pupil = OneEuroFilter()
        self.prev_points = None

    def smooth(self, new_points, face_dis):
        if self.prev_points is None:
            self.prev_points = new_points.copy()
            return new_points
        dis = cult_dis(self.prev_points, new_points) / face_dis
        smooth_points = new_points.copy()

        # smooth face
        if np.mean(dis[self.face_down_idx]) < 0.005:
            ratio_tmp = np.mean(dis[self.face_down_idx]) / 0.005
            fcmin_tmp = 0.05 * ratio_tmp
            beta_tmp = 0.05 * ratio_tmp
            smooth_points[self.face_idx] = self.filter_face(
                new_points[self.face_idx],
                self.prev_points[self.face_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        elif np.mean(dis[self.face_down_idx]) < 0.02:
            ratio_tmp = (np.mean(dis[self.face_down_idx]) - 0.005) / (0.02 - 0.005)
            fcmin_tmp = 0.05 + (0.3 - 0.05) * ratio_tmp
            beta_tmp = 0.05 + (0.3 - 0.05) * ratio_tmp
            smooth_points[self.face_idx] = self.filter_face(
                new_points[self.face_idx],
                self.prev_points[self.face_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        else:
            smooth_points[self.face_idx] = self.filter_face(
                new_points[self.face_idx],
                self.prev_points[self.face_idx],
                fcmin=0.3,
                beta=0.3,
            )
        # smooth nose
        if np.mean(dis[self.nose_idx]) < 0.003:
            # stable
            ratio_tmp = np.mean(dis[self.nose_idx]) / 0.003
            fcmin_tmp = 0.03 * ratio_tmp
            beta_tmp = 0.03 * ratio_tmp
            smooth_points[self.nose_idx] = self.filter_nose(
                new_points[self.nose_idx],
                self.prev_points[self.nose_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        elif np.mean(dis[self.nose_idx]) < 0.02:
            ratio_tmp = (np.mean(dis[self.nose_idx]) - 0.003) / (0.02 - 0.003)
            fcmin_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            beta_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            smooth_points[self.nose_idx] = self.filter_nose(
                new_points[self.nose_idx],
                self.prev_points[self.nose_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        else:
            # filter
            smooth_points[self.nose_idx] = self.filter_nose(
                new_points[self.nose_idx],
                self.prev_points[self.nose_idx],
                fcmin=0.7,
                beta=0.7,
            )
        # smooth eyebrow
        if np.mean(dis[self.eyebrow_idx]) < 0.003:
            # stable
            ratio_tmp = np.mean(dis[self.eyebrow_idx]) / 0.003
            fcmin_tmp = 0.02 * ratio_tmp
            beta_tmp = 0.02 * ratio_tmp
            smooth_points[self.eyebrow_idx] = self.filter_eyebrow(
                new_points[self.eyebrow_idx],
                self.prev_points[self.eyebrow_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        elif np.mean(dis[self.eyebrow_idx]) < 0.02:
            # filter
            ratio_tmp = (np.mean(dis[self.eyebrow_idx]) - 0.003) / (0.02 - 0.003)
            fcmin_tmp = 0.02 + (0.5 - 0.02) * ratio_tmp
            beta_tmp = 0.02 + (0.5 - 0.02) * ratio_tmp
            smooth_points[self.eyebrow_idx] = self.filter_eyebrow(
                new_points[self.eyebrow_idx],
                self.prev_points[self.eyebrow_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        else:
            # filter
            smooth_points[self.eyebrow_idx] = self.filter_eyebrow(
                new_points[self.eyebrow_idx],
                self.prev_points[self.eyebrow_idx],
                fcmin=0.5,
                beta=0.5,
            )
        # smooth eye
        if np.mean(dis[self.left_eye_idx]) < 0.003:
            # stable
            ratio_tmp = np.mean(dis[self.left_eye_idx]) / 0.003
            fcmin_tmp = 0.03 * ratio_tmp
            beta_tmp = 0.03 * ratio_tmp
            smooth_points[self.left_eye_idx] = self.filter_left_eye(
                new_points[self.left_eye_idx],
                self.prev_points[self.left_eye_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        elif np.mean(dis[self.left_eye_idx]) < 0.02:
            # filter
            ratio_tmp = (np.mean(dis[self.left_eye_idx]) - 0.003) / (0.02 - 0.003)
            fcmin_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            beta_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            smooth_points[self.left_eye_idx] = self.filter_left_eye(
                new_points[self.left_eye_idx],
                self.prev_points[self.left_eye_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        else:
            # fast
            smooth_points[self.left_eye_idx] = self.filter_left_eye(
                new_points[self.left_eye_idx],
                self.prev_points[self.left_eye_idx],
                fcmin=0.7,
                beta=0.7,
            )
        if np.mean(dis[self.right_eye_idx]) < 0.003:
            # stable
            ratio_tmp = np.mean(dis[self.right_eye_idx]) / 0.003
            fcmin_tmp = 0.03 * ratio_tmp
            beta_tmp = 0.03 * ratio_tmp
            smooth_points[self.right_eye_idx] = self.filter_right_eye(
                new_points[self.right_eye_idx],
                self.prev_points[self.right_eye_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        elif np.mean(dis[self.right_eye_idx]) < 0.02:
            # filter
            ratio_tmp = (np.mean(dis[self.right_eye_idx]) - 0.003) / (0.02 - 0.003)
            fcmin_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            beta_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            smooth_points[self.right_eye_idx] = self.filter_right_eye(
                new_points[self.right_eye_idx],
                self.prev_points[self.right_eye_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        else:
            # fast
            smooth_points[self.right_eye_idx] = self.filter_right_eye(
                new_points[self.right_eye_idx],
                self.prev_points[self.right_eye_idx],
                fcmin=0.7,
                beta=0.7,
            )

        # smooth mouth
        if np.mean(dis[self.mouth_idx]) < 0.003:
            # stable
            ratio_tmp = np.mean(dis[self.mouth_idx]) / 0.003
            fcmin_tmp = 0.05 * ratio_tmp
            beta_tmp = 0.05 * ratio_tmp
            smooth_points[self.mouth_idx] = self.filter_mouth(
                new_points[self.mouth_idx],
                self.prev_points[self.mouth_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        elif np.mean(dis[self.mouth_idx]) < 0.02:
            # filter
            ratio_tmp = (np.mean(dis[self.mouth_idx]) - 0.003) / (0.02 - 0.003)
            fcmin_tmp = 0.05 + (0.7 - 0.05) * ratio_tmp
            beta_tmp = 0.05 + (0.7 - 0.05) * ratio_tmp
            smooth_points[self.mouth_idx] = self.filter_mouth(
                new_points[self.mouth_idx],
                self.prev_points[self.mouth_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        else:
            # fast
            smooth_points[self.mouth_idx] = self.filter_mouth(
                new_points[self.mouth_idx],
                self.prev_points[self.mouth_idx],
                fcmin=0.7,
                beta=0.7,
            )

        # smooth pupil
        if np.mean(dis[self.left_pupil_idx]) < 0.003:
            # stable
            ratio_tmp = np.mean(dis[self.left_pupil_idx]) / 0.003
            fcmin_tmp = 0.03 * ratio_tmp
            beta_tmp = 0.03 * ratio_tmp
            smooth_points[self.left_pupil_idx] = self.filter_left_pupil(
                new_points[self.left_pupil_idx],
                self.prev_points[self.left_pupil_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        elif np.mean(dis[self.left_pupil_idx]) < 0.02:
            # filter
            ratio_tmp = (np.mean(dis[self.left_pupil_idx]) - 0.003) / (0.02 - 0.003)
            fcmin_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            beta_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            smooth_points[self.left_pupil_idx] = self.filter_left_pupil(
                new_points[self.left_pupil_idx],
                self.prev_points[self.left_pupil_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        else:
            # fast
            smooth_points[self.left_pupil_idx] = self.filter_left_pupil(
                new_points[self.left_pupil_idx],
                self.prev_points[self.left_pupil_idx],
                fcmin=0.7,
                beta=0.7,
            )
        if np.mean(dis[self.right_pupil_idx]) < 0.003:
            # stable
            ratio_tmp = np.mean(dis[self.right_pupil_idx]) / 0.003
            fcmin_tmp = 0.03 * ratio_tmp
            beta_tmp = 0.03 * ratio_tmp
            smooth_points[self.right_pupil_idx] = self.filter_right_pupil(
                new_points[self.right_pupil_idx],
                self.prev_points[self.right_pupil_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        elif np.mean(dis[self.right_pupil_idx]) < 0.02:
            # filter
            ratio_tmp = (np.mean(dis[self.right_pupil_idx]) - 0.003) / (0.02 - 0.003)
            fcmin_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            beta_tmp = 0.03 + (0.7 - 0.03) * ratio_tmp
            smooth_points[self.right_pupil_idx] = self.filter_right_pupil(
                new_points[self.right_pupil_idx],
                self.prev_points[self.right_pupil_idx],
                fcmin=fcmin_tmp,
                beta=beta_tmp,
            )
        else:
            # fast
            smooth_points[self.right_pupil_idx] = self.filter_right_pupil(
                new_points[self.right_pupil_idx],
                self.prev_points[self.right_pupil_idx],
                fcmin=0.7,
                beta=0.7,
            )

        # update pre points
        self.prev_points = smooth_points
        return smooth_points


class CameraDemo(object):
    def __init__(self, face_alignment_module, reset=False):
        self.face_alignment_module = face_alignment_module
        self.face_prob_th = 0.0001
        self.min_face = 96
        self.face_image_size = self.face_alignment_module.face_image_size
        self.trackingFaces = []
        self.reset = reset

    def reset_track(self):
        self.trackingFaces = []

    def forward(self, src_image, reset=False, pre_rect=None):

        if self.reset or reset:
            self.trackingFaces = []

        if len(self.trackingFaces) == 0:
            if pre_rect is not None:
                detected_faces = [pre_rect]
            else:
                detected_faces, _, _ = self.face_alignment_module.face_detector.detect(
                    src_image
                )
            for face_rect in detected_faces:
                new_tracking_object = {
                    "face_rect": face_rect,
                    "rotate_angle": 0.0,
                    "pre_kpt_222": None,
                    "face_dis": np.sqrt(
                        np.square((face_rect[2] - face_rect[0]))
                        + np.square((face_rect[3] - face_rect[1]))
                    ),
                    "smoother_222": Smoother222(),
                    "prob": 0,
                }
                self.trackingFaces.append(new_tracking_object)
        else:
            detected_faces, _, _ = self.face_alignment_module.face_detector.detect(
                src_image
            )
            for face_rect in detected_faces:
                new_tracking_object = {
                    "face_rect": face_rect,
                    "rotate_angle": 0.0,
                    "pre_kpt_222": None,
                    "face_dis": np.sqrt(
                        np.square((face_rect[2] - face_rect[0]))
                        + np.square((face_rect[3] - face_rect[1]))
                    ),
                    "smoother_222": Smoother222(),
                    "prob": 0,
                }
                self.trackingFaces.append(new_tracking_object)

        delete_idx_list = []
        for face_idx, tracking_face in enumerate(self.trackingFaces):
            if tracking_face["pre_kpt_222"] is not None:
                result_dict = self.face_alignment_module.forward(
                    src_image, pre_pts=tracking_face["pre_kpt_222"], iterations=3
                )
            else:
                result_dict = self.face_alignment_module.forward(
                    src_image, face_box=tracking_face["face_rect"], iterations=3
                )

            if result_dict["prob"] < self.face_prob_th:
                if not face_idx in delete_idx_list:
                    delete_idx_list.append(face_idx)
                continue

            landmarks_final = tracking_face["smoother_222"].smooth(
                result_dict["pt222"], tracking_face["face_dis"]
            )
            tracking_face["pre_kpt_222"] = landmarks_final

            left_eye_corner = landmarks_final[74]
            right_eye_corner = landmarks_final[96]

            radian = np.arctan2(
                right_eye_corner[1] - left_eye_corner[1],
                right_eye_corner[0] - left_eye_corner[0] + 0.00000001,
            )
            rotate_angle = np.rad2deg(radian)
            face_x_min, face_x_max = np.min(landmarks_final[:, 0]), np.max(
                landmarks_final[:, 0]
            )
            face_y_min, face_y_max = np.min(landmarks_final[:, 1]), np.max(
                landmarks_final[:, 1]
            )
            face_bbox = [face_x_min, face_y_min, face_x_max, face_y_max]
            face_dis = np.linalg.norm(landmarks_final[0] - landmarks_final[32])

            if (
                face_x_max - face_x_min < self.min_face
                or face_y_max - face_y_min < self.min_face
            ):
                if not face_idx in delete_idx_list:
                    delete_idx_list.append(face_idx)

            euler_pred = result_dict["euler_rad"]
            pitch = np.rad2deg(euler_pred[0])
            yaw = np.rad2deg(euler_pred[1])
            roll = np.rad2deg(euler_pred[2])
            # print("pitch, yaw, roll", pitch, yaw, roll)

            # one filter model
            max_euler = abs(pitch) + (abs(yaw) * 0.6)
            face_dis *= 1.0 + max_euler / 18.0

            # two filter model
            tracking_face["face_rect"] = face_bbox
            tracking_face["rotate_angle"] = rotate_angle
            tracking_face["face_dis"] = face_dis
            tracking_face["prob"] = result_dict["prob"]
            tracking_face["pitch"] = pitch
            tracking_face["yaw"] = yaw
            tracking_face["roll"] = roll
            tracking_face["euler_rad"] = result_dict["euler_rad"]

        if len(self.trackingFaces) > 1:
            for face_idx, tracking_face_target in enumerate(self.trackingFaces):
                if face_idx in delete_idx_list:
                    continue
                for idx, tracking_face in enumerate(self.trackingFaces):
                    if idx in delete_idx_list:
                        continue
                    if face_idx == idx:
                        continue
                    iou_temp = self.count_iou(
                        tracking_face_target["face_rect"], tracking_face["face_rect"]
                    )
                    # prog 2
                    if iou_temp > 0.12:
                        if (
                            self.area(tracking_face_target["face_rect"])
                            - self.area(tracking_face["face_rect"])
                            < 0
                        ):
                            if not face_idx in delete_idx_list:
                                delete_idx_list.append(face_idx)
                        else:
                            if not idx in delete_idx_list:
                                delete_idx_list.append(idx)

        idx_offset = 0
        for delete_idx in sorted(delete_idx_list):
            self.trackingFaces.pop(delete_idx - idx_offset)
            idx_offset += 1

        return self.trackingFaces

    def count_iou(self, boxA, boxB):
        # determine the (x, y)-coordinates of the intersection rectangle
        xA = max(boxA[0], boxB[0])
        yA = max(boxA[1], boxB[1])
        xB = min(boxA[2], boxB[2])
        yB = min(boxA[3], boxB[3])

        # compute the area of intersection rectangle
        interArea = abs(max((xB - xA, 0)) * max((yB - yA), 0))
        if interArea == 0:
            return 0
        # compute the area of both the prediction and ground-truth
        # rectangles
        boxAArea = abs((boxA[2] - boxA[0]) * (boxA[3] - boxA[1]))
        boxBArea = abs((boxB[2] - boxB[0]) * (boxB[3] - boxB[1]))

        # compute the intersection over union by taking the intersection
        # area and dividing it by the sum of prediction + ground-truth
        # areas - the interesection area
        iou = interArea / float(boxAArea + boxBArea - interArea)

        # return the intersection over union value
        return iou

    def area(self, bbox):
        w = bbox[3] - bbox[1]
        h = bbox[2] - bbox[0]
        return w * h