1.1.0 支持 多人脸

This commit is contained in:
shadowcz007
2024-07-06 17:14:50 +08:00
parent 2b33e8cc8c
commit 95b64677d1
10 changed files with 1269 additions and 23 deletions
+43 -3
View File
@@ -1,16 +1,56 @@
[LivePortrait](https://github.com/KwaiVGI/LivePortrait)的Comfyui版本。
!! 支持多人脸
> [寻求帮助 Mixlab nodes discord](https://discord.gg/cXs9vZSqeK)
> [推荐:mixlab-nodes](https://github.com/shadowcz007/comfyui-mixlab-nodes)
### workflow
> 配合 [comfyui-mixlab-nodes](https://github.com/shadowcz007/comfyui-mixlab-nodes) 使用
[![alt text](example/1720078701040.png)](example/live_workflow.json)
> 全家福
[![alt text](example/1720256574305.png)](example/全家福模式-workflow.json)
[全家福 Workflow JSON](example/全家福模式-workflow.json)
[![alt text](example/1720256807930.png)](example/live_workflow.json)
[Workflow JSON](example/live_workflow.json)
[AppInfo workflow JSON](example/appinfo-workflow.json)
## models
### Nodes
##### Live Portrait,用于生成动态肖像。
输入参数:
source_image:输入一张静态图像,作为生成动态肖像的基础。
driving_video:输入一个驱动视频,决定动态肖像的动作和表情变化。
crop_info:输入人脸裁剪信息,用于指定人脸的裁剪区域。
输出:
video:输出生成的动态肖像视频。
video_concat:输出与驱动视频合成后的动态肖像视频(如果有的话)。
##### Face Crop Info,用于提取图像中的人脸裁剪信息。
可调参数:
face_sorting_direction:设置人脸排序方向,可选值为 "left-right"(从左到右)或 "large-small"(从大到小)。用于在图像中包含多张人脸时,确定哪张人脸优先处理。
face_index:指定要处理的人脸索引,默认值为 -1,表示处理所有检测到的人脸。可以设置为特定的索引值以选择特定的人脸。
debug:开启或关闭调试模式。设置为 true 时,会输出调试图像以便查看人脸检测和裁剪区域;设置为 false 时,不输出调试图像。
### models
[google drive](https://drive.google.com/drive/folders/1UtKgzKjFAOmZkhNK-OYT0caJ_w2XAnib)
@@ -22,7 +62,7 @@
``` Comfyui/models/insightface ```
## python环境依赖
### python环境依赖
[打包好的环境](https://pan.baidu.com/s/1oQCvtWR2seFloZDGsNGNTQ?pwd=MAI0)
+4 -2
View File
@@ -1,14 +1,16 @@
from .nodes.live_portrait import LivePortraitNode
from .nodes.live_portrait import LivePortraitNode,FaceCropInfo
NODE_CLASS_MAPPINGS = {
"LivePortraitNode": LivePortraitNode,
"FaceCropInfo":FaceCropInfo
}
# dict = { "key":value }
NODE_DISPLAY_NAME_MAPPINGS = {
"LivePortraitNode":"LivePortrait"
"LivePortraitNode":"Live Portrait",
"FaceCropInfo":"Face Crop Info"
}
# web ui的节点功能
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 425 KiB

File diff suppressed because one or more lines are too long
@@ -29,6 +29,14 @@ from .live_portrait_wrapper import LivePortraitWrapper
import comfy.utils
def add_index_to_filename(output_path, index):
directory, filename = osp.split(output_path)
basename, ext = osp.splitext(filename)
new_filename = f"{basename}_{index}{ext}"
new_output_path = osp.join(directory, new_filename)
return new_output_path
def make_abs_path(fn):
return osp.join(osp.dirname(osp.realpath(__file__)), fn)
@@ -44,9 +52,14 @@ class LivePortraitPipeline(object):
######## process reference portrait ########
# img_rgb = load_image_rgb(args.source_image)
img_rgb=args.source_image
# 增加人脸好的
crop_info=args.crop_info
img_rgb = resize_to_limit(img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
# log(f"Load source image from {args.source_image}")
crop_info = self.cropper.crop_single_image(img_rgb)
# todo 人脸检测并裁切 - 独立一个节点
crop_info = self.cropper.crop_single_image(img_rgb,src_face=crop_info)
source_lmk = crop_info['lmk_crop']
img_crop, img_crop_256x256 = crop_info['img_crop'], crop_info['img_crop_256x256']
if inference_cfg.flag_do_crop:
@@ -206,3 +219,194 @@ class LivePortraitPipeline(object):
images2video(I_p_lst, wfp=wfp, fps=video_fps)
return wfp, wfp_concat
def executeForAll(self, args):
inference_cfg = self.live_portrait_wrapper.cfg # for convenience
######## process reference portrait ########
# img_rgb = load_image_rgb(args.source_image)
img_rgb = args.source_image
# 增加人脸好的
crop_info_list = args.crop_info
img_rgb = resize_to_limit(img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
# log(f"Load source image from {args.source_image}")
# todo 人脸检测并裁切 - 独立一个节点
crop_info_list = [self.cropper.crop_single_image(img_rgb, src_face=crop_info) for crop_info in crop_info_list]
video_fps = cv2.VideoCapture(args.driving_info).get(cv2.CAP_PROP_FPS)
######## process driving info ########
self.driving_lmk_lst=None
self.n_frames=None
if is_video(args.driving_info):
log(f"Load from video file (mp4 mov avi etc...): {args.driving_info}")
# TODO: 这里track一下驱动视频 -> 构建模板
driving_rgb_lst = load_driving_info(args.driving_info)
driving_rgb_lst_256 = [cv2.resize(_, (256, 256)) for _ in driving_rgb_lst]
I_d_lst = self.live_portrait_wrapper.prepare_driving_videos(driving_rgb_lst_256)
self.n_frames = I_d_lst.shape[0]
if inference_cfg.flag_eye_retargeting or inference_cfg.flag_lip_retargeting:
self.driving_lmk_lst = self.cropper.get_retargeting_lmk_info(driving_rgb_lst)
# input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)
# elif is_template(args.driving_info):
# log(f"Load from video templates {args.driving_info}")
# with open(args.driving_info, 'rb') as f:
# template_lst, driving_lmk_lst = pickle.load(f)
# n_frames = template_lst[0]['n_frames']
# # input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)
# else:
# raise Exception("Unsupported driving types!")
#########################################
print('#driving_lmk_lst',self.driving_lmk_lst)
# 原图片---视频帧
img_rgbs=[img_rgb for i in range(self.n_frames)]
for index in range(len(crop_info_list)):
crop_info=crop_info_list[index]
# 地一张脸
source_lmk = crop_info['lmk_crop']
img_crop, img_crop_256x256 = crop_info['img_crop'], crop_info['img_crop_256x256']
if inference_cfg.flag_do_crop:
I_s = self.live_portrait_wrapper.prepare_source(img_crop_256x256)
x_s_info = self.live_portrait_wrapper.get_kp_info(I_s)
x_c_s = x_s_info['kp']
R_s = get_rotation_matrix(x_s_info['pitch'], x_s_info['yaw'], x_s_info['roll'])
f_s = self.live_portrait_wrapper.extract_feature_3d(I_s)
x_s = self.live_portrait_wrapper.transform_keypoint(x_s_info)
if inference_cfg.flag_lip_zero:
# let lip-open scalar to be 0 at first
c_d_lip_before_animation = [0.]
combined_lip_ratio_tensor_before_animation = self.live_portrait_wrapper.calc_combined_lip_ratio(c_d_lip_before_animation, source_lmk)
if combined_lip_ratio_tensor_before_animation[0][0] < inference_cfg.lip_zero_threshold:
inference_cfg.flag_lip_zero = False
else:
lip_delta_before_animation = self.live_portrait_wrapper.retarget_lip(x_s, combined_lip_ratio_tensor_before_animation)
############################################
if self.driving_lmk_lst!=None:
input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, self.driving_lmk_lst)
######## prepare for pasteback ########
if inference_cfg.flag_pasteback:
if inference_cfg.mask_crop is None:
inference_cfg.mask_crop = cv2.imread(make_abs_path('./utils/resources/mask_template.png'), cv2.IMREAD_COLOR)
mask_ori = _transform_img(inference_cfg.mask_crop, crop_info['M_c2o'], dsize=(img_rgb.shape[1], img_rgb.shape[0]))
mask_ori = mask_ori.astype(np.float32) / 255.
I_p_paste_lst = []
#########################################
I_p_lst = []
R_d_0, x_d_0_info = None, None
pbar = comfy.utils.ProgressBar(self.n_frames)
for i in track(range(self.n_frames), description='Animating...', total=self.n_frames):
if is_video(args.driving_info):
# extract kp info by M
I_d_i = I_d_lst[i]
x_d_i_info = self.live_portrait_wrapper.get_kp_info(I_d_i)
R_d_i = get_rotation_matrix(x_d_i_info['pitch'], x_d_i_info['yaw'], x_d_i_info['roll'])
if i == 0:
R_d_0 = R_d_i
x_d_0_info = x_d_i_info
if inference_cfg.flag_relative:
R_new = (R_d_i @ R_d_0.permute(0, 2, 1)) @ R_s
delta_new = x_s_info['exp'] + (x_d_i_info['exp'] - x_d_0_info['exp'])
scale_new = x_s_info['scale'] * (x_d_i_info['scale'] / x_d_0_info['scale'])
t_new = x_s_info['t'] + (x_d_i_info['t'] - x_d_0_info['t'])
else:
R_new = R_d_i
delta_new = x_d_i_info['exp']
scale_new = x_s_info['scale']
t_new = x_d_i_info['t']
t_new[..., 2].fill_(0) # zero tz
x_d_i_new = scale_new * (x_c_s @ R_new + delta_new) + t_new
# Algorithm 1:
if not inference_cfg.flag_stitching and not inference_cfg.flag_eye_retargeting and not inference_cfg.flag_lip_retargeting:
# without stitching or retargeting
if inference_cfg.flag_lip_zero:
x_d_i_new += lip_delta_before_animation.reshape(-1, x_s.shape[1], 3)
else:
pass
elif inference_cfg.flag_stitching and not inference_cfg.flag_eye_retargeting and not inference_cfg.flag_lip_retargeting:
# with stitching and without retargeting
if inference_cfg.flag_lip_zero:
x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new) + lip_delta_before_animation.reshape(-1, x_s.shape[1], 3)
else:
x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new)
else:
eyes_delta, lip_delta = None, None
if inference_cfg.flag_eye_retargeting:
c_d_eyes_i = input_eye_ratio_lst[i]
combined_eye_ratio_tensor = self.live_portrait_wrapper.calc_combined_eye_ratio(c_d_eyes_i, source_lmk)
# ∆_eyes,i = R_eyes(x_s; c_s,eyes, c_d,eyes,i)
eyes_delta = self.live_portrait_wrapper.retarget_eye(x_s, combined_eye_ratio_tensor)
if inference_cfg.flag_lip_retargeting:
c_d_lip_i = input_lip_ratio_lst[i]
combined_lip_ratio_tensor = self.live_portrait_wrapper.calc_combined_lip_ratio(c_d_lip_i, source_lmk)
# ∆_lip,i = R_lip(x_s; c_s,lip, c_d,lip,i)
lip_delta = self.live_portrait_wrapper.retarget_lip(x_s, combined_lip_ratio_tensor)
if inference_cfg.flag_relative: # use x_s
x_d_i_new = x_s + \
(eyes_delta.reshape(-1, x_s.shape[1], 3) if eyes_delta is not None else 0) + \
(lip_delta.reshape(-1, x_s.shape[1], 3) if lip_delta is not None else 0)
else: # use x_d,i
x_d_i_new = x_d_i_new + \
(eyes_delta.reshape(-1, x_s.shape[1], 3) if eyes_delta is not None else 0) + \
(lip_delta.reshape(-1, x_s.shape[1], 3) if lip_delta is not None else 0)
if inference_cfg.flag_stitching:
x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new)
out = self.live_portrait_wrapper.warp_decode(f_s, x_s, x_d_i_new)
I_p_i = self.live_portrait_wrapper.parse_output(out['out'])[0]
I_p_lst.append(I_p_i)
if inference_cfg.flag_pasteback:
# img_rgbs=[img_rgb,img_rgb,img_rgb] 新构建的视频帧
img_rgb0=img_rgbs[i]
I_p_i_to_ori = _transform_img(I_p_i, crop_info['M_c2o'], dsize=(img_rgb0.shape[1], img_rgb0.shape[0]))
I_p_i_to_ori_blend = np.clip(mask_ori * I_p_i_to_ori + (1 - mask_ori) * img_rgb0, 0, 255).astype(np.uint8)
out = np.hstack([I_p_i_to_ori, I_p_i_to_ori_blend])
I_p_paste_lst.append(I_p_i_to_ori_blend)
# 更新到img_rgbs
img_rgbs[i]=np.copy(I_p_i_to_ori_blend)
pbar.update(1)
directory, filename = os.path.split(args.output_path)
if not os.path.exists(directory):
mkdir(directory)
# wfp_concat = args.output_path_concat
# if is_video(args.driving_info):
# frames_concatenated = concat_frames(I_p_lst, driving_rgb_lst, img_crop_256x256)
# # save (driving frames, source image, drived frames) result
# # wfp_concat = osp.join(directory, f'{basename(args.source_image)}--{basename(args.driving_info)}_concat.mp4')
# # images2video(frames_concatenated, wfp=wfp_concat)
# images2video(frames_concatenated, wfp=wfp_concat, fps=video_fps)
# save drived result
wfp = args.output_path
if inference_cfg.flag_pasteback:
images2video(I_p_paste_lst, wfp=wfp, fps=video_fps)
return wfp
+64 -4
View File
@@ -1,6 +1,7 @@
# coding: utf-8
import numpy as np
from PIL import Image
import os.path as osp
from typing import List, Union, Tuple
from dataclasses import dataclass, field
@@ -61,28 +62,87 @@ class Cropper(object):
if hasattr(self.crop_cfg, k):
setattr(self.crop_cfg, k, v)
def crop_single_image(self, obj, **kwargs):
# 计算多张人脸结果
def crop_all_image(self, obj, **kwargs):
direction = kwargs.get('direction', 'large-small')
# face_index = kwargs.get('face_index', 0)
# 是否需要调试图片
is_debug = kwargs.get('debug', False)
# print('#crop_single_image',direction,face_index)
# crop and align a single image
if isinstance(obj, str):
img_rgb = load_image_rgb(obj)
elif isinstance(obj, np.ndarray):
img_rgb = obj
src_face = self.face_analysis_wrapper.get(
src_faces = self.face_analysis_wrapper.get(
img_rgb,
flag_do_landmark_2d_106=True,
direction=direction
)
img_with_box=None
if is_debug:
img_with_box = img_rgb.copy()
for index in range(len(src_faces)):
src_face=src_faces[index]
# 获取人脸框坐标
box = src_face['bbox'] # 假设 src_face 有一个 'box' 键,包含了人脸框坐标
# 绘制人脸框
x, y, w, h = map(int, box)
cv2.rectangle(img_with_box, (x, y), (w, h), (0, 255, 0), 2)
# 在图像上绘制索引
cv2.putText(
img_with_box,
str(index),
(x+10, y + 10), # 索引显示在框的xia方
cv2.FONT_HERSHEY_SIMPLEX,
0.9, # 字体大小
(0, 255, 0), # 绿色文本
4 # 文本线宽
)
img_with_box = Image.fromarray(img_with_box)
return (src_faces,img_with_box)
def crop_single_image(self, obj, **kwargs):
direction = kwargs.get('direction', 'large-small')
face_index = kwargs.get('face_index', 0)
src_face = kwargs.get('src_face', None)
# crop and align a single image
if isinstance(obj, str):
img_rgb = load_image_rgb(obj)
elif isinstance(obj, np.ndarray):
img_rgb = obj
print('#crop_single_image',direction,face_index,src_face)
if src_face==None:
src_face = self.face_analysis_wrapper.get(
img_rgb,
flag_do_landmark_2d_106=True,
direction=direction
)
if len(src_face) == 0:
log('No face detected in the source image.')
raise Exception("No face detected in the source image!")
elif len(src_face) > 1:
log(f'More than one face detected in the image, only pick one face by rule {direction}.')
src_face = src_face[0]
# 如果人脸数量超过face_index,则取最大值
if len(src_face)<=face_index:
face_index=len(src_face)-1
src_face = src_face[face_index]
pts = src_face.landmark_2d_106
# crop the face
@@ -14,7 +14,6 @@ from .timer import Timer
def sort_by_direction(faces, direction: str = 'large-small', face_center=None):
if len(faces) <= 0:
return faces
if direction == 'left-right':
return sorted(faces, key=lambda face: face['bbox'][0])
if direction == 'right-left':
+91 -10
View File
@@ -2,7 +2,7 @@ import os
import sys
import folder_paths
import numpy as np
# import torch
import torch
from PIL import Image
import folder_paths
# import comfy.utils
@@ -23,6 +23,11 @@ sys.path.append(current_directory)
def tensor2pil(image):
return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
# Convert PIL to Tensor
def pil2tensor(image):
return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0)
from .LivePortrait.src.live_portrait_pipeline import LivePortraitPipeline
@@ -39,7 +44,9 @@ class ArgumentConfig:
driving_info,
output_path='animations/v.mp4',
output_path_concat="",
device_id=0,
device_id=0,
crop_info =None,
face_index=0,
flag_lip_zero=True,
flag_eye_retargeting=False,
flag_lip_retargeting=False,
@@ -59,6 +66,8 @@ class ArgumentConfig:
self.driving_info = driving_info
self.output_path = output_path
self.output_path_concat=output_path_concat
self.crop_info=crop_info
self.face_index=face_index
self.device_id = device_id
self.flag_lip_zero = flag_lip_zero
self.flag_eye_retargeting = flag_eye_retargeting
@@ -161,6 +170,65 @@ inference_cfg = InferenceConfig(
crop_cfg = CropConfig()
# 人脸检测并裁切
class FaceCropInfo:
def __init__(self):
self.speaker = None
@classmethod
def INPUT_TYPES(s):
return {"required": {
"source_image": ("IMAGE",),
},
"optional":{
"face_sorting_direction":(["left-right","large-small"], {"default": "left-right"}),
"face_index":("INT", {"default": 0, "min": -1,"max":200, "step": 1, "display": "number"}),
"debug":("BOOLEAN", {"default": False},),
}
}
RETURN_TYPES = ("CROP_INFO","IMAGE",)
RETURN_NAMES = ("crop_info","debug_image",)
FUNCTION = "run"
OUTPUT_NODE = True
CATEGORY = "♾️Mixlab/Video"
INPUT_IS_LIST = False
OUTPUT_IS_LIST = (True,False,) #list 列表 [1,2,3]
def run(self,source_image,face_sorting_direction="left-right",face_index=0,debug=False):
pil_image=tensor2pil(source_image)
# Convert PIL image to NumPy array
opencv_image = np.array(pil_image)
# print('##---------------------------------#landmark_runner_ckpt',landmark_runner_ckpt)
live_portrait_pipeline = LivePortraitPipeline(
inference_cfg=inference_cfg,
crop_cfg=crop_cfg,
landmark_runner_ckpt=landmark_runner_ckpt,
insightface_pretrained_weights=insightface_pretrained_weights
)
crop_info,debug_image = live_portrait_pipeline.cropper.crop_all_image(
opencv_image,
direction=face_sorting_direction,
debug=debug
)
debug_image=pil2tensor(debug_image)
if face_index>-1:
#只输出一张 [face]
crop_info=[crop_info[face_index]]
return (crop_info,debug_image,)
class LivePortraitNode:
def __init__(self):
self.speaker = None
@@ -171,9 +239,9 @@ class LivePortraitNode:
"source_image": ("IMAGE",),
"driving_video":("SCENE_VIDEO",),
},
# "optional":{
# "skip_refine_text":("BOOLEAN", {"default": False},),
# }
"optional":{
"crop_info":("CROP_INFO", ),
}
}
RETURN_TYPES = ("SCENE_VIDEO","SCENE_VIDEO",)
@@ -183,12 +251,20 @@ class LivePortraitNode:
CATEGORY = "♾️Mixlab/Video"
INPUT_IS_LIST = False
INPUT_IS_LIST = True
OUTPUT_IS_LIST = (False,False,) #list 列表 [1,2,3]
def run(self,source_image,driving_video):
def run(self,source_image,driving_video,crop_info=None):
# print('#crop_info',crop_info,isinstance(crop_info, list))
if crop_info!=None and isinstance(crop_info, list)==False:
crop_info=[crop_info]
if crop_info!=None:
crop_info=[ [c] for c in crop_info]
pil_image=tensor2pil(source_image)
driving_video=driving_video[0]
pil_image=tensor2pil(source_image[0])
# Convert PIL image to NumPy array
opencv_image = np.array(pil_image)
@@ -214,7 +290,8 @@ class LivePortraitNode:
source_image=opencv_image,
driving_info=driving_video,
output_path=v_path,
output_path_concat=output_path_concat
output_path_concat=output_path_concat,
crop_info=crop_info,
)
# print('##---------------------------------#landmark_runner_ckpt',landmark_runner_ckpt)
@@ -226,7 +303,11 @@ class LivePortraitNode:
)
# run
live_portrait_pipeline.execute(args)
if crop_info==None:
live_portrait_pipeline.execute(args)
else:
print('#executeForAll',len(crop_info))
live_portrait_pipeline.executeForAll(args)
live_portrait_pipeline.live_portrait_wrapper=None
+2 -2
View File
@@ -1,7 +1,7 @@
[project]
name = "comfyui-liveportrait"
description = "The ComfyUI version of [a/LivePortrait](https://github.com/KwaiVGI/LivePortrait)."
version = "1.0.1"
version = "1.1.0"
license = "LICENSE"
dependencies = ["numpy>=1.26.4", "opencv-python-headless", "imageio>=2.34.2", "lmdb>=1.4.1", "timm>=1.0.7", "rich>=13.7.1", "ffmpeg>=1.4", "onnxruntime-gpu>=1.18.0", "onnx>=1.16.1", "scikit-image>=0.24.0", "albumentations>=1.4.10", "matplotlib>=3.9.0", "imageio-ffmpeg>=0.5.1"]
@@ -10,6 +10,6 @@ Repository = "https://github.com/shadowcz007/comfyui-liveportrait"
# Used by Comfy Registry https://comfyregistry.org
[tool.comfy]
PublisherId = "shadowcz"
PublisherId = ""
DisplayName = "comfyui-liveportrait"
Icon = ""