Add files via upload
This commit is contained in:
VangengLab
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,65 @@
|
||||
# coding: utf-8
|
||||
|
||||
"""
|
||||
All configs for user
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
import tyro
|
||||
from typing_extensions import Annotated
|
||||
from typing import Optional, Literal
|
||||
from .base_config import PrintableConfig, make_abs_path
|
||||
import cv2
|
||||
from numpy import ndarray
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
|
||||
@dataclass(repr=False) # use repr from PrintableConfig
|
||||
class ArgumentConfig(PrintableConfig):
|
||||
########## input arguments ##########
|
||||
source: Annotated[str, tyro.conf.arg(aliases=["-s"])] = make_abs_path('../../assets/examples/source/s12.jpg') # path to the source portrait (human/animal) or video (human)
|
||||
driving: Annotated[str, tyro.conf.arg(aliases=["-d"])] = "custom_nodes/ComfyUI-LivePortrait_v2/assets/examples/driving/d9.mp4" # path to driving video or template (.pkl format)
|
||||
#autodl-tmp/ComfyUI/custom_nodes/ComfyUI-LivePortrait_v2/assets/examples/driving/d9.mp4
|
||||
output_dir: Annotated[str, tyro.conf.arg(aliases=["-o"])] = "../../animations/" # directory to save output video
|
||||
#mask_crop: ndarray = field(default_factory=lambda: cv2.imread(make_abs_path('../utils/resources/mask_template.png'), cv2.IMREAD_COLOR))
|
||||
|
||||
########## inference arguments ##########
|
||||
flag_use_half_precision: bool = True # whether to use half precision (FP16). If black boxes appear, it might be due to GPU incompatibility; set to False.
|
||||
flag_crop_driving_video: bool = False # whether to crop the driving video, if the given driving info is a video
|
||||
device_id: int = 0 # gpu device id
|
||||
flag_force_cpu: bool = False # force cpu inference, WIP!
|
||||
flag_normalize_lip: bool = False # whether to let the lip to close state before animation, only take effect when flag_eye_retargeting and flag_lip_retargeting is False
|
||||
flag_source_video_eye_retargeting: bool = False # when the input is a source video, whether to let the eye-open scalar of each frame to be the same as the first source frame before the animation, only take effect when flag_eye_retargeting and flag_lip_retargeting is False, may cause the inter-frame jittering
|
||||
flag_eye_retargeting: bool = False # not recommend to be True, WIP; whether to transfer the eyes-open ratio of each driving frame to the source image or the corresponding source frame
|
||||
flag_lip_retargeting: bool = False # not recommend to be True, WIP; whether to transfer the lip-open ratio of each driving frame to the source image or the corresponding source frame
|
||||
flag_stitching: bool = True # recommend to True if head movement is small, False if head movement is large or the source image is an animal
|
||||
flag_relative_motion: bool = True # whether to use relative motion
|
||||
flag_pasteback: bool = True # whether to paste-back/stitch the animated face cropping from the face-cropping space to the original image space
|
||||
flag_do_crop: bool = True # whether to crop the source portrait or video to the face-cropping space
|
||||
driving_option: Literal["expression-friendly", "pose-friendly"] = "expression-friendly" # "expression-friendly" or "pose-friendly"; "expression-friendly" would adapt the driving motion with the global multiplier, and could be used when the source is a human image
|
||||
driving_multiplier: float = 1.0 # be used only when driving_option is "expression-friendly"
|
||||
driving_smooth_observation_variance: float = 3e-7 # smooth strength scalar for the animated video when the input is a source video, the larger the number, the smoother the animated video; too much smoothness would result in loss of motion accuracy
|
||||
audio_priority: Literal['source', 'driving'] = 'driving' # whether to use the audio from source or driving video
|
||||
animation_region: Literal["exp", "pose", "lip", "eyes", "all"] = "all" # the region where the animation was performed, "exp" means the expression, "pose" means the head pose, "all" means all regions
|
||||
########## source crop arguments ##########
|
||||
det_thresh: float = 0.15 # detection threshold
|
||||
scale: float = 2.3 # the ratio of face area is smaller if scale is larger
|
||||
vx_ratio: float = 0 # the ratio to move the face to left or right in cropping space
|
||||
vy_ratio: float = -0.125 # the ratio to move the face to up or down in cropping space
|
||||
flag_do_rot: bool = True # whether to conduct the rotation when flag_do_crop is True
|
||||
source_max_dim: int = 1280 # the max dim of height and width of source image or video, you can change it to a larger number, e.g., 1920
|
||||
source_division: int = 2 # make sure the height and width of source image or video can be divided by this number
|
||||
|
||||
########## driving crop arguments ##########
|
||||
scale_crop_driving_video: float = 2.2 # scale factor for cropping driving video
|
||||
vx_ratio_crop_driving_video: float = 0. # adjust y offset
|
||||
vy_ratio_crop_driving_video: float = -0.1 # adjust x offset
|
||||
########## gradio arguments ##########
|
||||
port: Annotated[int, tyro.conf.arg(aliases=["--port"])] = 6006 # port for gradio server
|
||||
listen: bool = True
|
||||
share: bool = False # whether to share the server to public
|
||||
server_name: Optional[str] = "127.0.0.1" # set the local server name, "0.0.0.0" to broadcast all
|
||||
flag_do_torch_compile: bool = False # whether to use torch.compile to accelerate generation
|
||||
gradio_temp_dir: Optional[str] = None # directory to save gradio temp files
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# coding: utf-8
|
||||
|
||||
"""
|
||||
pretty printing class
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import os.path as osp
|
||||
from typing import Tuple
|
||||
|
||||
|
||||
def make_abs_path(fn):
|
||||
return osp.join(osp.dirname(osp.realpath(__file__)), fn)
|
||||
|
||||
|
||||
class PrintableConfig: # pylint: disable=too-few-public-methods
|
||||
"""Printable Config defining str function"""
|
||||
|
||||
def __repr__(self):
|
||||
lines = [self.__class__.__name__ + ":"]
|
||||
for key, val in vars(self).items():
|
||||
if isinstance(val, Tuple):
|
||||
flattened_val = "["
|
||||
for item in val:
|
||||
flattened_val += str(item) + "\n"
|
||||
flattened_val = flattened_val.rstrip("\n")
|
||||
val = flattened_val + "]"
|
||||
lines += f"{key}: {str(val)}".split("\n")
|
||||
return "\n ".join(lines)
|
||||
@@ -0,0 +1,35 @@
|
||||
# coding: utf-8
|
||||
|
||||
"""
|
||||
parameters used for crop faces
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .base_config import PrintableConfig, make_abs_path
|
||||
|
||||
|
||||
@dataclass(repr=False) # use repr from PrintableConfig
|
||||
class CropConfig(PrintableConfig):
|
||||
insightface_root: str = make_abs_path("../../pretrained_weights/insightface")###############
|
||||
landmark_ckpt_path: str = make_abs_path("../../../../models/Liveportrait_v3/liveportrait/landmark.onnx")
|
||||
xpose_config_file_path: str = make_abs_path("../utils/dependencies/XPose/config_model/UniPose_SwinT.py")
|
||||
xpose_embedding_cache_path: str = make_abs_path('../utils/resources/clip_embedding')
|
||||
|
||||
xpose_ckpt_path: str = make_abs_path("../../../../models/Liveportrait_v3/liveportrait_animals/xpose.pth")#######################
|
||||
device_id: int = 0 # gpu device id
|
||||
flag_force_cpu: bool = False # force cpu inference, WIP
|
||||
det_thresh: float = 0.1 # detection threshold
|
||||
########## source image or video cropping option #
|
||||
dsize: int = 512 # crop size
|
||||
scale: float = 2.3 # scale factor
|
||||
vx_ratio: float = 0 # vx ratio
|
||||
vy_ratio: float = -0.125 # vy ratio +up, -down
|
||||
max_face_num: int = 0 # max face number, 0 mean no limit
|
||||
flag_do_rot: bool = True # whether to conduct the rotation when flag_do_crop is True
|
||||
animal_face_type: str = "animal_face_9" # animal_face_68 -> 68 landmark points, animal_face_9 -> 9 landmarks
|
||||
########## driving video auto cropping option ##########
|
||||
scale_crop_driving_video: float = 2.2 # 2.0 # scale factor for cropping driving video
|
||||
vx_ratio_crop_driving_video: float = 0.0 # adjust y offset
|
||||
vy_ratio_crop_driving_video: float = -0.1 # adjust x offset
|
||||
direction: str = "large-small" # direction of cropping
|
||||
@@ -0,0 +1,69 @@
|
||||
# coding: utf-8
|
||||
|
||||
"""
|
||||
config dataclass used for inference
|
||||
"""
|
||||
|
||||
import cv2
|
||||
from numpy import ndarray
|
||||
import pickle as pkl
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Literal, Tuple
|
||||
from .base_config import PrintableConfig, make_abs_path
|
||||
|
||||
def load_lip_array():
|
||||
with open(make_abs_path('../utils/resources/lip_array.pkl'), 'rb') as f:
|
||||
return pkl.load(f)
|
||||
|
||||
@dataclass(repr=False) # use repr from PrintableConfig
|
||||
class InferenceConfig(PrintableConfig):
|
||||
# HUMAN MODEL CONFIG, NOT EXPORTED PARAMS
|
||||
models_config: str = make_abs_path('./models.yaml') # portrait animation config
|
||||
checkpoint_F: str = make_abs_path('../../') # path to checkpoint of F
|
||||
checkpoint_M: str = make_abs_path('models/live/pretrained_weights/liveportrait/base_models/motion_extractor.pth') # path to checkpoint pf M
|
||||
checkpoint_G: str = make_abs_path('models/live/pretrained_weights/liveportrait/base_models/spade_generator.pth') # path to checkpoint of G
|
||||
checkpoint_W: str = make_abs_path('models/live/pretrained_weights/liveportrait/base_models/warping_module.pth') # path to checkpoint of W
|
||||
checkpoint_S: str = make_abs_path('models/live/pretrained_weights/liveportrait/retargeting_models/stitching_retargeting_module.pth') # path to checkpoint to S and R_eyes, R_lip
|
||||
|
||||
# ANIMAL MODEL CONFIG, NOT EXPORTED PARAMS
|
||||
checkpoint_F_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait_animals/base_models/appearance_feature_extractor.pth') # path to checkpoint of F
|
||||
checkpoint_M_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait_animals/base_models/motion_extractor.pth') # path to checkpoint pf M
|
||||
checkpoint_G_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait_animals/base_models/spade_generator.pth') # path to checkpoint of G
|
||||
checkpoint_W_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait_animals/base_models/warping_module.pth') # path to checkpoint of W
|
||||
checkpoint_S_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait/retargeting_models/stitching_retargeting_module.pth') # path to checkpoint to S and R_eyes, R_lip, NOTE: use human temporarily!
|
||||
|
||||
# EXPORTED PARAMS
|
||||
flag_use_half_precision: bool = True
|
||||
flag_crop_driving_video: bool = False
|
||||
device_id: int = 0
|
||||
flag_normalize_lip: bool = True
|
||||
flag_source_video_eye_retargeting: bool = False
|
||||
flag_eye_retargeting: bool = False
|
||||
flag_lip_retargeting: bool = False
|
||||
flag_stitching: bool = True
|
||||
flag_relative_motion: bool = True
|
||||
flag_pasteback: bool = True
|
||||
flag_do_crop: bool = True
|
||||
flag_do_rot: bool = True
|
||||
flag_force_cpu: bool = False
|
||||
flag_do_torch_compile: bool = False
|
||||
driving_option: str = "pose-friendly" # "expression-friendly" or "pose-friendly"
|
||||
driving_multiplier: float = 1.0
|
||||
driving_smooth_observation_variance: float = 3e-7 # smooth strength scalar for the animated video when the input is a source video, the larger the number, the smoother the animated video; too much smoothness would result in loss of motion accuracy
|
||||
source_max_dim: int = 1280 # the max dim of height and width of source image or video
|
||||
source_division: int = 2 # make sure the height and width of source image or video can be divided by this number
|
||||
animation_region: Literal["exp", "pose", "lip", "eyes", "all"] = "all" # the region where the animation was performed, "exp" means the expression, "pose" means the head pose
|
||||
|
||||
# NOT EXPORTED PARAMS
|
||||
lip_normalize_threshold: float = 0.03 # threshold for flag_normalize_lip
|
||||
source_video_eye_retargeting_threshold: float = 0.18 # threshold for eyes retargeting if the input is a source video
|
||||
anchor_frame: int = 0 # TO IMPLEMENT
|
||||
|
||||
input_shape: Tuple[int, int] = (256, 256) # input shape
|
||||
output_format: Literal['mp4', 'gif'] = 'mp4' # output video format
|
||||
crf: int = 15 # crf for output video
|
||||
output_fps: int = 25 # default output fps
|
||||
|
||||
mask_crop: ndarray = field(default_factory=lambda: cv2.imread(make_abs_path('../utils/resources/mask_template.png'), cv2.IMREAD_COLOR))
|
||||
lip_array: ndarray = field(default_factory=load_lip_array)
|
||||
size_gif: int = 256 # default gif size, TO IMPLEMENT
|
||||
@@ -0,0 +1,43 @@
|
||||
model_params:
|
||||
appearance_feature_extractor_params: # the F in the paper
|
||||
image_channel: 3
|
||||
block_expansion: 64
|
||||
num_down_blocks: 2
|
||||
max_features: 512
|
||||
reshape_channel: 32
|
||||
reshape_depth: 16
|
||||
num_resblocks: 6
|
||||
motion_extractor_params: # the M in the paper
|
||||
num_kp: 21
|
||||
backbone: convnextv2_tiny
|
||||
warping_module_params: # the W in the paper
|
||||
num_kp: 21
|
||||
block_expansion: 64
|
||||
max_features: 512
|
||||
num_down_blocks: 2
|
||||
reshape_channel: 32
|
||||
estimate_occlusion_map: True
|
||||
dense_motion_params:
|
||||
block_expansion: 32
|
||||
max_features: 1024
|
||||
num_blocks: 5
|
||||
reshape_depth: 16
|
||||
compress: 4
|
||||
spade_generator_params: # the G in the paper
|
||||
upscale: 2 # represents upsample factor 256x256 -> 512x512
|
||||
block_expansion: 64
|
||||
max_features: 512
|
||||
num_down_blocks: 2
|
||||
stitching_retargeting_module_params: # the S in the paper
|
||||
stitching:
|
||||
input_size: 126 # (21*3)*2
|
||||
hidden_sizes: [128, 128, 64]
|
||||
output_size: 65 # (21*3)+2(tx,ty)
|
||||
lip:
|
||||
input_size: 65 # (21*3)+2
|
||||
hidden_sizes: [128, 128, 64]
|
||||
output_size: 63 # (21*3)
|
||||
eye:
|
||||
input_size: 66 # (21*3)+3
|
||||
hidden_sizes: [256, 256, 128, 128, 64]
|
||||
output_size: 63 # (21*3)
|
||||
Binary file not shown.
Reference in New Issue
Block a user