diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/__pycache__/__init__.cpython-310.pyc b/config/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..5851c8d Binary files /dev/null and b/config/__pycache__/__init__.cpython-310.pyc differ diff --git a/config/__pycache__/argument_config.cpython-310.pyc b/config/__pycache__/argument_config.cpython-310.pyc new file mode 100644 index 0000000..aff78f0 Binary files /dev/null and b/config/__pycache__/argument_config.cpython-310.pyc differ diff --git a/config/__pycache__/base_config.cpython-310.pyc b/config/__pycache__/base_config.cpython-310.pyc new file mode 100644 index 0000000..3b03500 Binary files /dev/null and b/config/__pycache__/base_config.cpython-310.pyc differ diff --git a/config/__pycache__/crop_config.cpython-310.pyc b/config/__pycache__/crop_config.cpython-310.pyc new file mode 100644 index 0000000..1a2cffc Binary files /dev/null and b/config/__pycache__/crop_config.cpython-310.pyc differ diff --git a/config/__pycache__/inference_config.cpython-310.pyc b/config/__pycache__/inference_config.cpython-310.pyc new file mode 100644 index 0000000..24db061 Binary files /dev/null and b/config/__pycache__/inference_config.cpython-310.pyc differ diff --git a/config/argument_config.py b/config/argument_config.py new file mode 100644 index 0000000..f7ba883 --- /dev/null +++ b/config/argument_config.py @@ -0,0 +1,65 @@ +# coding: utf-8 + +""" +All configs for user +""" +from dataclasses import dataclass +import tyro +from typing_extensions import Annotated +from typing import Optional, Literal +from .base_config import PrintableConfig, make_abs_path +import cv2 +from numpy import ndarray + +from dataclasses import dataclass, field + + + +@dataclass(repr=False) # use repr from PrintableConfig +class ArgumentConfig(PrintableConfig): + ########## input arguments ########## + source: Annotated[str, tyro.conf.arg(aliases=["-s"])] = make_abs_path('../../assets/examples/source/s12.jpg') # path to the source portrait (human/animal) or video (human) + driving: Annotated[str, tyro.conf.arg(aliases=["-d"])] = "custom_nodes/ComfyUI-LivePortrait_v2/assets/examples/driving/d9.mp4" # path to driving video or template (.pkl format) + #autodl-tmp/ComfyUI/custom_nodes/ComfyUI-LivePortrait_v2/assets/examples/driving/d9.mp4 + output_dir: Annotated[str, tyro.conf.arg(aliases=["-o"])] = "../../animations/" # directory to save output video + #mask_crop: ndarray = field(default_factory=lambda: cv2.imread(make_abs_path('../utils/resources/mask_template.png'), cv2.IMREAD_COLOR)) + + ########## inference arguments ########## + flag_use_half_precision: bool = True # whether to use half precision (FP16). If black boxes appear, it might be due to GPU incompatibility; set to False. + flag_crop_driving_video: bool = False # whether to crop the driving video, if the given driving info is a video + device_id: int = 0 # gpu device id + flag_force_cpu: bool = False # force cpu inference, WIP! + flag_normalize_lip: bool = False # whether to let the lip to close state before animation, only take effect when flag_eye_retargeting and flag_lip_retargeting is False + flag_source_video_eye_retargeting: bool = False # when the input is a source video, whether to let the eye-open scalar of each frame to be the same as the first source frame before the animation, only take effect when flag_eye_retargeting and flag_lip_retargeting is False, may cause the inter-frame jittering + flag_eye_retargeting: bool = False # not recommend to be True, WIP; whether to transfer the eyes-open ratio of each driving frame to the source image or the corresponding source frame + flag_lip_retargeting: bool = False # not recommend to be True, WIP; whether to transfer the lip-open ratio of each driving frame to the source image or the corresponding source frame + flag_stitching: bool = True # recommend to True if head movement is small, False if head movement is large or the source image is an animal + flag_relative_motion: bool = True # whether to use relative motion + flag_pasteback: bool = True # whether to paste-back/stitch the animated face cropping from the face-cropping space to the original image space + flag_do_crop: bool = True # whether to crop the source portrait or video to the face-cropping space + driving_option: Literal["expression-friendly", "pose-friendly"] = "expression-friendly" # "expression-friendly" or "pose-friendly"; "expression-friendly" would adapt the driving motion with the global multiplier, and could be used when the source is a human image + driving_multiplier: float = 1.0 # be used only when driving_option is "expression-friendly" + driving_smooth_observation_variance: float = 3e-7 # smooth strength scalar for the animated video when the input is a source video, the larger the number, the smoother the animated video; too much smoothness would result in loss of motion accuracy + audio_priority: Literal['source', 'driving'] = 'driving' # whether to use the audio from source or driving video + animation_region: Literal["exp", "pose", "lip", "eyes", "all"] = "all" # the region where the animation was performed, "exp" means the expression, "pose" means the head pose, "all" means all regions + ########## source crop arguments ########## + det_thresh: float = 0.15 # detection threshold + scale: float = 2.3 # the ratio of face area is smaller if scale is larger + vx_ratio: float = 0 # the ratio to move the face to left or right in cropping space + vy_ratio: float = -0.125 # the ratio to move the face to up or down in cropping space + flag_do_rot: bool = True # whether to conduct the rotation when flag_do_crop is True + source_max_dim: int = 1280 # the max dim of height and width of source image or video, you can change it to a larger number, e.g., 1920 + source_division: int = 2 # make sure the height and width of source image or video can be divided by this number + + ########## driving crop arguments ########## + scale_crop_driving_video: float = 2.2 # scale factor for cropping driving video + vx_ratio_crop_driving_video: float = 0. # adjust y offset + vy_ratio_crop_driving_video: float = -0.1 # adjust x offset + ########## gradio arguments ########## + port: Annotated[int, tyro.conf.arg(aliases=["--port"])] = 6006 # port for gradio server + listen: bool = True + share: bool = False # whether to share the server to public + server_name: Optional[str] = "127.0.0.1" # set the local server name, "0.0.0.0" to broadcast all + flag_do_torch_compile: bool = False # whether to use torch.compile to accelerate generation + gradio_temp_dir: Optional[str] = None # directory to save gradio temp files + diff --git a/config/base_config.py b/config/base_config.py new file mode 100644 index 0000000..216b8be --- /dev/null +++ b/config/base_config.py @@ -0,0 +1,29 @@ +# coding: utf-8 + +""" +pretty printing class +""" + +from __future__ import annotations +import os.path as osp +from typing import Tuple + + +def make_abs_path(fn): + return osp.join(osp.dirname(osp.realpath(__file__)), fn) + + +class PrintableConfig: # pylint: disable=too-few-public-methods + """Printable Config defining str function""" + + def __repr__(self): + lines = [self.__class__.__name__ + ":"] + for key, val in vars(self).items(): + if isinstance(val, Tuple): + flattened_val = "[" + for item in val: + flattened_val += str(item) + "\n" + flattened_val = flattened_val.rstrip("\n") + val = flattened_val + "]" + lines += f"{key}: {str(val)}".split("\n") + return "\n ".join(lines) diff --git a/config/crop_config.py b/config/crop_config.py new file mode 100644 index 0000000..75413b1 --- /dev/null +++ b/config/crop_config.py @@ -0,0 +1,35 @@ +# coding: utf-8 + +""" +parameters used for crop faces +""" + +from dataclasses import dataclass + +from .base_config import PrintableConfig, make_abs_path + + +@dataclass(repr=False) # use repr from PrintableConfig +class CropConfig(PrintableConfig): + insightface_root: str = make_abs_path("../../pretrained_weights/insightface")############### + landmark_ckpt_path: str = make_abs_path("../../../../models/Liveportrait_v3/liveportrait/landmark.onnx") + xpose_config_file_path: str = make_abs_path("../utils/dependencies/XPose/config_model/UniPose_SwinT.py") + xpose_embedding_cache_path: str = make_abs_path('../utils/resources/clip_embedding') + + xpose_ckpt_path: str = make_abs_path("../../../../models/Liveportrait_v3/liveportrait_animals/xpose.pth")####################### + device_id: int = 0 # gpu device id + flag_force_cpu: bool = False # force cpu inference, WIP + det_thresh: float = 0.1 # detection threshold + ########## source image or video cropping option # + dsize: int = 512 # crop size + scale: float = 2.3 # scale factor + vx_ratio: float = 0 # vx ratio + vy_ratio: float = -0.125 # vy ratio +up, -down + max_face_num: int = 0 # max face number, 0 mean no limit + flag_do_rot: bool = True # whether to conduct the rotation when flag_do_crop is True + animal_face_type: str = "animal_face_9" # animal_face_68 -> 68 landmark points, animal_face_9 -> 9 landmarks + ########## driving video auto cropping option ########## + scale_crop_driving_video: float = 2.2 # 2.0 # scale factor for cropping driving video + vx_ratio_crop_driving_video: float = 0.0 # adjust y offset + vy_ratio_crop_driving_video: float = -0.1 # adjust x offset + direction: str = "large-small" # direction of cropping diff --git a/config/inference_config.py b/config/inference_config.py new file mode 100644 index 0000000..54366fb --- /dev/null +++ b/config/inference_config.py @@ -0,0 +1,69 @@ +# coding: utf-8 + +""" +config dataclass used for inference +""" + +import cv2 +from numpy import ndarray +import pickle as pkl +from dataclasses import dataclass, field +from typing import Literal, Tuple +from .base_config import PrintableConfig, make_abs_path + +def load_lip_array(): + with open(make_abs_path('../utils/resources/lip_array.pkl'), 'rb') as f: + return pkl.load(f) + +@dataclass(repr=False) # use repr from PrintableConfig +class InferenceConfig(PrintableConfig): + # HUMAN MODEL CONFIG, NOT EXPORTED PARAMS + models_config: str = make_abs_path('./models.yaml') # portrait animation config + checkpoint_F: str = make_abs_path('../../') # path to checkpoint of F + checkpoint_M: str = make_abs_path('models/live/pretrained_weights/liveportrait/base_models/motion_extractor.pth') # path to checkpoint pf M + checkpoint_G: str = make_abs_path('models/live/pretrained_weights/liveportrait/base_models/spade_generator.pth') # path to checkpoint of G + checkpoint_W: str = make_abs_path('models/live/pretrained_weights/liveportrait/base_models/warping_module.pth') # path to checkpoint of W + checkpoint_S: str = make_abs_path('models/live/pretrained_weights/liveportrait/retargeting_models/stitching_retargeting_module.pth') # path to checkpoint to S and R_eyes, R_lip + + # ANIMAL MODEL CONFIG, NOT EXPORTED PARAMS + checkpoint_F_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait_animals/base_models/appearance_feature_extractor.pth') # path to checkpoint of F + checkpoint_M_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait_animals/base_models/motion_extractor.pth') # path to checkpoint pf M + checkpoint_G_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait_animals/base_models/spade_generator.pth') # path to checkpoint of G + checkpoint_W_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait_animals/base_models/warping_module.pth') # path to checkpoint of W + checkpoint_S_animal: str = make_abs_path('models/live/pretrained_weights/liveportrait/retargeting_models/stitching_retargeting_module.pth') # path to checkpoint to S and R_eyes, R_lip, NOTE: use human temporarily! + + # EXPORTED PARAMS + flag_use_half_precision: bool = True + flag_crop_driving_video: bool = False + device_id: int = 0 + flag_normalize_lip: bool = True + flag_source_video_eye_retargeting: bool = False + flag_eye_retargeting: bool = False + flag_lip_retargeting: bool = False + flag_stitching: bool = True + flag_relative_motion: bool = True + flag_pasteback: bool = True + flag_do_crop: bool = True + flag_do_rot: bool = True + flag_force_cpu: bool = False + flag_do_torch_compile: bool = False + driving_option: str = "pose-friendly" # "expression-friendly" or "pose-friendly" + driving_multiplier: float = 1.0 + driving_smooth_observation_variance: float = 3e-7 # smooth strength scalar for the animated video when the input is a source video, the larger the number, the smoother the animated video; too much smoothness would result in loss of motion accuracy + source_max_dim: int = 1280 # the max dim of height and width of source image or video + source_division: int = 2 # make sure the height and width of source image or video can be divided by this number + animation_region: Literal["exp", "pose", "lip", "eyes", "all"] = "all" # the region where the animation was performed, "exp" means the expression, "pose" means the head pose + + # NOT EXPORTED PARAMS + lip_normalize_threshold: float = 0.03 # threshold for flag_normalize_lip + source_video_eye_retargeting_threshold: float = 0.18 # threshold for eyes retargeting if the input is a source video + anchor_frame: int = 0 # TO IMPLEMENT + + input_shape: Tuple[int, int] = (256, 256) # input shape + output_format: Literal['mp4', 'gif'] = 'mp4' # output video format + crf: int = 15 # crf for output video + output_fps: int = 25 # default output fps + + mask_crop: ndarray = field(default_factory=lambda: cv2.imread(make_abs_path('../utils/resources/mask_template.png'), cv2.IMREAD_COLOR)) + lip_array: ndarray = field(default_factory=load_lip_array) + size_gif: int = 256 # default gif size, TO IMPLEMENT diff --git a/config/models.yaml b/config/models.yaml new file mode 100644 index 0000000..131d1c6 --- /dev/null +++ b/config/models.yaml @@ -0,0 +1,43 @@ +model_params: + appearance_feature_extractor_params: # the F in the paper + image_channel: 3 + block_expansion: 64 + num_down_blocks: 2 + max_features: 512 + reshape_channel: 32 + reshape_depth: 16 + num_resblocks: 6 + motion_extractor_params: # the M in the paper + num_kp: 21 + backbone: convnextv2_tiny + warping_module_params: # the W in the paper + num_kp: 21 + block_expansion: 64 + max_features: 512 + num_down_blocks: 2 + reshape_channel: 32 + estimate_occlusion_map: True + dense_motion_params: + block_expansion: 32 + max_features: 1024 + num_blocks: 5 + reshape_depth: 16 + compress: 4 + spade_generator_params: # the G in the paper + upscale: 2 # represents upsample factor 256x256 -> 512x512 + block_expansion: 64 + max_features: 512 + num_down_blocks: 2 + stitching_retargeting_module_params: # the S in the paper + stitching: + input_size: 126 # (21*3)*2 + hidden_sizes: [128, 128, 64] + output_size: 65 # (21*3)+2(tx,ty) + lip: + input_size: 65 # (21*3)+2 + hidden_sizes: [128, 128, 64] + output_size: 63 # (21*3) + eye: + input_size: 66 # (21*3)+3 + hidden_sizes: [256, 256, 128, 128, 64] + output_size: 63 # (21*3) diff --git a/config/models/livepretrained_weights/insightface/models/buffalo_l.zip b/config/models/livepretrained_weights/insightface/models/buffalo_l.zip new file mode 100644 index 0000000..f512eb0 Binary files /dev/null and b/config/models/livepretrained_weights/insightface/models/buffalo_l.zip differ