update

add expression_editor
Update README.md
2024-08-03 14:15:34 +08:00 · 2024-08-03 14:09:45 +08:00 · 2024-07-14 14:44:11 +08:00 · 2024-07-13 22:19:59 +08:00 · 2024-07-13 22:19:12 +08:00 · 2024-07-13 22:14:23 +08:00
16 changed files with 3330 additions and 247 deletions
@@ -1,18 +1,49 @@
 [LivePortrait](https://github.com/KwaiVGI/LivePortrait)的Comfyui版本。

-!! 支持多人脸 
+!! 支持多人脸 、不同的脸 指定不同的动画

 > [寻求帮助 Mixlab nodes discord](https://discord.gg/cXs9vZSqeK)

 > [推荐：mixlab-nodes](https://github.com/shadowcz007/comfyui-mixlab-nodes)


+### 更新
+
+expression_editor：
+
+![alt text](./example/1722665711975.png)
+
+[示例工作流](./example/expression_workflow.json)
+
+表情代码：修改自[ComfyUI-AdvancedLivePortrait](https://github.com/PowerHouseMan/ComfyUI-AdvancedLivePortrait)
+
+face crop 模型参考[comfyui-ultralytics-yolo](https://github.com/shadowcz007/comfyui-ultralytics-yolo)
+
+下载  [face_yolov8m.pt 或者 face_yolov8n.pt](https://github.com/ultralytics/assets/releases/) 到 ```models/ultralytics/bbox/```
+
+
+### 教程
+[![image](https://github.com/user-attachments/assets/ab5d45c3-aeae-458f-bc8b-b017e7d1138c)](https://www.bilibili.com/video/BV1JW421R7sP/?buvid=ZE4865E83C2A9F8547C08310ED8406E72D1B&is_story_h5=false&mid=hSf%2B8X%2BJL2Hq%2F3zyc4No3A%3D%3D&p=1&plat_id=116&share_from=ugc&share_medium=iphone&share_plat=ios&share_session_id=B4772702-1A00-4E2D-8993-4725A2F52BB1&share_source=WEIXIN&share_tag=s_i&spmid=united.player-video-detail.0.0&timestamp=1720927658&unique_k=LAUWKu1&up_id=43149384&vd_source=6b8c7c3af882b1b8460fa6fa0ce1c69d)
+
+
+

 ### workflow
+
+
+
 > 配合 [comfyui-mixlab-nodes](https://github.com/shadowcz007/comfyui-mixlab-nodes) 使用

+
+> 支持视频模式 [video-to-video](example/v2v-workflow.json)
+
+
 > 全家福

+[![alt text](example/1720268832629.png)](example/mul-workflow.json)
+[不同脸对应不同的驱动视频 Workflow JSON](example/mul-workflow.json)
+
+
 [![alt text](example/1720256574305.png)](example/全家福模式-workflow.json)

 [全家福 Workflow JSON](example/全家福模式-workflow.json)
@@ -47,7 +78,7 @@ face_index：指定要处理的人脸索引，默认值为 -1，表示处理所

 debug：开启或关闭调试模式。设置为 true 时，会输出调试图像以便查看人脸检测和裁剪区域；设置为 false 时，不输出调试图像。

-
+##### Retargeting 可开关eye lip是否驱动


 ### models
@@ -1,16 +1,22 @@
-from .nodes.live_portrait import LivePortraitNode,FaceCropInfo
-
+from .nodes.live_portrait import LivePortraitNode,FaceCropInfo,Retargeting,LivePortraitVideoNode
+from .nodes.expression_editor import ExpressionEditor

 NODE_CLASS_MAPPINGS = {
    "LivePortraitNode": LivePortraitNode, 
-    "FaceCropInfo":FaceCropInfo
+    "LivePortraitVideoNode":LivePortraitVideoNode,
+    "FaceCropInfo":FaceCropInfo,
+    "Retargeting":Retargeting,
+    "ExpressionEditor_":ExpressionEditor
 }

 # dict = { "key":value }

 NODE_DISPLAY_NAME_MAPPINGS = {
   "LivePortraitNode":"Live Portrait",
-   "FaceCropInfo":"Face Crop Info"
+   "LivePortraitVideoNode":"Live Portrait for Video",
+   "FaceCropInfo":"Face Crop Info",
+   "Retargeting":"Retargeting",
+   "ExpressionEditor_":"Expression Editor"
 }

 # web ui的节点功能
@@ -0,0 +1,583 @@
+{
+  "last_node_id": 50,
+  "last_link_id": 82,
+  "nodes": [
+    {
+      "id": 22,
+      "type": "VideoCombine_Adv",
+      "pos": [
+        1722,
+        192
+      ],
+      "size": [
+        593.4324340820312,
+        809.4324340820312
+      ],
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_batch",
+          "type": "IMAGE",
+          "link": 65
+        }
+      ],
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoCombine_Adv"
+      },
+      "widgets_values": [
+        4,
+        0,
+        "Comfyui",
+        "video/h264-mp4",
+        false,
+        false,
+        false,
+        "/view?filename=Comfyui_00012_.mp4&subfolder=&type=temp&format=video%2Fh264-mp4"
+      ]
+    },
+    {
+      "id": 47,
+      "type": "ScenesNode_",
+      "pos": [
+        922,
+        796
+      ],
+      "size": {
+        "0": 299.8379821777344,
+        "1": 78
+      },
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "link": 70
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video frames (batch)",
+          "type": "IMAGE",
+          "links": [
+            69
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScenesNode_"
+      },
+      "widgets_values": [
+        10
+      ]
+    },
+    {
+      "id": 41,
+      "type": "ScenesNode_",
+      "pos": [
+        500,
+        852
+      ],
+      "size": {
+        "0": 308.9737854003906,
+        "1": 78
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "link": 61
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video frames (batch)",
+          "type": "IMAGE",
+          "links": [
+            62,
+            80
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScenesNode_"
+      },
+      "widgets_values": [
+        0
+      ]
+    },
+    {
+      "id": 42,
+      "type": "VideoCombine_Adv",
+      "pos": [
+        1310,
+        1156
+      ],
+      "size": [
+        483.5287170410156,
+        695.5287170410156
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_batch",
+          "type": "IMAGE",
+          "link": 62
+        },
+        {
+          "name": "frame_rate",
+          "type": "INT",
+          "link": 63,
+          "widget": {
+            "name": "frame_rate"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoCombine_Adv"
+      },
+      "widgets_values": [
+        4,
+        0,
+        "Comfyui",
+        "video/h264-mp4",
+        false,
+        false,
+        false,
+        "/view?filename=Comfyui_00010_.mp4&subfolder=&type=temp&format=video%2Fh264-mp4"
+      ]
+    },
+    {
+      "id": 46,
+      "type": "VideoCombine_Adv",
+      "pos": [
+        752,
+        1179
+      ],
+      "size": [
+        483.5287170410156,
+        699.5287170410156
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_batch",
+          "type": "IMAGE",
+          "link": 69
+        }
+      ],
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoCombine_Adv"
+      },
+      "widgets_values": [
+        4,
+        0,
+        "Comfyui",
+        "video/h264-mp4",
+        false,
+        false,
+        false,
+        "/view?filename=Comfyui_00011_.mp4&subfolder=&type=temp&format=video%2Fh264-mp4"
+      ]
+    },
+    {
+      "id": 39,
+      "type": "SwitchByIndex",
+      "pos": [
+        593,
+        294
+      ],
+      "size": {
+        "0": 315,
+        "1": 102
+      },
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "A",
+          "type": "*",
+          "link": 59
+        },
+        {
+          "name": "B",
+          "type": "*",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "list",
+          "type": "*",
+          "links": [
+            81
+          ],
+          "shape": 6,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SwitchByIndex"
+      },
+      "widgets_values": [
+        10,
+        "on"
+      ]
+    },
+    {
+      "id": 50,
+      "type": "LivePortraitVideoNode",
+      "pos": [
+        1216,
+        495
+      ],
+      "size": {
+        "0": 393,
+        "1": 46
+      },
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source_image_batch",
+          "type": "IMAGE",
+          "link": 80
+        },
+        {
+          "name": "driving_video",
+          "type": "SCENE_VIDEO",
+          "link": 81
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "SCENE_VIDEO",
+          "links": [
+            82
+          ],
+          "shape": 3
+        },
+        {
+          "name": "video_concat",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LivePortraitVideoNode"
+      }
+    },
+    {
+      "id": 43,
+      "type": "ScenesNode_",
+      "pos": [
+        1235,
+        326
+      ],
+      "size": {
+        "0": 299.8379821777344,
+        "1": 78
+      },
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "link": 82
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video frames (batch)",
+          "type": "IMAGE",
+          "links": [
+            65
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScenesNode_"
+      },
+      "widgets_values": [
+        0
+      ]
+    },
+    {
+      "id": 40,
+      "type": "LoadVideoAndSegment_",
+      "pos": [
+        140,
+        854
+      ],
+      "size": {
+        "0": 315,
+        "1": 214
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": [
+            61
+          ],
+          "shape": 6,
+          "slot_index": 0
+        },
+        {
+          "name": "scenes_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "links": [
+            63
+          ],
+          "shape": 3,
+          "slot_index": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideoAndSegment_"
+      },
+      "widgets_values": [
+        "WeChat_20240710164802.mp4",
+        24,
+        0,
+        null,
+        "video"
+      ]
+    },
+    {
+      "id": 1,
+      "type": "LoadVideoAndSegment_",
+      "pos": [
+        147,
+        207
+      ],
+      "size": {
+        "0": 324.6152038574219,
+        "1": 575.59765625
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": [
+            59,
+            70
+          ],
+          "shape": 6,
+          "slot_index": 0
+        },
+        {
+          "name": "scenes_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "links": [],
+          "shape": 3,
+          "slot_index": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideoAndSegment_"
+      },
+      "widgets_values": [
+        "d6.mp4",
+        24,
+        0,
+        null,
+        "video"
+      ]
+    }
+  ],
+  "links": [
+    [
+      59,
+      1,
+      0,
+      39,
+      0,
+      "*"
+    ],
+    [
+      61,
+      40,
+      0,
+      41,
+      0,
+      "SCENE_VIDEO"
+    ],
+    [
+      62,
+      41,
+      0,
+      42,
+      0,
+      "IMAGE"
+    ],
+    [
+      63,
+      40,
+      3,
+      42,
+      1,
+      "INT"
+    ],
+    [
+      65,
+      43,
+      0,
+      22,
+      0,
+      "IMAGE"
+    ],
+    [
+      69,
+      47,
+      0,
+      46,
+      0,
+      "IMAGE"
+    ],
+    [
+      70,
+      1,
+      0,
+      47,
+      0,
+      "SCENE_VIDEO"
+    ],
+    [
+      80,
+      41,
+      0,
+      50,
+      0,
+      "IMAGE"
+    ],
+    [
+      81,
+      39,
+      0,
+      50,
+      1,
+      "SCENE_VIDEO"
+    ],
+    [
+      82,
+      50,
+      0,
+      43,
+      0,
+      "SCENE_VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.7247295000000012,
+      "offset": [
+        10.69140058670257,
+        -97.28946419577976
+      ]
+    }
+  },
+  "version": 0.4
+}
@@ -12,7 +12,7 @@ import cv2
 import numpy as np
 import pickle,os
 import os.path as osp
-from rich.progress import track
+# from rich.progress import track

 # from .config.argument_config import ArgumentConfig
 from .config.inference_config import InferenceConfig
@@ -37,6 +37,26 @@ def add_index_to_filename(output_path, index):
    return new_output_path


+# 创建固定长度的list，不足的填充
+def create_drivings(elements, max_count, revert=False):
+    if not revert:
+        if max_count <= len(elements):
+            return elements[:max_count]
+        elif len(elements)>0:
+            return [elements[i % len(elements)] for i in range(max_count)]
+        else:
+            return [None for i in range(max_count)]
+    else:
+        if len(elements)==0:
+            return [None for i in range(max_count)]
+        extended_frames = elements + elements[-2:0:-1]  # 正向加反向中间部分
+        if max_count <= len(extended_frames):
+            return extended_frames[:max_count]
+        else:
+            return [extended_frames[i % len(extended_frames)] for i in range(max_count)]
+       
+
+
 def make_abs_path(fn):
    return osp.join(osp.dirname(osp.realpath(__file__)), fn)

@@ -55,6 +75,8 @@ class LivePortraitPipeline(object):
        # 增加人脸好的
        crop_info=args.crop_info

+        args.driving_info=args.driving_info[0]
+
        img_rgb = resize_to_limit(img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
        # log(f"Load source image from {args.source_image}")
        # todo 人脸检测并裁切 - 独立一个节点
@@ -116,8 +138,9 @@ class LivePortraitPipeline(object):
        R_d_0, x_d_0_info = None, None

        pbar = comfy.utils.ProgressBar(n_frames)
-
-        for i in track(range(n_frames), description='Animating...', total=n_frames):
+        print('Animating...',  n_frames)
+        for i in range(n_frames):
+        # track(range(n_frames), description='Animating...', total=n_frames):
    
            if is_video(args.driving_info):
                # extract kp info by M
@@ -213,12 +236,12 @@ class LivePortraitPipeline(object):

        # save drived result
        wfp = args.output_path
-        if inference_cfg.flag_pasteback:
+        if inference_cfg.flag_pasteback and args.source_video==False:
            images2video(I_p_paste_lst, wfp=wfp, fps=video_fps)
        else:
            images2video(I_p_lst, wfp=wfp, fps=video_fps)
-
-        return wfp, wfp_concat
+            
+        return (I_p_paste_lst if inference_cfg.flag_pasteback else I_p_lst, wfp, video_fps)

    def executeForAll(self, args):
        inference_cfg = self.live_portrait_wrapper.cfg  # for convenience
@@ -227,41 +250,77 @@ class LivePortraitPipeline(object):
        img_rgb = args.source_image
        # 增加人脸好的
        crop_info_list = args.crop_info
+        # eye lip
+        __eye__s=[c[0]['__eye__'] for c in crop_info_list]
+        __lip__s=[c[0]['__lip__'] for c in crop_info_list]
+
+        # 对齐多个驱动视频的长度
+        align_mode=args.align_mode

        img_rgb = resize_to_limit(img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
        # log(f"Load source image from {args.source_image}")
        # todo 人脸检测并裁切 - 独立一个节点
        crop_info_list = [self.cropper.crop_single_image(img_rgb, src_face=crop_info) for crop_info in crop_info_list]

+        video_fps = cv2.VideoCapture(args.driving_info[0]).get(cv2.CAP_PROP_FPS)

-        video_fps = cv2.VideoCapture(args.driving_info).get(cv2.CAP_PROP_FPS)
+        driving_infos=args.driving_info

        ######## process driving info ########
-        self.driving_lmk_lst=None
-        self.n_frames=None
-        if is_video(args.driving_info):
-            log(f"Load from video file (mp4 mov avi etc...): {args.driving_info}")
-            # TODO: 这里track一下驱动视频 -> 构建模板
-            driving_rgb_lst = load_driving_info(args.driving_info)
-            driving_rgb_lst_256 = [cv2.resize(_, (256, 256)) for _ in driving_rgb_lst]
-            I_d_lst = self.live_portrait_wrapper.prepare_driving_videos(driving_rgb_lst_256)
-            self.n_frames = I_d_lst.shape[0]
-            if inference_cfg.flag_eye_retargeting or inference_cfg.flag_lip_retargeting:
-                self.driving_lmk_lst = self.cropper.get_retargeting_lmk_info(driving_rgb_lst)
-                # input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)
-        # elif is_template(args.driving_info):
-        #     log(f"Load from video templates {args.driving_info}")
-        #     with open(args.driving_info, 'rb') as f:
-        #         template_lst, driving_lmk_lst = pickle.load(f)
-        #     n_frames = template_lst[0]['n_frames']
-        #     # input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)
-        # else:
-        #     raise Exception("Unsupported driving types!")
-        #########################################
-        print('#driving_lmk_lst',self.driving_lmk_lst)
+        driving_lmk_lst_s=[]
+        n_frames_s=[]
+        I_d_lst_s=[]
+
+        pbar = comfy.utils.ProgressBar(len(driving_infos))
+        for  z in range(len(driving_infos)):
+
+            driving_info=driving_infos[z]
+            crop_info=crop_info_list[z]
+            # print('###',z,len(driving_infos),len(crop_info_list))
+            # print('#crop_info_list[z]',crop_info_list[z])
+            __eye__=__eye__s[z]
+            __lip__=__lip__s[z]
+
+            if is_video(driving_info):
+                log(f"Load from video file (mp4 mov avi etc...): {driving_info}")
+                # TODO: 这里track一下驱动视频 -> 构建模板
+                driving_rgb_lst = load_driving_info(driving_info)
+                driving_rgb_lst_256 = [cv2.resize(_, (256, 256)) for _ in driving_rgb_lst]
+                I_d_lst = self.live_portrait_wrapper.prepare_driving_videos(driving_rgb_lst_256)
+                n_frames = I_d_lst.shape[0]
+                
+                I_d_lst_s.append(I_d_lst)
+                n_frames_s.append(n_frames)
+
+                source_lmk = crop_info['lmk_crop']
+                if __eye__ or __lip__:
+                    driving_lmk_lst = self.cropper.get_retargeting_lmk_info(driving_rgb_lst)
+                    driving_lmk_lst_s.append(driving_lmk_lst)
+                    input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(
+                        source_lmk, 
+                        driving_lmk_lst
+                        )
+            # elif is_template(args.driving_info):
+            #     log(f"Load from video templates {args.driving_info}")
+            #     with open(args.driving_info, 'rb') as f:
+            #         template_lst, driving_lmk_lst = pickle.load(f)
+            #     n_frames = template_lst[0]['n_frames']
+            #     # input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)
+            # else:
+            #     raise Exception("Unsupported driving types!")
+            #########################################
+            # print('#driving_lmk_lst',self.driving_lmk_lst)
+            pbar.update(1)
+
+        # 对齐
+        max_n_frames = max(n_frames_s)
+        n_frames_s=[max_n_frames for i in n_frames_s]
+        driving_lmk_lst_s= [create_drivings(d,max_n_frames,align_mode) for d in driving_lmk_lst_s]
+        I_d_lst_s= [create_drivings(i,max_n_frames,align_mode) for i in I_d_lst_s]
+

        # 原图片---视频帧
-        img_rgbs=[img_rgb for i in range(self.n_frames)]
+        img_rgbs=[img_rgb for i in range(n_frames_s[0])]
        

        for index in range(len(crop_info_list)):
@@ -269,6 +328,10 @@ class LivePortraitPipeline(object):
            crop_info=crop_info_list[index]
            # 地一张脸

+            __eye__=__eye__s[index]
+            __lip__=__lip__s[index]
+            
+            print('#crop_info',crop_info.keys())
            source_lmk = crop_info['lmk_crop']
            img_crop, img_crop_256x256 = crop_info['img_crop'], crop_info['img_crop_256x256']
            if inference_cfg.flag_do_crop:
@@ -289,8 +352,13 @@ class LivePortraitPipeline(object):
                else:
                    lip_delta_before_animation = self.live_portrait_wrapper.retarget_lip(x_s, combined_lip_ratio_tensor_before_animation)
            ############################################
-            if self.driving_lmk_lst!=None:
-                input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, self.driving_lmk_lst)
+
+            # 多个驱动视频
+            if 0 <= index < len(driving_lmk_lst_s):
+                driving_lmk_lst=driving_lmk_lst_s[index]
+                
+                if driving_lmk_lst!=None:
+                    input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)

            ######## prepare for pasteback ########
            if inference_cfg.flag_pasteback:
@@ -304,11 +372,19 @@ class LivePortraitPipeline(object):
            I_p_lst = []
            R_d_0, x_d_0_info = None, None

-            pbar = comfy.utils.ProgressBar(self.n_frames)
+            # 多个驱动视频
+            n_frames=n_frames_s[index]
+            driving_info=driving_infos[index]
+            I_d_lst=I_d_lst_s[index]

-            for i in track(range(self.n_frames), description='Animating...', total=self.n_frames):

-                if is_video(args.driving_info):
+            pbar = comfy.utils.ProgressBar(n_frames)
+
+            print('Animating...',  n_frames)
+            for i in range(n_frames):
+            # track(range(n_frames), description='Animating...', total=n_frames):
+
+                if is_video(driving_info):
                    # extract kp info by M
                    I_d_i = I_d_lst[i]
                    x_d_i_info = self.live_portrait_wrapper.get_kp_info(I_d_i)
@@ -333,13 +409,13 @@ class LivePortraitPipeline(object):
                x_d_i_new = scale_new * (x_c_s @ R_new + delta_new) + t_new

                # Algorithm 1:
-                if not inference_cfg.flag_stitching and not inference_cfg.flag_eye_retargeting and not inference_cfg.flag_lip_retargeting:
+                if not inference_cfg.flag_stitching and not __eye__ and not __lip__:
                    # without stitching or retargeting
                    if inference_cfg.flag_lip_zero:
                        x_d_i_new += lip_delta_before_animation.reshape(-1, x_s.shape[1], 3)
                    else:
                        pass
-                elif inference_cfg.flag_stitching and not inference_cfg.flag_eye_retargeting and not inference_cfg.flag_lip_retargeting:
+                elif inference_cfg.flag_stitching and not __eye__ and not __lip__:
                    # with stitching and without retargeting
                    if inference_cfg.flag_lip_zero:
                        x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new) + lip_delta_before_animation.reshape(-1, x_s.shape[1], 3)
@@ -347,12 +423,12 @@ class LivePortraitPipeline(object):
                        x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new)
                else:
                    eyes_delta, lip_delta = None, None
-                    if inference_cfg.flag_eye_retargeting:
+                    if __eye__:
                        c_d_eyes_i = input_eye_ratio_lst[i]
                        combined_eye_ratio_tensor = self.live_portrait_wrapper.calc_combined_eye_ratio(c_d_eyes_i, source_lmk)
                        # ∆_eyes,i = R_eyes(x_s; c_s,eyes, c_d,eyes,i)
                        eyes_delta = self.live_portrait_wrapper.retarget_eye(x_s, combined_eye_ratio_tensor)
-                    if inference_cfg.flag_lip_retargeting:
+                    if __lip__:
                        c_d_lip_i = input_lip_ratio_lst[i]
                        combined_lip_ratio_tensor = self.live_portrait_wrapper.calc_combined_lip_ratio(c_d_lip_i, source_lmk)
                        # ∆_lip,i = R_lip(x_s; c_s,lip, c_d,lip,i)
@@ -405,8 +481,8 @@ class LivePortraitPipeline(object):

        # save drived result
        wfp = args.output_path
-        if inference_cfg.flag_pasteback:
+        if inference_cfg.flag_pasteback and args.source_video==False:
            images2video(I_p_paste_lst, wfp=wfp, fps=video_fps)
        
-        return wfp
+        return (I_p_paste_lst, wfp, video_fps)

@@ -8,7 +8,7 @@ import os
 import cv2
 import numpy as np
 import pickle
-from rich.progress import track
+# from rich.progress import track
 from .utils.cropper import Cropper

 from .utils.io import load_driving_info
@@ -40,8 +40,8 @@ class TemplateMaker:

        templates = []

-
-        for i in track(range(n_frames), description='Making templates...', total=n_frames):
+        print('# Making templates...', n_frames)
+        for i in range(n_frames):
            I_d_i = I_d_lst[i]
            x_d_i_info = self.live_portrait_wrapper.get_kp_info(I_d_i)
            R_d_i = get_rotation_matrix(x_d_i_info['pitch'], x_d_i_info['yaw'], x_d_i_info['roll'])
@@ -123,7 +123,7 @@ class Cropper(object):
        elif isinstance(obj, np.ndarray):
            img_rgb = obj

-        print('#crop_single_image',direction,face_index,src_face)
+        # print('#crop_single_image',direction,face_index,src_face)

        if src_face==None:
            src_face = self.face_analysis_wrapper.get(
@@ -10,7 +10,7 @@ import subprocess
 import imageio
 import cv2

-from rich.progress import track
+# from rich.progress import track
 from .helper import prefix
 from .rprint import rprint as print

@@ -35,7 +35,8 @@ def images2video(images, wfp, **kwargs):
    )

    n = len(images)
-    for i in track(range(n), description='writing', transient=True):
+    print('writing',n)
+    for i in range(n):
        if image_mode.lower() == 'bgr':
            writer.append_data(images[i][..., ::-1])
        else:
@@ -83,7 +84,9 @@ def blend(img: np.ndarray, mask: np.ndarray, background_color=(255, 255, 255)):
 def concat_frames(I_p_lst, driving_rgb_lst, img_rgb):
    # TODO: add more concat style, e.g., left-down corner driving
    out_lst = []
-    for idx, _ in track(enumerate(I_p_lst), total=len(I_p_lst), description='Concatenating result...'):
+    print('Concatenating result...',len(I_p_lst))
+    for idx, _ in enumerate(I_p_lst):
+    # track(enumerate(I_p_lst), total=len(I_p_lst), description='Concatenating result...'):
        source_image_drived = I_p_lst[idx]
        image_drive = driving_rgb_lst[idx]

@@ -0,0 +1,533 @@
+import os
+import sys
+import numpy as np
+import torch
+import cv2
+from PIL import Image
+import folder_paths
+import copy,json
+from ultralytics import YOLO
+
+current_file_path = os.path.abspath(__file__)
+current_directory = os.path.dirname(current_file_path)
+sys.path.append(current_directory)
+from LivePortrait.src.live_portrait_wrapper import LivePortraitWrapper
+from LivePortrait.src.utils.camera import get_rotation_matrix
+
+def tensor2pil(image):
+    return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
+def pil2tensor(image):
+    return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0)
+def rgb_crop(rgb, region):
+    return rgb[region[1]:region[3], region[0]:region[2]]
+
+def get_rgb_size(rgb):
+    return rgb.shape[1], rgb.shape[0]
+def create_transform_matrix(x, y, scale=1):
+    return np.float32([[scale, 0, x], [0, scale, y]])
+
+def get_model_dir(m):
+    try:
+        return folder_paths.get_folder_paths(m)[0]
+    except:
+        return os.path.join(folder_paths.models_dir, m)
+
+def calc_crop_limit(center, img_size, crop_size):
+    pos = center - crop_size / 2
+    if pos < 0:
+        crop_size += pos * 2
+        pos = 0
+
+    pos2 = pos + crop_size
+
+    if img_size < pos2:
+        crop_size -= (pos2 - img_size) * 2
+        pos2 = img_size
+        pos = pos2 - crop_size
+
+    return pos, pos2, crop_size
+
+ 
+
+
+# 修改模型路径 ， 沿用 comfyui-liveportrait
+class InferenceConfig:
+    def __init__(self,
+                    models_config,
+                    checkpoint_F,
+                    checkpoint_M,
+                    checkpoint_G,
+                    checkpoint_W,
+                    checkpoint_S,
+                    mask_crop = None,
+                    flag_use_half_precision=True,
+                    flag_lip_zero=True,
+                    lip_zero_threshold=0.03,
+                    flag_eye_retargeting=False,
+                    flag_lip_retargeting=False,
+                    flag_stitching=True,
+                    flag_relative=True,
+                    anchor_frame=0,
+                    input_shape=(256, 256),
+                    output_format='mp4',
+                    output_fps=30,
+                    crf=15,
+                    flag_write_result=True,
+                    flag_pasteback=True,
+                    flag_write_gif=False,
+                    size_gif=256,
+                    ref_max_shape=1280,
+                    ref_shape_n=2,
+                    device_id=0,
+                    flag_do_crop=True,
+                    flag_do_rot=True):
+        self.models_config = models_config
+        self.checkpoint_F = checkpoint_F
+        self.checkpoint_M = checkpoint_M
+        self.checkpoint_G = checkpoint_G
+        self.checkpoint_W = checkpoint_W
+        self.checkpoint_S = checkpoint_S
+        self.flag_use_half_precision = flag_use_half_precision
+        self.flag_lip_zero = flag_lip_zero
+        self.lip_zero_threshold = lip_zero_threshold
+        self.flag_eye_retargeting = flag_eye_retargeting
+        self.flag_lip_retargeting = flag_lip_retargeting
+        self.flag_stitching = flag_stitching
+        self.flag_relative = flag_relative
+        self.anchor_frame = anchor_frame
+        self.input_shape = input_shape
+        self.output_format = output_format
+        self.output_fps = output_fps
+        self.crf = crf
+        self.flag_write_result = flag_write_result
+        self.flag_pasteback = flag_pasteback
+        self.flag_write_gif = flag_write_gif
+        self.size_gif = size_gif
+        self.ref_max_shape = ref_max_shape
+        self.ref_shape_n = ref_shape_n
+        self.device_id = device_id
+        self.flag_do_crop = flag_do_crop
+        self.flag_do_rot = flag_do_rot
+        self.mask_crop=mask_crop
+
+liveportrait_model=get_model_dir('liveportrait')
+
+inference_cfg = InferenceConfig(
+        models_config=os.path.join(current_directory,'LivePortrait','src','config','models.yaml'),
+        checkpoint_F=os.path.join(liveportrait_model,'base_models','appearance_feature_extractor.pth'),
+        checkpoint_M=os.path.join(liveportrait_model,'base_models','motion_extractor.pth') ,
+        checkpoint_G=os.path.join(liveportrait_model,'base_models','spade_generator.pth') ,
+        checkpoint_W=os.path.join(liveportrait_model,'base_models','warping_module.pth'),
+        checkpoint_S=os.path.join(liveportrait_model,'retargeting_models','stitching_retargeting_module.pth')
+    )
+
+class PreparedSrcImg:
+    def __init__(self, src_rgb, crop_trans_m, x_s_info, f_s_user, x_s_user, mask_ori):
+        self.src_rgb = src_rgb
+        self.crop_trans_m = crop_trans_m
+        self.x_s_info = x_s_info
+        self.f_s_user = f_s_user
+        self.x_s_user = x_s_user
+        self.mask_ori = mask_ori
+
+class LP_Engine:
+    pipeline = None
+    bbox_model = None
+    mask_img = None
+
+    def detect_face(self, image_rgb):
+
+        crop_factor = 1.7
+        bbox_drop_size = 10
+
+        if self.bbox_model == None:
+            bbox_model_path = os.path.join(get_model_dir("ultralytics"), "face_yolov8n.pt")
+            
+            # 沿用 comfyui-ultralytics-yolo
+            for fp in [os.path.join("bbox","face_yolov8m.pt"),os.path.join("bbox","face_yolov8n.pt"),"face_yolov8n.pt","face_yolov8m.pt",]:
+                np=os.path.join(get_model_dir("ultralytics"), fp)
+                if os.path.isfile(np):
+                    bbox_model_path=np
+
+            self.bbox_model = YOLO(bbox_model_path)
+
+        pred = self.bbox_model(image_rgb, conf=0.7, device="")
+        bboxes = pred[0].boxes.xyxy.cpu().numpy()
+
+        w, h = get_rgb_size(image_rgb)
+
+        # for x, label in zip(segmasks, detected_results[0]):
+        for x1, y1, x2, y2 in bboxes:
+            bbox_w = x2 - x1
+            bbox_h = y2 - y1
+
+            crop_w = bbox_w * crop_factor
+            crop_h = bbox_h * crop_factor
+
+            crop_w = max(crop_h, crop_w)
+            crop_h = crop_w
+
+            kernel_x = x1 + bbox_w / 2
+            kernel_y = y1 + bbox_h / 2
+
+            new_x1, new_x2, crop_w = calc_crop_limit(kernel_x, w, crop_w)
+
+            if crop_w < crop_h:
+                crop_h = crop_w
+
+            new_y1, new_y2, crop_h = calc_crop_limit(kernel_y, h, crop_h)
+
+            if crop_h < crop_w:
+                crop_w = crop_h
+                new_x1, new_x2, crop_w = calc_crop_limit(kernel_x, w, crop_w)
+
+            return [int(new_x1), int(new_y1), int(new_x2), int(new_y2)]
+
+        print("Failed to detect face!!")
+        return [0, 0, w, h]
+
+    def crop_face(self, rgb_img):
+        region = self.detect_face(rgb_img)
+        face_image = rgb_crop(rgb_img, region)
+        return face_image, region
+
+    def get_pipeline(self):
+        if self.pipeline == None:
+            print("Load pipeline...")
+            self.pipeline = LivePortraitWrapper(cfg=inference_cfg)
+
+        return self.pipeline
+
+    def prepare_src_image(self, img):
+        h, w = img.shape[:2]
+        input_shape = [256,256]
+        if h != input_shape[0] or w != input_shape[1]:
+            x = cv2.resize(img, (input_shape[0], input_shape[1]), interpolation = cv2.INTER_LINEAR)
+        else:
+            x = img.copy()
+
+        if x.ndim == 3:
+            x = x[np.newaxis].astype(np.float32) / 255.  # HxWx3 -> 1xHxWx3, normalized to 0~1
+        elif x.ndim == 4:
+            x = x.astype(np.float32) / 255.  # BxHxWx3, normalized to 0~1
+        else:
+            raise ValueError(f'img ndim should be 3 or 4: {x.ndim}')
+        x = np.clip(x, 0, 1)  # clip to 0~1
+        x = torch.from_numpy(x).permute(0, 3, 1, 2)  # 1xHxWx3 -> 1x3xHxW
+        x = x.cuda()
+        return x
+
+    def GetMask(self):
+        if self.mask_img is None:
+            path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "./LivePortrait/src/utils/resources/mask_template.png")
+            self.mask_img = cv2.imread(path, cv2.IMREAD_COLOR)
+        return self.mask_img
+
+    def prepare_source(self, source_image, is_video = False):
+        print("Prepare source...")
+        engine = self.get_pipeline()
+        source_image_np = (source_image * 255).byte().numpy()
+        img_rgb = source_image_np[0]
+        face_img, crop_region = self.crop_face(img_rgb)
+
+        scale = face_img.shape[0] / 512.
+        crop_trans_m = create_transform_matrix(crop_region[0], crop_region[1], scale)
+        mask_ori = cv2.warpAffine(self.GetMask(), crop_trans_m, get_rgb_size(img_rgb), cv2.INTER_LINEAR)
+        mask_ori = mask_ori.astype(np.float32) / 255.
+
+        psi_list = []
+        for img_rgb in source_image_np:
+            face_img = rgb_crop(img_rgb, crop_region)
+            i_s = self.prepare_src_image(face_img)
+            x_s_info = engine.get_kp_info(i_s)
+            f_s_user = engine.extract_feature_3d(i_s)
+            x_s_user = engine.transform_keypoint(x_s_info)
+            psi = PreparedSrcImg(img_rgb, crop_trans_m, x_s_info, f_s_user, x_s_user, mask_ori)
+            if is_video == False:
+                return psi
+            psi_list.append(psi)
+
+        return psi_list
+
+    def prepare_driving_video(self, face_images):
+        print("Prepare driving video...")
+        pipeline = self.get_pipeline()
+        f_img_np = (face_images * 255).byte().numpy()
+
+        out_list = []
+        for f_img in f_img_np:
+            i_d = pipeline.prepare_source(f_img)
+            d_info = pipeline.get_kp_info(i_d)
+            #out_list.append((d_info, get_rotation_matrix(d_info['pitch'], d_info['yaw'], d_info['roll'])))
+            out_list.append(d_info)
+
+        return out_list
+
+    def calc_fe(_, x_d_new, eyes, eyebrow, wink, pupil_x, pupil_y, mouth, eee, woo, smile,
+                rotate_pitch, rotate_yaw, rotate_roll):
+
+        x_d_new[0, 20, 1] += smile * -0.01
+        x_d_new[0, 14, 1] += smile * -0.02
+        x_d_new[0, 17, 1] += smile * 0.0065
+        x_d_new[0, 17, 2] += smile * 0.003
+        x_d_new[0, 13, 1] += smile * -0.00275
+        x_d_new[0, 16, 1] += smile * -0.00275
+        x_d_new[0, 3, 1] += smile * -0.0035
+        x_d_new[0, 7, 1] += smile * -0.0035
+
+        x_d_new[0, 19, 1] += mouth * 0.001
+        x_d_new[0, 19, 2] += mouth * 0.0001
+        x_d_new[0, 17, 1] += mouth * -0.0001
+        rotate_pitch -= mouth * 0.05
+
+        x_d_new[0, 20, 2] += eee * -0.001
+        x_d_new[0, 20, 1] += eee * -0.001
+        #x_d_new[0, 19, 1] += eee * 0.0006
+        x_d_new[0, 14, 1] += eee * -0.001
+
+        x_d_new[0, 14, 1] += woo * 0.001
+        x_d_new[0, 3, 1] += woo * -0.0005
+        x_d_new[0, 7, 1] += woo * -0.0005
+        x_d_new[0, 17, 2] += woo * -0.0005
+
+        x_d_new[0, 11, 1] += wink * 0.001
+        x_d_new[0, 13, 1] += wink * -0.0003
+        x_d_new[0, 17, 0] += wink * 0.0003
+        x_d_new[0, 17, 1] += wink * 0.0003
+        x_d_new[0, 3, 1] += wink * -0.0003
+        rotate_roll -= wink * 0.1
+        rotate_yaw -= wink * 0.1
+
+        if 0 < pupil_x:
+            x_d_new[0, 11, 0] += pupil_x * 0.0007
+            x_d_new[0, 15, 0] += pupil_x * 0.001
+        else:
+            x_d_new[0, 11, 0] += pupil_x * 0.001
+            x_d_new[0, 15, 0] += pupil_x * 0.0007
+
+        x_d_new[0, 11, 1] += pupil_y * -0.001
+        x_d_new[0, 15, 1] += pupil_y * -0.001
+        eyes -= pupil_y / 2.
+
+        x_d_new[0, 11, 1] += eyes * -0.001
+        x_d_new[0, 13, 1] += eyes * 0.0003
+        x_d_new[0, 15, 1] += eyes * -0.001
+        x_d_new[0, 16, 1] += eyes * 0.0003
+
+
+        if 0 < eyebrow:
+            x_d_new[0, 1, 1] += eyebrow * 0.001
+            x_d_new[0, 2, 1] += eyebrow * -0.001
+        else:
+            x_d_new[0, 1, 0] += eyebrow * -0.001
+            x_d_new[0, 2, 0] += eyebrow * 0.001
+            x_d_new[0, 1, 1] += eyebrow * 0.0003
+            x_d_new[0, 2, 1] += eyebrow * -0.0003
+
+
+        return torch.Tensor([rotate_pitch, rotate_yaw, rotate_roll])
+g_engine = LP_Engine()
+
+class ExpressionSet:
+    def __init__(self, erst = None, es = None):
+        if es != None:
+            self.e = copy.deepcopy(es.e)  # [:, :, :]
+            self.r = copy.deepcopy(es.r)  # [:]
+            self.s = copy.deepcopy(es.s)
+            self.t = copy.deepcopy(es.t)
+        elif erst != None:
+            self.e = erst[0]
+            self.r = erst[1]
+            self.s = erst[2]
+            self.t = erst[3]
+        else:
+            self.e = torch.from_numpy(np.zeros((1, 21, 3))).float().to(device='cuda')
+            self.r = torch.Tensor([0, 0, 0])
+            self.s = 0
+            self.t = 0
+    def div(self, value):
+        self.e /= value
+        self.r /= value
+        self.s /= value
+        self.t /= value
+    def add(self, other):
+        self.e += other.e
+        self.r += other.r
+        self.s += other.s
+        self.t += other.t
+    def sub(self, other):
+        self.e -= other.e
+        self.r -= other.r
+        self.s -= other.s
+        self.t -= other.t
+    def mul(self, value):
+        self.e *= value
+        self.r *= value
+        self.s *= value
+        self.t *= value
+
+    #def apply_ratio(self, ratio):        self.exp *= ratio
+
+ 
+
+class ExpressionEditor:
+    def __init__(self):
+        self.sample_image = None
+        self.src_image = None
+
+    @classmethod
+    def INPUT_TYPES(s):
+        display = "number"
+        #display = "slider"
+        return {
+            "required": {
+                "src_image": ("IMAGE",), 
+
+                "rotate_pitch": ("FLOAT", {"default": 0, "min": -50, "max": 50, "step": 0.2, "display": display}),
+                "rotate_yaw": ("FLOAT", {"default": 0, "min": -50, "max": 50, "step": 0.2, "display": display}),
+                "rotate_roll": ("FLOAT", {"default": 0, "min": -50, "max": 50, "step": 0.2, "display": display}),
+
+                "blink": ("FLOAT", {"default": 0, "min": -30, "max": 15, "step": 0.2, "display": display}),
+                "eyebrow": ("FLOAT", {"default": 0, "min": -30, "max": 25, "step": 0.2, "display": display}),
+                "wink": ("FLOAT", {"default": 0, "min": -10, "max": 25, "step": 0.2, "display": display}),
+                
+                "pupil_x": ("FLOAT", {"default": 0, "min": -15, "max": 15, "step": 0.2, "display": display}),
+                "pupil_y": ("FLOAT", {"default": 0, "min": -15, "max": 15, "step": 0.2, "display": display}),
+                
+                "aaa": ("FLOAT", {"default": 0, "min": -30, "max": 120, "step": 1, "display": display}),
+                "eee": ("FLOAT", {"default": 0, "min": -20, "max": 15, "step": 0.2, "display": display}),
+                "woo": ("FLOAT", {"default": 0, "min": -20, "max": 15, "step": 0.2, "display": display}),
+                
+                "smile": ("FLOAT", {"default": 0, "min": -0.3, "max": 1.3, "step": 0.01, "display": display}),
+
+                "src_weight": ("FLOAT", {"default": 1, "min": 0, "max": 1, "step": 0.01, "display": display}),
+                # "sample_ratio": ("FLOAT", {"default": 1, "min": 0, "max": 1, "step": 0.01, "display": display}),
+            },
+
+            "optional": {
+                
+                "expression_json":("STRING", {"forceInput": True,"dynamicPrompts": False}),
+                # "sample_image": ("IMAGE",), 
+            },
+        }
+
+    RETURN_TYPES = ("IMAGE","STRING",)
+    RETURN_NAMES = ("image","expression_json",)
+
+    FUNCTION = "run"
+
+    OUTPUT_NODE = True
+
+    CATEGORY = "♾️Mixlab/Face"
+
+    # INPUT_IS_LIST = False
+    # OUTPUT_IS_LIST = (False,)
+
+    def run(self,src_image, rotate_pitch, rotate_yaw, rotate_roll, blink, eyebrow, wink, pupil_x, pupil_y, aaa,
+             eee, woo, smile,
+            src_weight, expression_json=None):
+        
+        if expression_json!=None:
+            try:
+                dict_obj = json.loads(expression_json)
+                if "rotate_pitch" in dict_obj:
+                    rotate_pitch=dict_obj["rotate_pitch"]
+                if "rotate_yaw" in dict_obj:
+                    rotate_yaw=dict_obj["rotate_yaw"]
+                if "rotate_roll" in dict_obj:
+                    rotate_roll=dict_obj["rotate_roll"]
+                if "blink" in dict_obj:
+                    blink=dict_obj["blink"]
+                if "eyebrow" in dict_obj:
+                    eyebrow=dict_obj["eyebrow"]
+                if "wink" in dict_obj:
+                    wink=dict_obj["wink"]
+                if "pupil_x" in dict_obj:
+                    pupil_x=dict_obj["pupil_x"]
+                if "pupil_y" in dict_obj:
+                    pupil_y=dict_obj["pupil_y"]
+                if "aaa" in dict_obj:
+                    aaa=dict_obj["aaa"]
+                if "eee" in dict_obj:
+                    eee=dict_obj["eee"]
+                if "woo" in dict_obj:
+                    woo=dict_obj["woo"]
+                if "smile" in dict_obj:
+                    smile=dict_obj["smile"]
+                if "src_weight" in dict_obj:
+                    src_weight=dict_obj["src_weight"]
+            except:
+                print("#expression_json",expression_json)
+
+        print('#expression_json',expression_json)
+
+        expression_json={
+            "rotate_pitch":rotate_pitch,
+            "rotate_yaw":rotate_yaw,
+            "rotate_roll":rotate_roll, 
+            "blink":blink, 
+            "eyebrow":eyebrow, 
+            "wink":wink, 
+            "pupil_x":pupil_x, 
+            "pupil_y":pupil_y, 
+            "aaa":aaa, 
+            "eee":eee, 
+            "woo":woo, 
+            "smile":smile,
+            "src_weight":src_weight
+        }
+
+        rotate_yaw = -rotate_yaw
+
+        new_editor_link = None
+       
+        if id(src_image) != id(self.src_image):
+            self.psi = g_engine.prepare_source(src_image)
+            self.src_image = src_image
+        new_editor_link = []
+        new_editor_link.append(self.psi)
+        
+
+        pipeline = g_engine.get_pipeline()
+
+        psi = self.psi
+        s_info = psi.x_s_info
+        #delta_new = copy.deepcopy()
+        s_exp = s_info['exp'] * src_weight
+        s_exp[0, 5] = s_info['exp'][0, 5]
+        s_exp += s_info['kp']
+
+        es = ExpressionSet()
+
+        # if sample_image != None:
+        #     if id(self.sample_image) != id(sample_image):
+        #         self.sample_image = sample_image
+        #         d_image_np = (sample_image * 255).byte().numpy()
+        #         d_face, _ = g_engine.crop_face(d_image_np[0])
+        #         i_d = pipeline.prepare_source(d_face)
+        #         self.d_info = pipeline.get_kp_info(i_d)
+        #         self.d_info['exp'][0, 5, 0] = 0
+        #         self.d_info['exp'][0, 5, 1] = 0
+
+        #     # delta_new += s_exp * (1 - sample_ratio) + self.d_info['exp'] * sample_ratio
+        #     es.e += self.d_info['exp'] * sample_ratio
+
+        es.r = g_engine.calc_fe(es.e, blink, eyebrow, wink, pupil_x, pupil_y, aaa, eee, woo, smile,
+                                  rotate_pitch, rotate_yaw, rotate_roll)
+
+        new_rotate = get_rotation_matrix(s_info['pitch'] + es.r[0], s_info['yaw'] + es.r[1],
+                                         s_info['roll'] + es.r[2])
+        x_d_new = (s_info['scale'] * (1 + es.s)) * ((s_exp + es.e) @ new_rotate) + s_info['t']
+
+        x_d_new = pipeline.stitching(psi.x_s_user, x_d_new)
+
+        crop_out = pipeline.warp_decode(psi.f_s_user, psi.x_s_user, x_d_new)
+        crop_out = pipeline.parse_output(crop_out['out'])[0]
+
+        crop_with_fullsize = cv2.warpAffine(crop_out, psi.crop_trans_m, get_rgb_size(psi.src_rgb), cv2.INTER_LINEAR)
+        out = np.clip(psi.mask_ori * crop_with_fullsize + (1 - psi.mask_ori) * psi.src_rgb, 0, 255).astype(np.uint8)
+
+        out_img = pil2tensor(out)
+
+        return (out_img,json.dumps(expression_json) ,)
+
@@ -28,7 +28,7 @@ def pil2tensor(image):
    return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0)


-
+from .LivePortrait.src.utils.video import images2video
 from .LivePortrait.src.live_portrait_pipeline import LivePortraitPipeline

 def get_model_dir(m):
@@ -44,12 +44,14 @@ class ArgumentConfig:
                    driving_info,
                    output_path='animations/v.mp4',
                    output_path_concat="",
+                    source_video=False,
                    device_id=0, 
                    crop_info =None,
                    face_index=0,
+                    align_mode=True,
                    flag_lip_zero=True,
                    flag_eye_retargeting=False,
-                    flag_lip_retargeting=False,
+                    flag_lip_retargeting=False, 
                    flag_stitching=True,
                    flag_relative=True,
                    flag_pasteback=True,
@@ -63,15 +65,17 @@ class ArgumentConfig:
                    share=False,
                    server_name='0.0.0.0'):
        self.source_image = source_image
+        self.source_video=source_video
        self.driving_info = driving_info
        self.output_path = output_path
        self.output_path_concat=output_path_concat
        self.crop_info=crop_info
        self.face_index=face_index
+        self.align_mode=align_mode
        self.device_id = device_id
        self.flag_lip_zero = flag_lip_zero
        self.flag_eye_retargeting = flag_eye_retargeting
-        self.flag_lip_retargeting = flag_lip_retargeting
+        self.flag_lip_retargeting = flag_lip_retargeting 
        self.flag_stitching = flag_stitching
        self.flag_relative = flag_relative
        self.flag_pasteback = flag_pasteback
@@ -172,8 +176,6 @@ crop_cfg = CropConfig()

 # 人脸检测并裁切
 class FaceCropInfo:
-    def __init__(self):
-        self.speaker = None
    @classmethod
    def INPUT_TYPES(s):
        
@@ -194,7 +196,7 @@ class FaceCropInfo:

    OUTPUT_NODE = True

-    CATEGORY = "♾️Mixlab/Video"
+    CATEGORY = "♾️Mixlab/Video/LivePortrait"

    INPUT_IS_LIST = False
    OUTPUT_IS_LIST = (True,False,) #list 列表 [1,2,3]
@@ -225,9 +227,82 @@ class FaceCropInfo:
            #只输出一张 [face]
            crop_info=[crop_info[face_index]]

-        return (crop_info,debug_image,)
+
+        result=[]
+
+        for c in crop_info:
+            c['__eye__']=True
+            c['__lip__']=True
+            result.append(c)
+
+        return (result,debug_image,)


+# 人脸检测并裁切
+class Retargeting:
+    @classmethod
+    def INPUT_TYPES(s):
+        
+        return {"required": {
+                        "crop_info": ("CROP_INFO",),
+                        },
+                "optional":{ 
+                           "lip":("BOOLEAN", {"default": True},),
+                           "eye":("BOOLEAN", {"default": True},),
+                        }
+                }
+    
+    RETURN_TYPES = ("CROP_INFO",)
+    RETURN_NAMES = ("crop_info",)
+
+    FUNCTION = "run"
+
+    OUTPUT_NODE = True
+
+    CATEGORY = "♾️Mixlab/Video/LivePortrait"
+
+    INPUT_IS_LIST = False
+    OUTPUT_IS_LIST = (False,) #list 列表 [1,2,3]
+  
+    def run(self,crop_info,lip=True,eye=True):
+        crop_info['__eye__']=eye
+        crop_info['__lip__']=lip
+        return (crop_info,)
+
+
+
+# 驱动模板制作
+# class DriveVideoNode:
+
+#     @classmethod
+#     def INPUT_TYPES(s):
+        
+#         return {"required": {
+#                         "driving_video1":("SCENE_VIDEO",),
+#                         "driving_video2":("SCENE_VIDEO",),
+#                         },
+#                 # "optional":{ 
+#                 #         "face_index":("INT", {"default": 0, "min": -1,"max":200, "step": 1, "display": "number"}),
+                        
+#                 #         }
+#                 }
+    
+#     RETURN_TYPES = ("DRIVING_VIDEO",)
+#     RETURN_NAMES = ("driving_video",)
+
+#     FUNCTION = "run"
+
+#     OUTPUT_NODE = True
+
+#     CATEGORY = "♾️Mixlab/Video"
+
+#     INPUT_IS_LIST = False
+#     OUTPUT_IS_LIST = (True,) #list 列表 [1,2,3]
+  
+#     def run(self,driving_video1, driving_video2 ):
+        
+#         return ([driving_video1, driving_video2],)
+    

 class LivePortraitNode:
    def __init__(self):
@@ -241,6 +316,7 @@ class LivePortraitNode:
                        },
                "optional":{  
                            "crop_info":("CROP_INFO", ),
+                            "driving_video_reverse_align":("BOOLEAN", {"default": True},),
                        }
                }
    
@@ -249,12 +325,12 @@ class LivePortraitNode:

    FUNCTION = "run"

-    CATEGORY = "♾️Mixlab/Video"
+    CATEGORY = "♾️Mixlab/Video/LivePortrait"

    INPUT_IS_LIST = True
    OUTPUT_IS_LIST = (False,False,) #list 列表 [1,2,3]
  
-    def run(self,source_image,driving_video,crop_info=None):
+    def run(self,source_image,driving_video,crop_info=None,driving_video_reverse_align=True):
        # print('#crop_info',crop_info,isinstance(crop_info, list))
        if crop_info!=None and isinstance(crop_info, list)==False:
            crop_info=[crop_info]
@@ -262,8 +338,6 @@ class LivePortraitNode:
        if crop_info!=None:
            crop_info=[ [c] for c in crop_info]

-        driving_video=driving_video[0]
-
        pil_image=tensor2pil(source_image[0])
        # Convert PIL image to NumPy array
        opencv_image = np.array(pil_image)
@@ -286,14 +360,6 @@ class LivePortraitNode:
        v_path=os.path.join(output_dir, v_file)
        output_path_concat=os.path.join(output_dir, v_file_concat)

-        args =  ArgumentConfig(
-            source_image=opencv_image,
-            driving_info=driving_video,
-            output_path=v_path,
-            output_path_concat=output_path_concat,
-            crop_info=crop_info, 
-        )
-        
        # print('##---------------------------------#landmark_runner_ckpt',landmark_runner_ckpt)
        live_portrait_pipeline = LivePortraitPipeline(
            inference_cfg=inference_cfg,
@@ -304,9 +370,40 @@ class LivePortraitNode:

        # run
        if crop_info==None:
+
+            args =  ArgumentConfig(
+                source_image=opencv_image,
+                driving_info=[driving_video[0]],
+                output_path=v_path,
+                output_path_concat=output_path_concat,
+                crop_info=crop_info, 
+            )
+
            live_portrait_pipeline.execute(args)
        else:
-            print('#executeForAll',len(crop_info))
+
+            if len(driving_video)!=len(crop_info):
+                last_d=driving_video[-1]
+                ds=[]
+                #todo 视频的帧要对齐
+                for i in range(len(crop_info)):
+                    if i in driving_video:
+                        ds.append(driving_video[i])
+                    else:
+                        ds.append(last_d)
+                driving_video=ds
+            
+                
+            args =  ArgumentConfig(
+                source_image=opencv_image,
+                driving_info=driving_video,
+                output_path=v_path,
+                output_path_concat=output_path_concat,
+                crop_info=crop_info, 
+                align_mode=driving_video_reverse_align==False
+            )
+
+            # print('#executeForAll',len(crop_info))
            live_portrait_pipeline.executeForAll(args)

        live_portrait_pipeline.live_portrait_wrapper=None
@@ -315,3 +412,98 @@ class LivePortraitNode:

        return (v_path,output_path_concat,)

+
+
+
+
+class LivePortraitVideoNode:
+    
+    @classmethod
+    def INPUT_TYPES(s):
+        
+        return {"required": {
+                        "source_image_batch": ("IMAGE",),
+                        "driving_video":("SCENE_VIDEO",),  
+                        },
+                # "optional":{  
+                #             "driving_video_reverse_align":("BOOLEAN", {"default": True},),
+                #         }
+                }
+    
+    RETURN_TYPES = ("SCENE_VIDEO","SCENE_VIDEO",)
+    RETURN_NAMES = ("video","video_concat",)
+
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/Video/LivePortrait"
+
+    INPUT_IS_LIST = True
+    OUTPUT_IS_LIST = (False,False,) #list 列表 [1,2,3]
+  
+    def run(self,source_image_batch,driving_video):
+        source_video=True
+        driving_video_reverse_align=True
+        print('#source_image_batch',source_image_batch)
+        source_image_batch=source_image_batch[0]
+        
+        #获取临时目录：temp
+        output_dir = folder_paths.get_temp_directory()
+
+        def count_live_portrait_mp4_files(output_dir: str) -> int:
+            count = 0
+            for filename in os.listdir(output_dir):
+                if filename.startswith('live_portrait_') and filename.endswith('.mp4'):
+                    count += 1
+            return count
+
+        counter=count_live_portrait_mp4_files(output_dir)
+        
+        v_file = f"live_portrait_{counter:05}.mp4"
+        v_file_concat = f"live_portrait_concat_{counter:05}.mp4"
+
+        v_path=os.path.join(output_dir, v_file)
+        output_path_concat=os.path.join(output_dir, v_file_concat)
+
+        # print('##---------------------------------#landmark_runner_ckpt',landmark_runner_ckpt)
+        live_portrait_pipeline = LivePortraitPipeline(
+            inference_cfg=inference_cfg,
+            crop_cfg=crop_cfg,
+            landmark_runner_ckpt=landmark_runner_ckpt,
+            insightface_pretrained_weights=insightface_pretrained_weights
+        )
+
+        # run
+        crop_info=None
+        if crop_info==None:
+
+            frames=[]
+
+            for i in range(len(source_image_batch)):
+
+                source_image=source_image_batch[i]
+
+                pil_image=tensor2pil(source_image)
+                # Convert PIL image to NumPy array
+                opencv_image = np.array(pil_image)
+
+                args =  ArgumentConfig(
+                    source_image=opencv_image,
+                    driving_info=[driving_video[0]],
+                    output_path=v_path,
+                    output_path_concat=output_path_concat,
+                    crop_info=crop_info, 
+                    source_video=source_video
+                )
+
+                video_frames, v_path, video_fps=live_portrait_pipeline.execute(args)
+
+                frames.append(video_frames[i])
+        
+        images2video(frames, wfp=v_path, fps=video_fps)
+
+        live_portrait_pipeline.live_portrait_wrapper=None
+
+        live_portrait_pipeline=None
+
+        return (v_path,output_path_concat,)
+
@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-liveportrait"
 description = "The ComfyUI version of [a/LivePortrait](https://github.com/KwaiVGI/LivePortrait)."
-version = "1.1.0"
+version = "1.3.1"
 license = "LICENSE"
 dependencies = ["numpy>=1.26.4", "opencv-python-headless", "imageio>=2.34.2", "lmdb>=1.4.1", "timm>=1.0.7", "rich>=13.7.1", "ffmpeg>=1.4", "onnxruntime-gpu>=1.18.0", "onnx>=1.16.1", "scikit-image>=0.24.0", "albumentations>=1.4.10", "matplotlib>=3.9.0", "imageio-ffmpeg>=0.5.1"]

@@ -10,6 +10,6 @@ Repository = "https://github.com/shadowcz007/comfyui-liveportrait"
 #  Used by Comfy Registry https://comfyregistry.org

 [tool.comfy]
-PublisherId = ""
+PublisherId = "shadowcz"
 DisplayName = "comfyui-liveportrait"
 Icon = ""
@@ -5,4 +5,5 @@ lmdb>=1.4.1
 timm>=1.0.7
 rich>=13.7.1
 albumentations>=1.4.10
-insightface
+insightface
+ultralytics
Author	SHA1	Message	Date
shadowcz007	cf3d6f703a	update	2024-08-03 14:15:34 +08:00
shadowcz007	6d12180c92	add expression_editor	2024-08-03 14:09:45 +08:00
shadow	12b0379e96	Update README.md	2024-07-14 14:44:11 +08:00
shadowcz007	85c378858f	Update pyproject.toml	2024-07-13 22:19:59 +08:00
shadowcz007	3eb74a7523	Update 全家福模式-workflow.json	2024-07-13 22:19:12 +08:00
shadowcz007	c3b5f4abf5	fixbug	2024-07-13 22:14:23 +08:00
shadowcz007	3956039ead	1.3.0	2024-07-10 18:14:40 +08:00
shadowcz007	e737cc3400	Update README.md	2024-07-10 18:13:50 +08:00
shadowcz007	71e03d5b0b	video-to-video	2024-07-10 18:09:29 +08:00
shadowcz007	239f943a65	remove rich	2024-07-10 17:06:03 +08:00
shadowcz007	00797ae5c4	1.2.0 Retargeting	2024-07-07 23:55:53 +08:00
shadowcz007	c2429e4fbd	不同脸指定不同动画	2024-07-06 20:30:11 +08:00
shadowcz007	808f7fdd7a	update	2024-07-06 17:28:55 +08:00
shadowcz007	41b34f6c77	Update pyproject.toml	2024-07-06 17:18:43 +08:00