Update pyproject.toml

README.md

@@ -1,6 +1,6 @@
 [LivePortrait](https://github.com/KwaiVGI/LivePortrait)的Comfyui版本。
 
-!! 支持多人脸 
+!! 支持多人脸 、不同的脸 指定不同的动画
 
 > [寻求帮助 Mixlab nodes discord](https://discord.gg/cXs9vZSqeK)
 
@@ -9,10 +9,21 @@
 
 
 ### workflow
+
+
+
 > 配合 [comfyui-mixlab-nodes](https://github.com/shadowcz007/comfyui-mixlab-nodes) 使用
 
+
+> 支持视频模式 [video-to-video](example/v2v-workflow.json)
+
+
 > 全家福
 
+[![alt text](example/1720268832629.png)](example/mul-workflow.json)
+[不同脸对应不同的驱动视频 Workflow JSON](example/mul-workflow.json)
+
+
 [![alt text](example/1720256574305.png)](example/全家福模式-workflow.json)
 
 [全家福 Workflow JSON](example/全家福模式-workflow.json)
@@ -47,7 +58,7 @@ face_index：指定要处理的人脸索引，默认值为 -1，表示处理所
 
 debug：开启或关闭调试模式。设置为 true 时，会输出调试图像以便查看人脸检测和裁剪区域；设置为 false 时，不输出调试图像。
 
-
+##### Retargeting 可开关eye lip是否驱动
 
 
 ### models

__init__.py

@@ -1,16 +1,20 @@
-from .nodes.live_portrait import LivePortraitNode,FaceCropInfo
+from .nodes.live_portrait import LivePortraitNode,FaceCropInfo,Retargeting,LivePortraitVideoNode
 
 
 NODE_CLASS_MAPPINGS = {
     "LivePortraitNode": LivePortraitNode, 
-    "FaceCropInfo":FaceCropInfo
+    "LivePortraitVideoNode":LivePortraitVideoNode,
+    "FaceCropInfo":FaceCropInfo,
+    "Retargeting":Retargeting
 }
 
 # dict = { "key":value }
 
 NODE_DISPLAY_NAME_MAPPINGS = {
    "LivePortraitNode":"Live Portrait",
-   "FaceCropInfo":"Face Crop Info"
+   "LivePortraitVideoNode":"Live Portrait for Video",
+   "FaceCropInfo":"Face Crop Info",
+   "Retargeting":"Retargeting"
 }
 
 # web ui的节点功能

example/v2v-workflow.json

@@ -0,0 +1,583 @@
+{
+  "last_node_id": 50,
+  "last_link_id": 82,
+  "nodes": [
+    {
+      "id": 22,
+      "type": "VideoCombine_Adv",
+      "pos": [
+        1722,
+        192
+      ],
+      "size": [
+        593.4324340820312,
+        809.4324340820312
+      ],
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_batch",
+          "type": "IMAGE",
+          "link": 65
+        }
+      ],
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoCombine_Adv"
+      },
+      "widgets_values": [
+        4,
+        0,
+        "Comfyui",
+        "video/h264-mp4",
+        false,
+        false,
+        false,
+        "/view?filename=Comfyui_00012_.mp4&subfolder=&type=temp&format=video%2Fh264-mp4"
+      ]
+    },
+    {
+      "id": 47,
+      "type": "ScenesNode_",
+      "pos": [
+        922,
+        796
+      ],
+      "size": {
+        "0": 299.8379821777344,
+        "1": 78
+      },
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "link": 70
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video frames (batch)",
+          "type": "IMAGE",
+          "links": [
+            69
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScenesNode_"
+      },
+      "widgets_values": [
+        10
+      ]
+    },
+    {
+      "id": 41,
+      "type": "ScenesNode_",
+      "pos": [
+        500,
+        852
+      ],
+      "size": {
+        "0": 308.9737854003906,
+        "1": 78
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "link": 61
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video frames (batch)",
+          "type": "IMAGE",
+          "links": [
+            62,
+            80
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScenesNode_"
+      },
+      "widgets_values": [
+        0
+      ]
+    },
+    {
+      "id": 42,
+      "type": "VideoCombine_Adv",
+      "pos": [
+        1310,
+        1156
+      ],
+      "size": [
+        483.5287170410156,
+        695.5287170410156
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_batch",
+          "type": "IMAGE",
+          "link": 62
+        },
+        {
+          "name": "frame_rate",
+          "type": "INT",
+          "link": 63,
+          "widget": {
+            "name": "frame_rate"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoCombine_Adv"
+      },
+      "widgets_values": [
+        4,
+        0,
+        "Comfyui",
+        "video/h264-mp4",
+        false,
+        false,
+        false,
+        "/view?filename=Comfyui_00010_.mp4&subfolder=&type=temp&format=video%2Fh264-mp4"
+      ]
+    },
+    {
+      "id": 46,
+      "type": "VideoCombine_Adv",
+      "pos": [
+        752,
+        1179
+      ],
+      "size": [
+        483.5287170410156,
+        699.5287170410156
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_batch",
+          "type": "IMAGE",
+          "link": 69
+        }
+      ],
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoCombine_Adv"
+      },
+      "widgets_values": [
+        4,
+        0,
+        "Comfyui",
+        "video/h264-mp4",
+        false,
+        false,
+        false,
+        "/view?filename=Comfyui_00011_.mp4&subfolder=&type=temp&format=video%2Fh264-mp4"
+      ]
+    },
+    {
+      "id": 39,
+      "type": "SwitchByIndex",
+      "pos": [
+        593,
+        294
+      ],
+      "size": {
+        "0": 315,
+        "1": 102
+      },
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "A",
+          "type": "*",
+          "link": 59
+        },
+        {
+          "name": "B",
+          "type": "*",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "list",
+          "type": "*",
+          "links": [
+            81
+          ],
+          "shape": 6,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SwitchByIndex"
+      },
+      "widgets_values": [
+        10,
+        "on"
+      ]
+    },
+    {
+      "id": 50,
+      "type": "LivePortraitVideoNode",
+      "pos": [
+        1216,
+        495
+      ],
+      "size": {
+        "0": 393,
+        "1": 46
+      },
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source_image_batch",
+          "type": "IMAGE",
+          "link": 80
+        },
+        {
+          "name": "driving_video",
+          "type": "SCENE_VIDEO",
+          "link": 81
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "SCENE_VIDEO",
+          "links": [
+            82
+          ],
+          "shape": 3
+        },
+        {
+          "name": "video_concat",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LivePortraitVideoNode"
+      }
+    },
+    {
+      "id": 43,
+      "type": "ScenesNode_",
+      "pos": [
+        1235,
+        326
+      ],
+      "size": {
+        "0": 299.8379821777344,
+        "1": 78
+      },
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "link": 82
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video frames (batch)",
+          "type": "IMAGE",
+          "links": [
+            65
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScenesNode_"
+      },
+      "widgets_values": [
+        0
+      ]
+    },
+    {
+      "id": 40,
+      "type": "LoadVideoAndSegment_",
+      "pos": [
+        140,
+        854
+      ],
+      "size": {
+        "0": 315,
+        "1": 214
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": [
+            61
+          ],
+          "shape": 6,
+          "slot_index": 0
+        },
+        {
+          "name": "scenes_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "links": [
+            63
+          ],
+          "shape": 3,
+          "slot_index": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideoAndSegment_"
+      },
+      "widgets_values": [
+        "WeChat_20240710164802.mp4",
+        24,
+        0,
+        null,
+        "video"
+      ]
+    },
+    {
+      "id": 1,
+      "type": "LoadVideoAndSegment_",
+      "pos": [
+        147,
+        207
+      ],
+      "size": {
+        "0": 324.6152038574219,
+        "1": 575.59765625
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": [
+            59,
+            70
+          ],
+          "shape": 6,
+          "slot_index": 0
+        },
+        {
+          "name": "scenes_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "links": [],
+          "shape": 3,
+          "slot_index": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideoAndSegment_"
+      },
+      "widgets_values": [
+        "d6.mp4",
+        24,
+        0,
+        null,
+        "video"
+      ]
+    }
+  ],
+  "links": [
+    [
+      59,
+      1,
+      0,
+      39,
+      0,
+      "*"
+    ],
+    [
+      61,
+      40,
+      0,
+      41,
+      0,
+      "SCENE_VIDEO"
+    ],
+    [
+      62,
+      41,
+      0,
+      42,
+      0,
+      "IMAGE"
+    ],
+    [
+      63,
+      40,
+      3,
+      42,
+      1,
+      "INT"
+    ],
+    [
+      65,
+      43,
+      0,
+      22,
+      0,
+      "IMAGE"
+    ],
+    [
+      69,
+      47,
+      0,
+      46,
+      0,
+      "IMAGE"
+    ],
+    [
+      70,
+      1,
+      0,
+      47,
+      0,
+      "SCENE_VIDEO"
+    ],
+    [
+      80,
+      41,
+      0,
+      50,
+      0,
+      "IMAGE"
+    ],
+    [
+      81,
+      39,
+      0,
+      50,
+      1,
+      "SCENE_VIDEO"
+    ],
+    [
+      82,
+      50,
+      0,
+      43,
+      0,
+      "SCENE_VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.7247295000000012,
+      "offset": [
+        10.69140058670257,
+        -97.28946419577976
+      ]
+    }
+  },
+  "version": 0.4
+}

nodes/LivePortrait/src/live_portrait_pipeline.py

@@ -12,7 +12,7 @@ import cv2
 import numpy as np
 import pickle,os
 import os.path as osp
-from rich.progress import track
+# from rich.progress import track
 
 # from .config.argument_config import ArgumentConfig
 from .config.inference_config import InferenceConfig
@@ -37,6 +37,26 @@ def add_index_to_filename(output_path, index):
     return new_output_path
 
 
+# 创建固定长度的list，不足的填充
+def create_drivings(elements, max_count, revert=False):
+    if not revert:
+        if max_count <= len(elements):
+            return elements[:max_count]
+        elif len(elements)>0:
+            return [elements[i % len(elements)] for i in range(max_count)]
+        else:
+            return [None for i in range(max_count)]
+    else:
+        if len(elements)==0:
+            return [None for i in range(max_count)]
+        extended_frames = elements + elements[-2:0:-1]  # 正向加反向中间部分
+        if max_count <= len(extended_frames):
+            return extended_frames[:max_count]
+        else:
+            return [extended_frames[i % len(extended_frames)] for i in range(max_count)]
+       
+
+
 def make_abs_path(fn):
     return osp.join(osp.dirname(osp.realpath(__file__)), fn)
 
@@ -55,6 +75,8 @@ class LivePortraitPipeline(object):
         # 增加人脸好的
         crop_info=args.crop_info
 
+        args.driving_info=args.driving_info[0]
+
         img_rgb = resize_to_limit(img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
         # log(f"Load source image from {args.source_image}")
         # todo 人脸检测并裁切 - 独立一个节点
@@ -116,8 +138,9 @@ class LivePortraitPipeline(object):
         R_d_0, x_d_0_info = None, None
 
         pbar = comfy.utils.ProgressBar(n_frames)
-
-        for i in track(range(n_frames), description='Animating...', total=n_frames):
+        print('Animating...',  n_frames)
+        for i in range(n_frames):
+        # track(range(n_frames), description='Animating...', total=n_frames):
     
             if is_video(args.driving_info):
                 # extract kp info by M
@@ -213,12 +236,12 @@ class LivePortraitPipeline(object):
 
         # save drived result
         wfp = args.output_path
-        if inference_cfg.flag_pasteback:
+        if inference_cfg.flag_pasteback and args.source_video==False:
             images2video(I_p_paste_lst, wfp=wfp, fps=video_fps)
         else:
             images2video(I_p_lst, wfp=wfp, fps=video_fps)
-
-        return wfp, wfp_concat
+            
+        return (I_p_paste_lst if inference_cfg.flag_pasteback else I_p_lst, wfp, video_fps)
 
     def executeForAll(self, args):
         inference_cfg = self.live_portrait_wrapper.cfg  # for convenience
@@ -227,41 +250,77 @@ class LivePortraitPipeline(object):
         img_rgb = args.source_image
         # 增加人脸好的
         crop_info_list = args.crop_info
+        # eye lip
+        __eye__s=[c[0]['__eye__'] for c in crop_info_list]
+        __lip__s=[c[0]['__lip__'] for c in crop_info_list]
+
+        # 对齐多个驱动视频的长度
+        align_mode=args.align_mode
 
         img_rgb = resize_to_limit(img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
         # log(f"Load source image from {args.source_image}")
         # todo 人脸检测并裁切 - 独立一个节点
         crop_info_list = [self.cropper.crop_single_image(img_rgb, src_face=crop_info) for crop_info in crop_info_list]
 
+        video_fps = cv2.VideoCapture(args.driving_info[0]).get(cv2.CAP_PROP_FPS)
 
-        video_fps = cv2.VideoCapture(args.driving_info).get(cv2.CAP_PROP_FPS)
+        driving_infos=args.driving_info
 
         ######## process driving info ########
-        self.driving_lmk_lst=None
-        self.n_frames=None
-        if is_video(args.driving_info):
-            log(f"Load from video file (mp4 mov avi etc...): {args.driving_info}")
-            # TODO: 这里track一下驱动视频 -> 构建模板
-            driving_rgb_lst = load_driving_info(args.driving_info)
-            driving_rgb_lst_256 = [cv2.resize(_, (256, 256)) for _ in driving_rgb_lst]
-            I_d_lst = self.live_portrait_wrapper.prepare_driving_videos(driving_rgb_lst_256)
-            self.n_frames = I_d_lst.shape[0]
-            if inference_cfg.flag_eye_retargeting or inference_cfg.flag_lip_retargeting:
-                self.driving_lmk_lst = self.cropper.get_retargeting_lmk_info(driving_rgb_lst)
-                # input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)
-        # elif is_template(args.driving_info):
-        #     log(f"Load from video templates {args.driving_info}")
-        #     with open(args.driving_info, 'rb') as f:
-        #         template_lst, driving_lmk_lst = pickle.load(f)
-        #     n_frames = template_lst[0]['n_frames']
-        #     # input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)
-        # else:
-        #     raise Exception("Unsupported driving types!")
-        #########################################
-        print('#driving_lmk_lst',self.driving_lmk_lst)
+        driving_lmk_lst_s=[]
+        n_frames_s=[]
+        I_d_lst_s=[]
+
+        pbar = comfy.utils.ProgressBar(len(driving_infos))
+        for  z in range(len(driving_infos)):
+
+            driving_info=driving_infos[z]
+            crop_info=crop_info_list[z]
+            # print('###',z,len(driving_infos),len(crop_info_list))
+            # print('#crop_info_list[z]',crop_info_list[z])
+            __eye__=__eye__s[z]
+            __lip__=__lip__s[z]
+
+            if is_video(driving_info):
+                log(f"Load from video file (mp4 mov avi etc...): {driving_info}")
+                # TODO: 这里track一下驱动视频 -> 构建模板
+                driving_rgb_lst = load_driving_info(driving_info)
+                driving_rgb_lst_256 = [cv2.resize(_, (256, 256)) for _ in driving_rgb_lst]
+                I_d_lst = self.live_portrait_wrapper.prepare_driving_videos(driving_rgb_lst_256)
+                n_frames = I_d_lst.shape[0]
+                
+                I_d_lst_s.append(I_d_lst)
+                n_frames_s.append(n_frames)
+
+                source_lmk = crop_info['lmk_crop']
+                if __eye__ or __lip__:
+                    driving_lmk_lst = self.cropper.get_retargeting_lmk_info(driving_rgb_lst)
+                    driving_lmk_lst_s.append(driving_lmk_lst)
+                    input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(
+                        source_lmk, 
+                        driving_lmk_lst
+                        )
+            # elif is_template(args.driving_info):
+            #     log(f"Load from video templates {args.driving_info}")
+            #     with open(args.driving_info, 'rb') as f:
+            #         template_lst, driving_lmk_lst = pickle.load(f)
+            #     n_frames = template_lst[0]['n_frames']
+            #     # input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)
+            # else:
+            #     raise Exception("Unsupported driving types!")
+            #########################################
+            # print('#driving_lmk_lst',self.driving_lmk_lst)
+            pbar.update(1)
+
+        # 对齐
+        max_n_frames = max(n_frames_s)
+        n_frames_s=[max_n_frames for i in n_frames_s]
+        driving_lmk_lst_s= [create_drivings(d,max_n_frames,align_mode) for d in driving_lmk_lst_s]
+        I_d_lst_s= [create_drivings(i,max_n_frames,align_mode) for i in I_d_lst_s]
+
 
         # 原图片---视频帧
-        img_rgbs=[img_rgb for i in range(self.n_frames)]
+        img_rgbs=[img_rgb for i in range(n_frames_s[0])]
         
 
         for index in range(len(crop_info_list)):
@@ -269,6 +328,10 @@ class LivePortraitPipeline(object):
             crop_info=crop_info_list[index]
             # 地一张脸
 
+            __eye__=__eye__s[index]
+            __lip__=__lip__s[index]
+            
+            print('#crop_info',crop_info.keys())
             source_lmk = crop_info['lmk_crop']
             img_crop, img_crop_256x256 = crop_info['img_crop'], crop_info['img_crop_256x256']
             if inference_cfg.flag_do_crop:
@@ -289,8 +352,13 @@ class LivePortraitPipeline(object):
                 else:
                     lip_delta_before_animation = self.live_portrait_wrapper.retarget_lip(x_s, combined_lip_ratio_tensor_before_animation)
             ############################################
-            if self.driving_lmk_lst!=None:
-                input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, self.driving_lmk_lst)
+
+            # 多个驱动视频
+            if 0 <= index < len(driving_lmk_lst_s):
+                driving_lmk_lst=driving_lmk_lst_s[index]
+                
+                if driving_lmk_lst!=None:
+                    input_eye_ratio_lst, input_lip_ratio_lst = self.live_portrait_wrapper.calc_retargeting_ratio(source_lmk, driving_lmk_lst)
 
             ######## prepare for pasteback ########
             if inference_cfg.flag_pasteback:
@@ -304,11 +372,19 @@ class LivePortraitPipeline(object):
             I_p_lst = []
             R_d_0, x_d_0_info = None, None
 
-            pbar = comfy.utils.ProgressBar(self.n_frames)
+            # 多个驱动视频
+            n_frames=n_frames_s[index]
+            driving_info=driving_infos[index]
+            I_d_lst=I_d_lst_s[index]
 
-            for i in track(range(self.n_frames), description='Animating...', total=self.n_frames):
 
-                if is_video(args.driving_info):
+            pbar = comfy.utils.ProgressBar(n_frames)
+
+            print('Animating...',  n_frames)
+            for i in range(n_frames):
+            # track(range(n_frames), description='Animating...', total=n_frames):
+
+                if is_video(driving_info):
                     # extract kp info by M
                     I_d_i = I_d_lst[i]
                     x_d_i_info = self.live_portrait_wrapper.get_kp_info(I_d_i)
@@ -333,13 +409,13 @@ class LivePortraitPipeline(object):
                 x_d_i_new = scale_new * (x_c_s @ R_new + delta_new) + t_new
 
                 # Algorithm 1:
-                if not inference_cfg.flag_stitching and not inference_cfg.flag_eye_retargeting and not inference_cfg.flag_lip_retargeting:
+                if not inference_cfg.flag_stitching and not __eye__ and not __lip__:
                     # without stitching or retargeting
                     if inference_cfg.flag_lip_zero:
                         x_d_i_new += lip_delta_before_animation.reshape(-1, x_s.shape[1], 3)
                     else:
                         pass
-                elif inference_cfg.flag_stitching and not inference_cfg.flag_eye_retargeting and not inference_cfg.flag_lip_retargeting:
+                elif inference_cfg.flag_stitching and not __eye__ and not __lip__:
                     # with stitching and without retargeting
                     if inference_cfg.flag_lip_zero:
                         x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new) + lip_delta_before_animation.reshape(-1, x_s.shape[1], 3)
@@ -347,12 +423,12 @@ class LivePortraitPipeline(object):
                         x_d_i_new = self.live_portrait_wrapper.stitching(x_s, x_d_i_new)
                 else:
                     eyes_delta, lip_delta = None, None
-                    if inference_cfg.flag_eye_retargeting:
+                    if __eye__:
                         c_d_eyes_i = input_eye_ratio_lst[i]
                         combined_eye_ratio_tensor = self.live_portrait_wrapper.calc_combined_eye_ratio(c_d_eyes_i, source_lmk)
                         # ∆_eyes,i = R_eyes(x_s; c_s,eyes, c_d,eyes,i)
                         eyes_delta = self.live_portrait_wrapper.retarget_eye(x_s, combined_eye_ratio_tensor)
-                    if inference_cfg.flag_lip_retargeting:
+                    if __lip__:
                         c_d_lip_i = input_lip_ratio_lst[i]
                         combined_lip_ratio_tensor = self.live_portrait_wrapper.calc_combined_lip_ratio(c_d_lip_i, source_lmk)
                         # ∆_lip,i = R_lip(x_s; c_s,lip, c_d,lip,i)
@@ -405,8 +481,8 @@ class LivePortraitPipeline(object):
 
         # save drived result
         wfp = args.output_path
-        if inference_cfg.flag_pasteback:
+        if inference_cfg.flag_pasteback and args.source_video==False:
             images2video(I_p_paste_lst, wfp=wfp, fps=video_fps)
         
-        return wfp
+        return (I_p_paste_lst, wfp, video_fps)
 

nodes/LivePortrait/src/template_maker.py

@@ -8,7 +8,7 @@ import os
 import cv2
 import numpy as np
 import pickle
-from rich.progress import track
+# from rich.progress import track
 from .utils.cropper import Cropper
 
 from .utils.io import load_driving_info
@@ -40,8 +40,8 @@ class TemplateMaker:
 
         templates = []
 
-
-        for i in track(range(n_frames), description='Making templates...', total=n_frames):
+        print('# Making templates...', n_frames)
+        for i in range(n_frames):
             I_d_i = I_d_lst[i]
             x_d_i_info = self.live_portrait_wrapper.get_kp_info(I_d_i)
             R_d_i = get_rotation_matrix(x_d_i_info['pitch'], x_d_i_info['yaw'], x_d_i_info['roll'])

nodes/LivePortrait/src/utils/cropper.py

@@ -123,7 +123,7 @@ class Cropper(object):
         elif isinstance(obj, np.ndarray):
             img_rgb = obj
 
-        print('#crop_single_image',direction,face_index,src_face)
+        # print('#crop_single_image',direction,face_index,src_face)
 
         if src_face==None:
             src_face = self.face_analysis_wrapper.get(

nodes/LivePortrait/src/utils/video.py

@@ -10,7 +10,7 @@ import subprocess
 import imageio
 import cv2
 
-from rich.progress import track
+# from rich.progress import track
 from .helper import prefix
 from .rprint import rprint as print
 
@@ -35,7 +35,8 @@ def images2video(images, wfp, **kwargs):
     )
 
     n = len(images)
-    for i in track(range(n), description='writing', transient=True):
+    print('writing',n)
+    for i in range(n):
         if image_mode.lower() == 'bgr':
             writer.append_data(images[i][..., ::-1])
         else:
@@ -83,7 +84,9 @@ def blend(img: np.ndarray, mask: np.ndarray, background_color=(255, 255, 255)):
 def concat_frames(I_p_lst, driving_rgb_lst, img_rgb):
     # TODO: add more concat style, e.g., left-down corner driving
     out_lst = []
-    for idx, _ in track(enumerate(I_p_lst), total=len(I_p_lst), description='Concatenating result...'):
+    print('Concatenating result...',len(I_p_lst))
+    for idx, _ in enumerate(I_p_lst):
+    # track(enumerate(I_p_lst), total=len(I_p_lst), description='Concatenating result...'):
         source_image_drived = I_p_lst[idx]
         image_drive = driving_rgb_lst[idx]
 

nodes/live_portrait.py

@@ -28,7 +28,7 @@ def pil2tensor(image):
     return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0)
 
 
-
+from .LivePortrait.src.utils.video import images2video
 from .LivePortrait.src.live_portrait_pipeline import LivePortraitPipeline
 
 def get_model_dir(m):
@@ -44,12 +44,14 @@ class ArgumentConfig:
                     driving_info,
                     output_path='animations/v.mp4',
                     output_path_concat="",
+                    source_video=False,
                     device_id=0, 
                     crop_info =None,
                     face_index=0,
+                    align_mode=True,
                     flag_lip_zero=True,
                     flag_eye_retargeting=False,
-                    flag_lip_retargeting=False,
+                    flag_lip_retargeting=False, 
                     flag_stitching=True,
                     flag_relative=True,
                     flag_pasteback=True,
@@ -63,15 +65,17 @@ class ArgumentConfig:
                     share=False,
                     server_name='0.0.0.0'):
         self.source_image = source_image
+        self.source_video=source_video
         self.driving_info = driving_info
         self.output_path = output_path
         self.output_path_concat=output_path_concat
         self.crop_info=crop_info
         self.face_index=face_index
+        self.align_mode=align_mode
         self.device_id = device_id
         self.flag_lip_zero = flag_lip_zero
         self.flag_eye_retargeting = flag_eye_retargeting
-        self.flag_lip_retargeting = flag_lip_retargeting
+        self.flag_lip_retargeting = flag_lip_retargeting 
         self.flag_stitching = flag_stitching
         self.flag_relative = flag_relative
         self.flag_pasteback = flag_pasteback
@@ -172,8 +176,6 @@ crop_cfg = CropConfig()
 
 # 人脸检测并裁切
 class FaceCropInfo:
-    def __init__(self):
-        self.speaker = None
     @classmethod
     def INPUT_TYPES(s):
         
@@ -194,7 +196,7 @@ class FaceCropInfo:
 
     OUTPUT_NODE = True
 
-    CATEGORY = "♾️Mixlab/Video"
+    CATEGORY = "♾️Mixlab/Video/LivePortrait"
 
     INPUT_IS_LIST = False
     OUTPUT_IS_LIST = (True,False,) #list 列表 [1,2,3]
@@ -225,9 +227,82 @@ class FaceCropInfo:
             #只输出一张 [face]
             crop_info=[crop_info[face_index]]
 
-        return (crop_info,debug_image,)
+
+        result=[]
+
+        for c in crop_info:
+            c['__eye__']=True
+            c['__lip__']=True
+            result.append(c)
+
+        return (result,debug_image,)
 
 
+# 人脸检测并裁切
+class Retargeting:
+    @classmethod
+    def INPUT_TYPES(s):
+        
+        return {"required": {
+                        "crop_info": ("CROP_INFO",),
+                        },
+                "optional":{ 
+                           "lip":("BOOLEAN", {"default": True},),
+                           "eye":("BOOLEAN", {"default": True},),
+                        }
+                }
+    
+    RETURN_TYPES = ("CROP_INFO",)
+    RETURN_NAMES = ("crop_info",)
+
+    FUNCTION = "run"
+
+    OUTPUT_NODE = True
+
+    CATEGORY = "♾️Mixlab/Video/LivePortrait"
+
+    INPUT_IS_LIST = False
+    OUTPUT_IS_LIST = (False,) #list 列表 [1,2,3]
+  
+    def run(self,crop_info,lip=True,eye=True):
+        crop_info['__eye__']=eye
+        crop_info['__lip__']=lip
+        return (crop_info,)
+
+
+
+# 驱动模板制作
+# class DriveVideoNode:
+
+#     @classmethod
+#     def INPUT_TYPES(s):
+        
+#         return {"required": {
+#                         "driving_video1":("SCENE_VIDEO",),
+#                         "driving_video2":("SCENE_VIDEO",),
+#                         },
+#                 # "optional":{ 
+#                 #         "face_index":("INT", {"default": 0, "min": -1,"max":200, "step": 1, "display": "number"}),
+                        
+#                 #         }
+#                 }
+    
+#     RETURN_TYPES = ("DRIVING_VIDEO",)
+#     RETURN_NAMES = ("driving_video",)
+
+#     FUNCTION = "run"
+
+#     OUTPUT_NODE = True
+
+#     CATEGORY = "♾️Mixlab/Video"
+
+#     INPUT_IS_LIST = False
+#     OUTPUT_IS_LIST = (True,) #list 列表 [1,2,3]
+  
+#     def run(self,driving_video1, driving_video2 ):
+        
+#         return ([driving_video1, driving_video2],)
+    
 
 class LivePortraitNode:
     def __init__(self):
@@ -241,6 +316,7 @@ class LivePortraitNode:
                         },
                 "optional":{  
                             "crop_info":("CROP_INFO", ),
+                            "driving_video_reverse_align":("BOOLEAN", {"default": True},),
                         }
                 }
     
@@ -249,12 +325,12 @@ class LivePortraitNode:
 
     FUNCTION = "run"
 
-    CATEGORY = "♾️Mixlab/Video"
+    CATEGORY = "♾️Mixlab/Video/LivePortrait"
 
     INPUT_IS_LIST = True
     OUTPUT_IS_LIST = (False,False,) #list 列表 [1,2,3]
   
-    def run(self,source_image,driving_video,crop_info=None):
+    def run(self,source_image,driving_video,crop_info=None,driving_video_reverse_align=True):
         # print('#crop_info',crop_info,isinstance(crop_info, list))
         if crop_info!=None and isinstance(crop_info, list)==False:
             crop_info=[crop_info]
@@ -262,8 +338,6 @@ class LivePortraitNode:
         if crop_info!=None:
             crop_info=[ [c] for c in crop_info]
 
-        driving_video=driving_video[0]
-
         pil_image=tensor2pil(source_image[0])
         # Convert PIL image to NumPy array
         opencv_image = np.array(pil_image)
@@ -286,14 +360,6 @@ class LivePortraitNode:
         v_path=os.path.join(output_dir, v_file)
         output_path_concat=os.path.join(output_dir, v_file_concat)
 
-        args =  ArgumentConfig(
-            source_image=opencv_image,
-            driving_info=driving_video,
-            output_path=v_path,
-            output_path_concat=output_path_concat,
-            crop_info=crop_info, 
-        )
-        
         # print('##---------------------------------#landmark_runner_ckpt',landmark_runner_ckpt)
         live_portrait_pipeline = LivePortraitPipeline(
             inference_cfg=inference_cfg,
@@ -304,9 +370,40 @@ class LivePortraitNode:
 
         # run
         if crop_info==None:
+
+            args =  ArgumentConfig(
+                source_image=opencv_image,
+                driving_info=[driving_video[0]],
+                output_path=v_path,
+                output_path_concat=output_path_concat,
+                crop_info=crop_info, 
+            )
+
             live_portrait_pipeline.execute(args)
         else:
-            print('#executeForAll',len(crop_info))
+
+            if len(driving_video)!=len(crop_info):
+                last_d=driving_video[-1]
+                ds=[]
+                #todo 视频的帧要对齐
+                for i in range(len(crop_info)):
+                    if i in driving_video:
+                        ds.append(driving_video[i])
+                    else:
+                        ds.append(last_d)
+                driving_video=ds
+            
+                
+            args =  ArgumentConfig(
+                source_image=opencv_image,
+                driving_info=driving_video,
+                output_path=v_path,
+                output_path_concat=output_path_concat,
+                crop_info=crop_info, 
+                align_mode=driving_video_reverse_align==False
+            )
+
+            # print('#executeForAll',len(crop_info))
             live_portrait_pipeline.executeForAll(args)
 
         live_portrait_pipeline.live_portrait_wrapper=None
@@ -315,3 +412,98 @@ class LivePortraitNode:
 
         return (v_path,output_path_concat,)
 
+
+
+
+
+class LivePortraitVideoNode:
+    
+    @classmethod
+    def INPUT_TYPES(s):
+        
+        return {"required": {
+                        "source_image_batch": ("IMAGE",),
+                        "driving_video":("SCENE_VIDEO",),  
+                        },
+                # "optional":{  
+                #             "driving_video_reverse_align":("BOOLEAN", {"default": True},),
+                #         }
+                }
+    
+    RETURN_TYPES = ("SCENE_VIDEO","SCENE_VIDEO",)
+    RETURN_NAMES = ("video","video_concat",)
+
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/Video/LivePortrait"
+
+    INPUT_IS_LIST = True
+    OUTPUT_IS_LIST = (False,False,) #list 列表 [1,2,3]
+  
+    def run(self,source_image_batch,driving_video):
+        source_video=True
+        driving_video_reverse_align=True
+        print('#source_image_batch',source_image_batch)
+        source_image_batch=source_image_batch[0]
+        
+        #获取临时目录：temp
+        output_dir = folder_paths.get_temp_directory()
+
+        def count_live_portrait_mp4_files(output_dir: str) -> int:
+            count = 0
+            for filename in os.listdir(output_dir):
+                if filename.startswith('live_portrait_') and filename.endswith('.mp4'):
+                    count += 1
+            return count
+
+        counter=count_live_portrait_mp4_files(output_dir)
+        
+        v_file = f"live_portrait_{counter:05}.mp4"
+        v_file_concat = f"live_portrait_concat_{counter:05}.mp4"
+
+        v_path=os.path.join(output_dir, v_file)
+        output_path_concat=os.path.join(output_dir, v_file_concat)
+
+        # print('##---------------------------------#landmark_runner_ckpt',landmark_runner_ckpt)
+        live_portrait_pipeline = LivePortraitPipeline(
+            inference_cfg=inference_cfg,
+            crop_cfg=crop_cfg,
+            landmark_runner_ckpt=landmark_runner_ckpt,
+            insightface_pretrained_weights=insightface_pretrained_weights
+        )
+
+        # run
+        crop_info=None
+        if crop_info==None:
+
+            frames=[]
+
+            for i in range(len(source_image_batch)):
+
+                source_image=source_image_batch[i]
+
+                pil_image=tensor2pil(source_image)
+                # Convert PIL image to NumPy array
+                opencv_image = np.array(pil_image)
+
+                args =  ArgumentConfig(
+                    source_image=opencv_image,
+                    driving_info=[driving_video[0]],
+                    output_path=v_path,
+                    output_path_concat=output_path_concat,
+                    crop_info=crop_info, 
+                    source_video=source_video
+                )
+
+                video_frames, v_path, video_fps=live_portrait_pipeline.execute(args)
+
+                frames.append(video_frames[i])
+        
+        images2video(frames, wfp=v_path, fps=video_fps)
+
+        live_portrait_pipeline.live_portrait_wrapper=None
+
+        live_portrait_pipeline=None
+
+        return (v_path,output_path_concat,)
+

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-liveportrait"
 description = "The ComfyUI version of [a/LivePortrait](https://github.com/KwaiVGI/LivePortrait)."
-version = "1.1.0"
+version = "1.3.1"
 license = "LICENSE"
 dependencies = ["numpy>=1.26.4", "opencv-python-headless", "imageio>=2.34.2", "lmdb>=1.4.1", "timm>=1.0.7", "rich>=13.7.1", "ffmpeg>=1.4", "onnxruntime-gpu>=1.18.0", "onnx>=1.16.1", "scikit-image>=0.24.0", "albumentations>=1.4.10", "matplotlib>=3.9.0", "imageio-ffmpeg>=0.5.1"]
 
@@ -10,6 +10,6 @@ Repository = "https://github.com/shadowcz007/comfyui-liveportrait"
 #  Used by Comfy Registry https://comfyregistry.org
 
 [tool.comfy]
-PublisherId = ""
+PublisherId = "shadowcz"
 DisplayName = "comfyui-liveportrait"
 Icon = ""