1.3.0

README.md

@@ -9,8 +9,15 @@
 
 
 ### workflow
+
+
+
 > 配合 [comfyui-mixlab-nodes](https://github.com/shadowcz007/comfyui-mixlab-nodes) 使用
 
+
+> 支持视频模式 [video-to-video](example/v2v-workflow.json)
+
+
 > 全家福
 
 [![alt text](example/1720268832629.png)](example/mul-workflow.json)

__init__.py

@@ -1,8 +1,9 @@
-from .nodes.live_portrait import LivePortraitNode,FaceCropInfo,Retargeting
+from .nodes.live_portrait import LivePortraitNode,FaceCropInfo,Retargeting,LivePortraitVideoNode
 
 
 NODE_CLASS_MAPPINGS = {
     "LivePortraitNode": LivePortraitNode, 
+    "LivePortraitVideoNode":LivePortraitVideoNode,
     "FaceCropInfo":FaceCropInfo,
     "Retargeting":Retargeting
 }
@@ -11,6 +12,7 @@ NODE_CLASS_MAPPINGS = {
 
 NODE_DISPLAY_NAME_MAPPINGS = {
    "LivePortraitNode":"Live Portrait",
+   "LivePortraitVideoNode":"Live Portrait for Video",
    "FaceCropInfo":"Face Crop Info",
    "Retargeting":"Retargeting"
 }

example/v2v-workflow.json

@@ -0,0 +1,583 @@
+{
+  "last_node_id": 50,
+  "last_link_id": 82,
+  "nodes": [
+    {
+      "id": 22,
+      "type": "VideoCombine_Adv",
+      "pos": [
+        1722,
+        192
+      ],
+      "size": [
+        593.4324340820312,
+        809.4324340820312
+      ],
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_batch",
+          "type": "IMAGE",
+          "link": 65
+        }
+      ],
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoCombine_Adv"
+      },
+      "widgets_values": [
+        4,
+        0,
+        "Comfyui",
+        "video/h264-mp4",
+        false,
+        false,
+        false,
+        "/view?filename=Comfyui_00012_.mp4&subfolder=&type=temp&format=video%2Fh264-mp4"
+      ]
+    },
+    {
+      "id": 47,
+      "type": "ScenesNode_",
+      "pos": [
+        922,
+        796
+      ],
+      "size": {
+        "0": 299.8379821777344,
+        "1": 78
+      },
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "link": 70
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video frames (batch)",
+          "type": "IMAGE",
+          "links": [
+            69
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScenesNode_"
+      },
+      "widgets_values": [
+        10
+      ]
+    },
+    {
+      "id": 41,
+      "type": "ScenesNode_",
+      "pos": [
+        500,
+        852
+      ],
+      "size": {
+        "0": 308.9737854003906,
+        "1": 78
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "link": 61
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video frames (batch)",
+          "type": "IMAGE",
+          "links": [
+            62,
+            80
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScenesNode_"
+      },
+      "widgets_values": [
+        0
+      ]
+    },
+    {
+      "id": 42,
+      "type": "VideoCombine_Adv",
+      "pos": [
+        1310,
+        1156
+      ],
+      "size": [
+        483.5287170410156,
+        695.5287170410156
+      ],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_batch",
+          "type": "IMAGE",
+          "link": 62
+        },
+        {
+          "name": "frame_rate",
+          "type": "INT",
+          "link": 63,
+          "widget": {
+            "name": "frame_rate"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoCombine_Adv"
+      },
+      "widgets_values": [
+        4,
+        0,
+        "Comfyui",
+        "video/h264-mp4",
+        false,
+        false,
+        false,
+        "/view?filename=Comfyui_00010_.mp4&subfolder=&type=temp&format=video%2Fh264-mp4"
+      ]
+    },
+    {
+      "id": 46,
+      "type": "VideoCombine_Adv",
+      "pos": [
+        752,
+        1179
+      ],
+      "size": [
+        483.5287170410156,
+        699.5287170410156
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_batch",
+          "type": "IMAGE",
+          "link": 69
+        }
+      ],
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VideoCombine_Adv"
+      },
+      "widgets_values": [
+        4,
+        0,
+        "Comfyui",
+        "video/h264-mp4",
+        false,
+        false,
+        false,
+        "/view?filename=Comfyui_00011_.mp4&subfolder=&type=temp&format=video%2Fh264-mp4"
+      ]
+    },
+    {
+      "id": 39,
+      "type": "SwitchByIndex",
+      "pos": [
+        593,
+        294
+      ],
+      "size": {
+        "0": 315,
+        "1": 102
+      },
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "A",
+          "type": "*",
+          "link": 59
+        },
+        {
+          "name": "B",
+          "type": "*",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "list",
+          "type": "*",
+          "links": [
+            81
+          ],
+          "shape": 6,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "SwitchByIndex"
+      },
+      "widgets_values": [
+        10,
+        "on"
+      ]
+    },
+    {
+      "id": 50,
+      "type": "LivePortraitVideoNode",
+      "pos": [
+        1216,
+        495
+      ],
+      "size": {
+        "0": 393,
+        "1": 46
+      },
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "source_image_batch",
+          "type": "IMAGE",
+          "link": 80
+        },
+        {
+          "name": "driving_video",
+          "type": "SCENE_VIDEO",
+          "link": 81
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video",
+          "type": "SCENE_VIDEO",
+          "links": [
+            82
+          ],
+          "shape": 3
+        },
+        {
+          "name": "video_concat",
+          "type": "SCENE_VIDEO",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LivePortraitVideoNode"
+      }
+    },
+    {
+      "id": 43,
+      "type": "ScenesNode_",
+      "pos": [
+        1235,
+        326
+      ],
+      "size": {
+        "0": 299.8379821777344,
+        "1": 78
+      },
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "link": 82
+        }
+      ],
+      "outputs": [
+        {
+          "name": "video frames (batch)",
+          "type": "IMAGE",
+          "links": [
+            65
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ScenesNode_"
+      },
+      "widgets_values": [
+        0
+      ]
+    },
+    {
+      "id": 40,
+      "type": "LoadVideoAndSegment_",
+      "pos": [
+        140,
+        854
+      ],
+      "size": {
+        "0": 315,
+        "1": 214
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": [
+            61
+          ],
+          "shape": 6,
+          "slot_index": 0
+        },
+        {
+          "name": "scenes_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "links": [
+            63
+          ],
+          "shape": 3,
+          "slot_index": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideoAndSegment_"
+      },
+      "widgets_values": [
+        "WeChat_20240710164802.mp4",
+        24,
+        0,
+        null,
+        "video"
+      ]
+    },
+    {
+      "id": 1,
+      "type": "LoadVideoAndSegment_",
+      "pos": [
+        147,
+        207
+      ],
+      "size": {
+        "0": 324.6152038574219,
+        "1": 575.59765625
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "scenes_video",
+          "type": "SCENE_VIDEO",
+          "links": [
+            59,
+            70
+          ],
+          "shape": 6,
+          "slot_index": 0
+        },
+        {
+          "name": "scenes_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "fps",
+          "type": "INT",
+          "links": [],
+          "shape": 3,
+          "slot_index": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideoAndSegment_"
+      },
+      "widgets_values": [
+        "d6.mp4",
+        24,
+        0,
+        null,
+        "video"
+      ]
+    }
+  ],
+  "links": [
+    [
+      59,
+      1,
+      0,
+      39,
+      0,
+      "*"
+    ],
+    [
+      61,
+      40,
+      0,
+      41,
+      0,
+      "SCENE_VIDEO"
+    ],
+    [
+      62,
+      41,
+      0,
+      42,
+      0,
+      "IMAGE"
+    ],
+    [
+      63,
+      40,
+      3,
+      42,
+      1,
+      "INT"
+    ],
+    [
+      65,
+      43,
+      0,
+      22,
+      0,
+      "IMAGE"
+    ],
+    [
+      69,
+      47,
+      0,
+      46,
+      0,
+      "IMAGE"
+    ],
+    [
+      70,
+      1,
+      0,
+      47,
+      0,
+      "SCENE_VIDEO"
+    ],
+    [
+      80,
+      41,
+      0,
+      50,
+      0,
+      "IMAGE"
+    ],
+    [
+      81,
+      39,
+      0,
+      50,
+      1,
+      "SCENE_VIDEO"
+    ],
+    [
+      82,
+      50,
+      0,
+      43,
+      0,
+      "SCENE_VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.7247295000000012,
+      "offset": [
+        10.69140058670257,
+        -97.28946419577976
+      ]
+    }
+  },
+  "version": 0.4
+}

nodes/LivePortrait/src/live_portrait_pipeline.py

@@ -12,7 +12,7 @@ import cv2
 import numpy as np
 import pickle,os
 import os.path as osp
-from rich.progress import track
+# from rich.progress import track
 
 # from .config.argument_config import ArgumentConfig
 from .config.inference_config import InferenceConfig
@@ -138,8 +138,9 @@ class LivePortraitPipeline(object):
         R_d_0, x_d_0_info = None, None
 
         pbar = comfy.utils.ProgressBar(n_frames)
-
-        for i in track(range(n_frames), description='Animating...', total=n_frames):
+        print('Animating...',  n_frames)
+        for i in range(n_frames):
+        # track(range(n_frames), description='Animating...', total=n_frames):
     
             if is_video(args.driving_info):
                 # extract kp info by M
@@ -235,12 +236,12 @@ class LivePortraitPipeline(object):
 
         # save drived result
         wfp = args.output_path
-        if inference_cfg.flag_pasteback:
+        if inference_cfg.flag_pasteback and args.source_video==False:
             images2video(I_p_paste_lst, wfp=wfp, fps=video_fps)
         else:
             images2video(I_p_lst, wfp=wfp, fps=video_fps)
-
-        return wfp, wfp_concat
+            
+        return (I_p_paste_lst if inference_cfg.flag_pasteback else I_p_lst, wfp, video_fps)
 
     def executeForAll(self, args):
         inference_cfg = self.live_portrait_wrapper.cfg  # for convenience
@@ -379,7 +380,9 @@ class LivePortraitPipeline(object):
 
             pbar = comfy.utils.ProgressBar(n_frames)
 
-            for i in track(range(n_frames), description='Animating...', total=n_frames):
+            print('Animating...',  n_frames)
+            for i in range(n_frames):
+            # track(range(n_frames), description='Animating...', total=n_frames):
 
                 if is_video(driving_info):
                     # extract kp info by M
@@ -478,8 +481,8 @@ class LivePortraitPipeline(object):
 
         # save drived result
         wfp = args.output_path
-        if inference_cfg.flag_pasteback:
+        if inference_cfg.flag_pasteback and args.source_video==False:
             images2video(I_p_paste_lst, wfp=wfp, fps=video_fps)
         
-        return wfp
+        return (I_p_paste_lst, wfp, video_fps)
 

nodes/LivePortrait/src/template_maker.py

@@ -8,7 +8,7 @@ import os
 import cv2
 import numpy as np
 import pickle
-from rich.progress import track
+# from rich.progress import track
 from .utils.cropper import Cropper
 
 from .utils.io import load_driving_info
@@ -40,8 +40,8 @@ class TemplateMaker:
 
         templates = []
 
-
-        for i in track(range(n_frames), description='Making templates...', total=n_frames):
+        print('# Making templates...', n_frames)
+        for i in range(n_frames):
             I_d_i = I_d_lst[i]
             x_d_i_info = self.live_portrait_wrapper.get_kp_info(I_d_i)
             R_d_i = get_rotation_matrix(x_d_i_info['pitch'], x_d_i_info['yaw'], x_d_i_info['roll'])

nodes/LivePortrait/src/utils/video.py

@@ -10,7 +10,7 @@ import subprocess
 import imageio
 import cv2
 
-from rich.progress import track
+# from rich.progress import track
 from .helper import prefix
 from .rprint import rprint as print
 
@@ -35,7 +35,8 @@ def images2video(images, wfp, **kwargs):
     )
 
     n = len(images)
-    for i in track(range(n), description='writing', transient=True):
+    print('writing',n)
+    for i in range(n):
         if image_mode.lower() == 'bgr':
             writer.append_data(images[i][..., ::-1])
         else:
@@ -83,7 +84,9 @@ def blend(img: np.ndarray, mask: np.ndarray, background_color=(255, 255, 255)):
 def concat_frames(I_p_lst, driving_rgb_lst, img_rgb):
     # TODO: add more concat style, e.g., left-down corner driving
     out_lst = []
-    for idx, _ in track(enumerate(I_p_lst), total=len(I_p_lst), description='Concatenating result...'):
+    print('Concatenating result...',len(I_p_lst))
+    for idx, _ in enumerate(I_p_lst):
+    # track(enumerate(I_p_lst), total=len(I_p_lst), description='Concatenating result...'):
         source_image_drived = I_p_lst[idx]
         image_drive = driving_rgb_lst[idx]
 

nodes/live_portrait.py

@@ -28,7 +28,7 @@ def pil2tensor(image):
     return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0)
 
 
-
+from .LivePortrait.src.utils.video import images2video
 from .LivePortrait.src.live_portrait_pipeline import LivePortraitPipeline
 
 def get_model_dir(m):
@@ -44,6 +44,7 @@ class ArgumentConfig:
                     driving_info,
                     output_path='animations/v.mp4',
                     output_path_concat="",
+                    source_video=False,
                     device_id=0, 
                     crop_info =None,
                     face_index=0,
@@ -64,6 +65,7 @@ class ArgumentConfig:
                     share=False,
                     server_name='0.0.0.0'):
         self.source_image = source_image
+        self.source_video=source_video
         self.driving_info = driving_info
         self.output_path = output_path
         self.output_path_concat=output_path_concat
@@ -410,3 +412,98 @@ class LivePortraitNode:
 
         return (v_path,output_path_concat,)
 
+
+
+
+
+class LivePortraitVideoNode:
+    
+    @classmethod
+    def INPUT_TYPES(s):
+        
+        return {"required": {
+                        "source_image_batch": ("IMAGE",),
+                        "driving_video":("SCENE_VIDEO",),  
+                        },
+                # "optional":{  
+                #             "driving_video_reverse_align":("BOOLEAN", {"default": True},),
+                #         }
+                }
+    
+    RETURN_TYPES = ("SCENE_VIDEO","SCENE_VIDEO",)
+    RETURN_NAMES = ("video","video_concat",)
+
+    FUNCTION = "run"
+
+    CATEGORY = "♾️Mixlab/Video/LivePortrait"
+
+    INPUT_IS_LIST = True
+    OUTPUT_IS_LIST = (False,False,) #list 列表 [1,2,3]
+  
+    def run(self,source_image_batch,driving_video):
+        source_video=True
+        driving_video_reverse_align=True
+        print('#source_image_batch',source_image_batch)
+        source_image_batch=source_image_batch[0]
+        
+        #获取临时目录：temp
+        output_dir = folder_paths.get_temp_directory()
+
+        def count_live_portrait_mp4_files(output_dir: str) -> int:
+            count = 0
+            for filename in os.listdir(output_dir):
+                if filename.startswith('live_portrait_') and filename.endswith('.mp4'):
+                    count += 1
+            return count
+
+        counter=count_live_portrait_mp4_files(output_dir)
+        
+        v_file = f"live_portrait_{counter:05}.mp4"
+        v_file_concat = f"live_portrait_concat_{counter:05}.mp4"
+
+        v_path=os.path.join(output_dir, v_file)
+        output_path_concat=os.path.join(output_dir, v_file_concat)
+
+        # print('##---------------------------------#landmark_runner_ckpt',landmark_runner_ckpt)
+        live_portrait_pipeline = LivePortraitPipeline(
+            inference_cfg=inference_cfg,
+            crop_cfg=crop_cfg,
+            landmark_runner_ckpt=landmark_runner_ckpt,
+            insightface_pretrained_weights=insightface_pretrained_weights
+        )
+
+        # run
+        crop_info=None
+        if crop_info==None:
+
+            frames=[]
+
+            for i in range(len(source_image_batch)):
+
+                source_image=source_image_batch[i]
+
+                pil_image=tensor2pil(source_image)
+                # Convert PIL image to NumPy array
+                opencv_image = np.array(pil_image)
+
+                args =  ArgumentConfig(
+                    source_image=opencv_image,
+                    driving_info=[driving_video[0]],
+                    output_path=v_path,
+                    output_path_concat=output_path_concat,
+                    crop_info=crop_info, 
+                    source_video=source_video
+                )
+
+                video_frames, v_path, video_fps=live_portrait_pipeline.execute(args)
+
+                frames.append(video_frames[i])
+        
+        images2video(frames, wfp=v_path, fps=video_fps)
+
+        live_portrait_pipeline.live_portrait_wrapper=None
+
+        live_portrait_pipeline=None
+
+        return (v_path,output_path_concat,)
+

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-liveportrait"
 description = "The ComfyUI version of [a/LivePortrait](https://github.com/KwaiVGI/LivePortrait)."
-version = "1.2.0"
+version = "1.3.0"
 license = "LICENSE"
 dependencies = ["numpy>=1.26.4", "opencv-python-headless", "imageio>=2.34.2", "lmdb>=1.4.1", "timm>=1.0.7", "rich>=13.7.1", "ffmpeg>=1.4", "onnxruntime-gpu>=1.18.0", "onnx>=1.16.1", "scikit-image>=0.24.0", "albumentations>=1.4.10", "matplotlib>=3.9.0", "imageio-ffmpeg>=0.5.1"]