finish

README.md

@@ -1,2 +1,47 @@
 # ComfyUI-MimicMotion
 a comfyui custom node for [MimicMotion](https://github.com/Tencent/MimicMotion)
+[workflow](./doc/mimicmotion_workflow.json)
+
+## Example
+test on 2080ti 11GB torch==2.3.0+cu121 python 3.10.8
+- input
+
+refer_img
+<div>
+  <figure>
+  <img src="./doc/demo1.jpg" width="600px"/>
+  <figure>
+</div>
+
+refer_video
+
+- output
+
+## How to use
+make sure `ffmpeg` is worked in your commandline
+for Linux
+```
+apt update
+apt install ffmpeg
+```
+for Windows,you can install `ffmpeg` by [WingetUI](https://github.com/marticliment/WingetUI) automatically
+
+then!
+```
+## insatll xformers match your torch,for torch==2.1.0+cu121
+pip install xformers==0.0.22.post7
+
+## in ComfyUI/custom_nodes
+git clone https://github.com/AIFSH/ComfyUI-MimicMotion.git
+cd ComfyUI-MimicMotion
+pip install -r requirements.txt
+```
+weights will be downloaded from huggingface
+
+## Tutorial
+- [Demo](https://www.bilibili.com/video/BV1Br421L7rX)
+- QQ群：852228202
+
+## Thanks
+
+[MimicMotion](https://github.com/Tencent/MimicMotion)

doc/mimicmotion_workflow.json

@@ -0,0 +1,206 @@
+{
+  "last_node_id": 5,
+  "last_link_id": 6,
+  "nodes": [
+    {
+      "id": 2,
+      "type": "LoadImage",
+      "pos": [
+        39,
+        28
+      ],
+      "size": {
+        "0": 315,
+        "1": 314
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            4
+          ],
+          "shape": 3
+        },
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImage"
+      },
+      "widgets_values": [
+        "demo1.jpg",
+        "image"
+      ]
+    },
+    {
+      "id": 3,
+      "type": "LoadVideo",
+      "pos": [
+        51,
+        356
+      ],
+      "size": {
+        "0": 315,
+        "1": 612.4444580078125
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "VIDEO",
+          "type": "VIDEO",
+          "links": [
+            5
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadVideo"
+      },
+      "widgets_values": [
+        "demo.mp4",
+        "Video",
+        {
+          "hidden": false,
+          "paused": false,
+          "params": {}
+        }
+      ]
+    },
+    {
+      "id": 5,
+      "type": "MimicMotionNode",
+      "pos": [
+        459,
+        54
+      ],
+      "size": {
+        "0": 315,
+        "1": 294
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "ref_image",
+          "type": "IMAGE",
+          "link": 4,
+          "slot_index": 0
+        },
+        {
+          "name": "ref_video_path",
+          "type": "VIDEO",
+          "link": 5
+        }
+      ],
+      "outputs": [
+        {
+          "name": "VIDEO",
+          "type": "VIDEO",
+          "links": [
+            6
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "MimicMotionNode"
+      },
+      "widgets_values": [
+        576,
+        2,
+        8,
+        6,
+        4,
+        25,
+        2,
+        15,
+        415,
+        "randomize"
+      ]
+    },
+    {
+      "id": 4,
+      "type": "PreViewVideo",
+      "pos": [
+        816,
+        74
+      ],
+      "size": {
+        "0": 210,
+        "1": 377.77777099609375
+      },
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video",
+          "type": "VIDEO",
+          "link": 6
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "PreViewVideo"
+      },
+      "widgets_values": [
+        {
+          "hidden": false,
+          "paused": false,
+          "params": {}
+        }
+      ]
+    }
+  ],
+  "links": [
+    [
+      4,
+      2,
+      0,
+      5,
+      0,
+      "IMAGE"
+    ],
+    [
+      5,
+      3,
+      0,
+      5,
+      1,
+      "VIDEO"
+    ],
+    [
+      6,
+      5,
+      0,
+      4,
+      0,
+      "VIDEO"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 1.1000000000000005,
+      "offset": [
+        86.66106242715553,
+        12.114120018825606
+      ]
+    }
+  },
+  "version": 0.4
+}

nodes.py

@@ -64,6 +64,9 @@ class MimicMotionNode:
                 "tile_overlap": ("INT",{
                     "default": 6
                 }),
+                "decode_chunk_size":("INT",{
+                    "default": 8
+                }),
                 "num_inference_steps":  ("INT",{
                     "default": 25
                 }),
@@ -90,8 +93,8 @@ class MimicMotionNode:
 
     @torch.no_grad()
     def gen_video(self,ref_image,ref_video_path,resolution,sample_stride,
-                  tile_size,tile_overlap,num_inference_steps,guidance_scale,
-                  fps,seed):
+                  tile_size,tile_overlap,decode_chunk_size,num_inference_steps,
+                  guidance_scale,fps,seed):
         torch.set_default_dtype(torch.float16)
         infer_config = OmegaConf.load(os.path.join(now_dir,"test.yaml"))
         infer_config.base_model_path = svd_dir
@@ -109,6 +112,7 @@ class MimicMotionNode:
         task_config = {
             "tile_size": tile_size,
             "tile_overlap": tile_overlap,
+            "decode_chunk_size": decode_chunk_size,
             "num_inference_steps": num_inference_steps,
             "noise_aug_strength": 0,
             "guidance_scale": guidance_scale,
@@ -196,7 +200,7 @@ def run_pipeline(pipeline: MimicMotionPipeline, image_pixels, pose_pixels, devic
         height=pose_pixels.shape[-2], width=pose_pixels.shape[-1], fps=task_config["fps"],
         noise_aug_strength=task_config["noise_aug_strength"], num_inference_steps=task_config["num_inference_steps"],
         generator=generator, min_guidance_scale=task_config["guidance_scale"], 
-        max_guidance_scale=task_config["guidance_scale"], decode_chunk_size=8, output_type="pt", device=device
+        max_guidance_scale=task_config["guidance_scale"], decode_chunk_size=task_config['decode_chunk_size'], output_type="pt", device=device
     ).frames.cpu()
     video_frames = (frames * 255.0).to(torch.uint8)
 

requirements.txt

@@ -7,4 +7,5 @@ onnxruntime-gpu
 moviepy
 matplotlib
 opencv-python
-accelerate
+accelerate
+av