diff --git a/README.md b/README.md index 3e20549..2243b16 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,47 @@ # ComfyUI-MimicMotion a comfyui custom node for [MimicMotion](https://github.com/Tencent/MimicMotion) +[workflow](./doc/mimicmotion_workflow.json) + +## Example +test on 2080ti 11GB torch==2.3.0+cu121 python 3.10.8 +- input + +refer_img +
+
+ +
+
+ +refer_video + +- output + +## How to use +make sure `ffmpeg` is worked in your commandline +for Linux +``` +apt update +apt install ffmpeg +``` +for Windows,you can install `ffmpeg` by [WingetUI](https://github.com/marticliment/WingetUI) automatically + +then! +``` +## insatll xformers match your torch,for torch==2.1.0+cu121 +pip install xformers==0.0.22.post7 + +## in ComfyUI/custom_nodes +git clone https://github.com/AIFSH/ComfyUI-MimicMotion.git +cd ComfyUI-MimicMotion +pip install -r requirements.txt +``` +weights will be downloaded from huggingface + +## Tutorial +- [Demo](https://www.bilibili.com/video/BV1Br421L7rX) +- QQ群：852228202 + +## Thanks + +[MimicMotion](https://github.com/Tencent/MimicMotion) \ No newline at end of file diff --git a/doc/demo.mp4 b/doc/demo.mp4 new file mode 100644 index 0000000..79f42d5 Binary files /dev/null and b/doc/demo.mp4 differ diff --git a/doc/demo1.jpg b/doc/demo1.jpg new file mode 100644 index 0000000..37bf754 Binary files /dev/null and b/doc/demo1.jpg differ diff --git a/doc/mimicmotion_demo_20240702092927.mp4 b/doc/mimicmotion_demo_20240702092927.mp4 new file mode 100644 index 0000000..c0d9f72 Binary files /dev/null and b/doc/mimicmotion_demo_20240702092927.mp4 differ diff --git a/doc/mimicmotion_workflow.json b/doc/mimicmotion_workflow.json new file mode 100644 index 0000000..3db8cf4 --- /dev/null +++ b/doc/mimicmotion_workflow.json @@ -0,0 +1,206 @@ +{ + "last_node_id": 5, + "last_link_id": 6, + "nodes": [ + { + "id": 2, + "type": "LoadImage", + "pos": [ + 39, + 28 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 4 + ], + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "demo1.jpg", + "image" + ] + }, + { + "id": 3, + "type": "LoadVideo", + "pos": [ + 51, + 356 + ], + "size": { + "0": 315, + "1": 612.4444580078125 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 5 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "demo.mp4", + "Video", + { + "hidden": false, + "paused": false, + "params": {} + } + ] + }, + { + "id": 5, + "type": "MimicMotionNode", + "pos": [ + 459, + 54 + ], + "size": { + "0": 315, + "1": 294 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "ref_image", + "type": "IMAGE", + "link": 4, + "slot_index": 0 + }, + { + "name": "ref_video_path", + "type": "VIDEO", + "link": 5 + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 6 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "MimicMotionNode" + }, + "widgets_values": [ + 576, + 2, + 8, + 6, + 4, + 25, + 2, + 15, + 415, + "randomize" + ] + }, + { + "id": 4, + "type": "PreViewVideo", + "pos": [ + 816, + 74 + ], + "size": { + "0": 210, + "1": 377.77777099609375 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 6 + } + ], + "properties": { + "Node name for S&R": "PreViewVideo" + }, + "widgets_values": [ + { + "hidden": false, + "paused": false, + "params": {} + } + ] + } + ], + "links": [ + [ + 4, + 2, + 0, + 5, + 0, + "IMAGE" + ], + [ + 5, + 3, + 0, + 5, + 1, + "VIDEO" + ], + [ + 6, + 5, + 0, + 4, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.1000000000000005, + "offset": [ + 86.66106242715553, + 12.114120018825606 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes.py b/nodes.py index 1fb6e2a..f275fb1 100644 --- a/nodes.py +++ b/nodes.py @@ -64,6 +64,9 @@ class MimicMotionNode: "tile_overlap": ("INT",{ "default": 6 }), + "decode_chunk_size":("INT",{ + "default": 8 + }), "num_inference_steps": ("INT",{ "default": 25 }), @@ -90,8 +93,8 @@ class MimicMotionNode: @torch.no_grad() def gen_video(self,ref_image,ref_video_path,resolution,sample_stride, - tile_size,tile_overlap,num_inference_steps,guidance_scale, - fps,seed): + tile_size,tile_overlap,decode_chunk_size,num_inference_steps, + guidance_scale,fps,seed): torch.set_default_dtype(torch.float16) infer_config = OmegaConf.load(os.path.join(now_dir,"test.yaml")) infer_config.base_model_path = svd_dir @@ -109,6 +112,7 @@ class MimicMotionNode: task_config = { "tile_size": tile_size, "tile_overlap": tile_overlap, + "decode_chunk_size": decode_chunk_size, "num_inference_steps": num_inference_steps, "noise_aug_strength": 0, "guidance_scale": guidance_scale, @@ -196,7 +200,7 @@ def run_pipeline(pipeline: MimicMotionPipeline, image_pixels, pose_pixels, devic height=pose_pixels.shape[-2], width=pose_pixels.shape[-1], fps=task_config["fps"], noise_aug_strength=task_config["noise_aug_strength"], num_inference_steps=task_config["num_inference_steps"], generator=generator, min_guidance_scale=task_config["guidance_scale"], - max_guidance_scale=task_config["guidance_scale"], decode_chunk_size=8, output_type="pt", device=device + max_guidance_scale=task_config["guidance_scale"], decode_chunk_size=task_config['decode_chunk_size'], output_type="pt", device=device ).frames.cpu() video_frames = (frames * 255.0).to(torch.uint8) diff --git a/requirements.txt b/requirements.txt index 043abf2..aef251d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ onnxruntime-gpu moviepy matplotlib opencv-python -accelerate \ No newline at end of file +accelerate +av \ No newline at end of file