diff --git a/README.md b/README.md index c9f17a0..d684133 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,37 @@ # ComfyUI-MatAnyone + MatAnyone in ComfyUI + +Stable Video Matting with Consistent Memory Propagation: + +Download `matanyone.pth` from + +and place it in `checkpoint/matanyone.pth` + +## Workflow + +[workflow/workflow_mat_anyone.json](workflow/workflow_mat_anyone.json) + +![matanyone_example](workflow/mat_anyone_preview.png) +(Not a workflow-embedded image) + +Inputs: + +- src_video +- `forground_mask` (`IMAGE`) or `foreground_MASK` (`MASK`): The input mask. IMAGE option will automatically convert a black/white image to a mask. At least one option must be given. +- `solid_color` (optional): The solid color to create a screen. Defaults to Green Screen. +- `mask_frame`: The input mask's index (defaults to 0). Support first (0), last and middle frame. +- `n_warmup`: Number of iterations to warm up the model. Defaults to 10. + +Your input mask won't actually be in the final matte. Instead, the warmup process generate a new input mask, which is then propagated throughout the video. + +## Credit + +```cite +@InProceedings{yang2025matanyone, + title = {{MatAnyone}: Stable Video Matting with Consistent Memory Propagation}, + author = {Yang, Peiqing and Zhou, Shangchen and Zhao, Jixin and Tao, Qingyi and Loy, Chen Change}, + booktitle = {arXiv preprint arXiv:2501.14677}, + year = {2025} +} +``` diff --git a/mat_anyone.py b/mat_anyone.py index 9a6198d..db262c9 100644 --- a/mat_anyone.py +++ b/mat_anyone.py @@ -1,6 +1,5 @@ from pathlib import Path -import numpy as np import torch from omegaconf import OmegaConf from tqdm import tqdm diff --git a/workflow/mat_anyone_preview.png b/workflow/mat_anyone_preview.png new file mode 100644 index 0000000..463a75b Binary files /dev/null and b/workflow/mat_anyone_preview.png differ diff --git a/workflow/workflow_mat_anyone.json b/workflow/workflow_mat_anyone.json new file mode 100644 index 0000000..460258d --- /dev/null +++ b/workflow/workflow_mat_anyone.json @@ -0,0 +1,637 @@ +{ + "id": "9d77f655-23b7-4e29-8d05-763296629f0c", + "revision": 0, + "last_node_id": 43, + "last_link_id": 76, + "nodes": [ + { + "id": 3, + "type": "LoadImage", + "pos": [ + -180.32781982421875, + 472.85260009765625 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9, + 68 + ] + }, + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "test-sample1.png", + "image", + "" + ] + }, + { + "id": 2, + "type": "VHS_LoadVideo", + "pos": [ + -567.3414306640625, + 437.2431335449219 + ], + "size": [ + 352.79998779296875, + 463.1999816894531 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "meta_batch", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "localized_name": "vae", + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 41 + ] + }, + { + "localized_name": "frame_count", + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "localized_name": "video_info", + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "test-sample1.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "test-sample1.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + }, + "muted": false + } + } + }, + { + "id": 39, + "type": "SolidColorBatched", + "pos": [ + -202.20913696289062, + 987.6845092773438 + ], + "size": [ + 315, + 178 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 65 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 66 + } + ], + "outputs": [ + { + "localized_name": "solid", + "name": "solid", + "type": "IMAGE", + "links": [ + 64 + ] + } + ], + "properties": { + "Node name for S&R": "SolidColorBatched" + }, + "widgets_values": [ + 1, + 720, + 1280, + 0, + 255, + 0 + ] + }, + { + "id": 42, + "type": "VHS_VideoCombine", + "pos": [ + 146.1608428955078, + 669.3427734375 + ], + "size": [ + 315, + 513.9375 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 73 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "meta_batch", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "localized_name": "vae", + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "name": "frame_rate", + "type": "FLOAT", + "widget": { + "name": "frame_rate" + }, + "link": 74 + } + ], + "outputs": [ + { + "localized_name": "Filenames", + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "GB", + "format": "video/nvenc_h264-mp4", + "pix_fmt": "yuv420p", + "bitrate": 10, + "megabit": true, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "GB_00001.mp4", + "subfolder": "", + "type": "temp", + "format": "video/nvenc_h264-mp4", + "frame_rate": 30 + }, + "muted": false + } + } + }, + { + "id": 40, + "type": "VHS_VideoInfoLoaded", + "pos": [ + -174.8192596435547, + 835.6626586914062 + ], + "size": [ + 235.1999969482422, + 106 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "video_info", + "name": "video_info", + "type": "VHS_VIDEOINFO", + "link": 71 + } + ], + "outputs": [ + { + "localized_name": "fps🟦", + "name": "fps🟦", + "type": "FLOAT", + "links": [ + 74, + 75 + ] + }, + { + "localized_name": "frame_count🟦", + "name": "frame_count🟦", + "type": "INT", + "links": [] + }, + { + "localized_name": "duration🟦", + "name": "duration🟦", + "type": "FLOAT", + "links": null + }, + { + "localized_name": "width🟦", + "name": "width🟦", + "type": "INT", + "links": [ + 65 + ] + }, + { + "localized_name": "height🟦", + "name": "height🟦", + "type": "INT", + "links": [ + 66 + ] + } + ], + "properties": { + "Node name for S&R": "VHS_VideoInfoLoaded" + }, + "widgets_values": {} + }, + { + "id": 5, + "type": "VHS_VideoCombine", + "pos": [ + 474.8391418457031, + 666.7745971679688 + ], + "size": [ + 315, + 513.9375 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 72 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "meta_batch", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "localized_name": "vae", + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "name": "frame_rate", + "type": "FLOAT", + "widget": { + "name": "frame_rate" + }, + "link": 75 + } + ], + "outputs": [ + { + "localized_name": "Filenames", + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "Matte", + "format": "video/nvenc_h264-mp4", + "pix_fmt": "yuv420p", + "bitrate": 10, + "megabit": true, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Matte_00002.mp4", + "subfolder": "", + "type": "temp", + "format": "video/nvenc_h264-mp4", + "frame_rate": 30 + }, + "muted": false + } + } + }, + { + "id": 15, + "type": "MatAnyone", + "pos": [ + 171.0359649658203, + 475.4490966796875 + ], + "size": [ + 236.8000030517578, + 142 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "src_video", + "name": "src_video", + "type": "IMAGE", + "link": 42 + }, + { + "localized_name": "foreground_mask", + "name": "foreground_mask", + "shape": 7, + "type": "IMAGE", + "link": 68 + }, + { + "localized_name": "foreground_MASK", + "name": "foreground_MASK", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "localized_name": "solid_color", + "name": "solid_color", + "shape": 7, + "type": "IMAGE", + "link": 64 + } + ], + "outputs": [ + { + "localized_name": "matte", + "name": "matte", + "type": "IMAGE", + "links": [ + 36, + 72 + ] + }, + { + "localized_name": "green_screen", + "name": "green_screen", + "type": "IMAGE", + "links": [ + 73 + ] + } + ], + "properties": { + "Node name for S&R": "MatAnyone" + }, + "widgets_values": [ + 0, + 10 + ] + } + ], + "links": [ + [ + 19, + 7, + 0, + 5, + 4, + "FLOAT" + ], + [ + 42, + 2, + 0, + 15, + 0, + "IMAGE" + ], + [ + 49, + 7, + 0, + 33, + 4, + "FLOAT" + ], + [ + 64, + 39, + 0, + 15, + 3, + "IMAGE" + ], + [ + 65, + 40, + 3, + 39, + 0, + "INT" + ], + [ + 66, + 40, + 4, + 39, + 1, + "INT" + ], + [ + 68, + 3, + 0, + 15, + 1, + "IMAGE" + ], + [ + 71, + 2, + 3, + 40, + 0, + "VHS_VIDEOINFO" + ], + [ + 72, + 15, + 0, + 5, + 0, + "IMAGE" + ], + [ + 73, + 15, + 1, + 42, + 0, + "IMAGE" + ], + [ + 74, + 40, + 0, + 42, + 4, + "FLOAT" + ], + [ + 75, + 40, + 0, + 5, + 4, + "FLOAT" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1, + "offset": [ + 688.5119691614667, + -320.89544104869697 + ] + }, + "linkExtensions": [ + { + "id": 19, + "parentId": 3 + }, + { + "id": 49, + "parentId": 3 + } + ], + "reroutes": [ + { + "id": 3, + "pos": [ + 1139.3179931640625, + 206.41610717773438 + ], + "linkIds": [ + 19, + 49 + ] + } + ] + }, + "version": 0.4 +} \ No newline at end of file