diff --git a/.gitignore b/.gitignore index 24f99b0..fd7ccbc 100644 --- a/.gitignore +++ b/.gitignore @@ -120,7 +120,6 @@ logs/ # Visualization -*.png *.gif *.jpg *.obj diff --git a/README.md b/README.md index 33466b5..b8142fd 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,3 @@ If you can't run **install.bat** (e.g. you are a Linux user). Open the CMD/Shell # Examples - -https://github.com/Fannovel16/ComfyUI-MotionDiff/assets/16047777/a738dd25-10bc-40db-90c1-04aed9bcdf52 - - -https://github.com/Fannovel16/ComfyUI-MotionDiff/assets/16047777/56e97a22-da31-40f4-a3c1-9df2fd3d706f - - diff --git a/__init__.py b/__init__.py index 30822f4..4c99447 100644 --- a/__init__.py +++ b/__init__.py @@ -24,6 +24,7 @@ from mogen.smpl.rotation2xyz import Rotation2xyz from mogen.smpl.render_mesh import render_from_smpl import gc from pathlib import Path +from PIL import ImageColor def create_mdm_model(model_config): cfg = mmcv.Config.fromstring(model_config.config_code, '.py') @@ -239,7 +240,7 @@ class MotionDataVisualizer: "distance": ("FLOAT", {"default": 7.0, "min": 0.0, "max": 10.0, "step": 0.1}), "elevation": ("FLOAT", {"default": 120, "min": 0.0, "max": 300.0, "step": 0.1}), "rotation": ("FLOAT", {"default": -90, "min": -180, "max": 180, "step": 1}), - "poselinewidth": ("FLOAT", {"default": 8, "min": 0, "max": 50, "step": 0.1}), + "poselinewidth": ("FLOAT", {"default": 4, "min": 0, "max": 50, "step": 0.1}), }, "optional": { "opt_title": ("STRING", {"default": '' ,"multiline": False}), @@ -272,8 +273,8 @@ class SmplifyMotionData: return { "required": { "motion_data": ("MOTION_DATA", ), - "num_smplify_iters": ("INT", {"min": 10, "max": 1000, "default": 150}), - "smplify_step_size": ("FLOAT", {"min": 1e-4, "max": 5e-1, "step": 1e-4, "default": 1e-2}), + "num_smplify_iters": ("INT", {"min": 10, "max": 1000, "default": 50}), + "smplify_step_size": ("FLOAT", {"min": 1e-4, "max": 5e-1, "step": 1e-4, "default": 1e-1}), "smpl_model": (list(smpl_model_dicts.keys()), {"default": "SMPL_NEUTRAL.pkl"}) } } @@ -331,10 +332,11 @@ class RenderSMPLMesh: "smpl": ("SMPL", ), "draw_platform": ("BOOLEAN", {"default": False}), "depth_only": ("BOOLEAN", {"default": False}), - "yfov": ("FLOAT", {"default": np.pi / 3.0, "min": 0.1, "max": 10, "step": 0.1}), + "yfov": ("FLOAT", {"default": 0.75, "min": 0.1, "max": 10, "step": 0.05}), "move_x": ("FLOAT", {"default": 0,"min": -500, "max": 500, "step": 0.1}), "move_y": ("FLOAT", {"default": 0,"min": -500, "max": 500, "step": 0.1}), "move_z": ("FLOAT", {"default": 0,"min": -500, "max": 500, "step": 0.1}), + "background_hex_color": ("STRING", {"default": "#FFFFFF", "mutiline": False}) } } @@ -342,14 +344,21 @@ class RenderSMPLMesh: RETURN_NAMES = ("IMAGE", "DEPTH_MAP") CATEGORY = "MotionDiff/smpl" FUNCTION = "render" - def render(self, smpl, yfov, move_x, move_y, move_z, draw_platform, depth_only): + def render(self, smpl, yfov, move_x, move_y, move_z, draw_platform, depth_only, background_hex_color): smpl_model_path, motion_tensor, _ = smpl color_frames, depth_frames = render_from_smpl( motion_tensor.to(get_torch_device()), yfov, move_x, move_y, move_z, draw_platform,depth_only, smpl_model_path=smpl_model_path ) + bg_color = ImageColor.getcolor(background_hex_color, "RGB") color_frames = torch.from_numpy(color_frames[..., :3].astype(np.float32) / 255.) + white_mask = [ + (color_frames[..., 0] == 1.) & + (color_frames[..., 1] == 1.) & + (color_frames[..., 2] == 1.) + ] + color_frames[..., :3][white_mask] = torch.Tensor(bg_color) #Normalize to [0, 1] normalized_depth = (depth_frames - depth_frames.min()) / (depth_frames.max() - depth_frames.min()) diff --git a/examples/full_motiondiff_example.json b/examples/full_motiondiff_example.json new file mode 100644 index 0000000..ca9dd14 --- /dev/null +++ b/examples/full_motiondiff_example.json @@ -0,0 +1,656 @@ +{ + "last_node_id": 38, + "last_link_id": 47, + "nodes": [ + { + "id": 3, + "type": "EmptyMotionData", + "pos": [ + 1227, + -67 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MOTION_DATA", + "type": "MOTION_DATA", + "links": [ + 2, + 6 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyMotionData" + }, + "widgets_values": [ + 196 + ] + }, + { + "id": 4, + "type": "MotionDiffSimpleSampler", + "pos": [ + 1228, + -237 + ], + "size": { + "0": 315, + "1": 118 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "md_model", + "type": "MD_MODEL", + "link": 4 + }, + { + "name": "md_clip", + "type": "MD_CLIP", + "link": 5 + }, + { + "name": "md_cond", + "type": "MD_CONDITIONING", + "link": 3 + }, + { + "name": "motion_data", + "type": "MOTION_DATA", + "link": 6 + } + ], + "outputs": [ + { + "name": "MOTION_DATA", + "type": "MOTION_DATA", + "links": [ + 17, + 42 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "MotionDiffSimpleSampler" + }, + "widgets_values": [ + "ddpm" + ] + }, + { + "id": 1, + "type": "MotionDiffLoader", + "pos": [ + 1230, + -362 + ], + "size": { + "0": 315, + "1": 78 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "MD_MODEL", + "type": "MD_MODEL", + "links": [ + 4 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MD_CLIP", + "type": "MD_CLIP", + "links": [ + 1, + 5 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "MotionDiffLoader" + }, + "widgets_values": [ + "remodiffuse-human_ml3d" + ] + }, + { + "id": 33, + "type": "VHS_VideoCombine", + "pos": [ + 1823, + 62 + ], + "size": [ + 210, + 436 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 46 + } + ], + "outputs": [ + { + "name": "GIF", + "type": "GIF", + "links": null, + "shape": 3 + } + ], + "title": "Depth Map SMPL", + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": [ + 20, + 0, + "Depth_SMPL", + "video/h264-mp4", + false, + true, + null, + "/view?filename=Depth_SMPL_00022_.mp4&subfolder=&type=output&format=video%2Fh264-mp4" + ] + }, + { + "id": 35, + "type": "MotionDataVisualizer", + "pos": [ + 1926, + -362 + ], + "size": { + "0": 315, + "1": 178 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "motion_data", + "type": "MOTION_DATA", + "link": 42 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 43 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "MotionDataVisualizer" + }, + "widgets_values": [ + "pseudo-openpose", + 7, + 120, + -90, + 4, + "" + ] + }, + { + "id": 16, + "type": "SmplifyMotionData", + "pos": [ + 1579, + -373 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "motion_data", + "type": "MOTION_DATA", + "link": 17 + } + ], + "outputs": [ + { + "name": "SMPL", + "type": "SMPL", + "links": [ + 44 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "SmplifyMotionData" + }, + "widgets_values": [ + 10, + 0.5, + "SMPL_FEMALE.pkl" + ] + }, + { + "id": 34, + "type": "VHS_VideoCombine", + "pos": [ + 2048, + 63 + ], + "size": [ + 210, + 436 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 40 + } + ], + "outputs": [ + { + "name": "GIF", + "type": "GIF", + "links": null, + "shape": 3 + } + ], + "title": "Openpose SMPL", + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": [ + 20, + 0, + "OpenPose_SMPL", + "video/h264-mp4", + false, + true, + null, + "/view?filename=OpenPose_SMPL_00019_.mp4&subfolder=&type=output&format=video%2Fh264-mp4" + ] + }, + { + "id": 2, + "type": "MotionCLIPTextEncode", + "pos": [ + 1186, + 37 + ], + "size": { + "0": 387.9890441894531, + "1": 96 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "md_clip", + "type": "MD_CLIP", + "link": 1 + }, + { + "name": "motion_data", + "type": "MOTION_DATA", + "link": 2 + } + ], + "outputs": [ + { + "name": "MD_CONDITIONING", + "type": "MD_CONDITIONING", + "links": [ + 3 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "MotionCLIPTextEncode" + }, + "widgets_values": [ + "a person is running" + ] + }, + { + "id": 38, + "type": "RenderSMPLMesh", + "pos": [ + 1576, + -228 + ], + "size": { + "0": 315, + "1": 246 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "smpl", + "type": "SMPL", + "link": 44 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 45, + 47 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "DEPTH_MAP", + "type": "IMAGE", + "links": [ + 46 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "RenderSMPLMesh" + }, + "widgets_values": [ + false, + false, + 0.75, + 0, + 0, + 0, + "#FFFFFF" + ] + }, + { + "id": 27, + "type": "DWPreprocessor", + "pos": [ + 1919, + -129 + ], + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 47 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 40 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DWPreprocessor" + }, + "widgets_values": [ + "disable", + "enable", + "disable", + 1024 + ] + }, + { + "id": 36, + "type": "VHS_VideoCombine", + "pos": [ + 2274, + 69 + ], + "size": [ + 210, + 436 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 43 + } + ], + "outputs": [ + { + "name": "GIF", + "type": "GIF", + "links": null, + "shape": 3 + } + ], + "title": "Original", + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": [ + 20, + 0, + "Pseudo_OpenPose_SMPL", + "video/h264-mp4", + false, + true, + null, + "/view?filename=Pseudo_OpenPose_SMPL_00025_.mp4&subfolder=&type=output&format=video%2Fh264-mp4" + ] + }, + { + "id": 32, + "type": "VHS_VideoCombine", + "pos": [ + 1595, + 57 + ], + "size": [ + 210, + 436 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 45 + } + ], + "outputs": [ + { + "name": "GIF", + "type": "GIF", + "links": null, + "shape": 3 + } + ], + "title": "SMPL", + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": [ + 20, + 0, + "SMPL", + "video/h264-mp4", + false, + true, + null, + "/view?filename=SMPL_00022_.mp4&subfolder=&type=output&format=video%2Fh264-mp4" + ] + } + ], + "links": [ + [ + 1, + 1, + 1, + 2, + 0, + "MD_CLIP" + ], + [ + 2, + 3, + 0, + 2, + 1, + "MOTION_DATA" + ], + [ + 3, + 2, + 0, + 4, + 2, + "MD_CONDITIONING" + ], + [ + 4, + 1, + 0, + 4, + 0, + "MD_MODEL" + ], + [ + 5, + 1, + 1, + 4, + 1, + "MD_CLIP" + ], + [ + 6, + 3, + 0, + 4, + 3, + "MOTION_DATA" + ], + [ + 17, + 4, + 0, + 16, + 0, + "MOTION_DATA" + ], + [ + 40, + 27, + 0, + 34, + 0, + "IMAGE" + ], + [ + 42, + 4, + 0, + 35, + 0, + "MOTION_DATA" + ], + [ + 43, + 35, + 0, + 36, + 0, + "IMAGE" + ], + [ + 44, + 16, + 0, + 38, + 0, + "SMPL" + ], + [ + 45, + 38, + 0, + 32, + 0, + "IMAGE" + ], + [ + 46, + 38, + 1, + 33, + 0, + "IMAGE" + ], + [ + 47, + 38, + 0, + 27, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/examples/original.mp4 b/examples/original.mp4 deleted file mode 100644 index 9c4b40f..0000000 Binary files a/examples/original.mp4 and /dev/null differ diff --git a/examples/pseudo-openpose.mp4 b/examples/pseudo-openpose.mp4 deleted file mode 100644 index f2c1932..0000000 Binary files a/examples/pseudo-openpose.mp4 and /dev/null differ diff --git a/imgs/pipeline.png b/imgs/pipeline.png new file mode 100644 index 0000000..6cb95f1 Binary files /dev/null and b/imgs/pipeline.png differ diff --git a/imgs/teaser.png b/imgs/teaser.png new file mode 100644 index 0000000..2a476f4 Binary files /dev/null and b/imgs/teaser.png differ diff --git a/mogen/smpl/human-skin.png b/mogen/smpl/human-skin.png new file mode 100644 index 0000000..74b4282 Binary files /dev/null and b/mogen/smpl/human-skin.png differ