diff --git a/example_workflows/wanvideo_1_3B_ReCamMaster_example_01.json b/example_workflows/wanvideo_1_3B_ReCamMaster_example_01.json index 1283980..78da4a8 100644 --- a/example_workflows/wanvideo_1_3B_ReCamMaster_example_01.json +++ b/example_workflows/wanvideo_1_3B_ReCamMaster_example_01.json @@ -1,8 +1,8 @@ { "id": "c6e410bc-5e2c-460b-ae81-c91b6094fbb1", "revision": 0, - "last_node_id": 204, - "last_link_id": 336, + "last_node_id": 206, + "last_link_id": 341, "nodes": [ { "id": 42, @@ -101,7 +101,7 @@ 200 ], "flags": {}, - "order": 21, + "order": 22, "mode": 2, "inputs": [ { @@ -258,7 +258,7 @@ 174 ], "flags": {}, - "order": 34, + "order": 39, "mode": 0, "inputs": [ { @@ -371,7 +371,7 @@ 200 ], "flags": {}, - "order": 22, + "order": 23, "mode": 2, "inputs": [ { @@ -413,7 +413,7 @@ 86 ], "flags": {}, - "order": 24, + "order": 25, "mode": 0, "inputs": [ { @@ -647,9 +647,7 @@ "ver": "0.3.27", "Node name for S&R": "PreviewImage" }, - "widgets_values": [ - "" - ] + "widgets_values": [] }, { "id": 125, @@ -663,15 +661,12 @@ 190.28567504882812 ], "flags": {}, - "order": 40, + "order": 41, "mode": 0, "inputs": [ { "name": "text", "type": "STRING", - "widget": { - "name": "text" - }, "link": 215 } ], @@ -689,7 +684,7 @@ "Node name for S&R": "ShowText|pysssss" }, "widgets_values": [ - "", + "A man in a suit and tie walking down a hallway. He has a friendly expression and is looking directly at the camera. The hallway has beige walls adorned with framed black and white photographs. There is a door on the left side of the hallway and a poster on the wall. The lighting is soft and natural. The image is high quality and has a watermark in the bottom right corner.", "A man in a suit and tie walking down a hallway. He has a friendly expression and is looking directly at the camera. The hallway has beige walls adorned with framed black and white photographs. There is a door on the left side of the hallway and a poster on the wall. The lighting is soft and natural. The image is high quality and has a watermark in the bottom right corner." ] }, @@ -745,7 +740,7 @@ 261.5306701660156 ], "flags": {}, - "order": 41, + "order": 42, "mode": 0, "inputs": [ { @@ -842,7 +837,7 @@ "flags": { "collapsed": true }, - "order": 23, + "order": 24, "mode": 0, "inputs": [ { @@ -914,7 +909,7 @@ 266 ], "flags": {}, - "order": 28, + "order": 29, "mode": 0, "inputs": [ { @@ -922,28 +917,22 @@ "type": "IMAGE", "link": 244 }, - { - "name": "get_image_size", - "shape": 7, - "type": "IMAGE", - "link": null - }, { "name": "width_input", "shape": 7, "type": "INT", - "widget": { - "name": "width_input" - }, "link": null }, { "name": "height_input", "shape": 7, "type": "INT", - "widget": { - "name": "height_input" - }, + "link": null + }, + { + "name": "get_image_size", + "shape": 7, + "type": "IMAGE", "link": null } ], @@ -978,8 +967,6 @@ "lanczos", false, 16, - 0, - 0, "center" ] }, @@ -1025,50 +1012,6 @@ "color": "#2a363b", "bgcolor": "#3f5159" }, - { - "id": 74, - "type": "WidgetToString", - "pos": [ - 2128.5166015625, - -432.6599426269531 - ], - "size": [ - 315, - 154 - ], - "flags": {}, - "order": 29, - "mode": 0, - "inputs": [ - { - "name": "any_input", - "shape": 7, - "type": "*", - "link": 102 - } - ], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [ - 212 - ] - } - ], - "properties": { - "cnr_id": "comfyui-kjnodes", - "ver": "d57154c3a808b8a3f232ed293eaa2d000867c884", - "Node name for S&R": "WidgetToString" - }, - "widgets_values": [ - 0, - "camera_type", - false, - "", - 2 - ] - }, { "id": 58, "type": "WanVideoEncode", @@ -1139,7 +1082,7 @@ "flags": { "collapsed": true }, - "order": 42, + "order": 43, "mode": 0, "inputs": [ { @@ -1177,7 +1120,7 @@ 555.8994140625 ], "flags": {}, - "order": 35, + "order": 40, "mode": 0, "inputs": [ { @@ -1192,9 +1135,7 @@ "ver": "0.3.27", "Node name for S&R": "PreviewImage" }, - "widgets_values": [ - "" - ] + "widgets_values": [] }, { "id": 128, @@ -1292,7 +1233,7 @@ 274 ], "flags": {}, - "order": 39, + "order": 44, "mode": 0, "inputs": [ { @@ -1304,9 +1245,6 @@ "name": "caption", "shape": 7, "type": "STRING", - "widget": { - "name": "caption" - }, "link": null }, { @@ -1341,8 +1279,7 @@ "black", "FreeMonoBoldOblique.otf", "input", - "up", - "" + "up" ] }, { @@ -1353,7 +1290,7 @@ -1155.6121826171875 ], "size": [ - 390.5999755859375, + 421.6000061035156, 202 ], "flags": {}, @@ -1383,82 +1320,6 @@ 128 ] }, - { - "id": 138, - "type": "ReCamMasterPoseVisualizer", - "pos": [ - 1597.2598876953125, - 177.93458557128906 - ], - "size": [ - 349.6756591796875, - 130 - ], - "flags": {}, - "order": 31, - "mode": 0, - "inputs": [ - { - "name": "camera_poses", - "type": "CAMERAPOSES", - "link": 242 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 243 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WanVideoWrapper", - "ver": "2dc25c150ec4288e0b8689fc49fe2c9c8ab01999", - "Node name for S&R": "ReCamMasterPoseVisualizer" - }, - "widgets_values": [ - 0.10000000000000002, - 0.20000000000000004, - 0.4000000000000001, - 0.5000000000000001 - ] - }, - { - "id": 157, - "type": "GetNode", - "pos": [ - 1394.278076171875, - -105.60762786865234 - ], - "size": [ - 210, - 60 - ], - "flags": { - "collapsed": true - }, - "order": 14, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 274 - ] - } - ], - "title": "Get_InputLatents", - "properties": {}, - "widgets_values": [ - "InputLatents" - ], - "color": "#323", - "bgcolor": "#535" - }, { "id": 127, "type": "WanVideoExperimentalArgs", @@ -1471,7 +1332,7 @@ 130 ], "flags": {}, - "order": 15, + "order": 14, "mode": 0, "inputs": [], "outputs": [ @@ -1509,7 +1370,7 @@ "flags": { "collapsed": true }, - "order": 16, + "order": 15, "mode": 0, "inputs": [], "outputs": [ @@ -1543,7 +1404,7 @@ "flags": { "collapsed": true }, - "order": 17, + "order": 16, "mode": 0, "inputs": [], "outputs": [ @@ -1575,7 +1436,7 @@ 46 ], "flags": {}, - "order": 27, + "order": 28, "mode": 2, "inputs": [ { @@ -1617,7 +1478,7 @@ "flags": { "collapsed": true }, - "order": 44, + "order": 45, "mode": 0, "inputs": [ { @@ -1657,7 +1518,7 @@ "flags": { "collapsed": true }, - "order": 18, + "order": 17, "mode": 0, "inputs": [], "outputs": [ @@ -1689,7 +1550,7 @@ 688.150634765625 ], "flags": {}, - "order": 30, + "order": 34, "mode": 0, "inputs": [ { @@ -1754,6 +1615,12 @@ "shape": 7, "type": "EXPERIMENTALARGS", "link": 334 + }, + { + "name": "sigmas", + "shape": 7, + "type": "SIGMAS", + "link": null } ], "outputs": [ @@ -1781,8 +1648,7 @@ 0, 1, false, - "comfy", - "" + "comfy" ] }, { @@ -1797,7 +1663,7 @@ 178 ], "flags": {}, - "order": 19, + "order": 18, "mode": 0, "inputs": [], "outputs": [ @@ -1832,10 +1698,10 @@ ], "size": [ 908.9017944335938, - 912.1107788085938 + 334 ], "flags": {}, - "order": 43, + "order": 46, "mode": 0, "inputs": [ { @@ -1900,53 +1766,6 @@ } } }, - { - "id": 56, - "type": "WanVideoReCamMasterCameraEmbed", - "pos": [ - 1379.0372314453125, - -36.21258544921875 - ], - "size": [ - 356.0601806640625, - 79.98188781738281 - ], - "flags": {}, - "order": 25, - "mode": 0, - "inputs": [ - { - "name": "latents", - "type": "LATENT", - "link": 274 - } - ], - "outputs": [ - { - "name": "camera_embeds", - "type": "WANVIDIMAGE_EMBEDS", - "links": [ - 102, - 272 - ] - }, - { - "name": "camera_poses", - "type": "CAMERAPOSES", - "links": [ - 242 - ] - } - ], - "properties": { - "cnr_id": "ComfyUI-WanVideoWrapper", - "ver": "11e9166d0b00fe3b1e6ebb0a3d1db50ce7a56d58", - "Node name for S&R": "WanVideoReCamMasterCameraEmbed" - }, - "widgets_values": [ - "arc_right" - ] - }, { "id": 22, "type": "WanVideoModelLoader", @@ -1959,7 +1778,7 @@ 234 ], "flags": {}, - "order": 20, + "order": 19, "mode": 0, "inputs": [ { @@ -2017,6 +1836,248 @@ ], "color": "#223", "bgcolor": "#335" + }, + { + "id": 138, + "type": "ReCamMasterPoseVisualizer", + "pos": [ + 1758.8209228515625, + 168.25579833984375 + ], + "size": [ + 349.6756591796875, + 130 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "camera_poses", + "type": "CAMERAPOSES", + "link": 242 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 243 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "2dc25c150ec4288e0b8689fc49fe2c9c8ab01999", + "Node name for S&R": "ReCamMasterPoseVisualizer" + }, + "widgets_values": [ + 0.10000000000000002, + 0.20000000000000004, + 0.4000000000000001, + 0.5000000000000001 + ] + }, + { + "id": 157, + "type": "GetNode", + "pos": [ + 1339.9281005859375, + -96.67338562011719 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 274, + 338 + ] + } + ], + "title": "Get_InputLatents", + "properties": {}, + "widgets_values": [ + "InputLatents" + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 56, + "type": "WanVideoReCamMasterCameraEmbed", + "pos": [ + 1338.8333740234375, + -36.212589263916016 + ], + "size": [ + 356.0601806640625, + 79.98188781738281 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "camera_poses", + "type": "CAMERAPOSES", + "link": 340 + }, + { + "name": "latents", + "type": "LATENT", + "link": 274 + } + ], + "outputs": [ + { + "name": "camera_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 272 + ] + }, + { + "name": "camera_poses", + "type": "CAMERAPOSES", + "links": [ + 242 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "11e9166d0b00fe3b1e6ebb0a3d1db50ce7a56d58", + "Node name for S&R": "WanVideoReCamMasterCameraEmbed" + }, + "widgets_values": [] + }, + { + "id": 206, + "type": "WanVideoReCamMasterGenerateOrbitCamera", + "pos": [ + 1305.5355224609375, + 185.5100555419922 + ], + "size": [ + 384.8144836425781, + 82 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "camera_poses", + "type": "CAMERAPOSES", + "links": [] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "8257cd1f8abaa6504248b946f31c5173c0228b3d", + "Node name for S&R": "WanVideoReCamMasterGenerateOrbitCamera" + }, + "widgets_values": [ + 81, + 90 + ] + }, + { + "id": 74, + "type": "WidgetToString", + "pos": [ + 2128.5166015625, + -432.6599426269531 + ], + "size": [ + 315, + 154 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "any_input", + "shape": 7, + "type": "*", + "link": 341 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 212 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "d57154c3a808b8a3f232ed293eaa2d000867c884", + "Node name for S&R": "WidgetToString" + }, + "widgets_values": [ + 0, + "camera_type", + false, + "", + 2 + ] + }, + { + "id": 205, + "type": "WanVideoReCamMasterDefaultCamera", + "pos": [ + 1317.4481201171875, + -241.10047912597656 + ], + "size": [ + 388.8835754394531, + 58 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "latents", + "type": "LATENT", + "link": 338 + } + ], + "outputs": [ + { + "name": "camera_poses", + "type": "CAMERAPOSES", + "links": [ + 340, + 341 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "8257cd1f8abaa6504248b946f31c5173c0228b3d", + "Node name for S&R": "WanVideoReCamMasterDefaultCamera" + }, + "widgets_values": [ + "pan_right" + ] } ], "links": [ @@ -2060,14 +2121,6 @@ 1, "CONDITIONING" ], - [ - 102, - 56, - 0, - 74, - 0, - "*" - ], [ 210, 28, @@ -2257,7 +2310,7 @@ 157, 0, 56, - 0, + 1, "LATENT" ], [ @@ -2291,6 +2344,30 @@ 155, 6, "TEACACHEARGS" + ], + [ + 338, + 157, + 0, + 205, + 0, + "LATENT" + ], + [ + 340, + 205, + 0, + 56, + 0, + "CAMERAPOSES" + ], + [ + 341, + 205, + 0, + 74, + 0, + "*" ] ], "groups": [ @@ -2337,13 +2414,12 @@ "config": {}, "extra": { "ds": { - "scale": 0.7400249944258357, + "scale": 0.611590904484162, "offset": [ - 1311.6629502036258, - 1366.2150672288358 + 1176.5764579377562, + 1095.9393193240473 ] }, - "linkExtensions": [], "node_versions": { "ComfyUI-WanVideoWrapper": "5a2383621a05825d0d0437781afcb8552d9590fd", "comfy-core": "0.3.26", @@ -2352,7 +2428,8 @@ "VHS_latentpreview": true, "VHS_latentpreviewrate": 0, "VHS_MetadataImage": true, - "VHS_KeepIntermediate": true + "VHS_KeepIntermediate": true, + "frontendVersion": "1.16.7" }, "version": 0.4 } \ No newline at end of file diff --git a/recammaster/nodes.py b/recammaster/nodes.py index de4f203..c41f254 100644 --- a/recammaster/nodes.py +++ b/recammaster/nodes.py @@ -10,9 +10,16 @@ class Camera(object): c2w_mat = np.array(c2w).reshape(4, 4) self.c2w_mat = c2w_mat self.w2c_mat = np.linalg.inv(c2w_mat) - -class WanVideoReCamMasterCameraEmbed: +def parse_matrix(matrix_str): + rows = matrix_str.strip().split('] [') + matrix = [] + for row in rows: + row = row.replace('[', '').replace(']', '') + matrix.append(list(map(float, row.split()))) + return np.array(matrix) + +class WanVideoReCamMasterDefaultCamera: @classmethod def INPUT_TYPES(s): return {"required": { @@ -32,18 +39,16 @@ class WanVideoReCamMasterCameraEmbed: }, } - RETURN_TYPES = ("WANVIDIMAGE_EMBEDS", "CAMERAPOSES",) - RETURN_NAMES = ("camera_embeds", "camera_poses",) + RETURN_TYPES = ("CAMERAPOSES",) + RETURN_NAMES = ("camera_poses",) FUNCTION = "process" CATEGORY = "WanVideoWrapper" DESCRIPTION = "https://github.com/KwaiVGI/ReCamMaster" def process(self, camera_type, latents): - # load camera import json - from einops import rearrange - camera_data_path = os.path.join(script_directory, "camera_extrinsics.json") + camera_data_path = os.path.join(script_directory, "recam_extrinsics.json") with open(camera_data_path, 'r') as file: cam_data = json.load(file) @@ -65,10 +70,96 @@ class WanVideoReCamMasterCameraEmbed: } cam_idx = list(range(num_frames))[::4] - traj = [self.parse_matrix(cam_data[f"frame{idx}"][f"cam{int(camera_type_map[camera_type]):02d}"]) for idx in cam_idx] + traj = [parse_matrix(cam_data[f"frame{idx}"][f"cam{int(camera_type_map[camera_type]):02d}"]) for idx in cam_idx] traj = np.stack(traj).transpose(0, 2, 1) + + return (traj,) + +class WanVideoReCamMasterGenerateOrbitCamera: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "num_frames": ("INT", {"default": 81, "min": 1, "max": 1000, "step": 1, "tooltip": "Number of frames to generate"}), + "degrees": ("INT", {"default": 90, "min": -180, "max": 180, "step": 1, "tooltip": "Degrees to orbit"}), + }, + } + + RETURN_TYPES = ("CAMERAPOSES",) + RETURN_NAMES = ("camera_poses",) + FUNCTION = "process" + CATEGORY = "WanVideoWrapper" + DESCRIPTION = "https://github.com/KwaiVGI/ReCamMaster" + + def process(self, degrees, num_frames): + + def generate_orbit(num_frames=num_frames, degrees=degrees): + camera_data = [] + center = np.array([3390, 1380, 240]) # Center point of orbit + + for i in range(num_frames): + # Calculate angle from 0 to specified degrees + angle = i * degrees / (num_frames - 1) + angle_rad = np.radians(angle) + + # Calculate position - circular path around center + x = center[0] - np.cos(angle_rad) + y = center[1] - np.sin(angle_rad) + z = center[2] + + # Calculate direction from camera to center point + camera_pos = np.array([x, y, z]) + dir_to_center = center - camera_pos + + # Calculate the angle needed to face the center + look_angle = np.arctan2(dir_to_center[1], dir_to_center[0]) + + # Rotation matrix for facing the center (corrected) + cos_look = np.cos(look_angle) + sin_look = np.sin(look_angle) + + # Create transformation matrix directly + transform = np.array([ + [cos_look, -sin_look, 0, x], + [sin_look, cos_look, 0, y], + [0, 0, 1, z], + [0, 0, 0, 1] + ]) + + camera_data.append(transform) + + return camera_data + + # Generate orbit data + camera_transforms = generate_orbit(num_frames=num_frames, degrees=degrees) + + traj = camera_transforms[::4] + traj = np.stack(traj) + + return (traj,) +class WanVideoReCamMasterCameraEmbed: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "camera_poses": ("CAMERAPOSES",), + "latents": ("LATENT", {"tooltip": "source video"}), + }, + } + + RETURN_TYPES = ("WANVIDIMAGE_EMBEDS", "CAMERAPOSES",) + RETURN_NAMES = ("camera_embeds", "camera_poses",) + FUNCTION = "process" + CATEGORY = "WanVideoWrapper" + DESCRIPTION = "https://github.com/KwaiVGI/ReCamMaster" + + def process(self, camera_poses, latents): + from einops import rearrange + samples = latents["samples"].squeeze(0) + C, T, H, W = samples.shape + num_frames = (T - 1) * 4 + 1 + + c2ws = [] - for c2w in traj: + for c2w in camera_poses: c2w = c2w[:, [1, 2, 0, 3]] c2w[:3, 1] *= -1. c2w[:3, 3] /= 100 @@ -93,15 +184,7 @@ class WanVideoReCamMasterCameraEmbed: } } - return (embeds, traj,) - - def parse_matrix(self, matrix_str): - rows = matrix_str.strip().split('] [') - matrix = [] - for row in rows: - row = row.replace('[', '').replace(']', '') - matrix.append(list(map(float, row.split()))) - return np.array(matrix) + return (embeds, camera_poses,) def get_relative_pose(self, cam_params): abs_w2cs = [cam_param.w2c_mat for cam_param in cam_params] @@ -274,8 +357,12 @@ or a .txt file with RealEstate camera intrinsics and coordinates, in a 3D plot. NODE_CLASS_MAPPINGS = { "WanVideoReCamMasterCameraEmbed": WanVideoReCamMasterCameraEmbed, "ReCamMasterPoseVisualizer": ReCamMasterPoseVisualizer, + "WanVideoReCamMasterGenerateOrbitCamera": WanVideoReCamMasterGenerateOrbitCamera, + "WanVideoReCamMasterDefaultCamera": WanVideoReCamMasterDefaultCamera, } NODE_DISPLAY_NAME_MAPPINGS = { "WanVideoReCamMasterCameraEmbed": "WanVideo ReCamMaster Camera Embed", "ReCamMasterPoseVisualizer": "ReCamMaster Pose Visualizer", + "WanVideoReCamMasterGenerateOrbitCamera": "WanVideo ReCamMaster Generate Orbit Camera", + "WanVideoReCamMasterDefaultCamera": "WanVideo ReCamMaster Default Camera", } diff --git a/recammaster/camera_extrinsics.json b/recammaster/recam_extrinsics.json similarity index 100% rename from recammaster/camera_extrinsics.json rename to recammaster/recam_extrinsics.json