nodes/kijai-ComfyUI-WanVideoWrapper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ReCamMaster custom orbit cam generation node

Browse Source
This commit is contained in:
kijai
2025-04-17 02:48:54 +03:00
parent 8257cd1f8a
commit cc8450b1a7
3 changed files with 419 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
example_workflows/wanvideo_1_3B_ReCamMaster_example_01.json
+314 -237
View File
@@ -1,8 +1,8 @@
{
"id": "c6e410bc-5e2c-460b-ae81-c91b6094fbb1",
"revision": 0,
"last_node_id": 204,
"last_link_id": 336,
"last_node_id": 206,
"last_link_id": 341,
"nodes": [
{
"id": 42,
@@ -101,7 +101,7 @@
200
],
"flags": {},
"order": 21,
"order": 22,
"mode": 2,
"inputs": [
{
@@ -258,7 +258,7 @@
174
],
"flags": {},
"order": 34,
"order": 39,
"mode": 0,
"inputs": [
{
@@ -371,7 +371,7 @@
200
],
"flags": {},
"order": 22,
"order": 23,
"mode": 2,
"inputs": [
{
@@ -413,7 +413,7 @@
86
],
"flags": {},
"order": 24,
"order": 25,
"mode": 0,
"inputs": [
{
@@ -647,9 +647,7 @@
"ver": "0.3.27",
"Node name for S&R": "PreviewImage"
},
"widgets_values": [
""
]
"widgets_values": []
},
{
"id": 125,
@@ -663,15 +661,12 @@
190.28567504882812
],
"flags": {},
"order": 40,
"order": 41,
"mode": 0,
"inputs": [
{
"name": "text",
"type": "STRING",
"widget": {
"name": "text"
},
"link": 215
}
],
@@ -689,7 +684,7 @@
"Node name for S&R": "ShowText|pysssss"
},
"widgets_values": [
"",
"A man in a suit and tie walking down a hallway. He has a friendly expression and is looking directly at the camera. The hallway has beige walls adorned with framed black and white photographs. There is a door on the left side of the hallway and a poster on the wall. The lighting is soft and natural. The image is high quality and has a watermark in the bottom right corner.",
"A man in a suit and tie walking down a hallway. He has a friendly expression and is looking directly at the camera. The hallway has beige walls adorned with framed black and white photographs. There is a door on the left side of the hallway and a poster on the wall. The lighting is soft and natural. The image is high quality and has a watermark in the bottom right corner."
]
},
@@ -745,7 +740,7 @@
261.5306701660156
],
"flags": {},
"order": 41,
"order": 42,
"mode": 0,
"inputs": [
{
@@ -842,7 +837,7 @@
"flags": {
"collapsed": true
},
"order": 23,
"order": 24,
"mode": 0,
"inputs": [
{
@@ -914,7 +909,7 @@
266
],
"flags": {},
"order": 28,
"order": 29,
"mode": 0,
"inputs": [
{
@@ -922,28 +917,22 @@
"type": "IMAGE",
"link": 244
},
{
"name": "get_image_size",
"shape": 7,
"type": "IMAGE",
"link": null
},
{
"name": "width_input",
"shape": 7,
"type": "INT",
"widget": {
"name": "width_input"
},
"link": null
},
{
"name": "height_input",
"shape": 7,
"type": "INT",
"widget": {
"name": "height_input"
},
"link": null
},
{
"name": "get_image_size",
"shape": 7,
"type": "IMAGE",
"link": null
}
],
@@ -978,8 +967,6 @@
"lanczos",
false,
16,
0,
0,
"center"
]
},
@@ -1025,50 +1012,6 @@
"color": "#2a363b",
"bgcolor": "#3f5159"
},
{
"id": 74,
"type": "WidgetToString",
"pos": [
2128.5166015625,
-432.6599426269531
],
"size": [
315,
154
],
"flags": {},
"order": 29,
"mode": 0,
"inputs": [
{
"name": "any_input",
"shape": 7,
"type": "*",
"link": 102
}
],
"outputs": [
{
"name": "STRING",
"type": "STRING",
"links": [
212
]
}
],
"properties": {
"cnr_id": "comfyui-kjnodes",
"ver": "d57154c3a808b8a3f232ed293eaa2d000867c884",
"Node name for S&R": "WidgetToString"
},
"widgets_values": [
0,
"camera_type",
false,
"",
2
]
},
{
"id": 58,
"type": "WanVideoEncode",
@@ -1139,7 +1082,7 @@
"flags": {
"collapsed": true
},
"order": 42,
"order": 43,
"mode": 0,
"inputs": [
{
@@ -1177,7 +1120,7 @@
555.8994140625
],
"flags": {},
"order": 35,
"order": 40,
"mode": 0,
"inputs": [
{
@@ -1192,9 +1135,7 @@
"ver": "0.3.27",
"Node name for S&R": "PreviewImage"
},
"widgets_values": [
""
]
"widgets_values": []
},
{
"id": 128,
@@ -1292,7 +1233,7 @@
274
],
"flags": {},
"order": 39,
"order": 44,
"mode": 0,
"inputs": [
{
@@ -1304,9 +1245,6 @@
"name": "caption",
"shape": 7,
"type": "STRING",
"widget": {
"name": "caption"
},
"link": null
},
{
@@ -1341,8 +1279,7 @@
"black",
"FreeMonoBoldOblique.otf",
"input",
"up",
""
"up"
]
},
{
@@ -1353,7 +1290,7 @@
-1155.6121826171875
],
"size": [
390.5999755859375,
421.6000061035156,
202
],
"flags": {},
@@ -1383,82 +1320,6 @@
128
]
},
{
"id": 138,
"type": "ReCamMasterPoseVisualizer",
"pos": [
1597.2598876953125,
177.93458557128906
],
"size": [
349.6756591796875,
130
],
"flags": {},
"order": 31,
"mode": 0,
"inputs": [
{
"name": "camera_poses",
"type": "CAMERAPOSES",
"link": 242
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
243
]
}
],
"properties": {
"cnr_id": "ComfyUI-WanVideoWrapper",
"ver": "2dc25c150ec4288e0b8689fc49fe2c9c8ab01999",
"Node name for S&R": "ReCamMasterPoseVisualizer"
},
"widgets_values": [
0.10000000000000002,
0.20000000000000004,
0.4000000000000001,
0.5000000000000001
]
},
{
"id": 157,
"type": "GetNode",
"pos": [
1394.278076171875,
-105.60762786865234
],
"size": [
210,
60
],
"flags": {
"collapsed": true
},
"order": 14,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
274
]
}
],
"title": "Get_InputLatents",
"properties": {},
"widgets_values": [
"InputLatents"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 127,
"type": "WanVideoExperimentalArgs",
@@ -1471,7 +1332,7 @@
130
],
"flags": {},
"order": 15,
"order": 14,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1509,7 +1370,7 @@
"flags": {
"collapsed": true
},
"order": 16,
"order": 15,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1543,7 +1404,7 @@
"flags": {
"collapsed": true
},
"order": 17,
"order": 16,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1575,7 +1436,7 @@
46
],
"flags": {},
"order": 27,
"order": 28,
"mode": 2,
"inputs": [
{
@@ -1617,7 +1478,7 @@
"flags": {
"collapsed": true
},
"order": 44,
"order": 45,
"mode": 0,
"inputs": [
{
@@ -1657,7 +1518,7 @@
"flags": {
"collapsed": true
},
"order": 18,
"order": 17,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1689,7 +1550,7 @@
688.150634765625
],
"flags": {},
"order": 30,
"order": 34,
"mode": 0,
"inputs": [
{
@@ -1754,6 +1615,12 @@
"shape": 7,
"type": "EXPERIMENTALARGS",
"link": 334
},
{
"name": "sigmas",
"shape": 7,
"type": "SIGMAS",
"link": null
}
],
"outputs": [
@@ -1781,8 +1648,7 @@
0,
1,
false,
"comfy",
""
"comfy"
]
},
{
@@ -1797,7 +1663,7 @@
178
],
"flags": {},
"order": 19,
"order": 18,
"mode": 0,
"inputs": [],
"outputs": [
@@ -1832,10 +1698,10 @@
],
"size": [
908.9017944335938,
912.1107788085938
334
],
"flags": {},
"order": 43,
"order": 46,
"mode": 0,
"inputs": [
{
@@ -1900,53 +1766,6 @@
}
}
},
{
"id": 56,
"type": "WanVideoReCamMasterCameraEmbed",
"pos": [
1379.0372314453125,
-36.21258544921875
],
"size": [
356.0601806640625,
79.98188781738281
],
"flags": {},
"order": 25,
"mode": 0,
"inputs": [
{
"name": "latents",
"type": "LATENT",
"link": 274
}
],
"outputs": [
{
"name": "camera_embeds",
"type": "WANVIDIMAGE_EMBEDS",
"links": [
102,
272
]
},
{
"name": "camera_poses",
"type": "CAMERAPOSES",
"links": [
242
]
}
],
"properties": {
"cnr_id": "ComfyUI-WanVideoWrapper",
"ver": "11e9166d0b00fe3b1e6ebb0a3d1db50ce7a56d58",
"Node name for S&R": "WanVideoReCamMasterCameraEmbed"
},
"widgets_values": [
"arc_right"
]
},
{
"id": 22,
"type": "WanVideoModelLoader",
@@ -1959,7 +1778,7 @@
234
],
"flags": {},
"order": 20,
"order": 19,
"mode": 0,
"inputs": [
{
@@ -2017,6 +1836,248 @@
],
"color": "#223",
"bgcolor": "#335"
},
{
"id": 138,
"type": "ReCamMasterPoseVisualizer",
"pos": [
1758.8209228515625,
168.25579833984375
],
"size": [
349.6756591796875,
130
],
"flags": {},
"order": 35,
"mode": 0,
"inputs": [
{
"name": "camera_poses",
"type": "CAMERAPOSES",
"link": 242
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
243
]
}
],
"properties": {
"cnr_id": "ComfyUI-WanVideoWrapper",
"ver": "2dc25c150ec4288e0b8689fc49fe2c9c8ab01999",
"Node name for S&R": "ReCamMasterPoseVisualizer"
},
"widgets_values": [
0.10000000000000002,
0.20000000000000004,
0.4000000000000001,
0.5000000000000001
]
},
{
"id": 157,
"type": "GetNode",
"pos": [
1339.9281005859375,
-96.67338562011719
],
"size": [
210,
60
],
"flags": {
"collapsed": true
},
"order": 20,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "LATENT",
"type": "LATENT",
"links": [
274,
338
]
}
],
"title": "Get_InputLatents",
"properties": {},
"widgets_values": [
"InputLatents"
],
"color": "#323",
"bgcolor": "#535"
},
{
"id": 56,
"type": "WanVideoReCamMasterCameraEmbed",
"pos": [
1338.8333740234375,
-36.212589263916016
],
"size": [
356.0601806640625,
79.98188781738281
],
"flags": {},
"order": 30,
"mode": 0,
"inputs": [
{
"name": "camera_poses",
"type": "CAMERAPOSES",
"link": 340
},
{
"name": "latents",
"type": "LATENT",
"link": 274
}
],
"outputs": [
{
"name": "camera_embeds",
"type": "WANVIDIMAGE_EMBEDS",
"links": [
272
]
},
{
"name": "camera_poses",
"type": "CAMERAPOSES",
"links": [
242
]
}
],
"properties": {
"cnr_id": "ComfyUI-WanVideoWrapper",
"ver": "11e9166d0b00fe3b1e6ebb0a3d1db50ce7a56d58",
"Node name for S&R": "WanVideoReCamMasterCameraEmbed"
},
"widgets_values": []
},
{
"id": 206,
"type": "WanVideoReCamMasterGenerateOrbitCamera",
"pos": [
1305.5355224609375,
185.5100555419922
],
"size": [
384.8144836425781,
82
],
"flags": {},
"order": 21,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "camera_poses",
"type": "CAMERAPOSES",
"links": []
}
],
"properties": {
"cnr_id": "ComfyUI-WanVideoWrapper",
"ver": "8257cd1f8abaa6504248b946f31c5173c0228b3d",
"Node name for S&R": "WanVideoReCamMasterGenerateOrbitCamera"
},
"widgets_values": [
81,
90
]
},
{
"id": 74,
"type": "WidgetToString",
"pos": [
2128.5166015625,
-432.6599426269531
],
"size": [
315,
154
],
"flags": {},
"order": 31,
"mode": 0,
"inputs": [
{
"name": "any_input",
"shape": 7,
"type": "*",
"link": 341
}
],
"outputs": [
{
"name": "STRING",
"type": "STRING",
"links": [
212
]
}
],
"properties": {
"cnr_id": "comfyui-kjnodes",
"ver": "d57154c3a808b8a3f232ed293eaa2d000867c884",
"Node name for S&R": "WidgetToString"
},
"widgets_values": [
0,
"camera_type",
false,
"",
2
]
},
{
"id": 205,
"type": "WanVideoReCamMasterDefaultCamera",
"pos": [
1317.4481201171875,
-241.10047912597656
],
"size": [
388.8835754394531,
58
],
"flags": {},
"order": 27,
"mode": 0,
"inputs": [
{
"name": "latents",
"type": "LATENT",
"link": 338
}
],
"outputs": [
{
"name": "camera_poses",
"type": "CAMERAPOSES",
"links": [
340,
341
]
}
],
"properties": {
"cnr_id": "ComfyUI-WanVideoWrapper",
"ver": "8257cd1f8abaa6504248b946f31c5173c0228b3d",
"Node name for S&R": "WanVideoReCamMasterDefaultCamera"
},
"widgets_values": [
"pan_right"
]
}
],
"links": [
@@ -2060,14 +2121,6 @@
1,
"CONDITIONING"
],
[
102,
56,
0,
74,
0,
"*"
],
[
210,
28,
@@ -2257,7 +2310,7 @@
157,
0,
56,
0,
1,
"LATENT"
],
[
@@ -2291,6 +2344,30 @@
155,
6,
"TEACACHEARGS"
],
[
338,
157,
0,
205,
0,
"LATENT"
],
[
340,
205,
0,
56,
0,
"CAMERAPOSES"
],
[
341,
205,
0,
74,
0,
"*"
]
],
"groups": [
@@ -2337,13 +2414,12 @@
"config": {},
"extra": {
"ds": {
"scale": 0.7400249944258357,
"scale": 0.611590904484162,
"offset": [
1311.6629502036258,
1366.2150672288358
1176.5764579377562,
1095.9393193240473
]
},
"linkExtensions": [],
"node_versions": {
"ComfyUI-WanVideoWrapper": "5a2383621a05825d0d0437781afcb8552d9590fd",
"comfy-core": "0.3.26",
@@ -2352,7 +2428,8 @@
"VHS_latentpreview": true,
"VHS_latentpreviewrate": 0,
"VHS_MetadataImage": true,
"VHS_KeepIntermediate": true
"VHS_KeepIntermediate": true,
"frontendVersion": "1.16.7"
},
"version": 0.4
}
recammaster/nodes.py
+105 -18
View File
@@ -10,9 +10,16 @@ class Camera(object):
c2w_mat = np.array(c2w).reshape(4, 4)
self.c2w_mat = c2w_mat
self.w2c_mat = np.linalg.inv(c2w_mat)
class WanVideoReCamMasterCameraEmbed:
def parse_matrix(matrix_str):
rows = matrix_str.strip().split('] [')
matrix = []
for row in rows:
row = row.replace('[', '').replace(']', '')
matrix.append(list(map(float, row.split())))
return np.array(matrix)
class WanVideoReCamMasterDefaultCamera:
@classmethod
def INPUT_TYPES(s):
return {"required": {
@@ -32,18 +39,16 @@ class WanVideoReCamMasterCameraEmbed:
},
}
RETURN_TYPES = ("WANVIDIMAGE_EMBEDS", "CAMERAPOSES",)
RETURN_NAMES = ("camera_embeds", "camera_poses",)
RETURN_TYPES = ("CAMERAPOSES",)
RETURN_NAMES = ("camera_poses",)
FUNCTION = "process"
CATEGORY = "WanVideoWrapper"
DESCRIPTION = "https://github.com/KwaiVGI/ReCamMaster"
def process(self, camera_type, latents):
# load camera
import json
from einops import rearrange
camera_data_path = os.path.join(script_directory, "camera_extrinsics.json")
camera_data_path = os.path.join(script_directory, "recam_extrinsics.json")
with open(camera_data_path, 'r') as file:
cam_data = json.load(file)
@@ -65,10 +70,96 @@ class WanVideoReCamMasterCameraEmbed:
}
cam_idx = list(range(num_frames))[::4]
traj = [self.parse_matrix(cam_data[f"frame{idx}"][f"cam{int(camera_type_map[camera_type]):02d}"]) for idx in cam_idx]
traj = [parse_matrix(cam_data[f"frame{idx}"][f"cam{int(camera_type_map[camera_type]):02d}"]) for idx in cam_idx]
traj = np.stack(traj).transpose(0, 2, 1)
return (traj,)
class WanVideoReCamMasterGenerateOrbitCamera:
@classmethod
def INPUT_TYPES(s):
return {"required": {
"num_frames": ("INT", {"default": 81, "min": 1, "max": 1000, "step": 1, "tooltip": "Number of frames to generate"}),
"degrees": ("INT", {"default": 90, "min": -180, "max": 180, "step": 1, "tooltip": "Degrees to orbit"}),
},
}
RETURN_TYPES = ("CAMERAPOSES",)
RETURN_NAMES = ("camera_poses",)
FUNCTION = "process"
CATEGORY = "WanVideoWrapper"
DESCRIPTION = "https://github.com/KwaiVGI/ReCamMaster"
def process(self, degrees, num_frames):
def generate_orbit(num_frames=num_frames, degrees=degrees):
camera_data = []
center = np.array([3390, 1380, 240]) # Center point of orbit
for i in range(num_frames):
# Calculate angle from 0 to specified degrees
angle = i * degrees / (num_frames - 1)
angle_rad = np.radians(angle)
# Calculate position - circular path around center
x = center[0] - np.cos(angle_rad)
y = center[1] - np.sin(angle_rad)
z = center[2]
# Calculate direction from camera to center point
camera_pos = np.array([x, y, z])
dir_to_center = center - camera_pos
# Calculate the angle needed to face the center
look_angle = np.arctan2(dir_to_center[1], dir_to_center[0])
# Rotation matrix for facing the center (corrected)
cos_look = np.cos(look_angle)
sin_look = np.sin(look_angle)
# Create transformation matrix directly
transform = np.array([
[cos_look, -sin_look, 0, x],
[sin_look, cos_look, 0, y],
[0, 0, 1, z],
[0, 0, 0, 1]
])
camera_data.append(transform)
return camera_data
# Generate orbit data
camera_transforms = generate_orbit(num_frames=num_frames, degrees=degrees)
traj = camera_transforms[::4]
traj = np.stack(traj)
return (traj,)
class WanVideoReCamMasterCameraEmbed:
@classmethod
def INPUT_TYPES(s):
return {"required": {
"camera_poses": ("CAMERAPOSES",),
"latents": ("LATENT", {"tooltip": "source video"}),
},
}
RETURN_TYPES = ("WANVIDIMAGE_EMBEDS", "CAMERAPOSES",)
RETURN_NAMES = ("camera_embeds", "camera_poses",)
FUNCTION = "process"
CATEGORY = "WanVideoWrapper"
DESCRIPTION = "https://github.com/KwaiVGI/ReCamMaster"
def process(self, camera_poses, latents):
from einops import rearrange
samples = latents["samples"].squeeze(0)
C, T, H, W = samples.shape
num_frames = (T - 1) * 4 + 1
c2ws = []
for c2w in traj:
for c2w in camera_poses:
c2w = c2w[:, [1, 2, 0, 3]]
c2w[:3, 1] *= -1.
c2w[:3, 3] /= 100
@@ -93,15 +184,7 @@ class WanVideoReCamMasterCameraEmbed:
}
}
return (embeds, traj,)
def parse_matrix(self, matrix_str):
rows = matrix_str.strip().split('] [')
matrix = []
for row in rows:
row = row.replace('[', '').replace(']', '')
matrix.append(list(map(float, row.split())))
return np.array(matrix)
return (embeds, camera_poses,)
def get_relative_pose(self, cam_params):
abs_w2cs = [cam_param.w2c_mat for cam_param in cam_params]
@@ -274,8 +357,12 @@ or a .txt file with RealEstate camera intrinsics and coordinates, in a 3D plot.
NODE_CLASS_MAPPINGS = {
"WanVideoReCamMasterCameraEmbed": WanVideoReCamMasterCameraEmbed,
"ReCamMasterPoseVisualizer": ReCamMasterPoseVisualizer,
"WanVideoReCamMasterGenerateOrbitCamera": WanVideoReCamMasterGenerateOrbitCamera,
"WanVideoReCamMasterDefaultCamera": WanVideoReCamMasterDefaultCamera,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"WanVideoReCamMasterCameraEmbed": "WanVideo ReCamMaster Camera Embed",
"ReCamMasterPoseVisualizer": "ReCamMaster Pose Visualizer",
"WanVideoReCamMasterGenerateOrbitCamera": "WanVideo ReCamMaster Generate Orbit Camera",
"WanVideoReCamMasterDefaultCamera": "WanVideo ReCamMaster Default Camera",
}
recammaster/camera_extrinsics.json → recammaster/recam_extrinsics.json
View File