#Based on https://github.com/Mael-zys/T2M-GPT/blob/main/render_final.py from motiondiff_modules.mogen.smpl.rotation2xyz import Rotation2xyz import numpy as np from trimesh import Trimesh import os #https://stackoverflow.com/a/45756291 if os.name == 'posix' and "DISPLAY" not in os.environ: os.environ['PYOPENGL_PLATFORM'] = "egl" import torch import comfy.utils from motiondiff_modules.mogen.smpl.simplify_loc2rot import joints2smpl import pyrender from pyrender.shader_program import ShaderProgramCache from shapely import geometry import trimesh from pyrender.constants import RenderFlags from comfy.model_management import get_torch_device from tqdm import tqdm shader_dir = os.path.join(os.path.dirname(__file__), 'shaders') class WeakPerspectiveCamera(pyrender.Camera): def __init__(self, scale, translation, znear=pyrender.camera.DEFAULT_Z_NEAR, zfar=None, name=None): super(WeakPerspectiveCamera, self).__init__( znear=znear, zfar=zfar, name=name, ) self.scale = scale self.translation = translation def get_projection_matrix(self, width=None, height=None): P = np.eye(4) P[0, 0] = self.scale[0] P[1, 1] = self.scale[1] P[0, 3] = self.translation[0] * self.scale[0] P[1, 3] = -self.translation[1] * self.scale[1] P[2, 2] = -1 return P def render(motions): frames, njoints, nfeats = motions.shape MINS = motions.min(axis=0).min(axis=0) MAXS = motions.max(axis=0).max(axis=0) height_offset = MINS[1] motions[:, :, 1] -= height_offset trajec = motions[:, 0, [0, 2]] j2s = joints2smpl(num_frames=frames, device=get_torch_device()) rot2xyz = Rotation2xyz(device=get_torch_device()) faces = rot2xyz.smpl_model.faces print(f'Running SMPLify, it may take a few minutes.') motion_tensor, opt_dict = j2s.forward(motions) # [nframes, njoints, 3] vertices = rot2xyz(torch.tensor(motion_tensor).clone(), mask=None, pose_rep='rot6d', translation=True, glob=True, jointstype='vertices', vertstrans=True) frames = vertices.shape[3] # shape: 1, nb_frames, 3, nb_joints MINS = torch.min(torch.min(vertices[0], axis=0)[0], axis=1)[0] MAXS = torch.max(torch.max(vertices[0], axis=0)[0], axis=1)[0] # vertices[:,:,1,:] -= MINS[1] + 1e-5 out_list = [] minx = MINS[0] - 0.5 maxx = MAXS[0] + 0.5 minz = MINS[2] - 0.5 maxz = MAXS[2] + 0.5 polygon = geometry.Polygon([[minx, minz], [minx, maxz], [maxx, maxz], [maxx, minz]]) polygon_mesh = trimesh.creation.extrude_polygon(polygon, 1e-5) vid = [] for i in range(frames): if i % 10 == 0: print(i) mesh = Trimesh(vertices=vertices[0, :, :, i].squeeze().tolist(), faces=faces) base_color = (0.11, 0.53, 0.8, 0.5) ## OPAQUE rendering without alpha ## BLEND rendering consider alpha material = pyrender.MetallicRoughnessMaterial( metallicFactor=0.7, alphaMode='OPAQUE', baseColorFactor=base_color ) mesh = pyrender.Mesh.from_trimesh(mesh, material=material) polygon_mesh.visual.face_colors = [0, 0, 0, 0.21] polygon_render = pyrender.Mesh.from_trimesh(polygon_mesh, smooth=False) bg_color = [1, 1, 1, 0.8] scene = pyrender.Scene(bg_color=bg_color, ambient_light=(0.4, 0.4, 0.4)) sx, sy, tx, ty = [0.75, 0.75, 0, 0.10] camera = pyrender.PerspectiveCamera(yfov=(np.pi / 3.0)) light = pyrender.DirectionalLight(color=[1,1,1], intensity=300) scene.add(mesh) c = np.pi / 2 scene.add(polygon_render, pose=np.array([[ 1, 0, 0, 0], [ 0, np.cos(c), -np.sin(c), MINS[1].cpu().numpy()], [ 0, np.sin(c), np.cos(c), 0], [ 0, 0, 0, 1]])) light_pose = np.eye(4) light_pose[:3, 3] = [0, -1, 1] scene.add(light, pose=light_pose.copy()) light_pose[:3, 3] = [0, 1, 1] scene.add(light, pose=light_pose.copy()) light_pose[:3, 3] = [1, 1, 2] scene.add(light, pose=light_pose.copy()) c = -np.pi / 6 scene.add(camera, pose=[[ 1, 0, 0, (minx+maxx).cpu().numpy()/2], [ 0, np.cos(c), -np.sin(c), 1.5], [ 0, np.sin(c), np.cos(c), max(4, minz.cpu().numpy()+(1.5-MINS[1].cpu().numpy())*2, (maxx-minx).cpu().numpy())], [ 0, 0, 0, 1] ]) # render scene r = pyrender.OffscreenRenderer(960, 960) color, _ = r.render(scene, flags=RenderFlags.RGBA) # Image.fromarray(color).save(outdir+name+'_'+str(i)+'.png') vid.append(color) r.delete() out = np.stack(vid, axis=0) return out def render_from_smpl(thetas, yfov, move_x, move_y, move_z, x_rot, y_rot, z_rot, frame_width, frame_height, draw_platform=True, depth_only=False, normals=False, smpl_model_path=None, shape_parameters=None, normalized_to_vertices=False): if shape_parameters is not None: betas_tensor = torch.tensor([shape_parameters], dtype=torch.float32) batch_size = thetas.shape[3] betas_batch = betas_tensor.repeat(batch_size, 1) # Replicates the single sample across the batch betas_batch = betas_batch.to(device=get_torch_device()) else: betas_batch = None rot2xyz = Rotation2xyz(device=get_torch_device(), smpl_model_path=smpl_model_path, betas=betas_batch) faces = rot2xyz.smpl_model.faces vertices = rot2xyz(thetas.clone().to(get_torch_device()).detach(), mask=None, pose_rep='xyz' if normalized_to_vertices else 'rot6d', translation=True, glob=True, jointstype='vertices', vertstrans=True) frames = vertices.shape[3] # shape: 1, nb_frames, 3, nb_joints MINS = torch.min(torch.min(vertices[0], axis=0)[0], axis=1)[0] MAXS = torch.max(torch.max(vertices[0], axis=0)[0], axis=1)[0] minx = MINS[0] - 0.5 maxx = MAXS[0] + 0.5 minz = MINS[2] - 0.5 maxz = MAXS[2] + 0.5 if draw_platform: polygon = geometry.Polygon([[minx, minz], [minx, maxz], [maxx, maxz], [maxx, minz]]) polygon_mesh = trimesh.creation.extrude_polygon(polygon, 1e-5) polygon_mesh.visual.face_colors = [0, 0, 0, 0.21] polygon_render = pyrender.Mesh.from_trimesh(polygon_mesh, smooth=False) c = np.pi / 2 platform_pose=np.array([[ 1, 0, 0, 0], [ 0, np.cos(c), -np.sin(c), MINS[1].cpu().numpy()], [ 0, np.sin(c), np.cos(c), 0], [ 0, 0, 0, 1]]) base_color = (0.11, 0.53, 0.8, 0.5) material = pyrender.MetallicRoughnessMaterial( metallicFactor=0.7, alphaMode='OPAQUE', baseColorFactor=base_color ) x_translation = move_x #X-axis translation value y_translation = move_y # Y-axis translation value z_translation = move_z # Z-axis translation value initial_pos = [(minx+maxx).cpu().numpy()/2 + x_translation, y_translation, max(4, minz.cpu().numpy()+(1.5-MINS[1].cpu().numpy())*2, (maxx-minx).cpu().numpy()) + z_translation] alpha = np.radians(x_rot) beta = np.radians(y_rot) gamma = np.radians(z_rot) # Rotation matrix around X-axis R_x = [[1, 0, 0, 0], [0, np.cos(alpha), -np.sin(alpha), 0], [0, np.sin(alpha), np.cos(alpha), 0], [0, 0, 0, 1]] # Rotation matrix around Y-axis R_y = [[np.cos(beta), 0, np.sin(beta), 0], [0, 1, 0, 0], [-np.sin(beta), 0, np.cos(beta), 0], [0, 0, 0, 1]] # Rotation matrix around Z-axis R_z = [[np.cos(gamma), -np.sin(gamma), 0, 0], [np.sin(gamma), np.cos(gamma), 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]] # Combine rotations, order of multiplication depends on the desired rotation order R = np.dot(R_z, np.dot(R_y, R_x)) # Now, R is a 4x4 matrix that represents the rotation around X, Y, and Z # Translation vector T = [initial_pos[0], initial_pos[1], initial_pos[2], 1] # Combine the rotation and translation into the final transformation matrix camera_pose = np.dot(R, np.array([[1, 0, 0, T[0]], [0, 1, 0, T[1]], [0, 0, 1, T[2]], [0, 0, 0, 1]])) if normals and not depth_only: r = pyrender.OffscreenRenderer(frame_width, frame_height) r._renderer._program_cache = ShaderProgramCache(shader_dir=shader_dir) else: r = pyrender.OffscreenRenderer(frame_width, frame_height) light = pyrender.DirectionalLight(color=[1,1,1], intensity=300) light_positions = [ [0, -1, 1], [0, 1, 1], [1, 1, 2] ] # Create transformation matrices for each light light_poses = [np.eye(4) for _ in light_positions] for i, position in enumerate(light_positions): light_poses[i][:3, 3] = position #Build the scene camera = pyrender.PerspectiveCamera(yfov) bg_color = [1, 1, 1, 0.8] scene = pyrender.Scene(bg_color=bg_color, ambient_light=(0.4, 0.4, 0.4)) scene.add(camera, pose=camera_pose) if draw_platform: scene.add(polygon_render, pose=platform_pose) if not normals: for pose in light_poses: scene.add(light, pose=pose) # Render loop vid = [] vid_depth = [] print("Rendering SMPL human mesh...") pbar = comfy.utils.ProgressBar(frames) for i in tqdm(range(frames)): mesh = Trimesh(vertices=vertices[0, :, :, i].squeeze().tolist(), faces=faces) mesh = pyrender.Mesh.from_trimesh(mesh, material=material) mesh_node = pyrender.Node(mesh=mesh) scene.add_node(mesh_node) if depth_only: depth = r.render(scene, flags=RenderFlags.DEPTH_ONLY) color = np.ones([frame_height, frame_width, 4]) else: color, depth = r.render(scene, flags=RenderFlags.RGBA) vid.append(color) vid_depth.append(depth) scene.remove_node(mesh_node) pbar.update(1) r = None return np.stack(vid, axis=0), np.stack(vid_depth, axis=0) # verts_frames: list of [num_subjects, num_verts, 3] # cam_t_frames: list of [num_subjects, 3] def render_from_smpl_multiple_subjects(verts_frames, cam_t_frames, focal_length, fx_offset, fy_offset, move_x, move_y, move_z, x_rot, y_rot, z_rot, frame_width, frame_height, draw_platform=True, depth_only=False, normals=False, smpl_model_path=None): def vertices_to_trimesh(vertices, camera_translation, faces, rot_axis=[1,0,0], rot_angle=0,): mesh = trimesh.Trimesh(vertices + camera_translation, faces.copy()) rot = trimesh.transformations.rotation_matrix( np.radians(rot_angle), rot_axis) mesh.apply_transform(rot) rot = trimesh.transformations.rotation_matrix( np.radians(180), [1, 0, 0]) mesh.apply_transform(rot) return mesh rot2xyz = Rotation2xyz(device="cpu", smpl_model_path=smpl_model_path) faces = rot2xyz.smpl_model.faces MINS = torch.stack([verts_frame.min(0).values.min(0).values for verts_frame in verts_frames if verts_frame is not None]).min(0).values MAXS = torch.stack([verts_frame.max(0).values.max(0).values for verts_frame in verts_frames if verts_frame is not None]).max(0).values minx = MINS[0] - 0.5 maxx = MAXS[0] + 0.5 minz = MINS[2] - 0.5 maxz = MAXS[2] + 0.5 if draw_platform: polygon = geometry.Polygon([[minx, minz], [minx, maxz], [maxx, maxz], [maxx, minz]]) polygon_mesh = trimesh.creation.extrude_polygon(polygon, 1e-5) polygon_mesh.visual.face_colors = [0, 0, 0, 0.21] polygon_render = pyrender.Mesh.from_trimesh(polygon_mesh, smooth=False) c = np.pi / 2 platform_pose=np.array([[ 1, 0, 0, 0], [ 0, np.cos(c), -np.sin(c), MINS[1].cpu().numpy()], [ 0, np.sin(c), np.cos(c), 0], [ 0, 0, 0, 1]]) base_color = (0.11, 0.53, 0.8, 0.5) material = pyrender.MetallicRoughnessMaterial( metallicFactor=0.7, alphaMode='OPAQUE', baseColorFactor=base_color ) x_translation = move_x #X-axis translation value y_translation = move_y # Y-axis translation value z_translation = move_z # Z-axis translation value initial_pos = [(minx+maxx).cpu().numpy()/2 + x_translation, y_translation, max(4, minz.cpu().numpy()+(1.5-MINS[1].cpu().numpy())*2, (maxx-minx).cpu().numpy()) + z_translation] alpha = np.radians(x_rot) beta = np.radians(y_rot) gamma = np.radians(z_rot) # Rotation matrix around X-axis R_x = [[1, 0, 0, 0], [0, np.cos(alpha), -np.sin(alpha), 0], [0, np.sin(alpha), np.cos(alpha), 0], [0, 0, 0, 1]] # Rotation matrix around Y-axis R_y = [[np.cos(beta), 0, np.sin(beta), 0], [0, 1, 0, 0], [-np.sin(beta), 0, np.cos(beta), 0], [0, 0, 0, 1]] # Rotation matrix around Z-axis R_z = [[np.cos(gamma), -np.sin(gamma), 0, 0], [np.sin(gamma), np.cos(gamma), 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]] # Combine rotations, order of multiplication depends on the desired rotation order R = np.dot(R_z, np.dot(R_y, R_x)) # Now, R is a 4x4 matrix that represents the rotation around X, Y, and Z # Translation vector T = [initial_pos[0], initial_pos[1], initial_pos[2], 1] # Combine the rotation and translation into the final transformation matrix camera_pose = np.dot(R, np.array([[1, 0, 0, T[0]], [0, 1, 0, T[1]], [0, 0, 1, T[2]], [0, 0, 0, 1]])) if normals and not depth_only: r = pyrender.OffscreenRenderer(frame_width, frame_height) r._renderer._program_cache = ShaderProgramCache(shader_dir=shader_dir) else: r = pyrender.OffscreenRenderer(frame_width, frame_height) light = pyrender.DirectionalLight(color=[1,1,1], intensity=300) light_positions = [ [0, -1, 1], [0, 1, 1], [1, 1, 2] ] # Create transformation matrices for each light light_poses = [np.eye(4) for _ in light_positions] for i, position in enumerate(light_positions): light_poses[i][:3, 3] = position #Build the scene camera = pyrender.IntrinsicsCamera(fx=focal_length + fx_offset, fy=focal_length + fy_offset, cx=frame_width / 2, cy=frame_height / 2, zfar=1e12) bg_color = [1, 1, 1, 0.8] scene = pyrender.Scene(bg_color=bg_color, ambient_light=(0.4, 0.4, 0.4)) scene.add(camera, pose=camera_pose) if draw_platform: scene.add(polygon_render, pose=platform_pose) if not normals: for pose in light_poses: scene.add(light, pose=pose) # Render loop vid = [] vid_depth = [] print("Rendering SMPL human mesh...") pbar = comfy.utils.ProgressBar(len(verts_frames)) for i in tqdm(range(len(verts_frames))): subjects = verts_frames[i] cam_t_subjects = cam_t_frames[i] mesh_nodes = [] if subjects is None: vid.append(np.ones([frame_height, frame_width, 4], dtype=np.uint8)) vid_depth.append(np.zeros([frame_height, frame_width], dtype=np.float32)) continue for subject_vertices, cam_t in zip(subjects, cam_t_subjects): mesh = vertices_to_trimesh(subject_vertices, cam_t, faces) mesh = pyrender.Mesh.from_trimesh(mesh, material=material) mesh_node = pyrender.Node(mesh=mesh) scene.add_node(mesh_node) mesh_nodes.append(mesh_node) if depth_only: depth = r.render(scene, flags=RenderFlags.DEPTH_ONLY) color = np.zeros([frame_height, frame_width, 4]) else: color, depth = r.render(scene, flags=RenderFlags.RGBA) vid.append(color) vid_depth.append(depth) for mesh_node in mesh_nodes: scene.remove_node(mesh_node) pbar.update(1) r = None return np.stack(vid, axis=0), np.stack(vid_depth, axis=0)