diff --git a/.gitignore b/.gitignore index 6677453..11c3ff3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ __pycache__/ **/__pycache__/ *.py[cod] *$py.class - +input_images* # C extensions *.so diff --git a/cli_demo.py b/cli_demo.py new file mode 100644 index 0000000..33aaf8f --- /dev/null +++ b/cli_demo.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import os +import cv2 +import torch +import numpy as np +import sys +import shutil +from datetime import datetime +import glob +import gc +import time +import argparse + +sys.path.append("vggt/") + +from visual_util import predictions_to_glb +from vggt.models.vggt import VGGT +from vggt.utils.load_fn import load_and_preprocess_images +from vggt.utils.pose_enc import pose_encoding_to_extri_intri +from vggt.utils.geometry import unproject_depth_map_to_point_map + +device = "cuda" if torch.cuda.is_available() else "cpu" + +print("Initializing and loading VGGT model...") +model = VGGT.from_pretrained("facebook/VGGT-1B") # alternative loading method + +# model = VGGT() +# _URL = "https://huggingface.co/facebook/VGGT-1B/resolve/main/model.pt" +# model.load_state_dict(torch.hub.load_state_dict_from_url(_URL)) + +model.eval() +model = model.to(device) + + +# ------------------------------------------------------------------------- +# 1) Core model inference +# ------------------------------------------------------------------------- +def run_model(target_dir, model) -> dict: + """ + Run the VGGT model on images in the 'target_dir/images' folder and return predictions. + """ + print(f"Processing images from {target_dir}") + + # Device check + device = "cuda" if torch.cuda.is_available() else "cpu" + if not torch.cuda.is_available(): + raise ValueError("CUDA is not available. Check your environment.") + + # Move model to device + model = model.to(device) + model.eval() + + # Load and preprocess images + image_names = glob.glob(os.path.join(target_dir, "images", "*")) + image_names = sorted(image_names) + print(f"Found {len(image_names)} images") + if len(image_names) == 0: + raise ValueError("No images found. Check your input.") + + images = load_and_preprocess_images(image_names).to(device) + print(f"Preprocessed images shape: {images.shape}") + + # Run inference + print("Running inference...") + with torch.no_grad(): + with torch.cuda.amp.autocast(dtype=torch.bfloat16): + predictions = model(images) + + # Convert pose encoding to extrinsic and intrinsic matrices + print("Converting pose encoding to extrinsic and intrinsic matrices...") + extrinsic, intrinsic = pose_encoding_to_extri_intri(predictions["pose_enc"], images.shape[-2:]) + print(f"extrinsic.shape {extrinsic.shape} \n") + print(f"extrinsic {extrinsic}") + print(f"extrinsic.shape {intrinsic.shape} \n") + print(f"intrinsic {intrinsic}") + predictions["extrinsic"] = extrinsic + predictions["intrinsic"] = intrinsic + + # Convert tensors to numpy arrays + for key in predictions.keys(): + if isinstance(predictions[key], torch.Tensor): + predictions[key] = predictions[key].cpu().numpy().squeeze(0) # remove batch dimension + + # Generate world points from depth map + print("Computing world points from depth map...") + depth_map = predictions["depth"] # (S, H, W, 1) + world_points = unproject_depth_map_to_point_map(depth_map, predictions["extrinsic"], predictions["intrinsic"]) + print(world_points.shape) + print(world_points) + predictions["world_points_from_depth"] = world_points + + # Clean up + torch.cuda.empty_cache() + print(f"predictions are: {predictions}") + return predictions + + +# ------------------------------------------------------------------------- +# 2) Handle uploaded video/images --> produce target_dir + images folder +# ------------------------------------------------------------------------- +def handle_uploads(input_video, input_images): + """ + Create a new folder (target_dir/images) and copy user-provided images or extract frames from video. + Returns (target_dir, list of image paths). + """ + start_time = time.time() + gc.collect() + torch.cuda.empty_cache() + + # Create a unique folder name + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + target_dir = f"input_images_{timestamp}" + target_dir_images = os.path.join(target_dir, "images") + + # Clean up if folder already exists (unlikely) + if os.path.exists(target_dir): + shutil.rmtree(target_dir) + os.makedirs(target_dir) + os.makedirs(target_dir_images) + + image_paths = [] + + # --- Handle images --- + if input_images is not None: + for file_path in input_images: + dst_path = os.path.join(target_dir_images, os.path.basename(file_path)) + shutil.copy(file_path, dst_path) + image_paths.append(dst_path) + + # --- Handle video --- + if input_video is not None: + video_path = input_video + vs = cv2.VideoCapture(video_path) + fps = vs.get(cv2.CAP_PROP_FPS) + frame_interval = int(fps * 1) # extract 1 frame per second + + count = 0 + video_frame_num = 0 + while True: + gotit, frame = vs.read() + if not gotit: + break + count += 1 + if count % frame_interval == 0: + image_path = os.path.join(target_dir_images, f"{video_frame_num:06}.png") + cv2.imwrite(image_path, frame) + image_paths.append(image_path) + video_frame_num += 1 + + # Sort images + image_paths = sorted(image_paths) + + end_time = time.time() + print(f"Files copied to {target_dir_images}; took {end_time - start_time:.3f} seconds") + return target_dir, image_paths + + +# ------------------------------------------------------------------------- +# 3) Reconstruction: performs the 3D reconstruction and saves the GLB file +# ------------------------------------------------------------------------- +def gradio_demo( + target_dir, + conf_thres=3.0, + frame_filter="All", + mask_black_bg=False, + mask_white_bg=False, + show_cam=True, + mask_sky=False, + prediction_mode="Pointmap Regression", +): + """ + Perform reconstruction using the images in target_dir/images. + Returns the path to the GLB file and a log message. + """ + if not os.path.isdir(target_dir) or target_dir == "None": + return None, "No valid target directory found. Please provide valid input." + + start_time = time.time() + gc.collect() + torch.cuda.empty_cache() + + # Get list of image files in target directory + target_dir_images = os.path.join(target_dir, "images") + all_files = sorted(os.listdir(target_dir_images)) if os.path.isdir(target_dir_images) else [] + frame_filter_choices = ["All"] + [f"{i}: {filename}" for i, filename in enumerate(all_files)] + + print("Running run_model...") + with torch.no_grad(): + predictions = run_model(target_dir, model) + + # Save predictions to disk + prediction_save_path = os.path.join(target_dir, "predictions.npz") + np.savez(prediction_save_path, **predictions) + + if frame_filter is None: + frame_filter = "All" + + # Build a GLB file name + glbfile = os.path.join( + target_dir, + f"glbscene_{conf_thres}{frame_filter.replace('.', '').replace(':', '').replace(' ', '')}" + f"maskb{mask_black_bg}maskw{mask_white_bg}_cam{show_cam}_sky{mask_sky}_pred{prediction_mode.replace(' ', '')}.glb", + ) + + # Convert predictions to GLB + glbscene = predictions_to_glb( + predictions, + conf_thres=conf_thres, + filter_by_frames=frame_filter, + mask_black_bg=mask_black_bg, + mask_white_bg=mask_white_bg, + show_cam=show_cam, + mask_sky=mask_sky, + target_dir=target_dir, + prediction_mode=prediction_mode, + ) + glbscene.export(file_obj=glbfile) + + # Cleanup + del predictions + gc.collect() + torch.cuda.empty_cache() + + end_time = time.time() + print(f"Total time: {end_time - start_time:.2f} seconds") + log_msg = f"Reconstruction successful. Processed {len(all_files)} frames." + + return glbfile, log_msg + + +# ------------------------------------------------------------------------- +# 4) Main function to run from command line +# ------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser( + description="VGGT 3D Reconstruction from images/video (command-line mode)" + ) + parser.add_argument("--video", type=str, default=None, help="Path to input video file.") + parser.add_argument( + "--images", + type=str, + nargs="*", + default=None, + help="Paths to one or more input image files.", + ) + parser.add_argument( + "--image_dir", + type=str, + default=None, + help="Path to a directory containing input images.", + ) + parser.add_argument( + "--conf_thres", + type=float, + default=50.0, + help="Confidence threshold percentage (default: 50.0).", + ) + parser.add_argument( + "--mask_black_bg", + action="store_true", + help="Filter black background points (flag; default: False).", + ) + parser.add_argument( + "--mask_white_bg", + action="store_true", + help="Filter white background points (flag; default: False).", + ) + parser.add_argument( + "--show_cam", + action="store_true", + help="Include camera visualization (flag; default: False).", + ) + parser.add_argument( + "--mask_sky", + action="store_true", + help="Filter sky points (flag; default: False).", + ) + parser.add_argument( + "--prediction_mode", + type=str, + choices=["Depthmap and Camera Branch", "Pointmap Branch"], + default="Depthmap and Camera Branch", + help="Prediction mode to use.", + ) + args = parser.parse_args() + + # Ensure that at least one type of input is provided. + if args.video is None and args.image_dir is None and (args.images is None or len(args.images) == 0): + parser.error("You must provide at least a video file, an image directory, or one or more image files.") + + # If an image directory is provided, use it to build a list of image files. + image_list = None + if args.image_dir is not None: + image_list = [] + # Look for common image file types. + for ext in (".png", ".jpg", ".jpeg", ".bmp"): + image_list.extend(glob.glob(os.path.join(args.image_dir, f"*{ext}"))) + image_list = sorted(image_list) + if not image_list: + parser.error(f"No image files found in directory: {args.image_dir}") + elif args.images is not None and len(args.images) > 0: + image_list = args.images + + # Process inputs: copy images / extract frames from video + target_dir, image_paths = handle_uploads(args.video, image_list) + print("Images for reconstruction:") + for ipath in image_paths: + print(" ", ipath) + + # For command-line, we simply use "All" for frame filtering. + frame_filter = "All" + + # Run reconstruction + glbfile, log_msg = gradio_demo( + target_dir, + conf_thres=args.conf_thres, + frame_filter=frame_filter, + mask_black_bg=args.mask_black_bg, + mask_white_bg=args.mask_white_bg, + show_cam=args.show_cam, + mask_sky=args.mask_sky, + prediction_mode=args.prediction_mode, + ) + + print(log_msg) + print("Output GLB file saved at:", glbfile) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/deneme.txt b/deneme.txt new file mode 100644 index 0000000..fa84cdb --- /dev/null +++ b/deneme.txt @@ -0,0 +1,259 @@ +Initializing and loading VGGT model... +Initializing and loading VGGT model... +Files copied to input_images_20250318_113826_158596/images; took 0.213 seconds +Images for reconstruction: + input_images_20250318_113826_158596/images/sandalye1.jpg + input_images_20250318_113826_158596/images/sandalye2.jpg + input_images_20250318_113826_158596/images/sandalye3.jpg + input_images_20250318_113826_158596/images/sandalye4.jpg +Running run_model... +Processing images from input_images_20250318_113826_158596 +Found 4 images +Preprocessed images shape: torch.Size([4, 3, 518, 518]) +Running inference... +Converting pose encoding to extrinsic and intrinsic matrices... +extrinsic.shape torch.Size([1, 4, 3, 4]) + +extrinsic tensor([[[[ 1.0000e+00, 3.9872e-05, 2.8053e-04, -3.3790e-05], + [-3.9849e-05, 1.0000e+00, -8.0769e-05, 1.1223e-05], + [-2.8053e-04, 8.0758e-05, 1.0000e+00, -1.1633e-04]], + + [[ 8.7897e-01, -1.3935e-01, 4.5607e-01, -4.3532e-01], + [ 1.3684e-01, 9.8984e-01, 3.8704e-02, -2.3849e-02], + [-4.5683e-01, 2.8389e-02, 8.8910e-01, 1.1196e-01]], + + [[ 5.4024e-01, 2.4198e-01, -8.0597e-01, 7.7377e-01], + [-2.4684e-01, 9.6120e-01, 1.2313e-01, -1.2373e-01], + [ 8.0450e-01, 1.3242e-01, 5.7901e-01, 4.3251e-01]], + + [[-8.7065e-01, 1.3212e-01, -4.7383e-01, 4.1820e-01], + [-1.6372e-01, 8.3051e-01, 5.3240e-01, -5.3343e-01], + [ 4.6386e-01, 5.4111e-01, -7.0145e-01, 1.6244e+00]]]], + device='cuda:0') +extrinsic.shape torch.Size([1, 4, 3, 3]) + +intrinsic tensor([[[[635.5547, 0.0000, 259.0000], + [ 0.0000, 632.4077, 259.0000], + [ 0.0000, 0.0000, 1.0000]], + + [[673.3128, 0.0000, 259.0000], + [ 0.0000, 668.9824, 259.0000], + [ 0.0000, 0.0000, 1.0000]], + + [[647.9485, 0.0000, 259.0000], + [ 0.0000, 645.5082, 259.0000], + [ 0.0000, 0.0000, 1.0000]], + + [[657.2827, 0.0000, 259.0000], + [ 0.0000, 654.9764, 259.0000], + [ 0.0000, 0.0000, 1.0000]]]], device='cuda:0') +Computing world points from depth map... +(4, 518, 518, 3) +[[[[-0.32635777 -0.3277628 0.80045778] + [-0.32256861 -0.32521258 0.79423084] + [-0.32393473 -0.32785957 0.80069504] + ... + [ 0.3017545 -0.30691559 0.74978018] + [ 0.29972646 -0.30366574 0.74184234] + [ 0.29839372 -0.30114284 0.73568019]] + + [[-0.31818604 -0.31832302 0.78041975] + [-0.3173857 -0.31875226 0.78147245] + [-0.31967265 -0.32229703 0.79016251] + ... + [ 0.30374933 -0.30775166 0.75473664] + [ 0.30392252 -0.3067282 0.75222743] + [ 0.29965597 -0.30124899 0.73879209]] + + [[-0.31972387 -0.3186213 0.78419041] + [-0.32490816 -0.32504109 0.79998947] + [-0.32480285 -0.32619915 0.80283983] + ... + [ 0.30304874 -0.30585151 0.75299595] + [ 0.30336437 -0.30497799 0.75084603] + [ 0.2993832 -0.29980799 0.73811962]] + + ... + + [[-0.33901007 0.33658178 0.83136156] + [-0.32582639 0.32474639 0.80213335] + [-0.31179238 0.31196827 0.77057688] + ... + [ 0.30756927 0.30935648 0.7642391 ] + [ 0.29614456 0.29670384 0.73298793] + [ 0.27577959 0.27522561 0.67993796]] + + [[-0.34688456 0.34574441 0.85066747] + [-0.35423694 0.35443977 0.87205807] + [-0.33679196 0.33829626 0.83234564] + ... + [ 0.2973025 0.30019644 0.73872963] + [ 0.2851795 0.28683233 0.70584956] + [ 0.26956483 0.27007303 0.66461615]] + + [[-0.34381086 0.34401416 0.84313129] + [-0.34717483 0.34872546 0.85467628] + [-0.34365569 0.34653311 0.84930432] + ... + [ 0.28845039 0.29238992 0.71673515] + [ 0.29376423 0.29661765 0.72709672] + [ 0.29365914 0.29536141 0.72401809]]] + + + [[[-0.27946509 -0.29477718 0.69031692] + [-0.26531197 -0.29030559 0.68010908] + [-0.26449681 -0.29057286 0.68089283] + ... + [ 0.30653217 -0.34329744 0.87838063] + [ 0.30861049 -0.34085725 0.87222004] + [ 0.31028803 -0.33932096 0.86838007]] + + [[-0.2688719 -0.28984225 0.68177667] + [-0.25869788 -0.28679426 0.67483183] + [-0.24405615 -0.28214502 0.6641481 ] + ... + [ 0.30616927 -0.34339063 0.88149869] + [ 0.30734013 -0.34308526 0.8808186 ] + [ 0.31049251 -0.33810836 0.86809097]] + + [[-0.25766528 -0.28472603 0.67272506] + [-0.2610214 -0.28645689 0.67693901] + [-0.25939894 -0.28643477 0.67704571] + ... + [ 0.30660648 -0.34162906 0.87984014] + [ 0.30754778 -0.34185431 0.88052861] + [ 0.31080704 -0.33663796 0.86713213]] + + ... + + [[-0.26050822 0.39260375 0.78422529] + [-0.23273498 0.37591514 0.75874836] + [-0.19342003 0.35200265 0.72174699] + ... + [ 0.3749378 0.27278311 1.05199034] + [ 0.37321451 0.28736655 1.09755724] + [ 0.37564739 0.28077125 1.07865886]] + + [[-0.26893818 0.39936239 0.79274108] + [-0.27326109 0.40263491 0.79886063] + [-0.24693766 0.38678373 0.7747721 ] + ... + [ 0.3755698 0.27180026 1.0451034 ] + [ 0.37722163 0.26900011 1.03775596] + [ 0.38097745 0.25502119 0.99644519]] + + [[-0.23415218 0.37915543 0.75883711] + [-0.25543611 0.39302342 0.78156286] + [-0.244081 0.38648061 0.77219268] + ... + [ 0.38006765 0.25102234 0.97855636] + [ 0.38073792 0.25290644 0.9853092 ] + [ 0.38054548 0.25944408 1.00614963]]] + + + [[[-0.23509424 -0.41133813 1.08391587] + [-0.2495936 -0.40332443 1.06414579] + [-0.24803409 -0.4034823 1.06426301] + ... + [ 0.01272387 -0.23979517 0.54143505] + [ 0.01129287 -0.23923831 0.5401517 ] + [ 0.02229242 -0.24042661 0.54119231]] + + [[-0.25275668 -0.40132411 1.06256198] + [-0.26313683 -0.39544312 1.04792776] + [-0.25849008 -0.39717296 1.05189651] + ... + [ 0.01571067 -0.23917281 0.54219104] + [ 0.01542839 -0.23877725 0.54111911] + [-0.00361067 -0.23577585 0.53654601]] + + [[-0.26319948 -0.39502293 1.05013188] + [-0.2504369 -0.40083964 1.0641572 ] + [-0.25789636 -0.39645873 1.05314673] + ... + [ 0.00777232 -0.23702825 0.54087802] + [ 0.00771979 -0.23666892 0.53985876] + [-0.01090445 -0.23375314 0.53536792]] + + ... + + [[-0.17039119 0.40640899 1.60073241] + [-0.19096508 0.3885153 1.55737619] + [-0.19465709 0.38497204 1.54676109] + ... + [-0.02004279 0.38856792 0.6495685 ] + [ 0.0340838 0.42424778 0.66639606] + [ 0.03972201 0.4278182 0.66690115]] + + [[-0.15641601 0.42065152 1.62900647] + [-0.18368865 0.39695608 1.57258144] + [-0.19292603 0.38866549 1.55122962] + ... + [-0.03154959 0.38239768 0.64596448] + [-0.0069916 0.39855119 0.65292407] + [-0.0308108 0.38258874 0.64367914]] + + [[-0.19888478 0.38662463 1.54783547] + [-0.19650977 0.38824308 1.5489444 ] + [-0.22517428 0.3632996 1.49018289] + ... + [-0.0785455 0.35255071 0.63039331] + [-0.05018382 0.371291 0.63871399] + [-0.04201678 0.37657947 0.64020454]]] + + + [[[ 0.24408485 -0.35839285 1.02185841] + [ 0.22540079 -0.36147691 1.03593353] + [ 0.22383672 -0.36140655 1.03576627] + ... + [-0.33928249 -0.29961887 0.82623133] + [-0.33998192 -0.29910583 0.82418417] + [-0.33834201 -0.29527485 0.80836729]] + + [[ 0.22273443 -0.36127259 1.04010822] + [ 0.21128833 -0.36304563 1.04818302] + [ 0.21446391 -0.36209773 1.04408448] + ... + [-0.33926179 -0.29841579 0.8256513 ] + [-0.33948677 -0.29722773 0.82082631] + [-0.34343427 -0.30132617 0.83782139]] + + [[ 0.2181925 -0.36106792 1.04436677] + [ 0.22282258 -0.35983951 1.03905615] + [ 0.22128324 -0.35976713 1.03887214] + ... + [-0.34051235 -0.29900312 0.83243455] + [-0.34124158 -0.29853749 0.83059836] + [-0.34494142 -0.30232457 0.8462184 ]] + + ... + + [[ 0.71527314 0.81422378 1.12961781] + [ 0.63372771 0.72739113 1.16137676] + [ 0.63649916 0.73292072 1.15831854] + ... + [-0.36567945 1.18487171 0.38276409] + [-0.36379322 1.25170052 0.33228194] + [-0.3619157 1.32145128 0.27955365]] + + [[ 0.75121778 0.85613816 1.11595538] + [ 0.69141656 0.79312866 1.13858521] + [ 0.63250065 0.73083639 1.161045 ] + ... + [-0.36201864 1.24967404 0.33818433] + [-0.36173734 1.29338213 0.30484941] + [-0.36453465 1.29021636 0.30607512]] + + [[ 0.64923229 0.74657437 1.15902805] + [ 0.68824982 0.79207206 1.14100994] + [ 0.63004851 0.73044378 1.163124 ] + ... + [-0.35799937 1.32063673 0.28931232] + [-0.36057686 1.32153301 0.28752395] + [-0.37615691 1.11483374 0.43864225]]]] +Building GLB scene +Using Depthmap and Camera Branch +GLB Scene built +Total time: 2.53 seconds +Reconstruction successful. Processed 4 frames. +Output GLB file saved at: input_images_20250318_113826_158596/glbscene_50.0AllmaskbFalsemaskwFalse_camFalse_skyFalse_predDepthmapandCameraBranch.glb diff --git a/deneme2.txt b/deneme2.txt new file mode 100644 index 0000000..bfe74e6 --- /dev/null +++ b/deneme2.txt @@ -0,0 +1,1118 @@ +Initializing and loading VGGT model... +Files copied to input_images_20250318_115921_809868/images; took 0.218 seconds +Images for reconstruction: + input_images_20250318_115921_809868/images/sandalye1.jpg + input_images_20250318_115921_809868/images/sandalye2.jpg + input_images_20250318_115921_809868/images/sandalye3.jpg + input_images_20250318_115921_809868/images/sandalye4.jpg +Running run_model... +Processing images from input_images_20250318_115921_809868 +Found 4 images +Preprocessed images shape: torch.Size([4, 3, 518, 518]) +Running inference... +Converting pose encoding to extrinsic and intrinsic matrices... +extrinsic.shape torch.Size([1, 4, 3, 4]) + +extrinsic tensor([[[[ 1.0000e+00, 3.9872e-05, 2.8053e-04, -3.3790e-05], + [-3.9849e-05, 1.0000e+00, -8.0769e-05, 1.1223e-05], + [-2.8053e-04, 8.0758e-05, 1.0000e+00, -1.1633e-04]], + + [[ 8.7897e-01, -1.3935e-01, 4.5607e-01, -4.3532e-01], + [ 1.3684e-01, 9.8984e-01, 3.8704e-02, -2.3849e-02], + [-4.5683e-01, 2.8389e-02, 8.8910e-01, 1.1196e-01]], + + [[ 5.4024e-01, 2.4198e-01, -8.0597e-01, 7.7377e-01], + [-2.4684e-01, 9.6120e-01, 1.2313e-01, -1.2373e-01], + [ 8.0450e-01, 1.3242e-01, 5.7901e-01, 4.3251e-01]], + + [[-8.7065e-01, 1.3212e-01, -4.7383e-01, 4.1820e-01], + [-1.6372e-01, 8.3051e-01, 5.3240e-01, -5.3343e-01], + [ 4.6386e-01, 5.4111e-01, -7.0145e-01, 1.6244e+00]]]], + device='cuda:0') +extrinsic.shape torch.Size([1, 4, 3, 3]) + +intrinsic tensor([[[[635.5547, 0.0000, 259.0000], + [ 0.0000, 632.4077, 259.0000], + [ 0.0000, 0.0000, 1.0000]], + + [[673.3128, 0.0000, 259.0000], + [ 0.0000, 668.9824, 259.0000], + [ 0.0000, 0.0000, 1.0000]], + + [[647.9485, 0.0000, 259.0000], + [ 0.0000, 645.5082, 259.0000], + [ 0.0000, 0.0000, 1.0000]], + + [[657.2827, 0.0000, 259.0000], + [ 0.0000, 654.9764, 259.0000], + [ 0.0000, 0.0000, 1.0000]]]], device='cuda:0') +Computing world points from depth map... +(4, 518, 518, 3) +[[[[-0.32635777 -0.3277628 0.80045778] + [-0.32256861 -0.32521258 0.79423084] + [-0.32393473 -0.32785957 0.80069504] + ... + [ 0.3017545 -0.30691559 0.74978018] + [ 0.29972646 -0.30366574 0.74184234] + [ 0.29839372 -0.30114284 0.73568019]] + + [[-0.31818604 -0.31832302 0.78041975] + [-0.3173857 -0.31875226 0.78147245] + [-0.31967265 -0.32229703 0.79016251] + ... + [ 0.30374933 -0.30775166 0.75473664] + [ 0.30392252 -0.3067282 0.75222743] + [ 0.29965597 -0.30124899 0.73879209]] + + [[-0.31972387 -0.3186213 0.78419041] + [-0.32490816 -0.32504109 0.79998947] + [-0.32480285 -0.32619915 0.80283983] + ... + [ 0.30304874 -0.30585151 0.75299595] + [ 0.30336437 -0.30497799 0.75084603] + [ 0.2993832 -0.29980799 0.73811962]] + + ... + + [[-0.33901007 0.33658178 0.83136156] + [-0.32582639 0.32474639 0.80213335] + [-0.31179238 0.31196827 0.77057688] + ... + [ 0.30756927 0.30935648 0.7642391 ] + [ 0.29614456 0.29670384 0.73298793] + [ 0.27577959 0.27522561 0.67993796]] + + [[-0.34688456 0.34574441 0.85066747] + [-0.35423694 0.35443977 0.87205807] + [-0.33679196 0.33829626 0.83234564] + ... + [ 0.2973025 0.30019644 0.73872963] + [ 0.2851795 0.28683233 0.70584956] + [ 0.26956483 0.27007303 0.66461615]] + + [[-0.34381086 0.34401416 0.84313129] + [-0.34717483 0.34872546 0.85467628] + [-0.34365569 0.34653311 0.84930432] + ... + [ 0.28845039 0.29238992 0.71673515] + [ 0.29376423 0.29661765 0.72709672] + [ 0.29365914 0.29536141 0.72401809]]] + + + [[[-0.27946509 -0.29477718 0.69031692] + [-0.26531197 -0.29030559 0.68010908] + [-0.26449681 -0.29057286 0.68089283] + ... + [ 0.30653217 -0.34329744 0.87838063] + [ 0.30861049 -0.34085725 0.87222004] + [ 0.31028803 -0.33932096 0.86838007]] + + [[-0.2688719 -0.28984225 0.68177667] + [-0.25869788 -0.28679426 0.67483183] + [-0.24405615 -0.28214502 0.6641481 ] + ... + [ 0.30616927 -0.34339063 0.88149869] + [ 0.30734013 -0.34308526 0.8808186 ] + [ 0.31049251 -0.33810836 0.86809097]] + + [[-0.25766528 -0.28472603 0.67272506] + [-0.2610214 -0.28645689 0.67693901] + [-0.25939894 -0.28643477 0.67704571] + ... + [ 0.30660648 -0.34162906 0.87984014] + [ 0.30754778 -0.34185431 0.88052861] + [ 0.31080704 -0.33663796 0.86713213]] + + ... + + [[-0.26050822 0.39260375 0.78422529] + [-0.23273498 0.37591514 0.75874836] + [-0.19342003 0.35200265 0.72174699] + ... + [ 0.3749378 0.27278311 1.05199034] + [ 0.37321451 0.28736655 1.09755724] + [ 0.37564739 0.28077125 1.07865886]] + + [[-0.26893818 0.39936239 0.79274108] + [-0.27326109 0.40263491 0.79886063] + [-0.24693766 0.38678373 0.7747721 ] + ... + [ 0.3755698 0.27180026 1.0451034 ] + [ 0.37722163 0.26900011 1.03775596] + [ 0.38097745 0.25502119 0.99644519]] + + [[-0.23415218 0.37915543 0.75883711] + [-0.25543611 0.39302342 0.78156286] + [-0.244081 0.38648061 0.77219268] + ... + [ 0.38006765 0.25102234 0.97855636] + [ 0.38073792 0.25290644 0.9853092 ] + [ 0.38054548 0.25944408 1.00614963]]] + + + [[[-0.23509424 -0.41133813 1.08391587] + [-0.2495936 -0.40332443 1.06414579] + [-0.24803409 -0.4034823 1.06426301] + ... + [ 0.01272387 -0.23979517 0.54143505] + [ 0.01129287 -0.23923831 0.5401517 ] + [ 0.02229242 -0.24042661 0.54119231]] + + [[-0.25275668 -0.40132411 1.06256198] + [-0.26313683 -0.39544312 1.04792776] + [-0.25849008 -0.39717296 1.05189651] + ... + [ 0.01571067 -0.23917281 0.54219104] + [ 0.01542839 -0.23877725 0.54111911] + [-0.00361067 -0.23577585 0.53654601]] + + [[-0.26319948 -0.39502293 1.05013188] + [-0.2504369 -0.40083964 1.0641572 ] + [-0.25789636 -0.39645873 1.05314673] + ... + [ 0.00777232 -0.23702825 0.54087802] + [ 0.00771979 -0.23666892 0.53985876] + [-0.01090445 -0.23375314 0.53536792]] + + ... + + [[-0.17039119 0.40640899 1.60073241] + [-0.19096508 0.3885153 1.55737619] + [-0.19465709 0.38497204 1.54676109] + ... + [-0.02004279 0.38856792 0.6495685 ] + [ 0.0340838 0.42424778 0.66639606] + [ 0.03972201 0.4278182 0.66690115]] + + [[-0.15641601 0.42065152 1.62900647] + [-0.18368865 0.39695608 1.57258144] + [-0.19292603 0.38866549 1.55122962] + ... + [-0.03154959 0.38239768 0.64596448] + [-0.0069916 0.39855119 0.65292407] + [-0.0308108 0.38258874 0.64367914]] + + [[-0.19888478 0.38662463 1.54783547] + [-0.19650977 0.38824308 1.5489444 ] + [-0.22517428 0.3632996 1.49018289] + ... + [-0.0785455 0.35255071 0.63039331] + [-0.05018382 0.371291 0.63871399] + [-0.04201678 0.37657947 0.64020454]]] + + + [[[ 0.24408485 -0.35839285 1.02185841] + [ 0.22540079 -0.36147691 1.03593353] + [ 0.22383672 -0.36140655 1.03576627] + ... + [-0.33928249 -0.29961887 0.82623133] + [-0.33998192 -0.29910583 0.82418417] + [-0.33834201 -0.29527485 0.80836729]] + + [[ 0.22273443 -0.36127259 1.04010822] + [ 0.21128833 -0.36304563 1.04818302] + [ 0.21446391 -0.36209773 1.04408448] + ... + [-0.33926179 -0.29841579 0.8256513 ] + [-0.33948677 -0.29722773 0.82082631] + [-0.34343427 -0.30132617 0.83782139]] + + [[ 0.2181925 -0.36106792 1.04436677] + [ 0.22282258 -0.35983951 1.03905615] + [ 0.22128324 -0.35976713 1.03887214] + ... + [-0.34051235 -0.29900312 0.83243455] + [-0.34124158 -0.29853749 0.83059836] + [-0.34494142 -0.30232457 0.8462184 ]] + + ... + + [[ 0.71527314 0.81422378 1.12961781] + [ 0.63372771 0.72739113 1.16137676] + [ 0.63649916 0.73292072 1.15831854] + ... + [-0.36567945 1.18487171 0.38276409] + [-0.36379322 1.25170052 0.33228194] + [-0.3619157 1.32145128 0.27955365]] + + [[ 0.75121778 0.85613816 1.11595538] + [ 0.69141656 0.79312866 1.13858521] + [ 0.63250065 0.73083639 1.161045 ] + ... + [-0.36201864 1.24967404 0.33818433] + [-0.36173734 1.29338213 0.30484941] + [-0.36453465 1.29021636 0.30607512]] + + [[ 0.64923229 0.74657437 1.15902805] + [ 0.68824982 0.79207206 1.14100994] + [ 0.63004851 0.73044378 1.163124 ] + ... + [-0.35799937 1.32063673 0.28931232] + [-0.36057686 1.32153301 0.28752395] + [-0.37615691 1.11483374 0.43864225]]]] +predictions are: {'pose_enc': array([[-3.37897363e-05, 1.12232847e-05, -1.16327807e-04, + 4.03783160e-05, 1.40253847e-04, -1.99284423e-05, + 9.99912739e-01, 7.77416885e-01, 7.73941100e-01], + [-4.35319424e-01, -2.38493308e-02, 1.11960225e-01, + -2.65991432e-03, 2.35401675e-01, 7.12177083e-02, + 9.69018340e-01, 7.38768935e-01, 7.34434426e-01], + [ 7.73772895e-01, -1.23725161e-01, 4.32510823e-01, + 2.63635046e-03, -4.57028240e-01, -1.38719589e-01, + 8.74186814e-01, 7.63139784e-01, 7.60535240e-01], + [ 4.18201447e-01, -5.33430219e-01, 1.62442684e+00, + 8.42045713e-03, -9.06855464e-01, -2.86110312e-01, + 2.49913156e-01, 7.53128111e-01, 7.50727296e-01]], dtype=float32), 'depth': array([[[[0.8004065 ], + [0.7941787 ], + [0.8006431 ], + ..., + [0.7495544 ], + [0.7416174 ], + [0.7354558 ]], + + [[0.78036696], + [0.7814194 ], + [0.7901098 ], + ..., + [0.7545102 ], + [0.75200105], + [0.73856735]], + + [[0.784138 ], + [0.799938 ], + [0.80278826], + ..., + [0.7527699 ], + [0.75061995], + [0.7378951 ]], + + ..., + + [[0.8313675 ], + [0.80213463], + [0.7705732 ], + ..., + [0.76406145], + [0.73281246], + [0.6797665 ]], + + [[0.85067636], + [0.8720697 ], + [0.8323511 ], + ..., + [0.7385541 ], + [0.7056764 ], + [0.664446 ]], + + [[0.8431392 ], + [0.8546855 ], + [0.84931237], + ..., + [0.7165615 ], + [0.7269219 ], + [0.7238432 ]]], + + + [[[0.8450213 ], + [0.8296069 ], + [0.82992375], + ..., + [0.74315095], + [0.7367934 ], + [0.73265654]], + + [[0.832729 ], + [0.82199305], + [0.80593735], + ..., + [0.74608636], + [0.7449555 ], + [0.7323405 ]], + + [[0.8197069 ], + [0.8249376 ], + [0.8242919 ], + ..., + [0.744462 ], + [0.7446377 ], + [0.73138607]], + + ..., + + [[0.9393694 ], + [0.9035564 ], + [0.85201937], + ..., + [0.8837478 ], + [0.92546266], + [0.90736145]], + + [[0.9509837 ], + [0.95849234], + [0.9245999 ], + ..., + [0.877308 ], + [0.8699413 ], + [0.8310992 ]], + + [[0.9043747 ], + [0.93469703], + [0.9209929 ], + ..., + [0.8154963 ], + [0.8212476 ], + [0.84005034]]], + + + [[[0.8165018 ], + [0.7944513 ], + [0.7957529 ], + ..., + [0.7244871 ], + [0.72266656], + [0.73196083]], + + [[0.79125446], + [0.7752091 ], + [0.7810163 ], + ..., + [0.72741014], + [0.7266148 ], + [0.7090475 ]], + + [[0.77649057], + [0.7941085 ], + [0.7823124 ], + ..., + [0.7205475 ], + [0.71996266], + [0.7027654 ]], + + ..., + + [[1.2760837 ], + [1.232059 ], + [1.2224734 ], + ..., + [0.84394604], + [0.9019588 ], + [0.90726 ]], + + [[1.3055836 ], + [1.2478346 ], + [1.2269424 ], + ..., + [0.831785 ], + [0.8577106 ], + [0.83108145]], + + [[1.2199131 ], + [1.2226802 ], + [1.1622933 ], + ..., + [0.7810087 ], + [0.811125 ], + [0.8192587 ]]], + + + [[[0.8269379 ], + [0.80672926], + [0.80615914], + ..., + [0.72536004], + [0.7267492 ], + [0.7406776 ]], + + [[0.80267465], + [0.79074174], + [0.7956026 ], + ..., + [0.7264275 ], + [0.7303505 ], + [0.7143805 ]], + + [[0.7976914 ], + [0.80422896], + [0.80368316], + ..., + [0.7207715 ], + [0.7219732 ], + [0.7072511 ]], + + ..., + + [[1.6044316 ], + [1.4973423 ], + [1.5037652 ], + ..., + [1.8274568 ], + [1.8999041 ], + [1.975504 ]], + + [[1.6533688 ], + [1.5756605 ], + [1.4988701 ], + ..., + [1.8954905 ], + [1.9426546 ], + [1.9387842 ]], + + [[1.5165621 ], + [1.571919 ], + [1.4960619 ], + ..., + [1.9700347 ], + [1.9705786 ], + [1.7455028 ]]]], dtype=float32), 'depth_conf': array([[[1.0000988, 1.000005 , 1.0000015, ..., 1.0000001, 1.0000005, + 1.0000458], + [1.0000018, 1. , 1. , ..., 1. , 1. , + 1.0000029], + [1.0000001, 1. , 1. , ..., 1. , 1. , + 1.0000002], + ..., + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1.0000013, 1. , 1. , ..., 1. , 1. , + 1.0000147]], + + [[1.0000044, 1. , 1. , ..., 1. , 1.0000004, + 1.0000345], + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000014], + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000001], + ..., + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1.0000004, 1. , 1. , ..., 1. , 1. , + 1.0000064]], + + [[1.0000017, 1. , 1. , ..., 1. , 1. , + 1.0000036], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + ..., + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000004]], + + [[1.0000017, 1. , 1. , ..., 1. , 1. , + 1.0000013], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + ..., + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ]]], dtype=float32), 'world_points': array([[[[-0.38207695, -0.37434822, 0.8221682 ], + [-0.3659399 , -0.3726825 , 0.83042854], + [-0.3607235 , -0.36651862, 0.83189493], + ..., + [ 0.35602355, -0.36010328, 0.8068627 ], + [ 0.35760617, -0.36161196, 0.80047923], + [ 0.37884474, -0.3397629 , 0.67519534]], + + [[-0.3768304 , -0.35727033, 0.82274574], + [-0.3636568 , -0.36283225, 0.8284476 ], + [-0.36103976, -0.35936093, 0.8328328 ], + ..., + [ 0.35230455, -0.35493124, 0.8073631 ], + [ 0.35267535, -0.35174412, 0.7997998 ], + [ 0.3585685 , -0.34718755, 0.79951656]], + + [[-0.37289461, -0.3527929 , 0.8113627 ], + [-0.36289504, -0.3544993 , 0.82636696], + [-0.3597397 , -0.3538873 , 0.83367336], + ..., + [ 0.35135618, -0.3495785 , 0.8078512 ], + [ 0.3506884 , -0.3476811 , 0.80013776], + [ 0.35366994, -0.3438282 , 0.80015075]], + + ..., + + [[-0.36830413, 0.35748133, 0.76419127], + [-0.35492593, 0.34797996, 0.7916368 ], + [-0.34697104, 0.34429082, 0.79116416], + ..., + [ 0.3340354 , 0.33725494, 0.7647337 ], + [ 0.33699262, 0.33828783, 0.7597029 ], + [ 0.34208027, 0.33686462, 0.7498733 ]], + + [[-0.35946357, 0.35676062, 0.7551297 ], + [-0.34762868, 0.3525526 , 0.7893895 ], + [-0.3481523 , 0.34768277, 0.787237 ], + ..., + [ 0.33289802, 0.33926642, 0.75883836], + [ 0.33963385, 0.34004268, 0.7554627 ], + [ 0.34730786, 0.33893478, 0.73482615]], + + [[-0.31300932, 0.34076336, 0.6512748 ], + [-0.34658423, 0.35089338, 0.7654841 ], + [-0.34368286, 0.35195783, 0.77350545], + ..., + [ 0.32553282, 0.34684196, 0.74781704], + [ 0.33085003, 0.34468508, 0.7436888 ], + [ 0.23658852, 0.30123493, 0.55968964]]], + + + [[[-0.31517747, -0.32796425, 0.6956756 ], + [-0.3216872 , -0.33031985, 0.7150369 ], + [-0.3211644 , -0.32745454, 0.7179589 ], + ..., + [ 0.29960877, -0.39654562, 0.96303153], + [ 0.30712298, -0.39733675, 0.9615447 ], + [ 0.34398824, -0.3796651 , 0.7922976 ]], + + [[-0.31441432, -0.31001708, 0.6954669 ], + [-0.32174408, -0.32371062, 0.7162361 ], + [-0.3263315 , -0.32245383, 0.7235892 ], + ..., + [ 0.29798135, -0.3918182 , 0.9639515 ], + [ 0.30440602, -0.388992 , 0.960985 ], + [ 0.31573662, -0.38617328, 0.96469945]], + + [[-0.31575918, -0.30630842, 0.6816451 ], + [-0.32328364, -0.3169451 , 0.7153291 ], + [-0.328416 , -0.31827652, 0.72680545], + ..., + [ 0.29887557, -0.38684273, 0.961369 ], + [ 0.30352122, -0.38550383, 0.9593147 ], + [ 0.3101416 , -0.38415724, 0.96630037]], + + ..., + + [[-0.1742928 , 0.37868387, 0.6688211 ], + [-0.16806974, 0.36278448, 0.68365645], + [-0.16413157, 0.35764956, 0.6878358 ], + ..., + [ 0.3930191 , 0.2655134 , 0.9915667 ], + [ 0.39867783, 0.26677594, 0.98919153], + [ 0.41069597, 0.26502335, 0.983859 ]], + + [[-0.16518757, 0.37712416, 0.65953976], + [-0.15989457, 0.36525798, 0.6807556 ], + [-0.15932353, 0.36112165, 0.68165493], + ..., + [ 0.39757085, 0.26926363, 0.987552 ], + [ 0.40700194, 0.26820585, 0.98846155], + [ 0.42461094, 0.2722786 , 0.96192545]], + + [[-0.14644124, 0.3461824 , 0.5716882 ], + [-0.15905654, 0.36518747, 0.66400254], + [-0.15211369, 0.36627784, 0.674485 ], + ..., + [ 0.39915967, 0.27286488, 0.97812235], + [ 0.40582418, 0.27355766, 0.9734615 ], + [ 0.29239157, 0.24710833, 0.72397953]]], + + + [[[-0.24055123, -0.47555315, 1.1497489 ], + [-0.19349965, -0.48061496, 1.1566769 ], + [-0.17696705, -0.46833646, 1.1443497 ], + ..., + [ 0.14982298, -0.283005 , 0.57548344], + [ 0.14561023, -0.28357223, 0.5719869 ], + [ 0.18514743, -0.26552632, 0.4849398 ]], + + [[-0.22764814, -0.45216978, 1.1730809 ], + [-0.1856817 , -0.46743077, 1.1567357 ], + [-0.17669863, -0.46072018, 1.1519431 ], + ..., + [ 0.15052871, -0.27879876, 0.5779284 ], + [ 0.14420916, -0.27446195, 0.5738673 ], + [ 0.14493386, -0.26612192, 0.5714314 ]], + + [[-0.23258516, -0.4539681 , 1.1535513 ], + [-0.181767 , -0.46150786, 1.1605568 ], + [-0.17119458, -0.45782885, 1.1594217 ], + ..., + [ 0.15405403, -0.274233 , 0.5803859 ], + [ 0.14459692, -0.27119967, 0.5756619 ], + [ 0.13711502, -0.2668283 , 0.5761198 ]], + + ..., + + [[-0.45554873, 0.2417899 , 1.2232995 ], + [-0.4083282 , 0.23858957, 1.2188371 ], + [-0.3930919 , 0.23561895, 1.204185 ], + ..., + [-0.05860386, 0.39561367, 0.65330356], + [-0.05958155, 0.39870408, 0.64795417], + [-0.06153874, 0.40139127, 0.645928 ]], + + [[-0.4513191 , 0.23950979, 1.2133999 ], + [-0.4035942 , 0.24191493, 1.2178581 ], + [-0.40293935, 0.23431864, 1.2031474 ], + ..., + [-0.06532838, 0.39806318, 0.6496558 ], + [-0.06056233, 0.39879057, 0.6499951 ], + [-0.05666913, 0.39766705, 0.6369729 ]], + + [[-0.39089993, 0.2608139 , 1.0048475 ], + [-0.41036615, 0.2383695 , 1.1931527 ], + [-0.41098347, 0.23569778, 1.1992356 ], + ..., + [-0.07827827, 0.4016483 , 0.6466236 ], + [-0.06544286, 0.3962488 , 0.63554287], + [-0.09346812, 0.3663947 , 0.4739664 ]]], + + + [[[ 0.26599672, -0.42771608, 1.0628014 ], + [ 0.2798947 , -0.42830792, 1.0364765 ], + [ 0.29351172, -0.4140331 , 1.0120764 ], + ..., + [-0.3184959 , -0.31673416, 0.69558513], + [-0.31787267, -0.31612438, 0.70836174], + [-0.15245034, -0.32766804, 0.5663736 ]], + + [[ 0.2758744 , -0.4150789 , 1.0773362 ], + [ 0.28306904, -0.41604182, 1.0285493 ], + [ 0.28577894, -0.40328488, 1.0122644 ], + ..., + [-0.31102625, -0.30853462, 0.6971798 ], + [-0.3075836 , -0.30468467, 0.70801514], + [-0.28554764, -0.29588708, 0.69550085]], + + [[ 0.27397716, -0.41544712, 1.049261 ], + [ 0.27388158, -0.41776782, 1.0430392 ], + [ 0.28674185, -0.39752078, 1.0046728 ], + ..., + [-0.3048553 , -0.29980552, 0.6904644 ], + [-0.3128433 , -0.30089182, 0.70536673], + [-0.3118052 , -0.29874197, 0.7096921 ]], + + ..., + + [[ 0.13315384, 0.13102354, 1.4538139 ], + [ 0.1265733 , 0.14215223, 1.415358 ], + [ 0.1263011 , 0.14365917, 1.398633 ], + ..., + [-0.39899707, 0.2947014 , 1.0817419 ], + [-0.4066646 , 0.29733956, 1.0822508 ], + [-0.41065946, 0.29718485, 1.0956538 ]], + + [[ 0.12186164, 0.13410984, 1.4557084 ], + [ 0.13160238, 0.15125132, 1.4312451 ], + [ 0.12071934, 0.14570431, 1.4097552 ], + ..., + [-0.41316128, 0.291897 , 1.0904315 ], + [-0.40633118, 0.28890496, 1.1030972 ], + [-0.38653064, 0.2937777 , 1.0761658 ]], + + [[ 0.08610126, 0.16247226, 1.1858633 ], + [ 0.12233471, 0.14179857, 1.4216311 ], + [ 0.118797 , 0.14192834, 1.4256477 ], + ..., + [-0.4227929 , 0.2989565 , 1.1016617 ], + [-0.40015313, 0.30098018, 1.083959 ], + [-0.3857455 , 0.27955198, 0.7823402 ]]]], dtype=float32), 'world_points_conf': array([[[1.000235 , 1.0000138, 1.0000021, ..., 1.0000042, 1.0000052, + 1.0002447], + [1.0000091, 1.0000032, 1.0000002, ..., 1.0000004, 1.000001 , + 1.0000838], + [1.0000017, 1.0000002, 1. , ..., 1. , 1.0000001, + 1.0000285], + ..., + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000014], + [1.0000001, 1. , 1. , ..., 1. , 1. , + 1.0000011], + [1.0000044, 1.0000011, 1.0000004, ..., 1.0000006, 1.0000007, + 1.0000303]], + + [[1.0000181, 1.0000004, 1. , ..., 1.0000008, 1.0000012, + 1.000071 ], + [1.0000001, 1. , 1. , ..., 1. , 1.0000001, + 1.0000206], + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000057], + ..., + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000002], + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000001], + [1.0000021, 1.0000005, 1.0000001, ..., 1.0000001, 1.0000001, + 1.000006 ]], + + [[1.0000092, 1.0000002, 1. , ..., 1.0000001, 1.0000004, + 1.0000556], + [1.0000001, 1. , 1. , ..., 1. , 1. , + 1.000012 ], + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000026], + ..., + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1.0000004, 1.0000001, 1. , ..., 1. , 1. , + 1.0000031]], + + [[1.0000019, 1. , 1. , ..., 1. , 1. , + 1.0000043], + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000011], + [1. , 1. , 1. , ..., 1. , 1. , + 1.0000001], + ..., + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1. , 1. , 1. , ..., 1. , 1. , + 1. ], + [1.0000005, 1.0000002, 1.0000001, ..., 1. , 1. , + 1.000001 ]]], dtype=float32), 'images': array([[[[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]], + + [[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]], + + [[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]]], + + + [[[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]], + + [[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]], + + [[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]]], + + + [[[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]], + + [[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]], + + [[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]]], + + + [[[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]], + + [[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]], + + [[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]]]], dtype=float32), 'extrinsic': array([[[ 9.99999940e-01, 3.98716875e-05, 2.80530570e-04, + -3.37897363e-05], + [-3.98490338e-05, 1.00000000e+00, -8.07692631e-05, + 1.12232847e-05], + [-2.80533772e-04, 8.07580873e-05, 9.99999940e-01, + -1.16327807e-04]], + + [[ 8.78966391e-01, -1.39345944e-01, 4.56070989e-01, + -4.35319424e-01], + [ 1.36840075e-01, 9.89836752e-01, 3.87043022e-02, + -2.38493308e-02], + [-4.56829131e-01, 2.83890143e-02, 8.89101326e-01, + 1.11960225e-01]], + + [[ 5.40236592e-01, 2.41980508e-01, -8.05971384e-01, + 7.73772895e-01], + [-2.46837318e-01, 9.61202145e-01, 1.23132966e-01, + -1.23725161e-01], + [ 8.04497302e-01, 1.32422894e-01, 5.79006374e-01, + 4.32510823e-01]], + + [[-8.70646954e-01, 1.32123172e-01, -4.73832339e-01, + 4.18201447e-01], + [-1.63717493e-01, 8.30508351e-01, 5.32402456e-01, + -5.33430219e-01], + [ 4.63864446e-01, 5.41109264e-01, -7.01448679e-01, + 1.62442684e+00]]], dtype=float32), 'intrinsic': array([[[635.55475, 0. , 259. ], + [ 0. , 632.4077 , 259. ], + [ 0. , 0. , 1. ]], + + [[673.3128 , 0. , 259. ], + [ 0. , 668.98236, 259. ], + [ 0. , 0. , 1. ]], + + [[647.94855, 0. , 259. ], + [ 0. , 645.50824, 259. ], + [ 0. , 0. , 1. ]], + + [[657.28265, 0. , 259. ], + [ 0. , 654.97644, 259. ], + [ 0. , 0. , 1. ]]], dtype=float32), 'world_points_from_depth': array([[[[-0.32635777, -0.3277628 , 0.80045778], + [-0.32256861, -0.32521258, 0.79423084], + [-0.32393473, -0.32785957, 0.80069504], + ..., + [ 0.3017545 , -0.30691559, 0.74978018], + [ 0.29972646, -0.30366574, 0.74184234], + [ 0.29839372, -0.30114284, 0.73568019]], + + [[-0.31818604, -0.31832302, 0.78041975], + [-0.3173857 , -0.31875226, 0.78147245], + [-0.31967265, -0.32229703, 0.79016251], + ..., + [ 0.30374933, -0.30775166, 0.75473664], + [ 0.30392252, -0.3067282 , 0.75222743], + [ 0.29965597, -0.30124899, 0.73879209]], + + [[-0.31972387, -0.3186213 , 0.78419041], + [-0.32490816, -0.32504109, 0.79998947], + [-0.32480285, -0.32619915, 0.80283983], + ..., + [ 0.30304874, -0.30585151, 0.75299595], + [ 0.30336437, -0.30497799, 0.75084603], + [ 0.2993832 , -0.29980799, 0.73811962]], + + ..., + + [[-0.33901007, 0.33658178, 0.83136156], + [-0.32582639, 0.32474639, 0.80213335], + [-0.31179238, 0.31196827, 0.77057688], + ..., + [ 0.30756927, 0.30935648, 0.7642391 ], + [ 0.29614456, 0.29670384, 0.73298793], + [ 0.27577959, 0.27522561, 0.67993796]], + + [[-0.34688456, 0.34574441, 0.85066747], + [-0.35423694, 0.35443977, 0.87205807], + [-0.33679196, 0.33829626, 0.83234564], + ..., + [ 0.2973025 , 0.30019644, 0.73872963], + [ 0.2851795 , 0.28683233, 0.70584956], + [ 0.26956483, 0.27007303, 0.66461615]], + + [[-0.34381086, 0.34401416, 0.84313129], + [-0.34717483, 0.34872546, 0.85467628], + [-0.34365569, 0.34653311, 0.84930432], + ..., + [ 0.28845039, 0.29238992, 0.71673515], + [ 0.29376423, 0.29661765, 0.72709672], + [ 0.29365914, 0.29536141, 0.72401809]]], + + + [[[-0.27946509, -0.29477718, 0.69031692], + [-0.26531197, -0.29030559, 0.68010908], + [-0.26449681, -0.29057286, 0.68089283], + ..., + [ 0.30653217, -0.34329744, 0.87838063], + [ 0.30861049, -0.34085725, 0.87222004], + [ 0.31028803, -0.33932096, 0.86838007]], + + [[-0.2688719 , -0.28984225, 0.68177667], + [-0.25869788, -0.28679426, 0.67483183], + [-0.24405615, -0.28214502, 0.6641481 ], + ..., + [ 0.30616927, -0.34339063, 0.88149869], + [ 0.30734013, -0.34308526, 0.8808186 ], + [ 0.31049251, -0.33810836, 0.86809097]], + + [[-0.25766528, -0.28472603, 0.67272506], + [-0.2610214 , -0.28645689, 0.67693901], + [-0.25939894, -0.28643477, 0.67704571], + ..., + [ 0.30660648, -0.34162906, 0.87984014], + [ 0.30754778, -0.34185431, 0.88052861], + [ 0.31080704, -0.33663796, 0.86713213]], + + ..., + + [[-0.26050822, 0.39260375, 0.78422529], + [-0.23273498, 0.37591514, 0.75874836], + [-0.19342003, 0.35200265, 0.72174699], + ..., + [ 0.3749378 , 0.27278311, 1.05199034], + [ 0.37321451, 0.28736655, 1.09755724], + [ 0.37564739, 0.28077125, 1.07865886]], + + [[-0.26893818, 0.39936239, 0.79274108], + [-0.27326109, 0.40263491, 0.79886063], + [-0.24693766, 0.38678373, 0.7747721 ], + ..., + [ 0.3755698 , 0.27180026, 1.0451034 ], + [ 0.37722163, 0.26900011, 1.03775596], + [ 0.38097745, 0.25502119, 0.99644519]], + + [[-0.23415218, 0.37915543, 0.75883711], + [-0.25543611, 0.39302342, 0.78156286], + [-0.244081 , 0.38648061, 0.77219268], + ..., + [ 0.38006765, 0.25102234, 0.97855636], + [ 0.38073792, 0.25290644, 0.9853092 ], + [ 0.38054548, 0.25944408, 1.00614963]]], + + + [[[-0.23509424, -0.41133813, 1.08391587], + [-0.2495936 , -0.40332443, 1.06414579], + [-0.24803409, -0.4034823 , 1.06426301], + ..., + [ 0.01272387, -0.23979517, 0.54143505], + [ 0.01129287, -0.23923831, 0.5401517 ], + [ 0.02229242, -0.24042661, 0.54119231]], + + [[-0.25275668, -0.40132411, 1.06256198], + [-0.26313683, -0.39544312, 1.04792776], + [-0.25849008, -0.39717296, 1.05189651], + ..., + [ 0.01571067, -0.23917281, 0.54219104], + [ 0.01542839, -0.23877725, 0.54111911], + [-0.00361067, -0.23577585, 0.53654601]], + + [[-0.26319948, -0.39502293, 1.05013188], + [-0.2504369 , -0.40083964, 1.0641572 ], + [-0.25789636, -0.39645873, 1.05314673], + ..., + [ 0.00777232, -0.23702825, 0.54087802], + [ 0.00771979, -0.23666892, 0.53985876], + [-0.01090445, -0.23375314, 0.53536792]], + + ..., + + [[-0.17039119, 0.40640899, 1.60073241], + [-0.19096508, 0.3885153 , 1.55737619], + [-0.19465709, 0.38497204, 1.54676109], + ..., + [-0.02004279, 0.38856792, 0.6495685 ], + [ 0.0340838 , 0.42424778, 0.66639606], + [ 0.03972201, 0.4278182 , 0.66690115]], + + [[-0.15641601, 0.42065152, 1.62900647], + [-0.18368865, 0.39695608, 1.57258144], + [-0.19292603, 0.38866549, 1.55122962], + ..., + [-0.03154959, 0.38239768, 0.64596448], + [-0.0069916 , 0.39855119, 0.65292407], + [-0.0308108 , 0.38258874, 0.64367914]], + + [[-0.19888478, 0.38662463, 1.54783547], + [-0.19650977, 0.38824308, 1.5489444 ], + [-0.22517428, 0.3632996 , 1.49018289], + ..., + [-0.0785455 , 0.35255071, 0.63039331], + [-0.05018382, 0.371291 , 0.63871399], + [-0.04201678, 0.37657947, 0.64020454]]], + + + [[[ 0.24408485, -0.35839285, 1.02185841], + [ 0.22540079, -0.36147691, 1.03593353], + [ 0.22383672, -0.36140655, 1.03576627], + ..., + [-0.33928249, -0.29961887, 0.82623133], + [-0.33998192, -0.29910583, 0.82418417], + [-0.33834201, -0.29527485, 0.80836729]], + + [[ 0.22273443, -0.36127259, 1.04010822], + [ 0.21128833, -0.36304563, 1.04818302], + [ 0.21446391, -0.36209773, 1.04408448], + ..., + [-0.33926179, -0.29841579, 0.8256513 ], + [-0.33948677, -0.29722773, 0.82082631], + [-0.34343427, -0.30132617, 0.83782139]], + + [[ 0.2181925 , -0.36106792, 1.04436677], + [ 0.22282258, -0.35983951, 1.03905615], + [ 0.22128324, -0.35976713, 1.03887214], + ..., + [-0.34051235, -0.29900312, 0.83243455], + [-0.34124158, -0.29853749, 0.83059836], + [-0.34494142, -0.30232457, 0.8462184 ]], + + ..., + + [[ 0.71527314, 0.81422378, 1.12961781], + [ 0.63372771, 0.72739113, 1.16137676], + [ 0.63649916, 0.73292072, 1.15831854], + ..., + [-0.36567945, 1.18487171, 0.38276409], + [-0.36379322, 1.25170052, 0.33228194], + [-0.3619157 , 1.32145128, 0.27955365]], + + [[ 0.75121778, 0.85613816, 1.11595538], + [ 0.69141656, 0.79312866, 1.13858521], + [ 0.63250065, 0.73083639, 1.161045 ], + ..., + [-0.36201864, 1.24967404, 0.33818433], + [-0.36173734, 1.29338213, 0.30484941], + [-0.36453465, 1.29021636, 0.30607512]], + + [[ 0.64923229, 0.74657437, 1.15902805], + [ 0.68824982, 0.79207206, 1.14100994], + [ 0.63004851, 0.73044378, 1.163124 ], + ..., + [-0.35799937, 1.32063673, 0.28931232], + [-0.36057686, 1.32153301, 0.28752395], + [-0.37615691, 1.11483374, 0.43864225]]]])} +Building GLB scene +Using Depthmap and Camera Branch +GLB Scene built +Total time: 2.35 seconds +Reconstruction successful. Processed 4 frames. +Output GLB file saved at: input_images_20250318_115921_809868/glbscene_50.0AllmaskbFalsemaskwFalse_camFalse_skyFalse_predDepthmapandCameraBranch.glb diff --git a/merged_scene.glb b/merged_scene.glb new file mode 100644 index 0000000..cce0364 Binary files /dev/null and b/merged_scene.glb differ diff --git a/npz_reader.py b/npz_reader.py new file mode 100644 index 0000000..f79ad7c --- /dev/null +++ b/npz_reader.py @@ -0,0 +1,229 @@ +import numpy as np +import os +import sys +import argparse +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D + +def load_predictions(npz_file): + """Load and return the predictions from an NPZ file.""" + if not os.path.exists(npz_file): + print(f"Error: File {npz_file} does not exist.") + sys.exit(1) + + try: + # Load the NPZ file + data = np.load(npz_file, allow_pickle=True) + + # Convert to dictionary for easier handling + predictions = {key: data[key] for key in data.files} + + return predictions + except Exception as e: + print(f"Error loading predictions file: {e}") + sys.exit(1) + +def print_predictions_summary(predictions): + """Print a summary of the predictions.""" + print("\n======== PREDICTIONS SUMMARY ========") + print(f"Number of entries: {len(predictions)}") + + for key, value in predictions.items(): + if isinstance(value, np.ndarray): + print(f"\n{key}:") + print(f" Shape: {value.shape}") + print(f" Type: {value.dtype}") + + # Print some statistics for numerical arrays + if np.issubdtype(value.dtype, np.number): + try: + print(f" Min: {np.min(value)}") + print(f" Max: {np.max(value)}") + print(f" Mean: {np.mean(value)}") + except: + print(" (Could not compute statistics)") + + # Print a sample of the data + print(" Sample data:") + if value.size > 0: + # For multi-dimensional arrays, flatten for sample display + flat_value = value.reshape(-1) + sample_size = min(5, flat_value.size) + print(f" {flat_value[:sample_size]} ...") + else: + print(f"\n{key}: {value}") + +def print_full_array(key, array): + """Print the full contents of an array.""" + print(f"\n======== FULL CONTENTS OF '{key}' ========") + print(f"Shape: {array.shape}") + print(f"Type: {array.dtype}") + + # For large arrays, ask for confirmation + if array.size > 1000: + confirm = input(f"Warning: This array has {array.size} elements. Do you want to print it all? (y/n): ") + if confirm.lower() != 'y': + print("Printing canceled.") + return + + # Print the array + print("\nArray contents:") + print(array) + +def visualize_depth_maps(predictions): + """Visualize the depth maps from the predictions.""" + if 'depth' not in predictions: + print("No depth maps found in predictions.") + return + + depth_maps = predictions['depth'] + + # Determine how many depth maps to display + num_frames = depth_maps.shape[0] if depth_maps.ndim > 3 else 1 + + # For multi-dimensional arrays, adapt accordingly + if depth_maps.ndim == 4: # (frames, height, width, 1) + num_frames = min(4, depth_maps.shape[0]) # Show up to 4 frames + fig, axes = plt.subplots(1, num_frames, figsize=(15, 5)) + + if num_frames == 1: + axes = [axes] # Make it iterable for consistency + + for i, ax in enumerate(axes): + if i < depth_maps.shape[0]: + # Remove the channel dimension if it exists + if depth_maps.shape[-1] == 1: + depth_map = depth_maps[i, :, :, 0] + else: + depth_map = depth_maps[i] + + im = ax.imshow(depth_map, cmap='viridis') + ax.set_title(f"Frame {i}") + plt.colorbar(im, ax=ax) + + plt.tight_layout() + plt.show() + +def visualize_point_cloud(predictions): + """Visualize the 3D point cloud.""" + point_cloud_key = None + if 'world_points_from_depth' in predictions: + point_cloud_key = 'world_points_from_depth' + + if point_cloud_key is None: + print("No point cloud data found in predictions.") + return + + points = predictions[point_cloud_key] + + # Choose a specific frame if multiple are available + if points.ndim == 4: # (frames, height, width, 3) + frame_idx = 0 + print(f"Showing point cloud for frame {frame_idx} (shape: {points.shape})") + points = points[frame_idx] + + # Flatten the spatial dimensions to get a list of 3D points + points_flat = points.reshape(-1, 3) + + # Create a scatter plot in 3D + fig = plt.figure(figsize=(12, 10)) + ax = fig.add_subplot(111, projection='3d') + + # Downsample if there are too many points + max_points = 10000 + if len(points_flat) > max_points: + # Random sampling + idx = np.random.choice(len(points_flat), max_points, replace=False) + points_flat = points_flat[idx] + + # Plot the points + ax.scatter( + points_flat[:, 0], + points_flat[:, 1], + points_flat[:, 2], + c='b', + marker='.', + alpha=0.5, + s=1 + ) + + ax.set_xlabel('X') + ax.set_ylabel('Y') + ax.set_zlabel('Z') + ax.set_title('3D Point Cloud') + + # Equal aspect ratio + ax.set_box_aspect([1, 1, 1]) + + plt.tight_layout() + plt.show() + +def main(): + parser = argparse.ArgumentParser(description='Read and analyze predictions.npz file from VGGT model.') + parser.add_argument('npz_file', help='Path to the predictions.npz file') + parser.add_argument('--key', help='Specific key to print from the npz file') + parser.add_argument('--visualize', choices=['depth', 'points', 'all'], + help='Visualize specific data: depth maps or 3D points') + + args = parser.parse_args() + + # Load predictions + predictions = load_predictions(args.npz_file) + + # If a specific key is provided, print it and exit + if args.key: + if args.key in predictions: + print_full_array(args.key, predictions[args.key]) + return + else: + print(f"Error: Key '{args.key}' not found.") + print("Available keys:") + for key in predictions.keys(): + print(f"- {key}") + return + + # Print summary + print_predictions_summary(predictions) + + # Handle visualization request + if args.visualize: + if args.visualize == 'depth' or args.visualize == 'all': + visualize_depth_maps(predictions) + if args.visualize == 'points' or args.visualize == 'all': + visualize_point_cloud(predictions) + else: + # Interactive mode + while True: + print("\n======== OPTIONS ========") + print("1: List all available keys") + print("2: Print contents of a specific key") + print("3: Visualize depth maps") + print("4: Visualize 3D point cloud") + print("5: Exit") + + choice = input("\nEnter your choice (1-5): ") + + if choice == '1': + print("\nAvailable keys:") + for key in sorted(predictions.keys()): + print(f"- {key}") + elif choice == '2': + key = input("Enter the key name: ") + if key in predictions: + print_full_array(key, predictions[key]) + else: + print(f"Error: Key '{key}' not found.") + print("Available keys:") + for k in sorted(predictions.keys()): + print(f"- {k}") + elif choice == '3': + visualize_depth_maps(predictions) + elif choice == '4': + visualize_point_cloud(predictions) + elif choice == '5': + break + else: + print("Invalid choice. Please try again.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/reconstruct.py b/reconstruct.py new file mode 100644 index 0000000..69e2ed8 --- /dev/null +++ b/reconstruct.py @@ -0,0 +1,269 @@ +import numpy as np +import open3d as o3d +import copy +import trimesh +import os +import matplotlib.pyplot as plt +from scipy.spatial.transform import Rotation + + +def merge_point_clouds(prediction_sets, voxel_size=0.02): + """ + Register and merge multiple point clouds from overlapping image sets + + Args: + prediction_sets: List of prediction dictionaries containing point clouds and camera parameters + voxel_size: Resolution for downsampling and registration + + Returns: + merged_scene: Final trimesh.Scene with merged point cloud + """ + # 1. Extract point clouds and camera positions from each prediction set + point_clouds = [] + camera_positions = [] + + for i, predictions in enumerate(prediction_sets): + # Extract point cloud data + if "world_points" in predictions: + points = predictions["world_points"] + else: + points = predictions["world_points_from_depth"] + + # Get confidence values if available + if "world_points_conf" in predictions: + conf = predictions["world_points_conf"] + else: + conf = np.ones_like(points[..., 0]) + + # Get RGB images + images = predictions["images"] + + # Extract camera matrices + extrinsics = predictions["extrinsic"] + + # Create colored point cloud + vertices, colors = extract_colored_point_cloud(points, images, conf, threshold=50.0) + + # Create Open3D point cloud + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(vertices) + pcd.colors = o3d.utility.Vector3dVector(colors / 255.0) # Normalize colors to [0,1] + + # Downsample to speed up registration + pcd_down = pcd.voxel_down_sample(voxel_size) + + # Extract camera centers from extrinsic matrices + cam_centers = extract_camera_centers(extrinsics) + + point_clouds.append(pcd_down) + camera_positions.append(cam_centers) + + print(f"Point cloud {i}: {len(pcd_down.points)} points, {len(cam_centers)} cameras") + + # 2. Register point clouds using both camera positions and point features + transformations = register_point_clouds(point_clouds, camera_positions) + + # 3. Transform and merge point clouds + merged_cloud = o3d.geometry.PointCloud() + + for i, pcd in enumerate(point_clouds): + # Apply transformation + if i > 0: # First cloud is reference + pcd_transformed = copy.deepcopy(pcd) + pcd_transformed.transform(transformations[i-1]) + merged_cloud += pcd_transformed + else: + merged_cloud += pcd + + # 4. Clean and optimize final point cloud + print("Cleaning merged point cloud...") + merged_cloud = merged_cloud.voxel_down_sample(voxel_size) + merged_cloud, _ = merged_cloud.remove_statistical_outlier(nb_neighbors=20, std_ratio=2.0) + + # 5. Create trimesh scene for output + merged_scene = trimesh.Scene() + + # Convert merged point cloud to trimesh + vertices = np.asarray(merged_cloud.points) + colors = (np.asarray(merged_cloud.colors) * 255).astype(np.uint8) + merged_trimesh = trimesh.PointCloud(vertices=vertices, colors=colors) + merged_scene.add_geometry(merged_trimesh) + + return merged_scene + + +def extract_colored_point_cloud(points, images, confidence, threshold=50.0): + """Extract points and colors based on confidence threshold""" + S, H, W, _ = points.shape + vertices_3d = points.reshape(-1, 3) + + # Handle different image formats + if images.ndim == 4 and images.shape[1] == 3: # NCHW format + colors_rgb = np.transpose(images, (0, 2, 3, 1)) + else: # NHWC format + colors_rgb = images + + colors_rgb = (colors_rgb.reshape(-1, 3) * 255).astype(np.uint8) + conf = confidence.reshape(-1) + + # Apply confidence threshold + if threshold > 0: + conf_threshold = np.percentile(conf, threshold) + mask = (conf >= conf_threshold) & (conf > 1e-5) + vertices_3d = vertices_3d[mask] + colors_rgb = colors_rgb[mask] + + return vertices_3d, colors_rgb + + +def extract_camera_centers(extrinsics): + """Extract camera centers from extrinsic matrices""" + centers = [] + for ext in extrinsics: + # Create 4x4 transformation matrix + T = np.eye(4) + T[:3, :4] = ext + + # Get camera center in world coordinates (-R^T * t) + R = T[:3, :3] + t = T[:3, 3] + center = -np.dot(R.T, t) + centers.append(center) + + return np.array(centers) + + +def register_point_clouds(point_clouds, camera_positions): + """ + Register point clouds using both camera positions and ICP + + Returns: + List of transformation matrices to align each point cloud to the first one + """ + if len(point_clouds) <= 1: + return [] + + transformations = [] + target_cloud = point_clouds[0] + target_cameras = camera_positions[0] + + # Process each subsequent point cloud + for i in range(1, len(point_clouds)): + source_cloud = point_clouds[i] + source_cameras = camera_positions[i] + + # 1. Initial alignment using camera positions (if sequential) + initial_transformation = estimate_transformation_from_cameras(source_cameras, target_cameras) + + # Apply initial transformation + source_aligned = copy.deepcopy(source_cloud) + source_aligned.transform(initial_transformation) + + # 2. Fine registration using Colored ICP + # Prepare for colored ICP + source_aligned.estimate_normals() + target_cloud.estimate_normals() + + # Run Colored ICP (slower but more accurate for colored point clouds) + print(f"Running colored ICP for point cloud {i}...") + reg_p2p = o3d.pipelines.registration.registration_colored_icp( + source_aligned, target_cloud, 0.05, + initial_transformation, + o3d.pipelines.registration.TransformationEstimationForColoredICP(), + o3d.pipelines.registration.ICPConvergenceCriteria(max_iteration=200) + ) + + # Combine transformations + final_transformation = reg_p2p.transformation @ initial_transformation + + transformations.append(final_transformation) + print(f"Point cloud {i} registered with fitness: {reg_p2p.fitness}, RMSE: {reg_p2p.inlier_rmse}") + + return transformations + + +def estimate_transformation_from_cameras(source_cameras, target_cameras): + """ + Estimate transformation between point clouds using camera positions + using a simplified version of Umeyama algorithm + """ + if len(source_cameras) < 3 or len(target_cameras) < 3: + # Not enough camera positions for reliable estimation + return np.eye(4) + + # Find corresponding cameras by proximity + # This works if the camera positions are in sequential order + s_cam = source_cameras + t_cam = target_cameras + + # If the number of cameras differs, use the minimum length + min_len = min(len(s_cam), len(t_cam)) + + # Try to find best alignment using first and last few cameras + source_pts = np.vstack([s_cam[:min(3, min_len)], s_cam[-min(3, min_len):]]) + target_pts = np.vstack([t_cam[:min(3, min_len)], t_cam[-min(3, min_len):]]) + + # Calculate centroids + source_centroid = np.mean(source_pts, axis=0) + target_centroid = np.mean(target_pts, axis=0) + + # Center the points + source_centered = source_pts - source_centroid + target_centered = target_pts - target_centroid + + # Compute rotation using singular value decomposition + H = np.dot(source_centered.T, target_centered) + U, _, Vt = np.linalg.svd(H) + R = np.dot(Vt.T, U.T) + + # Ensure proper rotation matrix (det=1) + if np.linalg.det(R) < 0: + Vt[-1, :] *= -1 + R = np.dot(Vt.T, U.T) + + # Translation + t = target_centroid - np.dot(R, source_centroid) + + # Create transformation matrix + transformation = np.eye(4) + transformation[:3, :3] = R + transformation[:3, 3] = t + + return transformation + + +def main(): + """Example usage""" + import argparse + + parser = argparse.ArgumentParser(description='Merge multiple point clouds from VGGT predictions') + parser.add_argument('--prediction_dirs', type=str, nargs='+', required=True, + help='Directories containing prediction.npz files') + parser.add_argument('--output', type=str, default='merged_scene.glb', + help='Output GLB file path') + parser.add_argument('--voxel_size', type=float, default=0.02, + help='Voxel size for downsampling') + + args = parser.parse_args() + + # Load predictions + prediction_sets = [] + for dir_path in args.prediction_dirs: + npz_path = os.path.join(dir_path, 'predictions.npz') + if not os.path.exists(npz_path): + print(f"Warning: No predictions.npz found in {dir_path}") + continue + + predictions = dict(np.load(npz_path, allow_pickle=True)) + prediction_sets.append(predictions) + + # Merge point clouds + merged_scene = merge_point_clouds(prediction_sets, args.voxel_size) + + # Export merged scene + merged_scene.export(file_obj=args.output) + print(f"Merged scene saved to {args.output}") + + +if __name__ == "__main__": + main() \ No newline at end of file