Test TRELLIS models, still waiting for spconv to fix its error for python3.12 cu124

https://github.com/traveller59/spconv/issues/731

.gitignore

@@ -157,6 +157,7 @@ cython_debug/
 *.pth
 *.safetensors
 *.pkl
+*.pt
 .huggingface
 
 # Ignore test outputs

Checkpoints/CharacterGen/2D_Stage/models/checkpoint/scaler.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3c63c3d2e7b7f18788aca8a5a0e766ad4633f631d49192602d8532b6b0df7615
-size 559

Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/README.md

@@ -0,0 +1,16 @@
+---
+library_name: trellis
+pipeline_tag: image-to-3d
+license: mit
+language:
+- en
+---
+# TRELLIS Image Large
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+The image conditioned version of TRELLIS, a large 3D genetive model. It was introduced in the paper [Structured 3D Latents for Scalable and Versatile 3D Generation](https://huggingface.co/papers/2412.01506).
+
+Project page: https://trellis3d.github.io/
+
+Code: https://github.com/Microsoft/TRELLIS

Checkpoints/Diffusers/tencent/Hunyuan3D-1/README.md

@@ -6,6 +6,10 @@ license_link: https://huggingface.co/tencent/Hunyuan3D-1/blob/main/LICENSE.txt
 language:
 - en
 - zh
+tags:
+- image-to-3d
+- text-to-3d
+pipeline_tag: text-to-3d
 ---
 <!-- ## **Hunyuan3D-1.0** -->
 
@@ -237,4 +241,4 @@ If you found this repository helpful, please cite our report:
     archivePrefix={arXiv},
     primaryClass={cs.CV}
 }
-```
+```

Gen_3D_Modules/CharacterGen/Stage_3D/lrm/utils/ops.py

@@ -6,7 +6,7 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 from torch.autograd import Function
-from torch.cuda.amp import custom_bwd, custom_fwd
+from torch.amp import custom_bwd, custom_fwd
 
 from CharacterGen.Stage_3D import lrm
 from .typing import *
@@ -41,13 +41,13 @@ class _TruncExp(Function):  # pylint: disable=abstract-method
     # Implementation from torch-ngp:
     # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py
     @staticmethod
-    @custom_fwd(cast_inputs=torch.float32)
+    @custom_fwd(cast_inputs=torch.float32, device_type="cuda")
     def forward(ctx, x):  # pylint: disable=arguments-differ
         ctx.save_for_backward(x)
         return torch.exp(x)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, g):  # pylint: disable=arguments-differ
         x = ctx.saved_tensors[0]
         return g * torch.exp(torch.clamp(x, max=15))

Gen_3D_Modules/Hunyuan3D_V1/svrm/ldm/utils/ops.py

@@ -7,7 +7,7 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 from torch.autograd import Function
-from torch.cuda.amp import custom_bwd, custom_fwd
+from torch.amp import custom_bwd, custom_fwd
 from igl import fast_winding_number_for_meshes, point_mesh_squared_distance, read_obj
 
 from .typing import *
@@ -52,13 +52,13 @@ class _TruncExp(Function):  # pylint: disable=abstract-method
     # Implementation from torch-ngp:
     # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py
     @staticmethod
-    @custom_fwd(cast_inputs=torch.float32)
+    @custom_fwd(cast_inputs=torch.float32, device_type="cuda")
     def forward(ctx, x):  # pylint: disable=arguments-differ
         ctx.save_for_backward(x)
         return torch.exp(x)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, g):  # pylint: disable=arguments-differ
         x = ctx.saved_tensors[0]
         return g * torch.exp(torch.clamp(x, max=15))
@@ -68,14 +68,14 @@ class SpecifyGradient(Function):
     # Implementation from stable-dreamfusion
     # https://github.com/ashawkey/stable-dreamfusion
     @staticmethod
-    @custom_fwd
+    @custom_fwd(device_type="cuda")
     def forward(ctx, input_tensor, gt_grad):
         ctx.save_for_backward(gt_grad)
         # we return a dummy value 1, which will be scaled by amp's scaler so we get the scale in backward.
         return torch.ones([1], device=input_tensor.device, dtype=input_tensor.dtype)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, grad_scale):
         (gt_grad,) = ctx.saved_tensors
         gt_grad = gt_grad * grad_scale

Gen_3D_Modules/StableFast3D/sf3d/models/network.py

@@ -8,7 +8,7 @@ from einops import rearrange
 from jaxtyping import Float
 from torch import Tensor
 from torch.autograd import Function
-from torch.cuda.amp import custom_bwd, custom_fwd
+from torch.amp import custom_bwd, custom_fwd
 
 from StableFast3D.sf3d.models.utils import BaseModule, normalize
 
@@ -65,13 +65,13 @@ class _TruncExp(Function):  # pylint: disable=abstract-method
     # Implementation from torch-ngp:
     # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py
     @staticmethod
-    @custom_fwd(cast_inputs=torch.float32)
+    @custom_fwd(cast_inputs=torch.float32, device_type="cuda")
     def forward(ctx, x):  # pylint: disable=arguments-differ
         ctx.save_for_backward(x)
         return torch.exp(x)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, g):  # pylint: disable=arguments-differ
         x = ctx.saved_tensors[0]
         return g * torch.exp(torch.clamp(x, max=15))

Gen_3D_Modules/TriplaneGaussian/utils/ops.py

@@ -4,7 +4,7 @@ import numpy as np
 import torch
 import torch.nn.functional as F
 from torch.autograd import Function
-from torch.cuda.amp import custom_bwd, custom_fwd
+from torch.amp import custom_bwd, custom_fwd
 from pytorch3d import io
 from pytorch3d.renderer import (
     PointsRasterizationSettings, 
@@ -35,13 +35,13 @@ class _TruncExp(Function):  # pylint: disable=abstract-method
     # Implementation from torch-ngp:
     # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py
     @staticmethod
-    @custom_fwd(cast_inputs=torch.float32)
+    @custom_fwd(cast_inputs=torch.float32, device_type="cuda")
     def forward(ctx, x):  # pylint: disable=arguments-differ
         ctx.save_for_backward(x)
         return torch.exp(x)
 
     @staticmethod
-    @custom_bwd
+    @custom_bwd(device_type="cuda")
     def backward(ctx, g):  # pylint: disable=arguments-differ
         x = ctx.saved_tensors[0]
         return g * torch.exp(torch.clamp(x, max=15))

Gen_3D_Modules/craftsman/utils/checkpoint.py

@@ -36,7 +36,7 @@ def checkpoint(
 
 class CheckpointFunction(torch.autograd.Function):
     @staticmethod
-    @torch.cuda.amp.custom_fwd
+    @torch.amp.custom_fwd(device_type="cuda")
     def forward(ctx, run_function, length, *args):
         ctx.run_function = run_function
         ctx.input_tensors = list(args[:length])
@@ -47,7 +47,7 @@ class CheckpointFunction(torch.autograd.Function):
         return output_tensors
 
     @staticmethod
-    @torch.cuda.amp.custom_bwd
+    @torch.amp.custom_bwd(device_type="cuda")
     def backward(ctx, *output_grads):
         ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
         with torch.enable_grad():

README.md

@@ -252,6 +252,6 @@ This is an extensive node suite that enables ComfyUI to process 3D inputs (Mesh
   ```
 * If you encounter OpenGL errors (e.g., `[F glutil.cpp:338] eglInitialize() failed`), then set `force_cuda_rasterize` to true on corresponding node
 * If after the installation, your ComfyUI gets stuck at starting or running, you can follow the instructions in the following link to solve the problem: [Code Hangs Indefinitely When Evaluating Neuron Models on GPU](https://github.com/lava-nc/lava-dl/discussions/211)
-
+* [Fix "nvcc fatal : Could not set up the environment for Microsoft Visual Studio using 'C:/Program Files (x86)/Microsoft V isual Studio/.../vcvars64.bat'](https://github.com/NVlabs/instant-ngp/issues/709#issuecomment-1378155274)
 ## Supporters
 - [MrNeRF](https://twitter.com/janusch_patas)

_Pre_Builds/_Build_Scripts/build_config.yaml

@@ -13,7 +13,7 @@ libs_dir_name: "_Libs"
 # Folder used to store built wheels
 wheels_dir_name: "_Build_Wheels"
 
-build_base_packages: ["torch", "torchvision"]
+build_base_packages: ["torch", "torchvision", "xformers", "spconv-${cuda_version}"]
 # All packages will be automatically installed at beginning of install.py follwing orders from top to bottom 
 remote_packages:
   torch:

_Pre_Builds/_Build_Scripts/build_utils.py

@@ -140,12 +140,17 @@ def git_folder_parallel(repo_id: str, folder: str, recursive: bool=True, root_ou
     
 def install_remote_packages(package_names):
     for package_name in package_names:
-        package_attr = build_config.remote_packages[package_name]
-        if hasattr(package_attr, "version"):
-            package_name += f"=={package_attr.version}"
-        url_option = package_attr.url_option if hasattr(package_attr, "url_option") else "--index-url"
-            
-        subprocess.run([
-            PYTHON_PATH, "-s", "-m", "pip", "install", 
-            package_name, url_option, package_attr.url
-        ])
+        if package_name in build_config.remote_packages:
+            package_attr = build_config.remote_packages[package_name]
+            if hasattr(package_attr, "version"):
+                package_name += f"=={package_attr.version}"
+            if hasattr(package_attr, "url"):
+                url_option = package_attr.url_option if hasattr(package_attr, "url_option") else "--index-url"
+                
+                subprocess.run([
+                    PYTHON_PATH, "-s", "-m", "pip", "install", 
+                    package_name, url_option, package_attr.url
+                ])
+                continue
+
+        subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", package_name])

_Pre_Builds/_Build_Scripts/dependencies.txt

@@ -2,4 +2,5 @@ https://github.com/ashawkey/diff-gaussian-rasterization.git
 https://github.com/NVlabs/nvdiffrast.git
 https://github.com/ashawkey/kiuikit.git
 https://github.com/facebookresearch/pytorch3d.git
-https://github.com/rusty1s/pytorch_scatter.git
+https://github.com/rusty1s/pytorch_scatter.git
+https://github.com/EasternJournalist/utils3d.git

install.py

@@ -47,7 +47,7 @@ try:
             cstr(f"pip install {wheel_path} to {PYTHON_PATH}").msg.print()
     
     # Install packages that needs specify remote url
-    install_remote_packages(build_config.remote_packages.keys())
+    install_remote_packages(build_config.build_base_packages)
     
     # Get the target remote pre-built wheels directory name and path
     platform_config_name = get_platform_config_name()

nodes.py

@@ -93,6 +93,8 @@ from CRM_T2I_V3.model.crm.sampler import CRMSamplerV3
 from Hunyuan3D_V1.mvd.hunyuan3d_mvd_std_pipeline import HunYuan3D_MVD_Std_Pipeline
 from Hunyuan3D_V1.mvd.hunyuan3d_mvd_lite_pipeline import Hunyuan3D_MVD_Lite_Pipeline
 from Hunyuan3D_V1.infer import Views2Mesh
+from TRELLIS.trellis.pipelines import TrellisImageTo3DPipeline
+from TRELLIS.trellis.utils import render_utils, postprocessing_utils
 
 from .shared_utils.image_utils import (
     prepare_torch_img, torch_imgs_to_pils, troch_image_dilate, 
@@ -2592,8 +2594,8 @@ class Era3D_MVDiffusion_Model:
         images_pred = out[bsz:] 
         
         # [N, 3, H, W] -> [N, H, W, 3]
-        multiview_images = images_pred.permute(0, 2, 3, 1).to(reference_image.dtype, dtype=reference_image.device)   
-        multiview_normals = normals_pred.permute(0, 2, 3, 1).to(reference_image.dtype, dtype=reference_image.device)
+        multiview_images = images_pred.permute(0, 2, 3, 1).to(reference_image.device, dtype=reference_image.dtype)   
+        multiview_normals = normals_pred.permute(0, 2, 3, 1).to(reference_image.device, dtype=reference_image.dtype)
         
         azimuths = [0, 45, 90, 180, -90, -45]
         elevations = [0.0] * 6
@@ -3846,6 +3848,7 @@ class Hunyuan3D_V1_Reconstruction_Model:
                 "multiview_image_grid": ("IMAGE",),
                 "condition_image": ("IMAGE",),
                 "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
+                "target_face_count": ("INT", {"default": 90000, "min": 1}),
             }
         }
 
@@ -3860,7 +3863,7 @@ class Hunyuan3D_V1_Reconstruction_Model:
     CATEGORY = "Comfy3D/Algorithm"
     
     @torch.no_grad()
-    def run_model(self, hunyuan3d_v1_reconstruction_model, multiview_image_grid, condition_image, seed):
+    def run_model(self, hunyuan3d_v1_reconstruction_model, multiview_image_grid, condition_image, seed, target_face_count):
         mv_grid_pil = torch_imgs_to_pils(multiview_image_grid)[0]
         condition_pil = torch_imgs_to_pils(condition_image)[0]
         
@@ -3868,6 +3871,7 @@ class Hunyuan3D_V1_Reconstruction_Model:
             mv_grid_pil,
             condition_pil,
             seed=seed,
+			target_face_count=target_face_count
         )
         vertices, faces, vtx_colors = torch.from_numpy(vertices).to(DEVICE), torch.from_numpy(faces).to(torch.int64).to(DEVICE), torch.from_numpy(vtx_colors).to(DEVICE)
         mesh = Mesh(v=vertices, f=faces.to(torch.int64), vc=vtx_colors, device=DEVICE)
@@ -3875,5 +3879,111 @@ class Hunyuan3D_V1_Reconstruction_Model:
         
         return (mesh,)
     
+class Load_Trellis_Structured_3D_Latents_Models:
+    default_repo_id = "JeffreyXiang/TRELLIS-image-large"
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "repo_id": ("STRING", {"default": cls.default_repo_id, "multiline": False}),
+            },
+        }
+    
+    RETURN_TYPES = (
+        "TRELLIS_PIPE",
+    )
+    RETURN_NAMES = (
+        "trellis_pipe",
+    )
+    FUNCTION = "load_pipe"
+    CATEGORY = "Comfy3D/Import|Export"
+    
+    def load_pipe(self, repo_id):
+        
+        pipe = TrellisImageTo3DPipeline.from_pretrained(repo_id).to(DEVICE)
+        
+        return (pipe, )
+    
+    
+class Trellis_Structured_3D_Latents_Models:
+    
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "trellis_pipe": ("TRELLIS_PIPE",),
+                "reference_image": ("IMAGE",),
+                "reference_mask": ("MASK",),
+                "seed": ("INT", {"default": 1, "min": 0, "max": 0xffffffffffffffff}),
+                "sparse_structure_guidance_scale": ("FLOAT", {"default": 7.5, "min": 0.0, "step": 0.01}),
+                "sparse_structure_sample_steps": ("INT", {"default": 12, "min": 1}),
+                "structured_latent_guidance_scale": ("FLOAT", {"default": 3.0, "min": 0.0, "step": 0.01}),
+                "structured_latent_sample_steps": ("INT", {"default": 12, "min": 1}),
+            }
+        }
+    
+    RETURN_TYPES = (
+        "MESH",
+    )
+    RETURN_NAMES = (
+        "mesh",
+    )
+    FUNCTION = "run_model"
+    CATEGORY = "Comfy3D/Algorithm"
+    
+    @torch.no_grad()
+    def run_model(
+        self, 
+        trellis_pipe, 
+        reference_image, # [1, H, W, 3]
+        reference_mask,  # [1, H, W]
+        seed,
+        sparse_structure_guidance_scale,
+        sparse_structure_sample_steps,
+        structured_latent_guidance_scale,
+        structured_latent_sample_steps,
+    ):
+        single_image = torch_imgs_to_pils(reference_image, reference_mask)[0]
+
+        outputs = trellis_pipe.run(
+            single_image,
+            # Optional parameters
+            seed=seed,
+            formats=["gaussian", "mesh"],
+            sparse_structure_sampler_params={
+                "cfg_strength": sparse_structure_guidance_scale,
+                "steps": sparse_structure_sample_steps,
+            },
+            slat_sampler_params={
+                "cfg_strength": structured_latent_guidance_scale,
+                "steps": structured_latent_sample_steps,
+            },
+        )
+
+        # test
+        import imageio
+        import trimesh
+        video = render_utils.render_video(outputs['gaussian'][0])['color']
+        imageio.mimsave("sampled_gs.mp4", video, fps=30)
+        # GLB files can be extracted from the outputs
+        vertices, faces, uvs, texture = postprocessing_utils.to_glb(
+            outputs['gaussian'][0],
+            outputs['mesh'][0],
+            # Optional parameters
+            simplify=0.95,          # Ratio of triangles to remove in the simplification process
+            texture_size=1024,      # Size of the texture used for the GLB
+        )
+        print(f"Vertices: {vertices.shape}, Faces: {faces.shape}, UVs: {uvs.shape}, Texture: {texture.shape}")
+        texture = Image.fromarray(texture)
+        glb = trimesh.Trimesh(vertices, faces, visual=trimesh.visual.TextureVisuals(uv=uvs, image=texture))
+        glb.export("sampled.glb")
+        
+        vertices, faces, uvs, texture = torch.from_numpy(vertices).to(DEVICE), torch.from_numpy(faces).to(torch.int64).to(DEVICE), torch.from_numpy(uvs).to(DEVICE), torch.from_numpy(texture).to(DEVICE)
+        mesh = Mesh(v=vertices, f=faces, vt=uvs, albedo=texture, device=DEVICE)
+        mesh.auto_normal()
+
+        return (mesh,)
+    
 
     

requirements.txt

@@ -43,6 +43,9 @@ pymeshlab
 gpytoolbox
 PyMCubes
 libigl
+pyvista
+pymeshfix
+igraph
 
 # configs & extra
 torchtyping>=0.1.5
@@ -51,4 +54,5 @@ jaxtyping
 packaging
 OmegaConf
 pyhocon
-iopath
+iopath
+easydict