diff --git a/.gitignore b/.gitignore index df54d65..fd2ca86 100644 --- a/.gitignore +++ b/.gitignore @@ -157,6 +157,7 @@ cython_debug/ *.pth *.safetensors *.pkl +*.pt .huggingface # Ignore test outputs diff --git a/Checkpoints/CharacterGen/2D_Stage/models/checkpoint/scaler.pt b/Checkpoints/CharacterGen/2D_Stage/models/checkpoint/scaler.pt deleted file mode 100644 index 38ac882..0000000 --- a/Checkpoints/CharacterGen/2D_Stage/models/checkpoint/scaler.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3c63c3d2e7b7f18788aca8a5a0e766ad4633f631d49192602d8532b6b0df7615 -size 559 diff --git a/Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/.gitattributes b/Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/README.md b/Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/README.md new file mode 100644 index 0000000..e2979a1 --- /dev/null +++ b/Checkpoints/Diffusers/JeffreyXiang/TRELLIS-image-large/README.md @@ -0,0 +1,16 @@ +--- +library_name: trellis +pipeline_tag: image-to-3d +license: mit +language: +- en +--- +# TRELLIS Image Large + + + +The image conditioned version of TRELLIS, a large 3D genetive model. It was introduced in the paper [Structured 3D Latents for Scalable and Versatile 3D Generation](https://huggingface.co/papers/2412.01506). + +Project page: https://trellis3d.github.io/ + +Code: https://github.com/Microsoft/TRELLIS diff --git a/Checkpoints/Diffusers/tencent/Hunyuan3D-1/README.md b/Checkpoints/Diffusers/tencent/Hunyuan3D-1/README.md index f109947..c6c8afd 100644 --- a/Checkpoints/Diffusers/tencent/Hunyuan3D-1/README.md +++ b/Checkpoints/Diffusers/tencent/Hunyuan3D-1/README.md @@ -6,6 +6,10 @@ license_link: https://huggingface.co/tencent/Hunyuan3D-1/blob/main/LICENSE.txt language: - en - zh +tags: +- image-to-3d +- text-to-3d +pipeline_tag: text-to-3d --- @@ -237,4 +241,4 @@ If you found this repository helpful, please cite our report: archivePrefix={arXiv}, primaryClass={cs.CV} } -``` +``` \ No newline at end of file diff --git a/Gen_3D_Modules/CharacterGen/Stage_3D/lrm/utils/ops.py b/Gen_3D_Modules/CharacterGen/Stage_3D/lrm/utils/ops.py index 43acc59..652405c 100644 --- a/Gen_3D_Modules/CharacterGen/Stage_3D/lrm/utils/ops.py +++ b/Gen_3D_Modules/CharacterGen/Stage_3D/lrm/utils/ops.py @@ -6,7 +6,7 @@ import torch import torch.nn as nn import torch.nn.functional as F from torch.autograd import Function -from torch.cuda.amp import custom_bwd, custom_fwd +from torch.amp import custom_bwd, custom_fwd from CharacterGen.Stage_3D import lrm from .typing import * @@ -41,13 +41,13 @@ class _TruncExp(Function): # pylint: disable=abstract-method # Implementation from torch-ngp: # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py @staticmethod - @custom_fwd(cast_inputs=torch.float32) + @custom_fwd(cast_inputs=torch.float32, device_type="cuda") def forward(ctx, x): # pylint: disable=arguments-differ ctx.save_for_backward(x) return torch.exp(x) @staticmethod - @custom_bwd + @custom_bwd(device_type="cuda") def backward(ctx, g): # pylint: disable=arguments-differ x = ctx.saved_tensors[0] return g * torch.exp(torch.clamp(x, max=15)) diff --git a/Gen_3D_Modules/Hunyuan3D_V1/svrm/ldm/utils/ops.py b/Gen_3D_Modules/Hunyuan3D_V1/svrm/ldm/utils/ops.py index 85be586..6000780 100644 --- a/Gen_3D_Modules/Hunyuan3D_V1/svrm/ldm/utils/ops.py +++ b/Gen_3D_Modules/Hunyuan3D_V1/svrm/ldm/utils/ops.py @@ -7,7 +7,7 @@ import torch import torch.nn as nn import torch.nn.functional as F from torch.autograd import Function -from torch.cuda.amp import custom_bwd, custom_fwd +from torch.amp import custom_bwd, custom_fwd from igl import fast_winding_number_for_meshes, point_mesh_squared_distance, read_obj from .typing import * @@ -52,13 +52,13 @@ class _TruncExp(Function): # pylint: disable=abstract-method # Implementation from torch-ngp: # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py @staticmethod - @custom_fwd(cast_inputs=torch.float32) + @custom_fwd(cast_inputs=torch.float32, device_type="cuda") def forward(ctx, x): # pylint: disable=arguments-differ ctx.save_for_backward(x) return torch.exp(x) @staticmethod - @custom_bwd + @custom_bwd(device_type="cuda") def backward(ctx, g): # pylint: disable=arguments-differ x = ctx.saved_tensors[0] return g * torch.exp(torch.clamp(x, max=15)) @@ -68,14 +68,14 @@ class SpecifyGradient(Function): # Implementation from stable-dreamfusion # https://github.com/ashawkey/stable-dreamfusion @staticmethod - @custom_fwd + @custom_fwd(device_type="cuda") def forward(ctx, input_tensor, gt_grad): ctx.save_for_backward(gt_grad) # we return a dummy value 1, which will be scaled by amp's scaler so we get the scale in backward. return torch.ones([1], device=input_tensor.device, dtype=input_tensor.dtype) @staticmethod - @custom_bwd + @custom_bwd(device_type="cuda") def backward(ctx, grad_scale): (gt_grad,) = ctx.saved_tensors gt_grad = gt_grad * grad_scale diff --git a/Gen_3D_Modules/StableFast3D/sf3d/models/network.py b/Gen_3D_Modules/StableFast3D/sf3d/models/network.py index f830dc5..d2ed5ad 100644 --- a/Gen_3D_Modules/StableFast3D/sf3d/models/network.py +++ b/Gen_3D_Modules/StableFast3D/sf3d/models/network.py @@ -8,7 +8,7 @@ from einops import rearrange from jaxtyping import Float from torch import Tensor from torch.autograd import Function -from torch.cuda.amp import custom_bwd, custom_fwd +from torch.amp import custom_bwd, custom_fwd from StableFast3D.sf3d.models.utils import BaseModule, normalize @@ -65,13 +65,13 @@ class _TruncExp(Function): # pylint: disable=abstract-method # Implementation from torch-ngp: # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py @staticmethod - @custom_fwd(cast_inputs=torch.float32) + @custom_fwd(cast_inputs=torch.float32, device_type="cuda") def forward(ctx, x): # pylint: disable=arguments-differ ctx.save_for_backward(x) return torch.exp(x) @staticmethod - @custom_bwd + @custom_bwd(device_type="cuda") def backward(ctx, g): # pylint: disable=arguments-differ x = ctx.saved_tensors[0] return g * torch.exp(torch.clamp(x, max=15)) diff --git a/Gen_3D_Modules/TriplaneGaussian/utils/ops.py b/Gen_3D_Modules/TriplaneGaussian/utils/ops.py index 29f0fd4..e1530d2 100644 --- a/Gen_3D_Modules/TriplaneGaussian/utils/ops.py +++ b/Gen_3D_Modules/TriplaneGaussian/utils/ops.py @@ -4,7 +4,7 @@ import numpy as np import torch import torch.nn.functional as F from torch.autograd import Function -from torch.cuda.amp import custom_bwd, custom_fwd +from torch.amp import custom_bwd, custom_fwd from pytorch3d import io from pytorch3d.renderer import ( PointsRasterizationSettings, @@ -35,13 +35,13 @@ class _TruncExp(Function): # pylint: disable=abstract-method # Implementation from torch-ngp: # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py @staticmethod - @custom_fwd(cast_inputs=torch.float32) + @custom_fwd(cast_inputs=torch.float32, device_type="cuda") def forward(ctx, x): # pylint: disable=arguments-differ ctx.save_for_backward(x) return torch.exp(x) @staticmethod - @custom_bwd + @custom_bwd(device_type="cuda") def backward(ctx, g): # pylint: disable=arguments-differ x = ctx.saved_tensors[0] return g * torch.exp(torch.clamp(x, max=15)) diff --git a/Gen_3D_Modules/craftsman/utils/checkpoint.py b/Gen_3D_Modules/craftsman/utils/checkpoint.py index 363e2bf..bf4b8eb 100644 --- a/Gen_3D_Modules/craftsman/utils/checkpoint.py +++ b/Gen_3D_Modules/craftsman/utils/checkpoint.py @@ -36,7 +36,7 @@ def checkpoint( class CheckpointFunction(torch.autograd.Function): @staticmethod - @torch.cuda.amp.custom_fwd + @torch.amp.custom_fwd(device_type="cuda") def forward(ctx, run_function, length, *args): ctx.run_function = run_function ctx.input_tensors = list(args[:length]) @@ -47,7 +47,7 @@ class CheckpointFunction(torch.autograd.Function): return output_tensors @staticmethod - @torch.cuda.amp.custom_bwd + @torch.amp.custom_bwd(device_type="cuda") def backward(ctx, *output_grads): ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors] with torch.enable_grad(): diff --git a/README.md b/README.md index 27f9d80..8d035d7 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,6 @@ This is an extensive node suite that enables ComfyUI to process 3D inputs (Mesh ``` * If you encounter OpenGL errors (e.g., `[F glutil.cpp:338] eglInitialize() failed`), then set `force_cuda_rasterize` to true on corresponding node * If after the installation, your ComfyUI gets stuck at starting or running, you can follow the instructions in the following link to solve the problem: [Code Hangs Indefinitely When Evaluating Neuron Models on GPU](https://github.com/lava-nc/lava-dl/discussions/211) - +* [Fix "nvcc fatal : Could not set up the environment for Microsoft Visual Studio using 'C:/Program Files (x86)/Microsoft V isual Studio/.../vcvars64.bat'](https://github.com/NVlabs/instant-ngp/issues/709#issuecomment-1378155274) ## Supporters - [MrNeRF](https://twitter.com/janusch_patas) diff --git a/_Example_Workflows/_Example_Inputs_Files/squirrel_girl_0.png b/_Example_Workflows/_Example_Inputs_Files/squirrel_girl_0.png new file mode 100644 index 0000000..e739609 Binary files /dev/null and b/_Example_Workflows/_Example_Inputs_Files/squirrel_girl_0.png differ diff --git a/_Example_Workflows/_Example_Inputs_Files/squirrel_girl_1.png b/_Example_Workflows/_Example_Inputs_Files/squirrel_girl_1.png new file mode 100644 index 0000000..4646d8c Binary files /dev/null and b/_Example_Workflows/_Example_Inputs_Files/squirrel_girl_1.png differ diff --git a/_Pre_Builds/_Build_Scripts/build_config.yaml b/_Pre_Builds/_Build_Scripts/build_config.yaml index 6002787..47de545 100644 --- a/_Pre_Builds/_Build_Scripts/build_config.yaml +++ b/_Pre_Builds/_Build_Scripts/build_config.yaml @@ -13,7 +13,7 @@ libs_dir_name: "_Libs" # Folder used to store built wheels wheels_dir_name: "_Build_Wheels" -build_base_packages: ["torch", "torchvision"] +build_base_packages: ["torch", "torchvision", "xformers", "spconv-${cuda_version}"] # All packages will be automatically installed at beginning of install.py follwing orders from top to bottom remote_packages: torch: diff --git a/_Pre_Builds/_Build_Scripts/build_utils.py b/_Pre_Builds/_Build_Scripts/build_utils.py index 8414b07..21e490c 100644 --- a/_Pre_Builds/_Build_Scripts/build_utils.py +++ b/_Pre_Builds/_Build_Scripts/build_utils.py @@ -140,12 +140,17 @@ def git_folder_parallel(repo_id: str, folder: str, recursive: bool=True, root_ou def install_remote_packages(package_names): for package_name in package_names: - package_attr = build_config.remote_packages[package_name] - if hasattr(package_attr, "version"): - package_name += f"=={package_attr.version}" - url_option = package_attr.url_option if hasattr(package_attr, "url_option") else "--index-url" - - subprocess.run([ - PYTHON_PATH, "-s", "-m", "pip", "install", - package_name, url_option, package_attr.url - ]) \ No newline at end of file + if package_name in build_config.remote_packages: + package_attr = build_config.remote_packages[package_name] + if hasattr(package_attr, "version"): + package_name += f"=={package_attr.version}" + if hasattr(package_attr, "url"): + url_option = package_attr.url_option if hasattr(package_attr, "url_option") else "--index-url" + + subprocess.run([ + PYTHON_PATH, "-s", "-m", "pip", "install", + package_name, url_option, package_attr.url + ]) + continue + + subprocess.run([PYTHON_PATH, "-s", "-m", "pip", "install", package_name]) \ No newline at end of file diff --git a/_Pre_Builds/_Build_Scripts/dependencies.txt b/_Pre_Builds/_Build_Scripts/dependencies.txt index a7c08db..0a9ef29 100644 --- a/_Pre_Builds/_Build_Scripts/dependencies.txt +++ b/_Pre_Builds/_Build_Scripts/dependencies.txt @@ -2,4 +2,5 @@ https://github.com/ashawkey/diff-gaussian-rasterization.git https://github.com/NVlabs/nvdiffrast.git https://github.com/ashawkey/kiuikit.git https://github.com/facebookresearch/pytorch3d.git -https://github.com/rusty1s/pytorch_scatter.git \ No newline at end of file +https://github.com/rusty1s/pytorch_scatter.git +https://github.com/EasternJournalist/utils3d.git \ No newline at end of file diff --git a/install.py b/install.py index 18fa170..0fb4124 100644 --- a/install.py +++ b/install.py @@ -47,7 +47,7 @@ try: cstr(f"pip install {wheel_path} to {PYTHON_PATH}").msg.print() # Install packages that needs specify remote url - install_remote_packages(build_config.remote_packages.keys()) + install_remote_packages(build_config.build_base_packages) # Get the target remote pre-built wheels directory name and path platform_config_name = get_platform_config_name() diff --git a/nodes.py b/nodes.py index 88d7053..8bf365a 100644 --- a/nodes.py +++ b/nodes.py @@ -93,6 +93,8 @@ from CRM_T2I_V3.model.crm.sampler import CRMSamplerV3 from Hunyuan3D_V1.mvd.hunyuan3d_mvd_std_pipeline import HunYuan3D_MVD_Std_Pipeline from Hunyuan3D_V1.mvd.hunyuan3d_mvd_lite_pipeline import Hunyuan3D_MVD_Lite_Pipeline from Hunyuan3D_V1.infer import Views2Mesh +from TRELLIS.trellis.pipelines import TrellisImageTo3DPipeline +from TRELLIS.trellis.utils import render_utils, postprocessing_utils from .shared_utils.image_utils import ( prepare_torch_img, torch_imgs_to_pils, troch_image_dilate, @@ -2592,8 +2594,8 @@ class Era3D_MVDiffusion_Model: images_pred = out[bsz:] # [N, 3, H, W] -> [N, H, W, 3] - multiview_images = images_pred.permute(0, 2, 3, 1).to(reference_image.dtype, dtype=reference_image.device) - multiview_normals = normals_pred.permute(0, 2, 3, 1).to(reference_image.dtype, dtype=reference_image.device) + multiview_images = images_pred.permute(0, 2, 3, 1).to(reference_image.device, dtype=reference_image.dtype) + multiview_normals = normals_pred.permute(0, 2, 3, 1).to(reference_image.device, dtype=reference_image.dtype) azimuths = [0, 45, 90, 180, -90, -45] elevations = [0.0] * 6 @@ -3846,6 +3848,7 @@ class Hunyuan3D_V1_Reconstruction_Model: "multiview_image_grid": ("IMAGE",), "condition_image": ("IMAGE",), "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "target_face_count": ("INT", {"default": 90000, "min": 1}), } } @@ -3860,7 +3863,7 @@ class Hunyuan3D_V1_Reconstruction_Model: CATEGORY = "Comfy3D/Algorithm" @torch.no_grad() - def run_model(self, hunyuan3d_v1_reconstruction_model, multiview_image_grid, condition_image, seed): + def run_model(self, hunyuan3d_v1_reconstruction_model, multiview_image_grid, condition_image, seed, target_face_count): mv_grid_pil = torch_imgs_to_pils(multiview_image_grid)[0] condition_pil = torch_imgs_to_pils(condition_image)[0] @@ -3868,6 +3871,7 @@ class Hunyuan3D_V1_Reconstruction_Model: mv_grid_pil, condition_pil, seed=seed, + target_face_count=target_face_count ) vertices, faces, vtx_colors = torch.from_numpy(vertices).to(DEVICE), torch.from_numpy(faces).to(torch.int64).to(DEVICE), torch.from_numpy(vtx_colors).to(DEVICE) mesh = Mesh(v=vertices, f=faces.to(torch.int64), vc=vtx_colors, device=DEVICE) @@ -3875,5 +3879,111 @@ class Hunyuan3D_V1_Reconstruction_Model: return (mesh,) +class Load_Trellis_Structured_3D_Latents_Models: + default_repo_id = "JeffreyXiang/TRELLIS-image-large" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "repo_id": ("STRING", {"default": cls.default_repo_id, "multiline": False}), + }, + } + + RETURN_TYPES = ( + "TRELLIS_PIPE", + ) + RETURN_NAMES = ( + "trellis_pipe", + ) + FUNCTION = "load_pipe" + CATEGORY = "Comfy3D/Import|Export" + + def load_pipe(self, repo_id): + + pipe = TrellisImageTo3DPipeline.from_pretrained(repo_id).to(DEVICE) + + return (pipe, ) + + +class Trellis_Structured_3D_Latents_Models: + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "trellis_pipe": ("TRELLIS_PIPE",), + "reference_image": ("IMAGE",), + "reference_mask": ("MASK",), + "seed": ("INT", {"default": 1, "min": 0, "max": 0xffffffffffffffff}), + "sparse_structure_guidance_scale": ("FLOAT", {"default": 7.5, "min": 0.0, "step": 0.01}), + "sparse_structure_sample_steps": ("INT", {"default": 12, "min": 1}), + "structured_latent_guidance_scale": ("FLOAT", {"default": 3.0, "min": 0.0, "step": 0.01}), + "structured_latent_sample_steps": ("INT", {"default": 12, "min": 1}), + } + } + + RETURN_TYPES = ( + "MESH", + ) + RETURN_NAMES = ( + "mesh", + ) + FUNCTION = "run_model" + CATEGORY = "Comfy3D/Algorithm" + + @torch.no_grad() + def run_model( + self, + trellis_pipe, + reference_image, # [1, H, W, 3] + reference_mask, # [1, H, W] + seed, + sparse_structure_guidance_scale, + sparse_structure_sample_steps, + structured_latent_guidance_scale, + structured_latent_sample_steps, + ): + single_image = torch_imgs_to_pils(reference_image, reference_mask)[0] + + outputs = trellis_pipe.run( + single_image, + # Optional parameters + seed=seed, + formats=["gaussian", "mesh"], + sparse_structure_sampler_params={ + "cfg_strength": sparse_structure_guidance_scale, + "steps": sparse_structure_sample_steps, + }, + slat_sampler_params={ + "cfg_strength": structured_latent_guidance_scale, + "steps": structured_latent_sample_steps, + }, + ) + + # test + import imageio + import trimesh + video = render_utils.render_video(outputs['gaussian'][0])['color'] + imageio.mimsave("sampled_gs.mp4", video, fps=30) + # GLB files can be extracted from the outputs + vertices, faces, uvs, texture = postprocessing_utils.to_glb( + outputs['gaussian'][0], + outputs['mesh'][0], + # Optional parameters + simplify=0.95, # Ratio of triangles to remove in the simplification process + texture_size=1024, # Size of the texture used for the GLB + ) + print(f"Vertices: {vertices.shape}, Faces: {faces.shape}, UVs: {uvs.shape}, Texture: {texture.shape}") + texture = Image.fromarray(texture) + glb = trimesh.Trimesh(vertices, faces, visual=trimesh.visual.TextureVisuals(uv=uvs, image=texture)) + glb.export("sampled.glb") + + vertices, faces, uvs, texture = torch.from_numpy(vertices).to(DEVICE), torch.from_numpy(faces).to(torch.int64).to(DEVICE), torch.from_numpy(uvs).to(DEVICE), torch.from_numpy(texture).to(DEVICE) + mesh = Mesh(v=vertices, f=faces, vt=uvs, albedo=texture, device=DEVICE) + mesh.auto_normal() + + return (mesh,) + diff --git a/requirements.txt b/requirements.txt index 5704e6c..9ba2b80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,6 +43,9 @@ pymeshlab gpytoolbox PyMCubes libigl +pyvista +pymeshfix +igraph # configs & extra torchtyping>=0.1.5 @@ -51,4 +54,5 @@ jaxtyping packaging OmegaConf pyhocon -iopath \ No newline at end of file +iopath +easydict \ No newline at end of file