From 97afc83395e29ae7266fc155a5a235576f8e2345 Mon Sep 17 00:00:00 2001 From: MrForExample <62230687+MrForExample@users.noreply.github.com> Date: Fri, 9 Feb 2024 00:14:12 +0100 Subject: [PATCH] Integrate LGM & Update 3DGS to Mesh Conversion Algorithm --- README.md | 40 +- ...iew_Gaussian_Model_to_3DGS and_3DMesh.json | 627 ++++++++++ ...sformers_to_3DGS(DMTet and DiffRast).json} | 0 ...ssian_Transformers_to_3DGS_and_3DMesh.json | 580 ++++++++++ .../_Example_Inputs_Files/Cat_Coffee_rgba.png | Bin 0 -> 800192 bytes .../Cucumber_Shark_rgba.png | Bin 0 -> 575331 bytes .../_Example_Inputs_Files/ghost_rgba.png | Bin 0 -> 55677 bytes algorithms/diff_texturing.py | 7 +- algorithms/large_multiview_gaussian_model.py | 171 +++ algorithms/main_3DGS.py | 7 +- algorithms/nerf_marching_cubes_converter.py | 416 +++++++ install.bat | 9 - lgm/acc_configs/gpu1.yaml | 15 + lgm/acc_configs/gpu4.yaml | 15 + lgm/acc_configs/gpu6.yaml | 15 + lgm/acc_configs/gpu8.yaml | 15 + lgm/core/__init__.py | 0 lgm/core/attention.py | 156 +++ lgm/core/gs.py | 186 +++ lgm/core/options.py | 117 ++ lgm/core/provider_objaverse.py | 172 +++ lgm/core/unet.py | 319 ++++++ lgm/core/utils.py | 109 ++ lgm/mvdream/mv_unet.py | 1005 +++++++++++++++++ lgm/mvdream/pipeline_mvdream.py | 559 +++++++++ mesh_processer/mesh.py | 2 +- nodes.py | 257 ++++- requirements.txt | 21 +- shared_utils/camera_utils.py | 21 - shared_utils/common_utils.py | 16 +- shared_utils/image_utils.py | 4 +- 31 files changed, 4778 insertions(+), 83 deletions(-) create mode 100644 _Example_Workflows/Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json rename _Example_Workflows/{Triplane_Gaussian_Transformers_to_3DGS.json => Triplane_Gaussian_Transformers_to_3DGS(DMTet and DiffRast).json} (100%) create mode 100644 _Example_Workflows/Triplane_Gaussian_Transformers_to_3DGS_and_3DMesh.json create mode 100644 _Example_Workflows/_Example_Inputs_Files/Cat_Coffee_rgba.png create mode 100644 _Example_Workflows/_Example_Inputs_Files/Cucumber_Shark_rgba.png create mode 100644 _Example_Workflows/_Example_Inputs_Files/ghost_rgba.png create mode 100644 algorithms/large_multiview_gaussian_model.py create mode 100644 algorithms/nerf_marching_cubes_converter.py create mode 100644 lgm/acc_configs/gpu1.yaml create mode 100644 lgm/acc_configs/gpu4.yaml create mode 100644 lgm/acc_configs/gpu6.yaml create mode 100644 lgm/acc_configs/gpu8.yaml create mode 100644 lgm/core/__init__.py create mode 100644 lgm/core/attention.py create mode 100644 lgm/core/gs.py create mode 100644 lgm/core/options.py create mode 100644 lgm/core/provider_objaverse.py create mode 100644 lgm/core/unet.py create mode 100644 lgm/core/utils.py create mode 100644 lgm/mvdream/mv_unet.py create mode 100644 lgm/mvdream/pipeline_mvdream.py diff --git a/README.md b/README.md index dee4782..56386cd 100644 --- a/README.md +++ b/README.md @@ -12,26 +12,23 @@ ### Note: this project is still a WIP and not been released into ComFyUI package database yet ## Currently support: -- For use case please check [Example Workflows](./_Example_Workflows/). [**Last update: 04/02/2024**] +- For use case please check [Example Workflows](./_Example_Workflows/). [**Last update: 09/02/2024**] - **Note:** you need to put [Example Inputs Files & Folders](_Example_Workflows/_Example_Inputs_Files/) under ComfyUI Root Directory\ComfyUI\input folder before you can run the example workflow + +- **Large Multiview Gaussian Model**: [3DTopia/LGM](https://github.com/3DTopia/LGM) + - Enable single image to 3D Gaussian in less than 30 seconds on a RTX3080 GPU, later you can also convert 3D Gaussian to mesh + + + +- **Triplane Gaussian Transformers**: [VAST-AI-Research/TriplaneGaussian](https://github.com/VAST-AI-Research/TriplaneGaussian) + - Enable single image to 3D Gaussian in less than 10 seconds on a RTX3080 GPU, later you can also convert 3D Gaussian to mesh + + + - **Preview 3DGS and 3D Mesh**: 3D Visualization inside ComfyUI: - Using [gsplat.js](https://github.com/huggingface/gsplat.js/tree/main) and [three.js](https://github.com/mrdoob/three.js/tree/dev) for 3DGS & 3D Mesh visualization respectively - https://github.com/MrForExample/ComfyUI-3D-Pack/assets/62230687/9f3c56b1-afb3-4bf1-8845-ab1025a87463 - -- **Triplane Gaussian Transformers**: [VAST-AI-Research/TriplaneGaussian](https://github.com/VAST-AI-Research/TriplaneGaussian) - - Enable single image to 3D Gaussian, then you can use Deep Marching Tetrahedrons node to convert it to mesh - -
-
- |
- - - | -
7(51?`~m6kH28_pmyQTv^jmwi>7g+ xbKm0kO@!BKSBUO3LFKLL9lg8QvGpXuR4S_qP)Wklr_DrQVpru1
zDc^@Fc4sUNjZ5MAwTE@?1dd87SDF!H#gaxPOIosK9M{l{8TjD7>!})u;gBipTM4Vw
zh+2Qy;;!Y|@9yBi&O!d|V&C{{8fK8uPVUP<=x~y