From 44545fcc153262708f3669757516bfb7e5bd2c87 Mon Sep 17 00:00:00 2001 From: "Mr. For Example" <62230687+MrForExample@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:32:40 -0600 Subject: [PATCH] Remove a few not too useful nodes Remove NeuS, DMTet Refactor GSConverterNeRFMarchingCubes --- README.md | 7 - ...iew_Gaussian_Model_to_3DGS and_3DMesh.json | 684 +++++++------ ..._to_Instant-NGP_to_DiffRast_to_3DMesh.json | 959 ++++++++++++++++++ ...ssian_Transformers_to_3DGS_and_3DMesh.json | 248 +++-- ...iew_Gaussian_Model_to_3DGS and_3DMesh.json | 690 +++++++------ gen_3d_modules/CRM/model/crm/model.py | 2 +- .../CRM/util/flexicubes_geometry.py | 1 - .../geometry/rep_3d/flexicubes_geometry.py | 1 - .../LGM/nerf_marching_cubes_converter.py | 379 +++++++ mesh_processer/mesh_utils.py | 11 +- nodes.py | 203 +--- 11 files changed, 2247 insertions(+), 938 deletions(-) create mode 100644 _Example_Workflows/Multi-View-Images_to_Instant-NGP_to_DiffRast_to_3DMesh.json create mode 100644 gen_3d_modules/LGM/nerf_marching_cubes_converter.py diff --git a/README.md b/README.md index 232451e..26f7af7 100644 --- a/README.md +++ b/README.md @@ -132,13 +132,6 @@ This is an extensive node suite that enables ComfyUI to process 3D inputs (Mesh - Bake Multi-View images into UVTexture of given 3D mesh using [Nvdiffrast](https://github.com/NVlabs/nvdiffrast), supports: - Export to .obj, .ply, .glb -- **NeuS** - - Fit a coarse mesh from sparse multi-view images & normal maps, as little as 4 to 6 views, pretty good at reconstruct the shape from reference images but texture lacking details. - -- **Deep Marching Tetrahedrons** - - Allow convert 3DGS .ply file to 3D mesh
- *Note: I didn't spent time to turn the hyperprameters yet, the result will be improved in the future!* - - **Save & Load 3D file** - .obj, .ply, .glb for 3D Mesh - .ply for 3DGS diff --git a/_Example_Workflows/MVDream_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json b/_Example_Workflows/MVDream_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json index 1abd22c..7b80805 100644 --- a/_Example_Workflows/MVDream_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json +++ b/_Example_Workflows/MVDream_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json @@ -1,121 +1,7 @@ { - "last_node_id": 19, - "last_link_id": 27, + "last_node_id": 20, + "last_link_id": 29, "nodes": [ - { - "id": 9, - "type": "[Comfy3D] Save 3DGS", - "pos": [ - 1798, - 459 - ], - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 13 - } - ], - "outputs": [ - { - "name": "save_path", - "type": "STRING", - "links": [ - 11 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Save 3DGS" - }, - "widgets_values": [ - "LGMTest/FatCat.ply" - ] - }, - { - "id": 17, - "type": "[Comfy3D] Large Multiview Gaussian Model", - "pos": [ - 1270, - 461 - ], - "size": { - "0": 252, - "1": 46 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "multiview_images", - "type": "IMAGE", - "link": 26 - }, - { - "name": "lgm_model", - "type": "LGM_MODEL", - "link": 22 - } - ], - "outputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "links": [ - 19, - 20 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Large Multiview Gaussian Model" - } - }, - { - "id": 2, - "type": "[Comfy3D] Load Diffusers Pipeline", - "pos": [ - 380, - 345 - ], - "size": { - "0": 375.3948669433594, - "1": 83.33004760742188 - }, - "flags": {}, - "order": 0, - "mode": 0, - "outputs": [ - { - "name": "pipe", - "type": "DIFFUSERS_PIPE", - "links": [ - 23 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Load Diffusers Pipeline" - }, - "widgets_values": [ - "MVDreamPipeline", - "ashawkey/imagedream-ipmv-diffusers" - ] - }, { "id": 4, "type": "LoadImage", @@ -128,7 +14,7 @@ "1": 314 }, "flags": {}, - "order": 1, + "order": 0, "mode": 0, "outputs": [ { @@ -206,7 +92,7 @@ "1": 82 }, "flags": {}, - "order": 2, + "order": 1, "mode": 0, "outputs": [ { @@ -227,189 +113,6 @@ "big" ] }, - { - "id": 15, - "type": "[Comfy3D] Preview 3DMesh", - "pos": [ - 2326.1025658000017, - 214.90587429004222 - ], - "size": [ - 600, - 500 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "mesh_file_path", - "type": "STRING", - "link": 16, - "widget": { - "name": "mesh_file_path" - } - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Preview 3DMesh" - }, - "widgets_values": [ - "", - null - ] - }, - { - "id": 14, - "type": "[Comfy3D] Save 3D Mesh", - "pos": [ - 1996.1025658, - 214.90587429004222 - ], - "size": { - "0": 307.8253173828125, - "1": 60.248291015625 - }, - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "mesh", - "type": "MESH", - "link": 17 - } - ], - "outputs": [ - { - "name": "save_path", - "type": "STRING", - "links": [ - 16 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Save 3D Mesh" - }, - "widgets_values": [ - "LGMTest/FatCat.obj" - ] - }, - { - "id": 12, - "type": "[Comfy3D] Switch 3DGS Axis", - "pos": [ - 1558, - 460 - ], - "size": { - "0": 210, - "1": 106 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 19 - } - ], - "outputs": [ - { - "name": "switched_gs_ply", - "type": "GS_PLY", - "links": [ - 13 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Switch 3DGS Axis" - }, - "widgets_values": [ - "+x", - "-y", - "-z" - ] - }, - { - "id": 10, - "type": "[Comfy3D] Preview 3DGS", - "pos": [ - 2331, - 776 - ], - "size": [ - 600, - 500 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "gs_file_path", - "type": "STRING", - "link": 11, - "widget": { - "name": "gs_file_path" - } - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Preview 3DGS" - }, - "widgets_values": [ - "", - null - ] - }, - { - "id": 16, - "type": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes", - "pos": [ - 1566, - 215 - ], - "size": { - "0": 411.6000061035156, - "1": 65.46688079833984 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 20 - } - ], - "outputs": [ - { - "name": "mesh", - "type": "MESH", - "links": [ - 17 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes" - }, - "widgets_values": [ - "big" - ] - }, { "id": 18, "type": "[Comfy3D] MVDream Model", @@ -451,6 +154,12 @@ ], "shape": 3, "slot_index": 0 + }, + { + "name": "orbit_camposes", + "type": "ORBIT_CAMPOSES", + "links": null, + "shape": 3 } ], "properties": { @@ -473,10 +182,10 @@ 1279, 120 ], - "size": [ - 210, - 246 - ], + "size": { + "0": 210, + "1": 246 + }, "flags": {}, "order": 6, "mode": 0, @@ -490,6 +199,329 @@ "properties": { "Node name for S&R": "PreviewImage" } + }, + { + "id": 12, + "type": "[Comfy3D] Switch 3DGS Axis", + "pos": [ + 1560, + 759 + ], + "size": { + "0": 210, + "1": 106 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "link": 19 + } + ], + "outputs": [ + { + "name": "switched_gs_ply", + "type": "GS_PLY", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Switch 3DGS Axis" + }, + "widgets_values": [ + "+x", + "-y", + "-z" + ] + }, + { + "id": 14, + "type": "[Comfy3D] Save 3D Mesh", + "pos": [ + 1996.1025658, + 214.90587429004222 + ], + "size": { + "0": 242.42886352539062, + "1": 58 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "mesh", + "type": "MESH", + "link": 29 + } + ], + "outputs": [ + { + "name": "save_path", + "type": "STRING", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Save 3D Mesh" + }, + "widgets_values": [ + "LGMTest/FatCat.obj" + ] + }, + { + "id": 15, + "type": "[Comfy3D] Preview 3DMesh", + "pos": [ + 2267, + 215 + ], + "size": [ + 600, + 500 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "mesh_file_path", + "type": "STRING", + "link": 16, + "widget": { + "name": "mesh_file_path" + } + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Preview 3DMesh" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 10, + "type": "[Comfy3D] Preview 3DGS", + "pos": [ + 2266, + 769 + ], + "size": [ + 600, + 500 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "gs_file_path", + "type": "STRING", + "link": 11, + "widget": { + "name": "gs_file_path" + } + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Preview 3DGS" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 9, + "type": "[Comfy3D] Save 3DGS", + "pos": [ + 1856, + 759 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "link": 13 + } + ], + "outputs": [ + { + "name": "save_path", + "type": "STRING", + "links": [ + 11 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Save 3DGS" + }, + "widgets_values": [ + "LGMTest/FatCat.ply" + ] + }, + { + "id": 2, + "type": "[Comfy3D] Load Diffusers Pipeline", + "pos": [ + 382, + 284 + ], + "size": { + "0": 375.3948669433594, + "1": 130 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "pipe", + "type": "DIFFUSERS_PIPE", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Load Diffusers Pipeline" + }, + "widgets_values": [ + "MVDreamPipeline", + "ashawkey/imagedream-ipmv-diffusers", + "", + true + ] + }, + { + "id": 17, + "type": "[Comfy3D] Large Multiview Gaussian Model", + "pos": [ + 1270, + 461 + ], + "size": { + "0": 252, + "1": 46 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "multiview_images", + "type": "IMAGE", + "link": 26 + }, + { + "name": "lgm_model", + "type": "LGM_MODEL", + "link": 22 + } + ], + "outputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "links": [ + 19, + 28 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Large Multiview Gaussian Model" + } + }, + { + "id": 20, + "type": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes", + "pos": [ + 1568, + 217 + ], + "size": [ + 411.6000061035156, + 387.6719606496259 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "link": 28 + } + ], + "outputs": [ + { + "name": "mesh", + "type": "MESH", + "links": [ + 29 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "imgs", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "alphas", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes" + }, + "widgets_values": [ + "big", + 512, + 128, + 256, + 128, + 10, + 2048, + 512, + 512, + 512, + 512, + 1024, + false + ] } ], "links": [ @@ -525,14 +557,6 @@ 0, "STRING" ], - [ - 17, - 16, - 0, - 14, - 0, - "MESH" - ], [ 19, 17, @@ -541,14 +565,6 @@ 0, "GS_PLY" ], - [ - 20, - 17, - 0, - 16, - 0, - "GS_PLY" - ], [ 22, 1, @@ -596,10 +612,34 @@ 19, 0, "IMAGE" + ], + [ + 28, + 17, + 0, + 20, + 0, + "GS_PLY" + ], + [ + 29, + 20, + 0, + 14, + 0, + "MESH" ] ], "groups": [], "config": {}, - "extra": {}, + "extra": { + "ds": { + "scale": 0.7513148009015777, + "offset": [ + -509.38235438454456, + 470.4740393503744 + ] + } + }, "version": 0.4 } \ No newline at end of file diff --git a/_Example_Workflows/Multi-View-Images_to_Instant-NGP_to_DiffRast_to_3DMesh.json b/_Example_Workflows/Multi-View-Images_to_Instant-NGP_to_DiffRast_to_3DMesh.json new file mode 100644 index 0000000..e64f1e0 --- /dev/null +++ b/_Example_Workflows/Multi-View-Images_to_Instant-NGP_to_DiffRast_to_3DMesh.json @@ -0,0 +1,959 @@ +{ + "last_node_id": 27, + "last_link_id": 46, + "nodes": [ + { + "id": 6, + "type": "MaskToImage", + "pos": [ + 1490, + 740 + ], + "size": { + "0": 210, + "1": 26 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 6 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 4 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "MaskToImage" + } + }, + { + "id": 10, + "type": "[Comfy3D] Preview 3DMesh", + "pos": [ + 2400, + 684 + ], + "size": [ + 600, + 500 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "mesh_file_path", + "type": "STRING", + "link": 14, + "widget": { + "name": "mesh_file_path" + } + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Preview 3DMesh" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 9, + "type": "Reroute", + "pos": [ + 1237, + 553 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 11 + } + ], + "outputs": [ + { + "name": "", + "type": "ORBIT_CAMPOSES", + "links": [ + 19, + 34 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 5, + "type": "[Comfy3D] Stack Orbit Camera Poses", + "pos": [ + 362, + 389 + ], + "size": { + "0": 315, + "1": 586 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "orbit_camposes", + "type": "ORBIT_CAMPOSES", + "links": [ + 5, + 9 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "orbit_radius_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "elevation_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "azimuth_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "orbit_center_X_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "orbit_center_Y_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "orbit_center_Z_list", + "type": "FLOAT", + "links": null, + "shape": 6 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Stack Orbit Camera Poses" + }, + "widgets_values": [ + 1.75, + 1.75, + 0.1, + -45, + 45, + 45, + 0, + -0.01, + 6, + 0, + 0, + 0.1, + 0, + 0, + 0.1, + 0, + 0, + 0.1 + ] + }, + { + "id": 8, + "type": "Reroute", + "pos": [ + 808, + 553 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 9 + } + ], + "outputs": [ + { + "name": "", + "type": "ORBIT_CAMPOSES", + "links": [ + 11 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 16, + "type": "Reroute", + "pos": [ + 1526, + 189 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 21 + } + ], + "outputs": [ + { + "name": "", + "type": "IMAGE", + "links": [ + 37 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 17, + "type": "Reroute", + "pos": [ + 1551, + 226 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 23 + } + ], + "outputs": [ + { + "name": "", + "type": "MASK", + "links": [ + 38 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 15, + "type": "Reroute", + "pos": [ + 2124, + 552 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "", + "type": "*", + "link": 19 + } + ], + "outputs": [ + { + "name": "", + "type": "ORBIT_CAMPOSES", + "links": [ + 40 + ], + "slot_index": 0 + } + ], + "properties": { + "showOutputText": false, + "horizontal": false + } + }, + { + "id": 11, + "type": "[Comfy3D] Save 3D Mesh", + "pos": [ + 2035, + 684 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "mesh", + "type": "MESH", + "link": 36 + } + ], + "outputs": [ + { + "name": "save_path", + "type": "STRING", + "links": [ + 14 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Save 3D Mesh" + }, + "widgets_values": [ + "Instant_NGP_Test/CatCoffee.obj" + ] + }, + { + "id": 1, + "type": "[Comfy3D] Load 3DGS", + "pos": [ + 104, + 218 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "links": [ + 15 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Load 3DGS" + }, + "widgets_values": [ + "C:\\Users\\reall\\Softwares\\ComfyUI_windows_portable\\ComfyUI\\custom_nodes\\ComfyUI-3D-Pack\\_Example_Workflows\\_Example_Inputs_Files\\Meshes\\CatCoffee.ply" + ] + }, + { + "id": 12, + "type": "[Comfy3D] Switch 3DGS Axis", + "pos": [ + 466, + 219 + ], + "size": { + "0": 210, + "1": 106 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "link": 15 + } + ], + "outputs": [ + { + "name": "switched_gs_ply", + "type": "GS_PLY", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Switch 3DGS Axis" + }, + "widgets_values": [ + "+x", + "-y", + "-z" + ] + }, + { + "id": 18, + "type": "[Comfy3D] Preview 3DMesh", + "pos": [ + 3303, + 189 + ], + "size": [ + 600, + 500 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "mesh_file_path", + "type": "STRING", + "link": 25, + "widget": { + "name": "mesh_file_path" + } + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Preview 3DMesh" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 19, + "type": "[Comfy3D] Save 3D Mesh", + "pos": [ + 2945, + 188 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "mesh", + "type": "MESH", + "link": 41 + } + ], + "outputs": [ + { + "name": "save_path", + "type": "STRING", + "links": [ + 25 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Save 3D Mesh" + }, + "widgets_values": [ + "Instant_NGP_Test/CatCoffee_1.obj" + ] + }, + { + "id": 22, + "type": "[Comfy3D] Fitting Mesh With Multiview Images", + "pos": [ + 2358, + 188 + ], + "size": { + "0": 531.5999755859375, + "1": 406 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "reference_images", + "type": "IMAGE", + "link": 37 + }, + { + "name": "reference_masks", + "type": "MASK", + "link": 38 + }, + { + "name": "reference_orbit_camera_poses", + "type": "ORBIT_CAMPOSES", + "link": 40 + }, + { + "name": "mesh", + "type": "MESH", + "link": 39 + } + ], + "outputs": [ + { + "name": "trained_mesh", + "type": "MESH", + "links": [ + 41 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "baked_texture", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Fitting Mesh With Multiview Images" + }, + "widgets_values": [ + 49.1, + 1024, + 1024, + 2048, + 1, + 0.1, + false, + 0.0001, + 0.5, + 512, + 0, + false + ] + }, + { + "id": 13, + "type": "PreviewImage", + "pos": [ + 1506, + -109 + ], + "size": { + "0": 215.39028930664062, + "1": 246 + }, + "flags": { + "collapsed": false + }, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 17 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 7, + "type": "PreviewImage", + "pos": [ + 1742, + 740 + ], + "size": { + "0": 215.39028930664062, + "1": 246 + }, + "flags": { + "collapsed": false + }, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 21, + "type": "[Comfy3D] Instant NGP", + "pos": [ + 1570, + 288 + ], + "size": { + "0": 418.1999816894531, + "1": 290 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "reference_image", + "type": "IMAGE", + "link": 32 + }, + { + "name": "reference_mask", + "type": "MASK", + "link": 33 + }, + { + "name": "reference_orbit_camera_poses", + "type": "ORBIT_CAMPOSES", + "link": 34 + } + ], + "outputs": [ + { + "name": "mesh", + "type": "MESH", + "links": [ + 36, + 39 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Instant NGP" + }, + "widgets_values": [ + 49.1, + 1024, + 128, + 256, + 128, + 10, + 1024, + 1, + false + ] + }, + { + "id": 2, + "type": "[Comfy3D] Gaussian Splatting Orbit Renderer", + "pos": [ + 807, + 289 + ], + "size": { + "0": 556.800048828125, + "1": 198 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "link": 16 + }, + { + "name": "render_orbit_camera_poses", + "type": "ORBIT_CAMPOSES", + "link": 5 + } + ], + "outputs": [ + { + "name": "rendered_gs_images", + "type": "IMAGE", + "links": [ + 17, + 21, + 32 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "rendered_gs_masks", + "type": "MASK", + "links": [ + 6, + 23, + 33 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Gaussian Splatting Orbit Renderer" + }, + "widgets_values": [ + 1024, + 1024, + 49.1, + 1, + 1, + 1 + ] + } + ], + "links": [ + [ + 4, + 6, + 0, + 7, + 0, + "IMAGE" + ], + [ + 5, + 5, + 0, + 2, + 1, + "ORBIT_CAMPOSES" + ], + [ + 6, + 2, + 1, + 6, + 0, + "MASK" + ], + [ + 9, + 5, + 0, + 8, + 0, + "*" + ], + [ + 11, + 8, + 0, + 9, + 0, + "*" + ], + [ + 14, + 11, + 0, + 10, + 0, + "STRING" + ], + [ + 15, + 1, + 0, + 12, + 0, + "GS_PLY" + ], + [ + 16, + 12, + 0, + 2, + 0, + "GS_PLY" + ], + [ + 17, + 2, + 0, + 13, + 0, + "IMAGE" + ], + [ + 19, + 9, + 0, + 15, + 0, + "*" + ], + [ + 21, + 2, + 0, + 16, + 0, + "*" + ], + [ + 23, + 2, + 1, + 17, + 0, + "*" + ], + [ + 25, + 19, + 0, + 18, + 0, + "STRING" + ], + [ + 32, + 2, + 0, + 21, + 0, + "IMAGE" + ], + [ + 33, + 2, + 1, + 21, + 1, + "MASK" + ], + [ + 34, + 9, + 0, + 21, + 2, + "ORBIT_CAMPOSES" + ], + [ + 36, + 21, + 0, + 11, + 0, + "MESH" + ], + [ + 37, + 16, + 0, + 22, + 0, + "IMAGE" + ], + [ + 38, + 17, + 0, + 22, + 1, + "MASK" + ], + [ + 39, + 21, + 0, + 22, + 3, + "MESH" + ], + [ + 40, + 15, + 0, + 22, + 2, + "ORBIT_CAMPOSES" + ], + [ + 41, + 22, + 0, + 19, + 0, + "MESH" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.6934334949441353, + "offset": [ + -1086.556448784114, + 243.62758422318373 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/_Example_Workflows/Triplane_Gaussian_Transformers_to_3DGS_and_3DMesh.json b/_Example_Workflows/Triplane_Gaussian_Transformers_to_3DGS_and_3DMesh.json index 506d65a..cec3b78 100644 --- a/_Example_Workflows/Triplane_Gaussian_Transformers_to_3DGS_and_3DMesh.json +++ b/_Example_Workflows/Triplane_Gaussian_Transformers_to_3DGS_and_3DMesh.json @@ -69,40 +69,6 @@ 49.1 ] }, - { - "id": 12, - "type": "[Comfy3D] Preview 3DGS", - "pos": [ - 1590, - 800 - ], - "size": [ - 600, - 500 - ], - "flags": { - "collapsed": false - }, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "gs_file_path", - "type": "STRING", - "link": 13, - "widget": { - "name": "gs_file_path" - } - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Preview 3DGS" - }, - "widgets_values": [ - "", - null - ] - }, { "id": 14, "type": "[Comfy3D] Triplane Gaussian Transformers", @@ -152,38 +118,6 @@ 1.9000000000000001 ] }, - { - "id": 26, - "type": "[Comfy3D] Preview 3DMesh", - "pos": [ - 2328, - 796 - ], - "size": [ - 600, - 500 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "mesh_file_path", - "type": "STRING", - "link": 32, - "widget": { - "name": "mesh_file_path" - } - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Preview 3DMesh" - }, - "widgets_values": [ - "", - null - ] - }, { "id": 11, "type": "Note", @@ -250,45 +184,6 @@ "-x" ] }, - { - "id": 10, - "type": "[Comfy3D] Save 3DGS", - "pos": [ - 1288, - 800 - ], - "size": { - "0": 275.57391357421875, - "1": 58 - }, - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 12 - } - ], - "outputs": [ - { - "name": "save_path", - "type": "STRING", - "links": [ - 13 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Save 3DGS" - }, - "widgets_values": [ - "MeshTest\\3DGS_Picacho.ply" - ] - }, { "id": 34, "type": "[Comfy3D] Switch 3DGS Axis", @@ -339,7 +234,7 @@ ], "size": { "0": 411.6000061035156, - "1": 58 + "1": 386 }, "flags": {}, "order": 10, @@ -360,13 +255,37 @@ ], "shape": 3, "slot_index": 0 + }, + { + "name": "imgs", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "alphas", + "type": "MASK", + "links": null, + "shape": 3 } ], "properties": { "Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes" }, "widgets_values": [ - "big" + "big", + 512, + 128, + 256, + 128, + 10, + 2048, + 512, + 512, + 512, + 512, + 1024, + false ] }, { @@ -481,6 +400,111 @@ "a_pikachu_with_smily_face-removebg.png", "image" ] + }, + { + "id": 10, + "type": "[Comfy3D] Save 3DGS", + "pos": [ + 1226, + 1084 + ], + "size": { + "0": 275.57391357421875, + "1": 58 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "link": 12 + } + ], + "outputs": [ + { + "name": "save_path", + "type": "STRING", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Save 3DGS" + }, + "widgets_values": [ + "MeshTest\\3DGS_Picacho.ply" + ] + }, + { + "id": 12, + "type": "[Comfy3D] Preview 3DGS", + "pos": [ + 1529, + 1083 + ], + "size": [ + 600, + 500 + ], + "flags": { + "collapsed": false + }, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "gs_file_path", + "type": "STRING", + "link": 13, + "widget": { + "name": "gs_file_path" + } + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Preview 3DGS" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 26, + "type": "[Comfy3D] Preview 3DMesh", + "pos": [ + 2327, + 628 + ], + "size": [ + 600, + 500 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "mesh_file_path", + "type": "STRING", + "link": 32, + "widget": { + "name": "mesh_file_path" + } + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Preview 3DMesh" + }, + "widgets_values": [ + "", + null + ] } ], "links": [ @@ -575,6 +599,14 @@ ], "groups": [], "config": {}, - "extra": {}, + "extra": { + "ds": { + "scale": 0.6209213230591553, + "offset": [ + -672.2404394535945, + -402.3689017930591 + ] + } + }, "version": 0.4 } \ No newline at end of file diff --git a/_Example_Workflows/Wonder3D_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json b/_Example_Workflows/Wonder3D_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json index cd6eeee..3ac6f70 100644 --- a/_Example_Workflows/Wonder3D_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json +++ b/_Example_Workflows/Wonder3D_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json @@ -1,194 +1,7 @@ { - "last_node_id": 23, - "last_link_id": 36, + "last_node_id": 24, + "last_link_id": 38, "nodes": [ - { - "id": 17, - "type": "[Comfy3D] Large Multiview Gaussian Model", - "pos": [ - 1270, - 461 - ], - "size": { - "0": 252, - "1": 46 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "multiview_images", - "type": "IMAGE", - "link": 35 - }, - { - "name": "lgm_model", - "type": "LGM_MODEL", - "link": 22 - } - ], - "outputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "links": [ - 19, - 20 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Large Multiview Gaussian Model" - } - }, - { - "id": 15, - "type": "[Comfy3D] Preview 3DMesh", - "pos": [ - 2326.1025658000017, - 214.90587429004222 - ], - "size": [ - 600, - 500 - ], - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [ - { - "name": "mesh_file_path", - "type": "STRING", - "link": 16, - "widget": { - "name": "mesh_file_path" - } - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Preview 3DMesh" - }, - "widgets_values": [ - "", - null - ] - }, - { - "id": 12, - "type": "[Comfy3D] Switch 3DGS Axis", - "pos": [ - 1558, - 460 - ], - "size": { - "0": 210, - "1": 106 - }, - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 19 - } - ], - "outputs": [ - { - "name": "switched_gs_ply", - "type": "GS_PLY", - "links": [ - 13 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Switch 3DGS Axis" - }, - "widgets_values": [ - "+x", - "-y", - "-z" - ] - }, - { - "id": 10, - "type": "[Comfy3D] Preview 3DGS", - "pos": [ - 2331, - 776 - ], - "size": [ - 600, - 500 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [ - { - "name": "gs_file_path", - "type": "STRING", - "link": 11, - "widget": { - "name": "gs_file_path" - } - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Preview 3DGS" - }, - "widgets_values": [ - "", - null - ] - }, - { - "id": 16, - "type": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes", - "pos": [ - 1566, - 215 - ], - "size": { - "0": 411.6000061035156, - "1": 82 - }, - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 20 - } - ], - "outputs": [ - { - "name": "mesh", - "type": "MESH", - "links": [ - 17 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes" - }, - "widgets_values": [ - "big", - false - ] - }, { "id": 5, "type": "InvertMask", @@ -226,39 +39,6 @@ "Node name for S&R": "InvertMask" } }, - { - "id": 20, - "type": "[Comfy3D] Load Diffusers Pipeline", - "pos": [ - -300, - 330 - ], - "size": { - "0": 392.1646728515625, - "1": 86.24864196777344 - }, - "flags": {}, - "order": 0, - "mode": 0, - "outputs": [ - { - "name": "pipe", - "type": "DIFFUSERS_PIPE", - "links": [ - 29 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Load Diffusers Pipeline" - }, - "widgets_values": [ - "Wonder3DMVDiffusionPipeline", - "flamehaze1115/wonder3d-v1.0" - ] - }, { "id": 22, "type": "[Comfy3D] Wonder3D MVDiffusion Model", @@ -326,7 +106,7 @@ 461 ], "size": { - "0": 236.4545135498047, + "0": 243.60000610351562, "1": 76 }, "flags": {}, @@ -370,7 +150,7 @@ "1": 82 }, "flags": {}, - "order": 1, + "order": 0, "mode": 0, "outputs": [ { @@ -400,10 +180,10 @@ ], "size": { "0": 375.3948669433594, - "1": 83.33004760742188 + "1": 130 }, "flags": {}, - "order": 2, + "order": 1, "mode": 4, "outputs": [ { @@ -421,7 +201,9 @@ }, "widgets_values": [ "MVDreamPipeline", - "ashawkey/imagedream-ipmv-diffusers" + "ashawkey/imagedream-ipmv-diffusers", + "", + true ] }, { @@ -462,6 +244,12 @@ "links": [], "shape": 3, "slot_index": 0 + }, + { + "name": "orbit_camposes", + "type": "ORBIT_CAMPOSES", + "links": null, + "shape": 3 } ], "properties": { @@ -477,84 +265,6 @@ 0 ] }, - { - "id": 14, - "type": "[Comfy3D] Save 3D Mesh", - "pos": [ - 1996.1025658, - 214.90587429004222 - ], - "size": { - "0": 307.8253173828125, - "1": 60.248291015625 - }, - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "mesh", - "type": "MESH", - "link": 17 - } - ], - "outputs": [ - { - "name": "save_path", - "type": "STRING", - "links": [ - 16 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Save 3D Mesh" - }, - "widgets_values": [ - "LGMTest/KungFuCat.obj" - ] - }, - { - "id": 9, - "type": "[Comfy3D] Save 3DGS", - "pos": [ - 1798, - 459 - ], - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 13 - } - ], - "outputs": [ - { - "name": "save_path", - "type": "STRING", - "links": [ - 11 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Save 3DGS" - }, - "widgets_values": [ - "LGMTest/KungFuCat.ply" - ] - }, { "id": 4, "type": "LoadImage", @@ -567,7 +277,7 @@ "1": 314 }, "flags": {}, - "order": 3, + "order": 2, "mode": 0, "outputs": [ { @@ -622,6 +332,329 @@ "properties": { "Node name for S&R": "PreviewImage" } + }, + { + "id": 14, + "type": "[Comfy3D] Save 3D Mesh", + "pos": [ + 1996.1025658, + 214.90587429004222 + ], + "size": { + "0": 259.5017395019531, + "1": 58 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "mesh", + "type": "MESH", + "link": 37 + } + ], + "outputs": [ + { + "name": "save_path", + "type": "STRING", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Save 3D Mesh" + }, + "widgets_values": [ + "LGMTest/KungFuCat.obj" + ] + }, + { + "id": 15, + "type": "[Comfy3D] Preview 3DMesh", + "pos": [ + 2289, + 215 + ], + "size": [ + 600, + 500 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "mesh_file_path", + "type": "STRING", + "link": 16, + "widget": { + "name": "mesh_file_path" + } + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Preview 3DMesh" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 10, + "type": "[Comfy3D] Preview 3DGS", + "pos": [ + 2203, + 776 + ], + "size": [ + 600, + 500 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "gs_file_path", + "type": "STRING", + "link": 11, + "widget": { + "name": "gs_file_path" + } + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Preview 3DGS" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 12, + "type": "[Comfy3D] Switch 3DGS Axis", + "pos": [ + 1572, + 776 + ], + "size": { + "0": 210, + "1": 106 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "link": 19 + } + ], + "outputs": [ + { + "name": "switched_gs_ply", + "type": "GS_PLY", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Switch 3DGS Axis" + }, + "widgets_values": [ + "+x", + "-y", + "-z" + ] + }, + { + "id": 9, + "type": "[Comfy3D] Save 3DGS", + "pos": [ + 1813, + 777 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "link": 13 + } + ], + "outputs": [ + { + "name": "save_path", + "type": "STRING", + "links": [ + 11 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Save 3DGS" + }, + "widgets_values": [ + "LGMTest/KungFuCat.ply" + ] + }, + { + "id": 17, + "type": "[Comfy3D] Large Multiview Gaussian Model", + "pos": [ + 1270, + 461 + ], + "size": { + "0": 252, + "1": 46 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "multiview_images", + "type": "IMAGE", + "link": 35 + }, + { + "name": "lgm_model", + "type": "LGM_MODEL", + "link": 22 + } + ], + "outputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "links": [ + 19, + 38 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Large Multiview Gaussian Model" + } + }, + { + "id": 24, + "type": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes", + "pos": [ + 1566, + 216 + ], + "size": { + "0": 411.6000061035156, + "1": 386 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "link": 38 + } + ], + "outputs": [ + { + "name": "mesh", + "type": "MESH", + "links": [ + 37 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "imgs", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "alphas", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes" + }, + "widgets_values": [ + "big", + 1024, + 128, + 256, + 128, + 10, + 2048, + 512, + 512, + 512, + 512, + 1024, + false + ] + }, + { + "id": 20, + "type": "[Comfy3D] Load Diffusers Pipeline", + "pos": [ + -264, + 278 + ], + "size": { + "0": 392.1646728515625, + "1": 130 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "pipe", + "type": "DIFFUSERS_PIPE", + "links": [ + 29 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Load Diffusers Pipeline" + }, + "widgets_values": [ + "Wonder3DMVDiffusionPipeline", + "flamehaze1115/wonder3d-v1.0", + "", + true + ] } ], "links": [ @@ -657,14 +690,6 @@ 0, "STRING" ], - [ - 17, - 16, - 0, - 14, - 0, - "MESH" - ], [ 19, 17, @@ -673,14 +698,6 @@ 0, "GS_PLY" ], - [ - 20, - 17, - 0, - 16, - 0, - "GS_PLY" - ], [ 22, 1, @@ -760,6 +777,22 @@ 19, 0, "IMAGE" + ], + [ + 37, + 24, + 0, + 14, + 0, + "MESH" + ], + [ + 38, + 17, + 0, + 24, + 0, + "GS_PLY" ] ], "groups": [ @@ -772,11 +805,18 @@ 436 ], "color": "#3f789e", - "font_size": 24, - "locked": false + "font_size": 24 } ], "config": {}, - "extra": {}, + "extra": { + "ds": { + "scale": 0.6209213230591556, + "offset": [ + -137.46041222315475, + 303.4617473928127 + ] + } + }, "version": 0.4 } \ No newline at end of file diff --git a/gen_3d_modules/CRM/model/crm/model.py b/gen_3d_modules/CRM/model/crm/model.py index 504a22f..fa4ca39 100644 --- a/gen_3d_modules/CRM/model/crm/model.py +++ b/gen_3d_modules/CRM/model/crm/model.py @@ -41,7 +41,7 @@ class ConvolutionalReconstructionModel(nn.Module): self.dec.c_dim = specs["DecoderSpecs"]["c_dim"] self.dec.plane_resolution = specs["DecoderSpecs"]["plane_resolution"] - self.geo_type = specs["Train"].get("geo_type", "flex") # "dmtet" or "flex" + self.geo_type = specs["Train"].get("geo_type", "flex") # only support "flex" self.unet2 = UNetPP(in_channels=self.dec.c_dim) diff --git a/gen_3d_modules/CRM/util/flexicubes_geometry.py b/gen_3d_modules/CRM/util/flexicubes_geometry.py index 8bc8622..b290b84 100644 --- a/gen_3d_modules/CRM/util/flexicubes_geometry.py +++ b/gen_3d_modules/CRM/util/flexicubes_geometry.py @@ -8,7 +8,6 @@ import torch from FlexiCubes.flexicubes import FlexiCubes -# from dmtet import sdf_reg_loss_batch import torch.nn.functional as F def get_center_boundary_index(grid_res, device): diff --git a/gen_3d_modules/InstantMesh/models/geometry/rep_3d/flexicubes_geometry.py b/gen_3d_modules/InstantMesh/models/geometry/rep_3d/flexicubes_geometry.py index e22fcd7..75681c7 100644 --- a/gen_3d_modules/InstantMesh/models/geometry/rep_3d/flexicubes_geometry.py +++ b/gen_3d_modules/InstantMesh/models/geometry/rep_3d/flexicubes_geometry.py @@ -11,7 +11,6 @@ import numpy as np import os from . import Geometry from FlexiCubes.flexicubes import FlexiCubes -from .dmtet import sdf_reg_loss_batch import torch.nn.functional as F def get_center_boundary_index(grid_res, device): diff --git a/gen_3d_modules/LGM/nerf_marching_cubes_converter.py b/gen_3d_modules/LGM/nerf_marching_cubes_converter.py new file mode 100644 index 0000000..3d7e9bd --- /dev/null +++ b/gen_3d_modules/LGM/nerf_marching_cubes_converter.py @@ -0,0 +1,379 @@ +import tqdm +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import comfy.utils + +import nerfacc +import nvdiffrast.torch as dr + +from kiui.mesh_utils import clean_mesh, decimate_mesh +from kiui.mesh_utils import laplacian_smooth_loss, normal_consistency +from kiui.op import inverse_sigmoid +from kiui.cam import orbit_camera, get_perspective +from kiui.nn import MLP, trunc_exp + +from mesh_processer.mesh import Mesh +from mesh_processer.mesh_utils import marching_cubes_density_to_mesh, color_func_to_albedo +from LGM.core.options import Options +from LGM.core.gs import GaussianRenderer +from LGM.core.utils import get_rays + +# Triple renderer of gaussians, gaussian, and diso mesh. +# gaussian --> nerf --> mesh +class GSConverterNeRFMarchingCubes(nn.Module): + def __init__(self, opt: Options, gs_ply): + super().__init__() + from kiui.gridencoder import GridEncoder + + self.opt = opt + self.device = torch.device("cuda") + + # gs renderer + self.tan_half_fov = np.tan(0.5 * np.deg2rad(opt.fovy)) + self.proj_matrix = torch.zeros(4, 4, dtype=torch.float32, device=self.device) + self.proj_matrix[0, 0] = 1 / self.tan_half_fov + self.proj_matrix[1, 1] = 1 / self.tan_half_fov + self.proj_matrix[2, 2] = (opt.zfar + opt.znear) / (opt.zfar - opt.znear) + self.proj_matrix[3, 2] = - (opt.zfar * opt.znear) / (opt.zfar - opt.znear) + self.proj_matrix[2, 3] = 1 + + self.gs_renderer = GaussianRenderer(opt) + + self.gaussians = self.gs_renderer.create_from_ply(gs_ply).to(self.device) + + # nerf renderer + if not self.opt.force_cuda_rast: + self.glctx = dr.RasterizeGLContext() + else: + self.glctx = dr.RasterizeCudaContext() + + self.step = 0 + self.render_step_size = 5e-3 + self.aabb = torch.tensor([-1.0, -1.0, -1.0, 1.0, 1.0, 1.0], device=self.device) + self.estimator = nerfacc.OccGridEstimator(roi_aabb=self.aabb, resolution=64, levels=1) + + self.encoder_density = GridEncoder(num_levels=12) # VMEncoder(output_dim=16, mode='sum') + self.encoder = GridEncoder(num_levels=12) + self.mlp_density = MLP(self.encoder_density.output_dim, 1, 32, 2, bias=False) + self.mlp = MLP(self.encoder.output_dim, 3, 32, 2, bias=False) + + # mesh renderer + self.proj = torch.from_numpy(get_perspective(self.opt.fovy)).float().to(self.device) + self.v = self.f = None + self.vt = self.ft = None + self.deform = None + self.albedo = None + + + @torch.no_grad() + def render_gs(self, pose): + + cam_poses = torch.from_numpy(pose).unsqueeze(0).to(self.device) + cam_poses[:, :3, 1:3] *= -1 # invert up & forward direction + + # cameras needed by gaussian rasterizer + cam_view = torch.inverse(cam_poses).transpose(1, 2) # [V, 4, 4] + cam_view_proj = cam_view @ self.proj_matrix # [V, 4, 4] + cam_pos = - cam_poses[:, :3, 3] # [V, 3] + + out = self.gs_renderer.render(self.gaussians.unsqueeze(0), cam_view.unsqueeze(0), cam_view_proj.unsqueeze(0), cam_pos.unsqueeze(0)) + image = out['image'].squeeze(1).squeeze(0) # [C, H, W] + alpha = out['alpha'].squeeze(2).squeeze(1).squeeze(0) # [H, W] + + return image, alpha + + def get_density(self, xs): + # xs: [..., 3] + xs = xs.to(self.device) + prefix = xs.shape[:-1] + xs = xs.view(-1, 3) + feats = self.encoder_density(xs) + density = trunc_exp(self.mlp_density(feats)) + density = density.view(*prefix, 1) + return density + + def render_nerf(self, pose): + + pose = torch.from_numpy(pose.astype(np.float32)).to(self.device) + + # get rays + resolution = self.opt.output_size + rays_o, rays_d = get_rays(pose, resolution, resolution, self.opt.fovy) + hw = rays_o.shape[0] * rays_o.shape[1] + rays_o = rays_o.view(hw, 3) + rays_d = rays_d.view(hw, 3) + + # update occ grid + if self.training: + def occ_eval_fn(xs): + sigmas = self.get_density(xs) + return self.render_step_size * sigmas + + self.estimator.update_every_n_steps(self.step, occ_eval_fn=occ_eval_fn, occ_thre=0.01, n=8) + self.step += 1 + + # render + def sigma_fn(t_starts, t_ends, ray_indices): + t_origins = rays_o[ray_indices] + t_dirs = rays_d[ray_indices] + xs = t_origins + t_dirs * (t_starts + t_ends)[:, None] / 2.0 + sigmas = self.get_density(xs) + return sigmas.squeeze(-1) + + with torch.no_grad(): + ray_indices, t_starts, t_ends = self.estimator.sampling( + rays_o, + rays_d, + sigma_fn=sigma_fn, + near_plane=0.01, + far_plane=100, + render_step_size=self.render_step_size, + stratified=self.training, + cone_angle=0, + ) + + t_origins = rays_o[ray_indices] + t_dirs = rays_d[ray_indices] + xs = t_origins + t_dirs * (t_starts + t_ends)[:, None] / 2.0 + sigmas = self.get_density(xs).squeeze(-1) + rgbs = torch.sigmoid(self.mlp(self.encoder(xs))) + + n_rays=rays_o.shape[0] + weights, trans, alphas = nerfacc.render_weight_from_density(t_starts, t_ends, sigmas, ray_indices=ray_indices, n_rays=n_rays) + color = nerfacc.accumulate_along_rays(weights, values=rgbs, ray_indices=ray_indices, n_rays=n_rays) + alpha = nerfacc.accumulate_along_rays(weights, values=None, ray_indices=ray_indices, n_rays=n_rays) + + color = color + 1 * (1.0 - alpha) + + color = color.view(resolution, resolution, 3).clamp(0, 1).permute(2, 0, 1).contiguous() + alpha = alpha.view(resolution, resolution).clamp(0, 1).contiguous() + + return color, alpha + + def fit_nerf(self, iters=512, resolution=128): + + self.opt.output_size = resolution + + optimizer = torch.optim.Adam([ + {'params': self.encoder_density.parameters(), 'lr': 1e-2}, + {'params': self.encoder.parameters(), 'lr': 1e-2}, + {'params': self.mlp_density.parameters(), 'lr': 1e-3}, + {'params': self.mlp.parameters(), 'lr': 1e-3}, + ]) + + print(f"[INFO] fitting nerf...") + imgs, alphas = [], [] + + comfy_pbar = comfy.utils.ProgressBar(iters) + pbar = tqdm.trange(iters) + for i in pbar: + + ver = np.random.randint(-45, 45) + hor = np.random.randint(-180, 180) + rad = np.random.uniform(1.5, 3.0) + + pose = orbit_camera(ver, hor, rad) + + image_gt, alpha_gt = self.render_gs(pose) + imgs.append(image_gt.permute(1, 2, 0).unsqueeze(0)) + alphas.append(alpha_gt.unsqueeze(0)) + image_pred, alpha_pred = self.render_nerf(pose) + + # if i % 200 == 0: + # kiui.vis.plot_image(image_gt, alpha_gt, image_pred, alpha_pred) + + loss_mse = F.mse_loss(image_pred, image_gt) + 0.1 * F.mse_loss(alpha_pred, alpha_gt) + loss = loss_mse #+ 0.1 * self.encoder_density.tv_loss() #+ 0.0001 * self.encoder_density.density_loss() + + loss.backward() + self.encoder_density.grad_total_variation(1e-8) + + optimizer.step() + optimizer.zero_grad() + + pbar.set_description(f"MSE = {loss_mse.item():.6f}") + comfy_pbar.update_absolute(i + 1) + + print(f"[INFO] finished fitting nerf!") + + return torch.cat(imgs, dim=0), torch.cat(alphas, dim=0) + + def render_mesh(self, pose): + + h = w = self.opt.output_size + + v = self.v + self.deform + f = self.f + + pose = torch.from_numpy(pose.astype(np.float32)).to(v.device) + + # get v_clip and render rgb + v_cam = torch.matmul(F.pad(v, pad=(0, 1), mode='constant', value=1.0), torch.inverse(pose).T).float().unsqueeze(0) + v_clip = v_cam @ self.proj.T + + rast, rast_db = dr.rasterize(self.glctx, v_clip, f, (h, w)) + + alpha = torch.clamp(rast[..., -1:], 0, 1).contiguous() # [1, H, W, 1] + alpha = dr.antialias(alpha, rast, v_clip, f).clamp(0, 1).squeeze(-1).squeeze(0) # [H, W] important to enable gradients! + + if self.albedo is None: + xyzs, _ = dr.interpolate(v.unsqueeze(0), rast, f) # [1, H, W, 3] + xyzs = xyzs.view(-1, 3) + mask = (alpha > 0).view(-1) + image = torch.zeros_like(xyzs, dtype=torch.float32) + if mask.any(): + masked_albedo = torch.sigmoid(self.mlp(self.encoder(xyzs[mask].detach(), bound=1))) + image[mask] = masked_albedo.float() + else: + texc, texc_db = dr.interpolate(self.vt.unsqueeze(0), rast, self.ft, rast_db=rast_db, diff_attrs='all') + image = torch.sigmoid(dr.texture(self.albedo.unsqueeze(0), texc, uv_da=texc_db)) # [1, H, W, 3] + + image = image.view(1, h, w, 3) + # image = dr.antialias(image, rast, v_clip, f).clamp(0, 1) + image = image.squeeze(0).permute(2, 0, 1).contiguous() # [3, H, W] + image = alpha * image + (1 - alpha) + + return image, alpha + + def fit_mesh(self, iters=2048, remesh_after_n_iteration=512, resolution=512, grid_size=256, S=128, density_thresh=10, decimate_target=5e4): + + self.opt.output_size = resolution + + vertices, triangles = marching_cubes_density_to_mesh(self.get_density, grid_size, S, density_thresh, decimate_target) + + self.v = torch.from_numpy(vertices).contiguous().float().to(self.device) + self.f = torch.from_numpy(triangles).contiguous().int().to(self.device) + self.deform = nn.Parameter(torch.zeros_like(self.v)).to(self.device) + + # Here coarse mesh already resemble the target shape closely + #mcube_mesh = Mesh(v=self.v, f=self.f, albedo=None, device=self.device) + #mcube_mesh.auto_normal() + #mcube_mesh.auto_uv() + + # fit mesh from gs + lr_factor = 1 + optimizer = torch.optim.Adam([ + {'params': self.encoder.parameters(), 'lr': 1e-3 * lr_factor}, + {'params': self.mlp.parameters(), 'lr': 1e-3 * lr_factor}, + {'params': self.deform, 'lr': 1e-4}, + ]) + + print(f"[INFO] fitting mesh...") + comfy_pbar = comfy.utils.ProgressBar(iters) + pbar = tqdm.trange(iters) + for i in pbar: + + ver = np.random.randint(-10, 10) + hor = np.random.randint(-180, 180) + rad = self.opt.cam_radius # np.random.uniform(1, 2) + + pose = orbit_camera(ver, hor, rad) + + image_gt, alpha_gt = self.render_gs(pose) + image_pred, alpha_pred = self.render_mesh(pose) + + loss_mse = F.mse_loss(image_pred, image_gt) + 0.1 * F.mse_loss(alpha_pred, alpha_gt) + loss_lap = laplacian_smooth_loss(self.v + self.deform, self.f) + loss_normal = normal_consistency(self.v + self.deform, self.f) + loss_offsets = (self.deform ** 2).sum(-1).mean() + loss = loss_mse + 0.001 * loss_normal + 0.1 * loss_offsets * loss_lap * 0.01 + + loss.backward() + + optimizer.step() + optimizer.zero_grad() + + # remesh periodically + if i > 0 and i % remesh_after_n_iteration == 0: + vertices = (self.v + self.deform).detach().cpu().numpy() + triangles = self.f.detach().cpu().numpy() + vertices, triangles = clean_mesh(vertices, triangles, remesh=True, remesh_size=0.01) + if triangles.shape[0] > decimate_target: + vertices, triangles = decimate_mesh(vertices, triangles, decimate_target, optimalplacement=False) + self.v = torch.from_numpy(vertices).contiguous().float().to(self.device) + self.f = torch.from_numpy(triangles).contiguous().int().to(self.device) + self.deform = nn.Parameter(torch.zeros_like(self.v)).to(self.device) + lr_factor *= 0.5 + optimizer = torch.optim.Adam([ + {'params': self.encoder.parameters(), 'lr': 1e-3 * lr_factor}, + {'params': self.mlp.parameters(), 'lr': 1e-3 * lr_factor}, + {'params': self.deform, 'lr': 1e-4}, + ]) + + pbar.set_description(f"MSE = {loss_mse.item():.6f}") + comfy_pbar.update_absolute(i + 1) + + # last clean + vertices = (self.v + self.deform).detach().cpu().numpy() + triangles = self.f.detach().cpu().numpy() + vertices, triangles = clean_mesh(vertices, triangles, remesh=False) + self.v = torch.from_numpy(vertices).contiguous().float().to(self.device) + self.f = torch.from_numpy(triangles).contiguous().int().to(self.device) + self.deform = nn.Parameter(torch.zeros_like(self.v).to(self.device)) + + print(f"[INFO] finished fitting mesh!") + + def get_color(self, xs): + return torch.sigmoid(self.mlp(self.encoder(xs.to(self.device)))) + + # uv mesh refine + def fit_mesh_uv(self, iters=512, resolution=512, texture_resolution=1024, padding=2): + + self.opt.output_size = resolution + + # unwrap uv + print(f"[INFO] uv unwrapping...") + mesh = Mesh(v=self.v, f=self.f, albedo=None, device=self.device) + mesh.auto_normal() + mesh.auto_uv() + + self.vt = mesh.vt + self.ft = mesh.ft + + # get albedo map from NeRF + albedo = color_func_to_albedo(mesh, self.get_color, texture_resolution, device=self.device, glctx=self.glctx) + + # optimize texture + self.albedo = nn.Parameter(inverse_sigmoid(albedo)).to(self.device) + + optimizer = torch.optim.Adam([ + {'params': self.albedo, 'lr': 1e-3}, + ]) + + print(f"[INFO] fitting mesh texture...") + comfy_pbar = comfy.utils.ProgressBar(iters) + pbar = tqdm.trange(iters) + for i in pbar: + + # shrink to front view as we care more about it... + ver = np.random.randint(-5, 5) + hor = np.random.randint(-15, 15) + rad = self.opt.cam_radius # np.random.uniform(1, 2) + + pose = orbit_camera(ver, hor, rad) + + image_gt, alpha_gt = self.render_gs(pose) + image_pred, alpha_pred = self.render_mesh(pose) + + loss_mse = F.mse_loss(image_pred, image_gt) + loss = loss_mse + + loss.backward() + + optimizer.step() + optimizer.zero_grad() + + pbar.set_description(f"MSE = {loss_mse.item():.6f}") + comfy_pbar.update_absolute(i + 1) + + print(f"[INFO] finished fitting mesh texture!") + + + @torch.no_grad() + def get_mesh(self): + mesh = Mesh(v=self.v, f=self.f, vt=self.vt, ft=self.ft, albedo=torch.sigmoid(self.albedo), device=self.device) + mesh.auto_normal() + return mesh diff --git a/mesh_processer/mesh_utils.py b/mesh_processer/mesh_utils.py index 29256ff..963d924 100644 --- a/mesh_processer/mesh_utils.py +++ b/mesh_processer/mesh_utils.py @@ -494,14 +494,15 @@ def marching_cubes_density_to_mesh(get_density_func, grid_size=256, S=128, densi return vertices, triangles -def color_func_to_albedo(mesh, get_rgb_func, texture_resolution=1024, padding=2, batch_size=640000, device="cuda", force_cuda_rast=False): +def color_func_to_albedo(mesh, get_rgb_func, texture_resolution=1024, padding=2, batch_size=640000, device="cuda", force_cuda_rast=False, glctx=None): import nvdiffrast.torch as dr from kiui.op import uv_padding - if force_cuda_rast: - glctx = dr.RasterizeCudaContext() - else: - glctx = dr.RasterizeGLContext() + if glctx is None: + if force_cuda_rast: + glctx = dr.RasterizeCudaContext() + else: + glctx = dr.RasterizeGLContext() # render uv maps h = w = texture_resolution diff --git a/nodes.py b/nodes.py index 320e72b..1641f82 100644 --- a/nodes.py +++ b/nodes.py @@ -43,10 +43,6 @@ from .mesh_processer.mesh_utils import ( marching_cubes_density_to_mesh, color_func_to_albedo, ) -from .algorithms.nerf_marching_cubes_converter import GSConverterNeRFMarchingCubes -# TODO remove: NeuS, DMTet -from .algorithms.NeuS_runner import NeuSParams, NeuSRunner -from .algorithms.dmtet import DMTetMesh from FlexiCubes.flexicubes_trainer import FlexiCubesTrainer from DiffRastMesh.diff_mesh import DiffMesh, DiffMeshCameraController @@ -62,6 +58,7 @@ from TriplaneGaussian.utils.misc import todevice, get_device from LGM.core.options import config_defaults from LGM.mvdream.pipeline_mvdream import MVDreamPipeline from LGM.large_multiview_gaussian_model import LargeMultiviewGaussianModel +from LGM.nerf_marching_cubes_converter import GSConverterNeRFMarchingCubes from TripoSR.system import TSR from InstantMesh.utils.camera_util import oribt_camera_poses_to_input_cameras from CRM.model.crm.model import ConvolutionalReconstructionModel @@ -1288,66 +1285,6 @@ class Fitting_Mesh_With_Multiview_Images: cstr(f"[{self.__class__.__name__}] Number of reference images {ref_imgs_num} does not equal to number of masks {ref_masks_num}").error.print() return (trained_mesh, baked_texture, ) - -class Deep_Marching_Tetrahedrons: - - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "training_iterations": ("INT", {"default": 5000, "min": 1, "max": 100000}), - "points_cloud_fitting_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "step": 0.01}), - "mesh_smoothing_weight": ("FLOAT", {"default": 0.1, "min": 0.0, "step": 0.01}), - "chamfer_faces_sample_scale": ("FLOAT", {"default": 1.0, "min": 0.01, "step": 0.01}), - "mesh_scale": ("FLOAT", {"default": 1.0, "min": 0.01, "step": 0.01}), - "grid_resolution": ([128, 64, 32], ), - "geometry_learning_rate": ("FLOAT", {"default": 0.0001, "min": 0.00001, "step": 0.00001}), - "positional_encoding_multires": ("INT", {"default": 2, "min": 2}), - "mlp_internal_dims": ("INT", {"default": 128, "min": 8}), - "mlp_hidden_layer_num": ("INT", {"default": 5, "min": 1}), - }, - - "optional": { - "reference_points_cloud": ("POINTCLOUD",), - "reference_images": ("IMAGE",), - "reference_masks": ("MASK",), - } - } - - RETURN_TYPES = ( - "MESH", - ) - RETURN_NAMES = ( - "trained_mesh", - ) - FUNCTION = "run_dmtet" - CATEGORY = "Comfy3D/Algorithm" - - def run_dmtet(self, training_iterations, points_cloud_fitting_weight, mesh_smoothing_weight, chamfer_faces_sample_scale, mesh_scale, grid_resolution, geometry_learning_rate, - positional_encoding_multires, mlp_internal_dims, mlp_hidden_layer_num, - reference_points_cloud=None, reference_images=None, reference_masks=None): - - with torch.inference_mode(False): - dmtet_mesh = DMTetMesh( - training_iterations, - points_cloud_fitting_weight, - mesh_smoothing_weight, - chamfer_faces_sample_scale, - mesh_scale, - grid_resolution, - geometry_learning_rate, - positional_encoding_multires, - mlp_internal_dims, - mlp_hidden_layer_num - ) - - if reference_points_cloud is None and reference_images is None: - cstr(f"[{self.__class__.__name__}] reference_points_cloud and reference_images cannot both be None, you need at least provide one of them!").error.print() - raise Exception("User didn't provide necessary inputs, stop executing the code!") - - dmtet_mesh.training(reference_points_cloud, reference_images, reference_masks) - mesh = dmtet_mesh.get_mesh() - return (mesh, ) class Load_Triplane_Gaussian_Transformers: @@ -1801,6 +1738,17 @@ class Convert_3DGS_to_Mesh_with_NeRF_and_Marching_Cubes: "required": { "gs_ply": ("GS_PLY",), "gs_config": (['big', 'default', 'small', 'tiny'], ), + "training_nerf_iterations": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}), + "training_nerf_resolution": ("INT", {"default": 128, "min": 1, "max": 0xffffffffffffffff}), + "marching_cude_grids_resolution": ("INT", {"default": 256, "min": 1, "max": 0xffffffffffffffff}), + "marching_cude_grids_batch_size": ("INT", {"default": 128, "min": 1, "max": 0xffffffffffffffff}), + "marching_cude_threshold": ("FLOAT", {"default": 10.0, "min": 0.0, "step": 0.01}), + "training_mesh_iterations": ("INT", {"default": 2048, "min": 1, "max": 0xffffffffffffffff}), + "training_mesh_resolution": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}), + "remesh_after_n_iteration": ("INT", {"default": 512, "min": 128, "max": 100000}), + "training_albedo_iterations": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}), + "training_albedo_resolution": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}), + "texture_resolution": ("INT", {"default": 1024, "min": 128, "max": 8192}), "force_cuda_rast": ("BOOLEAN", {"default": False}), }, } @@ -1818,117 +1766,36 @@ class Convert_3DGS_to_Mesh_with_NeRF_and_Marching_Cubes: FUNCTION = "convert_gs_ply" CATEGORY = "Comfy3D/Algorithm" - def convert_gs_ply(self, gs_ply, gs_config, force_cuda_rast): + def convert_gs_ply( + self, + gs_ply, + gs_config, + training_nerf_iterations, + training_nerf_resolution, + marching_cude_grids_resolution, + marching_cude_grids_batch_size, + marching_cude_threshold, + training_mesh_iterations, + training_mesh_resolution, + remesh_after_n_iteration, + training_albedo_iterations, + training_albedo_resolution, + texture_resolution, + force_cuda_rast, + ): with torch.inference_mode(False): chosen_config = config_defaults[gs_config] chosen_config.force_cuda_rast = force_cuda_rast converter = GSConverterNeRFMarchingCubes(config_defaults[gs_config], gs_ply).cuda() - imgs, alphas = converter.fit_nerf() - converter.fit_mesh() - converter.fit_mesh_uv() + imgs, alphas = converter.fit_nerf(training_nerf_iterations, training_nerf_resolution) + converter.fit_mesh( + training_mesh_iterations, remesh_after_n_iteration, training_mesh_resolution, + marching_cude_grids_resolution, marching_cude_grids_batch_size, marching_cude_threshold + ) + converter.fit_mesh_uv(training_albedo_iterations, training_albedo_resolution, texture_resolution) return(converter.get_mesh(), imgs, alphas) -class NeuS: - NeuS_config_path = "NeuS.conf" - fix_cam_pose_dir = "NeuS/models/fixed_poses" - - @classmethod - def INPUT_TYPES(cls): - cls.config_path_abs = os.path.join(CONFIG_ROOT_PATH, cls.NeuS_config_path) - return { - "required": { - "reference_image": ("IMAGE",), - "reference_mask": ("MASK",), - "reference_normals": ("IMAGE",), - "training_iterations": ("INT", {"default": 1000, "min": 1, "max": 100000}), # longer time, better result. 1w will be ok for most cases - "batch_size": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}), - "learning_rate": ("FLOAT", {"default": 5e-4, "min": 0.00001, "step": 0.00001}), - "learning_rate_alpha": ("FLOAT", {"default": 5e-4, "min": 0.00001, "step": 0.00001}), - "color_loss_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "step": 0.01}), - "mesh_smoothing_weight": ("FLOAT", {"default": 0.1, "min": 0.0, "step": 0.01}), - "mask_loss_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "step": 0.01}), - "normal_loss_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "step": 0.01}), - "sparse_loss_weight": ("FLOAT", {"default": 0.1, "min": 0.0, "step": 0.01}), - "warm_up_end": ("INT", {"default": 500, "min": 0, "max": 0xffffffffffffffff}), - "anneal_end": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), - "use_white_background": ("BOOLEAN", {"default": True},), - "geometry_extract_resolution": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}), - "marching_cude_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "step": 0.01}), - }, - } - - RETURN_TYPES = ( - "MESH", - ) - RETURN_NAMES = ( - "mesh", - ) - FUNCTION = "run_NeuS" - CATEGORY = "Comfy3D/Algorithm" - - def run_NeuS( - self, - reference_image, # [num_views, H, W, 3] - reference_mask, # [num_views, H, W] - reference_normals, # [num_views, H, W, 3] - training_iterations, - batch_size, - learning_rate, - learning_rate_alpha, - color_loss_weight, - mesh_smoothing_weight, - mask_loss_weight, - normal_loss_weight, - sparse_loss_weight, - warm_up_end, - anneal_end, - use_white_background, - geometry_extract_resolution, - marching_cude_threshold - ): - mesh = None - num_views = reference_image.shape[0] - - if num_views in [6, 5, 4]: - abs_fix_cam_pose_dir = os.path.join(ROOT_PATH, self.fix_cam_pose_dir) - - with torch.inference_mode(False): - NeuS_params = NeuSParams( - training_iterations, - batch_size, - learning_rate, - learning_rate_alpha, - color_loss_weight, - mesh_smoothing_weight, - mask_loss_weight, - normal_loss_weight, - sparse_loss_weight, - warm_up_end, - anneal_end, - use_white_background - ) - - runner = NeuSRunner( - reference_image, - reference_mask, - reference_normals, - self.config_path_abs, - abs_fix_cam_pose_dir, - num_views, - NeuS_params - ) - - cstr(f"[{self.__class__.__name__}] Training NeuS...").msg.print() - runner.train() - - cstr(f"[{self.__class__.__name__}] Extracting Mesh...").msg.print() - mesh = runner.extract_mesh(resolution=geometry_extract_resolution, threshold=marching_cude_threshold) - else: - cstr(f"[{self.__class__.__name__}] Number of views must be one of the following: 6, 5, 4. But got {num_views}").error.print() - - return(mesh, ) - class Load_TripoSR_Model: checkpoints_dir = "TripoSR" default_ckpt_name = "model.ckpt"