nodes/MrForExample-ComfyUI-3D-Pack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove a few not too useful nodes

Browse Source 
Remove NeuS, DMTet
Refactor GSConverterNeRFMarchingCubes
This commit is contained in:
Mr. For Example
2024-07-03 15:32:40 -06:00
parent 2d356dc0cc
commit 44545fcc15
11 changed files with 2247 additions and 938 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
-7
View File
@@ -132,13 +132,6 @@ This is an extensive node suite that enables ComfyUI to process 3D inputs (Mesh
- Bake Multi-View images into UVTexture of given 3D mesh using [Nvdiffrast](https://github.com/NVlabs/nvdiffrast), supports:
- Export to .obj, .ply, .glb
- **NeuS**
- Fit a coarse mesh from sparse multi-view images & normal maps, as little as 4 to 6 views, pretty good at reconstruct the shape from reference images but texture lacking details.
- **Deep Marching Tetrahedrons**
- Allow convert 3DGS .ply file to 3D mesh <br>
*Note: I didn't spent time to turn the hyperprameters yet, the result will be improved in the future!*
- **Save & Load 3D file**
- .obj, .ply, .glb for 3D Mesh
- .ply for 3DGS
_Example_Workflows/MVDream_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json
+362 -322
View File
@@ -1,121 +1,7 @@
{
"last_node_id": 19,
"last_link_id": 27,
"last_node_id": 20,
"last_link_id": 29,
"nodes": [
{
"id": 9,
"type": "[Comfy3D] Save 3DGS",
"pos": [
1798,
459
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 13
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
11
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3DGS"
},
"widgets_values": [
"LGMTest/FatCat.ply"
]
},
{
"id": 17,
"type": "[Comfy3D] Large Multiview Gaussian Model",
"pos": [
1270,
461
],
"size": {
"0": 252,
"1": 46
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "multiview_images",
"type": "IMAGE",
"link": 26
},
{
"name": "lgm_model",
"type": "LGM_MODEL",
"link": 22
}
],
"outputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"links": [
19,
20
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Large Multiview Gaussian Model"
}
},
{
"id": 2,
"type": "[Comfy3D] Load Diffusers Pipeline",
"pos": [
380,
345
],
"size": {
"0": 375.3948669433594,
"1": 83.33004760742188
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "pipe",
"type": "DIFFUSERS_PIPE",
"links": [
23
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Load Diffusers Pipeline"
},
"widgets_values": [
"MVDreamPipeline",
"ashawkey/imagedream-ipmv-diffusers"
]
},
{
"id": 4,
"type": "LoadImage",
@@ -128,7 +14,7 @@
"1": 314
},
"flags": {},
"order": 1,
"order": 0,
"mode": 0,
"outputs": [
{
@@ -206,7 +92,7 @@
"1": 82
},
"flags": {},
"order": 2,
"order": 1,
"mode": 0,
"outputs": [
{
@@ -227,189 +113,6 @@
"big"
]
},
{
"id": 15,
"type": "[Comfy3D] Preview 3DMesh",
"pos": [
2326.1025658000017,
214.90587429004222
],
"size": [
600,
500
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "mesh_file_path",
"type": "STRING",
"link": 16,
"widget": {
"name": "mesh_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DMesh"
},
"widgets_values": [
"",
null
]
},
{
"id": 14,
"type": "[Comfy3D] Save 3D Mesh",
"pos": [
1996.1025658,
214.90587429004222
],
"size": {
"0": 307.8253173828125,
"1": 60.248291015625
},
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "mesh",
"type": "MESH",
"link": 17
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
16
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3D Mesh"
},
"widgets_values": [
"LGMTest/FatCat.obj"
]
},
{
"id": 12,
"type": "[Comfy3D] Switch 3DGS Axis",
"pos": [
1558,
460
],
"size": {
"0": 210,
"1": 106
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 19
}
],
"outputs": [
{
"name": "switched_gs_ply",
"type": "GS_PLY",
"links": [
13
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Switch 3DGS Axis"
},
"widgets_values": [
"+x",
"-y",
"-z"
]
},
{
"id": 10,
"type": "[Comfy3D] Preview 3DGS",
"pos": [
2331,
776
],
"size": [
600,
500
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "gs_file_path",
"type": "STRING",
"link": 11,
"widget": {
"name": "gs_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DGS"
},
"widgets_values": [
"",
null
]
},
{
"id": 16,
"type": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes",
"pos": [
1566,
215
],
"size": {
"0": 411.6000061035156,
"1": 65.46688079833984
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 20
}
],
"outputs": [
{
"name": "mesh",
"type": "MESH",
"links": [
17
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes"
},
"widgets_values": [
"big"
]
},
{
"id": 18,
"type": "[Comfy3D] MVDream Model",
@@ -451,6 +154,12 @@
],
"shape": 3,
"slot_index": 0
},
{
"name": "orbit_camposes",
"type": "ORBIT_CAMPOSES",
"links": null,
"shape": 3
}
],
"properties": {
@@ -473,10 +182,10 @@
1279,
120
],
"size": [
210,
246
],
"size": {
"0": 210,
"1": 246
},
"flags": {},
"order": 6,
"mode": 0,
@@ -490,6 +199,329 @@
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 12,
"type": "[Comfy3D] Switch 3DGS Axis",
"pos": [
1560,
759
],
"size": {
"0": 210,
"1": 106
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 19
}
],
"outputs": [
{
"name": "switched_gs_ply",
"type": "GS_PLY",
"links": [
13
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Switch 3DGS Axis"
},
"widgets_values": [
"+x",
"-y",
"-z"
]
},
{
"id": 14,
"type": "[Comfy3D] Save 3D Mesh",
"pos": [
1996.1025658,
214.90587429004222
],
"size": {
"0": 242.42886352539062,
"1": 58
},
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "mesh",
"type": "MESH",
"link": 29
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
16
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3D Mesh"
},
"widgets_values": [
"LGMTest/FatCat.obj"
]
},
{
"id": 15,
"type": "[Comfy3D] Preview 3DMesh",
"pos": [
2267,
215
],
"size": [
600,
500
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "mesh_file_path",
"type": "STRING",
"link": 16,
"widget": {
"name": "mesh_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DMesh"
},
"widgets_values": [
"",
null
]
},
{
"id": 10,
"type": "[Comfy3D] Preview 3DGS",
"pos": [
2266,
769
],
"size": [
600,
500
],
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "gs_file_path",
"type": "STRING",
"link": 11,
"widget": {
"name": "gs_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DGS"
},
"widgets_values": [
"",
null
]
},
{
"id": 9,
"type": "[Comfy3D] Save 3DGS",
"pos": [
1856,
759
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 13
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
11
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3DGS"
},
"widgets_values": [
"LGMTest/FatCat.ply"
]
},
{
"id": 2,
"type": "[Comfy3D] Load Diffusers Pipeline",
"pos": [
382,
284
],
"size": {
"0": 375.3948669433594,
"1": 130
},
"flags": {},
"order": 2,
"mode": 0,
"outputs": [
{
"name": "pipe",
"type": "DIFFUSERS_PIPE",
"links": [
23
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Load Diffusers Pipeline"
},
"widgets_values": [
"MVDreamPipeline",
"ashawkey/imagedream-ipmv-diffusers",
"",
true
]
},
{
"id": 17,
"type": "[Comfy3D] Large Multiview Gaussian Model",
"pos": [
1270,
461
],
"size": {
"0": 252,
"1": 46
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "multiview_images",
"type": "IMAGE",
"link": 26
},
{
"name": "lgm_model",
"type": "LGM_MODEL",
"link": 22
}
],
"outputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"links": [
19,
28
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Large Multiview Gaussian Model"
}
},
{
"id": 20,
"type": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes",
"pos": [
1568,
217
],
"size": [
411.6000061035156,
387.6719606496259
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 28
}
],
"outputs": [
{
"name": "mesh",
"type": "MESH",
"links": [
29
],
"shape": 3,
"slot_index": 0
},
{
"name": "imgs",
"type": "IMAGE",
"links": null,
"shape": 3
},
{
"name": "alphas",
"type": "MASK",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes"
},
"widgets_values": [
"big",
512,
128,
256,
128,
10,
2048,
512,
512,
512,
512,
1024,
false
]
}
],
"links": [
@@ -525,14 +557,6 @@
0,
"STRING"
],
[
17,
16,
0,
14,
0,
"MESH"
],
[
19,
17,
@@ -541,14 +565,6 @@
0,
"GS_PLY"
],
[
20,
17,
0,
16,
0,
"GS_PLY"
],
[
22,
1,
@@ -596,10 +612,34 @@
19,
0,
"IMAGE"
],
[
28,
17,
0,
20,
0,
"GS_PLY"
],
[
29,
20,
0,
14,
0,
"MESH"
]
],
"groups": [],
"config": {},
"extra": {},
"extra": {
"ds": {
"scale": 0.7513148009015777,
"offset": [
-509.38235438454456,
470.4740393503744
]
}
},
"version": 0.4
}
_Example_Workflows/Multi-View-Images_to_Instant-NGP_to_DiffRast_to_3DMesh.json
+959
View File
@@ -0,0 +1,959 @@
{
"last_node_id": 27,
"last_link_id": 46,
"nodes": [
{
"id": 6,
"type": "MaskToImage",
"pos": [
1490,
740
],
"size": {
"0": 210,
"1": 26
},
"flags": {},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "mask",
"type": "MASK",
"link": 6
}
],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
4
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "MaskToImage"
}
},
{
"id": 10,
"type": "[Comfy3D] Preview 3DMesh",
"pos": [
2400,
684
],
"size": [
600,
500
],
"flags": {},
"order": 15,
"mode": 0,
"inputs": [
{
"name": "mesh_file_path",
"type": "STRING",
"link": 14,
"widget": {
"name": "mesh_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DMesh"
},
"widgets_values": [
"",
null
]
},
{
"id": 9,
"type": "Reroute",
"pos": [
1237,
553
],
"size": [
75,
26
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "",
"type": "*",
"link": 11
}
],
"outputs": [
{
"name": "",
"type": "ORBIT_CAMPOSES",
"links": [
19,
34
],
"slot_index": 0
}
],
"properties": {
"showOutputText": false,
"horizontal": false
}
},
{
"id": 5,
"type": "[Comfy3D] Stack Orbit Camera Poses",
"pos": [
362,
389
],
"size": {
"0": 315,
"1": 586
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "orbit_camposes",
"type": "ORBIT_CAMPOSES",
"links": [
5,
9
],
"shape": 3,
"slot_index": 0
},
{
"name": "orbit_radius_list",
"type": "FLOAT",
"links": null,
"shape": 6
},
{
"name": "elevation_list",
"type": "FLOAT",
"links": null,
"shape": 6
},
{
"name": "azimuth_list",
"type": "FLOAT",
"links": null,
"shape": 6
},
{
"name": "orbit_center_X_list",
"type": "FLOAT",
"links": null,
"shape": 6
},
{
"name": "orbit_center_Y_list",
"type": "FLOAT",
"links": null,
"shape": 6
},
{
"name": "orbit_center_Z_list",
"type": "FLOAT",
"links": null,
"shape": 6
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Stack Orbit Camera Poses"
},
"widgets_values": [
1.75,
1.75,
0.1,
-45,
45,
45,
0,
-0.01,
6,
0,
0,
0.1,
0,
0,
0.1,
0,
0,
0.1
]
},
{
"id": 8,
"type": "Reroute",
"pos": [
808,
553
],
"size": [
75,
26
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"name": "",
"type": "*",
"link": 9
}
],
"outputs": [
{
"name": "",
"type": "ORBIT_CAMPOSES",
"links": [
11
],
"slot_index": 0
}
],
"properties": {
"showOutputText": false,
"horizontal": false
}
},
{
"id": 16,
"type": "Reroute",
"pos": [
1526,
189
],
"size": [
75,
26
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "",
"type": "*",
"link": 21
}
],
"outputs": [
{
"name": "",
"type": "IMAGE",
"links": [
37
],
"slot_index": 0
}
],
"properties": {
"showOutputText": false,
"horizontal": false
}
},
{
"id": 17,
"type": "Reroute",
"pos": [
1551,
226
],
"size": [
75,
26
],
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "",
"type": "*",
"link": 23
}
],
"outputs": [
{
"name": "",
"type": "MASK",
"links": [
38
],
"slot_index": 0
}
],
"properties": {
"showOutputText": false,
"horizontal": false
}
},
{
"id": 15,
"type": "Reroute",
"pos": [
2124,
552
],
"size": [
75,
26
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "",
"type": "*",
"link": 19
}
],
"outputs": [
{
"name": "",
"type": "ORBIT_CAMPOSES",
"links": [
40
],
"slot_index": 0
}
],
"properties": {
"showOutputText": false,
"horizontal": false
}
},
{
"id": 11,
"type": "[Comfy3D] Save 3D Mesh",
"pos": [
2035,
684
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "mesh",
"type": "MESH",
"link": 36
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
14
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3D Mesh"
},
"widgets_values": [
"Instant_NGP_Test/CatCoffee.obj"
]
},
{
"id": 1,
"type": "[Comfy3D] Load 3DGS",
"pos": [
104,
218
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 1,
"mode": 0,
"outputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"links": [
15
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Load 3DGS"
},
"widgets_values": [
"C:\\Users\\reall\\Softwares\\ComfyUI_windows_portable\\ComfyUI\\custom_nodes\\ComfyUI-3D-Pack\\_Example_Workflows\\_Example_Inputs_Files\\Meshes\\CatCoffee.ply"
]
},
{
"id": 12,
"type": "[Comfy3D] Switch 3DGS Axis",
"pos": [
466,
219
],
"size": {
"0": 210,
"1": 106
},
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 15
}
],
"outputs": [
{
"name": "switched_gs_ply",
"type": "GS_PLY",
"links": [
16
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Switch 3DGS Axis"
},
"widgets_values": [
"+x",
"-y",
"-z"
]
},
{
"id": 18,
"type": "[Comfy3D] Preview 3DMesh",
"pos": [
3303,
189
],
"size": [
600,
500
],
"flags": {},
"order": 17,
"mode": 0,
"inputs": [
{
"name": "mesh_file_path",
"type": "STRING",
"link": 25,
"widget": {
"name": "mesh_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DMesh"
},
"widgets_values": [
"",
null
]
},
{
"id": 19,
"type": "[Comfy3D] Save 3D Mesh",
"pos": [
2945,
188
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 16,
"mode": 0,
"inputs": [
{
"name": "mesh",
"type": "MESH",
"link": 41
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
25
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3D Mesh"
},
"widgets_values": [
"Instant_NGP_Test/CatCoffee_1.obj"
]
},
{
"id": 22,
"type": "[Comfy3D] Fitting Mesh With Multiview Images",
"pos": [
2358,
188
],
"size": {
"0": 531.5999755859375,
"1": 406
},
"flags": {},
"order": 14,
"mode": 0,
"inputs": [
{
"name": "reference_images",
"type": "IMAGE",
"link": 37
},
{
"name": "reference_masks",
"type": "MASK",
"link": 38
},
{
"name": "reference_orbit_camera_poses",
"type": "ORBIT_CAMPOSES",
"link": 40
},
{
"name": "mesh",
"type": "MESH",
"link": 39
}
],
"outputs": [
{
"name": "trained_mesh",
"type": "MESH",
"links": [
41
],
"shape": 3,
"slot_index": 0
},
{
"name": "baked_texture",
"type": "IMAGE",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Fitting Mesh With Multiview Images"
},
"widgets_values": [
49.1,
1024,
1024,
2048,
1,
0.1,
false,
0.0001,
0.5,
512,
0,
false
]
},
{
"id": 13,
"type": "PreviewImage",
"pos": [
1506,
-109
],
"size": {
"0": 215.39028930664062,
"1": 246
},
"flags": {
"collapsed": false
},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 17
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 7,
"type": "PreviewImage",
"pos": [
1742,
740
],
"size": {
"0": 215.39028930664062,
"1": 246
},
"flags": {
"collapsed": false
},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 4
}
],
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 21,
"type": "[Comfy3D] Instant NGP",
"pos": [
1570,
288
],
"size": {
"0": 418.1999816894531,
"1": 290
},
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "reference_image",
"type": "IMAGE",
"link": 32
},
{
"name": "reference_mask",
"type": "MASK",
"link": 33
},
{
"name": "reference_orbit_camera_poses",
"type": "ORBIT_CAMPOSES",
"link": 34
}
],
"outputs": [
{
"name": "mesh",
"type": "MESH",
"links": [
36,
39
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Instant NGP"
},
"widgets_values": [
49.1,
1024,
128,
256,
128,
10,
1024,
1,
false
]
},
{
"id": 2,
"type": "[Comfy3D] Gaussian Splatting Orbit Renderer",
"pos": [
807,
289
],
"size": {
"0": 556.800048828125,
"1": 198
},
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 16
},
{
"name": "render_orbit_camera_poses",
"type": "ORBIT_CAMPOSES",
"link": 5
}
],
"outputs": [
{
"name": "rendered_gs_images",
"type": "IMAGE",
"links": [
17,
21,
32
],
"shape": 3,
"slot_index": 0
},
{
"name": "rendered_gs_masks",
"type": "MASK",
"links": [
6,
23,
33
],
"shape": 3,
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Gaussian Splatting Orbit Renderer"
},
"widgets_values": [
1024,
1024,
49.1,
1,
1,
1
]
}
],
"links": [
[
4,
6,
0,
7,
0,
"IMAGE"
],
[
5,
5,
0,
2,
1,
"ORBIT_CAMPOSES"
],
[
6,
2,
1,
6,
0,
"MASK"
],
[
9,
5,
0,
8,
0,
"*"
],
[
11,
8,
0,
9,
0,
"*"
],
[
14,
11,
0,
10,
0,
"STRING"
],
[
15,
1,
0,
12,
0,
"GS_PLY"
],
[
16,
12,
0,
2,
0,
"GS_PLY"
],
[
17,
2,
0,
13,
0,
"IMAGE"
],
[
19,
9,
0,
15,
0,
"*"
],
[
21,
2,
0,
16,
0,
"*"
],
[
23,
2,
1,
17,
0,
"*"
],
[
25,
19,
0,
18,
0,
"STRING"
],
[
32,
2,
0,
21,
0,
"IMAGE"
],
[
33,
2,
1,
21,
1,
"MASK"
],
[
34,
9,
0,
21,
2,
"ORBIT_CAMPOSES"
],
[
36,
21,
0,
11,
0,
"MESH"
],
[
37,
16,
0,
22,
0,
"IMAGE"
],
[
38,
17,
0,
22,
1,
"MASK"
],
[
39,
21,
0,
22,
3,
"MESH"
],
[
40,
15,
0,
22,
2,
"ORBIT_CAMPOSES"
],
[
41,
22,
0,
19,
0,
"MESH"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.6934334949441353,
"offset": [
-1086.556448784114,
243.62758422318373
]
}
},
"version": 0.4
}
_Example_Workflows/Triplane_Gaussian_Transformers_to_3DGS_and_3DMesh.json
+140 -108
View File
@@ -69,40 +69,6 @@
49.1
]
},
{
"id": 12,
"type": "[Comfy3D] Preview 3DGS",
"pos": [
1590,
800
],
"size": [
600,
500
],
"flags": {
"collapsed": false
},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "gs_file_path",
"type": "STRING",
"link": 13,
"widget": {
"name": "gs_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DGS"
},
"widgets_values": [
"",
null
]
},
{
"id": 14,
"type": "[Comfy3D] Triplane Gaussian Transformers",
@@ -152,38 +118,6 @@
1.9000000000000001
]
},
{
"id": 26,
"type": "[Comfy3D] Preview 3DMesh",
"pos": [
2328,
796
],
"size": [
600,
500
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "mesh_file_path",
"type": "STRING",
"link": 32,
"widget": {
"name": "mesh_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DMesh"
},
"widgets_values": [
"",
null
]
},
{
"id": 11,
"type": "Note",
@@ -250,45 +184,6 @@
"-x"
]
},
{
"id": 10,
"type": "[Comfy3D] Save 3DGS",
"pos": [
1288,
800
],
"size": {
"0": 275.57391357421875,
"1": 58
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 12
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
13
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3DGS"
},
"widgets_values": [
"MeshTest\\3DGS_Picacho.ply"
]
},
{
"id": 34,
"type": "[Comfy3D] Switch 3DGS Axis",
@@ -339,7 +234,7 @@
],
"size": {
"0": 411.6000061035156,
"1": 58
"1": 386
},
"flags": {},
"order": 10,
@@ -360,13 +255,37 @@
],
"shape": 3,
"slot_index": 0
},
{
"name": "imgs",
"type": "IMAGE",
"links": null,
"shape": 3
},
{
"name": "alphas",
"type": "MASK",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes"
},
"widgets_values": [
"big"
"big",
512,
128,
256,
128,
10,
2048,
512,
512,
512,
512,
1024,
false
]
},
{
@@ -481,6 +400,111 @@
"a_pikachu_with_smily_face-removebg.png",
"image"
]
},
{
"id": 10,
"type": "[Comfy3D] Save 3DGS",
"pos": [
1226,
1084
],
"size": {
"0": 275.57391357421875,
"1": 58
},
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 12
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
13
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3DGS"
},
"widgets_values": [
"MeshTest\\3DGS_Picacho.ply"
]
},
{
"id": 12,
"type": "[Comfy3D] Preview 3DGS",
"pos": [
1529,
1083
],
"size": [
600,
500
],
"flags": {
"collapsed": false
},
"order": 9,
"mode": 0,
"inputs": [
{
"name": "gs_file_path",
"type": "STRING",
"link": 13,
"widget": {
"name": "gs_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DGS"
},
"widgets_values": [
"",
null
]
},
{
"id": 26,
"type": "[Comfy3D] Preview 3DMesh",
"pos": [
2327,
628
],
"size": [
600,
500
],
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "mesh_file_path",
"type": "STRING",
"link": 32,
"widget": {
"name": "mesh_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DMesh"
},
"widgets_values": [
"",
null
]
}
],
"links": [
@@ -575,6 +599,14 @@
],
"groups": [],
"config": {},
"extra": {},
"extra": {
"ds": {
"scale": 0.6209213230591553,
"offset": [
-672.2404394535945,
-402.3689017930591
]
}
},
"version": 0.4
}
_Example_Workflows/Wonder3D_to_Large_Multiview_Gaussian_Model_to_3DGS and_3DMesh.json
+365 -325
View File
@@ -1,194 +1,7 @@
{
"last_node_id": 23,
"last_link_id": 36,
"last_node_id": 24,
"last_link_id": 38,
"nodes": [
{
"id": 17,
"type": "[Comfy3D] Large Multiview Gaussian Model",
"pos": [
1270,
461
],
"size": {
"0": 252,
"1": 46
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "multiview_images",
"type": "IMAGE",
"link": 35
},
{
"name": "lgm_model",
"type": "LGM_MODEL",
"link": 22
}
],
"outputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"links": [
19,
20
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Large Multiview Gaussian Model"
}
},
{
"id": 15,
"type": "[Comfy3D] Preview 3DMesh",
"pos": [
2326.1025658000017,
214.90587429004222
],
"size": [
600,
500
],
"flags": {},
"order": 15,
"mode": 0,
"inputs": [
{
"name": "mesh_file_path",
"type": "STRING",
"link": 16,
"widget": {
"name": "mesh_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DMesh"
},
"widgets_values": [
"",
null
]
},
{
"id": 12,
"type": "[Comfy3D] Switch 3DGS Axis",
"pos": [
1558,
460
],
"size": {
"0": 210,
"1": 106
},
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 19
}
],
"outputs": [
{
"name": "switched_gs_ply",
"type": "GS_PLY",
"links": [
13
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Switch 3DGS Axis"
},
"widgets_values": [
"+x",
"-y",
"-z"
]
},
{
"id": 10,
"type": "[Comfy3D] Preview 3DGS",
"pos": [
2331,
776
],
"size": [
600,
500
],
"flags": {},
"order": 14,
"mode": 0,
"inputs": [
{
"name": "gs_file_path",
"type": "STRING",
"link": 11,
"widget": {
"name": "gs_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DGS"
},
"widgets_values": [
"",
null
]
},
{
"id": 16,
"type": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes",
"pos": [
1566,
215
],
"size": {
"0": 411.6000061035156,
"1": 82
},
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 20
}
],
"outputs": [
{
"name": "mesh",
"type": "MESH",
"links": [
17
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes"
},
"widgets_values": [
"big",
false
]
},
{
"id": 5,
"type": "InvertMask",
@@ -226,39 +39,6 @@
"Node name for S&R": "InvertMask"
}
},
{
"id": 20,
"type": "[Comfy3D] Load Diffusers Pipeline",
"pos": [
-300,
330
],
"size": {
"0": 392.1646728515625,
"1": 86.24864196777344
},
"flags": {},
"order": 0,
"mode": 0,
"outputs": [
{
"name": "pipe",
"type": "DIFFUSERS_PIPE",
"links": [
29
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Load Diffusers Pipeline"
},
"widgets_values": [
"Wonder3DMVDiffusionPipeline",
"flamehaze1115/wonder3d-v1.0"
]
},
{
"id": 22,
"type": "[Comfy3D] Wonder3D MVDiffusion Model",
@@ -326,7 +106,7 @@
461
],
"size": {
"0": 236.4545135498047,
"0": 243.60000610351562,
"1": 76
},
"flags": {},
@@ -370,7 +150,7 @@
"1": 82
},
"flags": {},
"order": 1,
"order": 0,
"mode": 0,
"outputs": [
{
@@ -400,10 +180,10 @@
],
"size": {
"0": 375.3948669433594,
"1": 83.33004760742188
"1": 130
},
"flags": {},
"order": 2,
"order": 1,
"mode": 4,
"outputs": [
{
@@ -421,7 +201,9 @@
},
"widgets_values": [
"MVDreamPipeline",
"ashawkey/imagedream-ipmv-diffusers"
"ashawkey/imagedream-ipmv-diffusers",
"",
true
]
},
{
@@ -462,6 +244,12 @@
"links": [],
"shape": 3,
"slot_index": 0
},
{
"name": "orbit_camposes",
"type": "ORBIT_CAMPOSES",
"links": null,
"shape": 3
}
],
"properties": {
@@ -477,84 +265,6 @@
0
]
},
{
"id": 14,
"type": "[Comfy3D] Save 3D Mesh",
"pos": [
1996.1025658,
214.90587429004222
],
"size": {
"0": 307.8253173828125,
"1": 60.248291015625
},
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "mesh",
"type": "MESH",
"link": 17
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
16
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3D Mesh"
},
"widgets_values": [
"LGMTest/KungFuCat.obj"
]
},
{
"id": 9,
"type": "[Comfy3D] Save 3DGS",
"pos": [
1798,
459
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 13
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
11
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3DGS"
},
"widgets_values": [
"LGMTest/KungFuCat.ply"
]
},
{
"id": 4,
"type": "LoadImage",
@@ -567,7 +277,7 @@
"1": 314
},
"flags": {},
"order": 3,
"order": 2,
"mode": 0,
"outputs": [
{
@@ -622,6 +332,329 @@
"properties": {
"Node name for S&R": "PreviewImage"
}
},
{
"id": 14,
"type": "[Comfy3D] Save 3D Mesh",
"pos": [
1996.1025658,
214.90587429004222
],
"size": {
"0": 259.5017395019531,
"1": 58
},
"flags": {},
"order": 13,
"mode": 0,
"inputs": [
{
"name": "mesh",
"type": "MESH",
"link": 37
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
16
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3D Mesh"
},
"widgets_values": [
"LGMTest/KungFuCat.obj"
]
},
{
"id": 15,
"type": "[Comfy3D] Preview 3DMesh",
"pos": [
2289,
215
],
"size": [
600,
500
],
"flags": {},
"order": 15,
"mode": 0,
"inputs": [
{
"name": "mesh_file_path",
"type": "STRING",
"link": 16,
"widget": {
"name": "mesh_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DMesh"
},
"widgets_values": [
"",
null
]
},
{
"id": 10,
"type": "[Comfy3D] Preview 3DGS",
"pos": [
2203,
776
],
"size": [
600,
500
],
"flags": {},
"order": 14,
"mode": 0,
"inputs": [
{
"name": "gs_file_path",
"type": "STRING",
"link": 11,
"widget": {
"name": "gs_file_path"
}
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Preview 3DGS"
},
"widgets_values": [
"",
null
]
},
{
"id": 12,
"type": "[Comfy3D] Switch 3DGS Axis",
"pos": [
1572,
776
],
"size": {
"0": 210,
"1": 106
},
"flags": {},
"order": 10,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 19
}
],
"outputs": [
{
"name": "switched_gs_ply",
"type": "GS_PLY",
"links": [
13
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Switch 3DGS Axis"
},
"widgets_values": [
"+x",
"-y",
"-z"
]
},
{
"id": 9,
"type": "[Comfy3D] Save 3DGS",
"pos": [
1813,
777
],
"size": {
"0": 315,
"1": 58
},
"flags": {},
"order": 12,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 13
}
],
"outputs": [
{
"name": "save_path",
"type": "STRING",
"links": [
11
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Save 3DGS"
},
"widgets_values": [
"LGMTest/KungFuCat.ply"
]
},
{
"id": 17,
"type": "[Comfy3D] Large Multiview Gaussian Model",
"pos": [
1270,
461
],
"size": {
"0": 252,
"1": 46
},
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "multiview_images",
"type": "IMAGE",
"link": 35
},
{
"name": "lgm_model",
"type": "LGM_MODEL",
"link": 22
}
],
"outputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"links": [
19,
38
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Large Multiview Gaussian Model"
}
},
{
"id": 24,
"type": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes",
"pos": [
1566,
216
],
"size": {
"0": 411.6000061035156,
"1": 386
},
"flags": {},
"order": 11,
"mode": 0,
"inputs": [
{
"name": "gs_ply",
"type": "GS_PLY",
"link": 38
}
],
"outputs": [
{
"name": "mesh",
"type": "MESH",
"links": [
37
],
"shape": 3,
"slot_index": 0
},
{
"name": "imgs",
"type": "IMAGE",
"links": null,
"shape": 3
},
{
"name": "alphas",
"type": "MASK",
"links": null,
"shape": 3
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Convert 3DGS to Mesh with NeRF and Marching Cubes"
},
"widgets_values": [
"big",
1024,
128,
256,
128,
10,
2048,
512,
512,
512,
512,
1024,
false
]
},
{
"id": 20,
"type": "[Comfy3D] Load Diffusers Pipeline",
"pos": [
-264,
278
],
"size": {
"0": 392.1646728515625,
"1": 130
},
"flags": {},
"order": 3,
"mode": 0,
"outputs": [
{
"name": "pipe",
"type": "DIFFUSERS_PIPE",
"links": [
29
],
"shape": 3,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "[Comfy3D] Load Diffusers Pipeline"
},
"widgets_values": [
"Wonder3DMVDiffusionPipeline",
"flamehaze1115/wonder3d-v1.0",
"",
true
]
}
],
"links": [
@@ -657,14 +690,6 @@
0,
"STRING"
],
[
17,
16,
0,
14,
0,
"MESH"
],
[
19,
17,
@@ -673,14 +698,6 @@
0,
"GS_PLY"
],
[
20,
17,
0,
16,
0,
"GS_PLY"
],
[
22,
1,
@@ -760,6 +777,22 @@
19,
0,
"IMAGE"
],
[
37,
24,
0,
14,
0,
"MESH"
],
[
38,
17,
0,
24,
0,
"GS_PLY"
]
],
"groups": [
@@ -772,11 +805,18 @@
436
],
"color": "#3f789e",
"font_size": 24,
"locked": false
"font_size": 24
}
],
"config": {},
"extra": {},
"extra": {
"ds": {
"scale": 0.6209213230591556,
"offset": [
-137.46041222315475,
303.4617473928127
]
}
},
"version": 0.4
}
gen_3d_modules/CRM/model/crm/model.py
+1 -1
View File
@@ -41,7 +41,7 @@ class ConvolutionalReconstructionModel(nn.Module):
self.dec.c_dim = specs["DecoderSpecs"]["c_dim"]
self.dec.plane_resolution = specs["DecoderSpecs"]["plane_resolution"]
self.geo_type = specs["Train"].get("geo_type", "flex") # "dmtet" or "flex"
self.geo_type = specs["Train"].get("geo_type", "flex") # only support "flex"
self.unet2 = UNetPP(in_channels=self.dec.c_dim)
gen_3d_modules/CRM/util/flexicubes_geometry.py
-1
View File
@@ -8,7 +8,6 @@
import torch
from FlexiCubes.flexicubes import FlexiCubes
# from dmtet import sdf_reg_loss_batch
import torch.nn.functional as F
def get_center_boundary_index(grid_res, device):
gen_3d_modules/InstantMesh/models/geometry/rep_3d/flexicubes_geometry.py
-1
View File
@@ -11,7 +11,6 @@ import numpy as np
import os
from . import Geometry
from FlexiCubes.flexicubes import FlexiCubes
from .dmtet import sdf_reg_loss_batch
import torch.nn.functional as F
def get_center_boundary_index(grid_res, device):
gen_3d_modules/LGM/nerf_marching_cubes_converter.py
+379
View File
@@ -0,0 +1,379 @@
import tqdm
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import comfy.utils
import nerfacc
import nvdiffrast.torch as dr
from kiui.mesh_utils import clean_mesh, decimate_mesh
from kiui.mesh_utils import laplacian_smooth_loss, normal_consistency
from kiui.op import inverse_sigmoid
from kiui.cam import orbit_camera, get_perspective
from kiui.nn import MLP, trunc_exp
from mesh_processer.mesh import Mesh
from mesh_processer.mesh_utils import marching_cubes_density_to_mesh, color_func_to_albedo
from LGM.core.options import Options
from LGM.core.gs import GaussianRenderer
from LGM.core.utils import get_rays
# Triple renderer of gaussians, gaussian, and diso mesh.
# gaussian --> nerf --> mesh
class GSConverterNeRFMarchingCubes(nn.Module):
def __init__(self, opt: Options, gs_ply):
super().__init__()
from kiui.gridencoder import GridEncoder
self.opt = opt
self.device = torch.device("cuda")
# gs renderer
self.tan_half_fov = np.tan(0.5 * np.deg2rad(opt.fovy))
self.proj_matrix = torch.zeros(4, 4, dtype=torch.float32, device=self.device)
self.proj_matrix[0, 0] = 1 / self.tan_half_fov
self.proj_matrix[1, 1] = 1 / self.tan_half_fov
self.proj_matrix[2, 2] = (opt.zfar + opt.znear) / (opt.zfar - opt.znear)
self.proj_matrix[3, 2] = - (opt.zfar * opt.znear) / (opt.zfar - opt.znear)
self.proj_matrix[2, 3] = 1
self.gs_renderer = GaussianRenderer(opt)
self.gaussians = self.gs_renderer.create_from_ply(gs_ply).to(self.device)
# nerf renderer
if not self.opt.force_cuda_rast:
self.glctx = dr.RasterizeGLContext()
else:
self.glctx = dr.RasterizeCudaContext()
self.step = 0
self.render_step_size = 5e-3
self.aabb = torch.tensor([-1.0, -1.0, -1.0, 1.0, 1.0, 1.0], device=self.device)
self.estimator = nerfacc.OccGridEstimator(roi_aabb=self.aabb, resolution=64, levels=1)
self.encoder_density = GridEncoder(num_levels=12) # VMEncoder(output_dim=16, mode='sum')
self.encoder = GridEncoder(num_levels=12)
self.mlp_density = MLP(self.encoder_density.output_dim, 1, 32, 2, bias=False)
self.mlp = MLP(self.encoder.output_dim, 3, 32, 2, bias=False)
# mesh renderer
self.proj = torch.from_numpy(get_perspective(self.opt.fovy)).float().to(self.device)
self.v = self.f = None
self.vt = self.ft = None
self.deform = None
self.albedo = None
@torch.no_grad()
def render_gs(self, pose):
cam_poses = torch.from_numpy(pose).unsqueeze(0).to(self.device)
cam_poses[:, :3, 1:3] *= -1 # invert up & forward direction
# cameras needed by gaussian rasterizer
cam_view = torch.inverse(cam_poses).transpose(1, 2) # [V, 4, 4]
cam_view_proj = cam_view @ self.proj_matrix # [V, 4, 4]
cam_pos = - cam_poses[:, :3, 3] # [V, 3]
out = self.gs_renderer.render(self.gaussians.unsqueeze(0), cam_view.unsqueeze(0), cam_view_proj.unsqueeze(0), cam_pos.unsqueeze(0))
image = out['image'].squeeze(1).squeeze(0) # [C, H, W]
alpha = out['alpha'].squeeze(2).squeeze(1).squeeze(0) # [H, W]
return image, alpha
def get_density(self, xs):
# xs: [..., 3]
xs = xs.to(self.device)
prefix = xs.shape[:-1]
xs = xs.view(-1, 3)
feats = self.encoder_density(xs)
density = trunc_exp(self.mlp_density(feats))
density = density.view(*prefix, 1)
return density
def render_nerf(self, pose):
pose = torch.from_numpy(pose.astype(np.float32)).to(self.device)
# get rays
resolution = self.opt.output_size
rays_o, rays_d = get_rays(pose, resolution, resolution, self.opt.fovy)
hw = rays_o.shape[0] * rays_o.shape[1]
rays_o = rays_o.view(hw, 3)
rays_d = rays_d.view(hw, 3)
# update occ grid
if self.training:
def occ_eval_fn(xs):
sigmas = self.get_density(xs)
return self.render_step_size * sigmas
self.estimator.update_every_n_steps(self.step, occ_eval_fn=occ_eval_fn, occ_thre=0.01, n=8)
self.step += 1
# render
def sigma_fn(t_starts, t_ends, ray_indices):
t_origins = rays_o[ray_indices]
t_dirs = rays_d[ray_indices]
xs = t_origins + t_dirs * (t_starts + t_ends)[:, None] / 2.0
sigmas = self.get_density(xs)
return sigmas.squeeze(-1)
with torch.no_grad():
ray_indices, t_starts, t_ends = self.estimator.sampling(
rays_o,
rays_d,
sigma_fn=sigma_fn,
near_plane=0.01,
far_plane=100,
render_step_size=self.render_step_size,
stratified=self.training,
cone_angle=0,
)
t_origins = rays_o[ray_indices]
t_dirs = rays_d[ray_indices]
xs = t_origins + t_dirs * (t_starts + t_ends)[:, None] / 2.0
sigmas = self.get_density(xs).squeeze(-1)
rgbs = torch.sigmoid(self.mlp(self.encoder(xs)))
n_rays=rays_o.shape[0]
weights, trans, alphas = nerfacc.render_weight_from_density(t_starts, t_ends, sigmas, ray_indices=ray_indices, n_rays=n_rays)
color = nerfacc.accumulate_along_rays(weights, values=rgbs, ray_indices=ray_indices, n_rays=n_rays)
alpha = nerfacc.accumulate_along_rays(weights, values=None, ray_indices=ray_indices, n_rays=n_rays)
color = color + 1 * (1.0 - alpha)
color = color.view(resolution, resolution, 3).clamp(0, 1).permute(2, 0, 1).contiguous()
alpha = alpha.view(resolution, resolution).clamp(0, 1).contiguous()
return color, alpha
def fit_nerf(self, iters=512, resolution=128):
self.opt.output_size = resolution
optimizer = torch.optim.Adam([
{'params': self.encoder_density.parameters(), 'lr': 1e-2},
{'params': self.encoder.parameters(), 'lr': 1e-2},
{'params': self.mlp_density.parameters(), 'lr': 1e-3},
{'params': self.mlp.parameters(), 'lr': 1e-3},
])
print(f"[INFO] fitting nerf...")
imgs, alphas = [], []
comfy_pbar = comfy.utils.ProgressBar(iters)
pbar = tqdm.trange(iters)
for i in pbar:
ver = np.random.randint(-45, 45)
hor = np.random.randint(-180, 180)
rad = np.random.uniform(1.5, 3.0)
pose = orbit_camera(ver, hor, rad)
image_gt, alpha_gt = self.render_gs(pose)
imgs.append(image_gt.permute(1, 2, 0).unsqueeze(0))
alphas.append(alpha_gt.unsqueeze(0))
image_pred, alpha_pred = self.render_nerf(pose)
# if i % 200 == 0:
# kiui.vis.plot_image(image_gt, alpha_gt, image_pred, alpha_pred)
loss_mse = F.mse_loss(image_pred, image_gt) + 0.1 * F.mse_loss(alpha_pred, alpha_gt)
loss = loss_mse #+ 0.1 * self.encoder_density.tv_loss() #+ 0.0001 * self.encoder_density.density_loss()
loss.backward()
self.encoder_density.grad_total_variation(1e-8)
optimizer.step()
optimizer.zero_grad()
pbar.set_description(f"MSE = {loss_mse.item():.6f}")
comfy_pbar.update_absolute(i + 1)
print(f"[INFO] finished fitting nerf!")
return torch.cat(imgs, dim=0), torch.cat(alphas, dim=0)
def render_mesh(self, pose):
h = w = self.opt.output_size
v = self.v + self.deform
f = self.f
pose = torch.from_numpy(pose.astype(np.float32)).to(v.device)
# get v_clip and render rgb
v_cam = torch.matmul(F.pad(v, pad=(0, 1), mode='constant', value=1.0), torch.inverse(pose).T).float().unsqueeze(0)
v_clip = v_cam @ self.proj.T
rast, rast_db = dr.rasterize(self.glctx, v_clip, f, (h, w))
alpha = torch.clamp(rast[..., -1:], 0, 1).contiguous() # [1, H, W, 1]
alpha = dr.antialias(alpha, rast, v_clip, f).clamp(0, 1).squeeze(-1).squeeze(0) # [H, W] important to enable gradients!
if self.albedo is None:
xyzs, _ = dr.interpolate(v.unsqueeze(0), rast, f) # [1, H, W, 3]
xyzs = xyzs.view(-1, 3)
mask = (alpha > 0).view(-1)
image = torch.zeros_like(xyzs, dtype=torch.float32)
if mask.any():
masked_albedo = torch.sigmoid(self.mlp(self.encoder(xyzs[mask].detach(), bound=1)))
image[mask] = masked_albedo.float()
else:
texc, texc_db = dr.interpolate(self.vt.unsqueeze(0), rast, self.ft, rast_db=rast_db, diff_attrs='all')
image = torch.sigmoid(dr.texture(self.albedo.unsqueeze(0), texc, uv_da=texc_db)) # [1, H, W, 3]
image = image.view(1, h, w, 3)
# image = dr.antialias(image, rast, v_clip, f).clamp(0, 1)
image = image.squeeze(0).permute(2, 0, 1).contiguous() # [3, H, W]
image = alpha * image + (1 - alpha)
return image, alpha
def fit_mesh(self, iters=2048, remesh_after_n_iteration=512, resolution=512, grid_size=256, S=128, density_thresh=10, decimate_target=5e4):
self.opt.output_size = resolution
vertices, triangles = marching_cubes_density_to_mesh(self.get_density, grid_size, S, density_thresh, decimate_target)
self.v = torch.from_numpy(vertices).contiguous().float().to(self.device)
self.f = torch.from_numpy(triangles).contiguous().int().to(self.device)
self.deform = nn.Parameter(torch.zeros_like(self.v)).to(self.device)
# Here coarse mesh already resemble the target shape closely
#mcube_mesh = Mesh(v=self.v, f=self.f, albedo=None, device=self.device)
#mcube_mesh.auto_normal()
#mcube_mesh.auto_uv()
# fit mesh from gs
lr_factor = 1
optimizer = torch.optim.Adam([
{'params': self.encoder.parameters(), 'lr': 1e-3 * lr_factor},
{'params': self.mlp.parameters(), 'lr': 1e-3 * lr_factor},
{'params': self.deform, 'lr': 1e-4},
])
print(f"[INFO] fitting mesh...")
comfy_pbar = comfy.utils.ProgressBar(iters)
pbar = tqdm.trange(iters)
for i in pbar:
ver = np.random.randint(-10, 10)
hor = np.random.randint(-180, 180)
rad = self.opt.cam_radius # np.random.uniform(1, 2)
pose = orbit_camera(ver, hor, rad)
image_gt, alpha_gt = self.render_gs(pose)
image_pred, alpha_pred = self.render_mesh(pose)
loss_mse = F.mse_loss(image_pred, image_gt) + 0.1 * F.mse_loss(alpha_pred, alpha_gt)
loss_lap = laplacian_smooth_loss(self.v + self.deform, self.f)
loss_normal = normal_consistency(self.v + self.deform, self.f)
loss_offsets = (self.deform ** 2).sum(-1).mean()
loss = loss_mse + 0.001 * loss_normal + 0.1 * loss_offsets * loss_lap * 0.01
loss.backward()
optimizer.step()
optimizer.zero_grad()
# remesh periodically
if i > 0 and i % remesh_after_n_iteration == 0:
vertices = (self.v + self.deform).detach().cpu().numpy()
triangles = self.f.detach().cpu().numpy()
vertices, triangles = clean_mesh(vertices, triangles, remesh=True, remesh_size=0.01)
if triangles.shape[0] > decimate_target:
vertices, triangles = decimate_mesh(vertices, triangles, decimate_target, optimalplacement=False)
self.v = torch.from_numpy(vertices).contiguous().float().to(self.device)
self.f = torch.from_numpy(triangles).contiguous().int().to(self.device)
self.deform = nn.Parameter(torch.zeros_like(self.v)).to(self.device)
lr_factor *= 0.5
optimizer = torch.optim.Adam([
{'params': self.encoder.parameters(), 'lr': 1e-3 * lr_factor},
{'params': self.mlp.parameters(), 'lr': 1e-3 * lr_factor},
{'params': self.deform, 'lr': 1e-4},
])
pbar.set_description(f"MSE = {loss_mse.item():.6f}")
comfy_pbar.update_absolute(i + 1)
# last clean
vertices = (self.v + self.deform).detach().cpu().numpy()
triangles = self.f.detach().cpu().numpy()
vertices, triangles = clean_mesh(vertices, triangles, remesh=False)
self.v = torch.from_numpy(vertices).contiguous().float().to(self.device)
self.f = torch.from_numpy(triangles).contiguous().int().to(self.device)
self.deform = nn.Parameter(torch.zeros_like(self.v).to(self.device))
print(f"[INFO] finished fitting mesh!")
def get_color(self, xs):
return torch.sigmoid(self.mlp(self.encoder(xs.to(self.device))))
# uv mesh refine
def fit_mesh_uv(self, iters=512, resolution=512, texture_resolution=1024, padding=2):
self.opt.output_size = resolution
# unwrap uv
print(f"[INFO] uv unwrapping...")
mesh = Mesh(v=self.v, f=self.f, albedo=None, device=self.device)
mesh.auto_normal()
mesh.auto_uv()
self.vt = mesh.vt
self.ft = mesh.ft
# get albedo map from NeRF
albedo = color_func_to_albedo(mesh, self.get_color, texture_resolution, device=self.device, glctx=self.glctx)
# optimize texture
self.albedo = nn.Parameter(inverse_sigmoid(albedo)).to(self.device)
optimizer = torch.optim.Adam([
{'params': self.albedo, 'lr': 1e-3},
])
print(f"[INFO] fitting mesh texture...")
comfy_pbar = comfy.utils.ProgressBar(iters)
pbar = tqdm.trange(iters)
for i in pbar:
# shrink to front view as we care more about it...
ver = np.random.randint(-5, 5)
hor = np.random.randint(-15, 15)
rad = self.opt.cam_radius # np.random.uniform(1, 2)
pose = orbit_camera(ver, hor, rad)
image_gt, alpha_gt = self.render_gs(pose)
image_pred, alpha_pred = self.render_mesh(pose)
loss_mse = F.mse_loss(image_pred, image_gt)
loss = loss_mse
loss.backward()
optimizer.step()
optimizer.zero_grad()
pbar.set_description(f"MSE = {loss_mse.item():.6f}")
comfy_pbar.update_absolute(i + 1)
print(f"[INFO] finished fitting mesh texture!")
@torch.no_grad()
def get_mesh(self):
mesh = Mesh(v=self.v, f=self.f, vt=self.vt, ft=self.ft, albedo=torch.sigmoid(self.albedo), device=self.device)
mesh.auto_normal()
return mesh
mesh_processer/mesh_utils.py
+6 -5
View File
@@ -494,14 +494,15 @@ def marching_cubes_density_to_mesh(get_density_func, grid_size=256, S=128, densi
return vertices, triangles
def color_func_to_albedo(mesh, get_rgb_func, texture_resolution=1024, padding=2, batch_size=640000, device="cuda", force_cuda_rast=False):
def color_func_to_albedo(mesh, get_rgb_func, texture_resolution=1024, padding=2, batch_size=640000, device="cuda", force_cuda_rast=False, glctx=None):
import nvdiffrast.torch as dr
from kiui.op import uv_padding
if force_cuda_rast:
glctx = dr.RasterizeCudaContext()
else:
glctx = dr.RasterizeGLContext()
if glctx is None:
if force_cuda_rast:
glctx = dr.RasterizeCudaContext()
else:
glctx = dr.RasterizeGLContext()
# render uv maps
h = w = texture_resolution
nodes.py
+35 -168
View File
@@ -43,10 +43,6 @@ from .mesh_processer.mesh_utils import (
marching_cubes_density_to_mesh,
color_func_to_albedo,
)
from .algorithms.nerf_marching_cubes_converter import GSConverterNeRFMarchingCubes
# TODO remove: NeuS, DMTet
from .algorithms.NeuS_runner import NeuSParams, NeuSRunner
from .algorithms.dmtet import DMTetMesh
from FlexiCubes.flexicubes_trainer import FlexiCubesTrainer
from DiffRastMesh.diff_mesh import DiffMesh, DiffMeshCameraController
@@ -62,6 +58,7 @@ from TriplaneGaussian.utils.misc import todevice, get_device
from LGM.core.options import config_defaults
from LGM.mvdream.pipeline_mvdream import MVDreamPipeline
from LGM.large_multiview_gaussian_model import LargeMultiviewGaussianModel
from LGM.nerf_marching_cubes_converter import GSConverterNeRFMarchingCubes
from TripoSR.system import TSR
from InstantMesh.utils.camera_util import oribt_camera_poses_to_input_cameras
from CRM.model.crm.model import ConvolutionalReconstructionModel
@@ -1288,66 +1285,6 @@ class Fitting_Mesh_With_Multiview_Images:
cstr(f"[{self.__class__.__name__}] Number of reference images {ref_imgs_num} does not equal to number of masks {ref_masks_num}").error.print()
return (trained_mesh, baked_texture, )
class Deep_Marching_Tetrahedrons:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"training_iterations": ("INT", {"default": 5000, "min": 1, "max": 100000}),
"points_cloud_fitting_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "step": 0.01}),
"mesh_smoothing_weight": ("FLOAT", {"default": 0.1, "min": 0.0, "step": 0.01}),
"chamfer_faces_sample_scale": ("FLOAT", {"default": 1.0, "min": 0.01, "step": 0.01}),
"mesh_scale": ("FLOAT", {"default": 1.0, "min": 0.01, "step": 0.01}),
"grid_resolution": ([128, 64, 32], ),
"geometry_learning_rate": ("FLOAT", {"default": 0.0001, "min": 0.00001, "step": 0.00001}),
"positional_encoding_multires": ("INT", {"default": 2, "min": 2}),
"mlp_internal_dims": ("INT", {"default": 128, "min": 8}),
"mlp_hidden_layer_num": ("INT", {"default": 5, "min": 1}),
},
"optional": {
"reference_points_cloud": ("POINTCLOUD",),
"reference_images": ("IMAGE",),
"reference_masks": ("MASK",),
}
}
RETURN_TYPES = (
"MESH",
)
RETURN_NAMES = (
"trained_mesh",
)
FUNCTION = "run_dmtet"
CATEGORY = "Comfy3D/Algorithm"
def run_dmtet(self, training_iterations, points_cloud_fitting_weight, mesh_smoothing_weight, chamfer_faces_sample_scale, mesh_scale, grid_resolution, geometry_learning_rate,
positional_encoding_multires, mlp_internal_dims, mlp_hidden_layer_num,
reference_points_cloud=None, reference_images=None, reference_masks=None):
with torch.inference_mode(False):
dmtet_mesh = DMTetMesh(
training_iterations,
points_cloud_fitting_weight,
mesh_smoothing_weight,
chamfer_faces_sample_scale,
mesh_scale,
grid_resolution,
geometry_learning_rate,
positional_encoding_multires,
mlp_internal_dims,
mlp_hidden_layer_num
)
if reference_points_cloud is None and reference_images is None:
cstr(f"[{self.__class__.__name__}] reference_points_cloud and reference_images cannot both be None, you need at least provide one of them!").error.print()
raise Exception("User didn't provide necessary inputs, stop executing the code!")
dmtet_mesh.training(reference_points_cloud, reference_images, reference_masks)
mesh = dmtet_mesh.get_mesh()
return (mesh, )
class Load_Triplane_Gaussian_Transformers:
@@ -1801,6 +1738,17 @@ class Convert_3DGS_to_Mesh_with_NeRF_and_Marching_Cubes:
"required": {
"gs_ply": ("GS_PLY",),
"gs_config": (['big', 'default', 'small', 'tiny'], ),
"training_nerf_iterations": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}),
"training_nerf_resolution": ("INT", {"default": 128, "min": 1, "max": 0xffffffffffffffff}),
"marching_cude_grids_resolution": ("INT", {"default": 256, "min": 1, "max": 0xffffffffffffffff}),
"marching_cude_grids_batch_size": ("INT", {"default": 128, "min": 1, "max": 0xffffffffffffffff}),
"marching_cude_threshold": ("FLOAT", {"default": 10.0, "min": 0.0, "step": 0.01}),
"training_mesh_iterations": ("INT", {"default": 2048, "min": 1, "max": 0xffffffffffffffff}),
"training_mesh_resolution": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}),
"remesh_after_n_iteration": ("INT", {"default": 512, "min": 128, "max": 100000}),
"training_albedo_iterations": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}),
"training_albedo_resolution": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}),
"texture_resolution": ("INT", {"default": 1024, "min": 128, "max": 8192}),
"force_cuda_rast": ("BOOLEAN", {"default": False}),
},
}
@@ -1818,117 +1766,36 @@ class Convert_3DGS_to_Mesh_with_NeRF_and_Marching_Cubes:
FUNCTION = "convert_gs_ply"
CATEGORY = "Comfy3D/Algorithm"
def convert_gs_ply(self, gs_ply, gs_config, force_cuda_rast):
def convert_gs_ply(
self,
gs_ply,
gs_config,
training_nerf_iterations,
training_nerf_resolution,
marching_cude_grids_resolution,
marching_cude_grids_batch_size,
marching_cude_threshold,
training_mesh_iterations,
training_mesh_resolution,
remesh_after_n_iteration,
training_albedo_iterations,
training_albedo_resolution,
texture_resolution,
force_cuda_rast,
):
with torch.inference_mode(False):
chosen_config = config_defaults[gs_config]
chosen_config.force_cuda_rast = force_cuda_rast
converter = GSConverterNeRFMarchingCubes(config_defaults[gs_config], gs_ply).cuda()
imgs, alphas = converter.fit_nerf()
converter.fit_mesh()
converter.fit_mesh_uv()
imgs, alphas = converter.fit_nerf(training_nerf_iterations, training_nerf_resolution)
converter.fit_mesh(
training_mesh_iterations, remesh_after_n_iteration, training_mesh_resolution,
marching_cude_grids_resolution, marching_cude_grids_batch_size, marching_cude_threshold
)
converter.fit_mesh_uv(training_albedo_iterations, training_albedo_resolution, texture_resolution)
return(converter.get_mesh(), imgs, alphas)
class NeuS:
NeuS_config_path = "NeuS.conf"
fix_cam_pose_dir = "NeuS/models/fixed_poses"
@classmethod
def INPUT_TYPES(cls):
cls.config_path_abs = os.path.join(CONFIG_ROOT_PATH, cls.NeuS_config_path)
return {
"required": {
"reference_image": ("IMAGE",),
"reference_mask": ("MASK",),
"reference_normals": ("IMAGE",),
"training_iterations": ("INT", {"default": 1000, "min": 1, "max": 100000}), # longer time, better result. 1w will be ok for most cases
"batch_size": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}),
"learning_rate": ("FLOAT", {"default": 5e-4, "min": 0.00001, "step": 0.00001}),
"learning_rate_alpha": ("FLOAT", {"default": 5e-4, "min": 0.00001, "step": 0.00001}),
"color_loss_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "step": 0.01}),
"mesh_smoothing_weight": ("FLOAT", {"default": 0.1, "min": 0.0, "step": 0.01}),
"mask_loss_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "step": 0.01}),
"normal_loss_weight": ("FLOAT", {"default": 1.0, "min": 0.0, "step": 0.01}),
"sparse_loss_weight": ("FLOAT", {"default": 0.1, "min": 0.0, "step": 0.01}),
"warm_up_end": ("INT", {"default": 500, "min": 0, "max": 0xffffffffffffffff}),
"anneal_end": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
"use_white_background": ("BOOLEAN", {"default": True},),
"geometry_extract_resolution": ("INT", {"default": 512, "min": 1, "max": 0xffffffffffffffff}),
"marching_cude_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "step": 0.01}),
},
}
RETURN_TYPES = (
"MESH",
)
RETURN_NAMES = (
"mesh",
)
FUNCTION = "run_NeuS"
CATEGORY = "Comfy3D/Algorithm"
def run_NeuS(
self,
reference_image, # [num_views, H, W, 3]
reference_mask, # [num_views, H, W]
reference_normals, # [num_views, H, W, 3]
training_iterations,
batch_size,
learning_rate,
learning_rate_alpha,
color_loss_weight,
mesh_smoothing_weight,
mask_loss_weight,
normal_loss_weight,
sparse_loss_weight,
warm_up_end,
anneal_end,
use_white_background,
geometry_extract_resolution,
marching_cude_threshold
):
mesh = None
num_views = reference_image.shape[0]
if num_views in [6, 5, 4]:
abs_fix_cam_pose_dir = os.path.join(ROOT_PATH, self.fix_cam_pose_dir)
with torch.inference_mode(False):
NeuS_params = NeuSParams(
training_iterations,
batch_size,
learning_rate,
learning_rate_alpha,
color_loss_weight,
mesh_smoothing_weight,
mask_loss_weight,
normal_loss_weight,
sparse_loss_weight,
warm_up_end,
anneal_end,
use_white_background
)
runner = NeuSRunner(
reference_image,
reference_mask,
reference_normals,
self.config_path_abs,
abs_fix_cam_pose_dir,
num_views,
NeuS_params
)
cstr(f"[{self.__class__.__name__}] Training NeuS...").msg.print()
runner.train()
cstr(f"[{self.__class__.__name__}] Extracting Mesh...").msg.print()
mesh = runner.extract_mesh(resolution=geometry_extract_resolution, threshold=marching_cude_threshold)
else:
cstr(f"[{self.__class__.__name__}] Number of views must be one of the following: 6, 5, 4. But got {num_views}").error.print()
return(mesh, )
class Load_TripoSR_Model:
checkpoints_dir = "TripoSR"
default_ckpt_name = "model.ckpt"