diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..945b6ee --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "cmake.sourceDirectory": "C:/Users/reall/Softwares/ComfyUI_windows_portable/ComfyUI/custom_nodes/ComfyUI-3D-Pack/diff-gaussian-rasterization" +} \ No newline at end of file diff --git a/_.txt b/_.txt new file mode 100644 index 0000000..d443e57 --- /dev/null +++ b/_.txt @@ -0,0 +1,165 @@ +fatal: destination path 'diff-gaussian-rasterization' already exists and is not an empty directory. +Processing c:\users\reall\softwares\comfyui_windows_portable\comfyui\custom_nodes\comfyui-3d-pack\diff-gaussian-rasterization + Preparing metadata (setup.py) ... done +Building wheels for collected packages: diff-gaussian-rasterization + Building wheel for diff-gaussian-rasterization (setup.py) ... error + error: subprocess-exited-with-error + + × python setup.py bdist_wheel did not run successfully. + │ exit code: 1 + ╰─> [148 lines of output] + running bdist_wheel + running build + running build_py + creating build + creating build\lib.win-amd64-cpython-311 + creating build\lib.win-amd64-cpython-311\diff_gaussian_rasterization + copying diff_gaussian_rasterization\__init__.py -> build\lib.win-amd64-cpython-311\diff_gaussian_rasterization + running build_ext + building 'diff_gaussian_rasterization._C' extension + creating C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311 + creating C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release + creating C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\cuda_rasterizer + Emitting ninja build file C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\build.ninja... + Compiling objects... + Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + [1/5] C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\bin\nvcc --generate-dependencies-with-compile --dependency-output C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\rasterize_points.obj.d -std=c++17 --use-local-env -Xcompiler /MD -Xcompiler /wd4819 -Xcompiler /wd4251 -Xcompiler /wd4244 -Xcompiler /wd4267 -Xcompiler /wd4275 -Xcompiler /wd4018 -Xcompiler /wd4190 -Xcompiler /wd4624 -Xcompiler /wd4067 -Xcompiler /wd4068 -Xcompiler /EHsc -Xcudafe --diag_suppress=base_class_has_different_dll_interface -Xcudafe --diag_suppress=field_without_dll_interface -Xcudafe --diag_suppress=dll_interface_conflict_none_assumed -Xcudafe --diag_suppress=dll_interface_conflict_dllexport_assumed -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\torch\csrc\api\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\TH -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\THC "-IC:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include" -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Include "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.37.32822\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\VS\include" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.22621.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\um" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\shared" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\winrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\cppwinrt" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um" -c C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\rasterize_points.cu -o C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\rasterize_points.obj -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -IC:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\third_party/glm/ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 + FAILED: C:/Users/reall/Softwares/ComfyUI_windows_portable/ComfyUI/custom_nodes/ComfyUI-3D-Pack/diff-gaussian-rasterization/build/temp.win-amd64-cpython-311/Release/rasterize_points.obj + C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\bin\nvcc --generate-dependencies-with-compile --dependency-output C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\rasterize_points.obj.d -std=c++17 --use-local-env -Xcompiler /MD -Xcompiler /wd4819 -Xcompiler /wd4251 -Xcompiler /wd4244 -Xcompiler /wd4267 -Xcompiler /wd4275 -Xcompiler /wd4018 -Xcompiler /wd4190 -Xcompiler /wd4624 -Xcompiler /wd4067 -Xcompiler /wd4068 -Xcompiler /EHsc -Xcudafe --diag_suppress=base_class_has_different_dll_interface -Xcudafe --diag_suppress=field_without_dll_interface -Xcudafe --diag_suppress=dll_interface_conflict_none_assumed -Xcudafe --diag_suppress=dll_interface_conflict_dllexport_assumed -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\torch\csrc\api\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\TH -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\THC "-IC:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include" -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Include "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.37.32822\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\VS\include" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.22621.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\um" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\shared" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\winrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\cppwinrt" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um" -c C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\rasterize_points.cu -o C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\rasterize_points.obj -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -IC:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\third_party/glm/ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 + C:/Users/reall/Softwares/ComfyUI_windows_portable/python_embeded/Lib/site-packages/torch/include\torch/csrc/python_headers.h(12): fatal error C1083: Cannot open include file: 'Python.h': No such file or directory + + rasterize_points.cu + + [2/5] C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\bin\nvcc --generate-dependencies-with-compile --dependency-output C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\cuda_rasterizer/forward.obj.d -std=c++17 --use-local-env -Xcompiler /MD -Xcompiler /wd4819 -Xcompiler /wd4251 -Xcompiler /wd4244 -Xcompiler /wd4267 -Xcompiler /wd4275 -Xcompiler /wd4018 -Xcompiler /wd4190 -Xcompiler /wd4624 -Xcompiler /wd4067 -Xcompiler /wd4068 -Xcompiler /EHsc -Xcudafe --diag_suppress=base_class_has_different_dll_interface -Xcudafe --diag_suppress=field_without_dll_interface -Xcudafe --diag_suppress=dll_interface_conflict_none_assumed -Xcudafe --diag_suppress=dll_interface_conflict_dllexport_assumed -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\torch\csrc\api\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\TH -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\THC "-IC:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include" -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Include "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.37.32822\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\VS\include" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.22621.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\um" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\shared" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\winrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\cppwinrt" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um" -c C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\cuda_rasterizer\forward.cu -o C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\cuda_rasterizer/forward.obj -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -IC:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\third_party/glm/ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 + C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include\cuda\std\detail\libcxx\include\support\atomic\atomic_msvc.h(15): warning C4005: '_Compiler_barrier': macro redefinition + C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC/14.37.32822/include\xatomic.h(56): note: see previous definition of '_Compiler_barrier' + C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include\cuda\std\detail\libcxx\include\support\atomic\atomic_msvc.h(15): warning C4005: '_Compiler_barrier': macro redefinition + C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC/14.37.32822/include\xatomic.h(56): note: see previous definition of '_Compiler_barrier' + C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\cuda_rasterizer\auxiliary.h(151): warning #177-D: variable "p_proj" was declared but never referenced + + float3 p_proj = { p_hom.x * p_w, p_hom.y * p_w, p_hom.z * p_w }; + + ^ + + + + Remark: The warnings can be suppressed with "-diag-suppress " + + + + forward.cu + + tmpxft_00002c5c_00000000-7_forward.cudafe1.cpp + + [3/5] C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\bin\nvcc --generate-dependencies-with-compile --dependency-output C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\cuda_rasterizer/backward.obj.d -std=c++17 --use-local-env -Xcompiler /MD -Xcompiler /wd4819 -Xcompiler /wd4251 -Xcompiler /wd4244 -Xcompiler /wd4267 -Xcompiler /wd4275 -Xcompiler /wd4018 -Xcompiler /wd4190 -Xcompiler /wd4624 -Xcompiler /wd4067 -Xcompiler /wd4068 -Xcompiler /EHsc -Xcudafe --diag_suppress=base_class_has_different_dll_interface -Xcudafe --diag_suppress=field_without_dll_interface -Xcudafe --diag_suppress=dll_interface_conflict_none_assumed -Xcudafe --diag_suppress=dll_interface_conflict_dllexport_assumed -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\torch\csrc\api\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\TH -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\THC "-IC:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include" -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Include "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.37.32822\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\VS\include" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.22621.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\um" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\shared" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\winrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\cppwinrt" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um" -c C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\cuda_rasterizer\backward.cu -o C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\cuda_rasterizer/backward.obj -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -IC:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\third_party/glm/ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 + C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include\cuda\std\detail\libcxx\include\support\atomic\atomic_msvc.h(15): warning C4005: '_Compiler_barrier': macro redefinition + C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC/14.37.32822/include\xatomic.h(56): note: see previous definition of '_Compiler_barrier' + C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include\cuda\std\detail\libcxx\include\support\atomic\atomic_msvc.h(15): warning C4005: '_Compiler_barrier': macro redefinition + C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC/14.37.32822/include\xatomic.h(56): note: see previous definition of '_Compiler_barrier' + C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\cuda_rasterizer\auxiliary.h(151): warning #177-D: variable "p_proj" was declared but never referenced + + float3 p_proj = { p_hom.x * p_w, p_hom.y * p_w, p_hom.z * p_w }; + + ^ + + + + Remark: The warnings can be suppressed with "-diag-suppress " + + + + backward.cu + + tmpxft_00002a5c_00000000-7_backward.cudafe1.cpp + + [4/5] cl /showIncludes /nologo /O2 /W3 /GL /DNDEBUG /MD /MD /wd4819 /wd4251 /wd4244 /wd4267 /wd4275 /wd4018 /wd4190 /wd4624 /wd4067 /wd4068 /EHsc -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\torch\csrc\api\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\TH -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\THC "-IC:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include" -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Include "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.37.32822\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\VS\include" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.22621.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\um" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\shared" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\winrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\cppwinrt" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um" -c C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\ext.cpp /FoC:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\ext.obj -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 /std:c++17 + FAILED: C:/Users/reall/Softwares/ComfyUI_windows_portable/ComfyUI/custom_nodes/ComfyUI-3D-Pack/diff-gaussian-rasterization/build/temp.win-amd64-cpython-311/Release/ext.obj + cl /showIncludes /nologo /O2 /W3 /GL /DNDEBUG /MD /MD /wd4819 /wd4251 /wd4244 /wd4267 /wd4275 /wd4018 /wd4190 /wd4624 /wd4067 /wd4068 /EHsc -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\torch\csrc\api\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\TH -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\THC "-IC:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include" -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Include "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.37.32822\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\VS\include" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.22621.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\um" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\shared" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\winrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\cppwinrt" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um" -c C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\ext.cpp /FoC:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\ext.obj -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 /std:c++17 + C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\torch/csrc/python_headers.h(12): fatal error C1083: Cannot open include file: 'Python.h': No such file or directory + [5/5] C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\bin\nvcc --generate-dependencies-with-compile --dependency-output C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\cuda_rasterizer/rasterizer_impl.obj.d -std=c++17 --use-local-env -Xcompiler /MD -Xcompiler /wd4819 -Xcompiler /wd4251 -Xcompiler /wd4244 -Xcompiler /wd4267 -Xcompiler /wd4275 -Xcompiler /wd4018 -Xcompiler /wd4190 -Xcompiler /wd4624 -Xcompiler /wd4067 -Xcompiler /wd4068 -Xcompiler /EHsc -Xcudafe --diag_suppress=base_class_has_different_dll_interface -Xcudafe --diag_suppress=field_without_dll_interface -Xcudafe --diag_suppress=dll_interface_conflict_none_assumed -Xcudafe --diag_suppress=dll_interface_conflict_dllexport_assumed -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\torch\csrc\api\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\TH -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\include\THC "-IC:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include" -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\include -IC:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Include "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.37.32822\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\VS\include" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.22621.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\um" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\shared" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\winrt" "-IC:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\cppwinrt" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um" -c C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\cuda_rasterizer\rasterizer_impl.cu -o C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\build\temp.win-amd64-cpython-311\Release\cuda_rasterizer/rasterizer_impl.obj -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -IC:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\third_party/glm/ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -gencode=arch=compute_86,code=compute_86 -gencode=arch=compute_86,code=sm_86 + C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include\cuda\std\detail\libcxx\include\support\atomic\atomic_msvc.h(15): warning C4005: '_Compiler_barrier': macro redefinition + C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC/14.37.32822/include\xatomic.h(56): note: see previous definition of '_Compiler_barrier' + C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.1\include\cuda\std\detail\libcxx\include\support\atomic\atomic_msvc.h(15): warning C4005: '_Compiler_barrier': macro redefinition + C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/MSVC/14.37.32822/include\xatomic.h(56): note: see previous definition of '_Compiler_barrier' + C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\cuda_rasterizer\auxiliary.h(151): warning #177-D: variable "p_proj" was declared but never referenced + float3 p_proj = { p_hom.x * p_w, p_hom.y * p_w, p_hom.z * p_w }; + ^ + + Remark: The warnings can be suppressed with "-diag-suppress " + + rasterizer_impl.cu + tmpxft_000011f0_00000000-7_rasterizer_impl.cudafe1.cpp + + ninja: build stopped: subcommand failed. + Traceback (most recent call last): + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\utils\cpp_extension.py", line 2100, in _run_ninja_build + subprocess.run( + File "subprocess.py", line 571, in run + subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1. + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "", line 2, in + File "", line 34, in + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\ComfyUI\custom_nodes\ComfyUI-3D-Pack\diff-gaussian-rasterization\setup.py", line 17, in + setup( + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\__init__.py", line 103, in setup + return distutils.core.setup(**attrs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\core.py", line 185, in setup + return run_commands(dist) + ^^^^^^^^^^^^^^^^^^ + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\core.py", line 201, in run_commands + dist.run_commands() + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\dist.py", line 969, in run_commands + self.run_command(cmd) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\dist.py", line 989, in run_command + super().run_command(command) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\dist.py", line 988, in run_command + cmd_obj.run() + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\wheel\bdist_wheel.py", line 369, in run + self.run_command("build") + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\cmd.py", line 318, in run_command + self.distribution.run_command(command) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\dist.py", line 989, in run_command + super().run_command(command) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\dist.py", line 988, in run_command + cmd_obj.run() + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\command\build.py", line 131, in run + self.run_command(cmd_name) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\cmd.py", line 318, in run_command + self.distribution.run_command(command) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\dist.py", line 989, in run_command + super().run_command(command) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\dist.py", line 988, in run_command + cmd_obj.run() + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\command\build_ext.py", line 88, in run + _build_ext.run(self) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\command\build_ext.py", line 345, in run + self.build_extensions() + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\utils\cpp_extension.py", line 873, in build_extensions + build_ext.build_extensions(self) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\command\build_ext.py", line 467, in build_extensions + self._build_extensions_serial() + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\command\build_ext.py", line 493, in _build_extensions_serial + self.build_extension(ext) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\command\build_ext.py", line 249, in build_extension + _build_ext.build_extension(self, ext) + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\setuptools\_distutils\command\build_ext.py", line 548, in build_extension + objects = self.compiler.compile( + ^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\utils\cpp_extension.py", line 845, in win_wrap_ninja_compile + _write_ninja_file_and_compile_objects( + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\utils\cpp_extension.py", line 1774, in _write_ninja_file_and_compile_objects + _run_ninja_build( + File "C:\Users\reall\Softwares\ComfyUI_windows_portable\python_embeded\Lib\site-packages\torch\utils\cpp_extension.py", line 2116, in _run_ninja_build + raise RuntimeError(message) from e + RuntimeError: Error compiling objects for extension + [end of output] + + note: This error originates from a subprocess, and is likely not a problem with pip. + ERROR: Failed building wheel for diff-gaussian-rasterization + Running setup.py clean for diff-gaussian-rasterization +Failed to build diff-gaussian-rasterization +ERROR: Could not build wheels for diff-gaussian-rasterization, which is required to install pyproject.toml-based projects \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..c08e14b --- /dev/null +++ b/__init__.py @@ -0,0 +1,20 @@ +import os +import sys +from nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +sys.path.append('.') # for portable version + +all_module_names = ( + "diff_rast", + "gaussian_splatting", + "shared_utils", + "mesh_processer", +) +for module_name in all_module_names: + module_path = os.path.join(os.path.dirname(__file__), module_name) + sys.path.append(module_path) + + +import diff_rast.diff_texturing + +__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] diff --git a/diff_rast/diff_mesh_renderer.py b/diff_rast/diff_mesh_renderer.py new file mode 100644 index 0000000..f19343d --- /dev/null +++ b/diff_rast/diff_mesh_renderer.py @@ -0,0 +1,152 @@ +import os +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import nvdiffrast.torch as dr + +from mesh_processer.mesh import Mesh, safe_normalize + +def scale_img_nhwc(x, size, mag='bilinear', min='bilinear'): + assert (x.shape[1] >= size[0] and x.shape[2] >= size[1]) or (x.shape[1] < size[0] and x.shape[2] < size[1]), "Trying to magnify image in one dimension and minify in the other" + y = x.permute(0, 3, 1, 2) # NHWC -> NCHW + if x.shape[1] > size[0] and x.shape[2] > size[1]: # Minification, previous size was bigger + y = torch.nn.functional.interpolate(y, size, mode=min) + else: # Magnification + if mag == 'bilinear' or mag == 'bicubic': + y = torch.nn.functional.interpolate(y, size, mode=mag, align_corners=True) + else: + y = torch.nn.functional.interpolate(y, size, mode=mag) + return y.permute(0, 2, 3, 1).contiguous() # NCHW -> NHWC + +def scale_img_hwc(x, size, mag='bilinear', min='bilinear'): + return scale_img_nhwc(x[None, ...], size, mag, min)[0] + +def scale_img_nhw(x, size, mag='bilinear', min='bilinear'): + return scale_img_nhwc(x[..., None], size, mag, min)[..., 0] + +def scale_img_hw(x, size, mag='bilinear', min='bilinear'): + return scale_img_nhwc(x[None, ..., None], size, mag, min)[0, ..., 0] + +def trunc_rev_sigmoid(x, eps=1e-6): + x = x.clamp(eps, 1 - eps) + return torch.log(x / (1 - x)) + +def make_divisible(x, m=8): + return int(math.ceil(x / m) * m) + +class Renderer(nn.Module): + def __init__(self, opt): + + super().__init__() + + self.opt = opt + + self.mesh = Mesh.load(self.opt.mesh, resize=False) + + if not self.opt.force_cuda_rast and (not self.opt.gui or os.name == 'nt'): + self.glctx = dr.RasterizeGLContext() + else: + self.glctx = dr.RasterizeCudaContext() + + # extract trainable parameters + self.v_offsets = nn.Parameter(torch.zeros_like(self.mesh.v)) + self.raw_albedo = nn.Parameter(trunc_rev_sigmoid(self.mesh.albedo)) + + + def get_params(self): + + params = [ + {'params': self.raw_albedo, 'lr': self.opt.texture_lr}, + ] + + if self.opt.train_geo: + params.append({'params': self.v_offsets, 'lr': self.opt.geom_lr}) + + return params + + @torch.no_grad() + def export_mesh(self, save_path): + self.mesh.v = (self.mesh.v + self.v_offsets).detach() + self.mesh.albedo = torch.sigmoid(self.raw_albedo.detach()) + self.mesh.write(save_path) + + + def render(self, pose, proj, h0, w0, ssaa=1, bg_color=1, texture_filter='linear-mipmap-linear'): + + # do super-sampling + if ssaa != 1: + h = make_divisible(h0 * ssaa, 8) + w = make_divisible(w0 * ssaa, 8) + else: + h, w = h0, w0 + + results = {} + + # get v + if self.opt.train_geo: + v = self.mesh.v + self.v_offsets # [N, 3] + else: + v = self.mesh.v + + pose = torch.from_numpy(pose.astype(np.float32)).to(v.device) + proj = torch.from_numpy(proj.astype(np.float32)).to(v.device) + + # get v_clip and render rgb + v_cam = torch.matmul(F.pad(v, pad=(0, 1), mode='constant', value=1.0), torch.inverse(pose).T).float().unsqueeze(0) + v_clip = v_cam @ proj.T + + rast, rast_db = dr.rasterize(self.glctx, v_clip, self.mesh.f, (h, w)) + + alpha = (rast[0, ..., 3:] > 0).float() + depth, _ = dr.interpolate(-v_cam[..., [2]], rast, self.mesh.f) # [1, H, W, 1] + depth = depth.squeeze(0) # [H, W, 1] + + texc, texc_db = dr.interpolate(self.mesh.vt.unsqueeze(0).contiguous(), rast, self.mesh.ft, rast_db=rast_db, diff_attrs='all') + albedo = dr.texture(self.raw_albedo.unsqueeze(0), texc, uv_da=texc_db, filter_mode=texture_filter) # [1, H, W, 3] + albedo = torch.sigmoid(albedo) + # get vn and render normal + if self.opt.train_geo: + i0, i1, i2 = self.mesh.f[:, 0].long(), self.mesh.f[:, 1].long(), self.mesh.f[:, 2].long() + v0, v1, v2 = v[i0, :], v[i1, :], v[i2, :] + + face_normals = torch.cross(v1 - v0, v2 - v0) + face_normals = safe_normalize(face_normals) + + vn = torch.zeros_like(v) + vn.scatter_add_(0, i0[:, None].repeat(1,3), face_normals) + vn.scatter_add_(0, i1[:, None].repeat(1,3), face_normals) + vn.scatter_add_(0, i2[:, None].repeat(1,3), face_normals) + + vn = torch.where(torch.sum(vn * vn, -1, keepdim=True) > 1e-20, vn, torch.tensor([0.0, 0.0, 1.0], dtype=torch.float32, device=vn.device)) + else: + vn = self.mesh.vn + + normal, _ = dr.interpolate(vn.unsqueeze(0).contiguous(), rast, self.mesh.fn) + normal = safe_normalize(normal[0]) + + # rotated normal (where [0, 0, 1] always faces camera) + rot_normal = normal @ pose[:3, :3] + viewcos = rot_normal[..., [2]] + + # antialias + albedo = dr.antialias(albedo, rast, v_clip, self.mesh.f).squeeze(0) # [H, W, 3] + albedo = alpha * albedo + (1 - alpha) * bg_color + + # ssaa + if ssaa != 1: + albedo = scale_img_hwc(albedo, (h0, w0)) + alpha = scale_img_hwc(alpha, (h0, w0)) + depth = scale_img_hwc(depth, (h0, w0)) + normal = scale_img_hwc(normal, (h0, w0)) + viewcos = scale_img_hwc(viewcos, (h0, w0)) + + results['image'] = albedo.clamp(0, 1) + results['alpha'] = alpha + results['depth'] = depth + results['normal'] = (normal + 1) / 2 + results['viewcos'] = viewcos + + return results \ No newline at end of file diff --git a/diff_rast/diff_texturing.py b/diff_rast/diff_texturing.py new file mode 100644 index 0000000..b07ad8f --- /dev/null +++ b/diff_rast/diff_texturing.py @@ -0,0 +1,4 @@ +import sys + +from shared_utils.common_utils import get_persistent_directory +from shared_utils.camera_utils import orbit_camera, OrbitCamera diff --git a/install.bat b/install.bat new file mode 100644 index 0000000..a93ad66 --- /dev/null +++ b/install.bat @@ -0,0 +1,31 @@ +@echo off + +set "requirements_txt=%~dp0\requirements.txt" +set "python_miniconda_exec=..\..\..\python_miniconda_env\ComfyUI\python.exe" +set "python_exec=..\..\..\python_embeded\python.exe" + +echo Installing ComfyUI-3D-Pack ... + + +if exist "%python_miniconda_exec%" ( + echo Installing with Miniconda Environment + "%python_miniconda_exec%" -s -m pip install -r "%requirements_txt%" + + git clone --recursive https://github.com/ashawkey/diff-gaussian-rasterization + "%python_miniconda_exec%" -s -m pip install ./diff-gaussian-rasterization + "%python_miniconda_exec%" -s -m pip install ./simple-knn + "%python_miniconda_exec%" -s -m pip install git+https://github.com/NVlabs/nvdiffrast/ + "%python_miniconda_exec%" -s -m pip install git+https://github.com/ashawkey/kiuikit +) +else if exist "%python_exec%" ( + echo Installing with ComfyUI Portable + "%python_exec%" -s -m pip install -r "%requirements_txt%" +) else ( + echo Installing with system Python + pip install -r "%requirements_txt%" +) + + +echo Install Finished! + +pause \ No newline at end of file diff --git a/mesh_processer/mesh.py b/mesh_processer/mesh.py new file mode 100644 index 0000000..b58ba09 --- /dev/null +++ b/mesh_processer/mesh.py @@ -0,0 +1,622 @@ +import os +import cv2 +import torch +import trimesh +import numpy as np + +def dot(x, y): + return torch.sum(x * y, -1, keepdim=True) + + +def length(x, eps=1e-20): + return torch.sqrt(torch.clamp(dot(x, x), min=eps)) + + +def safe_normalize(x, eps=1e-20): + return x / length(x, eps) + +class Mesh: + def __init__( + self, + v=None, + f=None, + vn=None, + fn=None, + vt=None, + ft=None, + albedo=None, + vc=None, # vertex color + device=None, + ): + self.device = device + self.v = v + self.vn = vn + self.vt = vt + self.f = f + self.fn = fn + self.ft = ft + # only support a single albedo + self.albedo = albedo + # support vertex color is no albedo + self.vc = vc + + self.ori_center = 0 + self.ori_scale = 1 + + @classmethod + def load(cls, path=None, resize=True, renormal=True, retex=False, front_dir='+z', **kwargs): + # assume init with kwargs + if path is None: + mesh = cls(**kwargs) + # obj supports face uv + elif path.endswith(".obj"): + mesh = cls.load_obj(path, **kwargs) + # trimesh only supports vertex uv, but can load more formats + else: + mesh = cls.load_trimesh(path, **kwargs) + + print(f"[Mesh loading] v: {mesh.v.shape}, f: {mesh.f.shape}") + # auto-normalize + if resize: + mesh.auto_size() + # auto-fix normal + if renormal or mesh.vn is None: + mesh.auto_normal() + print(f"[Mesh loading] vn: {mesh.vn.shape}, fn: {mesh.fn.shape}") + # auto-fix texcoords + if retex or (mesh.albedo is not None and mesh.vt is None): + mesh.auto_uv(cache_path=path) + print(f"[Mesh loading] vt: {mesh.vt.shape}, ft: {mesh.ft.shape}") + + # rotate front dir to +z + if front_dir != "+z": + # axis switch + if "-z" in front_dir: + T = torch.tensor([[1, 0, 0], [0, 1, 0], [0, 0, -1]], device=mesh.device, dtype=torch.float32) + elif "+x" in front_dir: + T = torch.tensor([[0, 0, 1], [0, 1, 0], [1, 0, 0]], device=mesh.device, dtype=torch.float32) + elif "-x" in front_dir: + T = torch.tensor([[0, 0, -1], [0, 1, 0], [1, 0, 0]], device=mesh.device, dtype=torch.float32) + elif "+y" in front_dir: + T = torch.tensor([[1, 0, 0], [0, 0, 1], [0, 1, 0]], device=mesh.device, dtype=torch.float32) + elif "-y" in front_dir: + T = torch.tensor([[1, 0, 0], [0, 0, -1], [0, 1, 0]], device=mesh.device, dtype=torch.float32) + else: + T = torch.tensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]], device=mesh.device, dtype=torch.float32) + # rotation (how many 90 degrees) + if '1' in front_dir: + T @= torch.tensor([[0, -1, 0], [1, 0, 0], [0, 0, 1]], device=mesh.device, dtype=torch.float32) + elif '2' in front_dir: + T @= torch.tensor([[1, 0, 0], [0, -1, 0], [0, 0, 1]], device=mesh.device, dtype=torch.float32) + elif '3' in front_dir: + T @= torch.tensor([[0, 1, 0], [-1, 0, 0], [0, 0, 1]], device=mesh.device, dtype=torch.float32) + mesh.v @= T + mesh.vn @= T + + return mesh + + # load from obj file + @classmethod + def load_obj(cls, path, albedo_path=None, device=None): + assert os.path.splitext(path)[-1] == ".obj" + + mesh = cls() + + # device + if device is None: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + mesh.device = device + + # load obj + with open(path, "r") as f: + lines = f.readlines() + + def parse_f_v(fv): + # pass in a vertex term of a face, return {v, vt, vn} (-1 if not provided) + # supported forms: (Only supports triangulated face) + # f v1 v2 v3 + # f v1/vt1 v2/vt2 v3/vt3 + # f v1/vt1/vn1 v2/vt2/vn2 v3/vt3/vn3 + # f v1//vn1 v2//vn2 v3//vn3 + xs = [int(x) - 1 if x != "" else -1 for x in fv.split("/")] + xs.extend([-1] * (3 - len(xs))) + return xs[0], xs[1], xs[2] + + # NOTE: we ignore usemtl, and assume the mesh ONLY uses one material (first in mtl) + vertices, texcoords, normals = [], [], [] + faces, tfaces, nfaces = [], [], [] + mtl_path = None + + for line in lines: + split_line = line.split() + # empty line + if len(split_line) == 0: + continue + prefix = split_line[0].lower() + # mtllib + if prefix == "mtllib": + mtl_path = split_line[1] + # usemtl + elif prefix == "usemtl": + pass # ignored + # v/vn/vt + elif prefix == "v": + vertices.append([float(v) for v in split_line[1:]]) + elif prefix == "vn": + normals.append([float(v) for v in split_line[1:]]) + elif prefix == "vt": + val = [float(v) for v in split_line[1:]] + texcoords.append([val[0], 1.0 - val[1]]) + elif prefix == "f": + vs = split_line[1:] + nv = len(vs) + v0, t0, n0 = parse_f_v(vs[0]) + for i in range(nv - 2): # triangulate (assume vertices are ordered) + v1, t1, n1 = parse_f_v(vs[i + 1]) + v2, t2, n2 = parse_f_v(vs[i + 2]) + faces.append([v0, v1, v2]) + tfaces.append([t0, t1, t2]) + nfaces.append([n0, n1, n2]) + + mesh.v = torch.tensor(vertices, dtype=torch.float32, device=device) + mesh.vt = ( + torch.tensor(texcoords, dtype=torch.float32, device=device) + if len(texcoords) > 0 + else None + ) + mesh.vn = ( + torch.tensor(normals, dtype=torch.float32, device=device) + if len(normals) > 0 + else None + ) + + mesh.f = torch.tensor(faces, dtype=torch.int32, device=device) + mesh.ft = ( + torch.tensor(tfaces, dtype=torch.int32, device=device) + if len(texcoords) > 0 + else None + ) + mesh.fn = ( + torch.tensor(nfaces, dtype=torch.int32, device=device) + if len(normals) > 0 + else None + ) + + # see if there is vertex color + use_vertex_color = False + if mesh.v.shape[1] == 6: + use_vertex_color = True + mesh.vc = mesh.v[:, 3:] + mesh.v = mesh.v[:, :3] + print(f"[load_obj] use vertex color: {mesh.vc.shape}") + + # try to load texture image + if not use_vertex_color: + # try to retrieve mtl file + mtl_path_candidates = [] + if mtl_path is not None: + mtl_path_candidates.append(mtl_path) + mtl_path_candidates.append(os.path.join(os.path.dirname(path), mtl_path)) + mtl_path_candidates.append(path.replace(".obj", ".mtl")) + + mtl_path = None + for candidate in mtl_path_candidates: + if os.path.exists(candidate): + mtl_path = candidate + break + + # if albedo_path is not provided, try retrieve it from mtl + if mtl_path is not None and albedo_path is None: + with open(mtl_path, "r") as f: + lines = f.readlines() + for line in lines: + split_line = line.split() + # empty line + if len(split_line) == 0: + continue + prefix = split_line[0] + # NOTE: simply use the first map_Kd as albedo! + if "map_Kd" in prefix: + albedo_path = os.path.join(os.path.dirname(path), split_line[1]) + print(f"[load_obj] use texture from: {albedo_path}") + break + + # still not found albedo_path, or the path doesn't exist + if albedo_path is None or not os.path.exists(albedo_path): + # init an empty texture + print(f"[load_obj] init empty albedo!") + # albedo = np.random.rand(1024, 1024, 3).astype(np.float32) + albedo = np.ones((1024, 1024, 3), dtype=np.float32) * np.array([0.5, 0.5, 0.5]) # default color + else: + albedo = cv2.imread(albedo_path, cv2.IMREAD_UNCHANGED) + albedo = cv2.cvtColor(albedo, cv2.COLOR_BGR2RGB) + albedo = albedo.astype(np.float32) / 255 + print(f"[load_obj] load texture: {albedo.shape}") + + # import matplotlib.pyplot as plt + # plt.imshow(albedo) + # plt.show() + + mesh.albedo = torch.tensor(albedo, dtype=torch.float32, device=device) + + return mesh + + @classmethod + def load_trimesh(cls, path, device=None): + mesh = cls() + + # device + if device is None: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + mesh.device = device + + # use trimesh to load ply/glb, assume only has one single RootMesh... + _data = trimesh.load(path) + if isinstance(_data, trimesh.Scene): + if len(_data.geometry) == 1: + _mesh = list(_data.geometry.values())[0] + else: + # manual concat, will lose texture + _concat = [] + for g in _data.geometry.values(): + if isinstance(g, trimesh.Trimesh): + _concat.append(g) + _mesh = trimesh.util.concatenate(_concat) + else: + _mesh = _data + + if _mesh.visual.kind == 'vertex': + vertex_colors = _mesh.visual.vertex_colors + vertex_colors = np.array(vertex_colors[..., :3]).astype(np.float32) / 255 + mesh.vc = torch.tensor(vertex_colors, dtype=torch.float32, device=device) + print(f"[load_trimesh] use vertex color: {mesh.vc.shape}") + elif _mesh.visual.kind == 'texture': + _material = _mesh.visual.material + if isinstance(_material, trimesh.visual.material.PBRMaterial): + texture = np.array(_material.baseColorTexture).astype(np.float32) / 255 + elif isinstance(_material, trimesh.visual.material.SimpleMaterial): + texture = np.array(_material.to_pbr().baseColorTexture).astype(np.float32) / 255 + else: + raise NotImplementedError(f"material type {type(_material)} not supported!") + mesh.albedo = torch.tensor(texture, dtype=torch.float32, device=device) + print(f"[load_trimesh] load texture: {texture.shape}") + else: + texture = np.ones((1024, 1024, 3), dtype=np.float32) * np.array([0.5, 0.5, 0.5]) + mesh.albedo = torch.tensor(texture, dtype=torch.float32, device=device) + print(f"[load_trimesh] failed to load texture.") + + vertices = _mesh.vertices + + try: + texcoords = _mesh.visual.uv + texcoords[:, 1] = 1 - texcoords[:, 1] + except Exception as e: + texcoords = None + + try: + normals = _mesh.vertex_normals + except Exception as e: + normals = None + + # trimesh only support vertex uv... + faces = tfaces = nfaces = _mesh.faces + + mesh.v = torch.tensor(vertices, dtype=torch.float32, device=device) + mesh.vt = ( + torch.tensor(texcoords, dtype=torch.float32, device=device) + if texcoords is not None + else None + ) + mesh.vn = ( + torch.tensor(normals, dtype=torch.float32, device=device) + if normals is not None + else None + ) + + mesh.f = torch.tensor(faces, dtype=torch.int32, device=device) + mesh.ft = ( + torch.tensor(tfaces, dtype=torch.int32, device=device) + if texcoords is not None + else None + ) + mesh.fn = ( + torch.tensor(nfaces, dtype=torch.int32, device=device) + if normals is not None + else None + ) + + return mesh + + # aabb + def aabb(self): + return torch.min(self.v, dim=0).values, torch.max(self.v, dim=0).values + + # unit size + @torch.no_grad() + def auto_size(self): + vmin, vmax = self.aabb() + self.ori_center = (vmax + vmin) / 2 + self.ori_scale = 1.2 / torch.max(vmax - vmin).item() + self.v = (self.v - self.ori_center) * self.ori_scale + + def auto_normal(self): + i0, i1, i2 = self.f[:, 0].long(), self.f[:, 1].long(), self.f[:, 2].long() + v0, v1, v2 = self.v[i0, :], self.v[i1, :], self.v[i2, :] + + face_normals = torch.cross(v1 - v0, v2 - v0) + + # Splat face normals to vertices + vn = torch.zeros_like(self.v) + vn.scatter_add_(0, i0[:, None].repeat(1, 3), face_normals) + vn.scatter_add_(0, i1[:, None].repeat(1, 3), face_normals) + vn.scatter_add_(0, i2[:, None].repeat(1, 3), face_normals) + + # Normalize, replace zero (degenerated) normals with some default value + vn = torch.where( + dot(vn, vn) > 1e-20, + vn, + torch.tensor([0.0, 0.0, 1.0], dtype=torch.float32, device=vn.device), + ) + vn = safe_normalize(vn) + + self.vn = vn + self.fn = self.f + + def auto_uv(self, cache_path=None, vmap=True): + # try to load cache + if cache_path is not None: + cache_path = os.path.splitext(cache_path)[0] + "_uv.npz" + if cache_path is not None and os.path.exists(cache_path): + data = np.load(cache_path) + vt_np, ft_np, vmapping = data["vt"], data["ft"], data["vmapping"] + else: + import xatlas + + v_np = self.v.detach().cpu().numpy() + f_np = self.f.detach().int().cpu().numpy() + atlas = xatlas.Atlas() + atlas.add_mesh(v_np, f_np) + chart_options = xatlas.ChartOptions() + # chart_options.max_iterations = 4 + atlas.generate(chart_options=chart_options) + vmapping, ft_np, vt_np = atlas[0] # [N], [M, 3], [N, 2] + + # save to cache + if cache_path is not None: + np.savez(cache_path, vt=vt_np, ft=ft_np, vmapping=vmapping) + + vt = torch.from_numpy(vt_np.astype(np.float32)).to(self.device) + ft = torch.from_numpy(ft_np.astype(np.int32)).to(self.device) + self.vt = vt + self.ft = ft + + if vmap: + # remap v/f to vt/ft, so each v correspond to a unique vt. (necessary for gltf) + vmapping = torch.from_numpy(vmapping.astype(np.int64)).long().to(self.device) + self.align_v_to_vt(vmapping) + + def align_v_to_vt(self, vmapping=None): + # remap v/f and vn/vn to vt/ft. + if vmapping is None: + ft = self.ft.view(-1).long() + f = self.f.view(-1).long() + vmapping = torch.zeros(self.vt.shape[0], dtype=torch.long, device=self.device) + vmapping[ft] = f # scatter, randomly choose one if index is not unique + + self.v = self.v[vmapping] + self.f = self.ft + # assume fn == f + if self.vn is not None: + self.vn = self.vn[vmapping] + self.fn = self.ft + + def to(self, device): + self.device = device + for name in ["v", "f", "vn", "fn", "vt", "ft", "albedo"]: + tensor = getattr(self, name) + if tensor is not None: + setattr(self, name, tensor.to(device)) + return self + + def write(self, path): + if path.endswith(".ply"): + self.write_ply(path) + elif path.endswith(".obj"): + self.write_obj(path) + elif path.endswith(".glb") or path.endswith(".gltf"): + self.write_glb(path) + else: + raise NotImplementedError(f"format {path} not supported!") + + # write to ply file (only geom) + def write_ply(self, path): + + v_np = self.v.detach().cpu().numpy() + f_np = self.f.detach().cpu().numpy() + + _mesh = trimesh.Trimesh(vertices=v_np, faces=f_np) + _mesh.export(path) + + # write to gltf/glb file (geom + texture) + def write_glb(self, path): + + assert self.vn is not None and self.vt is not None # should be improved to support export without texture... + + # assert self.v.shape[0] == self.vn.shape[0] and self.v.shape[0] == self.vt.shape[0] + if self.v.shape[0] != self.vt.shape[0]: + self.align_v_to_vt() + + # assume f == fn == ft + + import pygltflib + + f_np = self.f.detach().cpu().numpy().astype(np.uint32) + v_np = self.v.detach().cpu().numpy().astype(np.float32) + # vn_np = self.vn.detach().cpu().numpy().astype(np.float32) + vt_np = self.vt.detach().cpu().numpy().astype(np.float32) + + albedo = self.albedo.detach().cpu().numpy() + albedo = (albedo * 255).astype(np.uint8) + albedo = cv2.cvtColor(albedo, cv2.COLOR_RGB2BGR) + + f_np_blob = f_np.flatten().tobytes() + v_np_blob = v_np.tobytes() + # vn_np_blob = vn_np.tobytes() + vt_np_blob = vt_np.tobytes() + albedo_blob = cv2.imencode('.png', albedo)[1].tobytes() + + gltf = pygltflib.GLTF2( + scene=0, + scenes=[pygltflib.Scene(nodes=[0])], + nodes=[pygltflib.Node(mesh=0)], + meshes=[pygltflib.Mesh(primitives=[ + pygltflib.Primitive( + # indices to accessors (0 is triangles) + attributes=pygltflib.Attributes( + POSITION=1, TEXCOORD_0=2, + ), + indices=0, material=0, + ) + ])], + materials=[ + pygltflib.Material( + pbrMetallicRoughness=pygltflib.PbrMetallicRoughness( + baseColorTexture=pygltflib.TextureInfo(index=0, texCoord=0), + metallicFactor=0.0, + roughnessFactor=1.0, + ), + alphaCutoff=0, + doubleSided=True, + ) + ], + textures=[ + pygltflib.Texture(sampler=0, source=0), + ], + samplers=[ + pygltflib.Sampler(magFilter=pygltflib.LINEAR, minFilter=pygltflib.LINEAR_MIPMAP_LINEAR, wrapS=pygltflib.REPEAT, wrapT=pygltflib.REPEAT), + ], + images=[ + # use embedded (buffer) image + pygltflib.Image(bufferView=3, mimeType="image/png"), + ], + buffers=[ + pygltflib.Buffer(byteLength=len(f_np_blob) + len(v_np_blob) + len(vt_np_blob) + len(albedo_blob)) + ], + # buffer view (based on dtype) + bufferViews=[ + # triangles; as flatten (element) array + pygltflib.BufferView( + buffer=0, + byteLength=len(f_np_blob), + target=pygltflib.ELEMENT_ARRAY_BUFFER, # GL_ELEMENT_ARRAY_BUFFER (34963) + ), + # positions; as vec3 array + pygltflib.BufferView( + buffer=0, + byteOffset=len(f_np_blob), + byteLength=len(v_np_blob), + byteStride=12, # vec3 + target=pygltflib.ARRAY_BUFFER, # GL_ARRAY_BUFFER (34962) + ), + # texcoords; as vec2 array + pygltflib.BufferView( + buffer=0, + byteOffset=len(f_np_blob) + len(v_np_blob), + byteLength=len(vt_np_blob), + byteStride=8, # vec2 + target=pygltflib.ARRAY_BUFFER, + ), + # texture; as none target + pygltflib.BufferView( + buffer=0, + byteOffset=len(f_np_blob) + len(v_np_blob) + len(vt_np_blob), + byteLength=len(albedo_blob), + ), + ], + accessors=[ + # 0 = triangles + pygltflib.Accessor( + bufferView=0, + componentType=pygltflib.UNSIGNED_INT, # GL_UNSIGNED_INT (5125) + count=f_np.size, + type=pygltflib.SCALAR, + max=[int(f_np.max())], + min=[int(f_np.min())], + ), + # 1 = positions + pygltflib.Accessor( + bufferView=1, + componentType=pygltflib.FLOAT, # GL_FLOAT (5126) + count=len(v_np), + type=pygltflib.VEC3, + max=v_np.max(axis=0).tolist(), + min=v_np.min(axis=0).tolist(), + ), + # 2 = texcoords + pygltflib.Accessor( + bufferView=2, + componentType=pygltflib.FLOAT, + count=len(vt_np), + type=pygltflib.VEC2, + max=vt_np.max(axis=0).tolist(), + min=vt_np.min(axis=0).tolist(), + ), + ], + ) + + # set actual data + gltf.set_binary_blob(f_np_blob + v_np_blob + vt_np_blob + albedo_blob) + + # glb = b"".join(gltf.save_to_bytes()) + gltf.save(path) + + # write to obj file (geom + texture) + def write_obj(self, path): + + mtl_path = path.replace(".obj", ".mtl") + albedo_path = path.replace(".obj", "_albedo.png") + + v_np = self.v.detach().cpu().numpy() + vt_np = self.vt.detach().cpu().numpy() if self.vt is not None else None + vn_np = self.vn.detach().cpu().numpy() if self.vn is not None else None + f_np = self.f.detach().cpu().numpy() + ft_np = self.ft.detach().cpu().numpy() if self.ft is not None else None + fn_np = self.fn.detach().cpu().numpy() if self.fn is not None else None + + with open(path, "w") as fp: + fp.write(f"mtllib {os.path.basename(mtl_path)} \n") + + for v in v_np: + fp.write(f"v {v[0]} {v[1]} {v[2]} \n") + + if vt_np is not None: + for v in vt_np: + fp.write(f"vt {v[0]} {1 - v[1]} \n") + + if vn_np is not None: + for v in vn_np: + fp.write(f"vn {v[0]} {v[1]} {v[2]} \n") + + fp.write(f"usemtl defaultMat \n") + for i in range(len(f_np)): + fp.write( + f'f {f_np[i, 0] + 1}/{ft_np[i, 0] + 1 if ft_np is not None else ""}/{fn_np[i, 0] + 1 if fn_np is not None else ""} \ + {f_np[i, 1] + 1}/{ft_np[i, 1] + 1 if ft_np is not None else ""}/{fn_np[i, 1] + 1 if fn_np is not None else ""} \ + {f_np[i, 2] + 1}/{ft_np[i, 2] + 1 if ft_np is not None else ""}/{fn_np[i, 2] + 1 if fn_np is not None else ""} \n' + ) + + with open(mtl_path, "w") as fp: + fp.write(f"newmtl defaultMat \n") + fp.write(f"Ka 1 1 1 \n") + fp.write(f"Kd 1 1 1 \n") + fp.write(f"Ks 0 0 0 \n") + fp.write(f"Tr 1 \n") + fp.write(f"illum 1 \n") + fp.write(f"Ns 0 \n") + fp.write(f"map_Kd {os.path.basename(albedo_path)} \n") + + albedo = self.albedo.detach().cpu().numpy() + albedo = (albedo * 255).astype(np.uint8) + cv2.imwrite(albedo_path, cv2.cvtColor(albedo, cv2.COLOR_RGB2BGR)) \ No newline at end of file diff --git a/nodes.py b/nodes.py new file mode 100644 index 0000000..6a278e6 --- /dev/null +++ b/nodes.py @@ -0,0 +1,121 @@ +import nodes +import torch +import torch.nn.functional as F +import numpy as np +import os +import math + +from pytorch_msssim import SSIM, MS_SSIM + +import diff_rast.diff_texturing +from shared_utils.common_utils import cstr +from mesh_processer.mesh import Mesh + +MANIFEST = { + "name": "ComfyUI-3D-Pack", + "version": (0,0,1), + "author": "Mr. For Example", + "project": "https://github.com/MrForExample/ComfyUI-3D-Pack", + "description": "An extensive node suite that enables ComfyUI to process 3D inputs (Mesh & UV Texture, etc) using cutting edge algorithms (3DGS, NeRF, etc.)", +} + +class Load3DFile: + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "mesh_file_path": ("STRING", {"default": '', "multiline": False}), + "resize": ("BOOLEAN", {"default": False},), + "renormal": ("BOOLEAN", {"default": True},), + "retex": ("BOOLEAN", {"default": False},), + }, + } + + RETURN_TYPES = ( + "MESH", + ) + RETURN_NAMES = ( + "mesh", + ) + FUNCTION = "load_mesh" + CATEGORY = "ComfyUI-3D" + + SUPPORTED_3D_EXTENSIONS = ( + '.obj', + '.ply', + '.glb', + ) + + def load_mesh(self, mesh_file_path, resize, renormal, retex): + mesh = None + + if os.path.exists(mesh_file_path): + folder, file = os.path.split(self.filepath) + if file.lower().endswith(self.SUPPORTED_3D_EXTENSIONS): + mesh = Mesh.load(self.opt.mesh, resize, renormal, retex) + else: + cstr(f"File name {file} does not end with supported 3D file extensions: {self.SUPPORTED_3D_EXTENSIONS}").error.print() + else: + cstr(f"File {mesh_file_path} does not exist").error.print() + return (mesh, ) + +class GaussianSplatting: + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "reference_images": ("IMAGE",), + }, + } + + RETURN_TYPES = ( + "GS_RAW", + ) + RETURN_NAMES = ( + "raw_3DGS", + ) + FUNCTION = "run_3DGS" + CATEGORY = "ComfyUI-3D" + + def run_3DGS(self, reference_images): + raw_3DGS = None + + return (raw_3DGS, ) + +class BakeTextureToMesh: + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "reference_images": ("IMAGE",), + "mesh": ("MESH") + }, + } + + RETURN_TYPES = ( + "IMAGE", + ) + RETURN_NAMES = ( + "baked_texture", + ) + FUNCTION = "bake_texture" + CATEGORY = "ComfyUI-3D" + + def bake_texture(self, reference_images, mesh): + baked_texture = None + + return (baked_texture, ) + +NODE_CLASS_MAPPINGS = { + "Load3DFile": Load3DFile, + "3DGS": GaussianSplatting, + "BakeTextureToMesh": BakeTextureToMesh +} +NODE_DISPLAY_NAME_MAPPINGS = { + "Load3DFile": "Load 3D File", + "3DGS": "3D Gaussian Splatting", + "BakeTextureToMesh": "Bake Texture To Mesh", +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..767af68 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +#python-dev-tools +cmake +ninja +numpy +scipy +opencv-python +tqdm + +# for training differentiable tensors +pytorch_msssim + +# for dmtet and mesh import & export +trimesh +plyfile +pygltflib +xatlas \ No newline at end of file diff --git a/shared_utils/__init__.py b/shared_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shared_utils/camera_utils.py b/shared_utils/camera_utils.py new file mode 100644 index 0000000..d181a10 --- /dev/null +++ b/shared_utils/camera_utils.py @@ -0,0 +1,217 @@ +import numpy as np +import math +from scipy.spatial.transform import Rotation as R + +import torch + +def dot(x, y): + if isinstance(x, np.ndarray): + return np.sum(x * y, -1, keepdims=True) + else: + return torch.sum(x * y, -1, keepdim=True) + + +def length(x, eps=1e-20): + if isinstance(x, np.ndarray): + return np.sqrt(np.maximum(np.sum(x * x, axis=-1, keepdims=True), eps)) + else: + return torch.sqrt(torch.clamp(dot(x, x), min=eps)) + + +def safe_normalize(x, eps=1e-20): + return x / length(x, eps) + + +def look_at(campos, target, opengl=True): + # campos: [N, 3], camera/eye position + # target: [N, 3], object to look at + # return: [N, 3, 3], rotation matrix + if not opengl: + # camera forward aligns with -z + forward_vector = safe_normalize(target - campos) + up_vector = np.array([0, 1, 0], dtype=np.float32) + right_vector = safe_normalize(np.cross(forward_vector, up_vector)) + up_vector = safe_normalize(np.cross(right_vector, forward_vector)) + else: + # camera forward aligns with +z + forward_vector = safe_normalize(campos - target) + up_vector = np.array([0, 1, 0], dtype=np.float32) + right_vector = safe_normalize(np.cross(up_vector, forward_vector)) + up_vector = safe_normalize(np.cross(forward_vector, right_vector)) + R = np.stack([right_vector, up_vector, forward_vector], axis=1) + return R + + +# elevation & azimuth to pose (cam2world) matrix +def orbit_camera(elevation, azimuth, radius=1, is_degree=True, target=None, opengl=True): + # radius: scalar + # elevation: scalar, in (-90, 90), from +y to -y is (-90, 90) + # azimuth: scalar, in (-180, 180), from +z to +x is (0, 90) + # return: [4, 4], camera pose matrix + if is_degree: + elevation = np.deg2rad(elevation) + azimuth = np.deg2rad(azimuth) + x = radius * np.cos(elevation) * np.sin(azimuth) + y = - radius * np.sin(elevation) + z = radius * np.cos(elevation) * np.cos(azimuth) + if target is None: + target = np.zeros([3], dtype=np.float32) + campos = np.array([x, y, z]) + target # [3] + T = np.eye(4, dtype=np.float32) + T[:3, :3] = look_at(campos, target, opengl) + T[:3, 3] = campos + return T + +def get_look_at_camera_pose(target, target_to_cam_offset, look_distance=0.1, opengl=True): + """ + Calculate the pose (cam2world) matrix from target position the camera suppose to look at and offset vector from target to camera + + Args: + target (NDArray[float32], shape: 3): the target position the camera suppose to look at + target_to_cam_dir (NDArray[float32], shape: 3): offset direction from target to camera + look_distance (float, optional): length of offset vector from target to camera. + + Returns: + NDArray[float32]: shape: (4, 4), pose (cam2world) matrix + """ + + norm=np.linalg.norm(target_to_cam_offset) + if norm==0: + norm=np.finfo(np.float32).eps + target_to_cam_offset = look_distance * target_to_cam_offset / norm + + campos = target_to_cam_offset + target # [3] + T = np.eye(4, dtype=np.float32) + T[:3, :3] = look_at(campos, target, opengl) + T[:3, 3] = campos + return T + +class OrbitCamera: + def __init__(self, W, H, r=2, fovy=60, near=0.01, far=100): + self.W = W + self.H = H + self.radius = r # camera distance from center + self.fovy = np.deg2rad(fovy) # deg 2 rad + self.near = near + self.far = far + self.center = np.array([0, 0, 0], dtype=np.float32) # look at this point + self.rot = R.from_matrix(np.eye(3)) + self.up = np.array([0, 1, 0], dtype=np.float32) # need to be normalized! + + @property + def fovx(self): + return 2 * np.arctan(np.tan(self.fovy / 2) * self.W / self.H) + + @property + def campos(self): + return self.pose[:3, 3] + + # pose (c2w) + @property + def pose(self): + # first move camera to radius + res = np.eye(4, dtype=np.float32) + res[2, 3] = self.radius # opengl convention... + # rotate + rot = np.eye(4, dtype=np.float32) + rot[:3, :3] = self.rot.as_matrix() + res = rot @ res + # translate + res[:3, 3] -= self.center + return res + + # view (w2c) + @property + def view(self): + return np.linalg.inv(self.pose) + + # projection (perspective) + @property + def perspective(self): + y = np.tan(self.fovy / 2) + aspect = self.W / self.H + return np.array( + [ + [1 / (y * aspect), 0, 0, 0], + [0, -1 / y, 0, 0], + [ + 0, + 0, + -(self.far + self.near) / (self.far - self.near), + -(2 * self.far * self.near) / (self.far - self.near), + ], + [0, 0, -1, 0], + ], + dtype=np.float32, + ) + + # intrinsics + @property + def intrinsics(self): + focal = self.H / (2 * np.tan(self.fovy / 2)) + return np.array([focal, focal, self.W // 2, self.H // 2], dtype=np.float32) + + @property + def mvp(self): + return self.perspective @ np.linalg.inv(self.pose) # [4, 4] + + def orbit(self, dx, dy): + # rotate along camera up/side axis! + side = self.rot.as_matrix()[:3, 0] + rotvec_x = self.up * np.radians(-0.05 * dx) + rotvec_y = side * np.radians(-0.05 * dy) + self.rot = R.from_rotvec(rotvec_x) * R.from_rotvec(rotvec_y) * self.rot + + def scale(self, delta): + self.radius *= 1.1 ** (-delta) + + def pan(self, dx, dy, dz=0): + # pan in camera coordinate system (careful on the sensitivity!) + self.center += 0.0005 * self.rot.as_matrix()[:3, :3] @ np.array([-dx, -dy, dz]) + +def calculate_fovX(H, W, fovy): + return 2 * np.arctan(np.tan(fovy / 2) * W / H) + +def get_projection_matrix(znear, zfar, fovX, fovY): + tanHalfFovY = math.tan((fovY / 2)) + tanHalfFovX = math.tan((fovX / 2)) + + P = torch.zeros(4, 4) + + z_sign = 1.0 + + P[0, 0] = 1 / tanHalfFovX + P[1, 1] = 1 / tanHalfFovY + P[3, 2] = z_sign + P[2, 2] = z_sign * zfar / (zfar - znear) + P[2, 3] = -(zfar * znear) / (zfar - znear) + return P + + +class MiniCam: + def __init__(self, c2w, width, height, fovy, fovx, znear, zfar, projection_matrix=None): + # c2w (pose) should be in NeRF convention. + + self.image_width = width + self.image_height = height + self.FoVy = fovy + self.FoVx = fovx + self.znear = znear + self.zfar = zfar + + w2c = np.linalg.inv(c2w) + + # rectify... + w2c[1:3, :3] *= -1 + w2c[:3, 3] *= -1 + + self.world_view_transform = torch.tensor(w2c).transpose(0, 1).cuda() + self.projection_matrix = ( + getProjectionMatrix( + znear=self.znear, zfar=self.zfar, fovX=self.FoVx, fovY=self.FoVy + ) + .transpose(0, 1) + .cuda() + ) if projection_matrix is None else projection_matrix + self.full_proj_transform = self.world_view_transform @ self.projection_matrix + self.camera_center = -torch.tensor(c2w[:3, 3]).cuda() \ No newline at end of file diff --git a/shared_utils/common_utils.py b/shared_utils/common_utils.py new file mode 100644 index 0000000..5d447dc --- /dev/null +++ b/shared_utils/common_utils.py @@ -0,0 +1,90 @@ +import os +import sys + +class cstr(str): + # Modified from: WAS Node Suite + class color: + END = '\33[0m' + BOLD = '\33[1m' + ITALIC = '\33[3m' + UNDERLINE = '\33[4m' + BLINK = '\33[5m' + BLINK2 = '\33[6m' + SELECTED = '\33[7m' + + BLACK = '\33[30m' + RED = '\33[31m' + GREEN = '\33[32m' + YELLOW = '\33[33m' + BLUE = '\33[34m' + VIOLET = '\33[35m' + BEIGE = '\33[36m' + WHITE = '\33[37m' + + BLACKBG = '\33[40m' + REDBG = '\33[41m' + GREENBG = '\33[42m' + YELLOWBG = '\33[43m' + BLUEBG = '\33[44m' + VIOLETBG = '\33[45m' + BEIGEBG = '\33[46m' + WHITEBG = '\33[47m' + + GREY = '\33[90m' + LIGHTRED = '\33[91m' + LIGHTGREEN = '\33[92m' + LIGHTYELLOW = '\33[93m' + LIGHTBLUE = '\33[94m' + LIGHTVIOLET = '\33[95m' + LIGHTBEIGE = '\33[96m' + LIGHTWHITE = '\33[97m' + + GREYBG = '\33[100m' + LIGHTREDBG = '\33[101m' + LIGHTGREENBG = '\33[102m' + LIGHTYELLOWBG = '\33[103m' + LIGHTBLUEBG = '\33[104m' + LIGHTVIOLETBG = '\33[105m' + LIGHTBEIGEBG = '\33[106m' + LIGHTWHITEBG = '\33[107m' + + @staticmethod + def add_code(name, code): + if not hasattr(cstr.color, name.upper()): + setattr(cstr.color, name.upper(), code) + else: + raise ValueError(f"'cstr' object already contains a code with the name '{name}'.") + + def __new__(cls, text): + return super().__new__(cls, text) + + def __getattr__(self, attr): + if attr.lower().startswith("_cstr"): + code = getattr(self.color, attr.upper().lstrip("_cstr")) + modified_text = self.replace(f"__{attr[1:]}__", f"{code}") + return cstr(modified_text) + elif attr.upper() in dir(self.color): + code = getattr(self.color, attr.upper()) + modified_text = f"{code}{self}{self.color.END}" + return cstr(modified_text) + elif attr.lower() in dir(cstr): + return getattr(cstr, attr.lower()) + else: + raise AttributeError(f"'cstr' object has no attribute '{attr}'") + + def print(self, **kwargs): + print(self, **kwargs) + +#! MESSAGE TEMPLATES +cstr.color.add_code("msg", f"{cstr.color.BLUE}WAS Node Suite: {cstr.color.END}") +cstr.color.add_code("warning", f"{cstr.color.BLUE}WAS Node Suite {cstr.color.LIGHTYELLOW}[WARNING] {cstr.color.END}") +cstr.color.add_code("error", f"{cstr.color.RED}WAS Node Suite {cstr.color.END}[ERROR] {cstr.color.END}") + +def get_persistent_directory(folder_name): + if sys.platform == "win32": + folder = os.path.join(os.path.expanduser("~"), "AppData", "Local", folder_name) + else: + folder = os.path.join(os.path.expanduser("~"), "." + folder_name) + + os.makedirs(folder, exist_ok=True) + return folder \ No newline at end of file diff --git a/simple-knn/ext.cpp b/simple-knn/ext.cpp new file mode 100644 index 0000000..ae6cefe --- /dev/null +++ b/simple-knn/ext.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#include +#include "spatial.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("distCUDA2", &distCUDA2); +} diff --git a/simple-knn/setup.py b/simple-knn/setup.py new file mode 100644 index 0000000..580d2bd --- /dev/null +++ b/simple-knn/setup.py @@ -0,0 +1,35 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + +from setuptools import setup +from torch.utils.cpp_extension import CUDAExtension, BuildExtension +import os + +cxx_compiler_flags = [] + +if os.name == 'nt': + cxx_compiler_flags.append("/wd4624") + +setup( + name="simple_knn", + ext_modules=[ + CUDAExtension( + name="simple_knn._C", + sources=[ + "spatial.cu", + "simple_knn.cu", + "ext.cpp"], + extra_compile_args={"nvcc": [], "cxx": cxx_compiler_flags}) + ], + cmdclass={ + 'build_ext': BuildExtension + } +) diff --git a/simple-knn/simple_knn.cu b/simple-knn/simple_knn.cu new file mode 100644 index 0000000..e72e4c9 --- /dev/null +++ b/simple-knn/simple_knn.cu @@ -0,0 +1,221 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#define BOX_SIZE 1024 + +#include "cuda_runtime.h" +#include "device_launch_parameters.h" +#include "simple_knn.h" +#include +#include +#include +#include +#include +#include +#define __CUDACC__ +#include +#include + +namespace cg = cooperative_groups; + +struct CustomMin +{ + __device__ __forceinline__ + float3 operator()(const float3& a, const float3& b) const { + return { min(a.x, b.x), min(a.y, b.y), min(a.z, b.z) }; + } +}; + +struct CustomMax +{ + __device__ __forceinline__ + float3 operator()(const float3& a, const float3& b) const { + return { max(a.x, b.x), max(a.y, b.y), max(a.z, b.z) }; + } +}; + +__host__ __device__ uint32_t prepMorton(uint32_t x) +{ + x = (x | (x << 16)) & 0x030000FF; + x = (x | (x << 8)) & 0x0300F00F; + x = (x | (x << 4)) & 0x030C30C3; + x = (x | (x << 2)) & 0x09249249; + return x; +} + +__host__ __device__ uint32_t coord2Morton(float3 coord, float3 minn, float3 maxx) +{ + uint32_t x = prepMorton(((coord.x - minn.x) / (maxx.x - minn.x)) * ((1 << 10) - 1)); + uint32_t y = prepMorton(((coord.y - minn.y) / (maxx.y - minn.y)) * ((1 << 10) - 1)); + uint32_t z = prepMorton(((coord.z - minn.z) / (maxx.z - minn.z)) * ((1 << 10) - 1)); + + return x | (y << 1) | (z << 2); +} + +__global__ void coord2Morton(int P, const float3* points, float3 minn, float3 maxx, uint32_t* codes) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P) + return; + + codes[idx] = coord2Morton(points[idx], minn, maxx); +} + +struct MinMax +{ + float3 minn; + float3 maxx; +}; + +__global__ void boxMinMax(uint32_t P, float3* points, uint32_t* indices, MinMax* boxes) +{ + auto idx = cg::this_grid().thread_rank(); + + MinMax me; + if (idx < P) + { + me.minn = points[indices[idx]]; + me.maxx = points[indices[idx]]; + } + else + { + me.minn = { FLT_MAX, FLT_MAX, FLT_MAX }; + me.maxx = { -FLT_MAX,-FLT_MAX,-FLT_MAX }; + } + + __shared__ MinMax redResult[BOX_SIZE]; + + for (int off = BOX_SIZE / 2; off >= 1; off /= 2) + { + if (threadIdx.x < 2 * off) + redResult[threadIdx.x] = me; + __syncthreads(); + + if (threadIdx.x < off) + { + MinMax other = redResult[threadIdx.x + off]; + me.minn.x = min(me.minn.x, other.minn.x); + me.minn.y = min(me.minn.y, other.minn.y); + me.minn.z = min(me.minn.z, other.minn.z); + me.maxx.x = max(me.maxx.x, other.maxx.x); + me.maxx.y = max(me.maxx.y, other.maxx.y); + me.maxx.z = max(me.maxx.z, other.maxx.z); + } + __syncthreads(); + } + + if (threadIdx.x == 0) + boxes[blockIdx.x] = me; +} + +__device__ __host__ float distBoxPoint(const MinMax& box, const float3& p) +{ + float3 diff = { 0, 0, 0 }; + if (p.x < box.minn.x || p.x > box.maxx.x) + diff.x = min(abs(p.x - box.minn.x), abs(p.x - box.maxx.x)); + if (p.y < box.minn.y || p.y > box.maxx.y) + diff.y = min(abs(p.y - box.minn.y), abs(p.y - box.maxx.y)); + if (p.z < box.minn.z || p.z > box.maxx.z) + diff.z = min(abs(p.z - box.minn.z), abs(p.z - box.maxx.z)); + return diff.x * diff.x + diff.y * diff.y + diff.z * diff.z; +} + +template +__device__ void updateKBest(const float3& ref, const float3& point, float* knn) +{ + float3 d = { point.x - ref.x, point.y - ref.y, point.z - ref.z }; + float dist = d.x * d.x + d.y * d.y + d.z * d.z; + for (int j = 0; j < K; j++) + { + if (knn[j] > dist) + { + float t = knn[j]; + knn[j] = dist; + dist = t; + } + } +} + +__global__ void boxMeanDist(uint32_t P, float3* points, uint32_t* indices, MinMax* boxes, float* dists) +{ + int idx = cg::this_grid().thread_rank(); + if (idx >= P) + return; + + float3 point = points[indices[idx]]; + float best[3] = { FLT_MAX, FLT_MAX, FLT_MAX }; + + for (int i = max(0, idx - 3); i <= min(P - 1, idx + 3); i++) + { + if (i == idx) + continue; + updateKBest<3>(point, points[indices[i]], best); + } + + float reject = best[2]; + best[0] = FLT_MAX; + best[1] = FLT_MAX; + best[2] = FLT_MAX; + + for (int b = 0; b < (P + BOX_SIZE - 1) / BOX_SIZE; b++) + { + MinMax box = boxes[b]; + float dist = distBoxPoint(box, point); + if (dist > reject || dist > best[2]) + continue; + + for (int i = b * BOX_SIZE; i < min(P, (b + 1) * BOX_SIZE); i++) + { + if (i == idx) + continue; + updateKBest<3>(point, points[indices[i]], best); + } + } + dists[indices[idx]] = (best[0] + best[1] + best[2]) / 3.0f; +} + +void SimpleKNN::knn(int P, float3* points, float* meanDists) +{ + float3* result; + cudaMalloc(&result, sizeof(float3)); + size_t temp_storage_bytes; + + float3 init = { 0, 0, 0 }, minn, maxx; + + cub::DeviceReduce::Reduce(nullptr, temp_storage_bytes, points, result, P, CustomMin(), init); + thrust::device_vector temp_storage(temp_storage_bytes); + + cub::DeviceReduce::Reduce(temp_storage.data().get(), temp_storage_bytes, points, result, P, CustomMin(), init); + cudaMemcpy(&minn, result, sizeof(float3), cudaMemcpyDeviceToHost); + + cub::DeviceReduce::Reduce(temp_storage.data().get(), temp_storage_bytes, points, result, P, CustomMax(), init); + cudaMemcpy(&maxx, result, sizeof(float3), cudaMemcpyDeviceToHost); + + thrust::device_vector morton(P); + thrust::device_vector morton_sorted(P); + coord2Morton << <(P + 255) / 256, 256 >> > (P, points, minn, maxx, morton.data().get()); + + thrust::device_vector indices(P); + thrust::sequence(indices.begin(), indices.end()); + thrust::device_vector indices_sorted(P); + + cub::DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, morton.data().get(), morton_sorted.data().get(), indices.data().get(), indices_sorted.data().get(), P); + temp_storage.resize(temp_storage_bytes); + + cub::DeviceRadixSort::SortPairs(temp_storage.data().get(), temp_storage_bytes, morton.data().get(), morton_sorted.data().get(), indices.data().get(), indices_sorted.data().get(), P); + + uint32_t num_boxes = (P + BOX_SIZE - 1) / BOX_SIZE; + thrust::device_vector boxes(num_boxes); + boxMinMax << > > (P, points, indices_sorted.data().get(), boxes.data().get()); + boxMeanDist << > > (P, points, indices_sorted.data().get(), boxes.data().get(), meanDists); + + cudaFree(result); +} \ No newline at end of file diff --git a/simple-knn/simple_knn.h b/simple-knn/simple_knn.h new file mode 100644 index 0000000..3fcfdb8 --- /dev/null +++ b/simple-knn/simple_knn.h @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#ifndef SIMPLEKNN_H_INCLUDED +#define SIMPLEKNN_H_INCLUDED + +class SimpleKNN +{ +public: + static void knn(int P, float3* points, float* meanDists); +}; + +#endif \ No newline at end of file diff --git a/simple-knn/simple_knn/.gitkeep b/simple-knn/simple_knn/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/simple-knn/spatial.cu b/simple-knn/spatial.cu new file mode 100644 index 0000000..1a6a654 --- /dev/null +++ b/simple-knn/spatial.cu @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#include "spatial.h" +#include "simple_knn.h" + +torch::Tensor +distCUDA2(const torch::Tensor& points) +{ + const int P = points.size(0); + + auto float_opts = points.options().dtype(torch::kFloat32); + torch::Tensor means = torch::full({P}, 0.0, float_opts); + + SimpleKNN::knn(P, (float3*)points.contiguous().data(), means.contiguous().data()); + + return means; +} \ No newline at end of file diff --git a/simple-knn/spatial.h b/simple-knn/spatial.h new file mode 100644 index 0000000..280c953 --- /dev/null +++ b/simple-knn/spatial.h @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#include + +torch::Tensor distCUDA2(const torch::Tensor& points); \ No newline at end of file