diff --git a/README.md b/README.md
index 7d1bedc..f2b8542 100644
--- a/README.md
+++ b/README.md
@@ -1,23 +1,23 @@
# ComfyUI-3D-Pack
An extensive node suite that enables ComfyUI to process 3D inputs (Mesh & UV Texture, etc) using cutting edge algorithms (3DGS, NeRF, Differentiable Rendering, SDS/VSD Optimization, etc.)
+
+Features —
+Roadmap —
+Install —
+Run —
+Tips
+
+
### Note: this project is still a WIP and not been released into ComFyUI package database yet
-ps: I'll show some generated result soon
-
----
-
@@ -32,45 +32,71 @@ ps: I'll show some generated result soon
-- Stack Orbit Camera Poses node to automatically generate all range of camera pose combinations
+- **Stack Orbit Camera Poses**: Automatically generate all range of camera pose combinations
- You can use it to conditioning the [StableZero123 (You need to Download the checkpoint first)](https://comfyanonymous.github.io/ComfyUI_examples/3d/), with full range of camera poses in one prompt pass
- You can use it to generate the orbit camera poses and directly input to other 3D process node (e.g. GaussianSplatting and BakeTextureToMesh)
- Example usage:
+
+
+
+ Loading scene...
+
+
+
+
+
+
+
+
+
+
+ Loading scene...
+
+
+
+
+
+
+
+
diff --git a/web/js/LittlestTokyo.glb b/web/js/LittlestTokyo.glb
new file mode 100644
index 0000000..f2c7e04
Binary files /dev/null and b/web/js/LittlestTokyo.glb differ
diff --git a/web/js/gsVisualizer.js b/web/js/gsVisualizer.js
new file mode 100644
index 0000000..6c9005d
--- /dev/null
+++ b/web/js/gsVisualizer.js
@@ -0,0 +1,74 @@
+import * as SPLAT from 'gsplat';
+import { api } from '/scripts/api.js'
+
+const visualizer = document.getElementById("visualizer");
+const canvas = document.getElementById("canvas");
+const progressDialog = document.getElementById("progress-dialog");
+const progressIndicator = document.getElementById("progress-indicator");
+
+const renderer = new SPLAT.WebGLRenderer(canvas);
+const scene = new SPLAT.Scene();
+const camera = new SPLAT.Camera();
+const controls = new SPLAT.OrbitControls(camera, canvas);
+controls.orbitSpeed = 3.0;
+
+// Handle window reseize event
+const handleResize = () => {
+ renderer.setSize(window.innerWidth, window.innerHeight);
+};
+handleResize();
+window.addEventListener("resize", handleResize);
+
+
+var lastFilepath = "";
+var needUpdate = false;
+
+function frameUpdate() {
+
+ var filepath = visualizer.getAttribute("filepath");
+ if (filepath == lastFilepath){
+ if (needUpdate){
+ controls.update();
+ renderer.render( scene, camera );
+ }
+ requestAnimationFrame( frameUpdate );
+ } else {
+ needUpdate = false;
+ scene.reset();
+ progressDialog.open = true;
+ lastFilepath = filepath;
+ main(lastFilepath);
+ }
+};
+
+const onProgress = function ( progress ) {
+ progressIndicator.value = progress * 100;
+};
+
+async function main(filepath="") {
+ // Check if file name is valid
+ if (/^.+\.[a-zA-Z]+$/.test(filepath)){
+
+ let params = {"filepath": filepath};
+ const url = api.apiURL('/viewfile?' + new URLSearchParams(params));
+ var splat = null;
+
+ var fileExt = filepath.split('.').pop().toLowerCase();
+ if (fileExt == "ply"){
+ splat = await SPLAT.PLYLoader.LoadAsync(url, scene, onProgress);
+ } else if (fileExt == "splat") {
+ splat = await SPLAT.Loader.LoadAsync(url, scene, onProgress);
+ } else {
+ throw new Error(`File extension name has to be either .ply or .splat, got .${fileExt}`);
+ }
+
+ needUpdate = true;
+ }
+
+ progressDialog.close();
+
+ frameUpdate();
+}
+
+//main("C:/Users/reall/Softwares/ComfyUI_windows_portable/ComfyUI/output/bonsai.splat");
+main();
\ No newline at end of file
diff --git a/web/js/threeVisualizer.js b/web/js/threeVisualizer.js
new file mode 100644
index 0000000..aca3e4a
--- /dev/null
+++ b/web/js/threeVisualizer.js
@@ -0,0 +1,147 @@
+import * as THREE from 'three';
+import { api } from '/scripts/api.js'
+
+import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
+import { RoomEnvironment } from 'three/addons/environments/RoomEnvironment.js';
+
+import { MTLLoader } from 'three/addons/loaders/MTLLoader.js';
+import { OBJLoader } from 'three/addons/loaders/OBJLoader.js';
+import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
+import { DRACOLoader } from 'three/addons/loaders/DRACOLoader.js';
+
+const visualizer = document.getElementById("visualizer");
+const container = document.getElementById( 'container' );
+const progressDialog = document.getElementById("progress-dialog");
+const progressIndicator = document.getElementById("progress-indicator");
+
+const renderer = new THREE.WebGLRenderer( { antialias: true } );
+renderer.setPixelRatio( window.devicePixelRatio );
+renderer.setSize( window.innerWidth, window.innerHeight );
+container.appendChild( renderer.domElement );
+
+const pmremGenerator = new THREE.PMREMGenerator( renderer );
+
+// scene
+const scene = new THREE.Scene();
+scene.background = new THREE.Color( 0x000000 );
+scene.environment = pmremGenerator.fromScene( new RoomEnvironment( renderer ), 0.04 ).texture;
+
+const ambientLight = new THREE.AmbientLight( 0xffffff );
+
+const camera = new THREE.PerspectiveCamera( 40, window.innerWidth / window.innerHeight, 1, 100 );
+camera.position.set( 5, 2, 8 );
+const pointLight = new THREE.PointLight( 0xffffff, 15 );
+camera.add( pointLight );
+
+const controls = new OrbitControls( camera, renderer.domElement );
+controls.target.set( 0, 0.5, 0 );
+controls.update();
+controls.enablePan = true;
+controls.enableDamping = true;
+
+// Handle window reseize event
+window.onresize = function () {
+
+ camera.aspect = window.innerWidth / window.innerHeight;
+ camera.updateProjectionMatrix();
+
+ renderer.setSize( window.innerWidth, window.innerHeight );
+
+};
+
+
+var lastFilepath = "";
+var needUpdate = false;
+
+function frameUpdate() {
+
+ var filepath = visualizer.getAttribute("filepath");
+ if (filepath == lastFilepath){
+ if (needUpdate){
+ controls.update();
+ renderer.render( scene, camera );
+ }
+ requestAnimationFrame( frameUpdate );
+ } else {
+ needUpdate = false;
+ scene.clear();
+ progressDialog.open = true;
+ lastFilepath = filepath;
+ main(lastFilepath);
+ }
+}
+
+const onProgress = function ( xhr ) {
+ if ( xhr.lengthComputable ) {
+ progressIndicator.value = xhr.loaded / xhr.total * 100;
+ }
+};
+const onError = function ( e ) {
+ console.error( e );
+};
+
+async function main(filepath="") {
+ // Check if file name is valid
+ if (/^.+\.[a-zA-Z]+$/.test(filepath)){
+
+ let params = {"filepath": filepath};
+ const url = api.apiURL('/viewfile?' + new URLSearchParams(params));
+
+ var filepathSplit = filepath.split('.');
+ var fileExt = filepathSplit.pop().toLowerCase();
+ var filepathNoExt = filepathSplit.join(".");
+
+ if (fileExt == "obj"){
+ const loader = new OBJLoader();
+
+ var mtlFolderpath = filepath.substring(0, Math.max(filepath.lastIndexOf("/"), filepath.lastIndexOf("\\"))) + "/";
+ var mtlFilepath = filepathNoExt.replace(/^.*[\\\/]/, '') + ".mtl";
+
+ const mtlLoader = new MTLLoader();
+ mtlLoader.setPath(api.apiURL('/viewfile?' + new URLSearchParams({"filepath": mtlFolderpath})));
+ mtlLoader.load( mtlFilepath, function ( mtl ) {
+ mtl.preload();
+ loader.setMaterials( mtl );
+ }, onProgress, onError );
+
+ loader.load( url, function ( obj ) {
+ obj.scale.setScalar( 5 );
+ scene.add( obj );
+
+ }, onProgress, onError );
+
+
+ } else if (fileExt == "glb") {
+ const dracoLoader = new DRACOLoader();
+ dracoLoader.setDecoderPath( 'https://unpkg.com/three@latest/examples/jsm/libs/draco/gltf/' );
+ const loader = new GLTFLoader();
+ loader.setDRACOLoader( dracoLoader );
+
+ loader.load( url, function ( gltf ) {
+ const model = gltf.scene;
+ model.position.set( 1, 1, 0 );
+ model.scale.set( 0.01, 0.01, 0.01 );
+
+ scene.add( model );
+
+ }, onProgress, onError );
+
+ } else if (fileExt == "ply") {
+
+ } else {
+ throw new Error(`File extension name has to be either .ply or .splat, got .${fileExt}`);
+ }
+
+ needUpdate = true;
+ }
+
+ scene.add( ambientLight );
+ scene.add( camera );
+
+ progressDialog.close();
+
+ frameUpdate();
+}
+
+//main("C:/Users/reall/Softwares/ComfyUI_windows_portable/ComfyUI/output/MeshTest/Mesh_01.obj");
+main();
\ No newline at end of file
diff --git a/web/style/gsStyle.css b/web/style/gsStyle.css
new file mode 100644
index 0000000..997fab4
--- /dev/null
+++ b/web/style/gsStyle.css
@@ -0,0 +1,12 @@
+body,
+html {
+ margin: 0;
+ padding: 0;
+ overflow: hidden;
+ background-color: #000;
+}
+
+canvas {
+ width: 100vw;
+ height: 100vh;
+}
\ No newline at end of file
diff --git a/web/style/progressStyle.css b/web/style/progressStyle.css
new file mode 100644
index 0000000..91fc468
--- /dev/null
+++ b/web/style/progressStyle.css
@@ -0,0 +1,36 @@
+dialog {
+ width: 100%;
+ text-align: center;
+ max-width: 20em;
+ color: white;
+ background-color: #000;
+ border: none;
+ position: relative;
+ transform: translate(-50%, -50%);
+}
+
+#progress-container {
+ position: absolute;
+ top: 50%;
+ left: 50%;
+}
+
+progress {
+ width: 100%;
+ height: 1em;
+ border: none;
+ background-color: #fff;
+ color: #eee;
+}
+
+progress::-webkit-progress-bar {
+ background-color: #333;
+}
+
+progress::-webkit-progress-value {
+ background-color: #eee;
+}
+
+progress::-moz-progress-bar {
+ background-color: #eee;
+}
\ No newline at end of file
diff --git a/web/style/threeStyle.css b/web/style/threeStyle.css
new file mode 100644
index 0000000..dab2f09
--- /dev/null
+++ b/web/style/threeStyle.css
@@ -0,0 +1,91 @@
+body {
+ margin: 0;
+ background-color: #000;
+ color: #fff;
+ font-family: Monospace;
+ font-size: 13px;
+ line-height: 24px;
+ overscroll-behavior: none;
+}
+
+a {
+ color: #ff0;
+ text-decoration: none;
+}
+
+a:hover {
+ text-decoration: underline;
+}
+
+button {
+ cursor: pointer;
+ text-transform: uppercase;
+}
+
+#info {
+ position: absolute;
+ top: 0px;
+ width: 100%;
+ padding: 10px;
+ box-sizing: border-box;
+ text-align: center;
+ -moz-user-select: none;
+ -webkit-user-select: none;
+ -ms-user-select: none;
+ user-select: none;
+ pointer-events: none;
+ z-index: 1; /* TODO Solve this in HTML */
+}
+
+a, button, input, select {
+ pointer-events: auto;
+}
+
+.lil-gui {
+ z-index: 2 !important; /* TODO Solve this in HTML */
+}
+
+@media all and ( max-width: 640px ) {
+ .lil-gui.root {
+ right: auto;
+ top: auto;
+ max-height: 50%;
+ max-width: 80%;
+ bottom: 0;
+ left: 0;
+ }
+}
+
+#overlay {
+ position: absolute;
+ font-size: 16px;
+ z-index: 2;
+ top: 0;
+ left: 0;
+ width: 100%;
+ height: 100%;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ flex-direction: column;
+ background: rgba(0,0,0,0.7);
+}
+
+ #overlay button {
+ background: transparent;
+ border: 0;
+ border: 1px solid rgb(255, 255, 255);
+ border-radius: 4px;
+ color: #ffffff;
+ padding: 12px 18px;
+ text-transform: uppercase;
+ cursor: pointer;
+ }
+
+#notSupported {
+ width: 50%;
+ margin: auto;
+ background-color: #f00;
+ margin-top: 20px;
+ padding: 10px;
+}
\ No newline at end of file
diff --git a/web/visualization.js b/web/visualization.js
new file mode 100644
index 0000000..ab527a6
--- /dev/null
+++ b/web/visualization.js
@@ -0,0 +1,165 @@
+import { app } from "/scripts/app.js"
+
+class Visualizer {
+ constructor(node, container, visualSrc) {
+ this.node = node
+
+ this.iframe = document.createElement('iframe')
+ Object.assign(this.iframe, {
+ scrolling: "no",
+ overflow: "hidden",
+ })
+ this.iframe.src = "/extensions/ComfyUI-3D-Pack/html/" + visualSrc + ".html"
+ container.appendChild(this.iframe)
+ }
+
+ updateVisual(filepath) {
+ const iframeDocument = this.iframe.contentWindow.document
+ const previewScript = iframeDocument.getElementById('visualizer')
+ previewScript.setAttribute("filepath", filepath)
+ }
+
+ remove() {
+ this.container.remove()
+ }
+}
+
+function createVisualizer(node, inputName, typeName, inputData, app) {
+ node.name = inputName
+
+ const widget = {
+ type: typeName,
+ name: "preview3d",
+ callback: () => {},
+ draw : function(ctx, node, widgetWidth, widgetY, widgetHeight) {
+ const margin = 10
+ const top_offset = 5
+ const visible = app.canvas.ds.scale > 0.5 && this.type === typeName
+ const w = widgetWidth - margin * 4
+ const clientRectBound = ctx.canvas.getBoundingClientRect()
+ const transform = new DOMMatrix()
+ .scaleSelf(
+ clientRectBound.width / ctx.canvas.width,
+ clientRectBound.height / ctx.canvas.height
+ )
+ .multiplySelf(ctx.getTransform())
+ .translateSelf(margin, margin + widgetY)
+
+ Object.assign(this.visualizer.style, {
+ left: `${transform.a * margin + transform.e}px`,
+ top: `${transform.d + transform.f + top_offset}px`,
+ width: `${(w * transform.a)}px`,
+ height: `${(w * transform.d - widgetHeight - (margin * 15) * transform.d)}px`,
+ position: "absolute",
+ overflow: "hidden",
+ zIndex: app.graph._nodes.indexOf(node),
+ })
+
+ Object.assign(this.visualizer.children[0].style, {
+ transformOrigin: "50% 50%",
+ width: '100%',
+ height: '100%',
+ border: '0 none',
+ })
+
+ this.visualizer.hidden = !visible
+ },
+ }
+
+ const container = document.createElement('div')
+ container.id = `Comfy3D_${inputName}`
+
+ node.visualizer = new Visualizer(node, container, typeName)
+ widget.visualizer = container
+ widget.parent = node
+
+ document.body.appendChild(widget.visualizer)
+
+ node.addCustomWidget(widget)
+
+ node.updateParameters = (params) => {
+ node.visualizer.updateVisual(params.filepath)
+ }
+
+ // Events for drawing backgound
+ node.onDrawBackground = function (ctx) {
+ if (!this.flags.collapsed) {
+ node.visualizer.iframe.hidden = false
+ } else {
+ node.visualizer.iframe.hidden = true
+ }
+ }
+
+ // Make sure visualization iframe is always inside the node when resize the node
+ node.onResize = function () {
+ let [w, h] = this.size
+ if (w <= 600) w = 600
+ if (h <= 500) h = 500
+
+ if (w > 600) {
+ h = w - 100
+ }
+
+ this.size = [w, h]
+ }
+
+ // Events for remove nodes
+ node.onRemoved = () => {
+ for (let w in node.widgets) {
+ if (node.widgets[w].visualizer) {
+ node.widgets[w].visualizer.remove()
+ }
+ }
+ }
+
+
+ return {
+ widget: widget,
+ }
+}
+
+function registerVisualizer(nodeType, nodeData, nodeClassName, typeName){
+ if (nodeData.name == nodeClassName) {
+ console.log("[3D Visualizer] Registering node: " + nodeData.name)
+
+ const onNodeCreated = nodeType.prototype.onNodeCreated
+
+ nodeType.prototype.onNodeCreated = async function() {
+ const r = onNodeCreated
+ ? onNodeCreated.apply(this, arguments)
+ : undefined
+
+ let Preview3DNode = app.graph._nodes.filter(
+ (wi) => wi.type == nodeClassName
+ )
+ let nodeName = `Preview3DNode_${Preview3DNode.length}`
+
+ console.log(`[Comfy3D] Create: ${nodeName}`)
+
+ const result = await createVisualizer.apply(this, [this, nodeName, typeName, {}, app])
+
+ this.setSize([600, 500])
+
+ return r
+ }
+
+ nodeType.prototype.onExecuted = async function(message) {
+ if (message?.previews) {
+ this.updateParameters(message.previews[0])
+ }
+ }
+ }
+}
+
+app.registerExtension({
+ name: "Mr.ForExample.Visualizer.GS",
+
+ async init (app) {
+
+ },
+
+ async beforeRegisterNodeDef(nodeType, nodeData, app) {
+ registerVisualizer(nodeType, nodeData, "[Comfy3D] Preview 3DGS", "gsVisualizer")
+ registerVisualizer(nodeType, nodeData, "[Comfy3D] Preview 3DMesh", "threeVisualizer")
+ },
+})
\ No newline at end of file
diff --git a/webserver/server.py b/webserver/server.py
new file mode 100644
index 0000000..cb9d3f2
--- /dev/null
+++ b/webserver/server.py
@@ -0,0 +1,34 @@
+import server
+import folder_paths as comfy_paths
+import os
+import time
+import subprocess
+
+from shared_utils.common_utils import cstr
+
+web = server.web
+
+SUPPORTED_VIEW_EXTENSIONS = (
+ '.png',
+ '.jpg',
+ '.jpeg ',
+ '.mtl',
+ '.obj',
+ '.glb',
+ '.ply',
+ '.splat'
+)
+
+@server.PromptServer.instance.routes.get("/extensions/ComfyUI-3D-Pack/html/viewfile")
+async def view_file(request):
+ query = request.rel_url.query
+ # Security check to see if query client is local
+ if request.remote == "127.0.0.1" and "filepath" in query:
+ filepath = query["filepath"]
+
+ cstr(f"[Server Query view_file] Get file {filepath}").msg.print()
+
+ if filepath.lower().endswith(SUPPORTED_VIEW_EXTENSIONS) and os.path.exists(filepath):
+ return web.FileResponse(filepath)
+
+ return web.Response(status=404)
-- Load 3D file (.obj, .ply, .glb)
- - Also have node that support save and load 3DGS .ply file
-- 3D Gaussian Splatting, with:
+ - Coordinate system:
+ - Azimuth: In top view, from angle 0 rotate 360 degree with step -90 you get (0, -90, -180/180, 90, 0), in this case camera rotates clock-wise, vice versa.
+ - Elevation: 0 when camera points horizontally forward, pointing down to the ground is negitive angle, vice versa.
+
+- **3D Gaussian Splatting**
- [Improved Differential Gaussian Rasterization](https://github.com/ashawkey/diff-gaussian-rasterization)
- Better Compactness-based Densification method from [Gsgen](https://gsgen3d.github.io/),
- Support initialize gaussians from given 3D mesh (Optional)
- Support mini-batch optimazation
- Multi-View images as inputs
- - Export to .ply support
-
+ - Export to standard 3DGS .ply format supported
+
+- **Gaussian Splatting Orbit Renderer**
+ - Render 3DGS to images sequences or video, given a 3DGS file and camera poses generated by **Stack Orbit Camera Poses** node
+
- Bake Multi-View images into UVTexture of given 3D mesh using [Nvdiffrast](https://github.com/NVlabs/nvdiffrast), supports:
- Export to .obj, .ply, .glb
-- Deep Marching Tetrahedrons
+- **Deep Marching Tetrahedrons**
- Allow convert 3DGS .ply file to 3D mesh