From 992782c1fcd1b8b32ded7a6b5fab57ea2a5fce4b Mon Sep 17 00:00:00 2001 From: Bruno Fargnoli Date: Thu, 21 May 2026 00:13:29 +0200 Subject: [PATCH] Fixed Pixal3D --- README.md | 1 + .../MeshWithTexturing_Pixal3D.json | 1694 ++++++++--------- nodes.py | 122 +- trellis2/modules/sparse/conv/config.py | 2 +- trellis2/pipelines/trellis2_image_to_3d.py | 114 +- .../mixins/image_conditioned_proj.py | 4 +- trellis2/utils/camera.py | 2 +- 7 files changed, 996 insertions(+), 943 deletions(-) diff --git a/README.md b/README.md index 79d4825..5ccb1b8 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ | Date | Description | | --- | --- | +| **2026-05-20** | Fixed Pixal3D
The node "Mesh with Voxel Advanced Generator" is compatible with Pixal3D | | **2026-05-13** | Added support for Pixal3D-T model
It's not compatible with all nodes
Check in the folder example_workflows | | **2026-04-20** | Recreated all Workflows | | **2026-04-20** | Added node "Sparse MultiView Generator"
Added node "ImageCond MultiView Generator"
Added node "Shape MultiView Generator"
Added node "Shape Cascade MultiView Generator"
Added node "Tex Slat MultiView Generator" | diff --git a/example_workflows/MeshWithTexturing_Pixal3D.json b/example_workflows/MeshWithTexturing_Pixal3D.json index ddc0231..e676676 100644 --- a/example_workflows/MeshWithTexturing_Pixal3D.json +++ b/example_workflows/MeshWithTexturing_Pixal3D.json @@ -1,200 +1,28 @@ { "id": "0cc31dc7-54c1-494d-b20e-d9f4aabc66bc", "revision": 0, - "last_node_id": 296, - "last_link_id": 593, + "last_node_id": 302, + "last_link_id": 628, "nodes": [ - { - "id": 193, - "type": "Trellis2FillHolesWithCuMesh", - "pos": [ - 184.40535063876197, - 149.86491249810132 - ], - "size": [ - 312.4361328125, - 58 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "link": 397 - } - ], - "outputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "links": [ - 356 - ] - } - ], - "properties": { - "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "dc66f263e832b103a81a021bf6cd53eec8ff28af", - "Node name for S&R": "Trellis2FillHolesWithCuMesh" - }, - "widgets_values": [ - 1 - ] - }, - { - "id": 161, - "type": "Trellis2ReconstructMeshWithQuad", - "pos": [ - 189.04847750020792, - 267.26177075303883 - ], - "size": [ - 331.5878996659427, - 142.11360851901668 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "link": 356 - } - ], - "outputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "links": [ - 398 - ] - } - ], - "properties": { - "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "d4d66cfc9f1ce5b1c3b9a2e504ee4b24ffad48a5", - "Node name for S&R": "Trellis2ReconstructMeshWithQuad", - "widget_ue_connectable": {} - }, - "widgets_values": [ - 1, - 1024, - true, - true - ] - }, - { - "id": 218, - "type": "Trellis2SimplifyMesh", - "pos": [ - 219.30428174016674, - 473.8612441499314 - ], - "size": [ - 270, - 82 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "link": 398 - }, - { - "name": "target_face_num", - "type": "INT", - "widget": { - "name": "target_face_num" - }, - "link": 454 - } - ], - "outputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "links": [ - 463 - ] - } - ], - "properties": { - "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "ef2286b47ce0ef0e681dffe0209aadd96de50f72", - "Node name for S&R": "Trellis2SimplifyMesh" - }, - "widgets_values": [ - 500000, - "Cumesh" - ] - }, - { - "id": 250, - "type": "Trellis2FillHolesNicelyWithMeshlib", - "pos": [ - 198.6610452102749, - 613.5101575092269 - ], - "size": [ - 328.5106969603829, - 46 - ], - "flags": {}, - "order": 14, - "mode": 4, - "inputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "link": 463 - } - ], - "outputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "links": [ - 488 - ] - }, - { - "name": "holes_filled", - "type": "INT", - "links": null - } - ], - "properties": { - "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "9d1f80ec9b4d5c4b966e7595ee99e01a1d0c9f19", - "Node name for S&R": "Trellis2FillHolesNicelyWithMeshlib" - }, - "widgets_values": [] - }, { "id": 203, "type": "Trellis2ExportMesh", "pos": [ - 170.5575082791613, - 1283.095561654488 + 197.09343630579113, + 1339.3518340228675 ], "size": [ 270, 102 ], "flags": {}, - "order": 17, + "order": 16, "mode": 0, "inputs": [ { "name": "trimesh", "type": "TRIMESH", - "link": 574 + "link": 628 }, { "name": "filename_prefix", @@ -230,186 +58,48 @@ ] }, { - "id": 257, - "type": "Trellis2SimplifyMesh", + "id": 194, + "type": "Trellis2PreProcessImage", "pos": [ - 222.22584571304807, - 705.7312451177193 + -1218.495705914309, + 437.03938555658084 ], "size": [ - 270, - 82 + 281.7837890625, + 106 ], "flags": {}, - "order": 15, - "mode": 4, - "inputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "link": 488 - }, - { - "name": "target_face_num", - "type": "INT", - "widget": { - "name": "target_face_num" - }, - "link": 487 - } - ], - "outputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "links": [ - 572 - ] - } - ], - "properties": { - "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "ef2286b47ce0ef0e681dffe0209aadd96de50f72", - "Node name for S&R": "Trellis2SimplifyMesh" - }, - "widgets_values": [ - 500000, - "Cumesh" - ] - }, - { - "id": 217, - "type": "Trellis2DecodeLatents", - "pos": [ - -48.8569261943387, - 548.0253905072441 - ], - "size": [ - 210, - 122 - ], - "flags": {}, - "order": 10, + "order": 4, "mode": 0, "inputs": [ { - "name": "pipeline", - "type": "TRELLIS2PIPELINE", - "link": 592 - }, - { - "name": "shape_slat", - "type": "SHAPE_SLAT", - "link": 536 - }, - { - "name": "texture_slat", - "shape": 7, - "type": "TEXTURE_SLAT", - "link": 591 - }, - { - "name": "resolution", - "type": "INT", - "widget": { - "name": "resolution" - }, - "link": 538 + "name": "image", + "type": "IMAGE", + "link": 344 } ], "outputs": [ { - "name": "mesh", - "type": "MESHWITHVOXEL", + "name": "image", + "type": "IMAGE", "links": [ - 397 + 394, + 515, + 522, + 589, + 621 ] - }, - { - "name": "bvh", - "type": "BVH", - "links": [ - 573 - ] - }, - { - "name": "pipeline", - "type": "TRELLIS2PIPELINE", - "links": [] } ], "properties": { "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "ef2286b47ce0ef0e681dffe0209aadd96de50f72", - "Node name for S&R": "Trellis2DecodeLatents" + "ver": "7ce26deae425d114f3deb78e802d6196e5987a4a", + "Node name for S&R": "Trellis2PreProcessImage" }, "widgets_values": [ - 0, - true - ] - }, - { - "id": 290, - "type": "Trellis2UnWrapAndRasterizer", - "pos": [ - 104.1595211548057, - 859.3392446479718 - ], - "size": [ - 419.15234375, - 338 - ], - "flags": {}, - "order": 16, - "mode": 0, - "inputs": [ - { - "name": "mesh", - "type": "MESHWITHVOXEL", - "link": 572 - }, - { - "name": "bvh", - "type": "BVH", - "link": 573 - } - ], - "outputs": [ - { - "name": "trimesh", - "type": "TRIMESH", - "links": [ - 574 - ] - }, - { - "name": "base_color_texture", - "type": "IMAGE", - "links": null - }, - { - "name": "metallic_roughness_texture", - "type": "IMAGE", - "links": null - } - ], - "properties": { - "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "86d13d9eac4a2bd4395954c7184d3aa4fa81a9d8", - "Node name for S&R": "Trellis2UnWrapAndRasterizer" - }, - "widgets_values": [ - 60, - 0, - 1, - 1, - 2048, - "OPAQUE", + 5, false, - false, - false, - "telea", - "None" + 1024 ] }, { @@ -444,15 +134,15 @@ "type": "IMAGE_COND", "links": [ 521, - 526 + 619 ] }, { "name": "cond_1024", "type": "IMAGE_COND", "links": [ - 530, - 587 + 587, + 618 ] }, { @@ -466,10 +156,10 @@ "name": "moge_camera_config", "type": "MOGE_CAM_CONFIG", "links": [ - 579, 580, 582, - 590 + 590, + 622 ] } ], @@ -482,87 +172,6 @@ 1 ] }, - { - "id": 232, - "type": "Preview3D", - "pos": [ - 584.4700506848733, - 118.27654774792936 - ], - "size": [ - 1196.026883643203, - 1247.4148518428156 - ], - "flags": {}, - "order": 18, - "mode": 0, - "inputs": [ - { - "name": "camera_info", - "shape": 7, - "type": "LOAD3D_CAMERA", - "link": null - }, - { - "name": "bg_image", - "shape": 7, - "type": "IMAGE", - "link": null - }, - { - "name": "model_file", - "type": "STRING,FILE_3D_GLB,FILE_3D_GLTF,FILE_3D_FBX,FILE_3D_OBJ,FILE_3D_STL,FILE_3D_USDZ,FILE_3D", - "widget": { - "name": "model_file" - }, - "link": 429 - } - ], - "outputs": [], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.16.4", - "Node name for S&R": "Preview3D", - "Last Time Model File": "C:/Git/ComfyUI/output/Cyberman_P3D_00001_.glb", - "Resource Folder": "Git/ComfyUI/output", - "Scene Config": { - "showGrid": true, - "backgroundColor": "#282828", - "backgroundImage": "", - "backgroundRenderMode": "tiled" - }, - "Camera Config": { - "cameraType": "perspective", - "fov": 37, - "state": { - "position": { - "x": -0.797953149079233, - "y": 2.996032511737236, - "z": 7.973882397278798 - }, - "target": { - "x": 0.2576038454794706, - "y": 2.541570255543307, - "z": -0.43926597484543517 - }, - "zoom": 1, - "cameraType": "perspective" - } - }, - "Light Config": { - "intensity": 4.16 - }, - "Model Config": { - "upDirection": "original", - "materialMode": "normal", - "showSkeleton": false - } - }, - "widgets_values": [ - "C:/Git/ComfyUI/output/Cyberman_P3D_00001_.glb", - "" - ] - }, { "id": 39, "type": "Trellis2LoadModel", @@ -614,48 +223,318 @@ ] }, { - "id": 194, - "type": "Trellis2PreProcessImage", + "id": 209, + "type": "PrimitiveInt", "pos": [ - -1218.495705914309, - 437.03938555658084 + -1845.7531239022012, + 263.9724281177611 ], "size": [ - 281.7837890625, - 106 + 270, + 82 ], "flags": {}, - "order": 4, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 454 + ] + } + ], + "title": "Target Face Number", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "PrimitiveInt" + }, + "widgets_values": [ + 500000, + "fixed" + ] + }, + { + "id": 217, + "type": "Trellis2DecodeLatents", + "pos": [ + -1.720577411358802, + 450.11445763826464 + ], + "size": [ + 210, + 122 + ], + "flags": {}, + "order": 10, "mode": 0, "inputs": [ { - "name": "image", - "type": "IMAGE", - "link": 344 + "name": "pipeline", + "type": "TRELLIS2PIPELINE", + "link": 614 + }, + { + "name": "shape_slat", + "type": "SHAPE_SLAT", + "link": 617 + }, + { + "name": "texture_slat", + "shape": 7, + "type": "TEXTURE_SLAT", + "link": 615 + }, + { + "name": "resolution", + "type": "INT", + "widget": { + "name": "resolution" + }, + "link": 616 } ], "outputs": [ { - "name": "image", - "type": "IMAGE", + "name": "mesh", + "type": "MESHWITHVOXEL", "links": [ - 394, - 515, - 522, - 532, - 589 + 625 + ] + }, + { + "name": "bvh", + "type": "BVH", + "links": [ + 627 + ] + }, + { + "name": "pipeline", + "type": "TRELLIS2PIPELINE", + "links": [] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Trellis2", + "ver": "ef2286b47ce0ef0e681dffe0209aadd96de50f72", + "Node name for S&R": "Trellis2DecodeLatents" + }, + "widgets_values": [ + 0, + true + ] + }, + { + "id": 219, + "type": "PrimitiveString", + "pos": [ + -1858.1285668527714, + 133.64378105828288 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 402 + ] + } + ], + "title": "Name", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "PrimitiveString" + }, + "widgets_values": [ + "DragonRing_P3D_1024" + ] + }, + { + "id": 215, + "type": "Trellis2ShapeGenerator", + "pos": [ + -841.6517809699204, + 940.6150509649334 + ], + "size": [ + 335.24609375, + 402 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "TRELLIS2PIPELINE", + "link": 525 + }, + { + "name": "image_cond", + "type": "IMAGE_COND", + "link": 619 + }, + { + "name": "coords", + "type": "COORDS", + "link": 524 + }, + { + "name": "image", + "shape": 7, + "type": "IMAGE", + "link": 515 + }, + { + "name": "moge_camera_config", + "shape": 7, + "type": "MOGE_CAM_CONFIG", + "link": 582 + } + ], + "outputs": [ + { + "name": "shape_slat", + "type": "SHAPE_SLAT", + "links": [ + 620 + ] + }, + { + "name": "resolution", + "type": "INT", + "links": [ + 609 + ] + }, + { + "name": "pipeline", + "type": "TRELLIS2PIPELINE", + "links": [ + 608 ] } ], "properties": { "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "7ce26deae425d114f3deb78e802d6196e5987a4a", - "Node name for S&R": "Trellis2PreProcessImage" + "ver": "ef2286b47ce0ef0e681dffe0209aadd96de50f72", + "Node name for S&R": "Trellis2ShapeGenerator" }, "widgets_values": [ - 5, + 512, + 12, + 7.5, + 0.5, + 3, + "euler", + 0.6, + 1, false, - 1024 + 0, + 2, + 0.92 + ] + }, + { + "id": 296, + "type": "Trellis2TexSlatGenerator", + "pos": [ + -378.5116589100683, + 664.4975815766863 + ], + "size": [ + 346.18029615893147, + 426 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "TRELLIS2PIPELINE", + "link": 613 + }, + { + "name": "image_cond", + "type": "IMAGE_COND", + "link": 587 + }, + { + "name": "shape_slat", + "type": "SHAPE_SLAT", + "link": 611 + }, + { + "name": "image", + "shape": 7, + "type": "IMAGE", + "link": 589 + }, + { + "name": "moge_camera_config", + "shape": 7, + "type": "MOGE_CAM_CONFIG", + "link": 590 + }, + { + "name": "from_resolution", + "shape": 7, + "type": "INT", + "widget": { + "name": "from_resolution" + }, + "link": 612 + } + ], + "outputs": [ + { + "name": "texture_slat", + "type": "TEXTURE_SLAT", + "links": [ + 615 + ] + }, + { + "name": "pipeline", + "type": "TRELLIS2PIPELINE", + "links": [ + 614 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Trellis2", + "ver": "fe543271876ff0db265c7b5989a1060362476696", + "Node name for S&R": "Trellis2TexSlatGenerator" + }, + "widgets_values": [ + 1024, + 12, + 1, + 0, + 3, + "euler", + 0.6, + 0.9, + false, + 0, + 2, + 0.92, + 1536 ] }, { @@ -708,7 +587,7 @@ "name": "sparse_structure_resolution", "type": "INT", "links": [ - 535 + 610 ] }, { @@ -727,65 +606,81 @@ "widgets_values": [ 1000, "fixed", - 50, + 12, 7.5, - 0.01, + 0.7, 5, - "heun", + "euler", 32, - 0.1, - 1, - true, + 0.6, 1, false, - 0.65, + 1, + false, + 0, 2, "flood_fill", 0.92, - true + false ] }, { - "id": 215, - "type": "Trellis2ShapeGenerator", + "id": 300, + "type": "Trellis2ShapeCascadeGenerator", "pos": [ - -841.1718308975657, - 945.0367148651351 + -383.6869870959995, + 132.63013165669145 ], "size": [ - 335.24609375, - 402 + 335.9791015625, + 474 ], "flags": {}, - "order": 7, + "order": 8, "mode": 0, "inputs": [ { "name": "pipeline", "type": "TRELLIS2PIPELINE", - "link": 525 + "link": 608 }, { "name": "image_cond", "type": "IMAGE_COND", - "link": 526 + "link": 618 }, { - "name": "coords", - "type": "COORDS", - "link": 524 + "name": "shape_slat", + "type": "SHAPE_SLAT", + "link": 620 }, { "name": "image", "shape": 7, "type": "IMAGE", - "link": 515 + "link": 621 }, { "name": "moge_camera_config", "shape": 7, "type": "MOGE_CAM_CONFIG", - "link": 582 + "link": 622 + }, + { + "name": "from_resolution", + "type": "INT", + "widget": { + "name": "from_resolution" + }, + "link": 609 + }, + { + "name": "sparse_structure_resolution", + "type": "INT", + "widget": { + "name": "sparse_structure_resolution" + }, + "link": 610 } ], "outputs": [ @@ -793,93 +688,67 @@ "name": "shape_slat", "type": "SHAPE_SLAT", "links": [ - 531 + 611, + 617 ] }, { "name": "resolution", "type": "INT", "links": [ - 534 + 612, + 616 ] }, { "name": "pipeline", "type": "TRELLIS2PIPELINE", "links": [ - 529 + 613 ] + }, + { + "name": "num_tokens", + "type": "INT", + "links": null } ], "properties": { "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "ef2286b47ce0ef0e681dffe0209aadd96de50f72", - "Node name for S&R": "Trellis2ShapeGenerator" + "ver": "9e856420a774524ea05eac3d0bb14349e62b52c3", + "Node name for S&R": "Trellis2ShapeCascadeGenerator" }, "widgets_values": [ - 512, - 25, + 0, + 1536, + 32, + 999999, + 12, 7.5, - 0.01, + 0.5, 3, - "heun", - 0.1, + "euler", + 0.6, 1, false, - 0.65, + 0, 2, 0.92 ] }, - { - "id": 209, - "type": "PrimitiveInt", - "pos": [ - -1845.7531239022012, - 263.9724281177611 - ], - "size": [ - 270, - 82 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "INT", - "type": "INT", - "links": [ - 454, - 487 - ] - } - ], - "title": "Target Face Number", - "properties": { - "cnr_id": "comfy-core", - "ver": "0.16.4", - "Node name for S&R": "PrimitiveInt" - }, - "widgets_values": [ - 500000, - "fixed" - ] - }, { "id": 6, "type": "Trellis2LoadImageWithTransparency", "pos": [ - -1898.8408811895958, - 425.13094800380867 + -1883.142636861064, + 400.0903716535769 ], "size": [ - 631.9393495485403, - 765.9737728955563 + 604.3843674864397, + 774.9642385238385 ], "flags": {}, - "order": 2, + "order": 3, "mode": 0, "inputs": [], "outputs": [ @@ -909,243 +778,334 @@ "widget_ue_connectable": {} }, "widgets_values": [ - "Image_1024_00163_.png", + "Image_1024_00183_.png", "image" ] }, { - "id": 219, - "type": "PrimitiveString", + "id": 232, + "type": "Preview3D", "pos": [ - -1858.1285668527714, - 133.64378105828288 + 740.1097683045132, + 130.90395332568517 ], "size": [ - 270, - 58 + 1117.7988480977626, + 1161.0136948795248 ], "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [ - 402 - ] - } - ], - "title": "Name", - "properties": { - "cnr_id": "comfy-core", - "ver": "0.16.4", - "Node name for S&R": "PrimitiveString" - }, - "widgets_values": [ - "Cyberman_P3D" - ] - }, - { - "id": 274, - "type": "Trellis2ShapeCascadeGenerator", - "pos": [ - -420.008261589595, - 174.50398195526927 - ], - "size": [ - 335.9791015625, - 474 - ], - "flags": {}, - "order": 8, + "order": 17, "mode": 0, "inputs": [ { - "name": "pipeline", - "type": "TRELLIS2PIPELINE", - "link": 529 + "name": "camera_info", + "shape": 7, + "type": "LOAD3D_CAMERA", + "link": null }, { - "name": "image_cond", - "type": "IMAGE_COND", - "link": 530 - }, - { - "name": "shape_slat", - "type": "SHAPE_SLAT", - "link": 531 - }, - { - "name": "image", + "name": "bg_image", "shape": 7, "type": "IMAGE", - "link": 532 + "link": null }, { - "name": "moge_camera_config", - "shape": 7, - "type": "MOGE_CAM_CONFIG", - "link": 579 - }, - { - "name": "from_resolution", - "type": "INT", + "name": "model_file", + "type": "STRING,FILE_3D_GLB,FILE_3D_GLTF,FILE_3D_FBX,FILE_3D_OBJ,FILE_3D_STL,FILE_3D_USDZ,FILE_3D", "widget": { - "name": "from_resolution" + "name": "model_file" }, - "link": 534 + "link": 429 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "Preview3D", + "Last Time Model File": "C:/Git/ComfyUI/output/DragonRing_P3D_1024_00008_.glb", + "Resource Folder": "Git/ComfyUI/output", + "Scene Config": { + "showGrid": true, + "backgroundColor": "#282828", + "backgroundImage": "", + "backgroundRenderMode": "tiled" }, + "Camera Config": { + "cameraType": "perspective", + "fov": 37, + "state": { + "position": { + "x": 0.21005988267699321, + "y": 3.3289779923102216, + "z": 10.95146591033317 + }, + "target": { + "x": 0, + "y": 2.5, + "z": 0 + }, + "zoom": 1, + "cameraType": "perspective" + } + }, + "Light Config": { + "intensity": 3 + }, + "Model Config": { + "upDirection": "original", + "materialMode": "original", + "showSkeleton": false + } + }, + "widgets_values": [ + "C:/Git/ComfyUI/output/DragonRing_P3D_1024_00008_.glb", + "" + ] + }, + { + "id": 301, + "type": "Trellis2RemeshWithQuad", + "pos": [ + 243.85951305407565, + 144.83229212166844 + ], + "size": [ + 316.630859375, + 154 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ { - "name": "sparse_structure_resolution", - "type": "INT", - "widget": { - "name": "sparse_structure_resolution" - }, - "link": 535 + "name": "mesh", + "type": "MESHWITHVOXEL", + "link": 625 } ], "outputs": [ { - "name": "shape_slat", - "type": "SHAPE_SLAT", + "name": "mesh", + "type": "MESHWITHVOXEL", "links": [ - 536, - 588 + 624 ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Trellis2", + "ver": "e3d592ab351aae68917892e7b1ae27072008abaa", + "Node name for S&R": "Trellis2RemeshWithQuad" + }, + "widgets_values": [ + 1, + 0, + "Auto", + true, + true + ] + }, + { + "id": 218, + "type": "Trellis2SimplifyMesh", + "pos": [ + 263.87706551416335, + 378.57758582334964 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "mesh", + "type": "MESHWITHVOXEL", + "link": 624 }, { - "name": "resolution", + "name": "target_face_num", "type": "INT", + "widget": { + "name": "target_face_num" + }, + "link": 454 + } + ], + "outputs": [ + { + "name": "mesh", + "type": "MESHWITHVOXEL", "links": [ - 538, - 593 + 598 + ] + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Trellis2", + "ver": "ef2286b47ce0ef0e681dffe0209aadd96de50f72", + "Node name for S&R": "Trellis2SimplifyMesh" + }, + "widgets_values": [ + 500000, + "Cumesh" + ] + }, + { + "id": 299, + "type": "Trellis2FillHolesWithMeshlib", + "pos": [ + 283.6635226511138, + 535.9007069198656 + ], + "size": [ + 249.284765625, + 46 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "mesh", + "type": "MESHWITHVOXEL", + "link": 598 + } + ], + "outputs": [ + { + "name": "mesh", + "type": "MESHWITHVOXEL", + "links": [ + 599 ] }, { - "name": "pipeline", - "type": "TRELLIS2PIPELINE", - "links": [ - 586 - ] - }, - { - "name": "num_tokens", + "name": "holes_filled", "type": "INT", "links": null } ], "properties": { "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "86d13d9eac4a2bd4395954c7184d3aa4fa81a9d8", - "Node name for S&R": "Trellis2ShapeCascadeGenerator" + "ver": "9e856420a774524ea05eac3d0bb14349e62b52c3", + "Node name for S&R": "Trellis2FillHolesWithMeshlib" }, - "widgets_values": [ - 0, - 1536, - 32, - 999999, - 12, - 7.5, - 0.01, - 3, - "heun", - 0.1, - 1, - false, - 0.65, - 2, - 0.92 - ] + "widgets_values": [] }, { - "id": 296, - "type": "Trellis2TexSlatGenerator", + "id": 297, + "type": "Trellis2PostProcess2", "pos": [ - -403.3454100534395, - 722.8973335388696 + 171.11370646288293, + 626.9226027330353 ], "size": [ - 346.18029615893147, - 426 + 391.8560546875, + 274 ], "flags": {}, - "order": 9, - "mode": 0, + "order": 14, + "mode": 4, "inputs": [ { - "name": "pipeline", - "type": "TRELLIS2PIPELINE", - "link": 586 - }, - { - "name": "image_cond", - "type": "IMAGE_COND", - "link": 587 - }, - { - "name": "shape_slat", - "type": "SHAPE_SLAT", - "link": 588 - }, - { - "name": "image", - "shape": 7, - "type": "IMAGE", - "link": 589 - }, - { - "name": "moge_camera_config", - "shape": 7, - "type": "MOGE_CAM_CONFIG", - "link": 590 - }, - { - "name": "from_resolution", - "shape": 7, - "type": "INT", - "widget": { - "name": "from_resolution" - }, - "link": 593 + "name": "mesh", + "type": "MESHWITHVOXEL", + "link": 599 } ], "outputs": [ { - "name": "texture_slat", - "type": "TEXTURE_SLAT", + "name": "mesh", + "type": "MESHWITHVOXEL", "links": [ - 591 - ] - }, - { - "name": "pipeline", - "type": "TRELLIS2PIPELINE", - "links": [ - 592 + 626 ] } ], "properties": { "aux_id": "visualbruno/ComfyUI-Trellis2", - "ver": "fe543271876ff0db265c7b5989a1060362476696", - "Node name for S&R": "Trellis2TexSlatGenerator" + "ver": "9e856420a774524ea05eac3d0bb14349e62b52c3", + "Node name for S&R": "Trellis2PostProcess2" }, "widgets_values": [ - 1024, - 12, - 7.5, - 0, - 3, - "heun", - 0, - 0.9, false, - 0.65, - 2, - 0.92, - 0 + false, + false, + false, + false, + 4, + true, + 15, + false, + 1 + ] + }, + { + "id": 302, + "type": "Trellis2UnWrapAndRasterizer", + "pos": [ + 145.49616453212727, + 957.0341557837779 + ], + "size": [ + 419.15234375, + 338 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "mesh", + "type": "MESHWITHVOXEL", + "link": 626 + }, + { + "name": "bvh", + "type": "BVH", + "link": 627 + } + ], + "outputs": [ + { + "name": "trimesh", + "type": "TRIMESH", + "links": [ + 628 + ] + }, + { + "name": "base_color_texture", + "type": "IMAGE", + "links": null + }, + { + "name": "metallic_roughness_texture", + "type": "IMAGE", + "links": null + } + ], + "properties": { + "aux_id": "visualbruno/ComfyUI-Trellis2", + "ver": "e3d592ab351aae68917892e7b1ae27072008abaa", + "Node name for S&R": "Trellis2UnWrapAndRasterizer" + }, + "widgets_values": [ + 60, + 0, + 1, + 1, + 2048, + "OPAQUE", + false, + false, + true, + "telea", + "None" ] } ], @@ -1158,14 +1118,6 @@ 0, "IMAGE" ], - [ - 356, - 193, - 0, - 161, - 0, - "MESHWITHVOXEL" - ], [ 394, 194, @@ -1182,22 +1134,6 @@ 0, "TRELLIS2PIPELINE" ], - [ - 397, - 217, - 0, - 193, - 0, - "MESHWITHVOXEL" - ], - [ - 398, - 161, - 0, - 218, - 0, - "MESHWITHVOXEL" - ], [ 402, 219, @@ -1222,30 +1158,6 @@ 1, "INT" ], - [ - 463, - 218, - 0, - 250, - 0, - "MESHWITHVOXEL" - ], - [ - 487, - 209, - 0, - 257, - 1, - "INT" - ], - [ - 488, - 250, - 0, - 257, - 0, - "MESHWITHVOXEL" - ], [ 515, 194, @@ -1294,110 +1206,6 @@ 0, "TRELLIS2PIPELINE" ], - [ - 526, - 213, - 0, - 215, - 1, - "IMAGE_COND" - ], - [ - 529, - 215, - 2, - 274, - 0, - "TRELLIS2PIPELINE" - ], - [ - 530, - 213, - 1, - 274, - 1, - "IMAGE_COND" - ], - [ - 531, - 215, - 0, - 274, - 2, - "SHAPE_SLAT" - ], - [ - 532, - 194, - 0, - 274, - 3, - "IMAGE" - ], - [ - 534, - 215, - 1, - 274, - 5, - "INT" - ], - [ - 535, - 272, - 1, - 274, - 6, - "INT" - ], - [ - 536, - 274, - 0, - 217, - 1, - "SHAPE_SLAT" - ], - [ - 538, - 274, - 1, - 217, - 3, - "INT" - ], - [ - 572, - 257, - 0, - 290, - 0, - "MESHWITHVOXEL" - ], - [ - 573, - 217, - 1, - 290, - 1, - "BVH" - ], - [ - 574, - 290, - 0, - 203, - 0, - "TRIMESH" - ], - [ - 579, - 213, - 3, - 274, - 4, - "MOGE_CAM_CONFIG" - ], [ 580, 213, @@ -1414,14 +1222,6 @@ 4, "MOGE_CAM_CONFIG" ], - [ - 586, - 274, - 2, - 296, - 0, - "TRELLIS2PIPELINE" - ], [ 587, 213, @@ -1430,14 +1230,6 @@ 1, "IMAGE_COND" ], - [ - 588, - 274, - 0, - 296, - 2, - "SHAPE_SLAT" - ], [ 589, 194, @@ -1455,15 +1247,71 @@ "MOGE_CAM_CONFIG" ], [ - 591, - 296, + 598, + 218, 0, - 217, - 2, - "TEXTURE_SLAT" + 299, + 0, + "MESHWITHVOXEL" ], [ - 592, + 599, + 299, + 0, + 297, + 0, + "MESHWITHVOXEL" + ], + [ + 608, + 215, + 2, + 300, + 0, + "TRELLIS2PIPELINE" + ], + [ + 609, + 215, + 1, + 300, + 5, + "INT" + ], + [ + 610, + 272, + 1, + 300, + 6, + "INT" + ], + [ + 611, + 300, + 0, + 296, + 2, + "SHAPE_SLAT" + ], + [ + 612, + 300, + 1, + 296, + 5, + "INT" + ], + [ + 613, + 300, + 2, + 296, + 0, + "TRELLIS2PIPELINE" + ], + [ + 614, 296, 1, 217, @@ -1471,12 +1319,108 @@ "TRELLIS2PIPELINE" ], [ - 593, - 274, - 1, + 615, 296, - 5, + 0, + 217, + 2, + "TEXTURE_SLAT" + ], + [ + 616, + 300, + 1, + 217, + 3, "INT" + ], + [ + 617, + 300, + 0, + 217, + 1, + "SHAPE_SLAT" + ], + [ + 618, + 213, + 1, + 300, + 1, + "IMAGE_COND" + ], + [ + 619, + 213, + 0, + 215, + 1, + "IMAGE_COND" + ], + [ + 620, + 215, + 0, + 300, + 2, + "SHAPE_SLAT" + ], + [ + 621, + 194, + 0, + 300, + 3, + "IMAGE" + ], + [ + 622, + 213, + 3, + 300, + 4, + "MOGE_CAM_CONFIG" + ], + [ + 624, + 301, + 0, + 218, + 0, + "MESHWITHVOXEL" + ], + [ + 625, + 217, + 0, + 301, + 0, + "MESHWITHVOXEL" + ], + [ + 626, + 297, + 0, + 302, + 0, + "MESHWITHVOXEL" + ], + [ + 627, + 217, + 1, + 302, + 1, + "BVH" + ], + [ + 628, + 302, + 0, + 203, + 0, + "TRIMESH" ] ], "groups": [ @@ -1498,8 +1442,8 @@ "bounding": [ -1240.97479805832, 43.52609271270552, - 1788.199900924072, - 1349.2198429421912 + 1890.3083860572324, + 1417.1519197551795 ], "color": "#3f789e", "flags": {} @@ -1510,10 +1454,10 @@ "workflowRendererVersion": "LG", "ue_links": [], "ds": { - "scale": 0.5709797729819821, + "scale": 0.4718841099024762, "offset": [ - 1772.9050677529558, - 122.45414639513807 + 1940.655834201771, + 361.97588518415347 ] }, "links_added_by_ue": [], diff --git a/nodes.py b/nodes.py index cdf6bef..6ea6e4e 100644 --- a/nodes.py +++ b/nodes.py @@ -61,6 +61,12 @@ class AnyType(str): any = AnyType("*") +def inpaint_channel(channel, mask_inv, radius=1, mode = cv2.INPAINT_TELEA): + out = cv2.inpaint(channel, mask_inv, radius, mode) + if out.ndim == 2: + out = out[..., None] + return out + def rotate_triton_cache(): """ Creates a new cache directory and attempts to clean up old ones. @@ -1278,9 +1284,9 @@ class Trellis2UnWrapAndRasterizer: # Inpainting: fill gaps (dilation) to prevent black seams at UV boundaries mask_inv = (~mask).astype(np.uint8) base_color = cv2.inpaint(base_color, mask_inv, 3, inpainting) - metallic = cv2.inpaint(metallic, mask_inv, 1, inpainting)[..., None] - roughness = cv2.inpaint(roughness, mask_inv, 1, inpainting)[..., None] - alpha = cv2.inpaint(alpha, mask_inv, 1, inpainting)[..., None] + metallic = inpaint_channel(metallic, mask_inv, 1, inpainting) + roughness = inpaint_channel(roughness, mask_inv, 1, inpainting) + alpha = inpaint_channel(alpha, mask_inv, 1, inpainting) # Create PBR material # Standard PBR packs Metallic and Roughness into Blue and Green channels @@ -1435,31 +1441,53 @@ class Trellis2MeshWithVoxelAdvancedGenerator: if generate_texture_slat: num_steps = 5 else: - num_steps = 4 - - pbar = ProgressBar(num_steps) + num_steps = 4 - mesh = pipeline.run(image=image_in, - seed=seed, - pipeline_type=pipeline_type, - sparse_structure_sampler_params = sparse_structure_sampler_params, - shape_slat_sampler_params = shape_slat_sampler_params, - tex_slat_sampler_params = tex_slat_sampler_params, - max_num_tokens = max_num_tokens, - sparse_structure_resolution = sparse_structure_resolution, - max_views = max_views, - generate_texture_slat=generate_texture_slat, - use_tiled=use_tiled_decoder, - pbar=pbar, - sampler=sampler, - fill_holes = fill_holes, - hole_iterations = hole_iterations, - verbose = verbose, - dino_lock = dino_lock, - dino_substeps = dino_substeps, - hole_fill_algorithm=hole_fill_algorithm, - dino_foundation_cap=dino_foundation_cap, - keep_only_shell=keep_only_shell)[0] + if pipeline.isPixal3D: + pipeline.load_moge_model() + + if isinstance(images, (list, tuple)): + image = images[0] + else: + image = images + + camera_params = pipeline.get_moge_camera_config(image) + + if not pipeline.keep_models_loaded: + pipeline.unload_moge_model() + + mesh = pipeline.run_pixal3d(image=image_in, + seed=seed, + pipeline_type=pipeline_type, + sparse_structure_sampler_params = sparse_structure_sampler_params, + shape_slat_sampler_params = shape_slat_sampler_params, + tex_slat_sampler_params = tex_slat_sampler_params, + max_num_tokens = max_num_tokens, + generate_texture_slat=generate_texture_slat, + camera_params=camera_params)[0] + else: + pbar = ProgressBar(num_steps) + mesh = pipeline.run(image=image_in, + seed=seed, + pipeline_type=pipeline_type, + sparse_structure_sampler_params = sparse_structure_sampler_params, + shape_slat_sampler_params = shape_slat_sampler_params, + tex_slat_sampler_params = tex_slat_sampler_params, + max_num_tokens = max_num_tokens, + sparse_structure_resolution = sparse_structure_resolution, + max_views = max_views, + generate_texture_slat=generate_texture_slat, + use_tiled=use_tiled_decoder, + pbar=pbar, + sampler=sampler, + fill_holes = fill_holes, + hole_iterations = hole_iterations, + verbose = verbose, + dino_lock = dino_lock, + dino_substeps = dino_substeps, + hole_fill_algorithm=hole_fill_algorithm, + dino_foundation_cap=dino_foundation_cap, + keep_only_shell=keep_only_shell)[0] vertices = mesh.vertices.cuda() faces = mesh.faces.cuda() @@ -1663,6 +1691,7 @@ class Trellis2PostProcessAndUnWrapAndRasterizer: "bvh": ("BVH",), "remove_inner_faces": ("BOOLEAN",{"default":True}), "inpainting": (["telea","ns"],{"default":"telea"}), + "reorient_vertices":(["None","90 degrees","-90 degrees"],{"default":"90 degrees"}), } } @@ -1672,7 +1701,7 @@ class Trellis2PostProcessAndUnWrapAndRasterizer: CATEGORY = "Trellis2Wrapper" OUTPUT_NODE = True - def process(self, mesh, mesh_cluster_threshold_cone_half_angle_rad, mesh_cluster_refine_iterations, mesh_cluster_global_iterations, mesh_cluster_smooth_strength, texture_size, remesh, remesh_band, remesh_project, target_face_num, simplify_method, fill_holes, texture_alpha_mode, dual_contouring_resolution, double_side_material, remove_floaters, bake_on_vertices,use_custom_normals,bvh,remove_inner_faces,inpainting): + def process(self, mesh, mesh_cluster_threshold_cone_half_angle_rad, mesh_cluster_refine_iterations, mesh_cluster_global_iterations, mesh_cluster_smooth_strength, texture_size, remesh, remesh_band, remesh_project, target_face_num, simplify_method, fill_holes, texture_alpha_mode, dual_contouring_resolution, double_side_material, remove_floaters, bake_on_vertices,use_custom_normals,bvh,remove_inner_faces,inpainting,reorient_vertices): pbar = ProgressBar(5 if not bake_on_vertices else 4) mesh_copy = copy.deepcopy(mesh) @@ -1707,10 +1736,7 @@ class Trellis2PostProcessAndUnWrapAndRasterizer: grid_size = np.array(grid_size) if isinstance(grid_size, np.ndarray): grid_size = torch.tensor(grid_size, dtype=torch.int32, device=coords.device) - voxel_size = (aabb[1] - aabb[0]) / grid_size - - if remove_floaters: - mesh_copy = remove_floater(mesh_copy) + voxel_size = (aabb[1] - aabb[0]) / grid_size vertices = mesh_copy.vertices faces = mesh_copy.faces @@ -1769,18 +1795,21 @@ class Trellis2PostProcessAndUnWrapAndRasterizer: else: resolution = int(dual_contouring_resolution) + scale = (resolution + 3 * remesh_band) / resolution * scale + print(f"Calculated scale: {scale}") + print('Performing Dual Contouring ...') # Perform Dual Contouring remeshing (rebuilds topology) cumesh.init(*CuMesh.remeshing.remesh_narrow_band_dc_quad( vertices, faces, center = center, - scale = scale * 1.1, # old calculation : (resolution + 3 * remesh_band) / resolution * scale, + scale = scale, resolution = resolution, band = remesh_band, project_back = remesh_project, # Snaps vertices back to original surface verbose = True, remove_inner_faces = remove_inner_faces, - #bvh = bvh, + bvh = bvh, )) new_vertices, new_faces = cumesh.read() @@ -1910,8 +1939,12 @@ class Trellis2PostProcessAndUnWrapAndRasterizer: normals_np = out_normals.cpu().numpy() # Swap Y and Z axes, invert Y (common conversion for GLB compatibility) - vertices_np[:, 1], vertices_np[:, 2] = vertices_np[:, 2].copy(), -vertices_np[:, 1].copy() - normals_np[:, 1], normals_np[:, 2] = normals_np[:, 2].copy(), -normals_np[:, 1].copy() + if reorient_vertices == '90 degrees': + vertices_np[:, 1], vertices_np[:, 2] = vertices_np[:, 2].copy(), -vertices_np[:, 1].copy() + normals_np[:, 1], normals_np[:, 2] = normals_np[:, 2].copy(), -normals_np[:, 1].copy() + elif reorient_vertices == '-90 degrees': + vertices_np[:, 1], vertices_np[:, 2] = -vertices_np[:, 2].copy(), vertices_np[:, 1].copy() + normals_np[:, 1], normals_np[:, 2] = -normals_np[:, 2].copy(), normals_np[:, 1].copy() # Create mesh with vertex colors using ColorVisuals if use_custom_normals: @@ -2021,9 +2054,9 @@ class Trellis2PostProcessAndUnWrapAndRasterizer: mask_inv = (~mask).astype(np.uint8) base_color = cv2.inpaint(base_color, mask_inv, 3, inpainting) - metallic = cv2.inpaint(metallic, mask_inv, 1, inpainting)[..., None] - roughness = cv2.inpaint(roughness, mask_inv, 1, inpainting)[..., None] - alpha = cv2.inpaint(alpha, mask_inv, 1, inpainting)[..., None] + metallic = inpaint_channel(metallic, mask_inv, 1, inpainting) + roughness = inpaint_channel(roughness, mask_inv, 1, inpainting) + alpha = inpaint_channel(alpha, mask_inv, 1, inpainting) # Create PBR material # Standard PBR packs Metallic and Roughness into Blue and Green channels @@ -2045,8 +2078,13 @@ class Trellis2PostProcessAndUnWrapAndRasterizer: normals_np = out_normals.cpu().numpy() # Swap Y and Z axes, invert Y (common conversion for GLB compatibility) - vertices_np[:, 1], vertices_np[:, 2] = vertices_np[:, 2].copy(), -vertices_np[:, 1].copy() - normals_np[:, 1], normals_np[:, 2] = normals_np[:, 2].copy(), -normals_np[:, 1].copy() + if reorient_vertices == '90 degrees': + vertices_np[:, 1], vertices_np[:, 2] = vertices_np[:, 2].copy(), -vertices_np[:, 1].copy() + normals_np[:, 1], normals_np[:, 2] = normals_np[:, 2].copy(), -normals_np[:, 1].copy() + elif reorient_vertices == '-90 degrees': + vertices_np[:, 1], vertices_np[:, 2] = -vertices_np[:, 2].copy(), vertices_np[:, 1].copy() + normals_np[:, 1], normals_np[:, 2] = -normals_np[:, 2].copy(), normals_np[:, 1].copy() + uvs_np[:, 1] = 1 - uvs_np[:, 1] # Flip UV V-coordinate if use_custom_normals: @@ -5156,7 +5194,7 @@ class Trellis2SaveImage: "required": { "images": ("IMAGE", {"tooltip": "The images to save."}), "filename_prefix": ("STRING", {"default": "ComfyUI", "tooltip": "The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."}), - "compress_level": ("INT",{"default":4,"min":1,"max":9,"step":1}), + "compress_level": ("INT",{"default":1,"min":1,"max":9,"step":1}), }, "hidden": { "prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO" diff --git a/trellis2/modules/sparse/conv/config.py b/trellis2/modules/sparse/conv/config.py index 574b38b..ac08489 100644 --- a/trellis2/modules/sparse/conv/config.py +++ b/trellis2/modules/sparse/conv/config.py @@ -1,3 +1,3 @@ SPCONV_ALGO = 'auto' # 'auto', 'implicit_gemm', 'native' -FLEX_GEMM_ALGO = 'implicit_gemm_splitk' # 'explicit_gemm', 'implicit_gemm', 'implicit_gemm_splitk', 'masked_implicit_gemm', 'masked_implicit_gemm_splitk' +FLEX_GEMM_ALGO = 'masked_implicit_gemm_splitk' # 'explicit_gemm', 'implicit_gemm', 'implicit_gemm_splitk', 'masked_implicit_gemm', 'masked_implicit_gemm_splitk' FLEX_GEMM_HASHMAP_RATIO = 2.0 # Ratio of hashmap size to input size diff --git a/trellis2/pipelines/trellis2_image_to_3d.py b/trellis2/pipelines/trellis2_image_to_3d.py index 9b8dcdb..61e9c07 100644 --- a/trellis2/pipelines/trellis2_image_to_3d.py +++ b/trellis2/pipelines/trellis2_image_to_3d.py @@ -239,14 +239,25 @@ class Trellis2ImageTo3DPipeline(Pipeline): facebook_model_path = os.path.join(folder_paths.models_dir,"facebook","dinov3-vitl16-pretrain-lvd1689m") pipeline._pretrained_args['image_cond_model']['args']['model_name'] = facebook_model_path - + pipeline.PIXAL3D_IMAGE_COND_CONFIGS["ss"]["model_name"] = facebook_model_path pipeline.PIXAL3D_IMAGE_COND_CONFIGS["shape_512"]["model_name"] = facebook_model_path pipeline.PIXAL3D_IMAGE_COND_CONFIGS["shape_1024"]["model_name"] = facebook_model_path pipeline.PIXAL3D_IMAGE_COND_CONFIGS["tex_1024"]["model_name"] = facebook_model_path + try: + from mmgp import safetensors2 as _mmgp_st2 + if 'C64' not in _mmgp_st2._map_to_dtype: + _mmgp_st2._map_to_dtype['C64'] = torch.complex64 + _mmgp_st2._map_to_dtype['C128'] = torch.complex128 + print("[Pixal3D] Patched mmgp.safetensors2._map_to_dtype with C64/C128") + except (ImportError, AttributeError): + print('mmgp not installed') + # mmgp not installed (comfy_env worker / non-Desktop ComfyUI) — no patch needed. + pass + return pipeline - + def load_moge_model(self): if hasattr(self,'moge_model') and self.moge_model is not None: return self.moge_model @@ -526,7 +537,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): # ========================================================================= # Proj mode condition building # ========================================================================= - + @torch.no_grad() def get_proj_cond_ss( self, @@ -635,6 +646,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): 'neg_cond': {'global': torch.zeros_like(z_global), 'proj': SparseTensor(feats=torch.zeros_like(z_proj_sparse), coords=coords)}, } + @torch.no_grad() def get_moge_camera_config(self, image): from ..utils.camera import get_camera_params_wild_moge @@ -679,7 +691,8 @@ class Trellis2ImageTo3DPipeline(Pipeline): output = output[:, :, :3] * output[:, :, 3:4] output = Image.fromarray((output * 255).astype(np.uint8)) return output - + + @torch.no_grad() def get_cond( self, image: Union[torch.Tensor, Image.Image, List[Image.Image]], @@ -766,20 +779,21 @@ class Trellis2ImageTo3DPipeline(Pipeline): neg_cond = torch.zeros_like(cond) return {"cond": cond, "neg_cond": neg_cond} + @torch.no_grad() def sample_sparse_structure( self, cond: dict, resolution: int, num_samples: int = 1, sampler_params: dict = {}, - fill_holes: bool = True, + fill_holes: bool = False, hole_structure: int = 1, hole_iterations: int = 1, hole_fill_algorithm: str = "remove_small_holes", - keep_only_shell: bool = True, - verbose: bool = True, + keep_only_shell: bool = False, + verbose: bool = False, dino_lock: float = 0.0, - dino_substeps: int = 4, + dino_substeps: int = 2, dino_foundation_cap: float = 0.92 ) -> torch.Tensor: """ @@ -951,6 +965,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): self._cleanup_cuda() return coords + @torch.no_grad() def sample_shape_slat( self, cond: dict, @@ -1008,6 +1023,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return slat + @torch.no_grad() def sample_shape_slat_cascade( self, lr_cond: dict, @@ -1144,6 +1160,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return slat, hr_resolution + @torch.no_grad() def decode_shape_slat( self, slat: SparseTensor, @@ -1178,6 +1195,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return ret + @torch.no_grad() def sample_tex_slat( self, cond: dict, @@ -1296,6 +1314,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): self._cleanup_cuda() out_mesh = [] for m in meshes: + m.fill_holes() out_mesh.append( MeshWithVoxel( m.vertices, m.faces, @@ -1315,6 +1334,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): self._cleanup_cuda() out_mesh = [] for m, v in zip(meshes, tex_voxels): + m.fill_holes() out_mesh.append( MeshWithVoxel( m.vertices, m.faces, @@ -1328,7 +1348,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): ) return out_mesh - @torch.no_grad() + @torch.inference_mode() def run( self, image: Image.Image, @@ -1701,6 +1721,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): else: return 'Euler' + @torch.no_grad() def sample_shape_slat_cascade_advanced( self, lr_cond: dict, @@ -1850,6 +1871,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return slat, hr_resolution + @torch.no_grad() def sample_tex_slat_advanced( self, cond: dict, @@ -2085,8 +2107,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return out_mesh - - @torch.no_grad() + @torch.inference_mode() def run_multiview( self, front: Image.Image, @@ -2336,6 +2357,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): else: return out_mesh + @torch.no_grad() def sample_sparse_structure_multiview( self, conds: dict, @@ -2543,6 +2565,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return coords + @torch.no_grad() def sample_shape_slat_multiview( self, conds: dict, @@ -2613,6 +2636,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return slat + @torch.no_grad() def sample_shape_slat_cascade_multiview( self, lr_conds: dict, @@ -2767,7 +2791,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return slat - + @torch.no_grad() def sample_tex_slat_multiview( self, conds: dict, @@ -2854,7 +2878,6 @@ class Trellis2ImageTo3DPipeline(Pipeline): return slat - def preprocess_mesh(self, mesh: trimesh.Trimesh) -> trimesh.Trimesh: """ Preprocess the input mesh. @@ -2875,6 +2898,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): mesh.vertices = vertices return mesh + @torch.no_grad() def encode_shape_slat( self, mesh: trimesh.Trimesh, @@ -3154,7 +3178,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return textured_mesh, baseColorTexture, metallicRoughnessTexture - @torch.no_grad() + @torch.inference_mode() def texture_mesh( self, mesh: trimesh.Trimesh, @@ -3239,7 +3263,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): out_mesh, baseColorTexture, metallicRoughnessTexture = self.postprocess_mesh(mesh, pbr_voxel, resolution, texture_size, texture_alpha_mode, double_side_material, bake_on_vertices, use_custom_normals, mesh_cluster_threshold_cone_half_angle_rad, inpainting) return out_mesh, baseColorTexture, metallicRoughnessTexture - @torch.no_grad() + @torch.inference_mode() def texture_mesh_multiview( self, mesh: trimesh.Trimesh, @@ -3375,6 +3399,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return coords; + @torch.no_grad() def sample_mesh_slat( self, mesh_slat, @@ -3470,7 +3495,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): return slat, hr_resolution - @torch.no_grad() + @torch.inference_mode() def refine_mesh( self, mesh: trimesh.Trimesh, @@ -3631,7 +3656,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): else: return out_mesh - @torch.no_grad() + @torch.inference_mode() def run_pixal3d( self, image: Image.Image, @@ -3641,7 +3666,7 @@ class Trellis2ImageTo3DPipeline(Pipeline): sparse_structure_sampler_params: dict = {}, shape_slat_sampler_params: dict = {}, tex_slat_sampler_params: dict = {}, - preprocess_image: bool = True, + preprocess_image: bool = False, return_latent: bool = False, pipeline_type: Optional[str] = None, max_num_tokens: int = 49152, @@ -3667,7 +3692,10 @@ class Trellis2ImageTo3DPipeline(Pipeline): max_num_tokens (int): The maximum number of tokens to use. """ # Check pipeline type - pipeline_type = pipeline_type or self.default_pipeline_type + if pipeline_type == '1536_cascade': + hr_resolution = 1536 + else: + hr_resolution = 1024 # Extract camera params camera_angle_x = camera_params['camera_angle_x'] @@ -3679,35 +3707,51 @@ class Trellis2ImageTo3DPipeline(Pipeline): torch.manual_seed(seed) # ---- Stage 1: Sparse Structure (proj) ---- + image_cond_model = self.load_pixal3d_image_cond_ss() cond_ss = self.get_proj_cond_ss( [image], camera_angle_x=camera_angle_x, distance=distance, mesh_scale=mesh_scale, + image_cond_model=image_cond_model ) + del image_cond_model + if not self.keep_models_loaded: + self.unload_pixal3d_image_cond_ss() ss_res = 32 + self.load_sparse_structure_model() coords = self.sample_sparse_structure( cond_ss, ss_res, num_samples, sparse_structure_sampler_params ) + if not self.keep_models_loaded: + self.unload_sparse_structure_model() del cond_ss torch.cuda.empty_cache() # ---- Stage 2: Shape LR 512 (proj) ---- + image_cond_model = self.load_pixal3d_image_cond_shape_512() cond_shape_lr = self.get_proj_cond_shape( - self.image_cond_model_shape_512, [image], coords, + image_cond_model, [image], coords, camera_angle_x=camera_angle_x, distance=distance, mesh_scale=mesh_scale, ) + del image_cond_model + if not self.keep_models_loaded: + self.unload_pixal3d_image_cond_shape_512() + self.load_shape_slat_flow_model_512() lr_slat = self.sample_shape_slat( cond_shape_lr, self.models['shape_slat_flow_model_512'], coords, shape_slat_sampler_params ) + if not self.keep_models_loaded: + self.unload_shape_slat_flow_model_512() del cond_shape_lr torch.cuda.empty_cache() # ---- Stage 3a: Upsample LR → HR ---- + self.load_shape_slat_decoder() if self.low_vram: self.models['shape_slat_decoder'].to(self.device) self.models['shape_slat_decoder'].low_vram = True @@ -3716,6 +3760,9 @@ class Trellis2ImageTo3DPipeline(Pipeline): self.models['shape_slat_decoder'].cpu() self.models['shape_slat_decoder'].low_vram = False + if not self.keep_models_loaded: + self.unload_shape_slat_decoder() + lr_resolution = 512 actual_hr_resolution = hr_resolution while True: @@ -3735,18 +3782,26 @@ class Trellis2ImageTo3DPipeline(Pipeline): torch.cuda.empty_cache() # ---- Stage 3b: Shape HR (proj) ---- + image_cond_model = self.load_pixal3d_image_cond_shape_1024() cond_shape_hr = self.get_proj_cond_shape( - self.image_cond_model_shape_1024, [image], hr_coords_unique, + image_cond_model, [image], hr_coords_unique, camera_angle_x=camera_angle_x, distance=distance, mesh_scale=mesh_scale, grid_resolution_override=actual_grid_res, ) + del image_cond_model + if not self.keep_models_loaded: + self.unload_pixal3d_image_cond_shape_1024() + + self.load_shape_slat_flow_model_1024() + noise_hr = SparseTensor( feats=torch.randn(hr_coords_unique.shape[0], self.models['shape_slat_flow_model_1024'].in_channels).to(self.device), coords=hr_coords_unique, ) sampler_params_hr = {**self.shape_slat_sampler_params, **shape_slat_sampler_params} + flow_model_hr = self.models['shape_slat_flow_model_1024'] if self.low_vram: flow_model_hr.to(self.device) @@ -3763,23 +3818,38 @@ class Trellis2ImageTo3DPipeline(Pipeline): std = torch.tensor(self.shape_slat_normalization['std'])[None].to(hr_slat.device) mean = torch.tensor(self.shape_slat_normalization['mean'])[None].to(hr_slat.device) shape_slat = hr_slat * std + mean + del flow_model_hr + if not self.keep_models_loaded: + self.unload_shape_slat_flow_model_1024() del cond_shape_hr, noise_hr, hr_slat, hr_coords_unique torch.cuda.empty_cache() if generate_texture_slat: # ---- Stage 4: Texture (proj) ---- + image_cond_model = self.load_pixal3d_image_cond_tex_1024() + tex_grid_res = actual_hr_resolution // 16 cond_tex = self.get_proj_cond_shape( - self.image_cond_model_tex_1024, [image], shape_slat.coords, + image_cond_model, [image], shape_slat.coords, camera_angle_x=camera_angle_x, distance=distance, mesh_scale=mesh_scale, grid_resolution_override=tex_grid_res, ) + + if not self.keep_models_loaded: + self.unload_pixal3d_image_cond_tex_1024() + + self.load_tex_slat_flow_model_1024() + tex_slat = self.sample_tex_slat( cond_tex, self.models['tex_slat_flow_model_1024'], shape_slat, tex_slat_sampler_params ) + + if not self.keep_models_loaded: + self.unload_tex_slat_flow_model_1024() + del cond_tex torch.cuda.empty_cache() diff --git a/trellis2/trainers/flow_matching/mixins/image_conditioned_proj.py b/trellis2/trainers/flow_matching/mixins/image_conditioned_proj.py index 24ab5af..7a06c84 100644 --- a/trellis2/trainers/flow_matching/mixins/image_conditioned_proj.py +++ b/trellis2/trainers/flow_matching/mixins/image_conditioned_proj.py @@ -67,7 +67,7 @@ def project_points_to_image_batch( points_homogeneous = torch.cat([points_3d_batch, ones], dim=-1) # [B, N, 4] # Compute world to camera transformation matrix - world_to_camera = torch.linalg.inv(transform_matrix) # [B, 4, 4] + world_to_camera = torch.linalg.inv(transform_matrix.float()).to(transform_matrix.dtype) # linalg.inv requires fp32+ # Batch transform to camera coordinate system: [B, N, 4] @ [B, 4, 4]^T -> [B, N, 3] points_camera = torch.bmm(points_homogeneous, world_to_camera.transpose(-2, -1))[..., :3] # [B, N, 3] @@ -409,7 +409,7 @@ class DinoV3ProjFeatureExtractor(nn.Module): # (current behavior, default). >1 enables a streaming wrapper that tiles NAF and # the projection together, capping the transient HR-feature-map peak. Set transiently # by the pipeline (see Pixal3DImageTo3DPipeline.get_proj_cond_shape). - self.naf_tile_factor: int = 4 + self.naf_tile_factor: int = 1 # proj_channels: the output dimension of proj features # Without NAF: embed_dim (e.g. 1024) diff --git a/trellis2/utils/camera.py b/trellis2/utils/camera.py index 0d2ee33..3a9a87c 100644 --- a/trellis2/utils/camera.py +++ b/trellis2/utils/camera.py @@ -19,7 +19,7 @@ def distance_from_fov(camera_angle_x, grid_point, target_point, mesh_scale, imag distance_x = f_pixels * xw / x_ndc - yw return {"distance_from_x": float(distance_x), "f_pixels": float(f_pixels)} -def get_camera_params_wild_moge(pil_image, moge_model, device="cuda", mesh_scale=1.0, extend_pixel=0, image_resolution=1024): +def get_camera_params_wild_moge(pil_image, moge_model, device="cuda", mesh_scale=1.0, extend_pixel=0, image_resolution=512): #pil_image = Image.open(image_path).convert("RGB") width, height = pil_image.size image_np = np.array(pil_image).astype(np.float32) / 255.0