From feaa3e03b1c0f4d25995a850b0bc186b87996663 Mon Sep 17 00:00:00 2001 From: Denis Kochetov Date: Wed, 7 May 2025 21:51:08 +0000 Subject: [PATCH] add multiview --- .../hunyuan3d-dit-v2-mv-fast/config.yaml | 69 ++++++ .../hunyuan3d-dit-v2-mv/config.yaml | 68 ++++++ .../Hunyuan3D_V2/hy3dgen/texgen/pipelines.py | 4 +- nodes.py | 212 ++++++++++++------ 4 files changed, 284 insertions(+), 69 deletions(-) create mode 100644 Checkpoints/Diffusers/tencent/Hunyuan3D-2mv/hunyuan3d-dit-v2-mv-fast/config.yaml create mode 100644 Checkpoints/Diffusers/tencent/Hunyuan3D-2mv/hunyuan3d-dit-v2-mv/config.yaml diff --git a/Checkpoints/Diffusers/tencent/Hunyuan3D-2mv/hunyuan3d-dit-v2-mv-fast/config.yaml b/Checkpoints/Diffusers/tencent/Hunyuan3D-2mv/hunyuan3d-dit-v2-mv-fast/config.yaml new file mode 100644 index 0000000..0cb7655 --- /dev/null +++ b/Checkpoints/Diffusers/tencent/Hunyuan3D-2mv/hunyuan3d-dit-v2-mv-fast/config.yaml @@ -0,0 +1,69 @@ +model: + target: Hunyuan3D_V2.hy3dgen.shapegen.models.Hunyuan3DDiT + params: + in_channels: 64 + context_in_dim: 1536 + hidden_size: 1024 + mlp_ratio: 4.0 + num_heads: 16 + depth: 16 + depth_single_blocks: 32 + axes_dim: [ 64 ] + theta: 10000 + qkv_bias: true + guidance_embed: true + +vae: + target: Hunyuan3D_V2.hy3dgen.shapegen.models.ShapeVAE + params: + num_latents: 3072 + embed_dim: 64 + num_freqs: 8 + include_pi: false + heads: 16 + width: 1024 + num_decoder_layers: 16 + qkv_bias: false + qk_norm: true + scale_factor: 0.9990943042622529 + +conditioner: + target: Hunyuan3D_V2.hy3dgen.shapegen.models.SingleImageEncoder + params: + main_image_encoder: + type: DinoImageEncoderMV # dino giant + kwargs: + config: + attention_probs_dropout_prob: 0.0 + drop_path_rate: 0.0 + hidden_act: gelu + hidden_dropout_prob: 0.0 + hidden_size: 1536 + image_size: 518 + initializer_range: 0.02 + layer_norm_eps: 1.e-6 + layerscale_value: 1.0 + mlp_ratio: 4 + model_type: dinov2 + num_attention_heads: 24 + num_channels: 3 + num_hidden_layers: 40 + patch_size: 14 + qkv_bias: true + torch_dtype: float32 + use_swiglu_ffn: true + image_size: 518 + +scheduler: + target: Hunyuan3D_V2.hy3dgen.shapegen.schedulers.FlowMatchEulerDiscreteScheduler + params: + num_train_timesteps: 1000 + +image_processor: + target: Hunyuan3D_V2.hy3dgen.shapegen.preprocessors.MVImageProcessorV2 + params: + size: 512 + border_ratio: 0.15 + +pipeline: + target: Hunyuan3D_V2.hy3dgen.shapegen.pipelines.Hunyuan3DDiTFlowMatchingPipeline \ No newline at end of file diff --git a/Checkpoints/Diffusers/tencent/Hunyuan3D-2mv/hunyuan3d-dit-v2-mv/config.yaml b/Checkpoints/Diffusers/tencent/Hunyuan3D-2mv/hunyuan3d-dit-v2-mv/config.yaml new file mode 100644 index 0000000..29fac42 --- /dev/null +++ b/Checkpoints/Diffusers/tencent/Hunyuan3D-2mv/hunyuan3d-dit-v2-mv/config.yaml @@ -0,0 +1,68 @@ +model: + target: Hunyuan3D_V2.hy3dgen.shapegen.models.Hunyuan3DDiT + params: + in_channels: 64 + context_in_dim: 1536 + hidden_size: 1024 + mlp_ratio: 4.0 + num_heads: 16 + depth: 16 + depth_single_blocks: 32 + axes_dim: [ 64 ] + theta: 10000 + qkv_bias: true + +vae: + target: Hunyuan3D_V2.hy3dgen.shapegen.models.ShapeVAE + params: + num_latents: 3072 + embed_dim: 64 + num_freqs: 8 + include_pi: false + heads: 16 + width: 1024 + num_decoder_layers: 16 + qkv_bias: false + qk_norm: true + scale_factor: 0.9990943042622529 + +conditioner: + target: Hunyuan3D_V2.hy3dgen.shapegen.models.SingleImageEncoder + params: + main_image_encoder: + type: DinoImageEncoderMV # dino giant + kwargs: + config: + attention_probs_dropout_prob: 0.0 + drop_path_rate: 0.0 + hidden_act: gelu + hidden_dropout_prob: 0.0 + hidden_size: 1536 + image_size: 518 + initializer_range: 0.02 + layer_norm_eps: 1.e-6 + layerscale_value: 1.0 + mlp_ratio: 4 + model_type: dinov2 + num_attention_heads: 24 + num_channels: 3 + num_hidden_layers: 40 + patch_size: 14 + qkv_bias: true + torch_dtype: float32 + use_swiglu_ffn: true + image_size: 518 + +scheduler: + target: Hunyuan3D_V2.hy3dgen.shapegen.schedulers.FlowMatchEulerDiscreteScheduler + params: + num_train_timesteps: 1000 + +image_processor: + target: Hunyuan3D_V2.hy3dgen.shapegen.preprocessors.MVImageProcessorV2 + params: + size: 512 + border_ratio: 0.15 + +pipeline: + target: Hunyuan3D_V2.hy3dgen.shapegen.pipelines.Hunyuan3DDiTFlowMatchingPipeline \ No newline at end of file diff --git a/Gen_3D_Modules/Hunyuan3D_V2/hy3dgen/texgen/pipelines.py b/Gen_3D_Modules/Hunyuan3D_V2/hy3dgen/texgen/pipelines.py index 9f82240..2110588 100644 --- a/Gen_3D_Modules/Hunyuan3D_V2/hy3dgen/texgen/pipelines.py +++ b/Gen_3D_Modules/Hunyuan3D_V2/hy3dgen/texgen/pipelines.py @@ -93,8 +93,8 @@ class Hunyuan3DPaintPipeline: texture_size=self.config.texture_size) self.load_models() - - def to(self, device=None, dtype=None): # ⬅️  тот же уровень, что и __init__ + + def to(self, device=None, dtype=None): if device is not None: for m in self.models.values(): m.pipeline.to(device) diff --git a/nodes.py b/nodes.py index 0df710d..7036f71 100644 --- a/nodes.py +++ b/nodes.py @@ -97,6 +97,7 @@ from Hunyuan3D_V1.mvd.hunyuan3d_mvd_lite_pipeline import Hunyuan3D_MVD_Lite_Pipe from Hunyuan3D_V1.infer import Views2Mesh from Hunyuan3D_V2.hy3dgen.shapegen import FaceReducer, FloaterRemover, DegenerateFaceRemover, Hunyuan3DDiTFlowMatchingPipeline from Hunyuan3D_V2.hy3dgen.texgen import Hunyuan3DPaintPipeline +from Hunyuan3D_V2.hy3dgen.rembg import BackgroundRemover from TRELLIS.trellis.pipelines import TrellisImageTo3DPipeline from TRELLIS.trellis.utils import postprocessing_utils from TripoSG.pipelines.pipeline_triposg import TripoSGPipeline @@ -4262,11 +4263,14 @@ class Load_Hunyuan3D_V2_ShapeGen_Pipeline: _MODES = { "Hunyuan3D-2": ("Hunyuan3D-2", "hunyuan3d-dit-v2-0", 30), - "Hunyuan3D-2-Fast": ("Hunyuan3D-2", "hunyuan3d-dit-v2-0-fast", 5), + "Hunyuan3D-2-Fast": ("Hunyuan3D-2", "hunyuan3d-dit-v2-0-fast", 20), "Hunyuan3D-2-Turbo": ("Hunyuan3D-2", "hunyuan3d-dit-v2-0-turbo", 5), "Hunyuan3D-2mini": ("Hunyuan3D-2mini", "hunyuan3d-dit-v2-mini", 30), - "Hunyuan3D-2mini-Fast": ("Hunyuan3D-2mini", "hunyuan3d-dit-v2-mini-fast", 5), + "Hunyuan3D-2mini-Fast": ("Hunyuan3D-2mini", "hunyuan3d-dit-v2-mini-fast", 20), "Hunyuan3D-2mini-Turbo": ("Hunyuan3D-2mini", "hunyuan3d-dit-v2-mini-turbo", 5), + "Hunyuan3D-2mv": ("Hunyuan3D-2mv", "hunyuan3d-dit-v2-mv", 30), + "Hunyuan3D-2mv-Fast": ("Hunyuan3D-2mv", "hunyuan3d-dit-v2-mv-fast", 20), + "Hunyuan3D-2mv-Turbo": ("Hunyuan3D-2mv", "hunyuan3d-dit-v2-mv-turbo", 5), } @classmethod @@ -4290,7 +4294,6 @@ class Load_Hunyuan3D_V2_ShapeGen_Pipeline: repo_id=f"{Load_Hunyuan3D_V2_ShapeGen_Pipeline._REPO_ID_BASE}/{repo}", repo_type="model", local_dir=base_dir, - local_dir_use_symlinks=False, resume_download=True, ignore_patterns = HF_DOWNLOAD_IGNORE ) @@ -4329,51 +4332,6 @@ class Load_Hunyuan3D_V2_ShapeGen_Pipeline: pipe = self._build_pipe(repo, subfolder, use_safe, flash_vdm) pipe.default_steps = def_steps return (pipe,) - -class Hunyuan3D_V2_ShapeGen: - - CATEGORY = "Comfy3D/Algorithm" - RETURN_TYPES = ("MESH",) - RETURN_NAMES = ("mesh",) - FUNCTION = "run" - - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "shapegen_pipe" : ("DIFFUSERS_PIPE",), - "reference_image" : ("IMAGE",), - "reference_mask" : ("MASK",), - "seed" : ("INT", {"default": 1234, "min": 0, "max": 0xffffffffffffffff}), - "guidance_scale" : ("FLOAT", {"default": 5.0, "min": 0.0, "step": 0.1}), - "num_inference_steps": ("INT", {"default": 5, "min": 0}), - "octree_resolution" : ("INT", {"default": 256, "min": 64}), - } - } - - @torch.no_grad() - def run(self, shapegen_pipe, reference_image, reference_mask, seed, - guidance_scale, num_inference_steps, octree_resolution): - - img = torch_imgs_to_pils(reference_image, reference_mask)[0] - - steps = (shapegen_pipe.default_steps if num_inference_steps == 0 - else num_inference_steps) - gen = torch.Generator(device=shapegen_pipe.device).manual_seed(seed) - - mesh = shapegen_pipe( - image = img, - num_inference_steps = steps, - guidance_scale = guidance_scale, - generator = gen, - octree_resolution = octree_resolution, - output_type = "trimesh", - )[0] - - for fn in (FloaterRemover(), DegenerateFaceRemover(), FaceReducer()): - mesh = fn(mesh) - - return (Mesh.load_trimesh(given_mesh=mesh),) class Load_Hunyuan3D_V2_TexGen_Pipeline: CATEGORY = "Comfy3D/Algorithm" @@ -4403,7 +4361,6 @@ class Load_Hunyuan3D_V2_TexGen_Pipeline: repo_id=repo_id, repo_type="model", local_dir=base_dir, - local_dir_use_symlinks=False, resume_download=True, ignore_patterns = HF_DOWNLOAD_IGNORE ) @@ -4418,33 +4375,154 @@ class Load_Hunyuan3D_V2_TexGen_Pipeline: ) return (pipe.to("cuda",torch.float16),) -class Hunyuan3D_V2_Paint_Model_Turbo: - CATEGORY = "Comfy3D/Algorithm" - RETURN_TYPES = ("MESH",) - RETURN_NAMES = ("mesh",) - FUNCTION = "run" +class Hunyuan3D_V2_Paint_Model_Turbo_MV: + """ + Texture-painting pipeline using a list of PIL images. + If list contains 1 image → single-view; if >1 → multi-view mode. + """ + + CATEGORY = "Comfy3D/Algorithm" + RETURN_TYPES = ("MESH",) + RETURN_NAMES = ("mesh",) + FUNCTION = "run" @classmethod def INPUT_TYPES(cls): - return {"required": { - "hunyuan3d_v2_texgen_pipe": ("DIFFUSERS_PIPE",), - "reference_image": ("IMAGE",), - "reference_mask": ("MASK",), - "mesh": ("MESH",), - }} + return { + "required": { + "hunyuan3d_v2_texgen_pipe": ("DIFFUSERS_PIPE",), + "mesh": ("MESH",), + "images": ("LIST",), + } + } @torch.no_grad() - def run(self, hunyuan3d_v2_texgen_pipe, - reference_image, reference_mask, mesh): - - img_pil = torch_imgs_to_pils(reference_image, reference_mask)[0] + def run(self, hunyuan3d_v2_texgen_pipe, mesh, images): + if not isinstance(images, list) or len(images) == 0: + raise Exception("[Hunyuan3D_V2_Paint_Model_Turbo_MV] 'images' must be a non-empty list of PIL images") v_np = mesh.v.detach().cpu().numpy() f_np = mesh.f.detach().cpu().numpy() - tri = trimesh.Trimesh(vertices=v_np, faces=f_np) + tri = trimesh.Trimesh(vertices=v_np, faces=f_np) + + try: + textured = hunyuan3d_v2_texgen_pipe(tri, images) + except Exception as e: + raise Exception(f"[Hunyuan3D_V2_Paint_Model_Turbo_MV] Texture generation failed: {str(e)}") - textured = hunyuan3d_v2_texgen_pipe(tri, img_pil) m_out = Mesh.load_trimesh(given_mesh=textured) m_out.auto_normal() return (m_out,) - + +class Multi_Background_Remover: + """ + Converts 1 to 4 image inputs (front/back/left/right) to a list of processed PIL images. + Applies RGBA conversion and background removal. + Suitable for feeding directly into ShapeGen or Paint models. + """ + + CATEGORY = "Comfy3D/Preprocessors" + RETURN_TYPES = ("LIST",) # List of PIL images + RETURN_NAMES = ("images",) + FUNCTION = "run" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image_front": ("IMAGE",), + }, + "optional": { + "image_back": ("IMAGE",), + "image_left": ("IMAGE",), + } + } + + @torch.no_grad() + def run( + self, + image_front, + image_back=None, + image_left=None + ): + rmbg = BackgroundRemover() + + mv_inputs = { + k: v for k, v in { + "front": image_front, + "back": image_back, + "left": image_left + }.items() if v is not None + } + + images = [] + for key, tensor_img in mv_inputs.items(): + pil_img = torch_imgs_to_pils(tensor_img)[0].convert("RGBA") + if pil_img.mode == "RGB": + pil_img = rmbg(pil_img.convert("RGB")) + images.append(pil_img) + + return (images,) + +class Hunyuan3D_V2_ShapeGen_MV: + """ + Shape generation pipeline using a list of processed PIL images. + If len(images) == 1 → single-view; if >1 → multi-view dict. + """ + + CATEGORY = "Comfy3D/Algorithm" + RETURN_TYPES = ("MESH",) + RETURN_NAMES = ("mesh",) + FUNCTION = "run" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "shapegen_pipe": ("DIFFUSERS_PIPE",), + "images": ("LIST",), + "seed": ("INT", {"default": 1234, "min": 0, "max": 0xffffffffffffffff}), + "guidance_scale": ("FLOAT", {"default": 5.0, "min": 0.0, "step": 0.1}), + "num_inference_steps": ("INT", {"default": 5, "min": 0, "tooltip": "Turbo: 5; Fast/Standard: 30–40"}), + "octree_resolution": ("INT", {"default": 256, "min": 64}), + } + } + + @torch.no_grad() + def run( + self, + shapegen_pipe, + images, + seed=1234, + guidance_scale=5.0, + num_inference_steps=5, + octree_resolution=256 + ): + if not isinstance(images, list) or len(images) == 0: + raise Exception("[Hunyuan3D_V2_ShapeGen_MV] 'images' must be a non-empty list of PIL images") + + if len(images) == 1: + image = images[0] + else: + directions = ["front", "back", "left"] + image = {k: v for k, v in zip(directions, images)} + + steps = shapegen_pipe.default_steps if num_inference_steps == 0 else num_inference_steps + gen = torch.Generator(device=shapegen_pipe.device).manual_seed(seed) + + try: + mesh = shapegen_pipe( + image=image, + num_inference_steps=steps, + guidance_scale=guidance_scale, + generator=gen, + octree_resolution=octree_resolution, + output_type="trimesh", + )[0] + except Exception as e: + raise Exception(f"[Hunyuan3D_V2_ShapeGen_MV] Shape generation failed: {str(e)}") + + for fn in (FloaterRemover(), DegenerateFaceRemover(), FaceReducer()): + mesh = fn(mesh) + + return (Mesh.load_trimesh(given_mesh=mesh),) \ No newline at end of file