diff --git a/Gen_3D_Modules/MV_Adapter/mvadapter/utils/mesh_utils/render.py b/Gen_3D_Modules/MV_Adapter/mvadapter/utils/mesh_utils/render.py index e6d76cf..704f2d3 100644 --- a/Gen_3D_Modules/MV_Adapter/mvadapter/utils/mesh_utils/render.py +++ b/Gen_3D_Modules/MV_Adapter/mvadapter/utils/mesh_utils/render.py @@ -30,7 +30,7 @@ class RenderOutput: class NVDiffRastContextWrapper: def __init__(self, device: str, context_type: str = "gl"): if context_type == "gl": - self.ctx = dr.RasterizeGLContext(device=device) + self.ctx = dr.RasterizeCudaContext(device=device) elif context_type == "cuda": self.ctx = dr.RasterizeCudaContext(device=device) else: diff --git a/Gen_3D_Modules/MV_Adapter/mvadapter_node_utils.py b/Gen_3D_Modules/MV_Adapter/mvadapter_node_utils.py index da88942..3a98854 100644 --- a/Gen_3D_Modules/MV_Adapter/mvadapter_node_utils.py +++ b/Gen_3D_Modules/MV_Adapter/mvadapter_node_utils.py @@ -10,7 +10,7 @@ from typing import Optional, Union, List from torchvision import transforms from transformers import AutoModelForImageSegmentation -from diffusers import AutoencoderKL, DDPMScheduler, LCMScheduler, UNet2DConditionModel +from diffusers import AutoencoderKL, DDPMScheduler, UNet2DConditionModel from huggingface_hub import snapshot_download from .mvadapter.pipelines.pipeline_mvadapter_i2mv_sdxl import MVAdapterI2MVSDXLPipeline @@ -37,10 +37,9 @@ except ImportError: def prepare_pipeline( base_model: str, vae_model: Optional[str] = None, - unet_model: Optional[str] = None, lora_model: Optional[str] = None, adapter_path: str = "huanngzh/mv-adapter", - scheduler: str = "default", + scheduler: str = "ddpm", num_views: int = 6, device: str = "cuda", dtype: torch.dtype = torch.float16, @@ -53,40 +52,30 @@ def prepare_pipeline( Args: base_model: Base Stable Diffusion XL model vae_model: Custom VAE model (optional) - unet_model: Custom UNet model (optional) lora_model: LoRA model (optional) adapter_path: Path to adapter weights - scheduler: Scheduler type ("default", "ddpm", "lcm") + scheduler: Scheduler type ("ddpm") num_views: Number of views to generate device: Device for computation dtype: Data type use_mmgp: Use mmgp for memory optimization adapter_local_path: Local path for adapter download """ - # Load vae and unet if provided + # Load vae if provided pipe_kwargs = {} if vae_model is not None: pipe_kwargs["vae"] = AutoencoderKL.from_pretrained(vae_model) - if unet_model is not None: - pipe_kwargs["unet"] = UNet2DConditionModel.from_pretrained(unet_model) - + # Prepare pipeline pipe = MVAdapterI2MVSDXLPipeline.from_pretrained(base_model, **pipe_kwargs) - # Load scheduler if provided - if scheduler != "default": - scheduler_class = None - if scheduler == "ddpm": - scheduler_class = DDPMScheduler - elif scheduler == "lcm": - scheduler_class = LCMScheduler - - pipe.scheduler = ShiftSNRScheduler.from_scheduler( - pipe.scheduler, - shift_mode="interpolated", - shift_scale=8.0, - scheduler_class=scheduler_class, - ) + # Load DDPM scheduler + pipe.scheduler = ShiftSNRScheduler.from_scheduler( + pipe.scheduler, + shift_mode="interpolated", + shift_scale=8.0, + scheduler_class=DDPMScheduler, + ) pipe.init_custom_adapter( num_views=num_views, self_attn_processor=DecoupledMVRowColSelfAttnProcessor2_0 @@ -130,32 +119,6 @@ def prepare_pipeline( return pipe -# def create_bg_remover(device: str = "cuda"): -# """Create background removal function""" -# birefnet = AutoModelForImageSegmentation.from_pretrained( -# "ZhengPeng7/BiRefNet", trust_remote_code=True -# ) -# birefnet.to(device) -# transform_image = transforms.Compose([ -# transforms.Resize((1024, 1024)), -# transforms.ToTensor(), -# transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), -# ]) - -# def remove_bg(image): -# image_size = image.size -# input_images = transform_image(image).unsqueeze(0).to(device) -# with torch.no_grad(): -# preds = birefnet(input_images)[-1].sigmoid().cpu() -# pred = preds[0].squeeze() -# pred_pil = transforms.ToPILImage()(pred) -# mask = pred_pil.resize(image_size) -# image.putalpha(mask) -# return image - -# return remove_bg - - def preprocess_image(image: Image.Image, height: int, width: int) -> Image.Image: """Preprocess image for MV-Adapter""" image = np.array(image) @@ -251,9 +214,6 @@ def run_pipeline( # Prepare image reference_image = Image.open(image) if isinstance(image, str) else image - # if remove_bg_fn is not None: - # reference_image = remove_bg_fn(reference_image) - # reference_image = preprocess_image(reference_image, height, width) if reference_image.mode == "RGBA": reference_image = preprocess_image(reference_image, height, width) @@ -301,158 +261,6 @@ def pils_to_torch_tensor(images: List[Image.Image]) -> torch.Tensor: return torch.stack(tensors, dim=0) -def prepare_t2mv_pipeline( - base_model: str, - vae_model: Optional[str] = None, - unet_model: Optional[str] = None, - lora_model: Optional[str] = None, - adapter_path: str = "huanngzh/mv-adapter", - scheduler: str = "default", - num_views: int = 6, - device: str = "cuda", - dtype: torch.dtype = torch.float16, - use_mmgp: bool = False, - adapter_local_path: Optional[str] = None, -): - """ - Prepare MV-Adapter T2MV pipeline - - Args: - base_model: Base Stable Diffusion XL model - vae_model: Custom VAE model (optional) - unet_model: Custom UNet model (optional) - lora_model: LoRA model (optional) - adapter_path: Path to adapter weights - scheduler: Scheduler type ("default", "ddpm", "lcm") - num_views: Number of views to generate - device: Device for computation - dtype: Data type - use_mmgp: Use mmgp for memory optimization - adapter_local_path: Local path for adapter download - """ - # Load vae and unet if provided - pipe_kwargs = {} - if vae_model is not None: - pipe_kwargs["vae"] = AutoencoderKL.from_pretrained(vae_model) - if unet_model is not None: - pipe_kwargs["unet"] = UNet2DConditionModel.from_pretrained(unet_model) - - # Prepare pipeline - pipe = MVAdapterT2MVSDXLPipeline.from_pretrained(base_model, **pipe_kwargs) - - # Load scheduler if provided - if scheduler != "default": - scheduler_class = None - if scheduler == "ddpm": - scheduler_class = DDPMScheduler - elif scheduler == "lcm": - scheduler_class = LCMScheduler - - pipe.scheduler = ShiftSNRScheduler.from_scheduler( - pipe.scheduler, - shift_mode="interpolated", - shift_scale=8.0, - scheduler_class=scheduler_class, - ) - - pipe.init_custom_adapter(num_views=num_views) - - # Load adapter weights - weight_name = "mvadapter_t2mv_sdxl.safetensors" - pipe.load_custom_adapter(adapter_path, weight_name=weight_name, local_cache_dir=adapter_local_path) - - pipe.to(device=device, dtype=dtype) - pipe.cond_encoder.to(device=device, dtype=dtype) - - # load lora if provided - if lora_model is not None: - model_, name_ = lora_model.rsplit("/", 1) - pipe.load_lora_weights(model_, weight_name=name_) - - # Apply mmgp optimization if available and requested - if use_mmgp and MMGP_AVAILABLE: - import gc - torch.cuda.empty_cache() - gc.collect() - - all_models = { - "vae": pipe.vae, - "text_encoder": pipe.text_encoder, - "text_encoder_2": pipe.text_encoder_2, - "unet": pipe.unet, - "cond_encoder": pipe.cond_encoder, - } - - if hasattr(pipe, 'image_encoder') and pipe.image_encoder is not None: - all_models["image_encoder"] = pipe.image_encoder - - offload.profile(all_models, profile_type.HighRAM_LowVRAM) - print("mmgp profiling applied successfully.") - - # Enable VAE slicing for memory efficiency - pipe.enable_vae_slicing() - - return pipe - - -def run_t2mv_pipeline( - pipe, - num_views: int, - text: str, - height: int, - width: int, - num_inference_steps: int, - guidance_scale: float, - seed: int, - negative_prompt: str = "watermark, ugly, deformed, noisy, blurry, low contrast", - lora_scale: float = 1.0, - device: str = "cuda", - azimuth_deg: Optional[List[int]] = None, -): - """ - Execute text-to-multiview generation - """ - # Prepare cameras - if azimuth_deg is None: - azimuth_deg = [0, 45, 90, 180, 270, 315][:num_views] - - cameras = get_orthogonal_camera( - elevation_deg=[0] * num_views, - distance=[1.8] * num_views, - left=-0.55, - right=0.55, - bottom=-0.55, - top=0.55, - azimuth_deg=[x - 90 for x in azimuth_deg], - device=device, - ) - - plucker_embeds = get_plucker_embeds_from_cameras_ortho( - cameras.c2w, [1.1] * num_views, width - ) - control_images = ((plucker_embeds + 1.0) / 2.0).clamp(0, 1) - - pipe_kwargs = {} - if seed != -1 and isinstance(seed, int): - pipe_kwargs["generator"] = torch.Generator(device=device).manual_seed(seed) - - images = pipe( - text, - height=height, - width=width, - num_inference_steps=num_inference_steps, - guidance_scale=guidance_scale, - num_images_per_prompt=num_views, - control_image=control_images, - control_conditioning_scale=1.0, - negative_prompt=negative_prompt, - cross_attention_kwargs={"scale": lora_scale}, - **pipe_kwargs, - ).images - - return images - - def prepare_texture_pipeline( upscaler_ckpt_path: Optional[str] = None, inpaint_ckpt_path: Optional[str] = None, @@ -563,7 +371,6 @@ def run_texture_pipeline( if camera_elevation_deg is None: camera_elevation_deg = [0, 0, 0, 0, 89.99, -89.99] - # Adjust azimuth angles (subtract 90 as in original code) azimuth_deg_corrected = [x - 90 for x in camera_azimuth_deg] grid_image = make_image_grid(multiview_images, rows=1) @@ -641,7 +448,6 @@ def download_texture_checkpoints(texture_ckpt_dir, upscaler_ckpt_path, inpaint_c except (subprocess.CalledProcessError, FileNotFoundError): pass - # Try curl if wget failed if not success: try: subprocess.run([ @@ -652,7 +458,6 @@ def download_texture_checkpoints(texture_ckpt_dir, upscaler_ckpt_path, inpaint_c except (subprocess.CalledProcessError, FileNotFoundError): pass - # Try Python urllib as fallback if not success: try: import urllib.request @@ -662,11 +467,195 @@ def download_texture_checkpoints(texture_ckpt_dir, upscaler_ckpt_path, inpaint_c except Exception as e: print(f"Failed to download {ckpt['name']} using urllib: {e}") - # If all methods failed if not success: print(f"Failed to download {ckpt['name']}. Please download manually:") print(f"wget {ckpt['url']} -O {ckpt['path']}") print(f"or") print(f"curl -L {ckpt['url']} -o {ckpt['path']}") else: - print(f"{ckpt['name']} already exists at {ckpt['path']}") \ No newline at end of file + print(f"{ckpt['name']} already exists at {ckpt['path']}") + + +def prepare_tg2mv_pipeline( + base_model: str, + vae_model: Optional[str] = None, + lora_model: Optional[str] = None, + adapter_path: str = "huanngzh/mv-adapter", + scheduler: str = "ddpm", + num_views: int = 6, + device: str = "cuda", + dtype: torch.dtype = torch.float16, + use_mmgp: bool = False, + adapter_local_path: Optional[str] = None, +): + """ + Prepare MV-Adapter TG2MV (Text-Guided to Multi-View) pipeline + + Args: + base_model: Base Stable Diffusion XL model + vae_model: Custom VAE model (optional) + lora_model: LoRA model (optional) + adapter_path: Path to adapter weights + scheduler: Scheduler type ("ddpm") + num_views: Number of views to generate + device: Device for computation + dtype: Data type + use_mmgp: Use mmgp for memory optimization + adapter_local_path: Local path for adapter download + """ + import gc + torch.cuda.empty_cache() + gc.collect() + + pipe_kwargs = {} + if vae_model is not None: + pipe_kwargs["vae"] = AutoencoderKL.from_pretrained(vae_model, torch_dtype=dtype) + + pipe = MVAdapterT2MVSDXLPipeline.from_pretrained(base_model, torch_dtype=dtype, **pipe_kwargs) + + pipe.scheduler = ShiftSNRScheduler.from_scheduler( + pipe.scheduler, + shift_mode="interpolated", + shift_scale=8.0, + scheduler_class=DDPMScheduler, + ) + + pipe.init_custom_adapter( + num_views=num_views, self_attn_processor=DecoupledMVRowColSelfAttnProcessor2_0 + ) + + weight_name = "mvadapter_tg2mv_sdxl.safetensors" + pipe.load_custom_adapter(adapter_path, weight_name=weight_name, local_cache_dir=adapter_local_path) + + pipe.to(device=device, dtype=dtype) + pipe.cond_encoder.to(device=device, dtype=dtype) + + if lora_model is not None: + model_, name_ = lora_model.rsplit("/", 1) + pipe.load_lora_weights(model_, weight_name=name_) + + pipe.enable_vae_slicing() + pipe.enable_vae_tiling() + + if hasattr(pipe, 'enable_model_cpu_offload'): + if not use_mmgp: + pipe.enable_model_cpu_offload() + print("[INFO] Включен CPU offload для экономии VRAM") + + if use_mmgp and MMGP_AVAILABLE: + torch.cuda.empty_cache() + gc.collect() + + all_models = { + "vae": pipe.vae, + "text_encoder": pipe.text_encoder, + "text_encoder_2": pipe.text_encoder_2, + "unet": pipe.unet, + "cond_encoder": pipe.cond_encoder, + } + + if hasattr(pipe, 'image_encoder') and pipe.image_encoder is not None: + all_models["image_encoder"] = pipe.image_encoder + + try: + offload.profile(all_models, profile_type.HighRAM_LowVRAM) + print("mmgp profiling applied successfully.") + except Exception as e: + print(f"[WARNING] mmgp failed, continuing without it: {e}") + + torch.cuda.empty_cache() + gc.collect() + + return pipe + + +def run_tg2mv_pipeline( + pipe, + mesh_path: str, + num_views: int, + text: str, + height: int, + width: int, + num_inference_steps: int, + guidance_scale: float, + seed: int, + negative_prompt: str = "watermark, ugly, deformed, noisy, blurry, low contrast", + lora_scale: float = 1.0, + device: str = "cuda", +): + """ + Execute text-guided multiview generation (TG2MV) + + Args: + pipe: TG2MV pipeline + mesh_path: Path to 3D mesh file for guidance + num_views: Number of views to generate + text: Text prompt + height: Image height + width: Image width + num_inference_steps: Number of denoising steps + guidance_scale: Guidance scale + seed: Random seed + negative_prompt: Negative prompt + lora_scale: LoRA scale + device: Device for computation + """ + cameras = get_orthogonal_camera( + elevation_deg=[0, 0, 0, 0, 89.99, -89.99], + distance=[1.8] * num_views, + left=-0.55, + right=0.55, + bottom=-0.55, + top=0.55, + azimuth_deg=[x - 90 for x in [0, 90, 180, 270, 180, 180]], + device=device, + ) + ctx = NVDiffRastContextWrapper(device=device) + + mesh = load_mesh(mesh_path, rescale=True, device=device) + render_out = render( + ctx, + mesh, + cameras, + height=height, + width=width, + render_attr=False, + normal_background=0.0, + ) + + pos_images = tensor_to_image((render_out.pos + 0.5).clamp(0, 1), batched=True) + normal_images = tensor_to_image( + (render_out.normal / 2 + 0.5).clamp(0, 1), batched=True + ) + + control_images = ( + torch.cat( + [ + (render_out.pos + 0.5).clamp(0, 1), + (render_out.normal / 2 + 0.5).clamp(0, 1), + ], + dim=-1, + ) + .permute(0, 3, 1, 2) + .to(device) + ) + + pipe_kwargs = {} + if seed != -1 and isinstance(seed, int): + pipe_kwargs["generator"] = torch.Generator(device=device).manual_seed(seed) + + images = pipe( + text, + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + num_images_per_prompt=num_views, + control_image=control_images, + control_conditioning_scale=1.0, + negative_prompt=negative_prompt, + cross_attention_kwargs={"scale": lora_scale}, + **pipe_kwargs, + ).images + + return images, pos_images, normal_images \ No newline at end of file diff --git a/Gen_3D_Modules/Stable3DGen/__init__.py b/Gen_3D_Modules/Stable3DGen/__init__.py index 6a1207b..22a7919 100644 --- a/Gen_3D_Modules/Stable3DGen/__init__.py +++ b/Gen_3D_Modules/Stable3DGen/__init__.py @@ -24,18 +24,18 @@ if parent_dir not in sys.path: trellis_path = os.path.join(current_dir, "trellis") if os.path.exists(trellis_path) and trellis_path not in sys.path: sys.path.insert(0, trellis_path) - logger.info(f"Added trellis path to sys.path: {trellis_path}") + # logger.info(f"Added trellis path to sys.path: {trellis_path}") # Add stablx package path stablex_path = os.path.join(current_dir, "stablex") if os.path.exists(trellis_path) and trellis_path not in sys.path: sys.path.insert(0, trellis_path) - logger.info(f"Added stablex path to sys.path: {trellis_path}") + # logger.info(f"Added stablex path to sys.path: {trellis_path}") # Verify trellis package is importable try: import trellis - logger.info("Trellis package imported successfully") + # logger.info("Trellis package imported successfully") except ImportError as e: logger.error(f"Failed to import trellis package: {e}") logger.error(f"Current sys.path: {sys.path}") @@ -44,7 +44,7 @@ except ImportError as e: # Verify stablex package is importable try: import stablex - logger.info("stablex package imported successfully") + # logger.info("stablex package imported successfully") except ImportError as e: logger.error(f"Failed to import stablex package: {e}") logger.error(f"Current sys.path: {sys.path}") diff --git a/Gen_3D_Modules/Stable3DGen/trellis/pipelines/samplers/flow_euler.py b/Gen_3D_Modules/Stable3DGen/trellis/pipelines/samplers/flow_euler.py index 938bd8b..b453d01 100644 --- a/Gen_3D_Modules/Stable3DGen/trellis/pipelines/samplers/flow_euler.py +++ b/Gen_3D_Modules/Stable3DGen/trellis/pipelines/samplers/flow_euler.py @@ -112,12 +112,13 @@ class FlowEulerSampler(Sampler): t_pairs = list((t_seq[i], t_seq[i + 1]) for i in range(steps)) ret = edict({"samples": None, "pred_x_t": [], "pred_x_0": []}) comfy_pbar = comfy.utils.ProgressBar(steps) - for t, t_prev in tqdm(t_pairs, desc="Sampling", disable=not verbose): + for i, (t, t_prev) in enumerate(tqdm(t_pairs, desc="Sampling", disable=not verbose)): out = self.sample_once(model, sample, t, t_prev, cond, **kwargs) sample = out.pred_x_prev ret.pred_x_t.append(out.pred_x_prev) ret.pred_x_0.append(out.pred_x_0) - comfy_pbar.update_absolute(i + 1) + comfy_pbar.update_absolute(i + 1) + ret.samples = sample return ret diff --git a/README.md b/README.md index 57fd734..c05609b 100644 --- a/README.md +++ b/README.md @@ -49,15 +49,15 @@ Your ComfyUI Root Directory\python_embeded\python.exe install.py - **MV-Adapter**: [huanngzh/MV-Adapter](https://github.com/huanngzh/MV-Adapter) - Two generation methods: - IG2MV: Single image + 3D mesh to multi-view images - - T2MV: Text prompt to multi-view images + - TG2MV: Text prompt + 3D mesh to multi-view images - Texturing: Grid image + 3D mesh to textured mesh - Model weights: https://huggingface.co/huanngzh/mv-adapter - - Workflows: [IG2MV](./_Example_Workflows/MV-Adapter/MV-Adapter-ig2mv.json), [T2MV](./_Example_Workflows/MV-Adapter/MV-Adapter-t2mv.json), [Texturing](./_Example_Workflows/MV-Adapter/MV-Adapter-Texturing.json) + - Workflows: [IG2MV](./_Example_Workflows/MV-Adapter/MV-Adapter-ig2mv.json), [T2MV](./_Example_Workflows/MV-Adapter/MV-Adapter-tg2mv.json), [Texturing](./_Example_Workflows/MV-Adapter/MV-Adapter-Texturing.json) - - - - + + + + - **Stable3DGen**: [Stable-X/Stable3DGen](https://github.com/Stable-X/Stable3DGen) - Two models pipeline: - Stable3DGen: Single image to 3D Mesh diff --git a/_Example_Workflows/MV-Adapter/MV-Adapter-Texturing.json b/_Example_Workflows/MV-Adapter/MV-Adapter-Texturing.json index 0e82ea2..9c658e3 100644 --- a/_Example_Workflows/MV-Adapter/MV-Adapter-Texturing.json +++ b/_Example_Workflows/MV-Adapter/MV-Adapter-Texturing.json @@ -106,43 +106,6 @@ true ] }, - { - "id": 15, - "type": "LoadImage", - "pos": [ - 4655.7041015625, - 132.22010803222656 - ], - "size": [ - 325.982177734375, - 338.67474365234375 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 39 - ] - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "squirel_girl_grid.png", - "image" - ] - }, { "id": 26, "type": "[Comfy3D] MVAdapter Texture Projection", @@ -216,6 +179,43 @@ false ] }, + { + "id": 15, + "type": "LoadImage", + "pos": [ + 4655.7041015625, + 132.22010803222656 + ], + "size": [ + 325.982177734375, + 338.67474365234375 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 39 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "matermulti.png", + "image" + ] + }, { "id": 4, "type": "PrimitiveString", @@ -245,7 +245,7 @@ "Node name for S&R": "PrimitiveString" }, "widgets_values": [ - "output/StableGenTest/mesh.glb" + "output/StableGenTest/mater.glb" ] } ], @@ -295,15 +295,15 @@ "config": {}, "extra": { "ds": { - "scale": 4.321942375150668, + "scale": 0.8227024747918819, "offset": [ - -4633.552682414396, - -263.1857272782662 + -4472.103767929271, + 239.69671269689613 ] }, "frontendVersion": "1.20.4", "node_versions": { - "ComfyUI-3D-Pack": "626a5ee60d6fbe3d76d69cd89261d445dbca2bc4", + "ComfyUI-3D-Pack": "1f6bd293b66069de8bb2182bea8308a9441c961e", "comfy-core": "0.3.35" }, "VHS_latentpreview": false, diff --git a/_Example_Workflows/MV-Adapter/MV-Adapter-ig2mv.json b/_Example_Workflows/MV-Adapter/MV-Adapter-ig2mv.json index 074747f..bed43a9 100644 --- a/_Example_Workflows/MV-Adapter/MV-Adapter-ig2mv.json +++ b/_Example_Workflows/MV-Adapter/MV-Adapter-ig2mv.json @@ -1,36 +1,9 @@ { "id": "867889cd-eeb4-48b0-853c-21e125a93896", "revision": 0, - "last_node_id": 28, - "last_link_id": 25, + "last_node_id": 34, + "last_link_id": 41, "nodes": [ - { - "id": 8, - "type": "PreviewImage", - "pos": [ - 3033.075927734375, - -927.3829956054688 - ], - "size": [ - 370.7532958984375, - 288.39501953125 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, { "id": 26, "type": "[Comfy3D] Make Image Grid", @@ -49,7 +22,7 @@ { "name": "images", "type": "IMAGE", - "link": 22 + "link": 39 } ], "outputs": [ @@ -69,16 +42,180 @@ true ] }, + { + "id": 8, + "type": "PreviewImage", + "pos": [ + 3033.075927734375, + -927.3829956054688 + ], + "size": [ + 370.7532958984375, + 288.39501953125 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 38 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 32, + "type": "[Comfy3D] Load MVAdapter IG2MV Pipeline", + "pos": [ + 2138.81591796875, + -1144.373291015625 + ], + "size": [ + 353.38751220703125, + 238.59991455078125 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "mvadapter_pipe", + "type": "DIFFUSERS_PIPE", + "links": [ + 36 + ] + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Load MVAdapter IG2MV Pipeline" + }, + "widgets_values": [ + "stabilityai/stable-diffusion-xl-base-1.0", + "madebyollin/sdxl-vae-fp16-fix", + "huanngzh/mv-adapter", + "ddpm", + 6, + true, + true, + "" + ] + }, + { + "id": 3, + "type": "LoadImage", + "pos": [ + 2151.8310546875, + -841.9072265625 + ], + "size": [ + 346.8450622558594, + 326 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 37 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "squirrel_girl_1.png", + "image" + ] + }, + { + "id": 34, + "type": "[Comfy3D] MVAdapter IG2MV", + "pos": [ + 2552.5146484375, + -933.671875 + ], + "size": [ + 400, + 402 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "mvadapter_pipe", + "type": "DIFFUSERS_PIPE", + "link": 36 + }, + { + "name": "reference_image", + "type": "IMAGE", + "link": 37 + }, + { + "name": "mesh_path", + "type": "STRING", + "widget": { + "name": "mesh_path" + }, + "link": 41 + } + ], + "outputs": [ + { + "name": "multiview_images", + "type": "IMAGE", + "links": [ + 38, + 39 + ] + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] MVAdapter IG2MV" + }, + "widgets_values": [ + "", + "high quality", + "watermark, ugly, deformed, noisy, blurry, low contrast", + 50, + 3, + 1, + 768, + 768, + 459568704818520, + "randomize", + false, + 1 + ] + }, { "id": 27, "type": "PreviewImage", "pos": [ - 3032.88818359375, - -449.5286865234375 + 3035.859130859375, + -443.73687744140625 ], "size": [ 420.0350036621094, - 123.75502014160156 + 246 ], "flags": {}, "order": 6, @@ -96,43 +233,6 @@ }, "widgets_values": [] }, - { - "id": 3, - "type": "LoadImage", - "pos": [ - 2153.30078125, - -839.7021484375 - ], - "size": [ - 346.8450622558594, - 326 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 19 - ] - }, - { - "name": "MASK", - "type": "MASK", - "links": null - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "squirrel_girl_1.png", - "image" - ] - }, { "id": 9, "type": "PrimitiveString", @@ -145,7 +245,7 @@ 67.5549545288086 ], "flags": {}, - "order": 1, + "order": 2, "mode": 0, "inputs": [], "outputs": [ @@ -153,152 +253,20 @@ "name": "STRING", "type": "STRING", "links": [ - 7 + 41 ] } ], + "title": "Mesh Path", "properties": { "Node name for S&R": "PrimitiveString" }, "widgets_values": [ "output/StableGenTest/mesh.glb" ] - }, - { - "id": 7, - "type": "[Comfy3D] MVAdapter Image To MultiView", - "pos": [ - 2557.745361328125, - -938.2529296875 - ], - "size": [ - 406.614990234375, - 407.14501953125 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "mvadapter_pipe", - "type": "DIFFUSERS_PIPE", - "link": 25 - }, - { - "name": "reference_image", - "type": "IMAGE", - "link": 19 - }, - { - "name": "mesh_path", - "type": "STRING", - "widget": { - "name": "mesh_path" - }, - "link": 7 - } - ], - "outputs": [ - { - "name": "multiview_images", - "type": "IMAGE", - "links": [ - 6, - 22 - ] - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] MVAdapter Image To MultiView" - }, - "widgets_values": [ - "", - "high quality", - "watermark, ugly, deformed, noisy, blurry, low contrast", - 50, - 3, - 1, - 768, - 768, - 710491314281528, - "randomize", - false, - 1 - ] - }, - { - "id": 28, - "type": "[Comfy3D] Load MVAdapter IG2MV Pipeline", - "pos": [ - 2154.555908203125, - -1155.862060546875 - ], - "size": [ - 357.2724609375, - 257.3499755859375 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "mvadapter_pipe", - "type": "DIFFUSERS_PIPE", - "links": [ - 25 - ] - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Load MVAdapter IG2MV Pipeline" - }, - "widgets_values": [ - "stabilityai/stable-diffusion-xl-base-1.0", - "madebyollin/sdxl-vae-fp16-fix", - "huanngzh/mv-adapter", - "default", - 6, - true, - true, - "", - "" - ] } ], "links": [ - [ - 6, - 7, - 0, - 8, - 0, - "IMAGE" - ], - [ - 7, - 9, - 0, - 7, - 2, - "STRING" - ], - [ - 19, - 3, - 0, - 7, - 1, - "IMAGE" - ], - [ - 22, - 7, - 0, - 26, - 0, - "IMAGE" - ], [ 23, 26, @@ -308,28 +276,60 @@ "IMAGE" ], [ - 25, - 28, + 36, + 32, 0, - 7, + 34, 0, "DIFFUSERS_PIPE" + ], + [ + 37, + 3, + 0, + 34, + 1, + "IMAGE" + ], + [ + 38, + 34, + 0, + 8, + 0, + "IMAGE" + ], + [ + 39, + 34, + 0, + 26, + 0, + "IMAGE" + ], + [ + 41, + 9, + 0, + 34, + 2, + "STRING" ] ], "groups": [], "config": {}, "extra": { "ds": { - "scale": 0.9070294784580498, + "scale": 0.9523809523809523, "offset": [ - -1941.4116551684099, - 1211.3556830868408 + -1889.6151044555788, + 1117.4222501296233 ] }, "frontendVersion": "1.20.4", "node_versions": { - "comfy-core": "0.3.35", - "ComfyUI-3D-Pack": "626a5ee60d6fbe3d76d69cd89261d445dbca2bc4" + "ComfyUI-3D-Pack": "1f6bd293b66069de8bb2182bea8308a9441c961e", + "comfy-core": "0.3.35" }, "VHS_latentpreview": false, "VHS_latentpreviewrate": 0, diff --git a/_Example_Workflows/MV-Adapter/MV-Adapter-t2mv.json b/_Example_Workflows/MV-Adapter/MV-Adapter-t2mv.json deleted file mode 100644 index 11643d8..0000000 --- a/_Example_Workflows/MV-Adapter/MV-Adapter-t2mv.json +++ /dev/null @@ -1,160 +0,0 @@ -{ - "id": "fd4d59aa-116b-4ec5-920e-f918c36151fe", - "revision": 0, - "last_node_id": 3, - "last_link_id": 2, - "nodes": [ - { - "id": 2, - "type": "PreviewImage", - "pos": [ - 2673.054931640625, - -795.7294311523438 - ], - "size": [ - 291.89996337890625, - 246 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 1 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, - { - "id": 1, - "type": "[Comfy3D] MVAdapter Text To MultiView", - "pos": [ - 2183.7578125, - -848.2293701171875 - ], - "size": [ - 400, - 358 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "mvadapter_t2mv_pipe", - "type": "DIFFUSERS_PIPE", - "link": 2 - } - ], - "outputs": [ - { - "name": "multiview_images", - "type": "IMAGE", - "links": [ - 1 - ] - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] MVAdapter Text To MultiView" - }, - "widgets_values": [ - "Mater, a rusty and beat-up tow truck from the 2006 Disney/Pixar animated film 'Cars', with a rusty brown exterior, big blue eyes.", - "watermark, ugly, deformed, noisy, blurry, low contrast", - 6, - 50, - 7, - 768, - 768, - 937383047416961, - "randomize", - 1, - "0,45,90,180,270,315" - ] - }, - { - "id": 3, - "type": "[Comfy3D] Load MVAdapter T2MV Pipeline", - "pos": [ - 1833.75634765625, - -913.3286743164062 - ], - "size": [ - 298.8599548339844, - 250 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "mvadapter_t2mv_pipe", - "type": "DIFFUSERS_PIPE", - "links": [ - 2 - ] - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Load MVAdapter T2MV Pipeline" - }, - "widgets_values": [ - "stabilityai/stable-diffusion-xl-base-1.0", - "madebyollin/sdxl-vae-fp16-fix", - "huanngzh/mv-adapter", - "default", - 6, - true, - false, - "", - "" - ] - } - ], - "links": [ - [ - 1, - 1, - 0, - 2, - 0, - "IMAGE" - ], - [ - 2, - 3, - 0, - 1, - 0, - "DIFFUSERS_PIPE" - ] - ], - "groups": [], - "config": {}, - "extra": { - "ds": { - "scale": 1.2155062499999998, - "offset": [ - -1674.815702398016, - 990.4808470306099 - ] - }, - "node_versions": { - "comfy-core": "0.3.35", - "ComfyUI-3D-Pack": "354ab9f7b56129f1f0fc4a42bea1751b266fd222" - }, - "frontendVersion": "1.20.4", - "VHS_latentpreview": false, - "VHS_latentpreviewrate": 0, - "VHS_MetadataImage": true, - "VHS_KeepIntermediate": true - }, - "version": 0.4 -} \ No newline at end of file diff --git a/_Example_Workflows/MV-Adapter/MV-Adapter-tg2mv.json b/_Example_Workflows/MV-Adapter/MV-Adapter-tg2mv.json new file mode 100644 index 0000000..8430d12 --- /dev/null +++ b/_Example_Workflows/MV-Adapter/MV-Adapter-tg2mv.json @@ -0,0 +1,289 @@ +{ + "id": "69e3650c-32e7-4419-9cad-665856cf7b64", + "revision": 0, + "last_node_id": 6, + "last_link_id": 5, + "nodes": [ + { + "id": 5, + "type": "[Comfy3D] Make Image Grid", + "pos": [ + 3231.359130859375, + -755.6622924804688 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 5 + } + ], + "outputs": [ + { + "name": "image_grid", + "type": "IMAGE", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Make Image Grid" + }, + "widgets_values": [ + 1, + true + ] + }, + { + "id": 6, + "type": "PreviewImage", + "pos": [ + 3233.5966796875, + -612.426513671875 + ], + "size": [ + 420.0350036621094, + 246 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 3 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 4, + "type": "PreviewImage", + "pos": [ + 3230.8134765625, + -1096.0721435546875 + ], + "size": [ + 370.7532958984375, + 288.39501953125 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 1, + "type": "[Comfy3D] Load MVAdapter TG2MV Pipeline", + "pos": [ + 2334.81494140625, + -1060.3724365234375 + ], + "size": [ + 355.9496154785156, + 226 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "mvadapter_tg2mv_pipe", + "type": "DIFFUSERS_PIPE", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Load MVAdapter TG2MV Pipeline" + }, + "widgets_values": [ + "stabilityai/stable-diffusion-xl-base-1.0", + "madebyollin/sdxl-vae-fp16-fix", + "huanngzh/mv-adapter", + "ddpm", + 6, + true, + true, + "" + ] + }, + { + "id": 2, + "type": "[Comfy3D] MVAdapter TG2MV", + "pos": [ + 2758.3154296875, + -917.5724487304688 + ], + "size": [ + 400, + 358 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "mvadapter_tg2mv_pipe", + "type": "DIFFUSERS_PIPE", + "link": 1 + }, + { + "name": "mesh_path", + "type": "STRING", + "widget": { + "name": "mesh_path" + }, + "link": 2 + } + ], + "outputs": [ + { + "name": "multiview_images", + "type": "IMAGE", + "links": [ + 4, + 5 + ] + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] MVAdapter TG2MV" + }, + "widgets_values": [ + "", + "Mater, a rusty and beat-up tow truck from the 2006 Disney/Pixar animated film 'Cars', with a rusty brown exterior, big blue eyes.", + "watermark, ugly, deformed, noisy, blurry, low contrast", + 6, + 50, + 7, + 768, + 768, + 192532995061057, + "randomize", + 1 + ] + }, + { + "id": 3, + "type": "PrimitiveString", + "pos": [ + 2330.61767578125, + -762.8717651367188 + ], + "size": [ + 343.5000305175781, + 67.5549545288086 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 2 + ] + } + ], + "title": "Mesh Path", + "properties": { + "Node name for S&R": "PrimitiveString" + }, + "widgets_values": [ + "output/StableGenTest/mater.glb" + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 2, + 0, + "DIFFUSERS_PIPE" + ], + [ + 2, + 3, + 0, + 2, + 1, + "STRING" + ], + [ + 3, + 5, + 0, + 6, + 0, + "IMAGE" + ], + [ + 4, + 2, + 0, + 4, + 0, + "IMAGE" + ], + [ + 5, + 2, + 0, + 5, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.9523809523809523, + "offset": [ + -2165.4151899047965, + 1209.1222287673186 + ] + }, + "frontendVersion": "1.20.4", + "node_versions": { + "ComfyUI-3D-Pack": "1f6bd293b66069de8bb2182bea8308a9441c961e", + "comfy-core": "0.3.35" + }, + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/_Example_Workflows/_Example_Inputs_Files/matermulti.png b/_Example_Workflows/_Example_Inputs_Files/matermulti.png new file mode 100644 index 0000000..cdf5698 Binary files /dev/null and b/_Example_Workflows/_Example_Inputs_Files/matermulti.png differ diff --git a/_Example_Workflows/_Example_Inputs_Files/squirel_girl1_grid.png b/_Example_Workflows/_Example_Inputs_Files/squirel_girl1_grid.png deleted file mode 100644 index 6718f8d..0000000 Binary files a/_Example_Workflows/_Example_Inputs_Files/squirel_girl1_grid.png and /dev/null differ diff --git a/nodes.py b/nodes.py index b9cfd8c..faac023 100644 --- a/nodes.py +++ b/nodes.py @@ -107,8 +107,8 @@ from Stable3DGen.pipeline_builders import StableGenPipelineBuilder from MV_Adapter.mvadapter_node_utils import ( prepare_pipeline as mvadapter_prepare_pipeline, run_pipeline as mvadapter_run_pipeline, - prepare_t2mv_pipeline as mvadapter_prepare_t2mv_pipeline, - run_t2mv_pipeline as mvadapter_run_t2mv_pipeline, + prepare_tg2mv_pipeline as mvadapter_prepare_tg2mv_pipeline, + run_tg2mv_pipeline as mvadapter_run_tg2mv_pipeline, prepare_texture_pipeline as mvadapter_prepare_texture_pipeline, download_texture_checkpoints, ) @@ -4787,7 +4787,7 @@ class StableGen_StableX_Process_Image: except Exception as e: raise Exception(f"[StableGen_StableX_Process_Image] Image processing failed: {str(e)}") - +# --- MV START ---- class Load_MVAdapter_IG2MV_Pipeline: """Loader pipeline for MV-Adapter (Image to Multi-View)""" CATEGORY = "Comfy3D/Algorithm" @@ -4806,45 +4806,44 @@ class Load_MVAdapter_IG2MV_Pipeline: "vae_model": (["madebyollin/sdxl-vae-fp16-fix", "None"], {"default": "madebyollin/sdxl-vae-fp16-fix"}), "adapter_path": (["huanngzh/mv-adapter"], {"default": "huanngzh/mv-adapter"}), - "scheduler": (["default", "ddpm", "lcm"], {"default": "default"}), + "scheduler": (["ddpm"], {"default": "ddpm"}), "num_views": ("INT", {"default": 6, "min": 1, "max": 16}), "use_fp16": ("BOOLEAN", {"default": True}), "use_mmgp": ("BOOLEAN", {"default": True}), }, "optional": { - "unet_model": ("STRING", {"default": ""}), "lora_model": ("STRING", {"default": ""}), } } + @classmethod def load(cls, base_model, vae_model, adapter_path, scheduler, num_views, - use_fp16, use_mmgp, unet_model="", lora_model=""): - + use_fp16, use_mmgp, lora_model=""): dtype = torch.float16 if use_fp16 else torch.float32 vae_model = None if vae_model == "None" else vae_model - unet_model = None if not unet_model else unet_model lora_model = None if not lora_model else lora_model - pipe = mvadapter_prepare_pipeline( - base_model=base_model, - vae_model=vae_model, - unet_model=unet_model, - lora_model=lora_model, - adapter_path=adapter_path, - scheduler=scheduler, - num_views=num_views, - device=DEVICE_STR, - dtype=dtype, - use_mmgp=use_mmgp, - adapter_local_path=cls.CKPT_MVADAPTER_PATH - ) + # Подготавливаем параметры для пайплайна + pipeline_kwargs = { + "base_model": base_model, + "vae_model": vae_model, + "lora_model": lora_model, + "adapter_path": adapter_path, + "scheduler": scheduler, + "num_views": num_views, + "device": DEVICE_STR, + "dtype": dtype, + "use_mmgp": use_mmgp, + "adapter_local_path": cls.CKPT_MVADAPTER_PATH + } - print("MV-Adapter pipeline loaded successfully") + pipe = mvadapter_prepare_pipeline(**pipeline_kwargs) + + print("MV-Adapter IG2MV pipeline loaded successfully") return (pipe,) - -class MVAdapter_Image_To_MultiView: +class MVAdapter_IG2MV: """Generate multi-view images from single image and 3D mesh""" CATEGORY = "Comfy3D/Algorithm" RETURN_TYPES = ("IMAGE",) @@ -4902,23 +4901,14 @@ class MVAdapter_Image_To_MultiView: device=DEVICE_STR, ) - # grid_image = make_image_grid(images, rows=1) - - # grid_tensor = pils_to_torch_imgs([grid_image], device=DEVICE_STR) - - # print(f"Generated multiview images: {grid_tensor.shape}") - # return (grid_tensor,) - # return images - return_images = pils_to_torch_imgs(images, device=DEVICE_STR) return (return_images,) - -class Load_MVAdapter_T2MV_Pipeline: - """Loader pipeline for MV-Adapter Text-to-Multi-View""" +class Load_MVAdapter_TG2MV_Pipeline: + """Loader pipeline for MV-Adapter Text-Guided to Multi-View (TG2MV)""" CATEGORY = "Comfy3D/Algorithm" RETURN_TYPES = ("DIFFUSERS_PIPE",) - RETURN_NAMES = ("mvadapter_t2mv_pipe",) + RETURN_NAMES = ("mvadapter_tg2mv_pipe",) FUNCTION = "load" CKPT_MVADAPTER_PATH = os.path.join(CKPT_DIFFUSERS_PATH, "huanngzh", "MV-Adapter") @@ -4932,45 +4922,51 @@ class Load_MVAdapter_T2MV_Pipeline: "vae_model": (["madebyollin/sdxl-vae-fp16-fix", "None"], {"default": "madebyollin/sdxl-vae-fp16-fix"}), "adapter_path": (["huanngzh/mv-adapter"], {"default": "huanngzh/mv-adapter"}), - "scheduler": (["default", "ddpm", "lcm"], {"default": "default"}), + "scheduler": (["ddpm"], {"default": "ddpm"}), "num_views": ("INT", {"default": 6, "min": 1, "max": 16}), "use_fp16": ("BOOLEAN", {"default": True}), "use_mmgp": ("BOOLEAN", {"default": False}), }, "optional": { - "unet_model": ("STRING", {"default": ""}), "lora_model": ("STRING", {"default": ""}), } } @classmethod def load(cls, base_model, vae_model, adapter_path, scheduler, num_views, - use_fp16, use_mmgp, unet_model="", lora_model=""): + use_fp16, use_mmgp, lora_model=""): + + if torch.cuda.is_available(): + torch.cuda.empty_cache() dtype = torch.float16 if use_fp16 else torch.float32 vae_model = None if vae_model == "None" else vae_model - unet_model = None if not unet_model else unet_model lora_model = None if not lora_model else lora_model - pipe = mvadapter_prepare_t2mv_pipeline( - base_model=base_model, - vae_model=vae_model, - unet_model=unet_model, - lora_model=lora_model, - adapter_path=adapter_path, - scheduler=scheduler, - num_views=num_views, - device=DEVICE_STR, - dtype=dtype, - use_mmgp=use_mmgp, - adapter_local_path=cls.CKPT_MVADAPTER_PATH - ) + pipeline_kwargs = { + "base_model": base_model, + "vae_model": vae_model, + "lora_model": lora_model, + "adapter_path": adapter_path, + "scheduler": scheduler, + "num_views": num_views, + "device": DEVICE_STR, + "dtype": dtype, + "use_mmgp": use_mmgp, + "adapter_local_path": cls.CKPT_MVADAPTER_PATH + } - print("MV-Adapter T2MV pipeline loaded successfully") - return (pipe,) + try: + pipe = mvadapter_prepare_tg2mv_pipeline(**pipeline_kwargs) + print("MV-Adapter TG2MV pipeline loaded successfully") + return (pipe,) + + except Exception as e: + raise e -class MVAdapter_Text_To_MultiView: - """Generate multi-view images from text prompt""" + +class MVAdapter_TG2MV: + """Generate multi-view images from text prompt using 3D mesh guidance""" CATEGORY = "Comfy3D/Algorithm" RETURN_TYPES = ("IMAGE",) RETURN_NAMES = ("multiview_images",) @@ -4980,7 +4976,8 @@ class MVAdapter_Text_To_MultiView: def INPUT_TYPES(cls): return { "required": { - "mvadapter_t2mv_pipe": ("DIFFUSERS_PIPE",), + "mvadapter_tg2mv_pipe": ("DIFFUSERS_PIPE",), + "mesh_path": ("STRING", {"default": ""}), "prompt": ("STRING", {"default": "a high quality 3D model", "multiline": True}), "negative_prompt": ("STRING", {"default": "watermark, ugly, deformed, noisy, blurry, low contrast", "multiline": True}), "num_views": ("INT", {"default": 6, "min": 1, "max": 16}), @@ -4992,30 +4989,18 @@ class MVAdapter_Text_To_MultiView: }, "optional": { "lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.1}), - "azimuth_angles": ("STRING", {"default": "0,45,90,180,270,315"}), } } - def run(self, mvadapter_t2mv_pipe, prompt, negative_prompt, num_views, - num_inference_steps, guidance_scale, height, width, seed, - lora_scale=1.0, azimuth_angles="0,45,90,180,270,315"): + def run(self, mvadapter_tg2mv_pipe, mesh_path, prompt, negative_prompt, num_views, + num_inference_steps, guidance_scale, height, width, seed, lora_scale=1.0): - # Parse azimuth angles - try: - azimuth_deg = [int(x.strip()) for x in azimuth_angles.split(",")] - # Ensure we have enough angles for the number of views - if len(azimuth_deg) < num_views: - # Repeat the pattern if we don't have enough angles - while len(azimuth_deg) < num_views: - azimuth_deg.extend(azimuth_deg[:min(len(azimuth_deg), num_views - len(azimuth_deg))]) - azimuth_deg = azimuth_deg[:num_views] # Take only the first num_views - except: - # Fallback to default angles - azimuth_deg = [0, 45, 90, 180, 270, 315][:num_views] + if not mesh_path or not os.path.exists(mesh_path): + raise ValueError(f"Mesh path does not exist: {mesh_path}") - # Execute generation - images = mvadapter_run_t2mv_pipeline( - pipe=mvadapter_t2mv_pipe, + images, pos_images, normal_images = mvadapter_run_tg2mv_pipeline( + pipe=mvadapter_tg2mv_pipe, + mesh_path=mesh_path, num_views=num_views, text=prompt, height=height, @@ -5026,17 +5011,9 @@ class MVAdapter_Text_To_MultiView: negative_prompt=negative_prompt, lora_scale=lora_scale, device=DEVICE_STR, - azimuth_deg=azimuth_deg, ) - # Create image grid - # grid_image = make_image_grid(images, rows=1) - # # Convert PIL image to torch tensor using shared utils - # grid_tensor = pils_to_torch_imgs([grid_image], device=DEVICE_STR) - - # print(f"Generated T2MV multiview images: {grid_tensor.shape}") - # return (grid_tensor,) return_images = pils_to_torch_imgs(images, device=DEVICE_STR) return (return_images,) @@ -5063,11 +5040,9 @@ class Load_MVAdapter_Texture_Pipeline: @classmethod def load(cls, upscaler_ckpt_name, inpaint_ckpt_name, use_mmgp, auto_download): - # Form full paths from model names upscaler_ckpt_path = os.path.join(cls.TEXTURE_CKPT_DIR, upscaler_ckpt_name) if upscaler_ckpt_name.strip() else None inpaint_ckpt_path = os.path.join(cls.TEXTURE_CKPT_DIR, inpaint_ckpt_name) if inpaint_ckpt_name.strip() else None - # Auto-download checkpoints if enabled if auto_download: download_texture_checkpoints(cls.TEXTURE_CKPT_DIR, upscaler_ckpt_path, inpaint_ckpt_path) @@ -5132,11 +5107,9 @@ class MVAdapter_Texture_Projection: if not mesh_path or not os.path.exists(mesh_path): raise ValueError(f"Mesh file does not exist: {mesh_path}") - # Convert to absolute paths save_dir = os.path.abspath(save_dir) os.makedirs(save_dir, exist_ok=True) - # Parse camera angles with fallback to defaults try: azimuth_deg = [float(x.strip()) for x in camera_azimuth_deg.split(",")] elevation_deg = [float(x.strip()) for x in camera_elevation_deg.split(",")] @@ -5144,10 +5117,8 @@ class MVAdapter_Texture_Projection: azimuth_deg = [0, 90, 180, 270, 180, 180] elevation_deg = [0, 0, 0, 0, 89.99, -89.99] - # Adjust azimuth angles (subtract 90 as in original code) azimuth_deg_corrected = [x - 90 for x in azimuth_deg] - # Save grid image temporarily temp_grid_path = os.path.join(save_dir, f"{save_name}_temp_grid.png") pil_grid.save(temp_grid_path) @@ -5164,7 +5135,6 @@ class MVAdapter_Texture_Projection: inpaint_mode=inpaint_mode ) - # Prepare texture pipeline arguments texture_args = { "mesh_path": mesh_path, "save_dir": save_dir, @@ -5184,7 +5154,6 @@ class MVAdapter_Texture_Projection: "apply_dilate": apply_dilate, } - # Add PBR settings if enabled if create_pbr_model: texture_args.update({ "base_color_path": temp_grid_path, @@ -5193,11 +5162,9 @@ class MVAdapter_Texture_Projection: output = texture_pipeline(**texture_args) - # Cleanup temp file if os.path.exists(temp_grid_path): os.remove(temp_grid_path) - # Convert to absolute paths shaded_path = os.path.abspath(output.shaded_model_save_path) if output.shaded_model_save_path else "" pbr_path = os.path.abspath(output.pbr_model_save_path) if output.pbr_model_save_path else "" @@ -5208,8 +5175,7 @@ class MVAdapter_Texture_Projection: return (shaded_path, pbr_path) except Exception as e: - # Cleanup temp file on error if os.path.exists(temp_grid_path): os.remove(temp_grid_path) raise e - +