nodes/MrForExample-ComfyUI-3D-Pack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add t2mv

Browse Source
This commit is contained in:
Denis Kochetov
2025-05-30 07:38:39 +00:00
parent 76179ca7fe
commit 953b4919f7
3 changed files with 283 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Gen_3D_Modules/MV_Adapter/mvadapter_node_utils.py
+155 -1
View File
@@ -14,6 +14,7 @@ from diffusers import AutoencoderKL, DDPMScheduler, LCMScheduler, UNet2DConditio
from huggingface_hub import snapshot_download
from .mvadapter.pipelines.pipeline_mvadapter_i2mv_sdxl import MVAdapterI2MVSDXLPipeline
from .mvadapter.pipelines.pipeline_mvadapter_t2mv_sdxl import MVAdapterT2MVSDXLPipeline
from .mvadapter.models.attention_processor import DecoupledMVRowColSelfAttnProcessor2_0
from .mvadapter.schedulers.scheduling_shift_snr import ShiftSNRScheduler
from .mvadapter.utils.mesh_utils import (
@@ -23,6 +24,7 @@ from .mvadapter.utils.mesh_utils import (
render,
)
from .mvadapter.utils import make_image_grid, tensor_to_image
from .mvadapter.utils.geometry import get_plucker_embeds_from_cameras_ortho
try:
from mmgp import offload, profile_type
@@ -299,4 +301,156 @@ def pil_to_torch_tensor(image: Image.Image) -> torch.Tensor:
def pils_to_torch_tensor(images: List[Image.Image]) -> torch.Tensor:
"""Конвертация списка PIL изображений в torch tensor"""
tensors = [pil_to_torch_tensor(img).squeeze(0) for img in images]
return torch.stack(tensors, dim=0)
return torch.stack(tensors, dim=0)
def prepare_t2mv_pipeline(
base_model: str,
vae_model: Optional[str] = None,
unet_model: Optional[str] = None,
lora_model: Optional[str] = None,
adapter_path: str = "huanngzh/mv-adapter",
scheduler: str = "default",
num_views: int = 6,
device: str = "cuda",
dtype: torch.dtype = torch.float16,
use_mmgp: bool = False,
adapter_local_path: Optional[str] = None,
):
"""
Prepare MV-Adapter T2MV pipeline
Args:
base_model: Base Stable Diffusion XL model
vae_model: Custom VAE model (optional)
unet_model: Custom UNet model (optional)
lora_model: LoRA model (optional)
adapter_path: Path to adapter weights
scheduler: Scheduler type ("default", "ddpm", "lcm")
num_views: Number of views to generate
device: Device for computation
dtype: Data type
use_mmgp: Use mmgp for memory optimization
adapter_local_path: Local path for adapter download
"""
# Load vae and unet if provided
pipe_kwargs = {}
if vae_model is not None:
pipe_kwargs["vae"] = AutoencoderKL.from_pretrained(vae_model)
if unet_model is not None:
pipe_kwargs["unet"] = UNet2DConditionModel.from_pretrained(unet_model)
# Prepare pipeline
pipe = MVAdapterT2MVSDXLPipeline.from_pretrained(base_model, **pipe_kwargs)
# Load scheduler if provided
if scheduler != "default":
scheduler_class = None
if scheduler == "ddpm":
scheduler_class = DDPMScheduler
elif scheduler == "lcm":
scheduler_class = LCMScheduler
pipe.scheduler = ShiftSNRScheduler.from_scheduler(
pipe.scheduler,
shift_mode="interpolated",
shift_scale=8.0,
scheduler_class=scheduler_class,
)
pipe.init_custom_adapter(num_views=num_views)
# Load adapter weights
weight_name = "mvadapter_t2mv_sdxl.safetensors"
pipe.load_custom_adapter(adapter_path, weight_name=weight_name, local_cache_dir=adapter_local_path)
pipe.to(device=device, dtype=dtype)
pipe.cond_encoder.to(device=device, dtype=dtype)
# load lora if provided
if lora_model is not None:
model_, name_ = lora_model.rsplit("/", 1)
pipe.load_lora_weights(model_, weight_name=name_)
# Apply mmgp optimization if available and requested
if use_mmgp and MMGP_AVAILABLE:
import gc
torch.cuda.empty_cache()
gc.collect()
all_models = {
"vae": pipe.vae,
"text_encoder": pipe.text_encoder,
"text_encoder_2": pipe.text_encoder_2,
"unet": pipe.unet,
"cond_encoder": pipe.cond_encoder,
}
if hasattr(pipe, 'image_encoder') and pipe.image_encoder is not None:
all_models["image_encoder"] = pipe.image_encoder
offload.profile(all_models, profile_type.HighRAM_LowVRAM)
print("mmgp profiling applied successfully.")
# Enable VAE slicing for memory efficiency
pipe.enable_vae_slicing()
return pipe
def run_t2mv_pipeline(
pipe,
num_views: int,
text: str,
height: int,
width: int,
num_inference_steps: int,
guidance_scale: float,
seed: int,
negative_prompt: str = "watermark, ugly, deformed, noisy, blurry, low contrast",
lora_scale: float = 1.0,
device: str = "cuda",
azimuth_deg: Optional[List[int]] = None,
):
"""
Execute text-to-multiview generation
"""
# Prepare cameras
if azimuth_deg is None:
azimuth_deg = [0, 45, 90, 180, 270, 315][:num_views]
cameras = get_orthogonal_camera(
elevation_deg=[0] * num_views,
distance=[1.8] * num_views,
left=-0.55,
right=0.55,
bottom=-0.55,
top=0.55,
azimuth_deg=[x - 90 for x in azimuth_deg],
device=device,
)
plucker_embeds = get_plucker_embeds_from_cameras_ortho(
cameras.c2w, [1.1] * num_views, width
)
control_images = ((plucker_embeds + 1.0) / 2.0).clamp(0, 1)
pipe_kwargs = {}
if seed != -1 and isinstance(seed, int):
pipe_kwargs["generator"] = torch.Generator(device=device).manual_seed(seed)
images = pipe(
text,
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=guidance_scale,
num_images_per_prompt=num_views,
control_image=control_images,
control_conditioning_scale=1.0,
negative_prompt=negative_prompt,
cross_attention_kwargs={"scale": lora_scale},
**pipe_kwargs,
).images
return images
nodes.py
+128 -1
View File
@@ -109,6 +109,8 @@ from MV_Adapter.mvadapter_node_utils import (
prepare_pipeline as mvadapter_prepare_pipeline,
run_pipeline as mvadapter_run_pipeline,
create_bg_remover,
prepare_t2mv_pipeline as mvadapter_prepare_t2mv_pipeline,
run_t2mv_pipeline as mvadapter_run_t2mv_pipeline
)
from MV_Adapter.mvadapter.utils import make_image_grid
@@ -4913,4 +4915,129 @@ class MVAdapter_Image_To_MultiView:
print(f"Generated multiview images: {grid_tensor.shape}")
return (grid_tensor,)
class Load_MVAdapter_T2MV_Pipeline:
"""Loader pipeline for MV-Adapter Text-to-Multi-View"""
CATEGORY = "Comfy3D/Algorithm"
RETURN_TYPES = ("DIFFUSERS_PIPE",)
RETURN_NAMES = ("mvadapter_t2mv_pipe",)
FUNCTION = "load"
CKPT_MVADAPTER_PATH = os.path.join(CKPT_DIFFUSERS_PATH, "huanngzh", "MV-Adapter")
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"base_model": (["stabilityai/stable-diffusion-xl-base-1.0"],
{"default": "stabilityai/stable-diffusion-xl-base-1.0"}),
"vae_model": (["madebyollin/sdxl-vae-fp16-fix", "None"],
{"default": "madebyollin/sdxl-vae-fp16-fix"}),
"adapter_path": (["huanngzh/mv-adapter"], {"default": "huanngzh/mv-adapter"}),
"scheduler": (["default", "ddpm", "lcm"], {"default": "default"}),
"num_views": ("INT", {"default": 6, "min": 1, "max": 16}),
"use_fp16": ("BOOLEAN", {"default": True}),
"use_mmgp": ("BOOLEAN", {"default": False}),
},
"optional": {
"unet_model": ("STRING", {"default": ""}),
"lora_model": ("STRING", {"default": ""}),
}
}
@classmethod
def load(cls, base_model, vae_model, adapter_path, scheduler, num_views,
use_fp16, use_mmgp, unet_model="", lora_model=""):
dtype = torch.float16 if use_fp16 else torch.float32
vae_model = None if vae_model == "None" else vae_model
unet_model = None if not unet_model else unet_model
lora_model = None if not lora_model else lora_model
pipe = mvadapter_prepare_t2mv_pipeline(
base_model=base_model,
vae_model=vae_model,
unet_model=unet_model,
lora_model=lora_model,
adapter_path=adapter_path,
scheduler=scheduler,
num_views=num_views,
device=DEVICE_STR,
dtype=dtype,
use_mmgp=use_mmgp,
adapter_local_path=cls.CKPT_MVADAPTER_PATH
)
print("MV-Adapter T2MV pipeline loaded successfully")
return (pipe,)
class MVAdapter_Text_To_MultiView:
"""Generate multi-view images from text prompt"""
CATEGORY = "Comfy3D/Algorithm"
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES = ("multiview_images",)
FUNCTION = "run"
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"mvadapter_t2mv_pipe": ("DIFFUSERS_PIPE",),
"prompt": ("STRING", {"default": "a high quality 3D model", "multiline": True}),
"negative_prompt": ("STRING", {"default": "watermark, ugly, deformed, noisy, blurry, low contrast", "multiline": True}),
"num_views": ("INT", {"default": 6, "min": 1, "max": 16}),
"num_inference_steps": ("INT", {"default": 50, "min": 1, "max": 200}),
"guidance_scale": ("FLOAT", {"default": 7.0, "min": 0.0, "max": 20.0, "step": 0.1}),
"height": ("INT", {"default": 768, "min": 256, "max": 2048, "step": 8}),
"width": ("INT", {"default": 768, "min": 256, "max": 2048, "step": 8}),
"seed": ("INT", {"default": -1, "min": -1, "max": 0xffffffffffffffff}),
},
"optional": {
"lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.1}),
"azimuth_angles": ("STRING", {"default": "0,45,90,180,270,315"}),
}
}
def run(self, mvadapter_t2mv_pipe, prompt, negative_prompt, num_views,
num_inference_steps, guidance_scale, height, width, seed,
lora_scale=1.0, azimuth_angles="0,45,90,180,270,315"):
# Parse azimuth angles
try:
azimuth_deg = [int(x.strip()) for x in azimuth_angles.split(",")]
# Ensure we have enough angles for the number of views
if len(azimuth_deg) < num_views:
# Repeat the pattern if we don't have enough angles
while len(azimuth_deg) < num_views:
azimuth_deg.extend(azimuth_deg[:min(len(azimuth_deg), num_views - len(azimuth_deg))])
azimuth_deg = azimuth_deg[:num_views] # Take only the first num_views
except:
# Fallback to default angles
azimuth_deg = [0, 45, 90, 180, 270, 315][:num_views]
# Execute generation
images = mvadapter_run_t2mv_pipeline(
pipe=mvadapter_t2mv_pipe,
num_views=num_views,
text=prompt,
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=guidance_scale,
seed=seed,
negative_prompt=negative_prompt,
lora_scale=lora_scale,
device=DEVICE_STR,
azimuth_deg=azimuth_deg,
)
# Create image grid
grid_image = make_image_grid(images, rows=1)
# Convert PIL image to torch tensor using shared utils
grid_tensor = pils_to_torch_imgs([grid_image], device=DEVICE_STR)
print(f"Generated T2MV multiview images: {grid_tensor.shape}")
return (grid_tensor,)
stablegen_nodes.py
-408
View File
@@ -1,408 +0,0 @@
import torch
import os
import trimesh
import numpy as np
from collections import OrderedDict
from huggingface_hub import snapshot_download
# Import StableGen modules
try:
from .Gen_3D_Modules.Stable3DGen.trellis.pipelines.trellis_image_to_3d import TrellisImageTo3DPipeline as Stable3DGenTrellisImageTo3DPipeline
from .Gen_3D_Modules.Stable3DGen.stablex.pipeline_yoso import YosoPipeline
from .Gen_3D_Modules.Stable3DGen.stablex.controlnetvae import ControlNetVAEModel
from .Gen_3D_Modules.Stable3DGen.pipeline_builders import StableGenPipelineBuilder
except ImportError as e:
print(f"Warning: Could not import StableGen modules: {e}")
# Create dummy functions for fallback
def get_available_backends():
return {'flash_attn': True, 'sdpa': True}
def get_available_sparse_backends():
return {'spconv': True}
def set_attention_backend(backend):
return True
def set_sparse_backend(backend, algo):
return True
class StableGenPipelineBuilder:
@staticmethod
def build_trellis_pipeline(*args, **kwargs):
raise ImportError("StableGen modules not available")
@staticmethod
def build_stablex_pipeline(*args, **kwargs):
raise ImportError("StableGen modules not available")
# Import diffusers components for StableX
try:
from diffusers.models import AutoencoderKL, UNet2DConditionModel
from accelerate import init_empty_weights
from accelerate.utils import set_module_tensor_to_device
from safetensors.torch import load_file
except ImportError as e:
print(f"Warning: Could not import diffusers/accelerate: {e}")
import json
from .shared_utils.mesh_utils import Mesh
from .shared_utils.image_utils import torch_imgs_to_pils, pils_to_torch_imgs, pil_make_image_grid
ROOT_PATH = os.path.dirname(os.path.realpath(__file__))
CKPT_ROOT_PATH = os.path.join(ROOT_PATH, "Checkpoints")
CKPT_DIFFUSERS_PATH = os.path.join(CKPT_ROOT_PATH, "Diffusers")
WEIGHT_DTYPE = torch.float16
DEVICE_STR = "cuda" if torch.cuda.is_available() else "cpu"
DEVICE = torch.device(DEVICE_STR)
HF_DOWNLOAD_IGNORE = ["*.yaml", "*.json", "*.py", ".png", ".jpg", ".gif"]
class Load_StableGen_Trellis_Pipeline:
CATEGORY = "Comfy3D/Algorithm"
RETURN_TYPES = ("DIFFUSERS_PIPE",)
RETURN_NAMES = ("trellis_pipe",)
FUNCTION = "load"
_REPO_ID_BASE = "Stable-X"
CKPT_STABLEGEN_PATH = os.path.join(CKPT_DIFFUSERS_PATH, "Stable3DGen")
_MODES = {
"trellis-normal-v0-1": ("trellis-normal-v0-1", 12, 12), # (repo, ss_steps, slat_steps)
}
@classmethod
def INPUT_TYPES(cls):
available_attn, available_sparse = StableGenPipelineBuilder.get_available_backends()
return {
"required": {
"model_name": (list(cls._MODES.keys()),),
"dinov2_model": (["dinov2_vitl14_reg"],),
"use_fp16": ("BOOLEAN", {"default": True}),
"attn_backend": (available_attn,),
"sparse_backend": (available_sparse,),
"spconv_algo": (["implicit_gemm", "native", "auto"],),
"smooth_k": ("BOOLEAN", {"default": True}),
}
}
@classmethod
def _build_pipe(cls, repo: str, dinov2_model: str, use_fp16: bool, attn_backend: str,
sparse_backend: str, spconv_algo: str, smooth_k: bool):
return StableGenPipelineBuilder.build_trellis_pipeline(
repo=repo,
dinov2_model=dinov2_model,
use_fp16=use_fp16,
attn_backend=attn_backend,
sparse_backend=sparse_backend,
spconv_algo=spconv_algo,
smooth_k=smooth_k,
ckpt_path=cls.CKPT_STABLEGEN_PATH
)
def load(self, model_name, dinov2_model, use_fp16, attn_backend, sparse_backend, spconv_algo, smooth_k):
repo, ss_steps, slat_steps = self._MODES[model_name]
pipe = self.__class__._build_pipe(repo, dinov2_model, use_fp16, attn_backend, sparse_backend, spconv_algo, smooth_k)
# Store default steps
pipe.default_ss_steps = ss_steps
pipe.default_slat_steps = slat_steps
return (pipe,)
class Load_StableGen_StableX_Pipeline:
CATEGORY = "Comfy3D/Algorithm"
RETURN_TYPES = ("DIFFUSERS_PIPE",)
RETURN_NAMES = ("stablex_pipe",)
FUNCTION = "load"
_REPO_ID_BASE = "Stable-X"
CKPT_STABLEGEN_PATH = os.path.join(CKPT_DIFFUSERS_PATH, "Stable3DGen")
_MODES = {
"yoso-normal-v1-8-1": "yoso-normal-v1-8-1",
}
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model_name": (list(cls._MODES.keys()),),
"use_fp16": ("BOOLEAN", {"default": True}),
}
}
@classmethod
def _build_pipe(cls, repo: str, use_fp16: bool):
return StableGenPipelineBuilder.build_stablex_pipeline(
repo=repo,
use_fp16=use_fp16,
ckpt_path=cls.CKPT_STABLEGEN_PATH
)
def load(self, model_name, use_fp16):
repo = self._MODES[model_name]
pipe = self.__class__._build_pipe(repo, use_fp16)
return (pipe,)
class StableGen_Trellis_Image_To_3D:
"""
3D generation pipeline using Trellis model.
"""
CATEGORY = "Comfy3D/Algorithm"
RETURN_TYPES = ("MESH",)
RETURN_NAMES = ("mesh",)
FUNCTION = "run"
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"trellis_pipe": ("DIFFUSERS_PIPE",),
"images": ("IMAGE", {"list": True}),
"mode": (["single", "multi"], {"default": "single"}),
"seed": ("INT", {"default": 1234, "min": 0, "max": 0xffffffffffffffff}),
"ss_guidance_strength": ("FLOAT", {"default": 7.5, "min": 0.0, "step": 0.1}),
"ss_sampling_steps": ("INT", {"default": 12, "min": 1}),
"slat_guidance_strength": ("FLOAT", {"default": 3.0, "min": 0.0, "step": 0.1}),
"slat_sampling_steps": ("INT", {"default": 12, "min": 1}),
"mesh_simplify": ("FLOAT", {"default": 0.95, "min": 0.9, "max": 1.0, "step": 0.01}),
}
}
@torch.no_grad()
def run(
self,
trellis_pipe,
images,
mode="single",
seed=1234,
ss_guidance_strength=7.5,
ss_sampling_steps=12,
slat_guidance_strength=3.0,
slat_sampling_steps=12,
mesh_simplify=0.95
):
if isinstance(images, torch.Tensor):
images = torch_imgs_to_pils(images)
if not isinstance(images, list) or len(images) == 0:
raise Exception(f"[StableGen_Trellis_Image_To_3D] 'images' must be a non-empty list of PIL images. Got type: {type(images)}, len: {len(images) if hasattr(images, '__len__') else 'no len'}")
# Use inference context like in working version
with trellis_pipe.inference_context():
# Prepare input based on mode
if mode == "single":
if len(images) > 1:
print(f"Warning: Single mode selected but {len(images)} images provided. Using first image.")
image_input = images[0]
# Set up generation parameters for single image
pipeline_params = StableGenPipelineBuilder.create_trellis_pipeline_params(
seed=seed,
ss_sampling_steps=ss_sampling_steps,
ss_guidance_strength=ss_guidance_strength,
slat_sampling_steps=slat_sampling_steps,
slat_guidance_strength=slat_guidance_strength,
formats=["mesh"]
)
# Generate 3D mesh using single image method
outputs = trellis_pipe.run(
image_input,
**pipeline_params
)
else: # multi mode
# Set up generation parameters for multi image
pipeline_params = StableGenPipelineBuilder.create_trellis_pipeline_params(
seed=seed,
ss_sampling_steps=ss_sampling_steps,
ss_guidance_strength=ss_guidance_strength,
slat_sampling_steps=slat_sampling_steps,
slat_guidance_strength=slat_guidance_strength,
formats=["mesh"]
)
outputs = trellis_pipe.run_multi_image(
images,
**pipeline_params
)
try:
mesh_output = outputs['mesh'][0]
vertices = mesh_output.vertices.cpu().numpy()
faces = mesh_output.faces.cpu().numpy()
transformation_matrix = np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]])
vertices = vertices @ transformation_matrix
tri_mesh = trimesh.Trimesh(vertices, faces)
if mesh_simplify < 1.0:
try:
target_faces = int(len(faces) * mesh_simplify)
tri_mesh = tri_mesh.simplify_quadric_decimation(target_faces)
except Exception as e:
print(f"Warning: Mesh simplification failed: {e}. Using original mesh.")
# Convert to Mesh object
mesh = Mesh.load_trimesh(given_mesh=tri_mesh)
mesh.auto_normal()
return (mesh,)
except Exception as e:
raise Exception(f"[StableGen_Trellis_Image_To_3D] 3D generation failed: {str(e)}")
# Import MV-Adapter modules
try:
from .Gen_3D_Modules.MV_Adapter.mvadapter_node_utils import (
prepare_pipeline as mvadapter_prepare_pipeline,
run_pipeline as mvadapter_run_pipeline,
create_bg_remover,
)
from .Gen_3D_Modules.MV_Adapter.mvadapter.utils import make_image_grid
MVADAPTER_AVAILABLE = True
except ImportError as e:
print(f"Warning: Could not import MV-Adapter modules: {e}")
MVADAPTER_AVAILABLE = False
class Load_MVAdapter_Pipeline:
"""Loader pipeline for MV-Adapter (Image to Multi-View)"""
CATEGORY = "Comfy3D/Algorithm"
RETURN_TYPES = ("DIFFUSERS_PIPE",)
RETURN_NAMES = ("mvadapter_pipe",)
FUNCTION = "load"
CKPT_MVADAPTER_PATH = os.path.join(CKPT_DIFFUSERS_PATH, "huanngzh", "MV-Adapter")
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"base_model": (["stabilityai/stable-diffusion-xl-base-1.0"],
{"default": "stabilityai/stable-diffusion-xl-base-1.0"}),
"vae_model": (["madebyollin/sdxl-vae-fp16-fix", "None"],
{"default": "madebyollin/sdxl-vae-fp16-fix"}),
"adapter_path": (["huanngzh/mv-adapter"], {"default": "huanngzh/mv-adapter"}),
"scheduler": (["default", "ddpm", "lcm"], {"default": "default"}),
"num_views": ("INT", {"default": 6, "min": 1, "max": 16}),
"use_fp16": ("BOOLEAN", {"default": True}),
"use_mmgp": ("BOOLEAN", {"default": False}),
},
"optional": {
"unet_model": ("STRING", {"default": ""}),
"lora_model": ("STRING", {"default": ""}),
}
}
@classmethod
def load(self, base_model, vae_model, adapter_path, scheduler, num_views,
use_fp16, use_mmgp, unet_model="", lora_model=""):
dtype = torch.float16 if use_fp16 else torch.float32
vae_model = None if vae_model == "None" else vae_model
unet_model = None if not unet_model else unet_model
lora_model = None if not lora_model else lora_model
pipe = mvadapter_prepare_pipeline(
base_model=base_model,
vae_model=vae_model,
unet_model=unet_model,
lora_model=lora_model,
adapter_path=adapter_path,
scheduler=scheduler,
num_views=num_views,
device=DEVICE_STR,
dtype=dtype,
use_mmgp=use_mmgp,
adapter_local_path=cls.CKPT_MVADAPTER_PATH
)
print("MV-Adapter pipeline loaded successfully")
return (pipe,)
class MVAdapter_Image_To_MultiView:
"""Generate multi-view images from single image and 3D mesh"""
CATEGORY = "Comfy3D/Algorithm"
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES = ("multiview_images",)
FUNCTION = "run"
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"mvadapter_pipe": ("DIFFUSERS_PIPE",),
"mesh_path": ("STRING", {"default": ""}),
"reference_image": ("IMAGE",),
"prompt": ("STRING", {"default": "high quality", "multiline": True}),
"negative_prompt": ("STRING", {"default": "watermark, ugly, deformed, noisy, blurry, low contrast", "multiline": True}),
"num_inference_steps": ("INT", {"default": 50, "min": 1, "max": 200}),
"guidance_scale": ("FLOAT", {"default": 3.0, "min": 0.0, "max": 20.0, "step": 0.1}),
"reference_conditioning_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.1}),
"height": ("INT", {"default": 768, "min": 256, "max": 2048, "step": 8}),
"width": ("INT", {"default": 768, "min": 256, "max": 2048, "step": 8}),
"seed": ("INT", {"default": -1, "min": -1, "max": 0xffffffffffffffff}),
"remove_background": ("BOOLEAN", {"default": False}),
},
"optional": {
"lora_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.1}),
}
}
def run(self, mvadapter_pipe, mesh_path, reference_image, prompt, negative_prompt,
num_inference_steps, guidance_scale, reference_conditioning_scale,
height, width, seed, remove_background, lora_scale=1.0):
# Convert input image
if isinstance(reference_image, torch.Tensor):
reference_images = torch_imgs_to_pils(reference_image)
reference_image = reference_images[0]
# Check if mesh_path exists
if not mesh_path or not os.path.exists(mesh_path):
raise ValueError(f"Mesh path does not exist: {mesh_path}")
# Create background removal function if needed
remove_bg_fn = None
if remove_background:
remove_bg_fn = create_bg_remover(DEVICE_STR)
# Execute generation
num_views = 6 # Standard number of views
images, pos_images, normal_images, processed_ref_image = mvadapter_run_pipeline(
pipe=mvadapter_pipe,
mesh_path=mesh_path,
num_views=num_views,
text=prompt,
image=reference_image,
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=guidance_scale,
seed=seed,
remove_bg_fn=remove_bg_fn,
reference_conditioning_scale=reference_conditioning_scale,
negative_prompt=negative_prompt,
lora_scale=lora_scale,
device=DEVICE_STR,
)
# Create image grid
grid_image = make_image_grid(images, rows=1)
# Convert PIL image to torch tensor using shared utils
grid_tensor = pils_to_torch_imgs([grid_image], device=DEVICE_STR)
print(f"Generated multiview images: {grid_tensor.shape}")
return (grid_tensor,)