661 lines
21 KiB
Python
661 lines
21 KiB
Python
"""
|
|
Utilities for ComfyUI MV-Adapter nodes
|
|
"""
|
|
|
|
import os
|
|
import torch
|
|
import numpy as np
|
|
from PIL import Image
|
|
from typing import Optional, Union, List
|
|
from torchvision import transforms
|
|
from transformers import AutoModelForImageSegmentation
|
|
|
|
from diffusers import AutoencoderKL, DDPMScheduler, UNet2DConditionModel
|
|
from huggingface_hub import snapshot_download
|
|
|
|
from .mvadapter.pipelines.pipeline_mvadapter_i2mv_sdxl import MVAdapterI2MVSDXLPipeline
|
|
from .mvadapter.pipelines.pipeline_mvadapter_t2mv_sdxl import MVAdapterT2MVSDXLPipeline
|
|
from .mvadapter.models.attention_processor import DecoupledMVRowColSelfAttnProcessor2_0
|
|
from .mvadapter.schedulers.scheduling_shift_snr import ShiftSNRScheduler
|
|
from .mvadapter.utils.mesh_utils import (
|
|
NVDiffRastContextWrapper,
|
|
get_orthogonal_camera,
|
|
load_mesh,
|
|
render,
|
|
)
|
|
from .mvadapter.utils import make_image_grid, tensor_to_image
|
|
from .mvadapter.utils.geometry import get_plucker_embeds_from_cameras_ortho
|
|
|
|
try:
|
|
from mmgp import offload, profile_type
|
|
MMGP_AVAILABLE = True
|
|
except ImportError:
|
|
MMGP_AVAILABLE = False
|
|
print("Warning: mmgp not available, memory optimization disabled")
|
|
|
|
|
|
def prepare_pipeline(
|
|
base_model: str,
|
|
vae_model: Optional[str] = None,
|
|
lora_model: Optional[str] = None,
|
|
adapter_path: str = "huanngzh/mv-adapter",
|
|
scheduler: str = "ddpm",
|
|
num_views: int = 6,
|
|
device: str = "cuda",
|
|
dtype: torch.dtype = torch.float16,
|
|
use_mmgp: bool = False,
|
|
adapter_local_path: Optional[str] = None,
|
|
):
|
|
"""
|
|
Prepare MV-Adapter pipeline
|
|
|
|
Args:
|
|
base_model: Base Stable Diffusion XL model
|
|
vae_model: Custom VAE model (optional)
|
|
lora_model: LoRA model (optional)
|
|
adapter_path: Path to adapter weights
|
|
scheduler: Scheduler type ("ddpm")
|
|
num_views: Number of views to generate
|
|
device: Device for computation
|
|
dtype: Data type
|
|
use_mmgp: Use mmgp for memory optimization
|
|
adapter_local_path: Local path for adapter download
|
|
"""
|
|
# Load vae if provided
|
|
pipe_kwargs = {}
|
|
if vae_model is not None:
|
|
pipe_kwargs["vae"] = AutoencoderKL.from_pretrained(vae_model)
|
|
|
|
# Prepare pipeline
|
|
pipe = MVAdapterI2MVSDXLPipeline.from_pretrained(base_model, **pipe_kwargs)
|
|
|
|
# Load DDPM scheduler
|
|
pipe.scheduler = ShiftSNRScheduler.from_scheduler(
|
|
pipe.scheduler,
|
|
shift_mode="interpolated",
|
|
shift_scale=8.0,
|
|
scheduler_class=DDPMScheduler,
|
|
)
|
|
|
|
pipe.init_custom_adapter(
|
|
num_views=num_views, self_attn_processor=DecoupledMVRowColSelfAttnProcessor2_0
|
|
)
|
|
|
|
# Load adapter weights
|
|
weight_name = "mvadapter_ig2mv_sdxl.safetensors"
|
|
pipe.load_custom_adapter(adapter_path, weight_name=weight_name, local_cache_dir=adapter_local_path)
|
|
|
|
pipe.to(device=device, dtype=dtype)
|
|
pipe.cond_encoder.to(device=device, dtype=dtype)
|
|
|
|
# load lora if provided
|
|
if lora_model is not None:
|
|
model_, name_ = lora_model.rsplit("/", 1)
|
|
pipe.load_lora_weights(model_, weight_name=name_)
|
|
|
|
# Apply mmgp optimization if available and requested
|
|
if use_mmgp and MMGP_AVAILABLE:
|
|
import gc
|
|
torch.cuda.empty_cache()
|
|
gc.collect()
|
|
|
|
all_models = {
|
|
"vae": pipe.vae,
|
|
"text_encoder": pipe.text_encoder,
|
|
"text_encoder_2": pipe.text_encoder_2,
|
|
"unet": pipe.unet,
|
|
"cond_encoder": pipe.cond_encoder,
|
|
}
|
|
|
|
if hasattr(pipe, 'image_encoder') and pipe.image_encoder is not None:
|
|
all_models["image_encoder"] = pipe.image_encoder
|
|
|
|
offload.profile(all_models, profile_type.HighRAM_LowVRAM)
|
|
print("mmgp profiling applied successfully.")
|
|
|
|
# Enable VAE slicing for memory efficiency
|
|
pipe.enable_vae_slicing()
|
|
|
|
return pipe
|
|
|
|
|
|
def preprocess_image(image: Image.Image, height: int, width: int) -> Image.Image:
|
|
"""Preprocess image for MV-Adapter"""
|
|
image = np.array(image)
|
|
alpha = image[..., 3] > 0
|
|
H, W = alpha.shape
|
|
# get the bounding box of alpha
|
|
y, x = np.where(alpha)
|
|
y0, y1 = max(y.min() - 1, 0), min(y.max() + 1, H)
|
|
x0, x1 = max(x.min() - 1, 0), min(x.max() + 1, W)
|
|
image_center = image[y0:y1, x0:x1]
|
|
# resize the longer side to H * 0.9
|
|
H, W, _ = image_center.shape
|
|
if H > W:
|
|
W = int(W * (height * 0.9) / H)
|
|
H = int(height * 0.9)
|
|
else:
|
|
H = int(H * (width * 0.9) / W)
|
|
W = int(width * 0.9)
|
|
image_center = np.array(Image.fromarray(image_center).resize((W, H)))
|
|
# pad to H, W
|
|
start_h = (height - H) // 2
|
|
start_w = (width - W) // 2
|
|
image = np.zeros((height, width, 4), dtype=np.uint8)
|
|
image[start_h : start_h + H, start_w : start_w + W] = image_center
|
|
image = image.astype(np.float32) / 255.0
|
|
image = image[:, :, :3] * image[:, :, 3:4] + (1 - image[:, :, 3:4]) * 0.5
|
|
image = (image * 255).clip(0, 255).astype(np.uint8)
|
|
image = Image.fromarray(image)
|
|
|
|
return image
|
|
|
|
|
|
def run_pipeline(
|
|
pipe,
|
|
mesh_path: str,
|
|
num_views: int,
|
|
text: str,
|
|
image: Union[str, Image.Image],
|
|
height: int,
|
|
width: int,
|
|
num_inference_steps: int,
|
|
guidance_scale: float,
|
|
seed: int,
|
|
remove_bg_fn=None,
|
|
reference_conditioning_scale: float = 1.0,
|
|
negative_prompt: str = "watermark, ugly, deformed, noisy, blurry, low contrast",
|
|
lora_scale: float = 1.0,
|
|
device: str = "cuda",
|
|
):
|
|
"""
|
|
Execute multi-view image generation
|
|
"""
|
|
# Prepare cameras
|
|
cameras = get_orthogonal_camera(
|
|
elevation_deg=[0, 0, 0, 0, 89.99, -89.99],
|
|
distance=[1.8] * num_views,
|
|
left=-0.55,
|
|
right=0.55,
|
|
bottom=-0.55,
|
|
top=0.55,
|
|
azimuth_deg=[x - 90 for x in [0, 90, 180, 270, 180, 180]],
|
|
device=device,
|
|
)
|
|
ctx = NVDiffRastContextWrapper(device=device)
|
|
|
|
mesh = load_mesh(mesh_path, rescale=True, device=device)
|
|
render_out = render(
|
|
ctx,
|
|
mesh,
|
|
cameras,
|
|
height=height,
|
|
width=width,
|
|
render_attr=False,
|
|
normal_background=0.0,
|
|
)
|
|
|
|
pos_images = tensor_to_image((render_out.pos + 0.5).clamp(0, 1), batched=True)
|
|
normal_images = tensor_to_image(
|
|
(render_out.normal / 2 + 0.5).clamp(0, 1), batched=True
|
|
)
|
|
|
|
control_images = (
|
|
torch.cat(
|
|
[
|
|
(render_out.pos + 0.5).clamp(0, 1),
|
|
(render_out.normal / 2 + 0.5).clamp(0, 1),
|
|
],
|
|
dim=-1,
|
|
)
|
|
.permute(0, 3, 1, 2)
|
|
.to(device)
|
|
)
|
|
|
|
# Prepare image
|
|
reference_image = Image.open(image) if isinstance(image, str) else image
|
|
if reference_image.mode == "RGBA":
|
|
reference_image = preprocess_image(reference_image, height, width)
|
|
|
|
pipe_kwargs = {}
|
|
if seed != -1 and isinstance(seed, int):
|
|
pipe_kwargs["generator"] = torch.Generator(device=device).manual_seed(seed)
|
|
|
|
images = pipe(
|
|
text,
|
|
height=height,
|
|
width=width,
|
|
num_inference_steps=num_inference_steps,
|
|
guidance_scale=guidance_scale,
|
|
num_images_per_prompt=num_views,
|
|
control_image=control_images,
|
|
control_conditioning_scale=1.0,
|
|
reference_image=reference_image,
|
|
reference_conditioning_scale=reference_conditioning_scale,
|
|
negative_prompt=negative_prompt,
|
|
cross_attention_kwargs={"scale": lora_scale},
|
|
**pipe_kwargs,
|
|
).images
|
|
|
|
return images, pos_images, normal_images, reference_image
|
|
|
|
|
|
def create_image_grid(images: List[Image.Image], rows: int = 1) -> Image.Image:
|
|
return make_image_grid(images, rows=rows)
|
|
|
|
|
|
def pil_to_torch_tensor(image: Image.Image) -> torch.Tensor:
|
|
import numpy as np
|
|
array = np.array(image)
|
|
tensor = torch.from_numpy(array).float() / 255.0
|
|
|
|
# Add batch dimension if needed
|
|
if tensor.dim() == 3:
|
|
tensor = tensor.unsqueeze(0)
|
|
|
|
return tensor
|
|
|
|
|
|
def pils_to_torch_tensor(images: List[Image.Image]) -> torch.Tensor:
|
|
tensors = [pil_to_torch_tensor(img).squeeze(0) for img in images]
|
|
return torch.stack(tensors, dim=0)
|
|
|
|
|
|
def prepare_texture_pipeline(
|
|
upscaler_ckpt_path: Optional[str] = None,
|
|
inpaint_ckpt_path: Optional[str] = None,
|
|
device: str = "cuda",
|
|
use_mmgp: bool = False,
|
|
):
|
|
"""
|
|
Prepare texture projection pipeline
|
|
|
|
Args:
|
|
upscaler_ckpt_path: Path to upscaler checkpoint (optional)
|
|
inpaint_ckpt_path: Path to inpaint checkpoint (optional)
|
|
device: Device for computation
|
|
use_mmgp: Use mmgp for memory optimization
|
|
"""
|
|
from .mvadapter.pipelines.pipeline_texture import TexturePipeline
|
|
|
|
if upscaler_ckpt_path and not os.path.exists(upscaler_ckpt_path):
|
|
print(f"Warning: Upscaler checkpoint not found: {upscaler_ckpt_path}, using None")
|
|
upscaler_ckpt_path = None
|
|
|
|
if inpaint_ckpt_path and not os.path.exists(inpaint_ckpt_path):
|
|
print(f"Warning: Inpaint checkpoint not found: {inpaint_ckpt_path}, using None")
|
|
inpaint_ckpt_path = None
|
|
|
|
texture_pipe = TexturePipeline(
|
|
upscaler_ckpt_path=upscaler_ckpt_path,
|
|
inpaint_ckpt_path=inpaint_ckpt_path,
|
|
device=device
|
|
)
|
|
|
|
# Apply mmgp optimization if available and requested
|
|
if use_mmgp and MMGP_AVAILABLE:
|
|
import gc
|
|
torch.cuda.empty_cache()
|
|
gc.collect()
|
|
|
|
all_models = {}
|
|
|
|
if hasattr(texture_pipe, 'upscaler') and texture_pipe.upscaler is not None:
|
|
all_models["upscaler"] = texture_pipe.upscaler
|
|
|
|
if hasattr(texture_pipe, 'inpainter') and texture_pipe.inpainter is not None:
|
|
all_models["inpainter"] = texture_pipe.inpainter
|
|
|
|
if all_models:
|
|
offload.profile(all_models, profile_type.HighRAM_LowVRAM)
|
|
print("mmgp profiling applied to texture pipeline successfully.")
|
|
else:
|
|
print("No models loaded for texture pipeline, skipping mmgp profiling.")
|
|
|
|
return texture_pipe
|
|
|
|
|
|
def run_texture_pipeline(
|
|
texture_pipe,
|
|
multiview_images: List[Image.Image],
|
|
mesh_path: str,
|
|
save_dir: str,
|
|
save_name: str = "textured_model",
|
|
uv_size: int = 4096,
|
|
view_upscale: bool = True,
|
|
inpaint_mode: str = "view",
|
|
uv_unwarp: bool = True,
|
|
preprocess_mesh: bool = False,
|
|
move_to_center: bool = False,
|
|
front_x: bool = True,
|
|
camera_azimuth_deg: List[float] = None,
|
|
camera_elevation_deg: List[float] = None,
|
|
camera_distance: float = 1.0,
|
|
camera_ortho_scale: float = 1.1,
|
|
debug_mode: bool = False,
|
|
apply_dilate: bool = True,
|
|
):
|
|
"""
|
|
Execute texture projection pipeline
|
|
|
|
Args:
|
|
texture_pipe: Prepared texture pipeline
|
|
multiview_images: List of PIL images (6 views)
|
|
mesh_path: Path to 3D mesh file
|
|
save_dir: Directory to save results
|
|
save_name: Name for saved files
|
|
uv_size: UV texture resolution
|
|
view_upscale: Enable view upscaling
|
|
inpaint_mode: Inpainting mode ("none", "uv", "view")
|
|
uv_unwarp: Enable UV unwrapping
|
|
preprocess_mesh: Enable mesh preprocessing
|
|
move_to_center: Move mesh to center
|
|
front_x: Front face along X axis
|
|
camera_azimuth_deg: Camera azimuth angles
|
|
camera_elevation_deg: Camera elevation angles
|
|
camera_distance: Camera distance
|
|
camera_ortho_scale: Orthographic scale
|
|
debug_mode: Enable debug mode
|
|
apply_dilate: Apply dilate to remove texture seams
|
|
"""
|
|
from .mvadapter.pipelines.pipeline_texture import ModProcessConfig
|
|
from .mvadapter.utils import make_image_grid
|
|
|
|
if not mesh_path or not os.path.exists(mesh_path):
|
|
raise ValueError(f"Mesh file does not exist: {mesh_path}")
|
|
|
|
os.makedirs(save_dir, exist_ok=True)
|
|
|
|
if camera_azimuth_deg is None:
|
|
camera_azimuth_deg = [0, 90, 180, 270, 180, 180]
|
|
if camera_elevation_deg is None:
|
|
camera_elevation_deg = [0, 0, 0, 0, 89.99, -89.99]
|
|
|
|
azimuth_deg_corrected = [x - 90 for x in camera_azimuth_deg]
|
|
|
|
grid_image = make_image_grid(multiview_images, rows=1)
|
|
temp_grid_path = os.path.join(save_dir, f"{save_name}_temp_grid.png")
|
|
grid_image.save(temp_grid_path)
|
|
|
|
try:
|
|
rgb_process_config = ModProcessConfig(
|
|
view_upscale=view_upscale,
|
|
inpaint_mode=inpaint_mode
|
|
)
|
|
|
|
output = texture_pipe(
|
|
mesh_path=mesh_path,
|
|
save_dir=save_dir,
|
|
save_name=save_name,
|
|
move_to_center=move_to_center,
|
|
front_x=front_x,
|
|
uv_unwarp=uv_unwarp,
|
|
preprocess_mesh=preprocess_mesh,
|
|
uv_size=uv_size,
|
|
rgb_path=temp_grid_path,
|
|
rgb_process_config=rgb_process_config,
|
|
camera_elevation_deg=camera_elevation_deg,
|
|
camera_azimuth_deg=azimuth_deg_corrected,
|
|
camera_distance=camera_distance,
|
|
camera_ortho_scale=camera_ortho_scale,
|
|
debug_mode=debug_mode,
|
|
apply_dilate=apply_dilate,
|
|
)
|
|
|
|
if os.path.exists(temp_grid_path):
|
|
os.remove(temp_grid_path)
|
|
|
|
return output
|
|
|
|
except Exception as e:
|
|
if os.path.exists(temp_grid_path):
|
|
os.remove(temp_grid_path)
|
|
raise e
|
|
|
|
|
|
def download_texture_checkpoints(texture_ckpt_dir, upscaler_ckpt_path, inpaint_ckpt_path):
|
|
"""Download texture checkpoint files if they don't exist"""
|
|
import subprocess
|
|
|
|
os.makedirs(texture_ckpt_dir, exist_ok=True)
|
|
|
|
checkpoints = [
|
|
{
|
|
"url": "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.1/RealESRGAN_x2plus.pth",
|
|
"path": upscaler_ckpt_path,
|
|
"name": "RealESRGAN upscaler"
|
|
},
|
|
{
|
|
"url": "https://github.com/Sanster/models/releases/download/add_big_lama/big-lama.pt",
|
|
"path": inpaint_ckpt_path,
|
|
"name": "Big-LaMa inpainter"
|
|
}
|
|
]
|
|
|
|
for ckpt in checkpoints:
|
|
if not os.path.exists(ckpt["path"]):
|
|
print(f"Downloading {ckpt['name']} to {ckpt['path']}...")
|
|
|
|
success = False
|
|
|
|
# Try wget first
|
|
try:
|
|
subprocess.run([
|
|
"wget", ckpt["url"], "-O", ckpt["path"]
|
|
], check=True, capture_output=True, text=True)
|
|
print(f"Successfully downloaded {ckpt['name']} using wget")
|
|
success = True
|
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
pass
|
|
|
|
if not success:
|
|
try:
|
|
subprocess.run([
|
|
"curl", "-L", ckpt["url"], "-o", ckpt["path"]
|
|
], check=True, capture_output=True, text=True)
|
|
print(f"Successfully downloaded {ckpt['name']} using curl")
|
|
success = True
|
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
pass
|
|
|
|
if not success:
|
|
try:
|
|
import urllib.request
|
|
urllib.request.urlretrieve(ckpt["url"], ckpt["path"])
|
|
print(f"Successfully downloaded {ckpt['name']} using urllib")
|
|
success = True
|
|
except Exception as e:
|
|
print(f"Failed to download {ckpt['name']} using urllib: {e}")
|
|
|
|
if not success:
|
|
print(f"Failed to download {ckpt['name']}. Please download manually:")
|
|
print(f"wget {ckpt['url']} -O {ckpt['path']}")
|
|
print(f"or")
|
|
print(f"curl -L {ckpt['url']} -o {ckpt['path']}")
|
|
else:
|
|
print(f"{ckpt['name']} already exists at {ckpt['path']}")
|
|
|
|
|
|
def prepare_tg2mv_pipeline(
|
|
base_model: str,
|
|
vae_model: Optional[str] = None,
|
|
lora_model: Optional[str] = None,
|
|
adapter_path: str = "huanngzh/mv-adapter",
|
|
scheduler: str = "ddpm",
|
|
num_views: int = 6,
|
|
device: str = "cuda",
|
|
dtype: torch.dtype = torch.float16,
|
|
use_mmgp: bool = False,
|
|
adapter_local_path: Optional[str] = None,
|
|
):
|
|
"""
|
|
Prepare MV-Adapter TG2MV (Text-Guided to Multi-View) pipeline
|
|
|
|
Args:
|
|
base_model: Base Stable Diffusion XL model
|
|
vae_model: Custom VAE model (optional)
|
|
lora_model: LoRA model (optional)
|
|
adapter_path: Path to adapter weights
|
|
scheduler: Scheduler type ("ddpm")
|
|
num_views: Number of views to generate
|
|
device: Device for computation
|
|
dtype: Data type
|
|
use_mmgp: Use mmgp for memory optimization
|
|
adapter_local_path: Local path for adapter download
|
|
"""
|
|
import gc
|
|
torch.cuda.empty_cache()
|
|
gc.collect()
|
|
|
|
pipe_kwargs = {}
|
|
if vae_model is not None:
|
|
pipe_kwargs["vae"] = AutoencoderKL.from_pretrained(vae_model, torch_dtype=dtype)
|
|
|
|
pipe = MVAdapterT2MVSDXLPipeline.from_pretrained(base_model, torch_dtype=dtype, **pipe_kwargs)
|
|
|
|
pipe.scheduler = ShiftSNRScheduler.from_scheduler(
|
|
pipe.scheduler,
|
|
shift_mode="interpolated",
|
|
shift_scale=8.0,
|
|
scheduler_class=DDPMScheduler,
|
|
)
|
|
|
|
pipe.init_custom_adapter(
|
|
num_views=num_views, self_attn_processor=DecoupledMVRowColSelfAttnProcessor2_0
|
|
)
|
|
|
|
weight_name = "mvadapter_tg2mv_sdxl.safetensors"
|
|
pipe.load_custom_adapter(adapter_path, weight_name=weight_name, local_cache_dir=adapter_local_path)
|
|
|
|
pipe.to(device=device, dtype=dtype)
|
|
pipe.cond_encoder.to(device=device, dtype=dtype)
|
|
|
|
if lora_model is not None:
|
|
model_, name_ = lora_model.rsplit("/", 1)
|
|
pipe.load_lora_weights(model_, weight_name=name_)
|
|
|
|
pipe.enable_vae_slicing()
|
|
pipe.enable_vae_tiling()
|
|
|
|
if hasattr(pipe, 'enable_model_cpu_offload'):
|
|
if not use_mmgp:
|
|
pipe.enable_model_cpu_offload()
|
|
print("[INFO] CPU offload for memory optimization")
|
|
|
|
if use_mmgp and MMGP_AVAILABLE:
|
|
torch.cuda.empty_cache()
|
|
gc.collect()
|
|
|
|
all_models = {
|
|
"vae": pipe.vae,
|
|
"text_encoder": pipe.text_encoder,
|
|
"text_encoder_2": pipe.text_encoder_2,
|
|
"unet": pipe.unet,
|
|
"cond_encoder": pipe.cond_encoder,
|
|
}
|
|
|
|
if hasattr(pipe, 'image_encoder') and pipe.image_encoder is not None:
|
|
all_models["image_encoder"] = pipe.image_encoder
|
|
|
|
try:
|
|
offload.profile(all_models, profile_type.HighRAM_LowVRAM)
|
|
print("mmgp profiling applied successfully.")
|
|
except Exception as e:
|
|
print(f"[WARNING] mmgp failed, continuing without it: {e}")
|
|
|
|
torch.cuda.empty_cache()
|
|
gc.collect()
|
|
|
|
return pipe
|
|
|
|
|
|
def run_tg2mv_pipeline(
|
|
pipe,
|
|
mesh_path: str,
|
|
num_views: int,
|
|
text: str,
|
|
height: int,
|
|
width: int,
|
|
num_inference_steps: int,
|
|
guidance_scale: float,
|
|
seed: int,
|
|
negative_prompt: str = "watermark, ugly, deformed, noisy, blurry, low contrast",
|
|
lora_scale: float = 1.0,
|
|
device: str = "cuda",
|
|
):
|
|
"""
|
|
Execute text-guided multiview generation (TG2MV)
|
|
|
|
Args:
|
|
pipe: TG2MV pipeline
|
|
mesh_path: Path to 3D mesh file for guidance
|
|
num_views: Number of views to generate
|
|
text: Text prompt
|
|
height: Image height
|
|
width: Image width
|
|
num_inference_steps: Number of denoising steps
|
|
guidance_scale: Guidance scale
|
|
seed: Random seed
|
|
negative_prompt: Negative prompt
|
|
lora_scale: LoRA scale
|
|
device: Device for computation
|
|
"""
|
|
cameras = get_orthogonal_camera(
|
|
elevation_deg=[0, 0, 0, 0, 89.99, -89.99],
|
|
distance=[1.8] * num_views,
|
|
left=-0.55,
|
|
right=0.55,
|
|
bottom=-0.55,
|
|
top=0.55,
|
|
azimuth_deg=[x - 90 for x in [0, 90, 180, 270, 180, 180]],
|
|
device=device,
|
|
)
|
|
ctx = NVDiffRastContextWrapper(device=device)
|
|
|
|
mesh = load_mesh(mesh_path, rescale=True, device=device)
|
|
render_out = render(
|
|
ctx,
|
|
mesh,
|
|
cameras,
|
|
height=height,
|
|
width=width,
|
|
render_attr=False,
|
|
normal_background=0.0,
|
|
)
|
|
|
|
pos_images = tensor_to_image((render_out.pos + 0.5).clamp(0, 1), batched=True)
|
|
normal_images = tensor_to_image(
|
|
(render_out.normal / 2 + 0.5).clamp(0, 1), batched=True
|
|
)
|
|
|
|
control_images = (
|
|
torch.cat(
|
|
[
|
|
(render_out.pos + 0.5).clamp(0, 1),
|
|
(render_out.normal / 2 + 0.5).clamp(0, 1),
|
|
],
|
|
dim=-1,
|
|
)
|
|
.permute(0, 3, 1, 2)
|
|
.to(device)
|
|
)
|
|
|
|
pipe_kwargs = {}
|
|
if seed != -1 and isinstance(seed, int):
|
|
pipe_kwargs["generator"] = torch.Generator(device=device).manual_seed(seed)
|
|
|
|
images = pipe(
|
|
text,
|
|
height=height,
|
|
width=width,
|
|
num_inference_steps=num_inference_steps,
|
|
guidance_scale=guidance_scale,
|
|
num_images_per_prompt=num_views,
|
|
control_image=control_images,
|
|
control_conditioning_scale=1.0,
|
|
negative_prompt=negative_prompt,
|
|
cross_attention_kwargs={"scale": lora_scale},
|
|
**pipe_kwargs,
|
|
).images
|
|
|
|
return images, pos_images, normal_images |