smthemex-ComfyUI_Sapiens/Sapiens_Pytorch/normal.py

# !/usr/bin/env python
# -*- coding: UTF-8 -*-
from enum import Enum
import time
import cv2
import numpy as np
import torch
import torch.nn.functional as F

from .common import create_preprocessor, TaskType, download_hf_model,ImageProcessorNormal,ModelManager


class SapiensNormalType(Enum):
    OFF = "off"
    NORMAL_03B = "sapiens_0.3b_normal_render_people_epoch_66.pth"
    NORMAL_06B = "sapiens_0.6b_normal_render_people_epoch_200.pth"
    NORMAL_1B = "sapiens_1b_normal_render_people_epoch_115.pth"
    NORMAL_2B = "sapiens_2b_normal_render_people_epoch_70.pth"
    NORMAL_03B_T = "sapiens_0.3b_normal_render_people_epoch_66_torchscript.pt2"
    NORMAL_06B_T = "sapiens_0.6b_normal_render_people_epoch_200_torchscript.pt2"
    NORMAL_1B_T = "sapiens_1b_normal_render_people_epoch_115_torchscript.pt2"
    NORMAL_2B_T = "sapiens_2b_normal_render_people_epoch_70_torchscript.pt2"
    NORMAL_03B_16 = "sapiens_0.3b_normal_render_people_epoch_66_bfloat16.pt2"
    NORMAL_06B_16 = "sapiens_0.6b_normal_render_people_epoch_200_bfloat16.pt2"
    NORMAL_1B_16 = "sapiens_1b_normal_render_people_epoch_115_bfloat16.pt2"
    NORMAL_2B_16 = "sapiens_2b_normal_render_people_epoch_70_bfloat16.pt2"


def draw_normal_map(normal_map: np.ndarray) -> np.ndarray:
    # Normalize the normal map
    normal_map_norm = np.linalg.norm(normal_map, axis=-1, keepdims=True)
    normal_map_normalized = normal_map / (normal_map_norm + 1e-5)  # Add a small epsilon to avoid division by zero
    normal_map = ((normal_map_normalized + 1) / 2 * 255).astype(np.uint8)

    # Convert to BGR
    return cv2.cvtColor(normal_map, cv2.COLOR_RGB2BGR)


def postprocess_normal(results: torch.Tensor, img_shape: tuple[int, int]) -> np.ndarray:
    result = results[0].detach().cpu()

    # Upsample the result to the original image size
    logits = F.interpolate(result.unsqueeze(0), size=img_shape, mode="bilinear").squeeze(0)

    # Covert to numpy array
    normal_map = logits.float().numpy().transpose(1, 2, 0)

    return normal_map


class SapiensNormal():
    def __init__(self,
                 type: SapiensNormalType = SapiensNormalType.NORMAL_03B,local_normal="",pt_type="float32_torch",model_dir="",img_size=(1024, 768),use_torchscript=True,
                 device: torch.device = torch.device("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"),
                 dtype: torch.dtype = torch.float32):
        self.local_normal=local_normal
        self.model_dir = model_dir
        self.pt_type = pt_type
        self.img_size = img_size
        self.use_torchscript = use_torchscript
        self.device = device
        self.dtype = dtype
        if self.local_normal:
            path= self.local_normal
        else:
            path = download_hf_model(type.value, TaskType.NORMAL,self.model_dir,dtype=self.pt_type)
        if self.use_torchscript:
            model = torch.jit.load(path)
            model = model.eval()
            model.to(device).to(dtype)
        else:
            model = torch.export.load(path).module()
            dtype = torch.half if self.dtype=="float16" else torch.bfloat16
            model.to(dtype)
            model = torch.compile(model, mode="max-autotune", fullgraph=True)
        self.model = model
        self.preprocessor = create_preprocessor(input_size=self.img_size)  # Only these values seem to work well(1024, 768)

    def __call__(self, img: np.ndarray) -> np.ndarray:
        start = time.perf_counter()

        # Model expects BGR, but we change to RGB here because the preprocessor will switch the channels also
        input = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        tensor = self.preprocessor(input).to(self.device).to(self.dtype)

        with torch.inference_mode():
            results = self.model(tensor)

        normals = postprocess_normal(results, img.shape[:2])
        #print(f"Normal inference took: {time.perf_counter() - start:.4f} seconds")
        return normals
    
    def enable_model_cpu_offload(self):
        self.model.to("cpu")
        torch.cuda.empty_cache()
    def move_to_cuda(self):
        self.model.to("cuda")

class NormalSapiens():
    def __init__(self,
                 type: SapiensNormalType = SapiensNormalType.NORMAL_03B,local_normal="",
                 pt_type="float32_torch", model_dir="", use_torchscript=True,
                 dtype: torch.dtype = torch.float32):
        self.local_normal = local_normal
        self.model_dir = model_dir
        self.pt_type = pt_type
        self.use_torchscript = use_torchscript
        self.model_name = type
        self.device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
        self.dtype = dtype
        self.image_processor = ImageProcessorNormal()
        if self.local_normal:
            path = self.local_normal
        else:
            path = download_hf_model(self.model_name.value, TaskType.NORMAL, self.model_dir, dtype=self.pt_type)
        self.model = ModelManager.load_model(path, self.device)
    
    def __call__(self, image, if_seg, seg_in,select_obj,RGB_BG):
        #start = time.perf_counter()
        with torch.inference_mode():
            result = self.image_processor.process_image(image, self.model, if_seg, seg_in,select_obj,RGB_BG, model_dtype=self.dtype)
        #print(f"Normal inference took: {time.perf_counter() - start:.4f} seconds")
        return result
    
    def enable_model_cpu_offload(self):
        self.model.to("cpu")
        torch.cuda.empty_cache()
    
    def move_to_cuda(self):
        self.model.to("cuda")