hackkhai-ComfyUI-Image-Matting/nodes.py

import os
import sys
import math
import copy
import torch
import numpy as np
import cv2
from tqdm import trange
import folder_paths

import comfy.model_management
from comfy.utils import ProgressBar
from transformers import VitMatteForImageMatting, VitMatteImageProcessor

import cv2
import numpy as np
import logging
from PIL import Image

logger = logging.getLogger('comfyui_image_matting')

def FB_blur_fusion_foreground_estimator_1(image, alpha, r=90):
    alpha = alpha[:, :, None]
    return FB_blur_fusion_foreground_estimator(image, F=image, B=image, alpha=alpha, r=r)[0]


def FB_blur_fusion_foreground_estimator_2(image, alpha, r=90):
    alpha = alpha[:, :, None]
    F, blur_B = FB_blur_fusion_foreground_estimator(
        image, image, image, alpha, r)
    return FB_blur_fusion_foreground_estimator(image, F, blur_B, alpha, r=6)[0]


def FB_blur_fusion_foreground_estimator(image, F, B, alpha, r=90):
    blurred_alpha = cv2.blur(alpha, (r, r))[:, :, None]

    blurred_FA = cv2.blur(F * alpha, (r, r))
    blurred_F = blurred_FA / (blurred_alpha + 1e-5)

    blurred_B1A = cv2.blur(B * (1 - alpha), (r, r))
    blurred_B = blurred_B1A / ((1 - blurred_alpha) + 1e-5)
    F = blurred_F + alpha * \
        (image - alpha * blurred_F - (1 - alpha) * blurred_B)
    F = np.clip(F, 0, 1)
    return F, blurred_B

matting_model_dir_name = "matting_models"
matting_model_list = {
    "vitmatte_small (103 MB)": {
        "model_url": "hustvl/vitmatte-small-composition-1k"
    },
    "vitmatte_base (387 MB)": {
        "model_url": "hustvl/vitmatte-base-composition-1k"
    },
}
def list_matting_models():
    return list(matting_model_list.keys())

def load_matting_model(model_name):
    model_url = matting_model_list[model_name]["model_url"]
    matting_model_checkpoint_path = get_local_filepath_(
        matting_model_list[model_name]["model_url"], matting_model_dir_name,f"{model_url}")
    matting_model = VitMatteForImageMatting.from_pretrained(matting_model_checkpoint_path)
    preprocessor = VitMatteImageProcessor.from_pretrained(matting_model_checkpoint_path)
    matting_model_device = comfy.model_management.get_torch_device()
    matting_model = matting_model.to(matting_model_device)
    matting_model.eval()
    return (matting_model, preprocessor)

def get_local_filepath_(url, dirname, local_file_name=None):

    destination = folder_paths.get_full_path(dirname, local_file_name)
    if destination:
        logger.warn(f'using extra model: {destination}')
        return destination

    folder = os.path.join(folder_paths.models_dir, dirname)
    if not os.path.exists(folder):
        os.makedirs(folder)

    destination = os.path.join(folder, local_file_name)
    if not os.path.exists(destination):
        
        # download_url_to_file(url, destination)
        logger.warn(f'downloading {url} to {destination}')
        VitMatteForImageMatting.from_pretrained(url).save_pretrained(destination)
        VitMatteImageProcessor.from_pretrained(url).save_pretrained(destination)
    return destination

class MattingModelLoader:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "model_name": (list_matting_models(), ),
            }
        }
    CATEGORY = "image_matting"
    FUNCTION = "main"
    RETURN_TYPES = ("MATTING_MODEL", "MATTING_PREPROCESSOR")

    def main(self, model_name):
        matting_model,matting_preprocessor = load_matting_model(model_name)
        return (matting_model, matting_preprocessor)
def gen_trimap(alpha, kernel_size=25):
        
        k_size = kernel_size
        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,
                                            (k_size, k_size))
        dilated = cv2.dilate(alpha, kernel)
        eroded = cv2.erode(alpha, kernel)
        trimap = np.zeros(alpha.shape)
        trimap.fill(128)
        trimap[eroded > 254.5] = 255
        trimap[dilated < 0.5] = 0

        return trimap

class CreateTrimap:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "mask": ('MASK', {}),
                "kernel_size": ("INT", {
                    "default": 9,
                    "min": 0,
                    "max": 50,
                    "step": 0.01
                }),
            }
        }
    CATEGORY = "image_matting"
    FUNCTION = "main"
    RETURN_TYPES = ("MASK",)

    def main(self, mask, kernel_size):
        res_masks = []
        for mask_ in mask:
            
            mask_ = mask_.cpu().numpy()*255
            mask_ = mask_.astype(np.uint8)
            mask_ = gen_trimap(mask_, kernel_size=kernel_size)/255.
            
            res_masks.extend([torch.from_numpy(mask_).unsqueeze(0)])
        return (torch.cat(res_masks, dim=0),)


    
class ApplyMatting:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "matting_model": ('MATTING_MODEL', {}),
                "matting_preprocessor": ('MATTING_PREPROCESSOR', {}),
                "image": ('IMAGE', {}),
                "trimap": ("MASK", {}),
            }
        }
    CATEGORY = "image_matting"
    FUNCTION = "main"
    RETURN_TYPES = ("IMAGE", "MASK")

    def main(self, matting_model,matting_preprocessor, image, trimap):
        res_images = []
        res_masks = []
        for item, tri in zip(image, trimap):
            item = Image.fromarray(
                np.clip(255. * item.cpu().numpy(), 0, 255).astype(np.uint8)).convert('RGBA')
            print(tri.shape)
            tri = Image.fromarray(
                np.clip(255. * tri.cpu().numpy(), 0, 255).astype(np.uint8)).convert('L')
            
            alpha = matting_predict(
                matting_model,
                matting_preprocessor,
                item,
                tri
            )
            alpha = (alpha*255).astype(np.uint8)
            item_shape = item.size
            alpha = cv2.resize(alpha, (item_shape[0], item_shape[1]), interpolation=cv2.INTER_NEAREST)
            ffe_foreground = FB_blur_fusion_foreground_estimator(
                np.array(item.convert("RGB"))/255.,np.array(item.convert("RGB"))/255.,np.array(item.convert("RGB"))/255., alpha[:,:,None]/255., r=90
            )[0]*255
            ffe_foreground = ffe_foreground.astype(np.uint8)
            
            #H,W,C
            alpha = alpha.astype(np.uint8)[..., None] 
            print(ffe_foreground.shape, alpha.shape)
            #make it rgba with alpha
            ffe_foreground = np.concatenate([ffe_foreground, alpha], axis=2)
            
            #resize image from HWC to CHW
            res_images.extend([torch.from_numpy(ffe_foreground).unsqueeze(0)/255.])
            res_masks.extend([torch.from_numpy(alpha[:,:,0]).unsqueeze(0)/255.])
        
        if len(res_images) == 0:
            _, height, width, _ = image.size()
            empty_mask = torch.zeros((1, height, width), dtype=torch.uint8, device="cpu")
            return (empty_mask, empty_mask)
        return (torch.cat(res_images, dim=0), torch.cat(res_masks, dim=0))

def matting_predict(
    matting_model,
    matting_preprocessor,
    image,
    tri
):
    def load_matting_image(image_pil, tri, image_preprocessor):

        inputs = image_preprocessor(images=image_pil, trimaps=tri, return_tensors="pt")
        return inputs

    inputs = load_matting_image(image.convert("RGB"), tri, matting_preprocessor)
    for key in inputs:
        inputs[key] = inputs[key].to(comfy.model_management.get_torch_device())
    with torch.no_grad():
        alphas = matting_model(**inputs).alphas[0][0].cpu().numpy()
    return alphas

NODE_CLASS_MAPPINGS = {
    "MattingModelLoader": MattingModelLoader,
    "ApplyMatting": ApplyMatting,
    "CreateTrimap": CreateTrimap,
}

NODE_DISPLAY_NAME_MAPPINGS = {
    "MattingModelLoader": "Load Matting Model",
    "ApplyMatting": "Apply Matting",
    "CreateTrimap": "Create Trimap",
}