159 lines
6.4 KiB
Python
159 lines
6.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
# Please use this implementation in your products
|
|
# This implementation may produce slightly different results from Saining Xie's official implementations,
|
|
# but it generates smoother edges and is more suitable for ControlNet as well as other image-to-image translations.
|
|
# Different from official models and other implementations, this is an RGB-input model (rather than BGR)
|
|
# and in this way it works better for gradio's RGB protocol
|
|
|
|
from abc import ABCMeta
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import torch
|
|
import torchvision
|
|
from einops import rearrange
|
|
from scepter.modules.annotator.base_annotator import BaseAnnotator
|
|
from scepter.modules.annotator.registry import ANNOTATORS
|
|
from scepter.modules.utils.config import dict_to_yaml
|
|
from scepter.modules.utils.distribute import we
|
|
from scepter.modules.utils.file_system import FS
|
|
from torchvision.transforms import InterpolationMode
|
|
|
|
|
|
def nms(x, t, s):
|
|
x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s)
|
|
|
|
f1 = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]], dtype=np.uint8)
|
|
f2 = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=np.uint8)
|
|
f3 = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=np.uint8)
|
|
f4 = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]], dtype=np.uint8)
|
|
|
|
y = np.zeros_like(x)
|
|
|
|
for f in [f1, f2, f3, f4]:
|
|
np.putmask(y, cv2.dilate(x, kernel=f) == x, x)
|
|
|
|
z = np.zeros_like(y, dtype=np.uint8)
|
|
z[y > t] = 255
|
|
return z
|
|
|
|
|
|
class DoubleConvBlock(torch.nn.Module):
|
|
def __init__(self, input_channel, output_channel, layer_number):
|
|
super().__init__()
|
|
self.convs = torch.nn.Sequential()
|
|
self.convs.append(
|
|
torch.nn.Conv2d(in_channels=input_channel,
|
|
out_channels=output_channel,
|
|
kernel_size=(3, 3),
|
|
stride=(1, 1),
|
|
padding=1))
|
|
for i in range(1, layer_number):
|
|
self.convs.append(
|
|
torch.nn.Conv2d(in_channels=output_channel,
|
|
out_channels=output_channel,
|
|
kernel_size=(3, 3),
|
|
stride=(1, 1),
|
|
padding=1))
|
|
self.projection = torch.nn.Conv2d(in_channels=output_channel,
|
|
out_channels=1,
|
|
kernel_size=(1, 1),
|
|
stride=(1, 1),
|
|
padding=0)
|
|
|
|
def __call__(self, x, down_sampling=False):
|
|
h = x
|
|
if down_sampling:
|
|
h = torch.nn.functional.max_pool2d(h,
|
|
kernel_size=(2, 2),
|
|
stride=(2, 2))
|
|
for conv in self.convs:
|
|
h = conv(h)
|
|
h = torch.nn.functional.relu(h)
|
|
return h, self.projection(h)
|
|
|
|
|
|
class ControlNetHED_Apache2(torch.nn.Module):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.norm = torch.nn.Parameter(torch.zeros(size=(1, 3, 1, 1)))
|
|
self.block1 = DoubleConvBlock(input_channel=3,
|
|
output_channel=64,
|
|
layer_number=2)
|
|
self.block2 = DoubleConvBlock(input_channel=64,
|
|
output_channel=128,
|
|
layer_number=2)
|
|
self.block3 = DoubleConvBlock(input_channel=128,
|
|
output_channel=256,
|
|
layer_number=3)
|
|
self.block4 = DoubleConvBlock(input_channel=256,
|
|
output_channel=512,
|
|
layer_number=3)
|
|
self.block5 = DoubleConvBlock(input_channel=512,
|
|
output_channel=512,
|
|
layer_number=3)
|
|
|
|
def __call__(self, x):
|
|
h = x - self.norm
|
|
h, projection1 = self.block1(h)
|
|
h, projection2 = self.block2(h, down_sampling=True)
|
|
h, projection3 = self.block3(h, down_sampling=True)
|
|
h, projection4 = self.block4(h, down_sampling=True)
|
|
h, projection5 = self.block5(h, down_sampling=True)
|
|
return projection1, projection2, projection3, projection4, projection5
|
|
|
|
|
|
@ANNOTATORS.register_class()
|
|
class HedAnnotator(BaseAnnotator, metaclass=ABCMeta):
|
|
para_dict = {}
|
|
|
|
def __init__(self, cfg, logger=None):
|
|
super().__init__(cfg, logger=logger)
|
|
self.netNetwork = ControlNetHED_Apache2().float().eval()
|
|
pretrained_model = cfg.get('PRETRAINED_MODEL', None)
|
|
if pretrained_model:
|
|
with FS.get_from(pretrained_model, wait_finish=True) as local_path:
|
|
self.netNetwork.load_state_dict(torch.load(local_path))
|
|
|
|
@torch.no_grad()
|
|
@torch.inference_mode()
|
|
@torch.autocast('cuda', enabled=False)
|
|
def forward(self, image):
|
|
if isinstance(image, torch.Tensor):
|
|
if len(image.shape) == 3:
|
|
image = rearrange(image, 'h w c -> 1 c h w')
|
|
B, C, H, W = image.shape
|
|
elif len(image.shape) == 4:
|
|
B, C, H, W = image.shape
|
|
else:
|
|
raise "Unsurpport input image's shape"
|
|
elif isinstance(image, np.ndarray):
|
|
image = torch.from_numpy(image.copy()).float()
|
|
if len(image.shape) == 3:
|
|
image = rearrange(image, 'h w c -> 1 c h w')
|
|
B, C, H, W = image.shape
|
|
elif len(image.shape) == 4:
|
|
B, C, H, W = image.shape
|
|
else:
|
|
raise "Unsurpport input image's shape"
|
|
else:
|
|
raise "Unsurpport input image's type"
|
|
transform = torchvision.transforms.Resize(
|
|
(H, W), interpolation=InterpolationMode.BILINEAR, antialias=True)
|
|
edges = self.netNetwork(image.to(we.device_id))
|
|
edges = [transform(e) for e in edges]
|
|
edges = torch.cat(edges, dim=1)
|
|
edges = 1 / (1 +
|
|
torch.exp(-torch.mean(edges, dim=1).type(torch.float)))
|
|
edges = edges.cpu().numpy()
|
|
edges = 255 - (edges * 255.0).clip(0, 255).astype(np.uint8)
|
|
return edges[..., None].repeat(3, -1)
|
|
|
|
@staticmethod
|
|
def get_config_template():
|
|
return dict_to_yaml('ANNOTATORS',
|
|
__class__.__name__,
|
|
HedAnnotator.para_dict,
|
|
set_name=True)
|