# -*- coding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. # Please use this implementation in your products # This implementation may produce slightly different results from Saining Xie's official implementations, # but it generates smoother edges and is more suitable for ControlNet as well as other image-to-image translations. # Different from official models and other implementations, this is an RGB-input model (rather than BGR) # and in this way it works better for gradio's RGB protocol from abc import ABCMeta import cv2 import numpy as np import torch import torchvision from einops import rearrange from scepter.modules.annotator.base_annotator import BaseAnnotator from scepter.modules.annotator.registry import ANNOTATORS from scepter.modules.utils.config import dict_to_yaml from scepter.modules.utils.distribute import we from scepter.modules.utils.file_system import FS from torchvision.transforms import InterpolationMode def nms(x, t, s): x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s) f1 = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]], dtype=np.uint8) f2 = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=np.uint8) f3 = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=np.uint8) f4 = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]], dtype=np.uint8) y = np.zeros_like(x) for f in [f1, f2, f3, f4]: np.putmask(y, cv2.dilate(x, kernel=f) == x, x) z = np.zeros_like(y, dtype=np.uint8) z[y > t] = 255 return z class DoubleConvBlock(torch.nn.Module): def __init__(self, input_channel, output_channel, layer_number): super().__init__() self.convs = torch.nn.Sequential() self.convs.append( torch.nn.Conv2d(in_channels=input_channel, out_channels=output_channel, kernel_size=(3, 3), stride=(1, 1), padding=1)) for i in range(1, layer_number): self.convs.append( torch.nn.Conv2d(in_channels=output_channel, out_channels=output_channel, kernel_size=(3, 3), stride=(1, 1), padding=1)) self.projection = torch.nn.Conv2d(in_channels=output_channel, out_channels=1, kernel_size=(1, 1), stride=(1, 1), padding=0) def __call__(self, x, down_sampling=False): h = x if down_sampling: h = torch.nn.functional.max_pool2d(h, kernel_size=(2, 2), stride=(2, 2)) for conv in self.convs: h = conv(h) h = torch.nn.functional.relu(h) return h, self.projection(h) class ControlNetHED_Apache2(torch.nn.Module): def __init__(self): super().__init__() self.norm = torch.nn.Parameter(torch.zeros(size=(1, 3, 1, 1))) self.block1 = DoubleConvBlock(input_channel=3, output_channel=64, layer_number=2) self.block2 = DoubleConvBlock(input_channel=64, output_channel=128, layer_number=2) self.block3 = DoubleConvBlock(input_channel=128, output_channel=256, layer_number=3) self.block4 = DoubleConvBlock(input_channel=256, output_channel=512, layer_number=3) self.block5 = DoubleConvBlock(input_channel=512, output_channel=512, layer_number=3) def __call__(self, x): h = x - self.norm h, projection1 = self.block1(h) h, projection2 = self.block2(h, down_sampling=True) h, projection3 = self.block3(h, down_sampling=True) h, projection4 = self.block4(h, down_sampling=True) h, projection5 = self.block5(h, down_sampling=True) return projection1, projection2, projection3, projection4, projection5 @ANNOTATORS.register_class() class HedAnnotator(BaseAnnotator, metaclass=ABCMeta): para_dict = {} def __init__(self, cfg, logger=None): super().__init__(cfg, logger=logger) self.netNetwork = ControlNetHED_Apache2().float().eval() pretrained_model = cfg.get('PRETRAINED_MODEL', None) if pretrained_model: with FS.get_from(pretrained_model, wait_finish=True) as local_path: self.netNetwork.load_state_dict(torch.load(local_path)) @torch.no_grad() @torch.inference_mode() @torch.autocast('cuda', enabled=False) def forward(self, image): if isinstance(image, torch.Tensor): if len(image.shape) == 3: image = rearrange(image, 'h w c -> 1 c h w') B, C, H, W = image.shape elif len(image.shape) == 4: B, C, H, W = image.shape else: raise "Unsurpport input image's shape" elif isinstance(image, np.ndarray): image = torch.from_numpy(image.copy()).float() if len(image.shape) == 3: image = rearrange(image, 'h w c -> 1 c h w') B, C, H, W = image.shape elif len(image.shape) == 4: B, C, H, W = image.shape else: raise "Unsurpport input image's shape" else: raise "Unsurpport input image's type" transform = torchvision.transforms.Resize( (H, W), interpolation=InterpolationMode.BILINEAR, antialias=True) edges = self.netNetwork(image.to(we.device_id)) edges = [transform(e) for e in edges] edges = torch.cat(edges, dim=1) edges = 1 / (1 + torch.exp(-torch.mean(edges, dim=1).type(torch.float))) edges = edges.cpu().numpy() edges = 255 - (edges * 255.0).clip(0, 255).astype(np.uint8) return edges[..., None].repeat(3, -1) @staticmethod def get_config_template(): return dict_to_yaml('ANNOTATORS', __class__.__name__, HedAnnotator.para_dict, set_name=True)