import torch import torch.nn.functional as F from torch import nn from torch.nn import BatchNorm2d def make_coordinate_grid(spatial_size, type): """ Create a meshgrid [-1,1] x [-1,1] of given spatial_size. """ h, w = spatial_size x = torch.arange(w).type(type) y = torch.arange(h).type(type) x = 2 * (x / (w - 1)) - 1 y = 2 * (y / (h - 1)) - 1 yy = y.view(-1, 1).repeat(1, w) xx = x.view(1, -1).repeat(h, 1) meshed = torch.cat([xx.unsqueeze_(2), yy.unsqueeze_(2)], 2) return meshed def kp2gaussian(kp, spatial_size, kp_variance): """ Transform a keypoint into gaussian like representation """ mean = kp coordinate_grid = make_coordinate_grid(spatial_size, mean.type()) number_of_leading_dimensions = len(mean.shape) - 1 shape = (1,) * number_of_leading_dimensions + coordinate_grid.shape coordinate_grid = coordinate_grid.view(*shape) repeats = mean.shape[:number_of_leading_dimensions] + (1, 1, 1) coordinate_grid = coordinate_grid.repeat(*repeats) # Preprocess kp shape shape = mean.shape[:number_of_leading_dimensions] + (1, 1, 2) mean = mean.view(*shape) mean_sub = coordinate_grid - mean out = torch.exp(-0.5 * (mean_sub**2).sum(-1) / kp_variance) return out class UpBlock2d(nn.Module): """ Upsampling block for use in decoder. """ def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1): super(UpBlock2d, self).__init__() self.conv = nn.Conv2d( in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, padding=padding, groups=groups, ) self.norm = BatchNorm2d(out_features, affine=True) def forward(self, x): out = F.interpolate(x, scale_factor=2) out = self.conv(out) out = self.norm(out) out = F.relu(out) return out class DownBlock2d(nn.Module): """ Downsampling block for use in encoder. """ def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1): super(DownBlock2d, self).__init__() self.conv = nn.Conv2d( in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, padding=padding, groups=groups, ) self.norm = BatchNorm2d(out_features, affine=True) self.pool = nn.AvgPool2d(kernel_size=(2, 2)) def forward(self, x): out = self.conv(x) out = self.norm(out) out = F.relu(out) out = self.pool(out) return out class Encoder(nn.Module): """ Hourglass Encoder """ def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256): super(Encoder, self).__init__() down_blocks = [] for i in range(num_blocks): down_blocks.append( DownBlock2d( in_features if i == 0 else min(max_features, block_expansion * (2**i)), min(max_features, block_expansion * (2 ** (i + 1))), kernel_size=3, padding=1, ) ) self.down_blocks = nn.ModuleList(down_blocks) def forward(self, x): outs = [x] for down_block in self.down_blocks: outs.append(down_block(outs[-1])) return outs class Decoder(nn.Module): """ Hourglass Decoder """ def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256): super(Decoder, self).__init__() up_blocks = [] for i in range(num_blocks)[::-1]: in_filters = (1 if i == num_blocks - 1 else 2) * min( max_features, block_expansion * (2 ** (i + 1)) ) out_filters = min(max_features, block_expansion * (2**i)) up_blocks.append( UpBlock2d(in_filters, out_filters, kernel_size=3, padding=1) ) self.up_blocks = nn.ModuleList(up_blocks) self.out_filters = block_expansion + in_features def forward(self, x): out = x.pop() cnt = 0 for up_block in self.up_blocks: out = up_block(out) skip = x.pop() out = torch.cat([out, skip], dim=1) cnt += 1 return out class Hourglass(nn.Module): """ Hourglass architecture. """ def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256): super(Hourglass, self).__init__() self.encoder = Encoder(block_expansion, in_features, num_blocks, max_features) self.decoder = Decoder(block_expansion, in_features, num_blocks, max_features) self.out_filters = self.decoder.out_filters def forward(self, x): return self.decoder(self.encoder(x)) class AntiAliasInterpolation2d(nn.Module): """ Band-limited downsampling, for better preservation of the input signal. """ def __init__(self, channels, scale): super(AntiAliasInterpolation2d, self).__init__() sigma = (1 / scale - 1) / 2 kernel_size = 2 * round(sigma * 4) + 1 self.ka = kernel_size // 2 self.kb = self.ka - 1 if kernel_size % 2 == 0 else self.ka kernel_size = [kernel_size, kernel_size] sigma = [sigma, sigma] # The gaussian kernel is the product of the # gaussian function of each dimension. kernel = 1 meshgrids = torch.meshgrid( [torch.arange(size, dtype=torch.float32) for size in kernel_size] ) for size, std, mgrid in zip(kernel_size, sigma, meshgrids): mean = (size - 1) / 2 kernel *= torch.exp(-((mgrid - mean) ** 2) / (2 * std**2)) # Make sure sum of values in gaussian kernel equals 1. kernel = kernel / torch.sum(kernel) # Reshape to depthwise convolutional weight kernel = kernel.view(1, 1, *kernel.size()) kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1)) self.register_buffer("weight", kernel) self.groups = channels self.scale = scale inv_scale = 1 / scale self.int_inv_scale = int(inv_scale) def forward(self, input): if self.scale == 1.0: return input out = F.pad(input, (self.ka, self.kb, self.ka, self.kb)) out = F.conv2d(out, weight=self.weight, groups=self.groups) out = out[:, :, :: self.int_inv_scale, :: self.int_inv_scale] return out class KPDetector(nn.Module): """ Detecting keypoints. Return keypoint positions. """ def __init__( self, block_expansion=32, num_kp=10, num_channels=3, max_features=1024, num_blocks=5, temperature=0.1, scale_factor=0.25, pad=0, ): super(KPDetector, self).__init__() self.predictor = Hourglass( block_expansion, in_features=num_channels, max_features=max_features, num_blocks=num_blocks, ) self.kp = nn.Conv2d( in_channels=self.predictor.out_filters, out_channels=num_kp, kernel_size=(7, 7), padding=pad, ) self.num_kp = num_kp # We do not need the Jacobian (from FOMM). # if estimate_jacobian: # self.num_jacobian_maps = 1 if single_jacobian_map else num_kp # self.jacobian = nn.Conv2d(in_channels=self.predictor.out_filters, # out_channels=4 * self.num_jacobian_maps, kernel_size=(7, 7), padding=pad) # self.jacobian.weight.data.zero_() # self.jacobian.bias.data.copy_(torch.tensor([1, 0, 0, 1] * self.num_jacobian_maps, dtype=torch.float)) # else: # self.jacobian = None self.temperature = temperature self.scale_factor = scale_factor if self.scale_factor != 1: self.down = AntiAliasInterpolation2d(num_channels, self.scale_factor) def gaussian2kp(self, heatmap): shape = heatmap.shape heatmap = heatmap.unsqueeze(-1) grid = ( make_coordinate_grid(shape[2:], heatmap.type()).unsqueeze_(0).unsqueeze_(0) ) value = (heatmap * grid).sum(dim=(2, 3)) return value def forward(self, x): with torch.no_grad(): if self.scale_factor != 1: x = self.down(x) feature_map = self.predictor(x) prediction = self.kp(feature_map) final_shape = prediction.shape heatmap = prediction.view(final_shape[0], final_shape[1], -1) heatmap = F.softmax(heatmap / self.temperature, dim=2) heatmap = heatmap.view(*final_shape) out = self.gaussian2kp(heatmap) heatmap = heatmap.unsqueeze(2) # We do not need the Jacobian (from FOMM). # if self.jacobian is not None: # jacobian_map = self.jacobian(feature_map) # jacobian_map = jacobian_map.reshape(final_shape[0], self.num_jacobian_maps, 4, final_shape[2], # final_shape[3]) # jacobian = heatmap * jacobian_map # jacobian = jacobian.view(final_shape[0], final_shape[1], 4, -1) # jacobian = jacobian.sum(dim=-1) # jacobian = jacobian.view(jacobian.shape[0], jacobian.shape[1], 2, 2) return out, {"feature_map": feature_map, "heatmap": heatmap}