309 lines
9.3 KiB
Python
309 lines
9.3 KiB
Python
import torch
|
|
import torch.nn.functional as F
|
|
from torch import nn
|
|
from torch.nn import BatchNorm2d
|
|
|
|
|
|
def make_coordinate_grid(spatial_size, type):
|
|
"""
|
|
Create a meshgrid [-1,1] x [-1,1] of given spatial_size.
|
|
"""
|
|
h, w = spatial_size
|
|
x = torch.arange(w).type(type)
|
|
y = torch.arange(h).type(type)
|
|
|
|
x = 2 * (x / (w - 1)) - 1
|
|
y = 2 * (y / (h - 1)) - 1
|
|
|
|
yy = y.view(-1, 1).repeat(1, w)
|
|
xx = x.view(1, -1).repeat(h, 1)
|
|
|
|
meshed = torch.cat([xx.unsqueeze_(2), yy.unsqueeze_(2)], 2)
|
|
|
|
return meshed
|
|
|
|
|
|
def kp2gaussian(kp, spatial_size, kp_variance):
|
|
"""
|
|
Transform a keypoint into gaussian like representation
|
|
"""
|
|
mean = kp
|
|
|
|
coordinate_grid = make_coordinate_grid(spatial_size, mean.type())
|
|
number_of_leading_dimensions = len(mean.shape) - 1
|
|
shape = (1,) * number_of_leading_dimensions + coordinate_grid.shape
|
|
coordinate_grid = coordinate_grid.view(*shape)
|
|
repeats = mean.shape[:number_of_leading_dimensions] + (1, 1, 1)
|
|
coordinate_grid = coordinate_grid.repeat(*repeats)
|
|
|
|
# Preprocess kp shape
|
|
shape = mean.shape[:number_of_leading_dimensions] + (1, 1, 2)
|
|
mean = mean.view(*shape)
|
|
|
|
mean_sub = coordinate_grid - mean
|
|
|
|
out = torch.exp(-0.5 * (mean_sub**2).sum(-1) / kp_variance)
|
|
|
|
return out
|
|
|
|
|
|
class UpBlock2d(nn.Module):
|
|
"""
|
|
Upsampling block for use in decoder.
|
|
"""
|
|
|
|
def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1):
|
|
super(UpBlock2d, self).__init__()
|
|
|
|
self.conv = nn.Conv2d(
|
|
in_channels=in_features,
|
|
out_channels=out_features,
|
|
kernel_size=kernel_size,
|
|
padding=padding,
|
|
groups=groups,
|
|
)
|
|
self.norm = BatchNorm2d(out_features, affine=True)
|
|
|
|
def forward(self, x):
|
|
out = F.interpolate(x, scale_factor=2)
|
|
out = self.conv(out)
|
|
out = self.norm(out)
|
|
out = F.relu(out)
|
|
return out
|
|
|
|
|
|
class DownBlock2d(nn.Module):
|
|
"""
|
|
Downsampling block for use in encoder.
|
|
"""
|
|
|
|
def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1):
|
|
super(DownBlock2d, self).__init__()
|
|
self.conv = nn.Conv2d(
|
|
in_channels=in_features,
|
|
out_channels=out_features,
|
|
kernel_size=kernel_size,
|
|
padding=padding,
|
|
groups=groups,
|
|
)
|
|
self.norm = BatchNorm2d(out_features, affine=True)
|
|
self.pool = nn.AvgPool2d(kernel_size=(2, 2))
|
|
|
|
def forward(self, x):
|
|
out = self.conv(x)
|
|
out = self.norm(out)
|
|
out = F.relu(out)
|
|
out = self.pool(out)
|
|
return out
|
|
|
|
|
|
class Encoder(nn.Module):
|
|
"""
|
|
Hourglass Encoder
|
|
"""
|
|
|
|
def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256):
|
|
super(Encoder, self).__init__()
|
|
|
|
down_blocks = []
|
|
for i in range(num_blocks):
|
|
down_blocks.append(
|
|
DownBlock2d(
|
|
in_features
|
|
if i == 0
|
|
else min(max_features, block_expansion * (2**i)),
|
|
min(max_features, block_expansion * (2 ** (i + 1))),
|
|
kernel_size=3,
|
|
padding=1,
|
|
)
|
|
)
|
|
self.down_blocks = nn.ModuleList(down_blocks)
|
|
|
|
def forward(self, x):
|
|
outs = [x]
|
|
for down_block in self.down_blocks:
|
|
outs.append(down_block(outs[-1]))
|
|
return outs
|
|
|
|
|
|
class Decoder(nn.Module):
|
|
"""
|
|
Hourglass Decoder
|
|
"""
|
|
|
|
def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256):
|
|
super(Decoder, self).__init__()
|
|
|
|
up_blocks = []
|
|
|
|
for i in range(num_blocks)[::-1]:
|
|
in_filters = (1 if i == num_blocks - 1 else 2) * min(
|
|
max_features, block_expansion * (2 ** (i + 1))
|
|
)
|
|
out_filters = min(max_features, block_expansion * (2**i))
|
|
up_blocks.append(
|
|
UpBlock2d(in_filters, out_filters, kernel_size=3, padding=1)
|
|
)
|
|
|
|
self.up_blocks = nn.ModuleList(up_blocks)
|
|
self.out_filters = block_expansion + in_features
|
|
|
|
def forward(self, x):
|
|
out = x.pop()
|
|
cnt = 0
|
|
for up_block in self.up_blocks:
|
|
out = up_block(out)
|
|
skip = x.pop()
|
|
out = torch.cat([out, skip], dim=1)
|
|
cnt += 1
|
|
return out
|
|
|
|
|
|
class Hourglass(nn.Module):
|
|
"""
|
|
Hourglass architecture.
|
|
"""
|
|
|
|
def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256):
|
|
super(Hourglass, self).__init__()
|
|
self.encoder = Encoder(block_expansion, in_features, num_blocks, max_features)
|
|
self.decoder = Decoder(block_expansion, in_features, num_blocks, max_features)
|
|
self.out_filters = self.decoder.out_filters
|
|
|
|
def forward(self, x):
|
|
return self.decoder(self.encoder(x))
|
|
|
|
|
|
class AntiAliasInterpolation2d(nn.Module):
|
|
"""
|
|
Band-limited downsampling, for better preservation of the input signal.
|
|
"""
|
|
|
|
def __init__(self, channels, scale):
|
|
super(AntiAliasInterpolation2d, self).__init__()
|
|
sigma = (1 / scale - 1) / 2
|
|
kernel_size = 2 * round(sigma * 4) + 1
|
|
self.ka = kernel_size // 2
|
|
self.kb = self.ka - 1 if kernel_size % 2 == 0 else self.ka
|
|
|
|
kernel_size = [kernel_size, kernel_size]
|
|
sigma = [sigma, sigma]
|
|
# The gaussian kernel is the product of the
|
|
# gaussian function of each dimension.
|
|
kernel = 1
|
|
meshgrids = torch.meshgrid(
|
|
[torch.arange(size, dtype=torch.float32) for size in kernel_size]
|
|
)
|
|
for size, std, mgrid in zip(kernel_size, sigma, meshgrids):
|
|
mean = (size - 1) / 2
|
|
kernel *= torch.exp(-((mgrid - mean) ** 2) / (2 * std**2))
|
|
|
|
# Make sure sum of values in gaussian kernel equals 1.
|
|
kernel = kernel / torch.sum(kernel)
|
|
# Reshape to depthwise convolutional weight
|
|
kernel = kernel.view(1, 1, *kernel.size())
|
|
kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1))
|
|
|
|
self.register_buffer("weight", kernel)
|
|
self.groups = channels
|
|
self.scale = scale
|
|
inv_scale = 1 / scale
|
|
self.int_inv_scale = int(inv_scale)
|
|
|
|
def forward(self, input):
|
|
if self.scale == 1.0:
|
|
return input
|
|
|
|
out = F.pad(input, (self.ka, self.kb, self.ka, self.kb))
|
|
out = F.conv2d(out, weight=self.weight, groups=self.groups)
|
|
out = out[:, :, :: self.int_inv_scale, :: self.int_inv_scale]
|
|
|
|
return out
|
|
|
|
|
|
class KPDetector(nn.Module):
|
|
"""
|
|
Detecting keypoints. Return keypoint positions.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
block_expansion=32,
|
|
num_kp=10,
|
|
num_channels=3,
|
|
max_features=1024,
|
|
num_blocks=5,
|
|
temperature=0.1,
|
|
scale_factor=0.25,
|
|
pad=0,
|
|
):
|
|
super(KPDetector, self).__init__()
|
|
|
|
self.predictor = Hourglass(
|
|
block_expansion,
|
|
in_features=num_channels,
|
|
max_features=max_features,
|
|
num_blocks=num_blocks,
|
|
)
|
|
|
|
self.kp = nn.Conv2d(
|
|
in_channels=self.predictor.out_filters,
|
|
out_channels=num_kp,
|
|
kernel_size=(7, 7),
|
|
padding=pad,
|
|
)
|
|
|
|
self.num_kp = num_kp
|
|
|
|
# We do not need the Jacobian (from FOMM).
|
|
# if estimate_jacobian:
|
|
# self.num_jacobian_maps = 1 if single_jacobian_map else num_kp
|
|
# self.jacobian = nn.Conv2d(in_channels=self.predictor.out_filters,
|
|
# out_channels=4 * self.num_jacobian_maps, kernel_size=(7, 7), padding=pad)
|
|
# self.jacobian.weight.data.zero_()
|
|
# self.jacobian.bias.data.copy_(torch.tensor([1, 0, 0, 1] * self.num_jacobian_maps, dtype=torch.float))
|
|
# else:
|
|
# self.jacobian = None
|
|
|
|
self.temperature = temperature
|
|
self.scale_factor = scale_factor
|
|
if self.scale_factor != 1:
|
|
self.down = AntiAliasInterpolation2d(num_channels, self.scale_factor)
|
|
|
|
def gaussian2kp(self, heatmap):
|
|
shape = heatmap.shape
|
|
heatmap = heatmap.unsqueeze(-1)
|
|
grid = (
|
|
make_coordinate_grid(shape[2:], heatmap.type()).unsqueeze_(0).unsqueeze_(0)
|
|
)
|
|
value = (heatmap * grid).sum(dim=(2, 3))
|
|
return value
|
|
|
|
def forward(self, x):
|
|
with torch.no_grad():
|
|
if self.scale_factor != 1:
|
|
x = self.down(x)
|
|
|
|
feature_map = self.predictor(x)
|
|
prediction = self.kp(feature_map)
|
|
|
|
final_shape = prediction.shape
|
|
heatmap = prediction.view(final_shape[0], final_shape[1], -1)
|
|
heatmap = F.softmax(heatmap / self.temperature, dim=2)
|
|
heatmap = heatmap.view(*final_shape)
|
|
|
|
out = self.gaussian2kp(heatmap)
|
|
heatmap = heatmap.unsqueeze(2)
|
|
|
|
# We do not need the Jacobian (from FOMM).
|
|
# if self.jacobian is not None:
|
|
# jacobian_map = self.jacobian(feature_map)
|
|
# jacobian_map = jacobian_map.reshape(final_shape[0], self.num_jacobian_maps, 4, final_shape[2],
|
|
# final_shape[3])
|
|
# jacobian = heatmap * jacobian_map
|
|
# jacobian = jacobian.view(final_shape[0], final_shape[1], 4, -1)
|
|
# jacobian = jacobian.sum(dim=-1)
|
|
# jacobian = jacobian.view(jacobian.shape[0], jacobian.shape[1], 2, 2)
|
|
|
|
return out, {"feature_map": feature_map, "heatmap": heatmap}
|