import math import torch import torch.nn as nn from einops import rearrange class PositionalEncoding(nn.Module): def __init__(self, num_octaves, start_octave): super().__init__() self.num_octaves = num_octaves self.start_octave = start_octave def forward(self, coords): embed_fns = [] batch_size, num_points, dim = coords.shape octaves = torch.arange(self.start_octave, self.start_octave + self.num_octaves) octaves = octaves.float().to(coords) multipliers = 2**octaves * math.pi coords = coords.unsqueeze(-1) while len(multipliers.shape) < len(coords.shape): multipliers = multipliers.unsqueeze(0) scaled_coords = coords * multipliers sines = torch.sin(scaled_coords).reshape( batch_size, num_points, dim * self.num_octaves ) cosines = torch.cos(scaled_coords).reshape( batch_size, num_points, dim * self.num_octaves ) result = torch.cat((sines, cosines), -1) return result class FSRTPosEncoder(nn.Module): def __init__( self, kp_octaves=4, kp_start_octave=-1, pix_start_octave=-1, pix_octaves=16 ): super().__init__() self.kp_encoding = PositionalEncoding( num_octaves=kp_octaves, start_octave=kp_start_octave ) self.pix_encoding = PositionalEncoding( num_octaves=pix_octaves, start_octave=pix_start_octave ) def forward(self, pixels, kps=None): if len(pixels.shape) == 4: batchsize, height, width, _ = pixels.shape pixels = pixels.flatten(1, 2) pix_enc = self.pix_encoding(pixels) pix_enc = pix_enc.view(batchsize, height, width, pix_enc.shape[-1]) pix_enc = pix_enc.permute((0, 3, 1, 2)) if kps is not None: kp_enc = self.kp_encoding(kps.unsqueeze(1)) kp_enc = kp_enc.view(batchsize, kp_enc.shape[-1], 1, 1).repeat( 1, 1, height, width ) x = torch.cat((kp_enc, pix_enc), 1) else: pix_enc = self.pix_encoding(pixels) if kps is not None: kp_enc = self.kp_encoding(kps) x = torch.cat((kp_enc, pix_enc), -1) return x # Transformer implementation based on ViT # https://github.com/lucidrains/vit-pytorch/blob/main/vit_pytorch/vit.py class PreNorm(nn.Module): def __init__(self, dim, fn): super().__init__() self.norm = nn.LayerNorm(dim) self.fn = fn def forward(self, x, **kwargs): return self.fn(self.norm(x), **kwargs) class FeedForward(nn.Module): def __init__(self, dim, hidden_dim, dropout=0.0): super().__init__() self.net = nn.Sequential( nn.Linear(dim, hidden_dim), nn.GELU(), nn.Dropout(dropout), nn.Linear(hidden_dim, dim), nn.Dropout(dropout), ) def forward(self, x): return self.net(x) class Attention(nn.Module): def __init__( self, dim, heads=8, dim_head=64, dropout=0.0, selfatt=True, kv_dim=None ): super().__init__() inner_dim = dim_head * heads project_out = not (heads == 1 and dim_head == dim) self.heads = heads self.scale = dim_head**-0.5 self.attend = nn.Softmax(dim=-1) if selfatt: self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) else: self.to_q = nn.Linear(dim, inner_dim, bias=False) self.to_kv = nn.Linear(kv_dim, inner_dim * 2, bias=False) self.to_out = ( nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout)) if project_out else nn.Identity() ) def forward(self, x, z=None): if z is None: qkv = self.to_qkv(x).chunk(3, dim=-1) else: q = self.to_q(x) k, v = self.to_kv(z).chunk(2, dim=-1) qkv = (q, k, v) q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), qkv) dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale attn = self.attend(dots) out = torch.matmul(attn, v) out = rearrange(out, "b h n d -> b n (h d)") return self.to_out(out) class Transformer(nn.Module): def __init__( self, dim, depth, heads, dim_head, mlp_dim, dropout=0.0, selfatt=True, kv_dim=None, ): super().__init__() self.layers = nn.ModuleList([]) for _ in range(depth): self.layers.append( nn.ModuleList( [ PreNorm( dim, Attention( dim, heads=heads, dim_head=dim_head, dropout=dropout, selfatt=selfatt, kv_dim=kv_dim, ), ), PreNorm(dim, FeedForward(dim, mlp_dim, dropout=dropout)), ] ) ) def forward(self, x, z=None): for attn, ff in self.layers: x = attn(x, z=z) + x x = ff(x) + x return x