import random import torch import torch.nn as nn import torch.nn.functional as F def init_weights(net): for m in net.modules(): if isinstance(m, nn.Conv2d): m.weight.data.normal_(0, 0.02) if m.bias is not None: m.bias.data.zero_() elif isinstance(m, nn.ConvTranspose2d): m.weight.data.normal_(0, 0.02) if m.bias is not None: m.bias.data.zero_() elif isinstance(m, nn.Linear): m.weight.data.normal_(0, 0.02) if m.bias is not None: m.bias.data.zero_() def feature_wct_simple(content_feat, style_feat, alpha=1): target_feature = Bw_wct_core(content_feat, style_feat) target_feature = target_feature.view_as(content_feat) target_feature = alpha * target_feature + (1 - alpha) * content_feat return target_feature def adaptive_get_keys(feat_skips, start_layer_idx, last_layer_idx, target_feat): B, C, th, tw = target_feat.shape results = [] target_conv_layer = "conv" + str(last_layer_idx) + "_1" _, _, h, w = feat_skips[target_conv_layer].shape for i in range(start_layer_idx, last_layer_idx + 1): target_conv_layer = "conv" + str(i) + "_1" if i == last_layer_idx: results.append(mean_variance_norm(feat_skips[target_conv_layer])) else: results.append( mean_variance_norm(F.interpolate(feat_skips[target_conv_layer], (h, w))) ) return F.interpolate(torch.cat(results, dim=1), (th, tw)) def adaptive_gram_weight(image, level, ratio, encoder): if level == 0: encoded_features = image else: encoded_features = encoder.get_features(image, level) # B x C x W x H global_gram = gram_matrix(encoded_features) B, C, w, h = encoded_features.size() target_w, target_h = w // ratio, h // ratio patches = extract_image_patches(encoded_features, target_w, target_h) _, patches_num, _, _, _ = patches.size() cos = torch.nn.CosineSimilarity(eps=1e-6) intra_gram_statistic = [] inter_gram_statistic = [] comb = torch.combinations(torch.arange(patches_num), r=2) if patches_num >= 10: sampling_num = int(comb.size(0) * 0.05) else: sampling_num = comb.size(0) for idx in range(B): if patches_num < 2: continue cos_gram = [] for patch in range(0, patches_num): cos_gram.append( cos(global_gram, gram_matrix(patches[idx][patch].unsqueeze(0))) .mean() .item() ) intra_gram_statistic.append(torch.tensor(cos_gram)) cos_gram = [] for idxes in random.choices(list(comb), k=sampling_num): cos_gram.append( cos( gram_matrix(patches[idx][idxes[0]].unsqueeze(0)), gram_matrix(patches[idx][idxes[1]].unsqueeze(0)), ) .mean() .item() ) inter_gram_statistic.append(torch.tensor(cos_gram)) intra_gram_statistic = torch.stack(intra_gram_statistic).mean(dim=1) inter_gram_statistic = torch.stack(inter_gram_statistic).mean(dim=1) results = (intra_gram_statistic + inter_gram_statistic) / 2 ##For boosting value results = 1 / (1 + torch.exp(-10 * (results - 0.6))) return results def Bw_wct_core(content_feat, style_feat, weight=1, registers=None, device="cpu"): N, C, H, W = content_feat.size() cont_min = content_feat.min().item() cont_max = content_feat.max().item() whiten_cF, _, _ = SwitchWhiten2d(content_feat) _, wm_s, s_mean = SwitchWhiten2d(style_feat) targetFeature = torch.bmm(torch.inverse(wm_s), whiten_cF) targetFeature = targetFeature.view(N, C, H, W) targetFeature = targetFeature + s_mean.unsqueeze(2).expand_as(targetFeature) targetFeature.clamp_(cont_min, cont_max) return targetFeature def SwitchWhiten2d(x): N, C, H, W = x.size() in_data = x.view(N, C, -1) eye = in_data.data.new().resize_(C, C) eye = torch.nn.init.eye_(eye).view(1, C, C).expand(N, C, C) # calculate other statistics mean_in = in_data.mean(-1, keepdim=True) x_in = in_data - mean_in # (N x g) x C x C cov_in = torch.bmm(x_in, torch.transpose(x_in, 1, 2)).div(H * W) mean = mean_in cov = cov_in + 1e-5 * eye # perform whitening using Newton's iteration Ng, c, _ = cov.size() P = torch.eye(c).to(cov).expand(Ng, c, c) rTr = (cov * P).sum((1, 2), keepdim=True).reciprocal_() cov_N = cov * rTr for k in range(5): # P = torch.baddbmm(1.5, P, -0.5, torch.matrix_power(P, 3), cov_N) P = torch.baddbmm(P, torch.matrix_power(P, 3), cov_N, beta=1.5, alpha=-0.5) wm = P.mul_(rTr.sqrt()) x_hat = torch.bmm(wm, in_data - mean) return x_hat, wm, mean def mean_variance_norm(feat): size = feat.size() mean, std = calc_mean_std(feat) normalized_feat = (feat - mean.expand(size)) / std.expand(size) return normalized_feat def calc_mean_std(feat, eps=1e-5): # eps is a small value added to the variance to avoid divide-by-zero. size = feat.size() assert len(size) == 4 N, C = size[:2] feat_var = feat.view(N, C, -1).var(dim=2) + eps feat_std = feat_var.sqrt().view(N, C, 1, 1) feat_mean = feat.view(N, C, -1).mean(dim=2).view(N, C, 1, 1) return feat_mean, feat_std def gram_matrix(y): (b, ch, h, w) = y.size() features = y.view(b, ch, w * h) features_t = features.transpose(1, 2) gram = features.bmm(features_t) / (ch * h * w) return gram def extract_image_patches(x, kernel, stride=1): b, c, h, w = x.shape # Extract patches patches = x.unfold(2, kernel, stride).unfold(3, kernel, stride) patches = patches.contiguous().view(b, c, -1, kernel, kernel) patches = patches.permute(0, 2, 1, 3, 4).contiguous() # return patches.view(b, number_of_patches, c, h, w) return patches.view(b, -1, c, kernel, kernel)