190 lines
6.1 KiB
Python
190 lines
6.1 KiB
Python
from dataclasses import dataclass
|
|
from functools import partial
|
|
import cv2
|
|
import pandas as pd
|
|
import gc
|
|
import math
|
|
import lpips
|
|
from PIL import Image, ImageOps
|
|
import requests
|
|
import torch
|
|
from torch import nn
|
|
from torch.nn import functional as F
|
|
import torchvision
|
|
import torchvision.transforms as T
|
|
import torchvision.transforms.functional as TF
|
|
from tqdm import tqdm
|
|
from resize_right import resize
|
|
from guided_diffusion.script_util import create_model_and_diffusion, model_and_diffusion_defaults
|
|
import numpy as np
|
|
from functools import partial
|
|
from numpy import asarray
|
|
|
|
|
|
def append_dims(x, n):
|
|
return x[(Ellipsis, *(None,) * (n - x.ndim))]
|
|
|
|
|
|
def expand_to_planes(x, shape):
|
|
return append_dims(x, len(shape)).repeat([1, 1, *shape[2:]])
|
|
|
|
|
|
def t_to_alpha_sigma(t):
|
|
return torch.cos(t * math.pi / 2), torch.sin(t * math.pi / 2)
|
|
|
|
|
|
@dataclass
|
|
class DiffusionOutput:
|
|
v: torch.Tensor
|
|
pred: torch.Tensor
|
|
eps: torch.Tensor
|
|
|
|
|
|
class ConvBlock(nn.Sequential):
|
|
def __init__(self, c_in, c_out):
|
|
super().__init__(
|
|
nn.Conv2d(c_in, c_out, 3, padding=1),
|
|
nn.ReLU(inplace=True),
|
|
)
|
|
|
|
|
|
class SkipBlock(nn.Module):
|
|
def __init__(self, main, skip=None):
|
|
super().__init__()
|
|
self.main = nn.Sequential(*main)
|
|
self.skip = skip if skip else nn.Identity()
|
|
|
|
def forward(self, input):
|
|
return torch.cat([self.main(input), self.skip(input)], dim=1)
|
|
|
|
|
|
class FourierFeatures(nn.Module):
|
|
def __init__(self, in_features, out_features, std=1.):
|
|
super().__init__()
|
|
assert out_features % 2 == 0
|
|
self.weight = nn.Parameter(torch.randn(
|
|
[out_features // 2, in_features]) * std)
|
|
|
|
def forward(self, input):
|
|
f = 2 * math.pi * input @ self.weight.T
|
|
return torch.cat([f.cos(), f.sin()], dim=-1)
|
|
|
|
|
|
class SecondaryDiffusionImageNet(nn.Module):
|
|
def __init__(self):
|
|
super().__init__()
|
|
c = 64 # The base channel count
|
|
|
|
self.timestep_embed = FourierFeatures(1, 16)
|
|
|
|
self.net = nn.Sequential(
|
|
ConvBlock(3 + 16, c),
|
|
ConvBlock(c, c),
|
|
SkipBlock([
|
|
nn.AvgPool2d(2),
|
|
ConvBlock(c, c * 2),
|
|
ConvBlock(c * 2, c * 2),
|
|
SkipBlock([
|
|
nn.AvgPool2d(2),
|
|
ConvBlock(c * 2, c * 4),
|
|
ConvBlock(c * 4, c * 4),
|
|
SkipBlock([
|
|
nn.AvgPool2d(2),
|
|
ConvBlock(c * 4, c * 8),
|
|
ConvBlock(c * 8, c * 4),
|
|
nn.Upsample(scale_factor=2, mode='bilinear',
|
|
align_corners=False),
|
|
]),
|
|
ConvBlock(c * 8, c * 4),
|
|
ConvBlock(c * 4, c * 2),
|
|
nn.Upsample(scale_factor=2, mode='bilinear',
|
|
align_corners=False),
|
|
]),
|
|
ConvBlock(c * 4, c * 2),
|
|
ConvBlock(c * 2, c),
|
|
nn.Upsample(scale_factor=2, mode='bilinear',
|
|
align_corners=False),
|
|
]),
|
|
ConvBlock(c * 2, c),
|
|
nn.Conv2d(c, 3, 3, padding=1),
|
|
)
|
|
|
|
def forward(self, input, t):
|
|
timestep_embed = expand_to_planes(
|
|
self.timestep_embed(t[:, None]), input.shape)
|
|
v = self.net(torch.cat([input, timestep_embed], dim=1))
|
|
alphas, sigmas = map(
|
|
partial(append_dims, n=v.ndim), t_to_alpha_sigma(t))
|
|
pred = input * alphas - v * sigmas
|
|
eps = input * sigmas + v * alphas
|
|
return DiffusionOutput(v, pred, eps)
|
|
|
|
|
|
class SecondaryDiffusionImageNet2(nn.Module):
|
|
def __init__(self):
|
|
super().__init__()
|
|
c = 64 # The base channel count
|
|
cs = [c, c * 2, c * 2, c * 4, c * 4, c * 8]
|
|
|
|
self.timestep_embed = FourierFeatures(1, 16)
|
|
self.down = nn.AvgPool2d(2)
|
|
self.up = nn.Upsample(
|
|
scale_factor=2, mode='bilinear', align_corners=False)
|
|
|
|
self.net = nn.Sequential(
|
|
ConvBlock(3 + 16, cs[0]),
|
|
ConvBlock(cs[0], cs[0]),
|
|
SkipBlock([
|
|
self.down,
|
|
ConvBlock(cs[0], cs[1]),
|
|
ConvBlock(cs[1], cs[1]),
|
|
SkipBlock([
|
|
self.down,
|
|
ConvBlock(cs[1], cs[2]),
|
|
ConvBlock(cs[2], cs[2]),
|
|
SkipBlock([
|
|
self.down,
|
|
ConvBlock(cs[2], cs[3]),
|
|
ConvBlock(cs[3], cs[3]),
|
|
SkipBlock([
|
|
self.down,
|
|
ConvBlock(cs[3], cs[4]),
|
|
ConvBlock(cs[4], cs[4]),
|
|
SkipBlock([
|
|
self.down,
|
|
ConvBlock(cs[4], cs[5]),
|
|
ConvBlock(cs[5], cs[5]),
|
|
ConvBlock(cs[5], cs[5]),
|
|
ConvBlock(cs[5], cs[4]),
|
|
self.up,
|
|
]),
|
|
ConvBlock(cs[4] * 2, cs[4]),
|
|
ConvBlock(cs[4], cs[3]),
|
|
self.up,
|
|
]),
|
|
ConvBlock(cs[3] * 2, cs[3]),
|
|
ConvBlock(cs[3], cs[2]),
|
|
self.up,
|
|
]),
|
|
ConvBlock(cs[2] * 2, cs[2]),
|
|
ConvBlock(cs[2], cs[1]),
|
|
self.up,
|
|
]),
|
|
ConvBlock(cs[1] * 2, cs[1]),
|
|
ConvBlock(cs[1], cs[0]),
|
|
self.up,
|
|
]),
|
|
ConvBlock(cs[0] * 2, cs[0]),
|
|
nn.Conv2d(cs[0], 3, 3, padding=1),
|
|
)
|
|
|
|
def forward(self, input, t):
|
|
timestep_embed = expand_to_planes(
|
|
self.timestep_embed(t[:, None]), input.shape)
|
|
v = self.net(torch.cat([input, timestep_embed], dim=1))
|
|
alphas, sigmas = map(
|
|
partial(append_dims, n=v.ndim), t_to_alpha_sigma(t))
|
|
pred = input * alphas - v * sigmas
|
|
eps = input * sigmas + v * alphas
|
|
return DiffusionOutput(v, pred, eps)
|