277 lines
10 KiB
Python
277 lines
10 KiB
Python
"""
|
|
This node is a simple conversion of this repository into ComfyUI ecosystem:
|
|
https://github.com/rrmina/fast-neural-style-pytorch.git
|
|
|
|
Some of the code is written by ChatGPT4-o
|
|
"""
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
from torchvision import models
|
|
import time
|
|
import os
|
|
import folder_paths
|
|
import subprocess as sp
|
|
import sys
|
|
|
|
|
|
|
|
|
|
class VGG16(nn.Module):
|
|
def __init__(self, vgg_path=os.path.join(folder_paths.base_path, "custom_nodes\\ComfyUI-Fast-Style-Transfer\\vgg\\vgg16-00b39a1b.pth"), train=False):
|
|
super(VGG16, self).__init__()
|
|
# Load VGG Skeleton, Pretrained Weights
|
|
vgg16_features = models.vgg16(pretrained=False)
|
|
vgg16_features.load_state_dict(torch.load(vgg_path), strict=False)
|
|
self.features = vgg16_features.features
|
|
|
|
# Turn-off Gradient History (on for testing train)
|
|
for param in self.features.parameters():
|
|
param.requires_grad = train
|
|
|
|
def forward(self, x):
|
|
layers = {'3': 'relu1_2', '8': 'relu2_2', '15': 'relu3_3', '22': 'relu4_3'}
|
|
features = {}
|
|
for name, layer in self.features._modules.items():
|
|
x = layer(x)
|
|
if name in layers:
|
|
features[layers[name]] = x
|
|
if (name=='22'):
|
|
break
|
|
|
|
return features
|
|
|
|
|
|
class ConvLayer(nn.Module):
|
|
def __init__(self, in_channels, out_channels, kernel_size, stride, norm="instance"):
|
|
super(ConvLayer, self).__init__()
|
|
# Padding Layers
|
|
self.padding_size = kernel_size // 2
|
|
self.reflection_pad = nn.ReflectionPad2d(self.padding_size)
|
|
|
|
# Convolution Layer
|
|
self.conv_layer = nn.Conv2d(in_channels, out_channels, kernel_size, stride)
|
|
|
|
# Normalization Layers
|
|
self.norm_type = norm
|
|
if (norm=="instance"):
|
|
self.norm_layer = nn.InstanceNorm2d(out_channels, affine=True)
|
|
elif (norm=="batch"):
|
|
self.norm_layer = nn.BatchNorm2d(out_channels, affine=True)
|
|
|
|
def forward(self, x):
|
|
x = self.reflection_pad(x)
|
|
x = self.conv_layer(x)
|
|
if self.norm_type == "None":
|
|
out = x
|
|
else:
|
|
out = self.norm_layer(x)
|
|
return out
|
|
|
|
|
|
class ResidualLayer(nn.Module):
|
|
"""
|
|
Deep Residual Learning for Image Recognition
|
|
|
|
https://arxiv.org/abs/1512.03385
|
|
"""
|
|
def __init__(self, channels=128, kernel_size=3):
|
|
super(ResidualLayer, self).__init__()
|
|
self.conv1 = ConvLayer(channels, channels, kernel_size, stride=1)
|
|
self.relu = nn.ReLU()
|
|
self.conv2 = ConvLayer(channels, channels, kernel_size, stride=1)
|
|
|
|
def forward(self, x):
|
|
identity = x # preserve residual
|
|
out = self.relu(self.conv1(x)) # 1st conv layer + activation
|
|
out = self.conv2(out) # 2nd conv layer
|
|
out = out + identity # add residual
|
|
return out
|
|
|
|
|
|
class DeconvLayer(nn.Module):
|
|
def __init__(self, in_channels, out_channels, kernel_size, stride, output_padding, norm="instance"):
|
|
super(DeconvLayer, self).__init__()
|
|
|
|
# Transposed Convolution
|
|
padding_size = kernel_size // 2
|
|
self.conv_transpose = nn.ConvTranspose2d(in_channels, out_channels, kernel_size, stride, padding_size, output_padding)
|
|
|
|
# Normalization Layers
|
|
self.norm_type = norm
|
|
if (norm=="instance"):
|
|
self.norm_layer = nn.InstanceNorm2d(out_channels, affine=True)
|
|
elif (norm=="batch"):
|
|
self.norm_layer = nn.BatchNorm2d(out_channels, affine=True)
|
|
|
|
def forward(self, x):
|
|
x = self.conv_transpose(x)
|
|
if (self.norm_type=="None"):
|
|
out = x
|
|
else:
|
|
out = self.norm_layer(x)
|
|
return out
|
|
|
|
|
|
class TransformerNetwork(nn.Module):
|
|
"""Feedforward Transformation Network without Tanh
|
|
reference: https://arxiv.org/abs/1603.08155
|
|
exact architecture: https://cs.stanford.edu/people/jcjohns/papers/fast-style/fast-style-supp.pdf
|
|
"""
|
|
def __init__(self):
|
|
super(TransformerNetwork, self).__init__()
|
|
self.ConvBlock = nn.Sequential(
|
|
ConvLayer(3, 32, 9, 1),
|
|
nn.ReLU(),
|
|
ConvLayer(32, 64, 3, 2),
|
|
nn.ReLU(),
|
|
ConvLayer(64, 128, 3, 2),
|
|
nn.ReLU()
|
|
)
|
|
self.ResidualBlock = nn.Sequential(
|
|
ResidualLayer(128, 3),
|
|
ResidualLayer(128, 3),
|
|
ResidualLayer(128, 3),
|
|
ResidualLayer(128, 3),
|
|
ResidualLayer(128, 3)
|
|
)
|
|
self.DeconvBlock = nn.Sequential(
|
|
DeconvLayer(128, 64, 3, 2, 1),
|
|
nn.ReLU(),
|
|
DeconvLayer(64, 32, 3, 2, 1),
|
|
nn.ReLU(),
|
|
ConvLayer(32, 3, 9, 1, norm="None")
|
|
)
|
|
|
|
def forward(self, x):
|
|
x = self.ConvBlock(x)
|
|
x = self.ResidualBlock(x)
|
|
out = self.DeconvBlock(x)
|
|
return out
|
|
|
|
|
|
class TrainFastStyleTransfer:
|
|
def __init__(self):
|
|
pass
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
input_dir = folder_paths.get_input_directory()
|
|
files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))]
|
|
return {
|
|
"required": {
|
|
"style_img": (sorted(files), {"image_upload": True}),
|
|
"seed": ("INT", {"default": 30, "min": 0, "max": 999999, "step": 1,}),
|
|
"content_weight": ("INT", {"default": 14, "min": 1, "max": 128, "step": 1,}),
|
|
"style_weight": ("INT", {"default": 50, "min": 1, "max": 128, "step": 1,}),
|
|
"batch_size": ("INT", {"default": 4, "min": 1, "max": 128, "step": 1,}),
|
|
"train_img_size": ("INT", {"default": 256, "min": 256, "max": 2048, "step": 1,}),
|
|
"learning_rate": ("FLOAT", {"default": 0.001, "min": 0.0001, "max": 0.1, "step": 0.0001}),
|
|
"num_epochs": ("INT", {"default": 1, "min": 1, "max": 20, "step": 1,}),
|
|
"save_model_every": ("INT", {"default": 500, "min": 100, "max": 10000, "step": 1,}),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ()
|
|
OUTPUT_NODE = True
|
|
|
|
FUNCTION = "train"
|
|
|
|
CATEGORY = "Style Transfer"
|
|
|
|
|
|
def train(self, style_img, seed, batch_size, train_img_size, learning_rate, num_epochs, content_weight, style_weight, save_model_every):
|
|
save_model_path = os.path.join(folder_paths.base_path, "custom_nodes\\ComfyUI-Fast-Style-Transfer\\models\\")
|
|
dataset_path = os.path.join(folder_paths.base_path, "custom_nodes\\ComfyUI-Fast-Style-Transfer\\dataset\\")
|
|
vgg_path = os.path.join(folder_paths.base_path, "custom_nodes\\ComfyUI-Fast-Style-Transfer\\vgg\\vgg16-00b39a1b.pth")
|
|
save_image_path = os.path.join(folder_paths.base_path, "custom_nodes\\ComfyUI-Fast-Style-Transfer\\output\\")
|
|
style_image_path = folder_paths.get_annotated_filepath(style_img)
|
|
train_path = os.path.join(folder_paths.base_path, "custom_nodes\\ComfyUI-Fast-Style-Transfer\\train.py")
|
|
|
|
|
|
command = [
|
|
sys.executable, train_path,
|
|
'--train_image_size', str(train_img_size),
|
|
'--dataset_path', dataset_path,
|
|
'--vgg_path', vgg_path,
|
|
'--num_epochs', str(num_epochs),
|
|
'--style_image_path', style_image_path,
|
|
'--batch_size', str(batch_size),
|
|
'--content_weight', str(content_weight),
|
|
'--style_weight', str(style_weight),
|
|
'--adam_lr', str(learning_rate),
|
|
'--save_model_path', save_model_path,
|
|
'--save_image_path', save_image_path,
|
|
'--save_model_every', str(save_model_every),
|
|
'--seed', str(seed)
|
|
]
|
|
|
|
sp.run(command)
|
|
return ()
|
|
|
|
|
|
|
|
|
|
|
|
class FastStyleTransfer:
|
|
def __init__(self):
|
|
pass
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"content_img": ("IMAGE",),
|
|
"model": ([file for file in os.listdir(os.path.join(folder_paths.base_path, "custom_nodes\\ComfyUI-Fast-Style-Transfer\\models\\")) if file.endswith('.pth')], ),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
|
|
FUNCTION = "styleTransfer"
|
|
|
|
CATEGORY = "Style Transfer"
|
|
|
|
|
|
def encode_tensor(self, tensor):
|
|
tensor = tensor.permute(0, 3, 1, 2).contiguous() # Convert to [batch_size, channels, height, width]
|
|
return tensor[:, [2, 1, 0], :, :] * 255
|
|
|
|
def decode_tensor(self, tensor):
|
|
tensor = tensor[:, [2, 1, 0], :, :]
|
|
tensor = tensor.permute(0, 2, 3, 1).contiguous() # Convert to [batch_size, height, width, channels]
|
|
return tensor / 255
|
|
|
|
def styleTransfer(self, content_img, model):
|
|
# Device
|
|
device = ("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
# Load Transformer Network
|
|
net = TransformerNetwork().to(device)
|
|
model_path = os.path.join(folder_paths.base_path, "custom_nodes\\ComfyUI-Fast-Style-Transfer\\models\\") + model
|
|
net.load_state_dict(torch.load(model_path, map_location=device))
|
|
net = net.to(device)
|
|
|
|
with torch.no_grad():
|
|
torch.cuda.empty_cache()
|
|
starttime = time.time()
|
|
content_tensor = self.encode_tensor(content_img)
|
|
generated_tensor = net(content_tensor.to(device))
|
|
print("Transfer Time: {}".format(time.time() - starttime))
|
|
image = self.decode_tensor(generated_tensor)
|
|
return (image,)
|
|
|
|
|
|
# A dictionary that contains all nodes you want to export with their names
|
|
# NOTE: names should be globally unique
|
|
NODE_CLASS_MAPPINGS = {
|
|
"FastStyleTransfer": FastStyleTransfer,
|
|
"TrainFastStyleTransfer": TrainFastStyleTransfer
|
|
}
|
|
|
|
# A dictionary that contains the friendly/humanly readable titles for the nodes
|
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
|
"FastStyleTransfer": "Fast Style Transfer",
|
|
"TrainFastStyleTransfer": "Train Fast Style Transfer"
|
|
}
|