514 lines
14 KiB
Python
514 lines
14 KiB
Python
"""
|
|
@author: Fuou Marinas
|
|
@title: ComfyUI-StyleTransferPlus
|
|
@nickname: StyleTransferPlus
|
|
@description: A collection of style transfer nodes.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
import tqdm
|
|
from comfy.utils import ProgressBar
|
|
|
|
from .constants import (
|
|
CAST_DEFAULT,
|
|
CAST_NET_AE_PATH,
|
|
CAST_NET_DEC_B_PATH,
|
|
CAST_TYPES,
|
|
CAST_VGG_PATH,
|
|
EFDM_DEFAULT,
|
|
EFDM_PATH,
|
|
EFDM_STYLE_TYPES,
|
|
MICROAST_CONTENT_ENCODER_PATH,
|
|
MICROAST_DECODER_PATH,
|
|
MICROAST_MODULATOR_PATH,
|
|
MICROAST_STYLE_ENCODER_PATH,
|
|
UNIST_DEC_PATH,
|
|
UNIST_ENC_PATH,
|
|
UNIST_PATH,
|
|
)
|
|
from .module_cast import net as net_cast
|
|
from .module_cast.cast_model import inference_ucast, load_a_ckpt
|
|
from .module_efdm import net as net_efdm
|
|
from .module_efdm.efdm_model import inference_efdm
|
|
from .module_microast import net_microAST
|
|
from .module_microast.microast_model import inference_microast
|
|
from .module_neural_neighbor.neural_neighbor_model import inference_neural_neighbor
|
|
from .module_unist.model import video_Style_transfer
|
|
from .module_unist.unist_model import inference_unist
|
|
|
|
base_dir = Path(__file__).resolve().parent
|
|
|
|
|
|
class NeuralNeighbor:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"src_img": ("IMAGE",),
|
|
"style_img": ("IMAGE",),
|
|
"size": ([512, 1024], {"default": 512}),
|
|
"scale_long": (
|
|
"BOOLEAN",
|
|
{"default": True},
|
|
),
|
|
"flip": (
|
|
"BOOLEAN",
|
|
{"default": False},
|
|
),
|
|
"content_loss": (
|
|
"BOOLEAN",
|
|
{"default": False},
|
|
),
|
|
"colorize": (
|
|
"BOOLEAN",
|
|
{"default": True},
|
|
),
|
|
"content_weight": (
|
|
"FLOAT",
|
|
{"default": 0.75, "min": 0.01, "max": 1.00, "round": 0.01},
|
|
),
|
|
"max_iter": (
|
|
"INT",
|
|
{"default": 200, "min": 1, "step": 1},
|
|
),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
RETURN_NAMES = ("res_img",)
|
|
FUNCTION = "todo"
|
|
CATEGORY = "StyleTransferPlus"
|
|
|
|
def todo(
|
|
self,
|
|
src_img: torch.Tensor,
|
|
style_img: torch.Tensor,
|
|
size: int,
|
|
scale_long: bool,
|
|
flip: bool,
|
|
content_loss: bool,
|
|
colorize: bool,
|
|
content_weight: float,
|
|
max_iter: int,
|
|
):
|
|
print(f"{src_img.shape=}")
|
|
print(f"{style_img.shape=}")
|
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
params = {
|
|
"style_img": style_img,
|
|
"device": device,
|
|
"size": size,
|
|
"scale_long": scale_long,
|
|
"flip": flip,
|
|
"content_loss": content_loss,
|
|
"colorize": colorize,
|
|
"alpha": content_weight,
|
|
"max_iter": max_iter,
|
|
}
|
|
|
|
num_frames = src_img.size(0)
|
|
result: list[torch.Tensor] = []
|
|
with torch.inference_mode(mode=False):
|
|
for i in range(num_frames):
|
|
params["src_img"] = src_img[i].unsqueeze(0)
|
|
res_tensor = inference_neural_neighbor(**params)
|
|
result.append(res_tensor)
|
|
|
|
return (torch.cat(result, dim=0),)
|
|
|
|
|
|
class CAST:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"src_img": ("IMAGE",),
|
|
"style_img": ("IMAGE",),
|
|
"model_arch": (CAST_TYPES, {"default": CAST_DEFAULT}),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
RETURN_NAMES = ("res_img",)
|
|
FUNCTION = "todo"
|
|
CATEGORY = "StyleTransferPlus"
|
|
|
|
def todo(
|
|
self,
|
|
src_img: torch.Tensor,
|
|
style_img: torch.Tensor,
|
|
model_arch: str,
|
|
):
|
|
print(f"{src_img.shape=}")
|
|
print(f"{style_img.shape=}")
|
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
vgg = net_cast.vgg
|
|
vgg.load_state_dict(torch.load(f"{base_dir}/{CAST_VGG_PATH}"))
|
|
vgg = torch.nn.Sequential(*list(vgg.children())[:31])
|
|
|
|
netAE = net_cast.ADAIN_Encoder(vgg)
|
|
netDec_B = net_cast.Decoder()
|
|
|
|
netAE.load_state_dict(
|
|
load_a_ckpt(f"{base_dir}/models/{model_arch}_model/{CAST_NET_AE_PATH}")
|
|
)
|
|
netDec_B.load_state_dict(
|
|
load_a_ckpt(f"{base_dir}/models/{model_arch}_model/{CAST_NET_DEC_B_PATH}")
|
|
)
|
|
|
|
netAE = netAE.to(device).eval()
|
|
netDec_B = netDec_B.to(device).eval()
|
|
|
|
params = {
|
|
"style_img": style_img,
|
|
"device": device,
|
|
"netAE": netAE,
|
|
"netDec_B": netDec_B,
|
|
}
|
|
|
|
num_frames = src_img.size(0)
|
|
pbar = ProgressBar(num_frames)
|
|
|
|
result: list[torch.Tensor] = []
|
|
with torch.no_grad():
|
|
for i in range(num_frames):
|
|
params["src_img"] = src_img[i].unsqueeze(0)
|
|
res_tensor = inference_ucast(**params)
|
|
result.append(res_tensor.permute(0, 2, 3, 1))
|
|
pbar.update_absolute(i, num_frames)
|
|
|
|
return (torch.cat(result, dim=0),)
|
|
|
|
|
|
class EFDM:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"src_img": ("IMAGE",),
|
|
"style_img": ("IMAGE",),
|
|
"style_interp_weights": ("STRING", {"default": ""}),
|
|
"model_arch": (EFDM_STYLE_TYPES, {"default": EFDM_DEFAULT}),
|
|
"style_strength": (
|
|
"FLOAT",
|
|
{"default": 1.00, "min": 0.00, "max": 1.00, "round": 0.01},
|
|
),
|
|
"do_crop": (
|
|
"BOOLEAN",
|
|
{"default": False},
|
|
),
|
|
"preserve_color": (
|
|
"BOOLEAN",
|
|
{"default": False},
|
|
),
|
|
"size": (
|
|
"INT",
|
|
{"default": 512, "min": 1, "step": 1},
|
|
),
|
|
"use_cpu": (
|
|
"BOOLEAN",
|
|
{"default": False},
|
|
),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
RETURN_NAMES = ("res_img",)
|
|
FUNCTION = "todo"
|
|
CATEGORY = "StyleTransferPlus"
|
|
|
|
def todo(
|
|
self,
|
|
src_img: torch.Tensor,
|
|
style_img: torch.Tensor,
|
|
style_interp_weights: str,
|
|
model_arch: str,
|
|
style_strength: float,
|
|
do_crop: bool,
|
|
preserve_color: bool,
|
|
size: int,
|
|
use_cpu: bool,
|
|
):
|
|
print(f"{src_img.shape=}")
|
|
print(f"{style_img.shape=}")
|
|
device = torch.device(
|
|
"cuda" if torch.cuda.is_available() and not use_cpu else "cpu"
|
|
)
|
|
|
|
decoder = net_efdm.decoder
|
|
vgg = net_efdm.vgg
|
|
|
|
decoder.eval()
|
|
vgg.eval()
|
|
|
|
decoder.load_state_dict(torch.load(f"{base_dir}/{EFDM_PATH}"))
|
|
vgg.load_state_dict(torch.load(f"{base_dir}/{CAST_VGG_PATH}"))
|
|
|
|
vgg = torch.nn.Sequential(*list(vgg.children())[:31])
|
|
|
|
vgg.to(device)
|
|
decoder.to(device)
|
|
|
|
params = {
|
|
"style_img": style_img.permute(0, 3, 1, 2),
|
|
"device": device,
|
|
"decoder": decoder,
|
|
"vgg": vgg,
|
|
"alpha": style_strength,
|
|
"size": size,
|
|
"style_type": model_arch,
|
|
"do_crop": do_crop,
|
|
"preserve_color": preserve_color,
|
|
"style_interpolation_weights": deserialize_floats(style_interp_weights)
|
|
or None,
|
|
}
|
|
|
|
print(f"{params['style_interpolation_weights']=}")
|
|
|
|
num_frames = src_img.size(0)
|
|
pbar = ProgressBar(num_frames)
|
|
|
|
result: list[torch.Tensor] = []
|
|
with torch.no_grad():
|
|
for i in range(num_frames):
|
|
params["src_img"] = src_img[i].unsqueeze(0).permute(0, 3, 1, 2)
|
|
res_tensor = inference_efdm(**params)
|
|
result.append(res_tensor.permute(0, 2, 3, 1))
|
|
pbar.update_absolute(i, num_frames)
|
|
|
|
return (torch.cat(result, dim=0),)
|
|
|
|
|
|
class MicroAST:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"src_img": ("IMAGE",),
|
|
"style_img": ("IMAGE",),
|
|
"do_crop": (
|
|
"BOOLEAN",
|
|
{"default": False},
|
|
),
|
|
"size": (
|
|
"INT",
|
|
{"default": 512, "min": 1, "step": 1},
|
|
),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
RETURN_NAMES = ("res_img",)
|
|
FUNCTION = "todo"
|
|
CATEGORY = "StyleTransferPlus"
|
|
|
|
def todo(
|
|
self,
|
|
src_img: torch.Tensor,
|
|
style_img: torch.Tensor,
|
|
do_crop: bool,
|
|
size: int,
|
|
):
|
|
print(f"{src_img.shape=}")
|
|
print(f"{style_img.shape=}")
|
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
content_encoder = net_microAST.Encoder()
|
|
style_encoder = net_microAST.Encoder()
|
|
modulator = net_microAST.Modulator()
|
|
decoder = net_microAST.Decoder()
|
|
|
|
content_encoder.eval()
|
|
style_encoder.eval()
|
|
modulator.eval()
|
|
decoder.eval()
|
|
|
|
content_encoder.load_state_dict(
|
|
torch.load(f"{base_dir}/{MICROAST_CONTENT_ENCODER_PATH}")
|
|
)
|
|
style_encoder.load_state_dict(
|
|
torch.load(f"{base_dir}/{MICROAST_STYLE_ENCODER_PATH}")
|
|
)
|
|
modulator.load_state_dict(torch.load(f"{base_dir}/{MICROAST_MODULATOR_PATH}"))
|
|
decoder.load_state_dict(torch.load(f"{base_dir}/{MICROAST_DECODER_PATH}"))
|
|
|
|
network = net_microAST.TestNet(
|
|
content_encoder, style_encoder, modulator, decoder
|
|
)
|
|
network.to(device)
|
|
|
|
params = {
|
|
"style_img": style_img.permute(0, 3, 1, 2),
|
|
"device": device,
|
|
"alpha": 1.0,
|
|
"size": size,
|
|
"do_crop": do_crop,
|
|
"network": network,
|
|
}
|
|
|
|
num_frames = src_img.size(0)
|
|
pbar = ProgressBar(num_frames)
|
|
|
|
result: list[torch.Tensor] = []
|
|
with torch.no_grad():
|
|
for i in range(num_frames):
|
|
params["src_img"] = src_img[i].unsqueeze(0).permute(0, 3, 1, 2)
|
|
res_tensor = inference_microast(**params)
|
|
result.append(res_tensor.permute(0, 2, 3, 1))
|
|
pbar.update_absolute(i, num_frames)
|
|
|
|
return (torch.cat(result, dim=0),)
|
|
|
|
|
|
class UniST:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"src_img": ("IMAGE",),
|
|
"style_img": ("IMAGE",),
|
|
"do_crop": (
|
|
"BOOLEAN",
|
|
{"default": False},
|
|
),
|
|
"size": (
|
|
"INT",
|
|
{"default": 512, "min": 1, "step": 1},
|
|
),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
RETURN_NAMES = ("out_img",)
|
|
FUNCTION = "todo"
|
|
CATEGORY = "StyleTransferPlus"
|
|
|
|
def todo(
|
|
self,
|
|
src_img: torch.Tensor,
|
|
style_img: torch.Tensor,
|
|
do_crop: bool,
|
|
size: int,
|
|
):
|
|
print(f"{src_img.shape=}")
|
|
print(f"{style_img.shape=}")
|
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
network = video_Style_transfer(
|
|
ckpt_path=f"{base_dir}/{UNIST_PATH}",
|
|
encoder_path=f"{base_dir}/{UNIST_ENC_PATH}",
|
|
decoder_path=f"{base_dir}/{UNIST_DEC_PATH}",
|
|
)
|
|
|
|
network.eval()
|
|
network.to(device)
|
|
|
|
params = {
|
|
"src_style": style_img.permute(0, 3, 1, 2),
|
|
"device": device,
|
|
"size": size,
|
|
"do_crop": do_crop,
|
|
"network": network,
|
|
"content_type": "image",
|
|
}
|
|
|
|
num_frames = src_img.size(0)
|
|
pbar = ProgressBar(num_frames)
|
|
|
|
result: list[torch.Tensor] = []
|
|
with torch.no_grad():
|
|
for i in tqdm.tqdm(range(num_frames), desc="Generating"):
|
|
params["src"] = src_img[i].unsqueeze(0).permute(0, 3, 1, 2)
|
|
res_tensor = inference_unist(**params)
|
|
result.append(res_tensor.permute(0, 2, 3, 1))
|
|
pbar.update_absolute(i, num_frames)
|
|
|
|
return (torch.cat(result, dim=0),)
|
|
|
|
|
|
class UniST_Video:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"src_video": ("IMAGE",),
|
|
"style_video": ("IMAGE",),
|
|
"do_crop": (
|
|
"BOOLEAN",
|
|
{"default": False},
|
|
),
|
|
"size": (
|
|
"INT",
|
|
{"default": 512, "min": 1, "step": 1},
|
|
),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE",)
|
|
RETURN_NAMES = ("out_img",)
|
|
FUNCTION = "todo"
|
|
CATEGORY = "StyleTransferPlus"
|
|
|
|
def todo(
|
|
self,
|
|
src_video: torch.Tensor,
|
|
style_video: torch.Tensor,
|
|
do_crop: bool,
|
|
size: int,
|
|
):
|
|
print(f"{src_video.shape=}")
|
|
print(f"{style_video.shape=}")
|
|
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
|
|
network = video_Style_transfer(
|
|
ckpt_path=f"{base_dir}/{UNIST_PATH}",
|
|
encoder_path=f"{base_dir}/{UNIST_ENC_PATH}",
|
|
decoder_path=f"{base_dir}/{UNIST_DEC_PATH}",
|
|
)
|
|
|
|
network.eval()
|
|
network.to(device)
|
|
|
|
num_frames = src_video.size(0)
|
|
batch_size = 3
|
|
result = []
|
|
|
|
params = {
|
|
"src_style": style_video.permute(0, 3, 1, 2),
|
|
"device": device,
|
|
"size": size,
|
|
"do_crop": do_crop,
|
|
"network": network,
|
|
"content_type": "video",
|
|
}
|
|
|
|
pbar = ProgressBar(num_frames)
|
|
|
|
with torch.no_grad():
|
|
for i in tqdm.tqdm(range(0, num_frames, batch_size), "Generating"):
|
|
batch_end = min(i + batch_size, num_frames)
|
|
src_batch = src_video[i:batch_end].permute(0, 3, 1, 2)
|
|
|
|
params["src"] = src_batch
|
|
|
|
res_tensor = inference_unist(**params).permute(0, 2, 3, 1)
|
|
result.append(res_tensor)
|
|
|
|
pbar.update_absolute(batch_end, num_frames)
|
|
|
|
final_result = torch.cat(result, dim=0)
|
|
return (final_result,)
|
|
|
|
|
|
def serialize_floats(lst: list[float]):
|
|
return ",".join(map(str, lst))
|
|
|
|
|
|
def deserialize_floats(floats_str: str):
|
|
return [float(w) for w in floats_str.split(",") if w.strip()]
|