import sys sys.path.insert(0, "..") import os import struct import cv2 import numpy as np import torch.utils.data as data from PIL import Image cv2.setNumThreads(0) import pdb import random import matplotlib.pyplot as plt import torch import torchvision import torchvision.transforms as transforms from PIL import Image from skimage import color from utils.flowlib import read_flow from utils.util_distortion import CenterPad, CenterPadCrop_numpy from utils.warping import WarpingLayer import lib.functional as F class RGB2Lab(object): def __init__(self): pass def __call__(self, inputs): image_lab = color.rgb2lab(inputs) return image_lab class Normalize(object): def __init__(self): pass def __call__(self, inputs): inputs[0:1, :, :] = F.normalize(inputs[0:1, :, :], 50, 1) inputs[1:3, :, :] = F.normalize(inputs[1:3, :, :], (0, 0), (1, 1)) return inputs class ToTensor(object): def __init__(self): pass def __call__(self, inputs): outputs = F.to_mytensor(inputs) # permute channel and transform to tensor return outputs class RandomErasing(object): def __init__(self, probability=0.6, sl=0.05, sh=0.6): self.probability = probability self.sl = sl self.sh = sh def __call__(self, img): img = np.array(img) if random.uniform(0, 1) > self.probability: return Image.fromarray(img) area = img.shape[0] * img.shape[1] h0 = img.shape[0] w0 = img.shape[1] channel = img.shape[2] h = int(round(random.uniform(self.sl, self.sh) * h0)) w = int(round(random.uniform(self.sl, self.sh) * w0)) if w < img.shape[1] and h < img.shape[0]: x1 = random.randint(0, img.shape[0] - h) y1 = random.randint(0, img.shape[1] - w) img[x1 : x1 + h, y1 : y1 + w, :] = np.random.rand(h, w, channel) * 255 return Image.fromarray(img) return Image.fromarray(img) class CenterCrop(object): """ center crop the numpy array """ def __init__(self, image_size): # target size self.h0, self.w0 = image_size def __call__(self, input_numpy): if input_numpy.ndim == 3: h, w, channel = input_numpy.shape output_numpy = np.zeros((self.h0, self.w0, channel)) output_numpy = input_numpy[ (h - self.h0) // 2 : (h - self.h0) // 2 + self.h0, (w - self.w0) // 2 : (w - self.w0) // 2 + self.w0, : ] else: h, w = input_numpy.shape output_numpy = np.zeros((self.h0, self.w0)) output_numpy = input_numpy[ (h - self.h0) // 2 : (h - self.h0) // 2 + self.h0, (w - self.w0) // 2 : (w - self.w0) // 2 + self.w0 ] return output_numpy def parse_images(data_root): image_pairs = [] subdirs = sorted(os.listdir(data_root)) for subdir in subdirs: path = os.path.join(data_root, subdir) if not os.path.isdir(path): continue # parse_file = os.path.join(path, 'pairs_output.txt') parse_file = os.path.join(path, "pairs_output_new.txt") if os.path.exists(parse_file): f = open(parse_file, "r") lines = f.readlines() for line in lines: line = line.replace("\n", "") ( image1_name, image2_name, reference_video_name, reference_video_name1, reference_name1, reference_name2, reference_name3, reference_name4, reference_name5, reference_gt1, reference_gt2, reference_gt3, ) = line.split() image1_name = image1_name.split(".")[0] image2_name = image2_name.split(".")[0] reference_video_name = reference_video_name.split(".")[0] reference_video_name1 = reference_video_name1.split(".")[0] reference_name1 = reference_name1.split(".")[0] reference_name2 = reference_name2.split(".")[0] reference_name3 = reference_name3.split(".")[0] reference_name4 = reference_name4.split(".")[0] reference_name5 = reference_name5.split(".")[0] reference_gt1 = reference_gt1.split(".")[0] reference_gt2 = reference_gt2.split(".")[0] reference_gt3 = reference_gt3.split(".")[0] flow_forward_name = image1_name + "_forward" flow_backward_name = image1_name + "_backward" mask_name = image1_name + "_mask" item = ( image1_name + ".jpg", image2_name + ".jpg", reference_video_name + ".jpg", reference_name1 + ".JPEG", reference_name2 + ".JPEG", reference_name3 + ".JPEG", reference_name4 + ".JPEG", reference_name5 + ".JPEG", flow_forward_name + ".flo", flow_backward_name + ".flo", mask_name + ".pgm", reference_gt1 + ".jpg", reference_gt2 + ".jpg", reference_gt3 + ".jpg", path, ) image_pairs.append(item) f.close() else: raise (RuntimeError("Error when parsing pair_output_count.txt in subfolders of: " + path + "\n")) return image_pairs class VideosDataset(torch.utils.data.Dataset): def __init__( self, data_root, epoch, image_size, image_transform=None, use_google_reference=False, real_reference_probability=1, nonzero_placeholder_probability=0.5, ): self.data_root = data_root self.image_transform = image_transform self.CenterPad = CenterPad(image_size) self.ToTensor = ToTensor() self.CenterCrop = CenterCrop(image_size) assert len(self.data_root) > 0, "find no dataroot" self.epoch = epoch self.image_pairs = parse_images(self.data_root) self.real_len = len(self.image_pairs) print("##### parsing image pairs in %s: %d pairs #####" % (data_root, self.real_len)) self.image_pairs *= epoch self.use_google_reference = use_google_reference self.real_reference_probability = real_reference_probability self.nonzero_placeholder_probability = nonzero_placeholder_probability def __getitem__(self, index): ( image1_name, image2_name, reference_video_name, reference_name1, reference_name2, reference_name3, reference_name4, reference_name5, flow_forward_name, flow_backward_name, mask_name, reference_gt1, reference_gt2, reference_gt3, path, ) = self.image_pairs[index] try: # print('video loader') I1 = Image.open(os.path.join(path, "input_pad", image1_name)) I2 = Image.open(os.path.join(path, "input_pad", image2_name)) # randomly choose a reference frame from the video I_reference_video = Image.open( os.path.join(path, "reference_gt", random.choice([reference_gt1, reference_gt2, reference_gt3])) ) I_reference_video_real = Image.open( os.path.join( path, "reference", random.choice( [reference_name1, reference_name2, reference_name3, reference_name4, reference_name5] ), ) ) flow_forward = read_flow(os.path.join(path, "flow", flow_forward_name)) # numpy flow_backward = read_flow(os.path.join(path, "flow", flow_backward_name)) # numpy mask = Image.open(os.path.join(path, "mask", mask_name)) # binary mask mask = np.array(mask) mask[mask < 240] = 0 mask[mask >= 240] = 1 # transform I1 = self.image_transform(I1) I2 = self.image_transform(I2) I_reference_video = self.image_transform(self.CenterPad(I_reference_video)) I_reference_video_real = self.image_transform(self.CenterPad(I_reference_video_real)) flow_forward = self.ToTensor(self.CenterCrop(flow_forward)) flow_backward = self.ToTensor(self.CenterCrop(flow_backward)) mask = self.ToTensor(self.CenterCrop(mask)) if np.random.random() < self.real_reference_probability: I_reference_output = I_reference_video_real placeholder = torch.zeros_like(I1) self_ref_flag = torch.zeros_like(I1) else: I_reference_output = I_reference_video placeholder = I2 if np.random.random() < self.nonzero_placeholder_probability else torch.zeros_like(I1) self_ref_flag = torch.ones_like(I1) outputs = [] outputs.append(I1) outputs.append(I2) outputs.append(I_reference_output) # outputs.append(I_reference_video_real) outputs.append(flow_forward) outputs.append(flow_backward) outputs.append(mask) outputs.append(placeholder) outputs.append(self_ref_flag) except Exception as e: print("problem in, ", path) print(e) return self.__getitem__(np.random.randint(0, len(self.image_pairs))) return outputs def __len__(self): return len(self.image_pairs) import torchvision.utils as vutils from torch.autograd import Variable def batch_lab2rgb_transpose_mc(img_l_mc, img_ab_mc, nrow=8): if isinstance(img_l_mc, Variable): img_l_mc = img_l_mc.data.cpu() if isinstance(img_ab_mc, Variable): img_ab_mc = img_ab_mc.data.cpu() if img_l_mc.is_cuda: img_l_mc = img_l_mc.cpu() if img_ab_mc.is_cuda: img_ab_mc = img_ab_mc.cpu() assert img_l_mc.dim() == 4 and img_ab_mc.dim() == 4, "only for batch input" l_norm, ab_norm = 1.0, 1.0 l_mean, ab_mean = 50.0, 0 img_l = img_l_mc * l_norm + l_mean img_ab = img_ab_mc * ab_norm + ab_mean pred_lab = torch.cat((img_l, img_ab), dim=1) grid_lab = vutils.make_grid(pred_lab, nrow=nrow).numpy().astype("float64") grid_rgb = (np.clip(color.lab2rgb(grid_lab.transpose((1, 2, 0))), 0, 1) * 255).astype("uint8") return grid_rgb # if __name__ == "__main__": def test_videoloader(): # dataset initI1lization # image size = 256 train_dataset = VideosDataset(data_root="../../Dataset/video/video_pair", epoch=10) data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=1, shuffle=True, num_workers=0) iter_instance = iter(data_loader) sample = iter_instance.next() I1_lab, I2_lab, I_reference_lab, flow_forward, flow_backward, mask = sample # l: [-50,50], ab: [-128, 128] l_norm, ab_norm = 1.0, 1.0 l_mean, ab_mean = 50.0, 0 I1_l = I1_lab[:, 0:1, :, :] I1_ab = I1_lab[:, 1:3, :, :] I2_l = I2_lab[:, 0:1, :, :] I2_ab = I2_lab[:, 1:3, :, :] I_reference_l = I_reference_lab[:, 0:1, :, :] I_reference_ab = I_reference_lab[:, 1:3, :, :] # test image samples image_A_grid = batch_lab2rgb_transpose_mc(I1_l, I1_ab, nrow=4).astype(np.uint8) image_B_grid = batch_lab2rgb_transpose_mc(I2_l, I2_ab, nrow=4).astype(np.uint8) image_ref_grid = batch_lab2rgb_transpose_mc(I_reference_l, I_reference_ab, nrow=4).astype(np.uint8) plt.imsave("A.png", image_A_grid) plt.imsave("B.png", image_B_grid) plt.figure() plt.imshow(image_A_grid) plt.title("image A") plt.figure() plt.imshow(image_B_grid) plt.title("image B") plt.figure() plt.imshow(image_ref_grid) plt.title("image ref") # plt.show() # test warping print(I2_lab.shape) print(flow_forward.shape) warping_layer = WarpingLayer("gpu") I21_lab = warping_layer(I2_lab.cuda(), flow_forward.cuda()) I21_lab = I21_lab * mask.cuda() I21_l = I21_lab[:, 0:1, :, :] I21_ab = I21_lab[:, 1:3, :, :] image_warp_grid = batch_lab2rgb_transpose_mc(I21_l, I21_ab, nrow=4).astype(np.uint8) plt.imsave("BA.png", image_warp_grid) plt.figure() plt.imshow(image_warp_grid) plt.title("image warp") # plt.show() # image difference plt.figure() plt.imshow(np.abs(image_warp_grid - image_A_grid).astype(np.uint8)) plt.title("warp-A") plt.figure() plt.imshow(np.abs(image_B_grid - image_A_grid).astype(np.uint8)) plt.title("B-A") # plt.show() plt.imsave("difference_warp.png", np.abs(image_warp_grid - image_A_grid).astype(np.uint8)) plt.imsave("difference_BA.png", np.abs(image_B_grid - image_A_grid).astype(np.uint8)) return None