255 lines
9.3 KiB
Python
255 lines
9.3 KiB
Python
import os
|
|
import random
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from pycocotools import mask
|
|
|
|
from model.segment_anything.utils.transforms import ResizeLongestSide
|
|
|
|
from .grefer import G_REFER
|
|
from .refer import REFER
|
|
from torchvision import transforms
|
|
|
|
|
|
class ReferSegDataset(torch.utils.data.Dataset):
|
|
pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
|
|
pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
|
|
img_size = 1024
|
|
ignore_label = 255
|
|
|
|
def __init__(
|
|
self,
|
|
base_image_dir,
|
|
tokenizer,
|
|
samples_per_epoch=500 * 8 * 2 * 10,
|
|
precision: str = "fp32",
|
|
image_size: int = 224,
|
|
num_classes_per_sample: int = 3,
|
|
exclude_val=False,
|
|
refer_seg_data="refclef||refcoco||refcoco+||refcocog",
|
|
model_type="ori",
|
|
transform=ResizeLongestSide(1024),
|
|
):
|
|
self.model_type = model_type
|
|
self.exclude_val = exclude_val
|
|
self.samples_per_epoch = samples_per_epoch
|
|
self.num_classes_per_sample = num_classes_per_sample
|
|
|
|
self.base_image_dir = base_image_dir
|
|
self.tokenizer = tokenizer
|
|
self.precision = precision
|
|
self.transform = transform
|
|
self.image_preprocessor = transforms.Compose([
|
|
transforms.ToTensor(),
|
|
transforms.Resize((image_size, image_size), interpolation=3),
|
|
transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
|
|
])
|
|
|
|
DATA_DIR = os.path.join(base_image_dir, "refer_seg")
|
|
self.refer_seg_ds_list = refer_seg_data.split(
|
|
"||"
|
|
) # ['refclef', 'refcoco', 'refcoco+', 'refcocog']
|
|
self.refer_seg_data = {}
|
|
for ds in self.refer_seg_ds_list:
|
|
if ds == "refcocog":
|
|
splitBy = "umd"
|
|
else:
|
|
splitBy = "unc"
|
|
|
|
if ds == "grefcoco":
|
|
refer_api = G_REFER(DATA_DIR, ds, splitBy)
|
|
else:
|
|
refer_api = REFER(DATA_DIR, ds, splitBy)
|
|
|
|
ref_ids_train = refer_api.getRefIds(split="train")
|
|
images_ids_train = refer_api.getImgIds(ref_ids=ref_ids_train)
|
|
refs_train = refer_api.loadRefs(ref_ids=ref_ids_train)
|
|
|
|
refer_seg_ds = {}
|
|
refer_seg_ds["images"] = []
|
|
loaded_images = refer_api.loadImgs(image_ids=images_ids_train)
|
|
|
|
for item in loaded_images:
|
|
item = item.copy()
|
|
if ds == "refclef":
|
|
item["file_name"] = os.path.join(
|
|
DATA_DIR, "images/saiapr_tc-12", item["file_name"]
|
|
)
|
|
else:
|
|
item["file_name"] = os.path.join(
|
|
DATA_DIR, "images/mscoco/images/train2014", item["file_name"]
|
|
)
|
|
refer_seg_ds["images"].append(item)
|
|
refer_seg_ds["annotations"] = refer_api.Anns # anns_train
|
|
|
|
print(
|
|
"dataset {} (refs {}) (train split) has {} images and {} annotations.".format(
|
|
ds,
|
|
splitBy,
|
|
len(refer_seg_ds["images"]),
|
|
len(refer_seg_ds["annotations"]),
|
|
)
|
|
)
|
|
|
|
img2refs = {}
|
|
for ref in refs_train:
|
|
image_id = ref["image_id"]
|
|
img2refs[image_id] = img2refs.get(image_id, []) + [
|
|
ref,
|
|
]
|
|
refer_seg_ds["img2refs"] = img2refs
|
|
self.refer_seg_data[ds] = refer_seg_ds
|
|
|
|
def __len__(self):
|
|
return self.samples_per_epoch
|
|
|
|
def preprocess(self, x: torch.Tensor) -> torch.Tensor:
|
|
"""Normalize pixel values and pad to a square input."""
|
|
if self.model_type=="hq":
|
|
h, w = x.shape[-2:]
|
|
padh = self.img_size - h
|
|
padw = self.img_size - w
|
|
x = F.pad(x, (0, padw, 0, padh), value=128)
|
|
|
|
# Normalize colors
|
|
x = (x - self.pixel_mean) / self.pixel_std
|
|
|
|
if self.model_type=="effi" or self.model_type=="sam2":
|
|
x = F.interpolate(x.unsqueeze(0), (self.img_size, self.img_size), mode="bilinear").squeeze(0)
|
|
else:
|
|
# Pad
|
|
h, w = x.shape[-2:]
|
|
padh = self.img_size - h
|
|
padw = self.img_size - w
|
|
x = F.pad(x, (0, padw, 0, padh))
|
|
return x
|
|
|
|
def __getitem__(self, idx):
|
|
ds = random.randint(0, len(self.refer_seg_ds_list) - 1)
|
|
ds = self.refer_seg_ds_list[ds]
|
|
refer_seg_ds = self.refer_seg_data[ds]
|
|
images = refer_seg_ds["images"]
|
|
annotations = refer_seg_ds["annotations"]
|
|
img2refs = refer_seg_ds["img2refs"]
|
|
idx = random.randint(0, len(images) - 1)
|
|
image_info = images[idx]
|
|
image_path = image_info["file_name"]
|
|
image_id = image_info["id"]
|
|
refs = img2refs[image_id]
|
|
if len(refs) == 0:
|
|
return self.__getitem__(0)
|
|
|
|
sents = []
|
|
ann_ids = []
|
|
for ref in refs:
|
|
for sent in ref["sentences"]:
|
|
text = sent["sent"]
|
|
sents.append(text)
|
|
ann_ids.append(ref["ann_id"])
|
|
if len(sents) >= self.num_classes_per_sample:
|
|
sampled_inds = np.random.choice(
|
|
list(range(len(sents))), size=self.num_classes_per_sample, replace=False
|
|
)
|
|
else:
|
|
sampled_inds = list(range(len(sents)))
|
|
sampled_sents = np.vectorize(sents.__getitem__)(sampled_inds).tolist()
|
|
# sampled_ann_ids = np.vectorize(ann_ids.__getitem__)(sampled_inds).tolist()
|
|
sampled_ann_ids = [ann_ids[ind] for ind in sampled_inds]
|
|
sampled_classes = sampled_sents
|
|
image = cv2.imread(image_path)
|
|
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
|
|
|
|
# preprocess image for evf
|
|
image_evf = self.image_preprocessor(image)
|
|
|
|
image = self.transform.apply_image(image) # preprocess image for sam
|
|
resize = image.shape[:2]
|
|
|
|
image = self.preprocess(torch.from_numpy(image).permute(2, 0, 1).contiguous())
|
|
|
|
flag = False
|
|
masks = []
|
|
for ann_id in sampled_ann_ids:
|
|
if isinstance(ann_id, list):
|
|
flag = True
|
|
if -1 in ann_id:
|
|
assert len(ann_id) == 1
|
|
m = np.zeros((image_info["height"], image_info["width"])).astype(
|
|
np.uint8
|
|
)
|
|
else:
|
|
m_final = np.zeros(
|
|
(image_info["height"], image_info["width"])
|
|
).astype(np.uint8)
|
|
for ann_id_i in ann_id:
|
|
ann = annotations[ann_id_i]
|
|
|
|
if len(ann["segmentation"]) == 0:
|
|
m = np.zeros(
|
|
(image_info["height"], image_info["width"])
|
|
).astype(np.uint8)
|
|
else:
|
|
if type(ann["segmentation"][0]) == list: # polygon
|
|
rle = mask.frPyObjects(
|
|
ann["segmentation"],
|
|
image_info["height"],
|
|
image_info["width"],
|
|
)
|
|
else:
|
|
rle = ann["segmentation"]
|
|
for i in range(len(rle)):
|
|
if not isinstance(rle[i]["counts"], bytes):
|
|
rle[i]["counts"] = rle[i]["counts"].encode()
|
|
m = mask.decode(rle)
|
|
m = np.sum(
|
|
m, axis=2
|
|
) # sometimes there are multiple binary map (corresponding to multiple segs)
|
|
m = m.astype(np.uint8) # convert to np.uint8
|
|
m_final = m_final | m
|
|
m = m_final
|
|
masks.append(m)
|
|
continue
|
|
|
|
ann = annotations[ann_id]
|
|
|
|
if len(ann["segmentation"]) == 0:
|
|
m = np.zeros((image_info["height"], image_info["width"])).astype(
|
|
np.uint8
|
|
)
|
|
masks.append(m)
|
|
continue
|
|
|
|
if type(ann["segmentation"][0]) == list: # polygon
|
|
rle = mask.frPyObjects(
|
|
ann["segmentation"], image_info["height"], image_info["width"]
|
|
)
|
|
else:
|
|
rle = ann["segmentation"]
|
|
for i in range(len(rle)):
|
|
if not isinstance(rle[i]["counts"], bytes):
|
|
rle[i]["counts"] = rle[i]["counts"].encode()
|
|
m = mask.decode(rle)
|
|
m = np.sum(
|
|
m, axis=2
|
|
) # sometimes there are multiple binary map (corresponding to multiple segs)
|
|
m = m.astype(np.uint8) # convert to np.uint8
|
|
masks.append(m)
|
|
|
|
masks = np.stack(masks, axis=0)
|
|
|
|
masks = torch.from_numpy(masks)
|
|
label = torch.ones(masks.shape[1], masks.shape[2]) * self.ignore_label
|
|
|
|
return (
|
|
image_path,
|
|
image,
|
|
image_evf,
|
|
masks,
|
|
label,
|
|
resize,
|
|
sampled_classes,
|
|
)
|