288 lines
10 KiB
Python
288 lines
10 KiB
Python
# Copyright 2025 Bytedance Ltd. and/or its affiliates.
|
||
# SPDX-License-Identifier: Apache-2.0
|
||
|
||
import random
|
||
from PIL import Image
|
||
|
||
import cv2
|
||
import numpy as np
|
||
import torch
|
||
from torchvision import transforms
|
||
from torchvision.transforms import functional as F
|
||
from torchvision.transforms import InterpolationMode
|
||
|
||
|
||
class MaxLongEdgeMinShortEdgeResize(torch.nn.Module):
|
||
"""Resize the input image so that its longest side and shortest side are within a specified range,
|
||
ensuring that both sides are divisible by a specified stride.
|
||
|
||
Args:
|
||
max_size (int): Maximum size for the longest edge of the image.
|
||
min_size (int): Minimum size for the shortest edge of the image.
|
||
stride (int): Value by which the height and width of the image must be divisible.
|
||
max_pixels (int): Maximum pixels for the full image.
|
||
interpolation (InterpolationMode): Desired interpolation enum defined by
|
||
:class:`torchvision.transforms.InterpolationMode`. Default is ``InterpolationMode.BILINEAR``.
|
||
If input is Tensor, only ``InterpolationMode.NEAREST``, ``InterpolationMode.NEAREST_EXACT``,
|
||
``InterpolationMode.BILINEAR``, and ``InterpolationMode.BICUBIC`` are supported.
|
||
The corresponding Pillow integer constants, e.g., ``PIL.Image.BILINEAR`` are also accepted.
|
||
antialias (bool, optional): Whether to apply antialiasing (default is True).
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
max_size: int,
|
||
min_size: int,
|
||
stride: int,
|
||
max_pixels: int,
|
||
interpolation=InterpolationMode.BICUBIC,
|
||
antialias=True
|
||
):
|
||
super().__init__()
|
||
self.max_size = max_size
|
||
self.min_size = min_size
|
||
self.stride = stride
|
||
self.max_pixels = max_pixels
|
||
self.interpolation = interpolation
|
||
self.antialias = antialias
|
||
|
||
def _make_divisible(self, value, stride):
|
||
"""Ensure the value is divisible by the stride."""
|
||
return max(stride, int(round(value / stride) * stride))
|
||
|
||
def _apply_scale(self, width, height, scale):
|
||
new_width = round(width * scale)
|
||
new_height = round(height * scale)
|
||
new_width = self._make_divisible(new_width, self.stride)
|
||
new_height = self._make_divisible(new_height, self.stride)
|
||
return new_width, new_height
|
||
|
||
def forward(self, img, img_num=1):
|
||
"""
|
||
Args:
|
||
img (PIL Image): Image to be resized.
|
||
img_num (int): Number of images, used to change max_tokens.
|
||
Returns:
|
||
PIL Image or Tensor: Rescaled image with divisible dimensions.
|
||
"""
|
||
if isinstance(img, torch.Tensor):
|
||
height, width = img.shape[-2:]
|
||
else:
|
||
width, height = img.size
|
||
|
||
scale = min(self.max_size / max(width, height), 1.0)
|
||
scale = max(scale, self.min_size / min(width, height))
|
||
new_width, new_height = self._apply_scale(width, height, scale)
|
||
|
||
# Ensure the number of pixels does not exceed max_pixels
|
||
if new_width * new_height > self.max_pixels / img_num:
|
||
scale = self.max_pixels / img_num / (new_width * new_height)
|
||
new_width, new_height = self._apply_scale(new_width, new_height, scale)
|
||
|
||
# Ensure longest edge does not exceed max_size
|
||
if max(new_width, new_height) > self.max_size:
|
||
scale = self.max_size / max(new_width, new_height)
|
||
new_width, new_height = self._apply_scale(new_width, new_height, scale)
|
||
|
||
return F.resize(img, (new_height, new_width), self.interpolation, antialias=self.antialias)
|
||
|
||
|
||
class ImageTransform:
|
||
def __init__(
|
||
self,
|
||
max_image_size,
|
||
min_image_size,
|
||
image_stride,
|
||
max_pixels=14*14*9*1024,
|
||
image_mean=[0.5, 0.5, 0.5],
|
||
image_std=[0.5, 0.5, 0.5]
|
||
):
|
||
self.stride = image_stride
|
||
|
||
self.resize_transform = MaxLongEdgeMinShortEdgeResize(
|
||
max_size=max_image_size,
|
||
min_size=min_image_size,
|
||
stride=image_stride,
|
||
max_pixels=max_pixels,
|
||
)
|
||
self.to_tensor_transform = transforms.ToTensor()
|
||
self.normalize_transform = transforms.Normalize(mean=image_mean, std=image_std, inplace=True)
|
||
|
||
def __call__(self, img, img_num=1):
|
||
img = self.resize_transform(img, img_num=img_num)
|
||
img = self.to_tensor_transform(img)
|
||
img = self.normalize_transform(img)
|
||
return img
|
||
|
||
|
||
def decolorization(image):
|
||
gray_image = image.convert('L')
|
||
return Image.merge(image.mode, [gray_image] * 3) if image.mode in ('RGB', 'L') else gray_image
|
||
|
||
|
||
def downscale(image, scale_factor):
|
||
new_width = int(round(image.width * scale_factor))
|
||
new_height = int(round(image.height * scale_factor))
|
||
new_width = max(1, new_width)
|
||
new_height = max(1, new_height)
|
||
return image.resize((new_width, new_height), resample=Image.BICUBIC)
|
||
|
||
|
||
def crop(image, crop_factors):
|
||
target_h, target_w = crop_factors
|
||
img_w, img_h = image.size
|
||
|
||
if target_h > img_h or target_w > img_w:
|
||
raise ValueError("Crop size exceeds image dimensions")
|
||
|
||
x = random.randint(0, img_w - target_w)
|
||
y = random.randint(0, img_h - target_h)
|
||
|
||
return image.crop((x, y, x + target_w, y + target_h)), [[x, y], [x + target_w, y + target_h]]
|
||
|
||
|
||
def motion_blur_opencv(image, kernel_size=15, angle=0):
|
||
# 线性核
|
||
kernel = np.zeros((kernel_size, kernel_size), dtype=np.float32)
|
||
kernel[kernel_size // 2, :] = np.ones(kernel_size, dtype=np.float32)
|
||
|
||
# 旋转核
|
||
center = (kernel_size / 2 - 0.5, kernel_size / 2 - 0.5)
|
||
M = cv2.getRotationMatrix2D(center, angle, 1)
|
||
rotated_kernel = cv2.warpAffine(kernel, M, (kernel_size, kernel_size))
|
||
|
||
# 归一化核
|
||
rotated_kernel /= rotated_kernel.sum() if rotated_kernel.sum() != 0 else 1
|
||
|
||
img = np.array(image)
|
||
if img.ndim == 2:
|
||
blurred = cv2.filter2D(img, -1, rotated_kernel, borderType=cv2.BORDER_REFLECT)
|
||
else:
|
||
# 对于彩色图像，各通道独立卷积
|
||
blurred = np.zeros_like(img)
|
||
for c in range(img.shape[2]):
|
||
blurred[..., c] = cv2.filter2D(img[..., c], -1, rotated_kernel, borderType=cv2.BORDER_REFLECT)
|
||
|
||
return Image.fromarray(blurred.astype(np.uint8))
|
||
|
||
|
||
def shuffle_patch(image, num_splits, gap_size=2):
|
||
"""将图像分割为块（允许尺寸不整除），随机打乱后拼接，块间保留间隙"""
|
||
h_splits, w_splits = num_splits
|
||
img_w, img_h = image.size
|
||
|
||
base_patch_h = img_h // h_splits
|
||
patch_heights = [base_patch_h] * (h_splits - 1)
|
||
patch_heights.append(img_h - sum(patch_heights))
|
||
|
||
base_patch_w = img_w // w_splits
|
||
patch_widths = [base_patch_w] * (w_splits - 1)
|
||
patch_widths.append(img_w - sum(patch_widths))
|
||
|
||
patches = []
|
||
current_y = 0
|
||
for i in range(h_splits):
|
||
current_x = 0
|
||
patch_h = patch_heights[i]
|
||
for j in range(w_splits):
|
||
patch_w = patch_widths[j]
|
||
patch = image.crop((current_x, current_y, current_x + patch_w, current_y + patch_h))
|
||
patches.append(patch)
|
||
current_x += patch_w
|
||
current_y += patch_h
|
||
|
||
random.shuffle(patches)
|
||
|
||
total_width = sum(patch_widths) + (w_splits - 1) * gap_size
|
||
total_height = sum(patch_heights) + (h_splits - 1) * gap_size
|
||
new_image = Image.new(image.mode, (total_width, total_height), color=(255, 255, 255))
|
||
|
||
current_y = 0 # 当前行的起始 Y 坐标
|
||
patch_idx = 0 # 当前处理的块索引
|
||
for i in range(h_splits):
|
||
current_x = 0 # 当前列的起始 X 坐标
|
||
patch_h = patch_heights[i] # 当前行块的高度
|
||
for j in range(w_splits):
|
||
# 取出打乱后的块
|
||
patch = patches[patch_idx]
|
||
patch_w = patch_widths[j] # 当前列块的宽度
|
||
# 粘贴块（左上角坐标为 (current_x, current_y)）
|
||
new_image.paste(patch, (current_x, current_y))
|
||
# 更新 X 坐标（下一个块的起始位置 = 当前块宽度 + 间隙）
|
||
current_x += patch_w + gap_size
|
||
patch_idx += 1
|
||
# 更新 Y 坐标（下一行的起始位置 = 当前行高度 + 间隙）
|
||
current_y += patch_h + gap_size
|
||
|
||
return new_image
|
||
|
||
|
||
def inpainting(image, num_splits, blank_ratio=0.3, blank_color=(255, 255, 255)):
|
||
"""
|
||
图像分割后随机空白部分patch，用于inpainting任务
|
||
|
||
参数：
|
||
image: PIL.Image 输入图像（RGB模式）
|
||
h_splits: int 行分割数（垂直方向分割块数）
|
||
w_splits: int 列分割数（水平方向分割块数）
|
||
blank_ratio: float 空白patch的比例（0~1）
|
||
blank_color: tuple 空白区域的颜色（RGB，如白色(255,255,255)）
|
||
|
||
返回：
|
||
PIL.Image 处理后拼接的图像
|
||
"""
|
||
h_splits, w_splits = num_splits
|
||
img_w, img_h = image.size
|
||
|
||
base_patch_h = img_h // h_splits
|
||
patch_heights = [base_patch_h] * (h_splits - 1)
|
||
patch_heights.append(img_h - sum(patch_heights))
|
||
|
||
base_patch_w = img_w // w_splits
|
||
patch_widths = [base_patch_w] * (w_splits - 1)
|
||
patch_widths.append(img_w - sum(patch_widths))
|
||
|
||
patches = []
|
||
current_y = 0
|
||
for i in range(h_splits):
|
||
current_x = 0
|
||
patch_h = patch_heights[i]
|
||
for j in range(w_splits):
|
||
patch_w = patch_widths[j]
|
||
patch = image.crop((current_x, current_y, current_x + patch_w, current_y + patch_h))
|
||
patches.append(patch)
|
||
current_x += patch_w
|
||
current_y += patch_h
|
||
|
||
total_patches = h_splits * w_splits
|
||
num_blank = int(total_patches * blank_ratio)
|
||
num_blank = max(0, min(num_blank, total_patches))
|
||
blank_indices = random.sample(range(total_patches), num_blank)
|
||
|
||
processed_patches = []
|
||
for idx, patch in enumerate(patches):
|
||
if idx in blank_indices:
|
||
blank_patch = Image.new("RGB", patch.size, color=blank_color)
|
||
processed_patches.append(blank_patch)
|
||
else:
|
||
processed_patches.append(patch)
|
||
|
||
# 创建结果图像（尺寸与原图一致）
|
||
result_image = Image.new("RGB", (img_w, img_h))
|
||
current_y = 0
|
||
patch_idx = 0
|
||
for i in range(h_splits):
|
||
current_x = 0
|
||
patch_h = patch_heights[i]
|
||
for j in range(w_splits):
|
||
# 取出处理后的patch
|
||
patch = processed_patches[patch_idx]
|
||
patch_w = patch_widths[j]
|
||
# 粘贴到原位置
|
||
result_image.paste(patch, (current_x, current_y))
|
||
current_x += patch_w
|
||
patch_idx += 1
|
||
current_y += patch_h
|
||
|
||
return result_image
|