modelscope-scepter/scepter/modules/model/backbone/video/bricks/stems/embedding_stem.py

# -*- coding: utf-8 -*-
# Copyright (c) Alibaba, Inc. and its affiliates.

import torch.nn as nn
from scepter.modules.model.backbone.video.bricks.visualize_3d_module import \
    Visualize3DModule
from scepter.modules.model.registry import STEMS
from scepter.modules.utils.config import dict_to_yaml


@STEMS.register_class()
class PatchEmbedStem(Visualize3DModule):
    para_dict = {
        'IMAGE_SIZE': {
            'value': 224,
            'description': "the stem's input frame size!"
        },
        'PATCH_SIZE': {
            'value': 16,
            'description': "the stem's input patch size!"
        },
        'NUM_FRAMES': {
            'value': 16,
            'description': "the stem's input frame num!"
        },
        'NUM_INPUT_CHANNELS': {
            'value': 3,
            'description': "the stem's input channels num!"
        },
        'DIM': {
            'value': 768,
            'description': "the stem's input dim!"
        }
    }
    para_dict.update(Visualize3DModule.para_dict)

    def __init__(self, cfg, logger=None):
        super(PatchEmbedStem, self).__init__(cfg, logger=logger)
        image_size = cfg.get('IMAGE_SIZE', 224)
        patch_size = cfg.get('PATCH_SIZE', 16)
        num_frames = cfg.get('NUM_FRAMES', 16)
        num_input_channels = cfg.get('NUM_INPUT_CHANNELS', 3)
        dim = cfg.get('DIM', 768)
        num_patches_per_image = (image_size // patch_size)**2
        num_patches = num_patches_per_image * num_frames

        self.image_size = image_size
        self.patch_size = patch_size
        self.num_frames = num_frames
        self.num_patches = num_patches

        self.conv1 = nn.Conv3d(in_channels=num_input_channels,
                               out_channels=dim,
                               kernel_size=(1, patch_size, patch_size),
                               stride=(1, patch_size, patch_size),
                               bias=False)

    def forward(self, x):
        h, w, p = x.shape[3], x.shape[4], self.patch_size
        assert h % p == 0 and w % p == 0, f'height {h} and width {w} of video must be divisible by the patch size {p}'
        x = self.conv1(x)
        # b, c, t, h, w -> b, c, p (p: num patches)
        x = x.reshape(x.shape[0], x.shape[1], -1)
        # b, c, p -> b, p, c
        x = x.permute(0, 2, 1)
        return x

    @staticmethod
    def get_config_template():
        '''
        { "ENV" :
            { "description" : "",
              "A" : {
                    "value": 1.0,
                    "description": ""
               }
            }
        }
        :return:
        '''
        return dict_to_yaml('STEM',
                            __class__.__name__,
                            PatchEmbedStem.para_dict,
                            set_name=True)


@STEMS.register_class()
class TubeletEmbeddingStem(Visualize3DModule):
    para_dict = {
        'IMAGE_SIZE': {
            'value': 224,
            'description': "the stem's input frame size!"
        },
        'PATCH_SIZE': {
            'value': 16,
            'description': "the stem's input patch size!"
        },
        'NUM_FRAMES': {
            'value': 16,
            'description': "the stem's input frame num!"
        },
        'NUM_INPUT_CHANNELS': {
            'value': 3,
            'description': "the stem's input channels num!"
        },
        'TUBELET_SIZE': {
            'value': 2,
            'description': "the stem's tubelet size!"
        },
        'DIM': {
            'value': 768,
            'description': "the stem's input dim!"
        }
    }
    para_dict.update(Visualize3DModule.para_dict)

    def __init__(self, cfg, logger=None):
        super().__init__(cfg, logger=logger)
        image_size = cfg.get('IMAGE_SIZE', 224)
        patch_size = cfg.get('PATCH_SIZE', 16)
        num_frames = cfg.get('NUM_FRAMES', 16)
        num_input_channels = cfg.get('NUM_INPUT_CHANNELS', 3)
        tubelet_size = cfg.get('TUBELET_SIZE', 2)
        dim = cfg.get('DIM', 768)
        num_patches_per_image = (image_size // patch_size)**2
        num_patches = num_patches_per_image * num_frames

        self.image_size = image_size
        self.patch_size = patch_size
        self.num_frames = num_frames
        self.num_patches = num_patches

        self.conv1 = nn.Conv3d(in_channels=num_input_channels,
                               out_channels=dim,
                               kernel_size=(tubelet_size, patch_size,
                                            patch_size),
                               stride=(tubelet_size, patch_size, patch_size),
                               bias=False)

    def forward(self, x):
        h, w, p = x.shape[3], x.shape[4], self.patch_size
        assert h % p == 0 and w % p == 0, f'height {h} and width {w} of video must be divisible by the patch size {p}'
        x = self.conv1(x)
        # b, c, t, h, w -> b, c, p (p: num patches)
        x = x.reshape(x.shape[0], x.shape[1], -1)
        # b, c, p -> b, p, c
        x = x.permute(0, 2, 1)
        return x

    @staticmethod
    def get_config_template():
        '''
        { "ENV" :
            { "description" : "",
              "A" : {
                    "value": 1.0,
                    "description": ""
               }
            }
        }
        :return:
        '''
        return dict_to_yaml('STEM',
                            __class__.__name__,
                            TubeletEmbeddingStem.para_dict,
                            set_name=True)