388 lines
11 KiB
Python
388 lines
11 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
|
|
import datetime
|
|
import os
|
|
import os.path as osp
|
|
import random
|
|
import tempfile
|
|
import warnings
|
|
from abc import ABCMeta, abstractmethod
|
|
from copy import copy
|
|
from typing import Optional
|
|
|
|
from scepter.modules.utils.config import dict_to_yaml
|
|
from scepter.modules.utils.directory import get_md5
|
|
from scepter.modules.utils.file_clients.utils import remove_temp_path
|
|
from scepter.modules.utils.logger import get_logger
|
|
|
|
|
|
class BaseFs(object, metaclass=ABCMeta):
|
|
para_dict = {
|
|
'TEMP_DIR': {
|
|
'value':
|
|
None,
|
|
'description':
|
|
'default is None, means using system cache dir and auto remove! If you set dir, the data will'
|
|
' be saved in this temp dir without autoremoving default.'
|
|
},
|
|
'AUTO_CLEAN': {
|
|
'value':
|
|
False,
|
|
'description':
|
|
'when TEMP_DIR is not None, if you set AUTO_CLEAN to True, the data will be clean automatics.'
|
|
}
|
|
}
|
|
|
|
def __init__(self, cfg, logger=None):
|
|
self._target_local_mapper = {}
|
|
self._temp_files = set()
|
|
self.cfg = cfg
|
|
self.tmp_dir = cfg.get('TEMP_DIR', None)
|
|
self.enable_md5_path = cfg.get('ENABLE_MD5_PATH', True)
|
|
self.auto_clean = cfg.get('AUTO_CLEAN', False)
|
|
if self.tmp_dir is None:
|
|
self.auto_clean = True
|
|
if self.tmp_dir is not None:
|
|
if not os.path.exists(self.tmp_dir):
|
|
try:
|
|
os.makedirs(self.tmp_dir, exist_ok=True)
|
|
except Exception as e:
|
|
warnings.warn(
|
|
f'Create cache folder failed use default cache file{e}!'
|
|
.format(self.tmp_dir))
|
|
self.tmp_dir = None
|
|
# checking that the logger exists or not
|
|
if logger is None:
|
|
self.logger = get_logger(name='File System')
|
|
else:
|
|
self.logger = logger
|
|
|
|
# Functions without io
|
|
@abstractmethod
|
|
def get_prefix(self) -> str:
|
|
""" Get supported path prefix to determine which handler to use.
|
|
|
|
Returns:
|
|
A prefix.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def support_write(self) -> bool:
|
|
""" Return flag if this file system supports write operation.
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def support_link(self) -> bool:
|
|
""" Return if this file system supports create a soft link.
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
pass
|
|
|
|
def add_target_local_map(self, target_dir, local_dir):
|
|
""" Map target directory to local file system directory
|
|
|
|
Args:
|
|
target_dir (str): Target directory.
|
|
local_dir (str): Directory in local file system.
|
|
"""
|
|
self._target_local_mapper[target_dir] = local_dir
|
|
|
|
def map_to_local(self, target_path, etag='') -> (str, bool):
|
|
""" Map target path to local file path. (NO IO HERE).
|
|
|
|
Args:
|
|
target_path (str): Target file path.
|
|
|
|
Returns:
|
|
A local path and a flag indicates if the local path is a temporary file.
|
|
"""
|
|
for target_dir, local_dir in self._target_local_mapper.items():
|
|
if target_path.startswith(target_dir):
|
|
return osp.join(local_dir, osp.relpath(target_path,
|
|
target_dir)), False
|
|
else:
|
|
return self._make_temporary_file(target_path, etag=etag), True
|
|
|
|
def convert_to_local_path(self, target_path, etag='') -> str:
|
|
""" Deprecated. Use map_to_local() function instead.
|
|
"""
|
|
warnings.warn(
|
|
'Function convert_to_local_path is deprecated, use map_to_local() function instead.'
|
|
)
|
|
local_path, _ = self.map_to_local(target_path, etag=etag)
|
|
return local_path
|
|
|
|
def basename(self, target_path) -> str:
|
|
""" Get file name from target_path
|
|
|
|
Args:
|
|
target_path (str): Target file path.
|
|
|
|
Returns:
|
|
A file name.
|
|
"""
|
|
return osp.basename(target_path)
|
|
|
|
# Functions with heavy io
|
|
@abstractmethod
|
|
def get_object_to_local_file(self,
|
|
target_path,
|
|
local_path=None,
|
|
wait_finish=False) -> Optional[str]:
|
|
""" Transfer file object to local file.
|
|
If local_path is not None,
|
|
if path can be searched in local_mapper, download it as a persistent file
|
|
else, download it as a temporary file
|
|
else
|
|
download it as a persistent file
|
|
|
|
wait_finish when multi-processing download the same data, set wait_finish as True to avoid conflict
|
|
|
|
Args:
|
|
target_path (str): path of object in different file systems
|
|
local_path (Optional[str]): If not None, will write path to local_path.
|
|
|
|
Returns:
|
|
Local file path of the object, none means a failure happened.
|
|
"""
|
|
pass
|
|
|
|
# Functions with heavy io
|
|
@abstractmethod
|
|
def get_object(self, target_path):
|
|
""" Transfer file object to local file.
|
|
If local_path is not None,
|
|
if path can be searched in local_mapper, download it as a persistent file
|
|
else, download it as a temporary file
|
|
else
|
|
download it as a persistent file
|
|
|
|
Args:
|
|
target_path (str): path of object in different file systems
|
|
local_path (Optional[str]): If not None, will write path to local_path.
|
|
|
|
Returns:
|
|
Local file path of the object, none means a failure happened.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_object_stream(self, target_path, start, size, delimiter=None):
|
|
""" Transfer file object to local file.
|
|
If local_path is not None,
|
|
if path can be searched in local_mapper, download it as a persistent file
|
|
else, download it as a temporary file
|
|
else
|
|
download it as a persistent file
|
|
|
|
Args:
|
|
target_path (str): path of object in different file systems
|
|
start (int): object's start position.
|
|
size (int): object's bytes size.
|
|
delimiter (str): records's delimiter.
|
|
|
|
Returns:
|
|
Local file path of the object, none means a failure happened.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def put_object_from_local_file(self, local_path, target_path) -> bool:
|
|
""" Put local file to target file system path.
|
|
|
|
Args:
|
|
local_path (str): local file path of the object
|
|
target_path (str): target file path of the object
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def put_object(self, local_data, target_path) -> bool:
|
|
""" Put local file to target file system path.
|
|
|
|
Args:
|
|
local_path (binary): local data of the object
|
|
target_path (str): target file path of the object
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def make_link(self, target_link_path, target_path) -> bool:
|
|
""" Make soft link to target_path.
|
|
|
|
Args:
|
|
target_link_path (str):
|
|
target_path (str)
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def make_dir(self, target_dir) -> bool:
|
|
""" Make a directory.
|
|
If target_dir is already exists, return True.
|
|
|
|
Args:
|
|
target_dir (str):
|
|
|
|
Returns:
|
|
True if target_dir exists or created.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def remove(self, target_path) -> bool:
|
|
""" Remove target file.
|
|
|
|
Args:
|
|
target_path (str):
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_logging_handler(self, target_logging_path):
|
|
""" Get logging handler to target logging path.
|
|
|
|
Args:
|
|
target_logging_path:
|
|
|
|
Returns:
|
|
A handler which has a type of subclass of logging.Handler.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def walk_dir(self, file_dir, recurse=True):
|
|
pass
|
|
|
|
@abstractmethod
|
|
def put_dir_from_local_dir(self, local_dir, target_dir) -> bool:
|
|
""" Upload all contents in local_dir to target_dir, keep the file tree.
|
|
|
|
Args:
|
|
local_dir (str):
|
|
target_dir (str):
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
pass
|
|
|
|
def _make_temporary_file(self, target_path, etag=''):
|
|
""" Make a temporary file for target_path, which should have the same suffix.
|
|
|
|
Args:
|
|
target_path (str):
|
|
|
|
Returns:
|
|
A path (str).
|
|
"""
|
|
file_name = self.basename(target_path)
|
|
_, suffix = osp.splitext(file_name)
|
|
if self.tmp_dir is None:
|
|
rand_name = '{0:%Y%m%d%H%M%S%f}'.format(
|
|
datetime.datetime.now()) + '_' + ''.join(
|
|
[str(random.randint(1, 10)) for _ in range(5)])
|
|
# rand_name = get_md5(target_path)
|
|
if suffix:
|
|
rand_name += f'{suffix}'
|
|
tmp_file = osp.join(tempfile.gettempdir(), rand_name)
|
|
else:
|
|
if self.enable_md5_path:
|
|
cache_name = '{}{}{}'.format(etag, get_md5(target_path), suffix)
|
|
else:
|
|
cache_name = ''
|
|
tmp_file = osp.join(self.tmp_dir, cache_name)
|
|
return tmp_file
|
|
|
|
# Functions only for status, light io
|
|
@abstractmethod
|
|
def exists(self, target_path) -> bool:
|
|
""" Check if target_path exists.
|
|
|
|
Args:
|
|
target_path (str):
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def isfile(self, target_path) -> bool:
|
|
""" Check if target_path is a file.
|
|
|
|
Args:
|
|
target_path (str):
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def isdir(self, target_path) -> bool:
|
|
""" Check if target_path is a directory.
|
|
|
|
Args:
|
|
target_path (str):
|
|
|
|
Returns:
|
|
Bool.
|
|
"""
|
|
|
|
def add_temp_file(self, tmp_file):
|
|
self._temp_files.add(tmp_file)
|
|
|
|
def clear(self):
|
|
"""Delete all temp files
|
|
"""
|
|
if self.auto_clean:
|
|
for temp_local_file in self._temp_files:
|
|
remove_temp_path(temp_local_file)
|
|
|
|
# Functions for context
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
# if self.tmp_dir is None or self.auto_clean:
|
|
# for temp_local_file in self._temp_files:
|
|
# remove_temp_path(temp_local_file)
|
|
pass
|
|
|
|
def __del__(self):
|
|
pass
|
|
|
|
def copy(self):
|
|
obj = copy(self)
|
|
obj._temp_files = set(
|
|
) # A new obj to avoid confusing in multi-thread context.
|
|
return obj
|
|
|
|
@staticmethod
|
|
def get_config_template():
|
|
return dict_to_yaml('FILE_SYSTEMS',
|
|
__class__.__name__,
|
|
BaseFs.para_dict,
|
|
set_name=True)
|