153 lines
4.5 KiB
Python
153 lines
4.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
|
|
import os
|
|
import os.path as osp
|
|
import urllib.parse as parse
|
|
import urllib.request
|
|
from typing import Optional, Union
|
|
|
|
from scepter.modules.utils.file_clients.base_fs import BaseFs
|
|
from scepter.modules.utils.file_clients.registry import FILE_SYSTEMS
|
|
|
|
|
|
@FILE_SYSTEMS.register_class()
|
|
class HttpFs(BaseFs):
|
|
para_dict = {
|
|
'RETRY_TIMES': {
|
|
'value': 10,
|
|
'description': 'Retry get object times.'
|
|
}
|
|
}
|
|
para_dict.update(BaseFs.para_dict)
|
|
|
|
def __init__(self, cfg, logger):
|
|
super(HttpFs, self).__init__(cfg, logger=logger)
|
|
retry_times = cfg.get('RETRY_TIMES', 10)
|
|
self._retry_times = retry_times
|
|
|
|
def get_prefix(self) -> str:
|
|
return 'http'
|
|
|
|
def support_write(self) -> bool:
|
|
return False
|
|
|
|
def support_link(self) -> bool:
|
|
return False
|
|
|
|
def basename(self, target_path) -> str:
|
|
url = parse.unquote(target_path)
|
|
url = url.split('?')[0]
|
|
return osp.basename(url)
|
|
|
|
def get_object_to_local_file(self,
|
|
target_path,
|
|
local_path=None,
|
|
wait_finish=False) -> Optional[str]:
|
|
if local_path is None:
|
|
local_path, is_tmp = self.map_to_local(target_path)
|
|
else:
|
|
is_tmp = False
|
|
|
|
os.makedirs(osp.dirname(local_path), exist_ok=True)
|
|
|
|
retry = 0
|
|
while retry < self._retry_times:
|
|
try:
|
|
target_url = urllib.parse.quote(target_path,
|
|
safe=":/?#[]@!$&'()*+,;=%")
|
|
urllib.request.urlretrieve(target_url, local_path)
|
|
if osp.exists(local_path):
|
|
break
|
|
except Exception:
|
|
retry += 1
|
|
|
|
if retry >= self._retry_times:
|
|
return None
|
|
|
|
if is_tmp:
|
|
self.add_temp_file(local_path)
|
|
return local_path
|
|
|
|
def get_object(self, target_path):
|
|
try:
|
|
local_data = open(self.get_object_to_local_file(target_path),
|
|
'rb').read()
|
|
except Exception as e:
|
|
self.logger.error(f'Read {target_path} error {e}')
|
|
local_data = None
|
|
return local_data
|
|
|
|
def put_object(self, local_data, target_path):
|
|
raise NotImplementedError
|
|
|
|
def put_object_from_local_file(self, local_path, target_path) -> bool:
|
|
raise NotImplementedError
|
|
|
|
def make_link(self, target_link_path, target_path) -> bool:
|
|
raise NotImplementedError
|
|
|
|
def make_dir(self, target_dir) -> bool:
|
|
raise NotImplementedError
|
|
|
|
def remove(self, target_path) -> bool:
|
|
raise NotImplementedError
|
|
|
|
def get_logging_handler(self, target_logging_path):
|
|
raise NotImplementedError
|
|
|
|
def walk_dir(self, file_dir, recurse=True):
|
|
raise NotImplementedError
|
|
|
|
def put_dir_from_local_dir(self,
|
|
local_dir,
|
|
target_dir,
|
|
multi_thread=False) -> bool:
|
|
raise NotImplementedError
|
|
|
|
def size(self, target_path) -> Optional[int]:
|
|
raise NotImplementedError
|
|
|
|
def get_object_chunk_list(self,
|
|
target_path,
|
|
chunk_num=1,
|
|
delimiter=None) -> Optional[list]:
|
|
raise NotImplementedError
|
|
|
|
def get_object_stream(
|
|
self,
|
|
target_path,
|
|
start,
|
|
size=10000,
|
|
delimiter=None) -> (Union[bytes, str, None], Optional[int]):
|
|
raise NotImplementedError
|
|
|
|
def get_dir_to_local_dir(self,
|
|
target_path,
|
|
local_path=None,
|
|
wait_finish=False,
|
|
multi_thread=False,
|
|
timeout=3600,
|
|
worker_id=0) -> Optional[str]:
|
|
raise NotImplementedError
|
|
|
|
def get_url(self, target_path, set_public=False, lifecycle=3600 * 100):
|
|
return target_path
|
|
|
|
def exists(self, target_path) -> bool:
|
|
req = urllib.request.Request(target_path)
|
|
req.get_method = lambda: 'HEAD'
|
|
|
|
try:
|
|
urllib.request.urlopen(req)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def isfile(self, target_path) -> bool:
|
|
# Well for a http url, it should only be a file.
|
|
return True
|
|
|
|
def isdir(self, target_path) -> bool:
|
|
return False
|