update v0.0.4

scepter/modules/utils/config.py

@@ -603,3 +603,6 @@ class Config(object):
             return cfg_new
         else:
             return cfg
+
+    def pop(self, name):
+        self.cfg_dict.pop(name)

scepter/modules/utils/export_model.py

@@ -4,10 +4,10 @@ import io
 from io import BytesIO
 
 import onnx
+import onnxruntime
 import torch
 from torch.onnx import OperatorExportTypes
 
-import onnxruntime
 from scepter.modules.utils.distribute import we
 
 type_map = {

scepter/modules/utils/file_clients/local_fs.py

@@ -293,6 +293,9 @@ class LocalFs(BaseFs):
         return True
 
     def get_logging_handler(self, target_logging_path):
+        dirname = os.path.dirname(target_logging_path)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname, exist_ok=True)
         return logging.FileHandler(target_logging_path)
 
     def put_dir_from_local_dir(self,
@@ -303,11 +306,11 @@ class LocalFs(BaseFs):
         target_dir = self.reconstruct_path(target_dir)
         if local_dir == target_dir:
             return True
-        # cp -f local_dir/* target_dir/*
-        if not osp.exists(target_dir):
-            status = os.system(f'mkdir -p {target_dir}')
-            if status != 0:
-                return False
+        # # cp -f local_dir/* target_dir/*
+        # if not osp.exists(target_dir):
+        #     status = os.system(f'mkdir -p {target_dir}')
+        #     if status != 0:
+        #         return False
         try:
             shutil.copytree(local_dir, target_dir, symlinks=True)
         except Exception:

scepter/modules/utils/file_system.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import io
 import os
 import threading
 import time
@@ -327,6 +328,10 @@ class FileSystem(object):
                                                target_path_list):
                 if local_path is None or target_path is None:
                     flg = False
+                elif isinstance(local_path, io.BytesIO):
+                    flg = FS.put_object(local_path.getvalue(), target_path)
+                elif isinstance(local_path, bytes):
+                    flg = FS.put_object(local_path, target_path)
                 elif self.exists(local_path):
                     local_cache = self.get_from(local_path,
                                                 local_path + f'{time.time()}',

scepter/modules/utils/index.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+import os.path as osp
+
+from tqdm import tqdm
+
+from scepter.modules.utils.file_system import FS
+
+
+def init_1level_llfs(list_file,
+                     max_lines=1024,
+                     index_name='index',
+                     delimiter='\n'):
+    r"""Construct large-list-file index.
+    """
+    index_dir = osp.splitext(list_file)[0]
+    print(list_file)
+    with FS.get_from(list_file, wait_finish=True) as local_path:
+        print(local_path)
+        _num_split, index_stack, save_files = 0, [], []
+        with open(local_path, 'r', buffering=1000000) as f:
+            for line in tqdm(f):
+                index_stack.append(line.strip())
+                if len(index_stack) >= max_lines:
+                    save_file = f'{index_dir}/{index_name}/{_num_split + 1:09d}.txt'
+                    with FS.put_to(save_file) as cache_path:
+                        with open(cache_path, 'w') as f_w:
+                            f_w.write('\n'.join(index_stack))
+                    index_stack = []
+                    _num_split += 1
+                    save_files.append(save_file)
+
+    if len(index_stack) > 0:
+        save_file = f'{index_dir}/{index_name}/{_num_split + 1:06d}.txt'
+        with FS.put_to(save_file) as cache_path:
+            with open(cache_path, 'w') as f_w:
+                f_w.write('\n'.join(index_stack))
+        save_files.append(save_file)
+
+    # output meta-file
+    index_file = osp.join(index_dir, f'{index_name}.txt')
+    with FS.put_to(index_file) as cache_path:
+        with open(cache_path, 'w') as f_w:
+            f_w.write('\n'.join(save_files))
+    return index_file

scepter/modules/utils/probe.py

@@ -263,7 +263,8 @@ class ProbeData():
                         url = FS.get_url(one_path,
                                          lifecycle=3600 * 365 * 24).replace(
                                              '.oss-internal.aliyun-inc.',
-                                             '.oss.aliyuncs.')
+                                             '.oss.aliyuncs.').replace(
+                                                 '-internal', '')
                         one_rank += (
                             f'<td align="center"><input type="image" src="{url}" >'
                             f'<br><font size="4"><strong>{save_id}-{idx}|{one_label}<strong></font><br/></td>'