update v0.0.5
This commit is contained in:
Jingfeng727
+17
-17
@@ -14,8 +14,8 @@ Model modules are divided into backbones, necks, heads, loss, metrics, networks,
|
||||
Subclass registration:
|
||||
|
||||
```python
|
||||
from scepter.model.registry import BACKBONES
|
||||
from scepter.model.base_model import BaseModel
|
||||
from scepter.modules.model.registry import BACKBONES
|
||||
from scepter.modules.model.base_model import BaseModel
|
||||
|
||||
|
||||
@BACKBONES.register_class("ResNet")
|
||||
@@ -25,8 +25,8 @@ class ResNet(BaseModel):
|
||||
```
|
||||
|
||||
```python
|
||||
from scepter.model.registry import NECKS
|
||||
from scepter.model.base_model import BaseModel
|
||||
from scepter.modules.model.registry import NECKS
|
||||
from scepter.modules.model.base_model import BaseModel
|
||||
|
||||
|
||||
@NECKS.register_class()
|
||||
@@ -36,8 +36,8 @@ class GlobalAveragePooling(BaseModel):
|
||||
```
|
||||
|
||||
```python
|
||||
from scepter.model.registry import HEADS
|
||||
from scepter.model.base_model import BaseModel
|
||||
from scepter.modules.model.registry import HEADS
|
||||
from scepter.modules.model.base_model import BaseModel
|
||||
|
||||
|
||||
@HEADS.register_class()
|
||||
@@ -47,7 +47,7 @@ class ClassifierHead(BaseModel):
|
||||
```
|
||||
|
||||
```python
|
||||
from scepter.model.registry import LOSSES
|
||||
from scepter.modules.model.registry import LOSSES
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ class CrossEntropy(nn.Module):
|
||||
Actual usage：
|
||||
|
||||
```python
|
||||
from scepter.model.registry import BACKBONES, NECKS, HEADS, LOSSES
|
||||
from scepter.modules.model.registry import BACKBONES, NECKS, HEADS, LOSSES
|
||||
|
||||
backbone = BACKBONES.build(cfg.BACKBONE, logger=logger)
|
||||
neck = NECKS.build(cfg.NECK, logger=logger)
|
||||
@@ -83,8 +83,8 @@ To be implemented specifically as needed;
|
||||
Basic Usage Subclass registration:
|
||||
|
||||
```python
|
||||
from scepter.model.metrics.registry import METRICS
|
||||
from scepter.model.metrics.base_metric import BaseMetric
|
||||
from scepter.modules.model.metrics.registry import METRICS
|
||||
from scepter.modules.model.metrics.base_metric import BaseMetric
|
||||
|
||||
|
||||
@METRICS.register_class("AccuracyMetric")
|
||||
@@ -95,7 +95,7 @@ class AccuracyMetric(BaseMetric):
|
||||
Actual usage:
|
||||
|
||||
```python
|
||||
from scepter.model.metrics.registry import METRICS
|
||||
from scepter.modules.model.metrics.registry import METRICS
|
||||
|
||||
metric = METRICS.build(cfgs, logger)
|
||||
```
|
||||
@@ -117,8 +117,8 @@ Typically takes logits and labels as well as other necessary variables as inputs
|
||||
Subclass registration:
|
||||
|
||||
```python
|
||||
from scepter.model.registry import TOKENIZERS
|
||||
from scepter.model.tokenizers import BaseTokenizer
|
||||
from scepter.modules.model.registry import TOKENIZERS
|
||||
from scepter.modules.model.tokenizers import BaseTokenizer
|
||||
|
||||
|
||||
@TOKENIZERS.register_class()
|
||||
@@ -129,7 +129,7 @@ class BaseBertTokenizer(BaseTokenizer):
|
||||
Actual usage：
|
||||
|
||||
```python
|
||||
from scepter.model.registry import TOKENIZERS
|
||||
from scepter.modules.model.registry import TOKENIZERS
|
||||
|
||||
tokenizer = TOKENIZERS.build(cfgs, logger)
|
||||
```
|
||||
@@ -147,8 +147,8 @@ Takes a list of texts that need tokenization as input and outputs token id seque
|
||||
Subclass registration:
|
||||
|
||||
```python
|
||||
from scepter.model.registry import MODELS
|
||||
from scepter.model.networks.train_module import TrainModule
|
||||
from scepter.modules.model.registry import MODELS
|
||||
from scepter.modules.model.networks.train_module import TrainModule
|
||||
|
||||
|
||||
@MODELS.register_class()
|
||||
@@ -159,7 +159,7 @@ class Classifier(TrainModule):
|
||||
Actual usage：
|
||||
|
||||
```python
|
||||
from scepter.model.registry import MODELS
|
||||
from scepter.modules.model.registry import MODELS
|
||||
|
||||
model = MODELS.build(self.cfg.MODEL, logger=self.logger)
|
||||
```
|
||||
|
||||
@@ -9,8 +9,8 @@
|
||||
Usage when subclassing lr_schedulers:
|
||||
|
||||
```python
|
||||
from scepter.opt.lr_schedulers import LR_SCHEDULERS
|
||||
from scepter.opt.lr_schedulers.base_scheduler import BaseScheduler
|
||||
from scepter.modules.opt.lr_schedulers import LR_SCHEDULERS
|
||||
from scepter.modules.opt.lr_schedulers.base_scheduler import BaseScheduler
|
||||
|
||||
|
||||
@LR_SCHEDULERS.register_class()
|
||||
@@ -48,8 +48,8 @@ Sets up the schedule for the passed-in optimizer object;
|
||||
Usage when subclassing optimizers:
|
||||
|
||||
```python
|
||||
from scepter.opt.optimizers.base_optimizer import BaseOptimize
|
||||
from scepter.opt.optimizers.registry import OPTIMIZERS
|
||||
from scepter.modules.opt.optimizers.base_optimizer import BaseOptimize
|
||||
from scepter.modules.opt.optimizers.registry import OPTIMIZERS
|
||||
|
||||
|
||||
@OPTIMIZERS.register_class()
|
||||
|
||||
@@ -6,17 +6,17 @@ This is the File System Module, designed to handle file transfer functionalities
|
||||
|
||||
The component currently supports three types of IO Handler:
|
||||
|
||||
1. scepter.utils.file_clients.AliyunOssFs
|
||||
2. scepter.utils.file_clients.LocalFs
|
||||
3. scepter.utils.file_clients.HttpFs
|
||||
1. scepter.modules.utils.file_clients.AliyunOssFs
|
||||
2. scepter.modules.utils.file_clients.LocalFs
|
||||
3. scepter.modules.utils.file_clients.HttpFs
|
||||
|
||||
<hr/>
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```python
|
||||
from scepter.utils.file_system import FS
|
||||
from scepter.utils.config import Config
|
||||
from scepter.modules.utils.file_system import FS
|
||||
from scepter.modules.utils.config import Config
|
||||
|
||||
fs_cfg = Config(load=False, cfg_dict={
|
||||
"NAME": "AliyunOssFs",
|
||||
|
||||
@@ -4,18 +4,18 @@ Relies on SDKs, which are used to organize modules and SDKs that are frequently
|
||||
|
||||
## Overview
|
||||
|
||||
1. Parameter sdk (scepter.utils.config)
|
||||
2. Path sdk (scepter.utils.directory)
|
||||
3. PyTorch distributed sdk (scepter.utils.distribute)
|
||||
4. Model export sdk (scepter.utils.export_model)
|
||||
5. File system sdk (scepter.utils.file_system)
|
||||
6. Logging sdk (scepter.utils.logger)
|
||||
7. Video processing sdk (scepter.utils.video_reader), see the document (video_reader.md)
|
||||
8. Module registration sdk (scepter.utils.registry)
|
||||
9. Data sdk (scepter.utils.data)
|
||||
10. Model sdk (scepter.utils.model)
|
||||
11. Sampler sdk (scepter.utils.sampler)
|
||||
12. Probing sdk (scepter.utils.probe)
|
||||
1. Parameter sdk (scepter.modules.utils.config)
|
||||
2. Path sdk (scepter.modules.utils.directory)
|
||||
3. PyTorch distributed sdk (scepter.modules.utils.distribute)
|
||||
4. Model export sdk (scepter.modules.utils.export_model)
|
||||
5. File system sdk (scepter.modules.utils.file_system)
|
||||
6. Logging sdk (scepter.modules.utils.logger)
|
||||
7. Video processing sdk (scepter.modules.utils.video_reader), see the document (video_reader.md)
|
||||
8. Module registration sdk (scepter.modules.utils.registry)
|
||||
9. Data sdk (scepter.modules.utils.data)
|
||||
10. Model sdk (scepter.modules.utils.model)
|
||||
11. Sampler sdk (scepter.modules.utils.sampler)
|
||||
12. Probing sdk (scepter.modules.utils.probe)
|
||||
|
||||
<hr/>
|
||||
|
||||
@@ -24,7 +24,7 @@ Relies on SDKs, which are used to organize modules and SDKs that are frequently
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
from scepter.utils.config import Config
|
||||
from scepter.modules.utils.config import Config
|
||||
# Initialize Config object from a dict
|
||||
fs_cfg = Config(load=False, cfg_dict={"NAME": "LocalFs"})
|
||||
print(fs_cfg.NAME)
|
||||
@@ -105,7 +105,7 @@ print(fs_cfg.args)
|
||||
Some commonly used path functions
|
||||
### Basic Usage
|
||||
```python
|
||||
from scepter.utils.directory import osp_path
|
||||
from scepter.modules.utils.directory import osp_path
|
||||
# Automatically join paths based on the path prefix
|
||||
prefix = "xxxx"
|
||||
data_file = "example_videos/1.mp4"
|
||||
@@ -114,13 +114,13 @@ print(osp_path(prefix, data_file))
|
||||
# Also outputs as xxxx/example_videos/1.mp4
|
||||
data_file = "xxxx/example_videos/1.mp4"
|
||||
print(osp_path(prefix, data_file))
|
||||
from scepter.utils.directory import get_relative_folder
|
||||
from scepter.modules.utils.directory import get_relative_folder
|
||||
# Get the folder path at a specified level according to the path
|
||||
# By default, the last level xxxx/example_videos/
|
||||
print(get_relative_folder(data_file))
|
||||
# The second last level xxxx/
|
||||
print(get_relative_folder(data_file, keep_index=-2))
|
||||
from scepter.utils.directory import get_md5
|
||||
from scepter.modules.utils.directory import get_md5
|
||||
# Get the md5 code of the text/path 34a447fb46d0b786a3999c9dad01d470
|
||||
print(get_md5(data_file))
|
||||
```
|
||||
@@ -175,8 +175,8 @@ PyTorch distributed initialization SDK. By using this SDK, users can avoid focus
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
from scepter.utils.distribute import we
|
||||
from scepter.utils.config import Config
|
||||
from scepter.modules.utils.distribute import we
|
||||
from scepter.modules.utils.config import Config
|
||||
|
||||
cfg = Config(cfg_dict={}, load=False)
|
||||
|
||||
@@ -304,12 +304,12 @@ Since cloning is involved, this may cause additional GPU memory waste.
|
||||
**Returns**
|
||||
- **tensor** —— The output tensor on the CPU for process rank=0.
|
||||
|
||||
## 4. 模型导出sdk(scepter.utils.export_model)
|
||||
## 4. 模型导出sdk(scepter.modules.utils.export_model)
|
||||
APIs for exporting models to TorchScript/ONNX formats.
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
from scepter.utils.export_model import save_develop_model_multi_io
|
||||
from scepter.modules.utils.export_model import save_develop_model_multi_io
|
||||
|
||||
save_develop_model_multi_io(
|
||||
model,
|
||||
@@ -345,16 +345,16 @@ Supports importing and exporting models with multiple inputs and outputs
|
||||
**Returns**
|
||||
- **tensor** —— The output tensor on the CPU for process rank=0.
|
||||
|
||||
## 5. 文件系统sdk(scepter.utils.file_system)
|
||||
## 5. 文件系统sdk(scepter.modules.utils.file_system)
|
||||
Refer to [file_clients](file_clients.md)
|
||||
|
||||
## 6. Logging SDK(scepter.utils.logger)
|
||||
## 6. Logging SDK(scepter.modules.utils.logger)
|
||||
Used to instantiate a standard logging instance for printing information.
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
from scepter.utils.logger import get_logger, init_logger
|
||||
from scepter.modules.utils.logger import get_logger, init_logger
|
||||
|
||||
std_logger = get_logger(name="scepter")
|
||||
init_logger(std_logger, log_file="", dist_launcher="pytorch")
|
||||
@@ -405,14 +405,14 @@ Calculate the time remaining until completion based on the current usage time an
|
||||
**Returns**
|
||||
- **str** —— Formatted output.
|
||||
|
||||
## 7. Video Processing SDK (scepter.utils.video_reader)
|
||||
## 7. Video Processing SDK (scepter.modules.utils.video_reader)
|
||||
APIs for handling video reading.
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
from scepter.utils.video_reader.frame_sampler import do_frame_sample
|
||||
from scepter.utils.video_reader.video_reader import (
|
||||
from scepter.modules.utils.video_reader.frame_sampler import do_frame_sample
|
||||
from scepter.modules.utils.video_reader.video_reader import (
|
||||
VideoReaderWrapper, EasyVideoReader, FramesReaderWrapper
|
||||
)
|
||||
```
|
||||
@@ -554,14 +554,14 @@ Iterator, with each iteration returning a tensor of a segment.
|
||||
**Returns**
|
||||
- **tensor** —— The tensor of the video segment.
|
||||
|
||||
## 8. Module Registration SDK (scepter.utils.registry)
|
||||
## 8. Module Registration SDK (scepter.modules.utils.registry)
|
||||
Used for managing various registered classes.
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
from scepter.utils.registry import Registry
|
||||
from scepter.utils.config import Config
|
||||
from scepter.modules.utils.registry import Registry
|
||||
from scepter.modules.utils.config import Config
|
||||
|
||||
MODELS = Registry('MODELS')
|
||||
|
||||
@@ -614,14 +614,14 @@ Register a function
|
||||
**Returns**
|
||||
- **name** —— Registration name.
|
||||
|
||||
## 9. Data SDK(scepter.utils.data)
|
||||
## 9. Data SDK(scepter.modules.utils.data)
|
||||
Used for transferring data between devices
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
import torch
|
||||
from scepter.utils.data import transfer_data_to_numpy, transfer_data_to_cpu, transfer_data_to_cuda
|
||||
from scepter.modules.utils.data import transfer_data_to_numpy, transfer_data_to_cpu, transfer_data_to_cuda
|
||||
|
||||
data = {"a": torch.Tensor([0])}
|
||||
transfer_data_to_numpy(data)
|
||||
@@ -668,7 +668,7 @@ Used for operations such as loading and evaluating models
|
||||
|
||||
```python
|
||||
import torch
|
||||
from scepter.utils.model import move_model_to_cpu, load_pretrained,
|
||||
from scepter.modules.utils.model import move_model_to_cpu, load_pretrained,
|
||||
count_params, init_weights
|
||||
```
|
||||
<hr/>
|
||||
@@ -716,14 +716,14 @@ Initialize the parameters of the model modules.
|
||||
**Parameters**
|
||||
- **module** —— The torch.nn.Module model instance.
|
||||
|
||||
## 11. Sampler SDK(scepter.utils.sampler)
|
||||
## 11. Sampler SDK(scepter.modules.utils.sampler)
|
||||
Samplers are quite universal, and in most cases, custom development is not required. Here are provided several common types of sampler.
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
import torch
|
||||
from scepter.utils.sampler import MultiFoldDistributedSampler,
|
||||
from scepter.modules.utils.sampler import MultiFoldDistributedSampler,
|
||||
EvalDistributedSampler, MultiLevelBatchSampler, MixtureOfSamplers
|
||||
```
|
||||
<hr/>
|
||||
@@ -830,17 +830,17 @@ A sampler for multi-level indexing of large-scale data.
|
||||
|
||||
Iterator, each iteration returns an index of a sample.
|
||||
|
||||
## 12. Prober SDK(scepter.utils.probe)
|
||||
## 12. Prober SDK(scepter.modules.utils.probe)
|
||||
Used for probing variable statistics of various components.
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
from scepter.model.base_model import BaseModel
|
||||
from scepter.utils.config import Config
|
||||
from scepter.utils.file_system import FS
|
||||
from scepter.utils.probe import ProbeData
|
||||
from scepter.modules.model.base_model import BaseModel
|
||||
from scepter.modules.utils.config import Config
|
||||
from scepter.modules.utils.file_system import FS
|
||||
from scepter.modules.utils.probe import ProbeData
|
||||
|
||||
|
||||
class TestModel(BaseModel):
|
||||
|
||||
+17
-17
@@ -15,8 +15,8 @@
|
||||
子类注册：
|
||||
|
||||
```python
|
||||
from scepter.model.registry import BACKBONES
|
||||
from scepter.model.base_model import BaseModel
|
||||
from scepter.modules.model.registry import BACKBONES
|
||||
from scepter.modules.model.base_model import BaseModel
|
||||
|
||||
|
||||
@BACKBONES.register_class("ResNet")
|
||||
@@ -26,8 +26,8 @@ class ResNet(BaseModel):
|
||||
```
|
||||
|
||||
```python
|
||||
from scepter.model.registry import NECKS
|
||||
from scepter.model.base_model import BaseModel
|
||||
from scepter.modules.model.registry import NECKS
|
||||
from scepter.modules.model.base_model import BaseModel
|
||||
|
||||
|
||||
@NECKS.register_class()
|
||||
@@ -37,8 +37,8 @@ class GlobalAveragePooling(BaseModel):
|
||||
```
|
||||
|
||||
```python
|
||||
from scepter.model.registry import HEADS
|
||||
from scepter.model.base_model import BaseModel
|
||||
from scepter.modules.model.registry import HEADS
|
||||
from scepter.modules.model.base_model import BaseModel
|
||||
|
||||
|
||||
@HEADS.register_class()
|
||||
@@ -48,7 +48,7 @@ class ClassifierHead(BaseModel):
|
||||
```
|
||||
|
||||
```python
|
||||
from scepter.model.registry import LOSSES
|
||||
from scepter.modules.model.registry import LOSSES
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
@@ -60,7 +60,7 @@ class CrossEntropy(nn.Module):
|
||||
实际调用：
|
||||
|
||||
```python
|
||||
from scepter.model.registry import BACKBONES, NECKS, HEADS, LOSSES, TUNERS
|
||||
from scepter.modules.model.registry import BACKBONES, NECKS, HEADS, LOSSES, TUNERS
|
||||
|
||||
backbone = BACKBONES.build(cfg.BACKBONE, logger=logger)
|
||||
neck = NECKS.build(cfg.NECK, logger=logger)
|
||||
@@ -85,8 +85,8 @@ tuner = TUNERS.build(cfg.TUNER, logger=logger)
|
||||
子类注册：
|
||||
|
||||
```python
|
||||
from scepter.model.metrics.registry import METRICS
|
||||
from scepter.model.metrics.base_metric import BaseMetric
|
||||
from scepter.modules.model.metrics.registry import METRICS
|
||||
from scepter.modules.model.metrics.base_metric import BaseMetric
|
||||
|
||||
|
||||
@METRICS.register_class("AccuracyMetric")
|
||||
@@ -97,7 +97,7 @@ class AccuracyMetric(BaseMetric):
|
||||
实际用法：
|
||||
|
||||
```python
|
||||
from scepter.model.metrics.registry import METRICS
|
||||
from scepter.modules.model.metrics.registry import METRICS
|
||||
|
||||
metric = METRICS.build(cfgs, logger)
|
||||
```
|
||||
@@ -119,8 +119,8 @@ metric = METRICS.build(cfgs, logger)
|
||||
子类注册：
|
||||
|
||||
```python
|
||||
from scepter.model.registry import TOKENIZERS
|
||||
from scepter.model.tokenizers import BaseTokenizer
|
||||
from scepter.modules.model.registry import TOKENIZERS
|
||||
from scepter.modules.model.tokenizers import BaseTokenizer
|
||||
|
||||
|
||||
@TOKENIZERS.register_class()
|
||||
@@ -131,7 +131,7 @@ class BaseBertTokenizer(BaseTokenizer):
|
||||
实际用法：
|
||||
|
||||
```python
|
||||
from scepter.model.registry import TOKENIZERS
|
||||
from scepter.modules.model.registry import TOKENIZERS
|
||||
|
||||
tokenizer = TOKENIZERS.build(cfgs, logger)
|
||||
```
|
||||
@@ -149,8 +149,8 @@ tokenizer = TOKENIZERS.build(cfgs, logger)
|
||||
子类注册：
|
||||
|
||||
```python
|
||||
from scepter.model.registry import MODELS
|
||||
from scepter.model.networks.train_module import TrainModule
|
||||
from scepter.modules.model.registry import MODELS
|
||||
from scepter.modules.model.networks.train_module import TrainModule
|
||||
|
||||
|
||||
@MODELS.register_class()
|
||||
@@ -161,7 +161,7 @@ class Classifier(TrainModule):
|
||||
实际用法：
|
||||
|
||||
```python
|
||||
from scepter.model.registry import MODELS
|
||||
from scepter.modules.model.registry import MODELS
|
||||
|
||||
model = MODELS.build(self.cfg.MODEL, logger=self.logger)
|
||||
```
|
||||
|
||||
@@ -9,8 +9,8 @@
|
||||
子lr_schedulers继承时用法：
|
||||
|
||||
```python
|
||||
from scepter.opt.lr_schedulers import LR_SCHEDULERS
|
||||
from scepter.opt.lr_schedulers.base_scheduler import BaseScheduler
|
||||
from scepter.modules.opt.lr_schedulers import LR_SCHEDULERS
|
||||
from scepter.modules.opt.lr_schedulers.base_scheduler import BaseScheduler
|
||||
|
||||
|
||||
@LR_SCHEDULERS.register_class()
|
||||
@@ -48,8 +48,8 @@ lr_schedulers的基类，支持注册操作，可根据需要自定义；
|
||||
子optimizers继承时用法：
|
||||
|
||||
```python
|
||||
from scepter.opt.optimizers.base_optimizer import BaseOptimize
|
||||
from scepter.opt.optimizers.registry import OPTIMIZERS
|
||||
from scepter.modules.opt.optimizers.base_optimizer import BaseOptimize
|
||||
from scepter.modules.opt.optimizers.registry import OPTIMIZERS
|
||||
|
||||
|
||||
@OPTIMIZERS.register_class()
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
|
||||
支持3类文件IO Handler:
|
||||
|
||||
1. scepter.utils.file_clients.AliyunOssFs
|
||||
2. scepter.utils.file_clients.LocalFs
|
||||
3. scepter.utils.file_clients.HttpFs
|
||||
4. scepter.utils.file_clients.ModelscopeFs
|
||||
1. scepter.modules.utils.file_clients.AliyunOssFs
|
||||
2. scepter.modules.utils.file_clients.LocalFs
|
||||
3. scepter.modules.utils.file_clients.HttpFs
|
||||
4. scepter.modules.utils.file_clients.ModelscopeFs
|
||||
|
||||
|
||||
<hr/>
|
||||
@@ -17,8 +17,8 @@
|
||||
## 基础用法
|
||||
|
||||
```python
|
||||
from scepter.utils.file_system import FS
|
||||
from scepter.utils.config import Config
|
||||
from scepter.modules.utils.file_system import FS
|
||||
from scepter.modules.utils.config import Config
|
||||
|
||||
fs_cfg = Config(load=False, cfg_dict={
|
||||
"NAME": "AliyunOssFs",
|
||||
|
||||
@@ -3,18 +3,18 @@
|
||||
依赖SDK，该部分用于对框架全局经常复用的模块和sdk进行整理，并根据功能相关性进行聚合。
|
||||
|
||||
## 总览
|
||||
1. 参数sdk(scepter.utils.config)
|
||||
2. 路径sdk(scepter.utils.directory)
|
||||
3. torch分布式sdk(scepter.utils.distribute)
|
||||
4. 模型导出sdk(scepter.utils.export_model)
|
||||
5. 文件系统sdk(scepter.utils.file_system)
|
||||
6. 日志sdk(scepter.utils.logger)
|
||||
7. 视频处理sdk(scepter.utils.video_reader)，文档参考(video_reader.md)
|
||||
8. 模块注册sdk(scepter.utils.registry)
|
||||
9. 数据sdk(scepter.utils.data)
|
||||
10. 模型sdk(scepter.utils.model)
|
||||
11. 采样器sdk(scepter.utils.sampler)
|
||||
12. 探针器sdk(scepter.utils.probe)
|
||||
1. 参数sdk(scepter.modules.utils.config)
|
||||
2. 路径sdk(scepter.modules.utils.directory)
|
||||
3. torch分布式sdk(scepter.modules.utils.distribute)
|
||||
4. 模型导出sdk(scepter.modules.utils.export_model)
|
||||
5. 文件系统sdk(scepter.modules.utils.file_system)
|
||||
6. 日志sdk(scepter.modules.utils.logger)
|
||||
7. 视频处理sdk(scepter.modules.utils.video_reader)，文档参考(video_reader.md)
|
||||
8. 模块注册sdk(scepter.modules.utils.registry)
|
||||
9. 数据sdk(scepter.modules.utils.data)
|
||||
10. 模型sdk(scepter.modules.utils.model)
|
||||
11. 采样器sdk(scepter.modules.utils.sampler)
|
||||
12. 探针器sdk(scepter.modules.utils.probe)
|
||||
|
||||
<hr/>
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
from scepter.utils.config import Config
|
||||
from scepter.modules.utils.config import Config
|
||||
|
||||
# 从一个dict对象 初始化 Config对象
|
||||
fs_cfg = Config(load=False, cfg_dict={"NAME": "LocalFs"})
|
||||
@@ -97,7 +97,7 @@ print(fs_cfg.args)
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
from scepter.utils.directory import osp_path
|
||||
from scepter.modules.utils.directory import osp_path
|
||||
|
||||
# 根据路径前缀进行自动化路径拼接
|
||||
prefix = "xxxx"
|
||||
@@ -108,7 +108,7 @@ print(osp_path(prefix, data_file))
|
||||
data_file = "xxxx/example_videos/1.mp4"
|
||||
print(osp_path(prefix, data_file))
|
||||
|
||||
from scepter.utils.directory import get_relative_folder
|
||||
from scepter.modules.utils.directory import get_relative_folder
|
||||
|
||||
# 根据路径获取指定层级的文件夹路径
|
||||
# 默认最后一级 xxxx/example_videos/
|
||||
@@ -116,7 +116,7 @@ print(get_relative_folder(data_file))
|
||||
# 倒数第二级 xxxx/
|
||||
print(get_relative_folder(data_file, keep_index=-2))
|
||||
|
||||
from scepter.utils.directory import get_md5
|
||||
from scepter.modules.utils.directory import get_md5
|
||||
|
||||
# 获取文本/路径的md5码 34a447fb46d0b786a3999c9dad01d470
|
||||
print(get_md5(data_file))
|
||||
@@ -172,8 +172,8 @@ torch分布式初始化sdk，使用该sdk，可以让用户不要关注torch的
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
from scepter.utils.distribute import we
|
||||
from scepter.utils.config import Config
|
||||
from scepter.modules.utils.distribute import we
|
||||
from scepter.modules.utils.config import Config
|
||||
|
||||
cfg = Config(cfg_dict={}, load=False)
|
||||
|
||||
@@ -304,12 +304,12 @@ we.init_env(cfg, fn, logger=None)
|
||||
**Returns**
|
||||
- **tensor** —— 输出的在进程rank=0上的cpu的tensor。
|
||||
|
||||
## 4. 模型导出sdk(scepter.utils.export_model)
|
||||
## 4. 模型导出sdk(scepter.modules.utils.export_model)
|
||||
用于模型导出为torchscript/Onnx格式的api。
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
from scepter.utils.export_model import save_develop_model_multi_io
|
||||
from scepter.modules.utils.export_model import save_develop_model_multi_io
|
||||
|
||||
save_develop_model_multi_io(
|
||||
model,
|
||||
@@ -347,16 +347,16 @@ input_type 一一对应。
|
||||
**Returns**
|
||||
- **tensor** —— 输出的在进程rank=0上的cpu的tensor。
|
||||
|
||||
## 5. 文件系统sdk(scepter.utils.file_system)
|
||||
## 5. 文件系统sdk(scepter.modules.utils.file_system)
|
||||
参考[file_clients](file_clients.md)
|
||||
|
||||
## 6. 日志sdk(scepter.utils.logger)
|
||||
## 6. 日志sdk(scepter.modules.utils.logger)
|
||||
用于实例化一个标准的日志实例，用于打印信息。
|
||||
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
from scepter.utils.logger import get_logger, init_logger
|
||||
from scepter.modules.utils.logger import get_logger, init_logger
|
||||
|
||||
std_logger = get_logger(name="scepter")
|
||||
init_logger(std_logger, log_file="", dist_launcher="pytorch")
|
||||
@@ -407,14 +407,14 @@ init_logger(std_logger, log_file="", dist_launcher="pytorch")
|
||||
**Returns**
|
||||
- **str** —— 格式化的输出。
|
||||
|
||||
## 7. 视频处理sdk(scepter.utils.video_reader)
|
||||
## 7. 视频处理sdk(scepter.modules.utils.video_reader)
|
||||
用于处理视频读取的api。
|
||||
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
from scepter.utils.video_reader.frame_sampler import do_frame_sample
|
||||
from scepter.utils.video_reader.video_reader import (
|
||||
from scepter.modules.utils.video_reader.frame_sampler import do_frame_sample
|
||||
from scepter.modules.utils.video_reader.video_reader import (
|
||||
VideoReaderWrapper, EasyVideoReader, FramesReaderWrapper
|
||||
)
|
||||
```
|
||||
@@ -556,14 +556,14 @@ overlap: Union[float, Fraction, str] = Fraction(0), transforms: Optional[Callabl
|
||||
**Returns**
|
||||
- **tensor** —— 视频片段的tensor。
|
||||
|
||||
## 8. 模块注册sdk(scepter.utils.registry)
|
||||
## 8. 模块注册sdk(scepter.modules.utils.registry)
|
||||
用于管理各种注册的类。
|
||||
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
from scepter.utils.registry import Registry
|
||||
from scepter.utils.config import Config
|
||||
from scepter.modules.utils.registry import Registry
|
||||
from scepter.modules.utils.config import Config
|
||||
|
||||
MODELS = Registry('MODELS')
|
||||
|
||||
@@ -616,14 +616,14 @@ build目标类的实例
|
||||
**Returns**
|
||||
- **name** —— 注册名称。
|
||||
|
||||
## 9. 数据sdk(scepter.utils.data)
|
||||
## 9. 数据sdk(scepter.modules.utils.data)
|
||||
用于数据在设备间转移
|
||||
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
import torch
|
||||
from scepter.utils.data import transfer_data_to_numpy, transfer_data_to_cpu, transfer_data_to_cuda
|
||||
from scepter.modules.utils.data import transfer_data_to_numpy, transfer_data_to_cpu, transfer_data_to_cuda
|
||||
|
||||
data = {"a": torch.Tensor([0])}
|
||||
transfer_data_to_numpy(data)
|
||||
@@ -670,7 +670,7 @@ transfer_data_to_cuda(data)
|
||||
|
||||
```python
|
||||
import torch
|
||||
from scepter.utils.model import move_model_to_cpu, load_pretrained,
|
||||
from scepter.modules.utils.model import move_model_to_cpu, load_pretrained,
|
||||
count_params, init_weights
|
||||
```
|
||||
<hr/>
|
||||
@@ -718,14 +718,14 @@ from scepter.utils.model import move_model_to_cpu, load_pretrained,
|
||||
**Parameters**
|
||||
- **module** —— torch.nn.Module模型实例。
|
||||
|
||||
## 11. 采样器sdk(scepter.utils.sampler)
|
||||
## 11. 采样器sdk(scepter.modules.utils.sampler)
|
||||
采样器比较具有通用性，大多数情况下不会进行定制开发，这里提供了几类常用的sampler采样器。
|
||||
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
import torch
|
||||
from scepter.utils.sampler import MultiFoldDistributedSampler,
|
||||
from scepter.modules.utils.sampler import MultiFoldDistributedSampler,
|
||||
EvalDistributedSampler, MultiLevelBatchSampler, MixtureOfSamplers
|
||||
```
|
||||
<hr/>
|
||||
@@ -832,17 +832,17 @@ from scepter.utils.sampler import MultiFoldDistributedSampler,
|
||||
|
||||
迭代器，每迭代一次得到一个样本的index
|
||||
|
||||
## 12. 探针器sdk(scepter.utils.probe)
|
||||
## 12. 探针器sdk(scepter.modules.utils.probe)
|
||||
用于探针各个组件的变量统计
|
||||
|
||||
### 基础用法
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
from scepter.model.base_model import BaseModel
|
||||
from scepter.utils.config import Config
|
||||
from scepter.utils.file_system import FS
|
||||
from scepter.utils.probe import ProbeData
|
||||
from scepter.modules.model.base_model import BaseModel
|
||||
from scepter.modules.utils.config import Config
|
||||
from scepter.modules.utils.file_system import FS
|
||||
from scepter.modules.utils.probe import ProbeData
|
||||
|
||||
|
||||
class TestModel(BaseModel):
|
||||
|
||||
Reference in New Issue
Block a user