diff --git a/asset/images/stylebooth/graffiti.jpeg b/asset/images/stylebooth/graffiti.jpeg new file mode 100644 index 0000000..f95b95e Binary files /dev/null and b/asset/images/stylebooth/graffiti.jpeg differ diff --git a/asset/images/stylebooth/kawaii.jpeg b/asset/images/stylebooth/kawaii.jpeg new file mode 100644 index 0000000..8f69857 Binary files /dev/null and b/asset/images/stylebooth/kawaii.jpeg differ diff --git a/asset/images/stylebooth/retrogame.jpeg b/asset/images/stylebooth/retrogame.jpeg new file mode 100644 index 0000000..0197c6f Binary files /dev/null and b/asset/images/stylebooth/retrogame.jpeg differ diff --git a/asset/images/stylebooth/vangogh.jpeg b/asset/images/stylebooth/vangogh.jpeg new file mode 100644 index 0000000..5d319d0 Binary files /dev/null and b/asset/images/stylebooth/vangogh.jpeg differ diff --git a/docs/en/scepter/model.md b/docs/en/scepter/model.md index 9ff4287..2554c8f 100644 --- a/docs/en/scepter/model.md +++ b/docs/en/scepter/model.md @@ -14,8 +14,8 @@ Model modules are divided into backbones, necks, heads, loss, metrics, networks, Subclass registration: ```python -from scepter.model.registry import BACKBONES -from scepter.model.base_model import BaseModel +from scepter.modules.model.registry import BACKBONES +from scepter.modules.model.base_model import BaseModel @BACKBONES.register_class("ResNet") @@ -25,8 +25,8 @@ class ResNet(BaseModel): ``` ```python -from scepter.model.registry import NECKS -from scepter.model.base_model import BaseModel +from scepter.modules.model.registry import NECKS +from scepter.modules.model.base_model import BaseModel @NECKS.register_class() @@ -36,8 +36,8 @@ class GlobalAveragePooling(BaseModel): ``` ```python -from scepter.model.registry import HEADS -from scepter.model.base_model import BaseModel +from scepter.modules.model.registry import HEADS +from scepter.modules.model.base_model import BaseModel @HEADS.register_class() @@ -47,7 +47,7 @@ class ClassifierHead(BaseModel): ``` ```python -from scepter.model.registry import LOSSES +from scepter.modules.model.registry import LOSSES import torch.nn as nn @@ -59,7 +59,7 @@ class CrossEntropy(nn.Module): Actual usage: ```python -from scepter.model.registry import BACKBONES, NECKS, HEADS, LOSSES +from scepter.modules.model.registry import BACKBONES, NECKS, HEADS, LOSSES backbone = BACKBONES.build(cfg.BACKBONE, logger=logger) neck = NECKS.build(cfg.NECK, logger=logger) @@ -83,8 +83,8 @@ To be implemented specifically as needed; Basic Usage Subclass registration: ```python -from scepter.model.metrics.registry import METRICS -from scepter.model.metrics.base_metric import BaseMetric +from scepter.modules.model.metrics.registry import METRICS +from scepter.modules.model.metrics.base_metric import BaseMetric @METRICS.register_class("AccuracyMetric") @@ -95,7 +95,7 @@ class AccuracyMetric(BaseMetric): Actual usage: ```python -from scepter.model.metrics.registry import METRICS +from scepter.modules.model.metrics.registry import METRICS metric = METRICS.build(cfgs, logger) ``` @@ -117,8 +117,8 @@ Typically takes logits and labels as well as other necessary variables as inputs Subclass registration: ```python -from scepter.model.registry import TOKENIZERS -from scepter.model.tokenizers import BaseTokenizer +from scepter.modules.model.registry import TOKENIZERS +from scepter.modules.model.tokenizers import BaseTokenizer @TOKENIZERS.register_class() @@ -129,7 +129,7 @@ class BaseBertTokenizer(BaseTokenizer): Actual usage: ```python -from scepter.model.registry import TOKENIZERS +from scepter.modules.model.registry import TOKENIZERS tokenizer = TOKENIZERS.build(cfgs, logger) ``` @@ -147,8 +147,8 @@ Takes a list of texts that need tokenization as input and outputs token id seque Subclass registration: ```python -from scepter.model.registry import MODELS -from scepter.model.networks.train_module import TrainModule +from scepter.modules.model.registry import MODELS +from scepter.modules.model.networks.train_module import TrainModule @MODELS.register_class() @@ -159,7 +159,7 @@ class Classifier(TrainModule): Actual usage: ```python -from scepter.model.registry import MODELS +from scepter.modules.model.registry import MODELS model = MODELS.build(self.cfg.MODEL, logger=self.logger) ``` diff --git a/docs/en/scepter/opt.md b/docs/en/scepter/opt.md index a986393..e5c4391 100644 --- a/docs/en/scepter/opt.md +++ b/docs/en/scepter/opt.md @@ -9,8 +9,8 @@ Usage when subclassing lr_schedulers: ```python -from scepter.opt.lr_schedulers import LR_SCHEDULERS -from scepter.opt.lr_schedulers.base_scheduler import BaseScheduler +from scepter.modules.opt.lr_schedulers import LR_SCHEDULERS +from scepter.modules.opt.lr_schedulers.base_scheduler import BaseScheduler @LR_SCHEDULERS.register_class() @@ -48,8 +48,8 @@ Sets up the schedule for the passed-in optimizer object; Usage when subclassing optimizers: ```python -from scepter.opt.optimizers.base_optimizer import BaseOptimize -from scepter.opt.optimizers.registry import OPTIMIZERS +from scepter.modules.opt.optimizers.base_optimizer import BaseOptimize +from scepter.modules.opt.optimizers.registry import OPTIMIZERS @OPTIMIZERS.register_class() diff --git a/docs/en/scepter/utils/file_clients.md b/docs/en/scepter/utils/file_clients.md index 78a5d87..024b39f 100644 --- a/docs/en/scepter/utils/file_clients.md +++ b/docs/en/scepter/utils/file_clients.md @@ -6,17 +6,17 @@ This is the File System Module, designed to handle file transfer functionalities The component currently supports three types of IO Handler: -1. scepter.utils.file_clients.AliyunOssFs -2. scepter.utils.file_clients.LocalFs -3. scepter.utils.file_clients.HttpFs +1. scepter.modules.utils.file_clients.AliyunOssFs +2. scepter.modules.utils.file_clients.LocalFs +3. scepter.modules.utils.file_clients.HttpFs
## Basic Usage ```python -from scepter.utils.file_system import FS -from scepter.utils.config import Config +from scepter.modules.utils.file_system import FS +from scepter.modules.utils.config import Config fs_cfg = Config(load=False, cfg_dict={ "NAME": "AliyunOssFs", diff --git a/docs/en/scepter/utils/utils.md b/docs/en/scepter/utils/utils.md index ef99523..42a628d 100644 --- a/docs/en/scepter/utils/utils.md +++ b/docs/en/scepter/utils/utils.md @@ -4,18 +4,18 @@ Relies on SDKs, which are used to organize modules and SDKs that are frequently ## Overview -1. Parameter sdk (scepter.utils.config) -2. Path sdk (scepter.utils.directory) -3. PyTorch distributed sdk (scepter.utils.distribute) -4. Model export sdk (scepter.utils.export_model) -5. File system sdk (scepter.utils.file_system) -6. Logging sdk (scepter.utils.logger) -7. Video processing sdk (scepter.utils.video_reader), see the document (video_reader.md) -8. Module registration sdk (scepter.utils.registry) -9. Data sdk (scepter.utils.data) -10. Model sdk (scepter.utils.model) -11. Sampler sdk (scepter.utils.sampler) -12. Probing sdk (scepter.utils.probe) +1. Parameter sdk (scepter.modules.utils.config) +2. Path sdk (scepter.modules.utils.directory) +3. PyTorch distributed sdk (scepter.modules.utils.distribute) +4. Model export sdk (scepter.modules.utils.export_model) +5. File system sdk (scepter.modules.utils.file_system) +6. Logging sdk (scepter.modules.utils.logger) +7. Video processing sdk (scepter.modules.utils.video_reader), see the document (video_reader.md) +8. Module registration sdk (scepter.modules.utils.registry) +9. Data sdk (scepter.modules.utils.data) +10. Model sdk (scepter.modules.utils.model) +11. Sampler sdk (scepter.modules.utils.sampler) +12. Probing sdk (scepter.modules.utils.probe)
@@ -24,7 +24,7 @@ Relies on SDKs, which are used to organize modules and SDKs that are frequently ### Basic Usage ```python -from scepter.utils.config import Config +from scepter.modules.utils.config import Config # Initialize Config object from a dict fs_cfg = Config(load=False, cfg_dict={"NAME": "LocalFs"}) print(fs_cfg.NAME) @@ -105,7 +105,7 @@ print(fs_cfg.args) Some commonly used path functions ### Basic Usage ```python -from scepter.utils.directory import osp_path +from scepter.modules.utils.directory import osp_path # Automatically join paths based on the path prefix prefix = "xxxx" data_file = "example_videos/1.mp4" @@ -114,13 +114,13 @@ print(osp_path(prefix, data_file)) # Also outputs as xxxx/example_videos/1.mp4 data_file = "xxxx/example_videos/1.mp4" print(osp_path(prefix, data_file)) -from scepter.utils.directory import get_relative_folder +from scepter.modules.utils.directory import get_relative_folder # Get the folder path at a specified level according to the path # By default, the last level xxxx/example_videos/ print(get_relative_folder(data_file)) # The second last level xxxx/ print(get_relative_folder(data_file, keep_index=-2)) -from scepter.utils.directory import get_md5 +from scepter.modules.utils.directory import get_md5 # Get the md5 code of the text/path 34a447fb46d0b786a3999c9dad01d470 print(get_md5(data_file)) ``` @@ -175,8 +175,8 @@ PyTorch distributed initialization SDK. By using this SDK, users can avoid focus ### Basic Usage ```python -from scepter.utils.distribute import we -from scepter.utils.config import Config +from scepter.modules.utils.distribute import we +from scepter.modules.utils.config import Config cfg = Config(cfg_dict={}, load=False) @@ -304,12 +304,12 @@ Since cloning is involved, this may cause additional GPU memory waste. **Returns** - **tensor** —— The output tensor on the CPU for process rank=0. -## 4. 模型导出sdk(scepter.utils.export_model) +## 4. 模型导出sdk(scepter.modules.utils.export_model) APIs for exporting models to TorchScript/ONNX formats. ### Basic Usage ```python -from scepter.utils.export_model import save_develop_model_multi_io +from scepter.modules.utils.export_model import save_develop_model_multi_io save_develop_model_multi_io( model, @@ -345,16 +345,16 @@ Supports importing and exporting models with multiple inputs and outputs **Returns** - **tensor** —— The output tensor on the CPU for process rank=0. -## 5. 文件系统sdk(scepter.utils.file_system) +## 5. 文件系统sdk(scepter.modules.utils.file_system) Refer to [file_clients](file_clients.md) -## 6. Logging SDK(scepter.utils.logger) +## 6. Logging SDK(scepter.modules.utils.logger) Used to instantiate a standard logging instance for printing information. ### Basic Usage ```python -from scepter.utils.logger import get_logger, init_logger +from scepter.modules.utils.logger import get_logger, init_logger std_logger = get_logger(name="scepter") init_logger(std_logger, log_file="", dist_launcher="pytorch") @@ -405,14 +405,14 @@ Calculate the time remaining until completion based on the current usage time an **Returns** - **str** —— Formatted output. -## 7. Video Processing SDK (scepter.utils.video_reader) +## 7. Video Processing SDK (scepter.modules.utils.video_reader) APIs for handling video reading. ### Basic Usage ```python -from scepter.utils.video_reader.frame_sampler import do_frame_sample -from scepter.utils.video_reader.video_reader import ( +from scepter.modules.utils.video_reader.frame_sampler import do_frame_sample +from scepter.modules.utils.video_reader.video_reader import ( VideoReaderWrapper, EasyVideoReader, FramesReaderWrapper ) ``` @@ -554,14 +554,14 @@ Iterator, with each iteration returning a tensor of a segment. **Returns** - **tensor** —— The tensor of the video segment. -## 8. Module Registration SDK (scepter.utils.registry) +## 8. Module Registration SDK (scepter.modules.utils.registry) Used for managing various registered classes. ### Basic Usage ```python -from scepter.utils.registry import Registry -from scepter.utils.config import Config +from scepter.modules.utils.registry import Registry +from scepter.modules.utils.config import Config MODELS = Registry('MODELS') @@ -614,14 +614,14 @@ Register a function **Returns** - **name** —— Registration name. -## 9. Data SDK(scepter.utils.data) +## 9. Data SDK(scepter.modules.utils.data) Used for transferring data between devices ### Basic Usage ```python import torch -from scepter.utils.data import transfer_data_to_numpy, transfer_data_to_cpu, transfer_data_to_cuda +from scepter.modules.utils.data import transfer_data_to_numpy, transfer_data_to_cpu, transfer_data_to_cuda data = {"a": torch.Tensor([0])} transfer_data_to_numpy(data) @@ -668,7 +668,7 @@ Used for operations such as loading and evaluating models ```python import torch -from scepter.utils.model import move_model_to_cpu, load_pretrained, +from scepter.modules.utils.model import move_model_to_cpu, load_pretrained, count_params, init_weights ```
@@ -716,14 +716,14 @@ Initialize the parameters of the model modules. **Parameters** - **module** —— The torch.nn.Module model instance. -## 11. Sampler SDK(scepter.utils.sampler) +## 11. Sampler SDK(scepter.modules.utils.sampler) Samplers are quite universal, and in most cases, custom development is not required. Here are provided several common types of sampler. ### Basic Usage ```python import torch -from scepter.utils.sampler import MultiFoldDistributedSampler, +from scepter.modules.utils.sampler import MultiFoldDistributedSampler, EvalDistributedSampler, MultiLevelBatchSampler, MixtureOfSamplers ```
@@ -830,17 +830,17 @@ A sampler for multi-level indexing of large-scale data. Iterator, each iteration returns an index of a sample. -## 12. Prober SDK(scepter.utils.probe) +## 12. Prober SDK(scepter.modules.utils.probe) Used for probing variable statistics of various components. ### Basic Usage ```python import numpy as np -from scepter.model.base_model import BaseModel -from scepter.utils.config import Config -from scepter.utils.file_system import FS -from scepter.utils.probe import ProbeData +from scepter.modules.model.base_model import BaseModel +from scepter.modules.utils.config import Config +from scepter.modules.utils.file_system import FS +from scepter.modules.utils.probe import ProbeData class TestModel(BaseModel): diff --git a/docs/zh_cn/scepter/model.md b/docs/zh_cn/scepter/model.md index 213453f..1a587dc 100644 --- a/docs/zh_cn/scepter/model.md +++ b/docs/zh_cn/scepter/model.md @@ -15,8 +15,8 @@ 子类注册: ```python -from scepter.model.registry import BACKBONES -from scepter.model.base_model import BaseModel +from scepter.modules.model.registry import BACKBONES +from scepter.modules.model.base_model import BaseModel @BACKBONES.register_class("ResNet") @@ -26,8 +26,8 @@ class ResNet(BaseModel): ``` ```python -from scepter.model.registry import NECKS -from scepter.model.base_model import BaseModel +from scepter.modules.model.registry import NECKS +from scepter.modules.model.base_model import BaseModel @NECKS.register_class() @@ -37,8 +37,8 @@ class GlobalAveragePooling(BaseModel): ``` ```python -from scepter.model.registry import HEADS -from scepter.model.base_model import BaseModel +from scepter.modules.model.registry import HEADS +from scepter.modules.model.base_model import BaseModel @HEADS.register_class() @@ -48,7 +48,7 @@ class ClassifierHead(BaseModel): ``` ```python -from scepter.model.registry import LOSSES +from scepter.modules.model.registry import LOSSES import torch.nn as nn @@ -60,7 +60,7 @@ class CrossEntropy(nn.Module): 实际调用: ```python -from scepter.model.registry import BACKBONES, NECKS, HEADS, LOSSES, TUNERS +from scepter.modules.model.registry import BACKBONES, NECKS, HEADS, LOSSES, TUNERS backbone = BACKBONES.build(cfg.BACKBONE, logger=logger) neck = NECKS.build(cfg.NECK, logger=logger) @@ -85,8 +85,8 @@ tuner = TUNERS.build(cfg.TUNER, logger=logger) 子类注册: ```python -from scepter.model.metrics.registry import METRICS -from scepter.model.metrics.base_metric import BaseMetric +from scepter.modules.model.metrics.registry import METRICS +from scepter.modules.model.metrics.base_metric import BaseMetric @METRICS.register_class("AccuracyMetric") @@ -97,7 +97,7 @@ class AccuracyMetric(BaseMetric): 实际用法: ```python -from scepter.model.metrics.registry import METRICS +from scepter.modules.model.metrics.registry import METRICS metric = METRICS.build(cfgs, logger) ``` @@ -119,8 +119,8 @@ metric = METRICS.build(cfgs, logger) 子类注册: ```python -from scepter.model.registry import TOKENIZERS -from scepter.model.tokenizers import BaseTokenizer +from scepter.modules.model.registry import TOKENIZERS +from scepter.modules.model.tokenizers import BaseTokenizer @TOKENIZERS.register_class() @@ -131,7 +131,7 @@ class BaseBertTokenizer(BaseTokenizer): 实际用法: ```python -from scepter.model.registry import TOKENIZERS +from scepter.modules.model.registry import TOKENIZERS tokenizer = TOKENIZERS.build(cfgs, logger) ``` @@ -149,8 +149,8 @@ tokenizer = TOKENIZERS.build(cfgs, logger) 子类注册: ```python -from scepter.model.registry import MODELS -from scepter.model.networks.train_module import TrainModule +from scepter.modules.model.registry import MODELS +from scepter.modules.model.networks.train_module import TrainModule @MODELS.register_class() @@ -161,7 +161,7 @@ class Classifier(TrainModule): 实际用法: ```python -from scepter.model.registry import MODELS +from scepter.modules.model.registry import MODELS model = MODELS.build(self.cfg.MODEL, logger=self.logger) ``` diff --git a/docs/zh_cn/scepter/opt.md b/docs/zh_cn/scepter/opt.md index c245b85..0d0c695 100644 --- a/docs/zh_cn/scepter/opt.md +++ b/docs/zh_cn/scepter/opt.md @@ -9,8 +9,8 @@ 子lr_schedulers继承时用法: ```python -from scepter.opt.lr_schedulers import LR_SCHEDULERS -from scepter.opt.lr_schedulers.base_scheduler import BaseScheduler +from scepter.modules.opt.lr_schedulers import LR_SCHEDULERS +from scepter.modules.opt.lr_schedulers.base_scheduler import BaseScheduler @LR_SCHEDULERS.register_class() @@ -48,8 +48,8 @@ lr_schedulers的基类,支持注册操作,可根据需要自定义; 子optimizers继承时用法: ```python -from scepter.opt.optimizers.base_optimizer import BaseOptimize -from scepter.opt.optimizers.registry import OPTIMIZERS +from scepter.modules.opt.optimizers.base_optimizer import BaseOptimize +from scepter.modules.opt.optimizers.registry import OPTIMIZERS @OPTIMIZERS.register_class() diff --git a/docs/zh_cn/scepter/utils/file_clients.md b/docs/zh_cn/scepter/utils/file_clients.md index b05cbc8..e70b98e 100644 --- a/docs/zh_cn/scepter/utils/file_clients.md +++ b/docs/zh_cn/scepter/utils/file_clients.md @@ -6,10 +6,10 @@ 支持3类文件IO Handler: -1. scepter.utils.file_clients.AliyunOssFs -2. scepter.utils.file_clients.LocalFs -3. scepter.utils.file_clients.HttpFs -4. scepter.utils.file_clients.ModelscopeFs +1. scepter.modules.utils.file_clients.AliyunOssFs +2. scepter.modules.utils.file_clients.LocalFs +3. scepter.modules.utils.file_clients.HttpFs +4. scepter.modules.utils.file_clients.ModelscopeFs
@@ -17,8 +17,8 @@ ## 基础用法 ```python -from scepter.utils.file_system import FS -from scepter.utils.config import Config +from scepter.modules.utils.file_system import FS +from scepter.modules.utils.config import Config fs_cfg = Config(load=False, cfg_dict={ "NAME": "AliyunOssFs", diff --git a/docs/zh_cn/scepter/utils/utils.md b/docs/zh_cn/scepter/utils/utils.md index 3140b43..0c1ee07 100644 --- a/docs/zh_cn/scepter/utils/utils.md +++ b/docs/zh_cn/scepter/utils/utils.md @@ -3,18 +3,18 @@ 依赖SDK,该部分用于对框架全局经常复用的模块和sdk进行整理,并根据功能相关性进行聚合。 ## 总览 -1. 参数sdk(scepter.utils.config) -2. 路径sdk(scepter.utils.directory) -3. torch分布式sdk(scepter.utils.distribute) -4. 模型导出sdk(scepter.utils.export_model) -5. 文件系统sdk(scepter.utils.file_system) -6. 日志sdk(scepter.utils.logger) -7. 视频处理sdk(scepter.utils.video_reader),文档参考(video_reader.md) -8. 模块注册sdk(scepter.utils.registry) -9. 数据sdk(scepter.utils.data) -10. 模型sdk(scepter.utils.model) -11. 采样器sdk(scepter.utils.sampler) -12. 探针器sdk(scepter.utils.probe) +1. 参数sdk(scepter.modules.utils.config) +2. 路径sdk(scepter.modules.utils.directory) +3. torch分布式sdk(scepter.modules.utils.distribute) +4. 模型导出sdk(scepter.modules.utils.export_model) +5. 文件系统sdk(scepter.modules.utils.file_system) +6. 日志sdk(scepter.modules.utils.logger) +7. 视频处理sdk(scepter.modules.utils.video_reader),文档参考(video_reader.md) +8. 模块注册sdk(scepter.modules.utils.registry) +9. 数据sdk(scepter.modules.utils.data) +10. 模型sdk(scepter.modules.utils.model) +11. 采样器sdk(scepter.modules.utils.sampler) +12. 探针器sdk(scepter.modules.utils.probe)
@@ -23,7 +23,7 @@ ### 基础用法 ```python -from scepter.utils.config import Config +from scepter.modules.utils.config import Config # 从一个dict对象 初始化 Config对象 fs_cfg = Config(load=False, cfg_dict={"NAME": "LocalFs"}) @@ -97,7 +97,7 @@ print(fs_cfg.args) ### 基础用法 ```python -from scepter.utils.directory import osp_path +from scepter.modules.utils.directory import osp_path # 根据路径前缀进行自动化路径拼接 prefix = "xxxx" @@ -108,7 +108,7 @@ print(osp_path(prefix, data_file)) data_file = "xxxx/example_videos/1.mp4" print(osp_path(prefix, data_file)) -from scepter.utils.directory import get_relative_folder +from scepter.modules.utils.directory import get_relative_folder # 根据路径获取指定层级的文件夹路径 # 默认最后一级 xxxx/example_videos/ @@ -116,7 +116,7 @@ print(get_relative_folder(data_file)) # 倒数第二级 xxxx/ print(get_relative_folder(data_file, keep_index=-2)) -from scepter.utils.directory import get_md5 +from scepter.modules.utils.directory import get_md5 # 获取文本/路径的md5码 34a447fb46d0b786a3999c9dad01d470 print(get_md5(data_file)) @@ -172,8 +172,8 @@ torch分布式初始化sdk,使用该sdk,可以让用户不要关注torch的 ### 基础用法 ```python -from scepter.utils.distribute import we -from scepter.utils.config import Config +from scepter.modules.utils.distribute import we +from scepter.modules.utils.config import Config cfg = Config(cfg_dict={}, load=False) @@ -304,12 +304,12 @@ we.init_env(cfg, fn, logger=None) **Returns** - **tensor** —— 输出的在进程rank=0上的cpu的tensor。 -## 4. 模型导出sdk(scepter.utils.export_model) +## 4. 模型导出sdk(scepter.modules.utils.export_model) 用于模型导出为torchscript/Onnx格式的api。 ### 基础用法 ```python -from scepter.utils.export_model import save_develop_model_multi_io +from scepter.modules.utils.export_model import save_develop_model_multi_io save_develop_model_multi_io( model, @@ -347,16 +347,16 @@ input_type 一一对应。 **Returns** - **tensor** —— 输出的在进程rank=0上的cpu的tensor。 -## 5. 文件系统sdk(scepter.utils.file_system) +## 5. 文件系统sdk(scepter.modules.utils.file_system) 参考[file_clients](file_clients.md) -## 6. 日志sdk(scepter.utils.logger) +## 6. 日志sdk(scepter.modules.utils.logger) 用于实例化一个标准的日志实例,用于打印信息。 ### 基础用法 ```python -from scepter.utils.logger import get_logger, init_logger +from scepter.modules.utils.logger import get_logger, init_logger std_logger = get_logger(name="scepter") init_logger(std_logger, log_file="", dist_launcher="pytorch") @@ -407,14 +407,14 @@ init_logger(std_logger, log_file="", dist_launcher="pytorch") **Returns** - **str** —— 格式化的输出。 -## 7. 视频处理sdk(scepter.utils.video_reader) +## 7. 视频处理sdk(scepter.modules.utils.video_reader) 用于处理视频读取的api。 ### 基础用法 ```python -from scepter.utils.video_reader.frame_sampler import do_frame_sample -from scepter.utils.video_reader.video_reader import ( +from scepter.modules.utils.video_reader.frame_sampler import do_frame_sample +from scepter.modules.utils.video_reader.video_reader import ( VideoReaderWrapper, EasyVideoReader, FramesReaderWrapper ) ``` @@ -556,14 +556,14 @@ overlap: Union[float, Fraction, str] = Fraction(0), transforms: Optional[Callabl **Returns** - **tensor** —— 视频片段的tensor。 -## 8. 模块注册sdk(scepter.utils.registry) +## 8. 模块注册sdk(scepter.modules.utils.registry) 用于管理各种注册的类。 ### 基础用法 ```python -from scepter.utils.registry import Registry -from scepter.utils.config import Config +from scepter.modules.utils.registry import Registry +from scepter.modules.utils.config import Config MODELS = Registry('MODELS') @@ -616,14 +616,14 @@ build目标类的实例 **Returns** - **name** —— 注册名称。 -## 9. 数据sdk(scepter.utils.data) +## 9. 数据sdk(scepter.modules.utils.data) 用于数据在设备间转移 ### 基础用法 ```python import torch -from scepter.utils.data import transfer_data_to_numpy, transfer_data_to_cpu, transfer_data_to_cuda +from scepter.modules.utils.data import transfer_data_to_numpy, transfer_data_to_cpu, transfer_data_to_cuda data = {"a": torch.Tensor([0])} transfer_data_to_numpy(data) @@ -670,7 +670,7 @@ transfer_data_to_cuda(data) ```python import torch -from scepter.utils.model import move_model_to_cpu, load_pretrained, +from scepter.modules.utils.model import move_model_to_cpu, load_pretrained, count_params, init_weights ```
@@ -718,14 +718,14 @@ from scepter.utils.model import move_model_to_cpu, load_pretrained, **Parameters** - **module** —— torch.nn.Module模型实例。 -## 11. 采样器sdk(scepter.utils.sampler) +## 11. 采样器sdk(scepter.modules.utils.sampler) 采样器比较具有通用性,大多数情况下不会进行定制开发,这里提供了几类常用的sampler采样器。 ### 基础用法 ```python import torch -from scepter.utils.sampler import MultiFoldDistributedSampler, +from scepter.modules.utils.sampler import MultiFoldDistributedSampler, EvalDistributedSampler, MultiLevelBatchSampler, MixtureOfSamplers ```
@@ -832,17 +832,17 @@ from scepter.utils.sampler import MultiFoldDistributedSampler, 迭代器,每迭代一次得到一个样本的index -## 12. 探针器sdk(scepter.utils.probe) +## 12. 探针器sdk(scepter.modules.utils.probe) 用于探针各个组件的变量统计 ### 基础用法 ```python import numpy as np -from scepter.model.base_model import BaseModel -from scepter.utils.config import Config -from scepter.utils.file_system import FS -from scepter.utils.probe import ProbeData +from scepter.modules.model.base_model import BaseModel +from scepter.modules.utils.config import Config +from scepter.modules.utils.file_system import FS +from scepter.modules.utils.probe import ProbeData class TestModel(BaseModel): diff --git a/readme.md b/readme.md index 2467a43..fcaddcc 100644 --- a/readme.md +++ b/readme.md @@ -21,6 +21,7 @@ - [Acknowledgement](#acknowledgement) ## 🎉 News +- [2024.04]: New [StyleBooth](https://ali-vilab.github.io/stylebooth-page/) demo on SCEPTER Studio, supporting `Text-Based Style Editing`. - [2024.03]: We optimize the training UI and checkpoint management. New [LAR-Gen](https://arxiv.org/abs/2403.19534) model has been added on SCEPTER Studio, supporting `zoom-out`, `virtual try on`, `inpainting`. - [2024.02]: We release new SCEdit controllable image synthesis models for SD v2.1 and SD XL. Multiple strategies applied to accelerate inference time for SCEPTER Studio. - [2024.01]: We release **SCEPTER Studio**, an integrated toolkit for data management, model training and inference based on [Gradio](https://www.gradio.app/). @@ -90,14 +91,14 @@ print(next(iter(ms_train_dataset))) #### CSV Format -For the data format used by SCEPTER Studio, please refer to [3D_example_csv.zip](https://modelscope.cn/api/v1/models/damo/scepter/repo?Revision=master&FilePath=datasets/3D_example_csv.zip). +For the data format used by SCEPTER Studio, please refer to [3D_example_csv.zip](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets/3D_example_csv.zip). #### TXT Format -To facilitate starting training in command-line mode, you can use a dataset in text format, please refer to [3D_example_txt.zip](https://modelscope.cn/api/v1/models/damo/scepter/repo?Revision=master&FilePath=datasets/3D_example_txt.zip) +To facilitate starting training in command-line mode, you can use a dataset in text format, please refer to [3D_example_txt.zip](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets/3D_example_txt.zip) ```shell -mkdir -p cache/datasets/ && wget 'https://modelscope.cn/api/v1/models/damo/scepter_scedit/repo?Revision=master&FilePath=dataset/3D_example_txt.zip' -O cache/datasets/3D_example_txt.zip && unzip cache/datasets/3D_example_txt.zip -d cache/datasets/ && rm cache/datasets/3D_example_txt.zip +mkdir -p cache/datasets/ && wget 'https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets/3D_example_txt.zip' -O cache/datasets/3D_example_txt.zip && unzip cache/datasets/3D_example_txt.zip -d cache/datasets/ && rm cache/datasets/3D_example_txt.zip ``` ### Training @@ -208,6 +209,25 @@ We deploy a work studio on Modelscope that includes only the inference tab, plea ## 🖼️ Gallery +### StyleBooth + + + + + + + + + + + + + + + +
Origin Image
Gold Dragon Tuner
Graffiti ArtAdorable Kawaiigame-retro gameVincent van Gogh
+ + ### LAR-Gen: Zoom Out @@ -340,6 +360,12 @@ We deploy a work studio on Modelscope that includes only the inference tab, plea |:---------:|:----------:|:----------:|:----------:| | SD XL | 🪄 | 🪄 | ⏳ | +- StyleBooth + +| **Text-Based** | **Exemplar-Based** | +|:--------------:|:-----------------:| +| 🪄 | ⏳ | + ### Model URL - ✅ indicates support for both training and inference. @@ -347,10 +373,11 @@ We deploy a work studio on Modelscope that includes only the inference tab, plea - ⏳ denotes that the module has not been integrated currently. - More models will be released in the future. -| Model | URL | -|--------|------------------------------------------------------------------------------------------------------------------------------------------------| +| Model | URL | +|--------|-------------------------------------------------------------------------------------------------------------------------------------------| | SCEdit | [ModelScope](https://modelscope.cn/models/iic/scepter_scedit/summary) [HuggingFace](https://huggingface.co/scepter-studio/scepter_scedit) | -| LAR-Gen | [ModelScope](https://www.modelscope.cn/models/iic/LARGEN/summary) | +| LAR-Gen | [ModelScope](https://www.modelscope.cn/models/iic/LARGEN/summary) | +| StyleBooth | [ModelScope](https://www.modelscope.cn/models/iic/stylebooth/summary) | PS: Scripts running within the SCEPTER framework will automatically fetch and load models based on the required dependency files, eliminating the need for manual downloads. diff --git a/requirements/framework.txt b/requirements/framework.txt index 3019164..2085b6c 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -2,7 +2,7 @@ albumentations bezier einops modelscope -ms-swift>=1.5.2 +ms-swift>=2.0.1 numpy open_clip_torch opencv-python diff --git a/requirements/scepter_studio.txt b/requirements/scepter_studio.txt index 029ab97..ddb7200 100644 --- a/requirements/scepter_studio.txt +++ b/requirements/scepter_studio.txt @@ -1,3 +1,6 @@ +bitsandbytes gradio>=3.47.1,<4.0.0 imagehash psutil +tiktoken +transformers_stream_generator diff --git a/scepter/methods/examples/generation/stable_diffusion_1.5_512_textlora.yaml b/scepter/methods/examples/generation/stable_diffusion_1.5_512_textlora.yaml new file mode 100644 index 0000000..8532812 --- /dev/null +++ b/scepter/methods/examples/generation/stable_diffusion_1.5_512_textlora.yaml @@ -0,0 +1,234 @@ +ENV: + BACKEND: nccl +SOLVER: + NAME: LatentDiffusionSolver + RESUME_FROM: + LOAD_MODEL_ONLY: True + USE_FSDP: False + SHARDING_STRATEGY: + USE_AMP: True + DTYPE: float16 + CHANNELS_LAST: True + MAX_STEPS: 2000 + MAX_EPOCHS: -1 + NUM_FOLDS: 1 + ACCU_STEP: 1 + EVAL_INTERVAL: 100 + # + WORK_DIR: ./cache/save_data/sd15_512_textlora + LOG_FILE: std_log.txt + # + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/data" + TUNER: + - + NAME: SwiftLoRA + R: 64 + LORA_ALPHA: 64 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "model.*(to_q|to_k|to_v|to_out.0|net.0.proj|net.2)$" + - + NAME: SwiftLoRA + R: 64 + LORA_ALPHA: 64 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "cond_stage_model.*(q_proj|k_proj|v_proj|out_proj|mlp.fc1|mlp.fc2)$" + # + MODEL: + NAME: LatentDiffusion + PARAMETERIZATION: eps + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-v1-5@v1-5-pruned-emaonly.safetensors + IGNORE_KEYS: [ ] + SCALE_FACTOR: 0.18215 + SIZE_FACTOR: 8 + # DEFAULT_N_PROMPT: 'lowres, error, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature' + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "scaled_linear" + "BETA_MIN": 0.00085 + "BETA_MAX": 0.012 + USE_EMA: False + # + DIFFUSION_MODEL: + NAME: DiffusionUNet + IN_CHANNELS: 4 + OUT_CHANNELS: 4 + MODEL_CHANNELS: 320 + NUM_HEADS: 8 + NUM_RES_BLOCKS: 2 + ATTENTION_RESOLUTIONS: [ 4, 2, 1 ] + CHANNEL_MULT: [ 1, 2, 4, 4 ] + CONV_RESAMPLE: True + DIMS: 2 + USE_CHECKPOINT: False + USE_SCALE_SHIFT_NORM: False + RESBLOCK_UPDOWN: False + USE_SPATIAL_TRANSFORMER: True + TRANSFORMER_DEPTH: 1 + CONTEXT_DIM: 768 + DISABLE_MIDDLE_SELF_ATTN: False + USE_LINEAR_IN_TRANSFORMER: False + PRETRAINED_MODEL: + IGNORE_KEYS: [] + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + EMBED_DIM: 4 + PRETRAINED_MODEL: + IGNORE_KEYS: [] + BATCH_SIZE: 4 + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + TOKENIZER: + NAME: ClipTokenizer + PRETRAINED_PATH: ms://AI-ModelScope/clip-vit-large-patch14 + LENGTH: 77 + CLEAN: True + # + COND_STAGE_MODEL: + NAME: FrozenCLIPEmbedder + FREEZE: True + LAYER: last + PRETRAINED_MODEL: ms://AI-ModelScope/clip-vit-large-patch14 + USE_GRAD: True + # + LOSS: + NAME: ReconstructLoss + LOSS_TYPE: l2 + # + SAMPLE_ARGS: + SAMPLER: ddim + SAMPLE_STEPS: 50 + SEED: 2023 + GUIDE_SCALE: 7.5 + GUIDE_RESCALE: 0.5 + DISCRETIZATION: trailing + IMAGE_SIZE: [512, 512] + RUN_TRAIN_N: False + # + OPTIMIZER: + NAME: AdamW + LEARNING_RATE: 0.064 + BETAS: [ 0.9, 0.999 ] + EPS: 1e-8 + WEIGHT_DECAY: 1e-2 + AMSGRAD: False + # + TRAIN_DATA: + NAME: ImageTextPairMSDataset + MODE: train + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_DATASET_SPLIT: train_short + MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' } + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + SAMPLER: + NAME: LoopSampler + TRANSFORMS: + - NAME: LoadImageFromFile + RGB_ORDER: RGB + BACKEND: pillow + - NAME: Resize + SIZE: 512 + INTERPOLATION: bilinear + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: CenterCrop + SIZE: 512 + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: ImageToTensor + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: Normalize + MEAN: [ 0.5, 0.5, 0.5 ] + STD: [ 0.5, 0.5, 0.5 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'image' ] + BACKEND: torchvision + - NAME: Select + KEYS: [ 'image', 'prompt' ] + META_KEYS: [ 'data_key' ] + # + EVAL_DATA: + NAME: ImageTextPairMSDataset + MODE: eval + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_REMAP_KEYS: { 'Image': 'Target:FILE' } + MS_DATASET_SPLIT: test_short + OUTPUT_SIZE: [512, 512] + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 4 + NUM_WORKERS: 4 + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/data" + # + TRANSFORMS: + - + NAME: Select + KEYS: ['prompt'] + META_KEYS: ['image_size'] + # + TRAIN_HOOKS: + - + NAME: BackwardHook + PRIORITY: 0 + - + NAME: LogHook + LOG_INTERVAL: 50 + - + NAME: CheckpointHook + INTERVAL: 1000 + - + NAME: ProbeDataHook + PROB_INTERVAL: 100 + # + EVAL_HOOKS: + - + NAME: ProbeDataHook + PROB_INTERVAL: 100 diff --git a/scepter/methods/examples/generation/stable_diffusion_xl_1024_textlora.yaml b/scepter/methods/examples/generation/stable_diffusion_xl_1024_textlora.yaml new file mode 100644 index 0000000..28ec143 --- /dev/null +++ b/scepter/methods/examples/generation/stable_diffusion_xl_1024_textlora.yaml @@ -0,0 +1,345 @@ +ENV: + BACKEND: nccl +SOLVER: + NAME: LatentDiffusionSolver + RESUME_FROM: + LOAD_MODEL_ONLY: True + USE_FSDP: False + SHARDING_STRATEGY: + USE_AMP: True + DTYPE: float16 + CHANNELS_LAST: True + MAX_STEPS: 2000 + MAX_EPOCHS: -1 + NUM_FOLDS: 1 + ACCU_STEP: 1 + EVAL_INTERVAL: 100 + # + WORK_DIR: ./cache/save_data/sdxl_1024_textlora + LOG_FILE: std_log.txt + # + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/data" + # + TUNER: + - + NAME: SwiftLoRA + R: 64 + LORA_ALPHA: 64 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "model.*(to_q|to_k|to_v|to_out.0|net.0.proj|net.2)$" + - + NAME: SwiftLoRA + R: 64 + LORA_ALPHA: 64 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "cond_stage_model.embedders.0.*(q_proj|k_proj|v_proj|out_proj|mlp.fc1|mlp.fc2)$" + # + MODEL: + NAME: LatentDiffusionXL + PARAMETERIZATION: eps + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-xl-base-1.0@sd_xl_base_1.0.safetensors + IGNORE_KEYS: [ ] + SCALE_FACTOR: 0.13025 + SIZE_FACTOR: 8 + # DEFAULT_N_PROMPT: 'lowres, error, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature' + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "scaled_linear" + "BETA_MIN": 0.00085 + "BETA_MAX": 0.0120 + USE_EMA: False + LOAD_REFINER: False + # + DIFFUSION_MODEL: + NAME: DiffusionUNetXL + PRETRAINED_MODEL: + IN_CHANNELS: 4 + OUT_CHANNELS: 4 + NUM_RES_BLOCKS: 2 + MODEL_CHANNELS: 320 + ATTENTION_RESOLUTIONS: [ 4, 2 ] + DROPOUT: 0 + CHANNEL_MULT: [ 1, 2, 4 ] + CONV_RESAMPLE: True + DIMS: 2 + NUM_CLASSES: sequential + USE_CHECKPOINT: False + NUM_HEADS: -1 + NUM_HEADS_CHANNELS: 64 + USE_SCALE_SHIFT_NORM: False + RESBLOCK_UPDOWN: False + USE_NEW_ATTENTION_ORDER: True + USE_SPATIAL_TRANSFORMER: True + TRANSFORMER_DEPTH: [ 1, 2, 10 ] + CONTEXT_DIM: 2048 + DISABLE_MIDDLE_SELF_ATTN: False + USE_LINEAR_IN_TRANSFORMER: True + ADM_IN_CHANNELS: 2816 + USE_SENTENCE_EMB: False + USE_WORD_MAPPING: False + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + EMBED_DIM: 4 + PRETRAINED_MODEL: + IGNORE_KEYS: [] + BATCH_SIZE: 1 + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + COND_STAGE_MODEL: + NAME: GeneralConditioner + PRETRAINED_MODEL: + USE_GRAD: True + EMBEDDERS: + - + NAME: FrozenCLIPEmbedder + PRETRAINED_MODEL: ms://AI-ModelScope/clip-vit-large-patch14 + TOKENIZER_PATH: ms://AI-ModelScope/clip-vit-large-patch14 + MAX_LENGTH: 77 + FREEZE: True + LAYER: hidden + LAYER_IDX: 11 + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "prompt" ] + LEGACY_UCG_VALUE: + - + NAME: FrozenOpenCLIPEmbedder2 + ARCH: ViT-bigG-14 + PRETRAINED_MODEL: + MAX_LENGTH: 77 + FREEZE: True + ALWAYS_RETURN_POOLED: True + LEGACY: False + LAYER: penultimate + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "prompt" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "original_size_as_tuple" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "crop_coords_top_left" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "target_size_as_tuple" ] + LEGACY_UCG_VALUE: + # + REFINER_MODEL: + NAME: DiffusionUNetXL + PRETRAINED_MODEL: + IN_CHANNELS: 4 + OUT_CHANNELS: 4 + NUM_RES_BLOCKS: 2 + MODEL_CHANNELS: 384 + ATTENTION_RESOLUTIONS: [ 4, 2 ] + DROPOUT: 0 + CHANNEL_MULT: [ 1, 2, 4, 4 ] + CONV_RESAMPLE: True + DIMS: 2 + NUM_CLASSES: sequential + USE_CHECKPOINT: False + NUM_HEADS: -1 + NUM_HEADS_CHANNELS: 64 + USE_SCALE_SHIFT_NORM: False + RESBLOCK_UPDOWN: False + USE_NEW_ATTENTION_ORDER: True + USE_SPATIAL_TRANSFORMER: True + TRANSFORMER_DEPTH: 4 + CONTEXT_DIM: [ 1280, 1280, 1280, 1280 ] + DISABLE_MIDDLE_SELF_ATTN: False + USE_LINEAR_IN_TRANSFORMER: True + ADM_IN_CHANNELS: 2560 + USE_SENTENCE_EMB: False + USE_WORD_MAPPING: False + # + REFINER_COND_MODEL: + NAME: GeneralConditioner + PRETRAINED_MODEL: + EMBEDDERS: + - + NAME: FrozenOpenCLIPEmbedder2 + ARCH: ViT-bigG-14 + PRETRAINED_MODEL: + MAX_LENGTH: 77 + FREEZE: True + ALWAYS_RETURN_POOLED: True + LEGACY: False + LAYER: penultimate + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "prompt" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "original_size_as_tuple" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "crop_coords_top_left" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "aesthetic_score" ] + LEGACY_UCG_VALUE: + # + LOSS: + NAME: ReconstructLoss + LOSS_TYPE: l2 + # + SAMPLE_ARGS: + SAMPLER: ddim + SAMPLE_STEPS: 50 + SEED: 2023 + GUIDE_SCALE: 5.0 + GUIDE_RESCALE: + DISCRETIZATION: trailing + IMAGE_SIZE: [1024, 1024] + RUN_TRAIN_N: False + # + OPTIMIZER: + NAME: AdamW + LEARNING_RATE: 0.064 + BETAS: [ 0.9, 0.999 ] + EPS: 1e-8 + WEIGHT_DECAY: 1e-2 + AMSGRAD: False + # + TRAIN_DATA: + NAME: ImageTextPairMSDataset + MODE: train + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_DATASET_SPLIT: train_short + MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' } + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + SAMPLER: + NAME: LoopSampler + TRANSFORMS: + - NAME: LoadImageFromFile + RGB_ORDER: RGB + BACKEND: pillow + - NAME: FlexibleResize + INTERPOLATION: bicubic + SIZE: 1024 + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: FlexibleCropXL + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: ImageToTensor + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: Normalize + MEAN: [ 0.5, 0.5, 0.5 ] + STD: [ 0.5, 0.5, 0.5 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: torchvision + - NAME: Select + KEYS: [ 'img', 'prompt', 'img_original_size_as_tuple', 'img_target_size_as_tuple', 'img_crop_coords_top_left' ] + META_KEYS: [ 'data_key', 'img_path' ] + - NAME: Rename + INPUT_KEY: [ 'img', 'img_original_size_as_tuple', 'img_target_size_as_tuple', 'img_crop_coords_top_left' ] + OUTPUT_KEY: [ 'image', 'original_size_as_tuple', 'target_size_as_tuple', 'crop_coords_top_left' ] + # + EVAL_DATA: + NAME: ImageTextPairMSDataset + MODE: eval + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_REMAP_KEYS: { 'Image': 'Target:FILE' } + MS_DATASET_SPLIT: test_short + OUTPUT_SIZE: [ 1024, 1024 ] + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 4 + NUM_WORKERS: 4 + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/data" + # + TRANSFORMS: + - NAME: Select + KEYS: [ 'prompt' ] + META_KEYS: [ 'image_size' ] + # + TRAIN_HOOKS: + - NAME: BackwardHook + PRIORITY: 0 + - NAME: LogHook + LOG_INTERVAL: 50 + - NAME: CheckpointHook + INTERVAL: 1000 + - NAME: ProbeDataHook + PROB_INTERVAL: 100 + # + EVAL_HOOKS: + - NAME: ProbeDataHook + PROB_INTERVAL: 100 diff --git a/scepter/methods/scedit/t2i/sd15_512_textsce_t2i_swift.yaml b/scepter/methods/scedit/t2i/sd15_512_textsce_t2i_swift.yaml new file mode 100644 index 0000000..77dcf20 --- /dev/null +++ b/scepter/methods/scedit/t2i/sd15_512_textsce_t2i_swift.yaml @@ -0,0 +1,234 @@ +ENV: + BACKEND: nccl +SOLVER: + NAME: LatentDiffusionSolver + RESUME_FROM: + LOAD_MODEL_ONLY: True + USE_FSDP: False + SHARDING_STRATEGY: + USE_AMP: True + DTYPE: float16 + CHANNELS_LAST: True + MAX_STEPS: 2000 + MAX_EPOCHS: -1 + NUM_FOLDS: 1 + ACCU_STEP: 1 + EVAL_INTERVAL: 100 + # + WORK_DIR: ./cache/save_data/sd15_512_textsce_t2i_swift + LOG_FILE: std_log.txt + # + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/data" + # + TUNER: + - + NAME: SwiftSCETuning + DIMS: [1280, 1280, 1280, 1280, 1280, 640, 640, 640, 320, 320, 320, 320] + TARGET_MODULES: model.lsc_identity\.\d+$ + DOWN_RATIO: 1.0 + TUNER_MODE: identity + - + NAME: SwiftLoRA + R: 64 + LORA_ALPHA: 64 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "cond_stage_model.*(q_proj|k_proj|v_proj|out_proj|mlp.fc1|mlp.fc2)$" + # + MODEL: + NAME: LatentDiffusion + PARAMETERIZATION: eps + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-v1-5@v1-5-pruned-emaonly.safetensors + IGNORE_KEYS: [ ] + SCALE_FACTOR: 0.18215 + SIZE_FACTOR: 8 + # DEFAULT_N_PROMPT: 'lowres, error, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature' + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "scaled_linear" + "BETA_MIN": 0.00085 + "BETA_MAX": 0.012 + USE_EMA: False + # + DIFFUSION_MODEL: + NAME: DiffusionUNet + IN_CHANNELS: 4 + OUT_CHANNELS: 4 + MODEL_CHANNELS: 320 + NUM_HEADS: 8 + NUM_RES_BLOCKS: 2 + ATTENTION_RESOLUTIONS: [ 4, 2, 1 ] + CHANNEL_MULT: [ 1, 2, 4, 4 ] + CONV_RESAMPLE: True + DIMS: 2 + USE_CHECKPOINT: False + USE_SCALE_SHIFT_NORM: False + RESBLOCK_UPDOWN: False + USE_SPATIAL_TRANSFORMER: True + TRANSFORMER_DEPTH: 1 + CONTEXT_DIM: 768 + DISABLE_MIDDLE_SELF_ATTN: False + USE_LINEAR_IN_TRANSFORMER: False + PRETRAINED_MODEL: + IGNORE_KEYS: [] + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + EMBED_DIM: 4 + PRETRAINED_MODEL: + IGNORE_KEYS: [] + BATCH_SIZE: 4 + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + TOKENIZER: + NAME: ClipTokenizer + PRETRAINED_PATH: ms://AI-ModelScope/clip-vit-large-patch14 + LENGTH: 77 + CLEAN: True + # + COND_STAGE_MODEL: + NAME: FrozenCLIPEmbedder + FREEZE: True + LAYER: last + PRETRAINED_MODEL: ms://AI-ModelScope/clip-vit-large-patch14 + USE_GRAD: True + # + LOSS: + NAME: ReconstructLoss + LOSS_TYPE: l2 + # + SAMPLE_ARGS: + SAMPLER: ddim + SAMPLE_STEPS: 50 + SEED: 2023 + GUIDE_SCALE: 7.5 + GUIDE_RESCALE: 0.5 + DISCRETIZATION: trailing + IMAGE_SIZE: [512, 512] + RUN_TRAIN_N: False + # + OPTIMIZER: + NAME: AdamW + LEARNING_RATE: 0.064 + BETAS: [ 0.9, 0.999 ] + EPS: 1e-8 + WEIGHT_DECAY: 1e-2 + AMSGRAD: False + # + TRAIN_DATA: + NAME: ImageTextPairMSDataset + MODE: train + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_DATASET_SPLIT: train_short + MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' } + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + SAMPLER: + NAME: LoopSampler + TRANSFORMS: + - NAME: LoadImageFromFile + RGB_ORDER: RGB + BACKEND: pillow + - NAME: Resize + SIZE: 512 + INTERPOLATION: bilinear + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: CenterCrop + SIZE: 512 + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: ImageToTensor + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: Normalize + MEAN: [ 0.5, 0.5, 0.5 ] + STD: [ 0.5, 0.5, 0.5 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'image' ] + BACKEND: torchvision + - NAME: Select + KEYS: [ 'image', 'prompt' ] + META_KEYS: [ 'data_key' ] + # + EVAL_DATA: + NAME: ImageTextPairMSDataset + MODE: eval + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_REMAP_KEYS: { 'Image': 'Target:FILE' } + MS_DATASET_SPLIT: test_short + OUTPUT_SIZE: [512, 512] + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 4 + NUM_WORKERS: 4 + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/data" + # + TRANSFORMS: + - + NAME: Select + KEYS: ['prompt'] + META_KEYS: ['image_size'] + # + TRAIN_HOOKS: + - + NAME: BackwardHook + PRIORITY: 0 + - + NAME: LogHook + LOG_INTERVAL: 50 + - + NAME: CheckpointHook + INTERVAL: 1000 + - + NAME: ProbeDataHook + PROB_INTERVAL: 100 + # + EVAL_HOOKS: + - + NAME: ProbeDataHook + PROB_INTERVAL: 100 diff --git a/scepter/methods/scedit/t2i/sdxl_1024_textsce_t2i_swift.yaml b/scepter/methods/scedit/t2i/sdxl_1024_textsce_t2i_swift.yaml new file mode 100644 index 0000000..26b8678 --- /dev/null +++ b/scepter/methods/scedit/t2i/sdxl_1024_textsce_t2i_swift.yaml @@ -0,0 +1,348 @@ +ENV: + BACKEND: nccl +SOLVER: + NAME: LatentDiffusionSolver + RESUME_FROM: + LOAD_MODEL_ONLY: True + USE_FSDP: False + SHARDING_STRATEGY: + USE_AMP: True + DTYPE: float16 + CHANNELS_LAST: True + MAX_STEPS: 2000 + MAX_EPOCHS: -1 + NUM_FOLDS: 1 + ACCU_STEP: 1 + EVAL_INTERVAL: 100 + # + WORK_DIR: ./cache/save_data/sdxl_1024_textsce_t2i_swift + LOG_FILE: std_log.txt + # + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/data" + # + TUNER: + - + NAME: SwiftSCETuning + DIMS: [1280, 1280, 640, 640, 640, 320, 320, 320, 320] + TARGET_MODULES: model.lsc_identity\.\d+$ + DOWN_RATIO: 1.0 + TUNER_MODE: identity + - + NAME: SwiftLoRA + R: 64 + LORA_ALPHA: 64 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "cond_stage_model.embedders.0.*(q_proj|k_proj|v_proj|out_proj|mlp.fc1|mlp.fc2)$" + # + MODEL: + NAME: LatentDiffusionXL + PARAMETERIZATION: eps + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-xl-base-1.0@sd_xl_base_1.0.safetensors + IGNORE_KEYS: [ ] + SCALE_FACTOR: 0.13025 + SIZE_FACTOR: 8 + # DEFAULT_N_PROMPT: 'lowres, error, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature' + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "scaled_linear" + "BETA_MIN": 0.00085 + "BETA_MAX": 0.0120 + USE_EMA: False + LOAD_REFINER: False + # + DIFFUSION_MODEL: + NAME: DiffusionUNetXL + PRETRAINED_MODEL: + IN_CHANNELS: 4 + OUT_CHANNELS: 4 + NUM_RES_BLOCKS: 2 + MODEL_CHANNELS: 320 + ATTENTION_RESOLUTIONS: [ 4, 2 ] + DROPOUT: 0 + CHANNEL_MULT: [ 1, 2, 4 ] + CONV_RESAMPLE: True + DIMS: 2 + NUM_CLASSES: sequential + USE_CHECKPOINT: False + NUM_HEADS: -1 + NUM_HEADS_CHANNELS: 64 + USE_SCALE_SHIFT_NORM: False + RESBLOCK_UPDOWN: False + USE_NEW_ATTENTION_ORDER: True + USE_SPATIAL_TRANSFORMER: True + TRANSFORMER_DEPTH: [ 1, 2, 10 ] + CONTEXT_DIM: 2048 + DISABLE_MIDDLE_SELF_ATTN: False + USE_LINEAR_IN_TRANSFORMER: True + ADM_IN_CHANNELS: 2816 + USE_SENTENCE_EMB: False + USE_WORD_MAPPING: False + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + EMBED_DIM: 4 + PRETRAINED_MODEL: + IGNORE_KEYS: [] + BATCH_SIZE: 1 + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + COND_STAGE_MODEL: + NAME: GeneralConditioner + PRETRAINED_MODEL: + USE_GRAD: True + EMBEDDERS: + - + NAME: FrozenCLIPEmbedder + PRETRAINED_MODEL: ms://AI-ModelScope/clip-vit-large-patch14 + TOKENIZER_PATH: ms://AI-ModelScope/clip-vit-large-patch14 + MAX_LENGTH: 77 + FREEZE: True + LAYER: hidden + LAYER_IDX: 11 + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "prompt" ] + LEGACY_UCG_VALUE: + - + NAME: FrozenOpenCLIPEmbedder2 + ARCH: ViT-bigG-14 + PRETRAINED_MODEL: + MAX_LENGTH: 77 + FREEZE: True + ALWAYS_RETURN_POOLED: True + LEGACY: False + LAYER: penultimate + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "prompt" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "original_size_as_tuple" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "crop_coords_top_left" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "target_size_as_tuple" ] + LEGACY_UCG_VALUE: + # + REFINER_MODEL: + NAME: DiffusionUNetXL + PRETRAINED_MODEL: + IN_CHANNELS: 4 + OUT_CHANNELS: 4 + NUM_RES_BLOCKS: 2 + MODEL_CHANNELS: 384 + ATTENTION_RESOLUTIONS: [ 4, 2 ] + DROPOUT: 0 + CHANNEL_MULT: [ 1, 2, 4, 4 ] + CONV_RESAMPLE: True + DIMS: 2 + NUM_CLASSES: sequential + USE_CHECKPOINT: False + NUM_HEADS: -1 + NUM_HEADS_CHANNELS: 64 + USE_SCALE_SHIFT_NORM: False + RESBLOCK_UPDOWN: False + USE_NEW_ATTENTION_ORDER: True + USE_SPATIAL_TRANSFORMER: True + TRANSFORMER_DEPTH: 4 + CONTEXT_DIM: [ 1280, 1280, 1280, 1280 ] + DISABLE_MIDDLE_SELF_ATTN: False + USE_LINEAR_IN_TRANSFORMER: True + ADM_IN_CHANNELS: 2560 + USE_SENTENCE_EMB: False + USE_WORD_MAPPING: False + REFINER_COND_MODEL: + NAME: GeneralConditioner + PRETRAINED_MODEL: + EMBEDDERS: + - + NAME: FrozenOpenCLIPEmbedder2 + ARCH: ViT-bigG-14 + PRETRAINED_MODEL: + MAX_LENGTH: 77 + FREEZE: True + ALWAYS_RETURN_POOLED: True + LEGACY: False + LAYER: penultimate + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "prompt" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "original_size_as_tuple" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "crop_coords_top_left" ] + LEGACY_UCG_VALUE: + - + NAME: ConcatTimestepEmbedderND + OUT_DIM: 256 + IS_TRAINABLE: False + UCG_RATE: 0.0 + INPUT_KEYS: [ "aesthetic_score" ] + LEGACY_UCG_VALUE: + # + LOSS: + NAME: ReconstructLoss + LOSS_TYPE: l2 + # + SAMPLE_ARGS: + SAMPLER: ddim + SAMPLE_STEPS: 50 + SEED: 2023 + GUIDE_SCALE: 7.5 + GUIDE_RESCALE: 0.5 + DISCRETIZATION: trailing + IMAGE_SIZE: [1024, 1024] + RUN_TRAIN_N: False + # + OPTIMIZER: + NAME: AdamW + LEARNING_RATE: 0.064 + BETAS: [ 0.9, 0.999 ] + EPS: 1e-8 + WEIGHT_DECAY: 1e-2 + AMSGRAD: False + # + TRAIN_DATA: + NAME: ImageTextPairMSDataset + MODE: train + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_DATASET_SPLIT: train_short + MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' } + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + SAMPLER: + NAME: LoopSampler + TRANSFORMS: + - NAME: LoadImageFromFile + RGB_ORDER: RGB + BACKEND: pillow + - NAME: FlexibleResize + INTERPOLATION: bicubic + SIZE: 1024 + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: FlexibleCropXL + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: ImageToTensor + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: Normalize + MEAN: [ 0.5, 0.5, 0.5 ] + STD: [ 0.5, 0.5, 0.5 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: torchvision + - NAME: Select + KEYS: [ 'img', 'prompt', 'img_original_size_as_tuple', 'img_target_size_as_tuple', 'img_crop_coords_top_left' ] + META_KEYS: [ 'data_key', 'img_path' ] + - NAME: Rename + INPUT_KEY: [ 'img', 'img_original_size_as_tuple', 'img_target_size_as_tuple', 'img_crop_coords_top_left' ] + OUTPUT_KEY: [ 'image', 'original_size_as_tuple', 'target_size_as_tuple', 'crop_coords_top_left' ] + # + EVAL_DATA: + NAME: ImageTextPairMSDataset + MODE: eval + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_REMAP_KEYS: { 'Image': 'Target:FILE' } + MS_DATASET_SPLIT: test_short + OUTPUT_SIZE: [ 1024, 1024 ] + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 4 + NUM_WORKERS: 4 + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/data" + # + TRANSFORMS: + - NAME: Select + KEYS: [ 'prompt' ] + META_KEYS: [ 'image_size' ] + # + TRAIN_HOOKS: + - + NAME: BackwardHook + PRIORITY: 0 + - + NAME: LogHook + LOG_INTERVAL: 50 + - + NAME: CheckpointHook + INTERVAL: 1000 + - + NAME: ProbeDataHook + PROB_INTERVAL: 100 + # + EVAL_HOOKS: + - + NAME: ProbeDataHook + PROB_INTERVAL: 100 diff --git a/scepter/methods/studio/extensions/tuners/official_tuners.yaml b/scepter/methods/studio/extensions/tuners/official_tuners.yaml index d195271..3fcec5c 100644 --- a/scepter/methods/studio/extensions/tuners/official_tuners.yaml +++ b/scepter/methods/studio/extensions/tuners/official_tuners.yaml @@ -1,7 +1,7 @@ TUNERS: - NAME: Azure-Dragon NAME_ZH: 青龙 - SOURCE: wanx + SOURCE: scepter DESCRIPTION: None BASE_MODEL: SD_XL1.0 MODEL_PATH: ms://damo/scepter_scedit@tuners_model/SD_XL1.0/azure_dragon/ @@ -10,7 +10,7 @@ TUNERS: PROMPT_EXAMPLE: Azure Dragon, 8K, high quality,Ultra High Detail.One of the Four Divine Creatures in Charge of Water. - NAME: Gold-Dragon NAME_ZH: 金龙 - SOURCE: wanx + SOURCE: scepter DESCRIPTION: None BASE_MODEL: SD_XL1.0 MODEL_PATH: ms://damo/scepter_scedit@tuners_model/SD_XL1.0/gold_dragon/ @@ -19,7 +19,7 @@ TUNERS: PROMPT_EXAMPLE: Chinese Gold Dragon in the clouds. Translucent Texture. Zbrush. Fuzzy Art. Exquisite Craftsmanship. 3D. 8K. Ultra High Detail - NAME: SpringFestival-Dragon NAME_ZH: 春节龙 - SOURCE: wanx + SOURCE: scepter DESCRIPTION: None BASE_MODEL: SD_XL1.0 MODEL_PATH: ms://damo/scepter_scedit@tuners_model/SD_XL1.0/spring_festival_dragon/ @@ -28,7 +28,7 @@ TUNERS: PROMPT_EXAMPLE: Chinese dragon. Spring Festival.Festive.Street.Lanterns.32K.High quality.expressive, dramatic, dreamlike and mysterious, Surrealism - NAME: Red-Dragon NAME_ZH: 红龙 - SOURCE: wanx + SOURCE: scepter DESCRIPTION: None BASE_MODEL: SD_XL1.0 MODEL_PATH: ms://damo/scepter_scedit@tuners_model/SD_XL1.0/red_dragon/ @@ -37,7 +37,7 @@ TUNERS: PROMPT_EXAMPLE: Traditional Red Dragon of China. Low Water Level. Studio Ghibli Style. Mural Illustration. White Background. High Detail - NAME: ChinesePunk-Dragon NAME_ZH: 中国朋克龙 - SOURCE: wanx + SOURCE: scepter DESCRIPTION: None BASE_MODEL: SD_XL1.0 MODEL_PATH: ms://damo/scepter_scedit@tuners_model/SD_XL1.0/chinese_punk_dragon/ @@ -46,7 +46,7 @@ TUNERS: PROMPT_EXAMPLE: uhd Image,Dragon,Chinese Dragon, Dunhuang Mural Style, Traditional Maritime Art Style - NAME: Cute-Dragon NAME_ZH: 喜庆龙 - SOURCE: wanx + SOURCE: scepter DESCRIPTION: None BASE_MODEL: SD_XL1.0 MODEL_PATH: ms://damo/scepter_scedit@tuners_model/SD_XL1.0/cute_dragon/ @@ -55,7 +55,7 @@ TUNERS: PROMPT_EXAMPLE: China Kawaii Dragon. Contest Winner. Minimalist Illustration. White Background. Flat Style. Digital Painting Style. Red. 32k uhd. Fun Comics. Fuzzy Art. Bold. Comic-Inspired Characters - NAME: Dragon-Baby NAME_ZH: 龙宝宝 - SOURCE: wanx + SOURCE: scepter DESCRIPTION: None BASE_MODEL: SD_XL1.0 MODEL_PATH: ms://damo/scepter_scedit@tuners_model/SD_XL1.0/baby_dragon/ @@ -64,7 +64,7 @@ TUNERS: PROMPT_EXAMPLE: Warm Colors, Soft,Chinese Dragon Baby, Felt Style,Dragon Baby, Best Quality, 3D Doll, Macaron Tones, Glittering Big Eyes, Winter,Dragon - NAME: Sloppy-Dragon NAME_ZH: 潦草龙 - SOURCE: wanx + SOURCE: scepter DESCRIPTION: None BASE_MODEL: SD_XL1.0 MODEL_PATH: ms://damo/scepter_scedit@tuners_model/SD_XL1.0/sloppy_dragon/ diff --git a/scepter/methods/studio/inference/edit/stylebooth_tb_pro.yaml b/scepter/methods/studio/inference/edit/stylebooth_tb_pro.yaml new file mode 100644 index 0000000..25c105c --- /dev/null +++ b/scepter/methods/studio/inference/edit/stylebooth_tb_pro.yaml @@ -0,0 +1,129 @@ +NAME: EDIT +IS_DEFAULT: False +DEFAULT_PARAS: + PARAS: + RESOLUTIONS: [[1024, 1024]] + INPUT: + IMAGE: + PROMPT: "" + NEGATIVE_PROMPT: "" + TARGET_SIZE_AS_TUPLE: [1024, 1024] + PROMPT_PREFIX: "" + SAMPLE: ddim + SAMPLE_STEPS: 50 + GUIDE_SCALE: + text: 7.5 + image: 1.5 + GUIDE_RESCALE: 0.5 + DISCRETIZATION: trailing + OUTPUT: + LATENT: + IMAGES: + SEED: + MODULES_PARAS: + FIRST_STAGE_MODEL: + FUNCTION: + - + NAME: encode + DTYPE: float16 + INPUT: ["IMAGE"] + - + NAME: decode + DTYPE: float16 + INPUT: ["LATENT"] + PARAS: + # SCALE_FACTOR DESCRIPTION: The vae embeding scale. TYPE: float default: 0.18215 + SCALE_FACTOR: 0.18215 + SIZE_FACTOR: 8 + DIFFUSION_MODEL: + FUNCTION: + - + NAME: forward + DTYPE: float16 + INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE", "GUIDE_RESCALE", "DISCRETIZATION"] + COND_STAGE_MODEL: + FUNCTION: + - + NAME: encode_text + DTYPE: float16 + INPUT: ["PROMPT", "NEGATIVE_PROMPT"] + +MODEL: + PRETRAINED_MODEL: ms://damo/stylebooth@models/stylebooth-tb-5000-0.bin + SCHEDULE: + PARAMETERIZATION: "eps" + TIMESTEPS: 1000 + ZERO_TERMINAL_SNR: False + SCHEDULE_ARGS: + # NAME DESCRIPTION: TYPE: default: '' + NAME: "scaled_linear" + BETA_MIN: 0.00085 + BETA_MAX: 0.0120 + # + DIFFUSION_MODEL: + NAME: DiffusionUNet + PRETRAINED_PATH: + IN_CHANNELS: 8 + OUT_CHANNELS: 4 + MODEL_CHANNELS: 320 + NUM_HEADS: 8 + NUM_RES_BLOCKS: 2 + ATTENTION_RESOLUTIONS: [ 4, 2, 1 ] + CHANNEL_MULT: [ 1, 2, 4, 4 ] + CONV_RESAMPLE: True + DIMS: 2 + USE_CHECKPOINT: False + USE_SCALE_SHIFT_NORM: False + RESBLOCK_UPDOWN: False + USE_SPATIAL_TRANSFORMER: True + TRANSFORMER_DEPTH: 1 + CONTEXT_DIM: 768 + DISABLE_MIDDLE_SELF_ATTN: False + USE_LINEAR_IN_TRANSFORMER: False + IGNORE_KEYS: [] + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + EMBED_DIM: 4 + IGNORE_KEYS: [] + BATCH_SIZE: 4 + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + TOKENIZER: + NAME: ClipTokenizer + PRETRAINED_PATH: ms://AI-ModelScope/clip-vit-large-patch14 + LENGTH: 77 + CLEAN: True + # + COND_STAGE_MODEL: + NAME: FrozenCLIPEmbedder + FREEZE: True + USE_GRAD: False + LAYER: last + PRETRAINED_MODEL: ms://AI-ModelScope/clip-vit-large-patch14 diff --git a/scepter/methods/studio/preprocess/preprocess.yaml b/scepter/methods/studio/preprocess/preprocess.yaml index c54f678..8e43da0 100644 --- a/scepter/methods/studio/preprocess/preprocess.yaml +++ b/scepter/methods/studio/preprocess/preprocess.yaml @@ -5,3 +5,169 @@ FILE_SYSTEM: # NAME DESCRIPTION: TYPE: default: '' NAME: LocalFs AUTO_CLEAN: False + +PROCESSORS: + - NAME: BlipImageBase + TYPE: caption + MODEL_PATH: ms://cubeai/blip-image-captioning-base + DEVICE: "gpu" + MEMORY: 1200 + PARAS: + - LANGUAGE_NAME: English + LANGUAGE_ZH_NAME: 英语 + - NAME: QWVL + TYPE: caption + MODEL_PATH: ms://qwen/Qwen-VL:v1.0.3 + DEVICE: "gpu" + MEMORY: 19968 + PARAS: + - PROMPT: 用中文描述这张图片 + LANGUAGE_NAME: Chinese + LANGUAGE_ZH_NAME: 中文 + MAX_NEW_TOKENS: + VALUE: 1024 + MAX: 2048 + STEP: 128 + MIN: 256 + MIN_NEW_TOKENS: + VALUE: 16 + MAX: 1024 + STEP: 16 + MIN: 0 + NUM_BEAMS: + VALUE: 1 + MAX: 12 + STEP: 1 + MIN: 1 + REPETITION_PENALTY: + VALUE: 1.0 + MAX: 100.0 + STEP: 1.0 + MIN: 1.0 + TEMPERATURE: + VALUE: 1.0 + MAX: 100.0 + STEP: 1.0 + MIN: 1.0 + - PROMPT: Generate the caption in English + LANGUAGE_NAME: English + LANGUAGE_ZH_NAME: 英语 + MAX_NEW_TOKENS: + VALUE: 1024 + MAX: 2048 + STEP: 128 + MIN: 256 + MIN_NEW_TOKENS: + VALUE: 16 + MAX: 1024 + STEP: 16 + MIN: 0 + NUM_BEAMS: + VALUE: 1 + MAX: 12 + STEP: 1 + MIN: 1 + REPETITION_PENALTY: + VALUE: 1.0 + MAX: 100.0 + STEP: 1.0 + MIN: 1.0 + TEMPERATURE: + VALUE: 1.0 + MAX: 100.0 + STEP: 1.0 + MIN: 1.0 + - + NAME: QWVLQuantize + TYPE: caption + DEVICE: "gpu" + MEMORY: 7885 + MODEL_PATH: ms://qwen/Qwen-VL:v1.0.3 + PARAS: + - PROMPT: 用中文描述这张图片 + LANGUAGE_NAME: Chinese + LANGUAGE_ZH_NAME: 中文 + MAX_NEW_TOKENS: + VALUE: 1024 + MAX: 2048 + STEP: 128 + MIN: 256 + MIN_NEW_TOKENS: + VALUE: 16 + MAX: 1024 + STEP: 16 + MIN: 0 + NUM_BEAMS: + VALUE: 1 + MAX: 12 + STEP: 1 + MIN: 1 + REPETITION_PENALTY: + VALUE: 1.0 + MAX: 100.0 + STEP: 1.0 + MIN: 1.0 + TEMPERATURE: + VALUE: 1.0 + MAX: 100.0 + STEP: 1.0 + MIN: 1.0 + - PROMPT: Generate the caption in English + LANGUAGE_NAME: English + LANGUAGE_ZH_NAME: 英语 + MAX_NEW_TOKENS: + VALUE: 1024 + MAX: 2048 + STEP: 128 + MIN: 256 + MIN_NEW_TOKENS: + VALUE: 16 + MAX: 1024 + STEP: 16 + MIN: 0 + NUM_BEAMS: + VALUE: 1 + MAX: 12 + STEP: 1 + MIN: 1 + REPETITION_PENALTY: + VALUE: 1.0 + MAX: 100.0 + STEP: 1.0 + MIN: 1.0 + TEMPERATURE: + VALUE: 1.0 + MAX: 100.0 + STEP: 1.0 + MIN: 1.0 + - + NAME: CenterCrop + TYPE: image + DEVICE: "cpu" + MEMORY: 10 + PARAS: + HEIGHT_RATIO: + VALUE: 1 + MAX: 20 + STEP: 1 + MIN: 1 + WIDTH_RATIO: + VALUE: 1 + MAX: 20 + STEP: 1 + MIN: 1 +# - NAME: PaddingCrop +# TYPE: image +# DEVICE: "cpu" +# MEMORY: 10 +# PARAS: +# HEIGHT_RATIO: +# VALUE: 3 +# MAX: 25 +# STEP: 1 +# MIN: 1 +# WIDTH_RATIO: +# VALUE: 4 +# MAX: 20 +# STEP: 1 +# MIN: 1 diff --git a/scepter/methods/studio/self_train/sd_xl/sdxl_pro.yaml b/scepter/methods/studio/self_train/sd_xl/sdxl_pro.yaml index fddf18f..afab33d 100644 --- a/scepter/methods/studio/self_train/sd_xl/sdxl_pro.yaml +++ b/scepter/methods/studio/self_train/sd_xl/sdxl_pro.yaml @@ -5,6 +5,7 @@ META: VERSION: 'SD_XL1.0' DESCRIPTION: "Stable Diffusion XL1.0" IS_DEFAULT: True + IS_SHARE: True INFERENCE_PARAS: INFERENCE_BATCH_SIZE: 1 INFERENCE_PREFIX: "" @@ -532,7 +533,6 @@ SOLVER: GUIDE_SCALE: 5.0 GUIDE_RESCALE: DISCRETIZATION: linspace - IMAGE_SIZE: [ 1024, 1024] RUN_TRAIN_N: False # OPTIMIZER DESCRIPTION: TYPE: default: '' OPTIMIZER: diff --git a/scepter/methods/studio/self_train/stable_diffusion/sd15_pro.yaml b/scepter/methods/studio/self_train/stable_diffusion/sd15_pro.yaml index f818291..8b0ee75 100644 --- a/scepter/methods/studio/self_train/stable_diffusion/sd15_pro.yaml +++ b/scepter/methods/studio/self_train/stable_diffusion/sd15_pro.yaml @@ -4,6 +4,7 @@ META: VERSION: 'SD1.5' DESCRIPTION: "Stable Diffusion v1.5" IS_DEFAULT: False + IS_SHARE: True INFERENCE_PARAS: INFERENCE_BATCH_SIZE: 1 INFERENCE_PREFIX: "" @@ -244,7 +245,6 @@ SOLVER: GUIDE_SCALE: 7.5 GUIDE_RESCALE: DISCRETIZATION: trailing - IMAGE_SIZE: [512, 512] RUN_TRAIN_N: False # OPTIMIZER: @@ -274,14 +274,14 @@ SOLVER: - NAME: LoadImageFromFile RGB_ORDER: RGB BACKEND: pillow - - NAME: Resize - SIZE: 512 + - NAME: FlexibleResize INTERPOLATION: bilinear + SIZE: [ 512, 512 ] INPUT_KEY: [ 'img' ] OUTPUT_KEY: [ 'img' ] BACKEND: pillow - - NAME: CenterCrop - SIZE: 512 + - NAME: FlexibleCenterCrop + SIZE: [ 512, 512 ] INPUT_KEY: [ 'img' ] OUTPUT_KEY: [ 'img' ] BACKEND: pillow diff --git a/scepter/methods/studio/self_train/stable_diffusion/sd21_pro.yaml b/scepter/methods/studio/self_train/stable_diffusion/sd21_pro.yaml index 257af14..50b842c 100644 --- a/scepter/methods/studio/self_train/stable_diffusion/sd21_pro.yaml +++ b/scepter/methods/studio/self_train/stable_diffusion/sd21_pro.yaml @@ -4,6 +4,7 @@ META: VERSION: 'SD2.1' DESCRIPTION: "Stable Diffusion v2.1" IS_DEFAULT: False + IS_SHARE: True INFERENCE_PARAS: INFERENCE_BATCH_SIZE: 1 INFERENCE_PREFIX: "" @@ -186,7 +187,6 @@ SOLVER: GUIDE_SCALE: 7.5 GUIDE_RESCALE: DISCRETIZATION: trailing - IMAGE_SIZE: [768, 768] RUN_TRAIN_N: False # OPTIMIZER: @@ -216,14 +216,14 @@ SOLVER: - NAME: LoadImageFromFile RGB_ORDER: RGB BACKEND: pillow - - NAME: Resize - SIZE: 768 + - NAME: FlexibleResize INTERPOLATION: bilinear + SIZE: [ 768, 768 ] INPUT_KEY: [ 'img' ] OUTPUT_KEY: [ 'img' ] BACKEND: pillow - - NAME: CenterCrop - SIZE: 768 + - NAME: FlexibleCenterCrop + SIZE: [ 768, 768 ] INPUT_KEY: [ 'img' ] OUTPUT_KEY: [ 'img' ] BACKEND: pillow diff --git a/scepter/methods/studio/tuner_manager/readme_en.md b/scepter/methods/studio/tuner_manager/readme_en.md new file mode 100644 index 0000000..40e5391 --- /dev/null +++ b/scepter/methods/studio/tuner_manager/readme_en.md @@ -0,0 +1,168 @@ +--- +frameworks: +- Pytorch +license: Apache License 2.0 +tasks: +- efficient-diffusion-tuning +--- + +

+ +

{MODEL_NAME}

+

+
+ +
+

+ +## Model Introduction +{MODEL_DESCRIPTION} + +## Model Parameters +
+ + + + + + + + + + + + + + + + + + + + + + + +
Base ModelTuner TypeTraining Parameters
Batch SizeEpochsLearning RateResolution
{BASE_MODEL}{TUNER_TYPE}{TRAIN_BATCH_SIZE}{TRAIN_EPOCH}{LEARNING_RATE}[{HEIGHT}, {WIDTH}]
+ + + + + + + + + + + + + + + + + + + +
Data TypeData SpaceData NameData Subset
{DATA_TYPE}{MS_DATA_SPACE}{MS_DATA_NAME}{MS_DATA_SUBNAME}
+ + +## Model Performance +Given the input "{EVAL_PROMPT}," the following image may be generated: + +![image]({IMAGE_PATH}) + +## Model Usage +### Command Line Execution +* Run using Scepter's SDK, taking care to use different configuration files in accordance with the different base models, as per the corresponding relationships shown below + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Base ModelLORASCETEXT_LORATEXT_SCE
SD1.5lora_cfgsce_cfgtext_lora_cfgtext_sce_cfg
SD2.1lora_cfgsce_cfgtext_lora_cfgtext_sce_cfg
SDXLlora_cfgsce_cfgtext_lora_cfgtext_sce_cfg
+ +* Running from Source Code + +```shell +git clone https://github.com/modelscope/scepter.git +cd scepter +pip install -r requirements/recommended.txt +PYTHONPATH=. python scepter/tools/run_inference.py + --pretrained_model {this model folder} + --cfg {lora_cfg} or {sce_cfg} or {text_lora_cfg} or {text_sce_cfg} + --prompt '{EVAL_PROMPT}' + --save_folder 'inference' +``` + +* Running after Installing Scepter (Recommended) +```shell +pip install scepter +python -m scepter/tools/run_inference.py + --pretrained_model {this model folder} + --cfg {lora_cfg} or {sce_cfg} or {text_lora_cfg} or {text_sce_cfg} + --prompt '{EVAL_PROMPT}' + --save_folder 'inference' +``` +### Running with Scepter Studio + +```shell +pip install scepter +# Launch Scepter Studio +python -m scepter.tools.webui +``` + +* Refer to the following guides for model usage. + +(video url) + +## Model Reference +If you wish to use this model for your own purposes, please cite it as follows. +```bibtex +@misc{{MODEL_NAME}, + title = {{MODEL_NAME}, {MODEL_URL}}, + author = {{USER_NAME}}, + year = {2024} +} +``` +This model was trained using [Scepter Studio](https://github.com/modelscope/scepter); [Scepter](https://github.com/modelscope/scepter) +is an algorithm framework and toolbox developed by the Alibaba Tongyi Wanxiang Team. It provides a suite of tools and models for image generation, editing, fine-tuning, data processing, and more. If you find our work beneficial for your research, +please cite as follows. +```bibtex +@misc{scepter, + title = {SCEPTER, https://github.com/modelscope/scepter}, + author = {SCEPTER}, + year = {2023} +} +``` diff --git a/scepter/methods/studio/tuner_manager/readme_zh.md b/scepter/methods/studio/tuner_manager/readme_zh.md new file mode 100644 index 0000000..204ebf4 --- /dev/null +++ b/scepter/methods/studio/tuner_manager/readme_zh.md @@ -0,0 +1,168 @@ +--- +frameworks: +- Pytorch +license: Apache License 2.0 +tasks: +- efficient-diffusion-tuning +--- + +

+ +

{MODEL_NAME}

+

+
+ +
+

+ +## 模型介绍 +{MODEL_DESCRIPTION} + +## 模型参数 + + + + + + + + + + + + + + + + + + + + + + + + +
基础模型微调类型训练参数
批次大小轮数学习率分辨率
{BASE_MODEL}{TUNER_TYPE}{TRAIN_BATCH_SIZE}{TRAIN_EPOCH}{LEARNING_RATE}[{HEIGHT}, {WIDTH}]
+ + + + + + + + + + + + + + + + + + + +
数据类型数据空间数据名称数据子集
{DATA_TYPE}{MS_DATA_SPACE}{MS_DATA_NAME}{MS_DATA_SUBNAME}
+ + +## 模型效果 + +输入 "{EVAL_PROMPT}",可能会得到如下图像: + +![image]({IMAGE_PATH}) + + +## 模型使用 +### 命令行运行 + +* 使用scepter的sdk进行运行,注意需要按照模型参数中基模型的不同使用不同的配置文件,其对应关系如下 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Base ModelLORASCETEXT_LORATEXT_SCE
SD1.5lora_cfgsce_cfgtext_lora_cfgtext_sce_cfg
SD2.1lora_cfgsce_cfgtext_lora_cfgtext_sce_cfg
SDXLlora_cfgsce_cfgtext_lora_cfgtext_sce_cfg
+ +* 从源码运行 + +```shell +git clone https://github.com/modelscope/scepter.git +cd scepter +pip install -r requirements/recommended.txt +PYTHONPATH=. python scepter/tools/run_inference.py + --pretrained_model {this model folder} + --cfg {lora_cfg} or {sce_cfg} or {text_lora_cfg} or {text_sce_cfg} + --prompt '{EVAL_PROMPT}' + --save_folder 'inference' +``` + +* 安装scepter后运行(推荐) +```shell +pip install scepter +python -m scepter/tools/run_inference.py + --pretrained_model {this model folder} + --cfg {lora_cfg} or {sce_cfg} or {text_lora_cfg} or {text_sce_cfg} + --prompt '{EVAL_PROMPT}' + --save_folder 'inference' +``` +### 使用Scepter Studio运行 +```shell +pip install scepter +启动scepter studio +python -m scepter.tools.webui +``` +* 参考以下指南使用模型 + + +## 模型引用 +如果你想使用该模型应用于自己的场景,请按照如下方式引用该模型。 +```bibtex +@misc{{MODEL_NAME}, + title = {{MODEL_NAME}, {MODEL_URL}}, + author = {{USER_NAME}}, + year = {2024} +} +``` +该模型是基于[Scepter Studio](https://github.com/modelscope/scepter)训练得到;[scepter](https://github.com/modelscope/scepter) +是由阿里巴巴通义万相团队开发的算法框架和工具箱,提供图像生成、编辑、微调、数据处理等一系列工具和模型。如果您觉得我们的工作有益于您的工作, +请按照如下方式引用。 +```bibtex +@misc{scepter, + title = {SCEPTER, https://github.com/modelscope/scepter}, + author = {SCEPTER}, + year = {2023} +} +``` diff --git a/scepter/methods/studio/tuner_manager/tuner_manager.yaml b/scepter/methods/studio/tuner_manager/tuner_manager.yaml index c0daf65..91e61dd 100644 --- a/scepter/methods/studio/tuner_manager/tuner_manager.yaml +++ b/scepter/methods/studio/tuner_manager/tuner_manager.yaml @@ -1,2 +1,14 @@ WORK_DIR: "tuner_manager" +SELF_TRAIN_DIR: "self_train" +EXPORT_DIR: "export_model" TUNER_LIST_YAML: "tuner_list.yaml" +README_EN: "scepter/methods/studio/tuner_manager/readme_en.md" +README_ZH: "scepter/methods/studio/tuner_manager/readme_zh.md" + +BASE_MODEL_VERSION: + - BASE_MODEL: 'SD_XL1.0' + TUNER_TYPE: [ 'TEXT_SCE', 'SCE', 'LORA', 'TEXT_LORA', 'FULL' ] + - BASE_MODEL: 'SD1.5' + TUNER_TYPE: [ 'TEXT_SCE', 'SCE', 'LORA', 'TEXT_LORA', 'FULL' ] + - BASE_MODEL: 'SD2.1' + TUNER_TYPE: [ 'SCE', 'LORA', 'FULL' ] diff --git a/scepter/modules/data/dataset/dataset.py b/scepter/modules/data/dataset/dataset.py index 22e9537..9721406 100644 --- a/scepter/modules/data/dataset/dataset.py +++ b/scepter/modules/data/dataset/dataset.py @@ -101,6 +101,10 @@ class ImageTextPairDataset(BaseDataset): 'NEGTIVE_PROMPT': { 'value': '', 'description': 'The default negtive prompt', + }, + 'DATA_NUM': { + 'value': '', + 'description': '', } } para_dict.update(BaseDataset.para_dict) @@ -108,6 +112,7 @@ class ImageTextPairDataset(BaseDataset): def __init__(self, cfg, logger=None): super(ImageTextPairDataset, self).__init__(cfg, logger=logger) self.p_zero = cfg.get('P_ZERO', 0.0) + self.real_number = cfg.get('DATA_NUM', None) self._default_item = { 'meta': {}, 'prompt': diff --git a/scepter/modules/data/dataset/registry.py b/scepter/modules/data/dataset/registry.py index a1e935e..b64d600 100644 --- a/scepter/modules/data/dataset/registry.py +++ b/scepter/modules/data/dataset/registry.py @@ -288,8 +288,12 @@ class DataObject(object): self.shuffle = False self.data_sampler_config.SEED = seed self.data_sampler_config.BATCH_SIZE = self.batch_size - self.sampler = SAMPLERS.build(self.data_sampler_config, - logger=self.logger) + sampler = SAMPLERS.build(self.data_sampler_config, + logger=self.logger) + if sampler_name.endswith('BatchSampler'): + self.batch_sampler = sampler + else: + self.sampler = sampler def _instantiate_multi_level_batch_sampler(self, sampler_config, batch_size, rank, seed): diff --git a/scepter/modules/data/sampler/__init__.py b/scepter/modules/data/sampler/__init__.py index 40ff88a..428ec1b 100644 --- a/scepter/modules/data/sampler/__init__.py +++ b/scepter/modules/data/sampler/__init__.py @@ -6,4 +6,4 @@ from scepter.modules.data.sampler.registry import SAMPLERS from scepter.modules.data.sampler.sampler import ( EvalDistributedSampler, LoopSampler, MixtureOfSamplers, MultiFoldDistributedSampler, MultiLevelBatchSampler, - MultiLevelBatchSamplerMultiSource) + MultiLevelBatchSamplerMultiSource, ResolutionBatchSampler) diff --git a/scepter/modules/data/sampler/sampler.py b/scepter/modules/data/sampler/sampler.py index 2f34563..b7c5899 100644 --- a/scepter/modules/data/sampler/sampler.py +++ b/scepter/modules/data/sampler/sampler.py @@ -7,7 +7,7 @@ import numbers import os import sys from collections.abc import Iterable -from typing import Optional +from typing import List, Optional import numpy as np import torch @@ -15,6 +15,8 @@ import torch.distributed as dist from scepter.modules.data.sampler.base_sampler import BaseSampler from scepter.modules.data.sampler.registry import SAMPLERS +from scepter.modules.data.utils.data_bucket import (BucketBatchIndex, + BucketManager) from scepter.modules.utils.config import dict_to_yaml from scepter.modules.utils.directory import osp_path from scepter.modules.utils.distribute import we @@ -589,3 +591,105 @@ class LoopSampler(BaseSampler): __class__.__name__, LoopSampler.para_dict, set_name=True) + + +@SAMPLERS.register_class() +class ResolutionBatchSampler(BaseSampler): + para_dict = {} + + def __init__(self, cfg, logger): + super().__init__(cfg, logger) + self.data_file = cfg.DATA_FILE + self.fields = cfg.get('FIELDS', []) + self.num_fields = len(self.fields) + self.delimiter = cfg.get('DELIMITER', ',') + self.path_prefix = cfg.get('PATH_PREFIX', '') + self.batch_size = cfg.BATCH_SIZE + max_reso = cfg.get('MAX_RESO', (1024, 1024)) + min_bucket_reso = cfg.get('MIN_BUCKET_RESO', 256) + max_bucket_reso = cfg.get('MAX_BUCKET_RESO', 1024) + bucket_reso_steps = cfg.get('BUCKET_RESO_STEPS', 64) + bucket_no_upscale = cfg.get('BUCKET_NO_UPSCALE', False) + rank = we.rank + self.rng = np.random.default_rng(self.seed + rank) + assert 'img_path' in self.fields and 'width' in self.fields and 'height' in self.fields + + self.bucket_manager = BucketManager(max_reso=max_reso, + min_size=min_bucket_reso, + max_size=max_bucket_reso, + reso_steps=bucket_reso_steps, + no_upscale=bucket_no_upscale) + if not bucket_no_upscale: + self.bucket_manager.make_buckets() + else: + self.logger.info( + 'min_bucket_reso and max_bucket_reso are ignored if bucket_no_upscale is set, ' + 'because bucket reso is defined by image size automatically / bucket_no_upscale' + ) + + self.data_map = {} + img_path_idx, width_idx, height_idx = self.fields.index( + 'img_path'), self.fields.index('width'), self.fields.index( + 'height') + with FS.get_from(self.data_file) as local_path: + with open(local_path) as f: + for i, line in enumerate(f): + items = line.strip() + item_sp = items.split(self.delimiter, self.num_fields - 1) + img_path, width, height = item_sp[img_path_idx], int( + item_sp[width_idx]), int(item_sp[height_idx]) + item_sp[img_path_idx] = os.path.join( + self.path_prefix, img_path) + bucket_reso, resized_size, ar_error = self.bucket_manager.select_bucket( + width, height) + self.bucket_manager.add_image(reso=bucket_reso, image=i) + self.data_map[i] = item_sp + + for i, (reso, bucket) in enumerate( + zip(self.bucket_manager.resos, self.bucket_manager.buckets)): + count = len(bucket) + if count > 0: + # self.logger.info(f"bucket {i}: resolution {reso}, bucket {bucket}, count: {len(bucket)}") + self.logger.info( + f'bucket {i}: resolution {reso}, count: {len(bucket)}') + + self.buckets_indices: List[BucketBatchIndex] = [] + for bucket_index, (reso, bucket) in enumerate( + zip(self.bucket_manager.resos, self.bucket_manager.buckets)): + batch_count = int(math.ceil(len(bucket) / self.batch_size)) + for batch_index in range(batch_count): + self.buckets_indices.append( + BucketBatchIndex(bucket_index, self.batch_size, + batch_index, reso)) + self.shuffle_buckets() + + def shuffle_buckets(self): + np.random.shuffle(self.buckets_indices) + self.bucket_manager.shuffle() + + def __iter__(self): + while True: + index = self.rng.choice(len(self.buckets_indices)) + bucket_reso = self.buckets_indices[index].bucket_reso + bucket_width, bucket_height = bucket_reso + bucket = self.bucket_manager.buckets[ + self.buckets_indices[index].bucket_index] + batches = self.rng.choice(bucket, self.batch_size) + # image_index = self.buckets_indices[index].batch_index * self.batch_size + # batch = bucket[image_index : image_index + self.batch_size] + fields = self.fields + ['image_size', 'prompt_prefix'] + batches = [ + self.data_map[idx] + [[bucket_height, bucket_width], fields] + for idx in batches + ] + yield batches + + def __len__(self): + return sys.maxsize + + @staticmethod + def get_config_template(): + return dict_to_yaml('SAMPLERS', + __class__.__name__, + ResolutionBatchSampler.para_dict, + set_name=True) diff --git a/scepter/modules/data/utils/__init__.py b/scepter/modules/data/utils/__init__.py new file mode 100644 index 0000000..75399c0 --- /dev/null +++ b/scepter/modules/data/utils/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. + +from scepter.modules.data.utils.data_bucket import BucketManager diff --git a/scepter/modules/data/utils/data_bucket.py b/scepter/modules/data/utils/data_bucket.py new file mode 100644 index 0000000..f27e709 --- /dev/null +++ b/scepter/modules/data/utils/data_bucket.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. + +import math +import random +from typing import List, NamedTuple + +import numpy as np + + +def make_bucket_resolutions(max_reso, + min_size=256, + max_size=1024, + divisible=64): + max_width, max_height = max_reso + max_area = (max_width // divisible) * (max_height // divisible) + + resos = set() + + size = int(math.sqrt(max_area)) * divisible + resos.add((size, size)) + + size = min_size + while size <= max_size: + width = size + height = min(max_size, (max_area // (width // divisible)) * divisible) + resos.add((width, height)) + resos.add((height, width)) + + # # make additional resos + # if width >= height and width - divisible >= min_size: + # resos.add((width - divisible, height)) + # resos.add((height, width - divisible)) + # if height >= width and height - divisible >= min_size: + # resos.add((width, height - divisible)) + # resos.add((height - divisible, width)) + + size += divisible + + resos = list(resos) + resos.sort() + return resos + + +class BucketBatchIndex(NamedTuple): + bucket_index: int + bucket_batch_size: int + batch_index: int + bucket_reso: List[int] + + +class BucketManager: + def __init__(self, + max_reso, + min_size=256, + max_size=1024, + reso_steps=64, + no_upscale=False) -> None: + self.no_upscale = no_upscale + if max_reso is None: + self.max_reso = None + self.max_area = None + else: + self.max_reso = max_reso + self.max_area = max_reso[0] * max_reso[1] + self.min_size = min_size + self.max_size = max_size + self.reso_steps = reso_steps + + self.resos = [] + self.reso_to_id = {} + self.buckets = [] + + def add_image(self, reso, image): + bucket_id = self.reso_to_id[reso] + self.buckets[bucket_id].append(image) + + def shuffle(self): + for bucket in self.buckets: + random.shuffle(bucket) + + def sort(self): + sorted_resos = self.resos.copy() + sorted_resos.sort() + + sorted_buckets = [] + sorted_reso_to_id = {} + for i, reso in enumerate(sorted_resos): + bucket_id = self.reso_to_id[reso] + sorted_buckets.append(self.buckets[bucket_id]) + sorted_reso_to_id[reso] = i + + self.resos = sorted_resos + self.buckets = sorted_buckets + self.reso_to_id = sorted_reso_to_id + + def make_buckets(self): + resos = make_bucket_resolutions(self.max_reso, self.min_size, + self.max_size, self.reso_steps) + self.set_predefined_resos(resos) + + def set_predefined_resos(self, resos): + self.predefined_resos = resos.copy() + self.predefined_resos_set = set(resos) + self.predefined_aspect_ratios = np.array([w / h for w, h in resos]) + + def add_if_new_reso(self, reso): + if reso not in self.reso_to_id: + bucket_id = len(self.resos) + self.reso_to_id[reso] = bucket_id + self.resos.append(reso) + self.buckets.append([]) + # print(reso, bucket_id, len(self.buckets)) + + def round_to_steps(self, x): + x = int(x + 0.5) + return x - x % self.reso_steps + + def select_bucket(self, image_width, image_height): + aspect_ratio = image_width / image_height + if not self.no_upscale: + reso = (image_width, image_height) + if reso in self.predefined_resos_set: + pass + else: + ar_errors = self.predefined_aspect_ratios - aspect_ratio + predefined_bucket_id = np.abs(ar_errors).argmin() + reso = self.predefined_resos[predefined_bucket_id] + + ar_reso = reso[0] / reso[1] + if aspect_ratio > ar_reso: + scale = reso[1] / image_height + else: + scale = reso[0] / image_width + + resized_size = (int(image_width * scale + 0.5), + int(image_height * scale + 0.5)) + # print("use predef", image_width, image_height, reso, resized_size) + else: + if image_width * image_height > self.max_area: + resized_width = math.sqrt(self.max_area * aspect_ratio) + resized_height = self.max_area / resized_width + assert abs(resized_width / resized_height - + aspect_ratio) < 1e-2, 'aspect is illegal' + + b_width_rounded = self.round_to_steps(resized_width) + b_height_in_wr = self.round_to_steps(b_width_rounded / + aspect_ratio) + ar_width_rounded = b_width_rounded / b_height_in_wr + + b_height_rounded = self.round_to_steps(resized_height) + b_width_in_hr = self.round_to_steps(b_height_rounded * + aspect_ratio) + ar_height_rounded = b_width_in_hr / b_height_rounded + + # print(b_width_rounded, b_height_in_wr, ar_width_rounded) + # print(b_width_in_hr, b_height_rounded, ar_height_rounded) + + if abs(ar_width_rounded - + aspect_ratio) < abs(ar_height_rounded - aspect_ratio): + resized_size = (b_width_rounded, + int(b_width_rounded / aspect_ratio + 0.5)) + else: + resized_size = (int(b_height_rounded * aspect_ratio + 0.5), + b_height_rounded) + # print(resized_size) + else: + resized_size = (image_width, image_height) + + bucket_width = resized_size[0] - resized_size[0] % self.reso_steps + bucket_height = resized_size[1] - resized_size[1] % self.reso_steps + # print("use arbitrary", image_width, image_height, resized_size, bucket_width, bucket_height) + + reso = (bucket_width, bucket_height) + self.add_if_new_reso(reso) + + ar_error = (reso[0] / reso[1]) - aspect_ratio + return reso, resized_size, ar_error + + +if __name__ == '__main__': + image_size_list = [(256, 256), (512, 378), (378, 512), (1024, 1024), + (768, 1024), (768, 768), (256, 1024), (512, 512)] + image_path_list = [f'image_path_{i}' for i in range(len(image_size_list))] + + max_reso = (512, 1024) + min_bucket_reso = 256 + max_bucket_reso = 1024 + bucket_reso_steps = 64 + bucket_no_upscale = False + bucket_manager = BucketManager(max_reso=max_reso, + min_size=min_bucket_reso, + max_size=max_bucket_reso, + reso_steps=bucket_reso_steps, + no_upscale=bucket_no_upscale) + if not bucket_no_upscale: + bucket_manager.make_buckets() + else: + print( + 'min_bucket_reso and max_bucket_reso are ignored if bucket_no_upscale is set, ' + 'because bucket reso is defined by image size automatically / bucket_no_upscale' + ) + + for i, (path, size) in enumerate(zip(image_path_list, image_size_list)): + image_width, image_height = size + bucket_reso, resized_size, ar_error = bucket_manager.select_bucket( + image_width, image_height) + print(i, size, bucket_reso, resized_size, ar_error) + bucket_manager.add_image(reso=bucket_reso, image=path) + + for i, (reso, bucket) in enumerate( + zip(bucket_manager.resos, bucket_manager.buckets)): + count = len(bucket) + if count > 0: + print( + f'bucket {i}: resolution {reso}, bucket {bucket}, count: {len(bucket)}' + ) + + batch_size = 2 + buckets_indices: List[BucketBatchIndex] = [] + for bucket_index, bucket in enumerate(bucket_manager.buckets): + batch_count = int(math.ceil(len(bucket) / batch_size)) + for batch_index in range(batch_count): + buckets_indices.append( + BucketBatchIndex(bucket_index, batch_size, batch_index)) + + def shuffle_buckets(): + random.shuffle(buckets_indices) + bucket_manager.shuffle() + + shuffle_buckets() diff --git a/scepter/modules/inference/control_inference.py b/scepter/modules/inference/control_inference.py index 406e6e6..10d8438 100644 --- a/scepter/modules/inference/control_inference.py +++ b/scepter/modules/inference/control_inference.py @@ -94,7 +94,7 @@ class ControlInference(): def get_control_input(self, control_model, control_cond_image, height, width): hints = [] - if control_cond_image and control_model: + if control_cond_image is not None and control_model is not None: if not isinstance(control_model, list): control_model = [control_model] if not isinstance(control_cond_image, list): diff --git a/scepter/modules/inference/diffusion_inference.py b/scepter/modules/inference/diffusion_inference.py index dd5efa9..31f767f 100644 --- a/scepter/modules/inference/diffusion_inference.py +++ b/scepter/modules/inference/diffusion_inference.py @@ -16,6 +16,7 @@ from scepter.modules.model.registry import (BACKBONES, EMBEDDERS, MODELS, from scepter.modules.utils.distribute import we from scepter.modules.utils.file_system import FS +from scepter.studio.utils.env import get_available_memory from .control_inference import ControlInference from .tuner_inference import TunerInference @@ -243,8 +244,19 @@ class DiffusionInference(): def unload(self, module): if module is None: return module - module['model'] = module['model'].to('cpu') - module['device'] = 'cpu' + mem = get_available_memory() + free_mem = int(mem['available'] / (1024**2)) + total_mem = int(mem['total'] / (1024**2)) + if free_mem < 0.5 * total_mem: + if module['model'] is not None: + module['model'] = module['model'].to('cpu') + del module['model'] + module['model'] = None + module['device'] = 'offline' + print('delete module') + else: + module['model'] = module['model'].to('cpu') + module['device'] = 'cpu' torch.cuda.empty_cache() torch.cuda.ipc_collect() return module diff --git a/scepter/modules/inference/largen_inference.py b/scepter/modules/inference/largen_inference.py index afbd454..4034d19 100644 --- a/scepter/modules/inference/largen_inference.py +++ b/scepter/modules/inference/largen_inference.py @@ -16,6 +16,7 @@ from scepter.modules.model.registry import (BACKBONES, EMBEDDERS, MODELS, from scepter.modules.model.utils.data_utils import crop_back from scepter.modules.utils.distribute import we from scepter.modules.utils.file_system import FS +from scepter.studio.utils.env import get_available_memory def get_model(model_tuple): @@ -245,8 +246,19 @@ class LargenInference(): def unload(self, module): if module is None: return module - module['model'] = module['model'].to('cpu') - module['device'] = 'cpu' + mem = get_available_memory() + free_mem = int(mem['available'] / (1024**2)) + total_mem = int(mem['total'] / (1024**2)) + if free_mem < 0.5 * total_mem: + if module['model'] is not None: + module['model'] = module['model'].to('cpu') + del module['model'] + module['model'] = None + module['device'] = 'offline' + print('delete module') + else: + module['model'] = module['model'].to('cpu') + module['device'] = 'cpu' torch.cuda.empty_cache() torch.cuda.ipc_collect() return module diff --git a/scepter/modules/inference/stylebooth_inference.py b/scepter/modules/inference/stylebooth_inference.py new file mode 100644 index 0000000..cd96f6b --- /dev/null +++ b/scepter/modules/inference/stylebooth_inference.py @@ -0,0 +1,663 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +import copy +import os.path +import random +from collections import OrderedDict + +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from PIL.Image import Image + +from scepter.modules.model.network.diffusion.diffusion import GaussianDiffusion +from scepter.modules.model.network.diffusion.schedules import noise_schedule +from scepter.modules.model.registry import (BACKBONES, EMBEDDERS, MODELS, + TOKENIZERS) +from scepter.modules.utils.distribute import we +from scepter.modules.utils.file_system import FS + +from ...studio.utils.env import get_available_memory +from .control_inference import ControlInference +from .tuner_inference import TunerInference + + +def get_model(model_tuple): + assert 'model' in model_tuple + return model_tuple['model'] + + +class StyleboothInference(): + ''' + define vae, unet, text-encoder, tuner, refiner components + support to load the components dynamicly. + create and load model when run this model at the first time. + ''' + def __init__(self, logger=None): + self.logger = logger + self.loaded_model = {} + self.loaded_model_name = [ + 'diffusion_model', 'first_stage_model', 'cond_stage_model' + ] + self.tuner_infer = TunerInference(self.logger) + self.control_infer = ControlInference(self.logger) + + def init_from_cfg(self, cfg): + self.name = cfg.NAME + self.is_default = cfg.get('IS_DEFAULT', False) + module_paras = self.load_default(cfg.get('DEFAULT_PARAS', None)) + assert cfg.have('MODEL') + cfg.MODEL = self.redefine_paras(cfg.MODEL) + self.diffusion = self.load_schedule(cfg.MODEL.SCHEDULE) + self.diffusion_model = self.infer_model( + cfg.MODEL.DIFFUSION_MODEL, module_paras.get( + 'DIFFUSION_MODEL', + None)) if cfg.MODEL.have('DIFFUSION_MODEL') else None + self.first_stage_model = self.infer_model( + cfg.MODEL.FIRST_STAGE_MODEL, + module_paras.get( + 'FIRST_STAGE_MODEL', + None)) if cfg.MODEL.have('FIRST_STAGE_MODEL') else None + self.cond_stage_model = self.infer_model( + cfg.MODEL.COND_STAGE_MODEL, + module_paras.get( + 'COND_STAGE_MODEL', + None)) if cfg.MODEL.have('COND_STAGE_MODEL') else None + self.refiner_cond_model = self.infer_model( + cfg.MODEL.REFINER_COND_MODEL, + module_paras.get( + 'REFINER_COND_MODEL', + None)) if cfg.MODEL.have('REFINER_COND_MODEL') else None + self.refiner_diffusion_model = self.infer_model( + cfg.MODEL.REFINER_MODEL, module_paras.get( + 'REFINER_MODEL', + None)) if cfg.MODEL.have('REFINER_MODEL') else None + self.tokenizer = TOKENIZERS.build( + cfg.MODEL.TOKENIZER, + logger=self.logger) if cfg.MODEL.have('TOKENIZER') else None + + if self.tokenizer is not None: + self.cond_stage_model['cfg'].KWARGS = { + 'vocab_size': self.tokenizer.vocab_size + } + + def redefine_paras(self, cfg): + if cfg.get('PRETRAINED_MODEL', None): + assert FS.isfile(cfg.PRETRAINED_MODEL) + with FS.get_from(cfg.PRETRAINED_MODEL, + wait_finish=True) as local_path: + if local_path.endswith('safetensors'): + from safetensors.torch import load_file as load_safetensors + sd = load_safetensors(local_path) + else: + sd = torch.load(local_path, map_location='cpu') + first_stage_model_path = os.path.join( + os.path.dirname(local_path), 'first_stage_model.pth') + cond_stage_model_path = os.path.join( + os.path.dirname(local_path), 'cond_stage_model.pth') + diffusion_model_path = os.path.join( + os.path.dirname(local_path), 'diffusion_model.pth') + if (not os.path.exists(first_stage_model_path) + or not os.path.exists(cond_stage_model_path) + or not os.path.exists(diffusion_model_path)): + self.logger.info( + 'Now read the whole model and rearrange the modules, it may take several mins.' + ) + first_stage_model = OrderedDict() + cond_stage_model = OrderedDict() + diffusion_model = OrderedDict() + for k, v in sd.items(): + if k.startswith('first_stage_model.'): + first_stage_model[k.replace( + 'first_stage_model.', '')] = v + elif k.startswith('conditioner.'): + cond_stage_model[k.replace('conditioner.', '')] = v + elif k.startswith('cond_stage_model.'): + if k.startswith('cond_stage_model.model.'): + cond_stage_model[k.replace( + 'cond_stage_model.model.', '')] = v + else: + cond_stage_model[k.replace( + 'cond_stage_model.', '')] = v + elif k.startswith('model.diffusion_model.'): + diffusion_model[k.replace('model.diffusion_model.', + '')] = v + else: + continue + if cfg.have('FIRST_STAGE_MODEL'): + with open(first_stage_model_path + 'cache', 'wb') as f: + torch.save(first_stage_model, f) + os.rename(first_stage_model_path + 'cache', + first_stage_model_path) + self.logger.info( + 'First stage model has been processed.') + if cfg.have('COND_STAGE_MODEL'): + with open(cond_stage_model_path + 'cache', 'wb') as f: + torch.save(cond_stage_model, f) + os.rename(cond_stage_model_path + 'cache', + cond_stage_model_path) + self.logger.info( + 'Cond stage model has been processed.') + if cfg.have('DIFFUSION_MODEL'): + with open(diffusion_model_path + 'cache', 'wb') as f: + torch.save(diffusion_model, f) + os.rename(diffusion_model_path + 'cache', + diffusion_model_path) + self.logger.info('Diffusion model has been processed.') + if not cfg.FIRST_STAGE_MODEL.get('PRETRAINED_MODEL', None): + cfg.FIRST_STAGE_MODEL.PRETRAINED_MODEL = first_stage_model_path + else: + cfg.FIRST_STAGE_MODEL.RELOAD_MODEL = first_stage_model_path + if not cfg.COND_STAGE_MODEL.get('PRETRAINED_MODEL', None): + cfg.COND_STAGE_MODEL.PRETRAINED_MODEL = cond_stage_model_path + else: + cfg.COND_STAGE_MODEL.RELOAD_MODEL = cond_stage_model_path + if not cfg.DIFFUSION_MODEL.get('PRETRAINED_MODEL', None): + cfg.DIFFUSION_MODEL.PRETRAINED_MODEL = diffusion_model_path + else: + cfg.DIFFUSION_MODEL.RELOAD_MODEL = diffusion_model_path + return cfg + + def init_from_modules(self, modules): + for k, v in modules.items(): + self.__setattr__(k, v) + + def infer_model(self, cfg, module_paras=None): + module = { + 'model': None, + 'cfg': cfg, + 'device': 'offline', + 'name': cfg.NAME, + 'function_info': {}, + 'paras': {} + } + if module_paras is None: + return module + function_info = {} + paras = { + k.lower(): v + for k, v in module_paras.get('PARAS', {}).items() + } + for function in module_paras.get('FUNCTION', []): + input_dict = {} + for inp in function.get('INPUT', []): + if inp.lower() in self.input: + input_dict[inp.lower()] = self.input[inp.lower()] + function_info[function.NAME] = { + 'dtype': function.get('DTYPE', 'float32'), + 'input': input_dict + } + module['paras'] = paras + module['function_info'] = function_info + return module + + def init_from_ckpt(self, path, model, ignore_keys=list()): + if path.endswith('safetensors'): + from safetensors.torch import load_file as load_safetensors + sd = load_safetensors(path) + else: + sd = torch.load(path, map_location='cpu') + + new_sd = OrderedDict() + for k, v in sd.items(): + ignored = False + for ik in ignore_keys: + if ik in k: + if we.rank == 0: + self.logger.info( + 'Ignore key {} from state_dict.'.format(k)) + ignored = True + break + if not ignored: + new_sd[k] = v + + missing, unexpected = model.load_state_dict(new_sd, strict=False) + if we.rank == 0: + self.logger.info( + f'Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys' + ) + if len(missing) > 0: + self.logger.info(f'Missing Keys:\n {missing}') + if len(unexpected) > 0: + self.logger.info(f'\nUnexpected Keys:\n {unexpected}') + + def load(self, module): + if module['device'] == 'offline': + if module['cfg'].NAME in MODELS.class_map: + model = MODELS.build(module['cfg'], logger=self.logger).eval() + elif module['cfg'].NAME in BACKBONES.class_map: + model = BACKBONES.build(module['cfg'], + logger=self.logger).eval() + elif module['cfg'].NAME in EMBEDDERS.class_map: + model = EMBEDDERS.build(module['cfg'], + logger=self.logger).eval() + else: + raise NotImplementedError + if module['cfg'].get('RELOAD_MODEL', None): + self.init_from_ckpt(module['cfg'].RELOAD_MODEL, model) + module['model'] = model + module['device'] = 'cpu' + if module['device'] == 'cpu': + module['device'] = we.device_id + module['model'] = module['model'].to(we.device_id) + return module + + def unload(self, module): + if module is None: + return module + mem = get_available_memory() + free_mem = int(mem['available'] / (1024**2)) + total_mem = int(mem['total'] / (1024**2)) + if free_mem < 0.5 * total_mem: + if module['model'] is not None: + module['model'] = module['model'].to('cpu') + del module['model'] + module['model'] = None + module['device'] = 'offline' + print('delete module') + else: + module['model'] = module['model'].to('cpu') + module['device'] = 'cpu' + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + return module + + def dynamic_load(self, module=None, name=''): + self.logger.info('Loading {} model'.format(name)) + if name == 'all': + for subname in self.loaded_model_name: + self.loaded_model[subname] = self.dynamic_load( + getattr(self, subname), subname) + elif name in self.loaded_model_name: + if name in self.loaded_model: + if module['cfg'] != self.loaded_model[name]['cfg']: + self.unload(self.loaded_model[name]) + module = self.load(module) + self.loaded_model[name] = module + return module + elif module['device'] == 'cpu': + module = self.load(module) + return module + else: + return module + else: + module = self.load(module) + self.loaded_model[name] = module + return module + else: + return self.load(module) + + def dynamic_unload(self, module=None, name='', skip_loaded=False): + self.logger.info('Unloading {} model'.format(name)) + if name == 'all': + for name, module in self.loaded_model.items(): + module = self.unload(self.loaded_model[name]) + self.loaded_model[name] = module + elif name in self.loaded_model_name: + if name in self.loaded_model: + if not skip_loaded: + module = self.unload(self.loaded_model[name]) + self.loaded_model[name] = module + else: + self.unload(module) + else: + self.unload(module) + + def load_default(self, cfg): + module_paras = {} + if cfg is not None: + self.paras = cfg.PARAS + self.input = {k.lower(): v for k, v in cfg.INPUT.items()} + self.output = {k.lower(): v for k, v in cfg.OUTPUT.items()} + module_paras = cfg.MODULES_PARAS + return module_paras + + def load_schedule(self, cfg): + parameterization = cfg.get('PARAMETERIZATION', 'eps') + assert parameterization in [ + 'eps', 'x0', 'v' + ], 'currently only supporting "eps" and "x0" and "v"' + num_timesteps = cfg.get('TIMESTEPS', 1000) + + schedule_args = { + k.lower(): v + for k, v in cfg.get('SCHEDULE_ARGS', { + 'NAME': 'logsnr_cosine_interp', + 'SCALE_MIN': 2.0, + 'SCALE_MAX': 4.0 + }).items() + } + + zero_terminal_snr = cfg.get('ZERO_TERMINAL_SNR', False) + if zero_terminal_snr: + assert parameterization == 'v', 'Now zero_terminal_snr only support v-prediction mode.' + sigmas = noise_schedule(schedule=schedule_args.pop('name'), + n=num_timesteps, + zero_terminal_snr=zero_terminal_snr, + **schedule_args) + diffusion = GaussianDiffusion(sigmas=sigmas, + prediction_type=parameterization) + return diffusion + + def get_batch(self, value_dict, num_samples=1): + batch = {} + batch_uc = {} + N = num_samples + device = we.device_id + for key in value_dict: + if key == 'prompt': + batch['prompt'] = value_dict['prompt'] + batch_uc['prompt'] = value_dict['negative_prompt'] + elif key == 'original_size_as_tuple': + batch['original_size_as_tuple'] = (torch.tensor( + value_dict['original_size_as_tuple']).to(device).repeat( + N, 1)) + elif key == 'crop_coords_top_left': + batch['crop_coords_top_left'] = (torch.tensor( + value_dict['crop_coords_top_left']).to(device).repeat( + N, 1)) + elif key == 'aesthetic_score': + batch['aesthetic_score'] = (torch.tensor( + [value_dict['aesthetic_score']]).to(device).repeat(N, 1)) + batch_uc['aesthetic_score'] = (torch.tensor([ + value_dict['negative_aesthetic_score'] + ]).to(device).repeat(N, 1)) + + elif key == 'target_size_as_tuple': + batch['target_size_as_tuple'] = (torch.tensor( + value_dict['target_size_as_tuple']).to(device).repeat( + N, 1)) + elif key == 'image': + batch[key] = self.load_image(value_dict[key], num_samples=N) + else: + batch[key] = value_dict[key] + + for key in batch.keys(): + if key not in batch_uc and isinstance(batch[key], torch.Tensor): + batch_uc[key] = torch.clone(batch[key]) + return batch, batch_uc + + def load_image(self, image, num_samples=1): + if isinstance(image, torch.Tensor): + pass + elif isinstance(image, Image): + pass + elif isinstance(image, Image): + pass + + def get_function_info(self, module, function_name=None): + all_function = module['function_info'] + if function_name in all_function: + return function_name, all_function[function_name]['dtype'] + if function_name is None and len(all_function) == 1: + for k, v in all_function.items(): + return k, v['dtype'] + + def encode_first_stage(self, x, **kwargs): + _, dtype = self.get_function_info(self.first_stage_model, 'encode') + with torch.autocast('cuda', + enabled=dtype == 'float16', + dtype=getattr(torch, dtype)): + z = get_model(self.first_stage_model).encode(x) + return self.first_stage_model['paras']['scale_factor'] * z + + def decode_first_stage(self, z): + _, dtype = self.get_function_info(self.first_stage_model, 'decode') + with torch.autocast('cuda', + enabled=dtype == 'float16', + dtype=getattr(torch, dtype)): + z = 1. / self.first_stage_model['paras']['scale_factor'] * z + return get_model(self.first_stage_model).decode(z) + + def encode_condition(self, data, data2=None, type='text'): + cond_stage_model = get_model(self.cond_stage_model) + assert hasattr(self, 'tokenizer') + with torch.autocast(device_type='cuda', enabled=False): + if type == 'image' and ( + hasattr(cond_stage_model, 'build_new_tokens') + and not hasattr(cond_stage_model, 'new_tokens_to_ids')): + cond_stage_model.build_new_tokens(self.tokenizer) + + if type == 'text': + text = self.tokenizer(data).to(we.device_id) + return cond_stage_model.encode_text(text) + elif type == 'image': + return cond_stage_model.encode_image(data) + elif type == 'hybrid': + text = self.tokenizer(data).to(we.device_id) + return cond_stage_model.encode_text(text, data2) + + def process_edit_image(self, images, height, width): + if not isinstance(images, list): + images = [images] + tensors = [] + for img in images: + w, h = img.size + if not h == height or not w == width: + scale = max(width / w, height / h) + new_size = (int(h * scale), int(w * scale)) + img = TF.resize(img, + new_size, + interpolation=TF.InterpolationMode.BICUBIC) + img = TF.center_crop(img, (height, width)) + tensor = TF.to_tensor(img).to(we.device_id) + tensors.append(tensor) + tensors = TF.normalize(torch.stack(tensors), + mean=[0.5, 0.5, 0.5], + std=[0.5, 0.5, 0.5]) + return tensors + + @torch.no_grad() + def __call__(self, + input, + num_samples=1, + intermediate_callback=None, + refine_strength=0, + img_to_img_strength=0, + cat_uc=True, + tuner_model=None, + control_model=None, + style_edit_image=None, + style_exemplar_image=None, + style_guide_scale_text=None, + style_guide_scale_image=None, + **kwargs): + + value_input = copy.deepcopy(self.input) + value_input.update(input) + print(value_input) + height, width = value_input['target_size_as_tuple'] + value_output = copy.deepcopy(self.output) + batch, batch_uc = self.get_batch(value_input, num_samples=1) + + # register tuner + if tuner_model is not None and tuner_model != '' and len( + tuner_model) > 0: + if not isinstance(tuner_model, list): + tuner_model = [tuner_model] + self.dynamic_load(self.diffusion_model, 'diffusion_model') + self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + self.tuner_infer.register_tuner(tuner_model, self.diffusion_model, + self.cond_stage_model) + self.dynamic_unload(self.diffusion_model, + 'diffusion_model', + skip_loaded=True) + self.dynamic_unload(self.cond_stage_model, + 'cond_stage_model', + skip_loaded=True) + + # register control + if control_model is not None and control_model != '': + self.dynamic_load(self.diffusion_model, 'diffusion_model') + self.control_infer.register_controllers(control_model, + self.diffusion_model) + self.dynamic_unload(self.diffusion_model, + 'diffusion_model', + skip_loaded=True) + + # first stage encode + image = input.pop('image', None) + if image is not None and img_to_img_strength > 0: + # run image2image + b, c, ori_width, ori_height = image.shape + if not (ori_width == width and ori_height == height): + image = F.interpolate(image, (width, height), mode='bicubic') + self.dynamic_load(self.first_stage_model, 'first_stage_model') + input_latent = self.encode_first_stage(image) + self.dynamic_unload(self.first_stage_model, + 'first_stage_model', + skip_loaded=True) + else: + input_latent = None + if 'input_latent' in value_output and input_latent is not None: + value_output['input_latent'] = input_latent + + # cond stage + self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + context = {} + if style_exemplar_image is not None: + if not isinstance(style_exemplar_image, list): + style_exemplar_image = [style_exemplar_image] + style_exemplar_image = [ + TF.resize(x, (224, 224), + interpolation=TF.InterpolationMode.BICUBIC) + for x in style_exemplar_image + ] + style_exemplar_image = [ + TF.to_tensor(x).to(we.device_id) for x in style_exemplar_image + ] + style_exemplar_image = TF.normalize( + torch.stack(style_exemplar_image), + mean=[0.48145466, 0.4578275, 0.40821073], + std=[0.26862954, 0.26130258, 0.27577711]) + image_feature = self.encode_condition(style_exemplar_image, + type='image') + context['crossattn'] = self.encode_condition(batch['prompt'], + image_feature, + type='hybrid') + else: + context['crossattn'] = self.encode_condition(batch['prompt']) + null_context = {} + null_context['crossattn'] = self.encode_condition(batch_uc['prompt']) + + self.dynamic_unload(self.cond_stage_model, + 'cond_stage_model', + skip_loaded=True) + + model_kwargs = [{'cond': context}] + + # style first stage encode + if style_edit_image is not None: + style_edit_image = self.process_edit_image(style_edit_image, + height, width) + self.dynamic_load(self.first_stage_model, 'first_stage_model') + cond_concat = self.encode_first_stage(style_edit_image) + cond_concat /= self.first_stage_model['paras']['scale_factor'] + self.dynamic_unload(self.first_stage_model, + 'first_stage_model', + skip_loaded=True) + + context['concat'] = cond_concat + null_context['concat'] = torch.zeros_like(cond_concat) + mid_context = {} + mid_context.update(null_context) + mid_context.update({'concat': cond_concat}) + model_kwargs.append({'cond': mid_context}) + model_kwargs.append({'cond': null_context}) + + # get noise + seed = kwargs.pop('seed', -1) + g = torch.Generator(device=we.device_id) + seed = seed if seed >= 0 else random.randint(0, 2**32 - 1) + g.manual_seed(seed) + if 'seed' in value_output: + value_output['seed'] = seed + for sample_id in range(num_samples): + if self.diffusion_model is not None: + noise = torch.empty( + 1, + 4, + height // self.first_stage_model['paras']['size_factor'], + width // self.first_stage_model['paras']['size_factor'], + device=we.device_id).normal_(generator=g) + + self.dynamic_load(self.diffusion_model, 'diffusion_model') + # UNet use input n_prompt + function_name, dtype = self.get_function_info( + self.diffusion_model) + with torch.autocast('cuda', + enabled=dtype == 'float16', + dtype=getattr(torch, dtype)): + latent = self.diffusion.sample( + noise=noise, + x=input_latent, + denoising_strength=img_to_img_strength + if input_latent is not None else 1.0, + refine_strength=refine_strength, + solver=value_input.get('sample', 'ddim'), + model=get_model(self.diffusion_model), + model_kwargs=model_kwargs, + steps=value_input.get('sample_steps', 50), + guide_scale={ + 'text': style_guide_scale_text, + 'image': style_guide_scale_image + }, + guide_rescale=value_input.get('guide_rescale', 0.5), + discretization=value_input.get('discretization', + 'trailing'), + show_progress=True, + seed=seed, + condition_fn=None, + clamp=None, + sharpness=value_input.get('sharpness', 0.0), + percentile=None, + t_max=None, + t_min=None, + discard_penultimate_step=None, + intermediate_callback=intermediate_callback, + cat_uc=value_input.get('cat_uc', cat_uc), + **kwargs) + + self.dynamic_unload(self.diffusion_model, + 'diffusion_model', + skip_loaded=True) + + if 'latent' in value_output: + if value_output['latent'] is None or ( + isinstance(value_output['latent'], list) + and len(value_output['latent']) < 1): + value_output['latent'] = [] + value_output['latent'].append(latent) + + self.dynamic_load(self.first_stage_model, 'first_stage_model') + x_samples = self.decode_first_stage(latent).float() + self.dynamic_unload(self.first_stage_model, + 'first_stage_model', + skip_loaded=True) + images = torch.clamp((x_samples + 1.0) / 2.0, min=0.0, max=1.0) + if 'images' in value_output: + if value_output['images'] is None or ( + isinstance(value_output['images'], list) + and len(value_output['images']) < 1): + value_output['images'] = [] + value_output['images'].append(images) + + for k, v in value_output.items(): + if isinstance(v, list): + value_output[k] = torch.cat(v, dim=0) + if isinstance(v, torch.Tensor): + value_output[k] = v.cpu() + + # unregister tuner + if tuner_model is not None and tuner_model != '' and len( + tuner_model) > 0: + self.tuner_infer.unregister_tuner(tuner_model, + self.diffusion_model, + self.cond_stage_model) + + # unregister control + if control_model is not None and control_model != '': + self.control_infer.unregister_controllers(control_model, + self.diffusion_model) + + return value_output diff --git a/scepter/modules/model/backbone/unet/unet_utils.py b/scepter/modules/model/backbone/unet/unet_utils.py index 50ab617..bfe5307 100644 --- a/scepter/modules/model/backbone/unet/unet_utils.py +++ b/scepter/modules/model/backbone/unet/unet_utils.py @@ -13,8 +13,9 @@ import torch.nn.functional as F import torchvision.transforms.functional as TF from einops import rearrange, repeat from packaging import version +from torch.utils.checkpoint import checkpoint -from scepter.modules.model.utils.basic_utils import checkpoint, default, exists +from scepter.modules.model.utils.basic_utils import default, exists try: import xformers @@ -360,8 +361,10 @@ class ResBlock(TimestepBlock): :param emb: an [N x emb_channels] Tensor of timestep embeddings. :return: an [N x C x ...] Tensor of outputs. """ - return checkpoint(self._forward, (x, emb), self.parameters(), - self.use_checkpoint) + if self.use_checkpoint: + return checkpoint(self._forward, x, emb) + else: + return self._forward(x, emb) def _forward(self, x, emb): if self.updown: @@ -422,8 +425,10 @@ class AttentionBlock(nn.Module): self.proj_out = zero_module(conv_nd(1, channels, channels, 1)) def forward(self, x): - return checkpoint(self._forward, (x, ), self.parameters(), - self.use_checkpoint) + if self.use_checkpoint: + return checkpoint(self._forward, x) + else: + return self._forward(x) def _forward(self, x): b, c, *spatial = x.shape @@ -986,8 +991,11 @@ class BasicTransformerBlock(nn.Module): self.use_checkpoint = use_checkpoint def forward(self, x, context=None): - return checkpoint(self._forward, (x, context), self.parameters(), - self.use_checkpoint) + + if self.use_checkpoint: + return checkpoint(self._forward, x, context) + else: + return self._forward(x, context) def _forward(self, x, context=None): x = self.attn1(self.norm1(x), diff --git a/scepter/modules/model/embedder/__init__.py b/scepter/modules/model/embedder/__init__.py index 959417a..f7b27b9 100644 --- a/scepter/modules/model/embedder/__init__.py +++ b/scepter/modules/model/embedder/__init__.py @@ -1,10 +1,7 @@ # -*- coding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from scepter.modules.model.embedder.embedder import (ConcatTimestepEmbedderND, - FrozenCLIPEmbedder, - FrozenOpenCLIPEmbedder, - FrozenOpenCLIPEmbedder2, - GeneralConditioner, - IPAdapterPlusEmbedder, - RefCrossEmbedder) +from scepter.modules.model.embedder.embedder import ( + ConcatTimestepEmbedderND, FrozenCLIPEmbedder, FrozenOpenCLIPEmbedder, + FrozenOpenCLIPEmbedder2, GeneralConditioner, IPAdapterPlusEmbedder, + RefCrossEmbedder) diff --git a/scepter/modules/model/head/__init__.py b/scepter/modules/model/head/__init__.py index cfd2b15..9f70332 100644 --- a/scepter/modules/model/head/__init__.py +++ b/scepter/modules/model/head/__init__.py @@ -1,8 +1,5 @@ # -*- coding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from scepter.modules.model.head.classifier_head import (ClassifierHead, - CosineLinearHead, - TransformerHead, - TransformerHeadx2, - VideoClassifierHead, - VideoClassifierHeadx2) +from scepter.modules.model.head.classifier_head import ( + ClassifierHead, CosineLinearHead, TransformerHead, TransformerHeadx2, + VideoClassifierHead, VideoClassifierHeadx2) diff --git a/scepter/modules/model/network/autoencoder/ae_kl.py b/scepter/modules/model/network/autoencoder/ae_kl.py index 8f0c2eb..bd88d14 100644 --- a/scepter/modules/model/network/autoencoder/ae_kl.py +++ b/scepter/modules/model/network/autoencoder/ae_kl.py @@ -52,7 +52,6 @@ class DiagonalGaussianDistribution(object): dim=dims) def mode(self): - # print('*** use DiagonalGaussianDistribution.mode() ***') return self.mean diff --git a/scepter/modules/model/network/diffusion/diffusion.py b/scepter/modules/model/network/diffusion/diffusion.py index 1eb7d12..bc98256 100644 --- a/scepter/modules/model/network/diffusion/diffusion.py +++ b/scepter/modules/model/network/diffusion/diffusion.py @@ -448,22 +448,23 @@ class GaussianDiffusion(object): # denoising t = self._sigma_to_t(sigma).repeat(len(xt)).round().long() - if isinstance( - model_kwargs[0]['cond'], dict) and \ - 'tar_x0' in model_kwargs[0]['cond'] and \ - 'tar_mask_latent' in model_kwargs[0]['cond']: - tar_x0 = model_kwargs[0]['cond']['tar_x0'] - tar_mask = model_kwargs[0]['cond']['tar_mask_latent'] + if isinstance(model_kwargs, list) and len(model_kwargs) == 2: + if isinstance( + model_kwargs[0]['cond'], dict) and \ + 'tar_x0' in model_kwargs[0]['cond'] and \ + 'tar_mask_latent' in model_kwargs[0]['cond']: + tar_x0 = model_kwargs[0]['cond']['tar_x0'] + tar_mask = model_kwargs[0]['cond']['tar_mask_latent'] - tar_xt = self.diffuse(x0=tar_x0, t=t) - xt = tar_xt * (1.0 - tar_mask) + xt * tar_mask + tar_xt = self.diffuse(x0=tar_x0, t=t) + xt = tar_xt * (1.0 - tar_mask) + xt * tar_mask - if isinstance(model_kwargs[0]['cond'], - dict) and 'ref_x0' in model_kwargs[0]['cond']: - model_kwargs[0]['cond']['ref_xt'] = self.diffuse( - x0=model_kwargs[0]['cond']['ref_x0'], t=t) - model_kwargs[1]['cond']['ref_xt'] = self.diffuse( - x0=model_kwargs[1]['cond']['ref_x0'], t=t) + if isinstance(model_kwargs[0]['cond'], + dict) and 'ref_x0' in model_kwargs[0]['cond']: + model_kwargs[0]['cond']['ref_xt'] = self.diffuse( + x0=model_kwargs[0]['cond']['ref_x0'], t=t) + model_kwargs[1]['cond']['ref_xt'] = self.diffuse( + x0=model_kwargs[1]['cond']['ref_x0'], t=t) if solver in ('onestep', 'multistep', 'multistep2', 'multistep3'): x0 = self.denoise(xt, diff --git a/scepter/modules/model/network/ldm/ldm.py b/scepter/modules/model/network/ldm/ldm.py index 4063bf5..4a6964f 100644 --- a/scepter/modules/model/network/ldm/ldm.py +++ b/scepter/modules/model/network/ldm/ldm.py @@ -336,7 +336,7 @@ class LatentDiffusion(TrainModule): h = int(meta['image_size'][0][0]) w = int(meta['image_size'][1][0]) image_size = [h, w] - if 'image_size' in kwargs: + if 'image_size' in kwargs and kwargs['image_size'] is not None: image_size = kwargs.pop('image_size') if isinstance(image_size, numbers.Number): image_size = [image_size, image_size] diff --git a/scepter/modules/model/utils/basic_utils.py b/scepter/modules/model/utils/basic_utils.py index 932739e..9448a91 100644 --- a/scepter/modules/model/utils/basic_utils.py +++ b/scepter/modules/model/utils/basic_utils.py @@ -2,8 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from inspect import isfunction -import torch - def exists(x): return x is not None @@ -15,62 +13,6 @@ def default(val, d): return d() if isfunction(d) else d -def checkpoint(func, inputs, params, flag): - """ - Evaluate a function without caching intermediate activations, allowing for - reduced memory at the expense of extra compute in the backward pass. - :param func: the function to evaluate. - :param inputs: the argument sequence to pass to `func`. - :param params: a sequence of parameters `func` depends on but does not - explicitly take as arguments. - :param flag: if False, disable gradient checkpointing. - """ - if flag: - args = tuple(inputs) + tuple(params) - return CheckpointFunction.apply(func, len(inputs), *args) - else: - return func(*inputs) - - -class CheckpointFunction(torch.autograd.Function): - @staticmethod - def forward(ctx, run_function, length, *args): - ctx.run_function = run_function - ctx.input_tensors = list(args[:length]) - ctx.input_params = list(args[length:]) - ctx.gpu_autocast_kwargs = { - 'enabled': torch.is_autocast_enabled(), - 'dtype': torch.get_autocast_gpu_dtype(), - 'cache_enabled': torch.is_autocast_cache_enabled() - } - with torch.no_grad(): - output_tensors = ctx.run_function(*ctx.input_tensors) - return output_tensors - - @staticmethod - def backward(ctx, *output_grads): - ctx.input_tensors = [ - x.detach().requires_grad_(True) for x in ctx.input_tensors - ] - with torch.enable_grad(), \ - torch.cuda.amp.autocast(**ctx.gpu_autocast_kwargs): - # Fixes a bug where the first op in run_function modifies the - # Tensor storage in place, which is not allowed for detach()'d - # Tensors. - shallow_copies = [x.view_as(x) for x in ctx.input_tensors] - output_tensors = ctx.run_function(*shallow_copies) - input_grads = torch.autograd.grad( - output_tensors, - ctx.input_tensors + ctx.input_params, - output_grads, - allow_unused=True, - ) - del ctx.input_tensors - del ctx.input_params - del output_tensors - return (None, None) + input_grads - - def disabled_train(self, mode=True): """Overwrite model.train with this function to make sure train/eval mode does not change anymore.""" diff --git a/scepter/modules/opt/optimizers/__init__.py b/scepter/modules/opt/optimizers/__init__.py index e51388a..675bcbb 100644 --- a/scepter/modules/opt/optimizers/__init__.py +++ b/scepter/modules/opt/optimizers/__init__.py @@ -1,10 +1,7 @@ # -*- coding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from scepter.modules.opt.optimizers.official_optimizers import (ASGD, LBFGS, - SGD, Adadelta, - Adagrad, Adam, - Adamax, AdamW, - RMSprop, Rprop, - SparseAdam) +from scepter.modules.opt.optimizers.official_optimizers import ( + ASGD, LBFGS, SGD, Adadelta, Adagrad, Adam, Adamax, AdamW, RMSprop, Rprop, + SparseAdam) from scepter.modules.opt.optimizers.registry import OPTIMIZERS diff --git a/scepter/modules/transform/image.py b/scepter/modules/transform/image.py index a0a7c5a..fea07d0 100644 --- a/scepter/modules/transform/image.py +++ b/scepter/modules/transform/image.py @@ -10,16 +10,11 @@ import torchvision.transforms as transforms import torchvision.transforms.functional as TF from scepter.modules.transform.registry import TRANSFORMS -from scepter.modules.transform.utils import (BACKEND_CV2, BACKEND_PILLOW, - BACKEND_TORCHVISION, - INPUT_CV2_TYPE_WARNING, - INPUT_PIL_TYPE_WARNING, - INPUT_TENSOR_TYPE_WARNING, - INTERPOLATION_STYLE, - INTERPOLATION_STYLE_CV2, - TORCHVISION_CAPABILITY, - is_cv2_image, is_pil_image, - is_tensor) +from scepter.modules.transform.utils import ( + BACKEND_CV2, BACKEND_PILLOW, BACKEND_TORCHVISION, INPUT_CV2_TYPE_WARNING, + INPUT_PIL_TYPE_WARNING, INPUT_TENSOR_TYPE_WARNING, INTERPOLATION_STYLE, + INTERPOLATION_STYLE_CV2, TORCHVISION_CAPABILITY, is_cv2_image, + is_pil_image, is_tensor) from scepter.modules.utils.config import dict_to_yaml if TORCHVISION_CAPABILITY: diff --git a/scepter/modules/transform/tensor.py b/scepter/modules/transform/tensor.py index 51568b5..ea281e2 100644 --- a/scepter/modules/transform/tensor.py +++ b/scepter/modules/transform/tensor.py @@ -260,6 +260,7 @@ class RenameMeta(object): self.input_key = cfg.INPUT_KEY self.output_key = cfg.OUTPUT_KEY self.force = cfg.get('FORCE', False) + self.move = cfg.get('MOVE', False) def __call__(self, item): if 'meta' in item: @@ -270,10 +271,16 @@ class RenameMeta(object): have_key_set = set(self.input_key) else: have_key_set = set(self.input_key + self.output_key) - for k, v in item['meta'].items(): - if k not in have_key_set: - data[k] = v - item['meta'] = data + if not self.move: + for k, v in item['meta'].items(): + if k not in have_key_set: + data[k] = v + item['meta'] = data + else: + for k, v in item.items(): + if k not in have_key_set: + data[k] = v + item.update(data) return item @staticmethod diff --git a/scepter/modules/utils/config.py b/scepter/modules/utils/config.py index da19f47..a109fdb 100644 --- a/scepter/modules/utils/config.py +++ b/scepter/modules/utils/config.py @@ -11,6 +11,13 @@ import yaml from scepter.modules.utils.model import StdMsg +_SECURE_KEYWORDS = [ + 'ENDPOINT', 'BUCKET', 'OSS_AK', 'OSS_SK', 'OSS', 'TOKEN', 'APPKEY' + 'SECRET', 'ACCESS_ID', 'ACCESS_KEY', 'PASSWORD', 'TEMP_DIR' +] # -> "*****" + +_SECURE_VALUEWORDS = ['oss://', 'oss-'] # -> "#####" + def dict_to_yaml(module_name, name, json_config, set_name=False): ''' @@ -517,8 +524,29 @@ class Config(object): def __repr__(self): return '{}\n'.format(self.dump()) - def dump(self): - return json.dumps(self.cfg_dict, indent=2) + def dump(self, is_secure=False): + if not is_secure: + return json.dumps(self.cfg_dict, indent=2) + else: + + def make_secure(cfg): + if isinstance(cfg, dict): + for key, val in cfg.items(): + if key in _SECURE_KEYWORDS and type(val) is str: + cfg[key] = '*****' + else: + cfg[key] = make_secure(cfg[key]) + elif isinstance(cfg, list): + cfg = [make_secure(t) for t in cfg] + elif isinstance(cfg, str): + for sval in _SECURE_VALUEWORDS: + if sval in cfg: + cfg = '#####' + return cfg + + cfg_dict_copy = copy.deepcopy(self.cfg_dict) + cfg_dict_copy = make_secure(cfg_dict_copy) + return json.dumps(cfg_dict_copy, indent=2) def deep_copy(self): return copy.deepcopy(self) @@ -604,5 +632,8 @@ class Config(object): else: return cfg + def __len__(self): + return len(self.cfg_dict) + def pop(self, name): self.cfg_dict.pop(name) diff --git a/scepter/modules/utils/file_clients/http_fs.py b/scepter/modules/utils/file_clients/http_fs.py index 7dcab7f..a6ad579 100644 --- a/scepter/modules/utils/file_clients/http_fs.py +++ b/scepter/modules/utils/file_clients/http_fs.py @@ -131,7 +131,7 @@ class HttpFs(BaseFs): worker_id=0) -> Optional[str]: raise NotImplementedError - def get_url(self, target_path, lifecycle=3600 * 100): + def get_url(self, target_path, set_public=False, lifecycle=3600 * 100): return target_path def exists(self, target_path) -> bool: diff --git a/scepter/modules/utils/file_clients/huggingface_fs.py b/scepter/modules/utils/file_clients/huggingface_fs.py index 150e1b1..a7375cf 100644 --- a/scepter/modules/utils/file_clients/huggingface_fs.py +++ b/scepter/modules/utils/file_clients/huggingface_fs.py @@ -181,7 +181,7 @@ class HuggingfaceFs(BaseFs): delimiter=None) -> (Union[bytes, str, None], Optional[int]): raise NotImplementedError - def get_url(self, target_path, lifecycle=3600 * 100): + def get_url(self, target_path, set_public=False, lifecycle=3600 * 100): return target_path def exists(self, target_path) -> bool: diff --git a/scepter/modules/utils/file_clients/modelscope_fs.py b/scepter/modules/utils/file_clients/modelscope_fs.py index 33a9f51..f71233c 100644 --- a/scepter/modules/utils/file_clients/modelscope_fs.py +++ b/scepter/modules/utils/file_clients/modelscope_fs.py @@ -199,7 +199,7 @@ class ModelscopeFs(BaseFs): delimiter=None) -> (Union[bytes, str, None], Optional[int]): raise NotImplementedError - def get_url(self, target_path, lifecycle=3600 * 100): + def get_url(self, target_path, set_public=False, lifecycle=3600 * 100): return target_path def exists(self, target_path) -> bool: diff --git a/scepter/studio/inference/inference.py b/scepter/studio/inference/inference.py index d71dbc7..324bd79 100644 --- a/scepter/studio/inference/inference.py +++ b/scepter/studio/inference/inference.py @@ -20,11 +20,13 @@ from scepter.studio.inference.inference_ui.largen_ui import LargenUI from scepter.studio.inference.inference_ui.mantra_ui import MantraUI from scepter.studio.inference.inference_ui.model_manage_ui import ModelManageUI from scepter.studio.inference.inference_ui.refiner_ui import RefinerUI +from scepter.studio.inference.inference_ui.stylebooth_ui import StyleboothUI from scepter.studio.inference.inference_ui.tuner_ui import TunerUI from scepter.studio.utils.env import init_env UI_MAP = [('diffusion', DiffusionUI), ('mantra', MantraUI), ('tuner', TunerUI), - ('control', ControlUI), ('refiner', RefinerUI), ('largen', LargenUI)] + ('control', ControlUI), ('refiner', RefinerUI), ('largen', LargenUI), + ('stylebooth', StyleboothUI)] class InferenceUI(): @@ -108,14 +110,16 @@ class InferenceUI(): self.tab_ui_kwargs[f'{name}_ui'] = ui self.__setattr__(f'{name}_ui', ui) - self.check_box_controlled_tabs = ['mantra', 'tuner', 'control', 'largen'] + self.check_box_controlled_tabs = [ + 'mantra', 'tuner', 'control', 'largen', 'stylebooth' + ] self.pipe_manager = pipe_manager assert len(self.component_names.check_box_for_setting) == len( self.check_box_controlled_tabs) def create_ui(self): # create model - self.model_manage_ui.create_ui() + self.model_manage_ui.create_ui(gallery_ui=self.gallery_ui) self.gallery_ui.create_ui() self.infer_info = gr.State(value=None) @@ -123,10 +127,10 @@ class InferenceUI(): def create_tab(name, ui): label = getattr(self.component_names, f'{name}_paras') if name in ['refiner']: - ui.create_ui() + ui.create_ui(gallery_ui=self.gallery_ui) else: with gr.TabItem(label=label, id=f'{name}_ui'): - ui.create_ui() + ui.create_ui(gallery_ui=self.gallery_ui) with gr.Row(variant='panel', equal_height=True): with gr.Accordion(label=self.component_names.advance_block_name, @@ -140,8 +144,10 @@ class InferenceUI(): def set_callbacks(self, manager): self.model_manage_ui.set_callbacks(**self.tab_ui_kwargs) - self.gallery_ui.set_callbacks(self, self.model_manage_ui, - **self.tab_ui_kwargs) + self.gallery_ui.set_callbacks(self, + self.model_manage_ui, + **self.tab_ui_kwargs, + manager=manager) for name, ui in self.tab_ui_kwargs.items(): ui.set_callbacks(self.model_manage_ui, **self.tab_ui_kwargs, @@ -152,8 +158,14 @@ class InferenceUI(): selected_tab = 'diffusion_ui' ui_tabs_state = [False] * len(args) largen_index = self.check_box_controlled_tabs.index('largen') - largen_key = self.component_names.check_box_for_setting[largen_index] + largen_key = self.component_names.check_box_for_setting[ + largen_index] largen_status = args[largen_index] + stylebooth_index = self.check_box_controlled_tabs.index( + 'stylebooth') + stylebooth_key = self.component_names.check_box_for_setting[ + stylebooth_index] + stylebooth_status = args[stylebooth_index] for key in check_box: i = self.component_names.check_box_for_setting.index(key) ui_tabs_state[i] = True @@ -163,16 +175,17 @@ class InferenceUI(): for key in check_box: i = self.component_names.check_box_for_setting.index(key) if ui_tabs_state[i] != args[i]: - if i in [largen_index]: + if i in [largen_index, stylebooth_index]: new_check_box_value = [key] for j in range(len(ui_tabs_state)): ui_tabs_state[j] = j == i else: new_check_box_value = [ k for k in check_box - if k not in [largen_key] + if k not in [largen_key, stylebooth_key] ] ui_tabs_state[largen_index] = False + ui_tabs_state[stylebooth_index] = False ui_tabs_updates = [gr.update(visible=v) for v in ui_tabs_state] @@ -183,7 +196,14 @@ class InferenceUI(): default_choices['diffusion_model']['choices'], value='LARGEN_LargenUNetXL', interactive=False) - elif largen_status: + elif ui_tabs_state[stylebooth_index]: + diffusion_model = gr.Dropdown( + label=self.model_manage_ui.component_names.diffusion_model, + choices=self.model_manage_ui. + default_choices['diffusion_model']['choices'], + value='EDIT_DiffusionUNet', + interactive=False) + elif largen_status or stylebooth_status: diffusion_model = gr.Dropdown( label=self.model_manage_ui.component_names.diffusion_model, choices=self.model_manage_ui. @@ -203,7 +223,8 @@ class InferenceUI(): choices=self.component_names.check_box_for_setting, value=new_check_box_value, show_label=False), gr.update( - selected=selected_tab), *ui_tabs_state, *ui_tabs_updates, diffusion_model + selected=selected_tab + ), *ui_tabs_state, *ui_tabs_updates, diffusion_model gr_states = [ self.tab_ui[name].state for name in self.check_box_controlled_tabs @@ -213,9 +234,14 @@ class InferenceUI(): ] self.check_box_for_setting.change( change_setting_tab, - inputs=[self.check_box_for_setting, self.model_manage_ui.diffusion_model, *gr_states], - outputs=[self.check_box_for_setting, self.setting_tab, *gr_states, - *gr_tabs, self.model_manage_ui.diffusion_model], + inputs=[ + self.check_box_for_setting, + self.model_manage_ui.diffusion_model, *gr_states + ], + outputs=[ + self.check_box_for_setting, self.setting_tab, *gr_states, + *gr_tabs, self.model_manage_ui.diffusion_model + ], queue=False) diff --git a/scepter/studio/inference/inference_manager/infer_runer.py b/scepter/studio/inference/inference_manager/infer_runer.py index deb4c75..6aa4112 100644 --- a/scepter/studio/inference/inference_manager/infer_runer.py +++ b/scepter/studio/inference/inference_manager/infer_runer.py @@ -2,6 +2,7 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from scepter.modules.inference.diffusion_inference import DiffusionInference from scepter.modules.inference.largen_inference import LargenInference +from scepter.modules.inference.stylebooth_inference import StyleboothInference from scepter.modules.utils.logger import get_logger @@ -99,6 +100,8 @@ class PipelineManager(): pipeline_name = cfg.NAME if 'LARGEN' in pipeline_name: PipelineBuilder = LargenInference + elif pipeline_name.startswith('EDIT'): + PipelineBuilder = StyleboothInference else: PipelineBuilder = DiffusionInference new_inference = PipelineBuilder(logger=self.logger) diff --git a/scepter/studio/inference/inference_ui/component_names.py b/scepter/studio/inference/inference_ui/component_names.py index 3e5b2a3..12fbcec 100644 --- a/scepter/studio/inference/inference_ui/component_names.py +++ b/scepter/studio/inference/inference_ui/component_names.py @@ -24,7 +24,8 @@ class InferenceUIName(): if language == 'en': self.advance_block_name = 'Advance Setting' self.check_box_for_setting = [ - 'Use Mantra', 'Use Tuners', 'Use Controller', 'LAR-Gen' + 'Use Mantra', 'Use Tuners', 'Use Controller', 'LAR-Gen', + 'StyleBooth' ] self.diffusion_paras = 'Generation Setting' self.mantra_paras = 'Mantra Book' @@ -32,15 +33,19 @@ class InferenceUIName(): self.control_paras = 'Controlable Generation' self.refiner_paras = 'Refiner Setting' self.largen_paras = 'LAR-Gen' + self.stylebooth_paras = 'StyleBooth' elif language == 'zh': self.advance_block_name = '生成选项' - self.check_box_for_setting = ['使用咒语', '使用微调', '使用控制', 'LAR-Gen'] + self.check_box_for_setting = [ + '使用咒语', '使用微调', '使用控制', 'LAR-Gen', 'StyleBooth' + ] self.diffusion_paras = '生成参数设置' self.mantra_paras = '咒语书' self.tuner_paras = '微调模型' self.control_paras = '可控生成' self.refiner_paras = 'Refine设置' self.largen_paras = 'LAR-Gen' + self.stylebooth_paras = 'StyleBooth' class ModelManageUIName(): @@ -499,3 +504,82 @@ class LargenUIName(): 1024 ], ] + + +class StyleboothUIName(): + def __init__(self, language='en'): + if language == 'en': + self.dropdown_name = 'Application' + self.apps = ['Text-based Style Editing'] + # self.apps = ["Text-based Style Editing", "Exemplar-based Style Editing"] + self.source_image = 'Source Image' + self.exemplar_image = 'Exemplar Image' + self.ins_format = 'Instruction Format (select or rewrite, en only)' + self.style_format = '{} (select or rewrite, en only)' + self.guide_scale_image = 'Guide Scale For Uncondition Image' + self.guide_scale_text = 'Guide Scale For Uncondition Text' + self.guide_rescale = 'Guide Rescale' + self.resolution = 'Resolution of Short Edge' + self.compose_button = 'Assemble Style Editing Instruction to Prompt' + elif language == 'zh': + self.dropdown_name = '应用' + self.apps = ['根据文本编辑风格'] + # self.apps = ["根据文本编辑风格", "根据风格样例编辑风格"] + self.source_image = '源图片' + self.exemplar_image = '样例图片' + self.ins_format = '指令模版 (选择或者新写,仅英文)' + self.style_format = '{} (选择或者新写,仅英文)' + self.guide_scale_image = '图片条件引导比例' + self.guide_scale_text = '文本条件引导比例' + self.guide_rescale = '引导缩放' + self.resolution = '短边分辨率' + self.compose_button = '组装风格编辑指令到Prompt栏' + self.tb_ins_format_choice = [ + 'Let this image be in the style of