From 01fd8335afd98f861ee5123ecfa38f641f33bf03 Mon Sep 17 00:00:00 2001 From: "zeyinzi.jzyz" Date: Thu, 18 Jul 2024 14:12:42 +0800 Subject: [PATCH] v1.0.3 update --- requirements/framework.txt | 3 +- requirements/scepter_studio.txt | 2 +- scepter/methods/edit/edit_512_lora.yaml | 2 +- .../examples/classification/example.yaml | 4 +- .../dit_pixart_alpha_1024_lora.yaml | 223 ++ .../examples/generation/dit_sd3_1024.yaml | 233 ++ .../generation/dit_sd3_1024_lora.yaml | 241 ++ .../generation/stable_diffusion_1.5_512.yaml | 7 +- .../stable_diffusion_1.5_512_lora.yaml | 7 +- .../stable_diffusion_1.5_512_textlora.yaml | 7 +- .../generation/stable_diffusion_2.1_512.yaml | 7 +- .../stable_diffusion_2.1_512_lora.yaml | 7 +- .../generation/stable_diffusion_2.1_768.yaml | 7 +- .../stable_diffusion_2.1_768_lora.yaml | 7 +- .../generation/stable_diffusion_xl_1024.yaml | 7 +- .../stable_diffusion_xl_1024_lora.yaml | 7 +- .../stable_diffusion_xl_1024_textlora.yaml | 7 +- .../scedit/ctr/sd15_512_sce_ctr_hed.yaml | 5 +- .../scedit/ctr/sd21_768_sce_ctr_canny.yaml | 5 +- .../scedit/ctr/sd21_768_sce_ctr_pose.yaml | 5 +- .../scedit/ctr/sdxl_1024_sce_ctr_canny.yaml | 5 +- .../scedit/ctr/sdxl_1024_sce_ctr_color.yaml | 5 +- .../ctr/sdxl_1024_sce_ctr_color_datatxt.yaml | 5 +- .../scedit/ctr/sdxl_1024_sce_ctr_depth.yaml | 5 +- .../methods/scedit/t2i/sd15_512_sce_t2i.yaml | 7 +- .../scedit/t2i/sd15_512_sce_t2i_swift.yaml | 7 +- .../t2i/sd15_512_textsce_t2i_swift.yaml | 7 +- .../methods/scedit/t2i/sd21_768_sce_t2i.yaml | 7 +- .../scedit/t2i/sd21_768_sce_t2i_swift.yaml | 7 +- .../methods/scedit/t2i/sdxl_1024_sce_t2i.yaml | 7 +- .../scedit/t2i/sdxl_1024_sce_t2i_datatxt.yaml | 5 +- .../scedit/t2i/sdxl_1024_sce_t2i_swift.yaml | 7 +- .../t2i/sdxl_1024_textsce_t2i_swift.yaml | 7 +- .../extensions/mantra_book/mantra_book.yaml | 2138 ++++++++++++++++- .../inference/dit/pixart_alpha_pro.yaml | 120 + .../methods/studio/inference/dit/sd3_pro.yaml | 148 ++ .../self_train/dit/pixart_alpha_pro.yaml | 264 ++ .../studio/self_train/dit/sd3_pro.yaml | 284 +++ .../self_train/edit/stylebooth_tb_pro.yaml | 7 +- .../studio/self_train/sd_xl/sdxl_pro.yaml | 5 +- .../self_train/stable_diffusion/sd15_pro.yaml | 5 +- .../self_train/stable_diffusion/sd21_pro.yaml | 5 +- .../modules/inference/diffusion_inference.py | 13 +- scepter/modules/inference/largen_inference.py | 10 +- scepter/modules/inference/pixart_inference.py | 209 ++ scepter/modules/inference/sd3_inference.py | 189 ++ .../modules/inference/stylebooth_inference.py | 10 +- scepter/modules/model/backbone/__init__.py | 4 +- .../modules/model/backbone/mmdit/__init__.py | 2 + scepter/modules/model/backbone/mmdit/sd3.py | 1330 ++++++++++ .../modules/model/backbone/pixart/__init__.py | 2 + .../model/backbone/pixart/pixart_alpha.py | 503 ++++ .../model/backbone/transformer/__init__.py | 0 .../model/backbone/transformer/attention.py | 760 ++++++ .../model/backbone/transformer/layers.py | 303 +++ .../model/backbone/transformer/patchify.py | 54 + .../model/backbone/transformer/pos_embed.py | 129 + scepter/modules/model/embedder/__init__.py | 6 +- scepter/modules/model/embedder/embedder.py | 306 ++- scepter/modules/model/network/__init__.py | 3 +- .../model/network/autoencoder/ae_kl.py | 42 +- .../model/network/diffusion/diffusion.py | 273 ++- .../model/network/diffusion/schedules.py | 13 +- .../model/network/diffusion/solvers.py | 8 +- scepter/modules/model/network/ldm/__init__.py | 2 + scepter/modules/model/network/ldm/ldm.py | 37 +- .../modules/model/network/ldm/ldm_pixart.py | 272 +++ scepter/modules/model/network/ldm/ldm_sd3.py | 236 ++ scepter/modules/model/tokenizer/tokenizer.py | 2 +- .../model/tokenizer/tokenizer_component.py | 132 + scepter/modules/solver/diffusion_solver.py | 11 +- scepter/modules/solver/hooks/checkpoint.py | 5 +- scepter/modules/utils/model.py | 8 +- .../inference_manager/infer_runer.py | 6 + .../inference/inference_ui/component_names.py | 28 +- .../inference/inference_ui/control_ui.py | 4 +- .../inference/inference_ui/gallery_ui.py | 4 +- .../inference/inference_ui/largen_ui.py | 37 +- .../inference/inference_ui/mantra_ui.py | 6 +- .../inference/inference_ui/model_manage_ui.py | 7 +- .../inference/inference_ui/stylebooth_ui.py | 5 +- .../studio/inference/inference_ui/tuner_ui.py | 7 +- .../caption_editor_ui/create_dataset_ui.py | 28 +- .../caption_editor_ui/dataset_gallery_ui.py | 23 +- .../self_train_ui/component_names.py | 12 +- .../self_train/self_train_ui/model_ui.py | 8 +- .../self_train/self_train_ui/trainer_ui.py | 233 +- .../tuner_manager/manager_ui/browser_ui.py | 28 +- .../tuner_manager/manager_ui/info_ui.py | 7 +- scepter/tools/convert_tuner.py | 2 - scepter/tools/process_watcher.py | 4 +- scepter/tools/webui.py | 1 - scepter/version.py | 2 +- tests/modules/test_diffusion_inference.py | 17 +- 94 files changed, 8776 insertions(+), 417 deletions(-) create mode 100644 scepter/methods/examples/generation/dit_pixart_alpha_1024_lora.yaml create mode 100644 scepter/methods/examples/generation/dit_sd3_1024.yaml create mode 100644 scepter/methods/examples/generation/dit_sd3_1024_lora.yaml create mode 100644 scepter/methods/studio/inference/dit/pixart_alpha_pro.yaml create mode 100644 scepter/methods/studio/inference/dit/sd3_pro.yaml create mode 100644 scepter/methods/studio/self_train/dit/pixart_alpha_pro.yaml create mode 100644 scepter/methods/studio/self_train/dit/sd3_pro.yaml create mode 100644 scepter/modules/inference/pixart_inference.py create mode 100644 scepter/modules/inference/sd3_inference.py create mode 100644 scepter/modules/model/backbone/mmdit/__init__.py create mode 100644 scepter/modules/model/backbone/mmdit/sd3.py create mode 100644 scepter/modules/model/backbone/pixart/__init__.py create mode 100644 scepter/modules/model/backbone/pixart/pixart_alpha.py create mode 100644 scepter/modules/model/backbone/transformer/__init__.py create mode 100644 scepter/modules/model/backbone/transformer/attention.py create mode 100644 scepter/modules/model/backbone/transformer/layers.py create mode 100644 scepter/modules/model/backbone/transformer/patchify.py create mode 100644 scepter/modules/model/backbone/transformer/pos_embed.py create mode 100644 scepter/modules/model/network/ldm/ldm_pixart.py create mode 100644 scepter/modules/model/network/ldm/ldm_sd3.py diff --git a/requirements/framework.txt b/requirements/framework.txt index 2085b6c..f01dc07 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -1,7 +1,8 @@ albumentations +beautifulsoup4 bezier einops -modelscope +modelscope==1.14.0 ms-swift>=2.0.1 numpy open_clip_torch diff --git a/requirements/scepter_studio.txt b/requirements/scepter_studio.txt index ddb7200..85766f4 100644 --- a/requirements/scepter_studio.txt +++ b/requirements/scepter_studio.txt @@ -1,5 +1,5 @@ bitsandbytes -gradio>=3.47.1,<4.0.0 +gradio imagehash psutil tiktoken diff --git a/scepter/methods/edit/edit_512_lora.yaml b/scepter/methods/edit/edit_512_lora.yaml index c18310c..49535bb 100644 --- a/scepter/methods/edit/edit_512_lora.yaml +++ b/scepter/methods/edit/edit_512_lora.yaml @@ -20,7 +20,7 @@ SOLVER: # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - diff --git a/scepter/methods/examples/classification/example.yaml b/scepter/methods/examples/classification/example.yaml index 3f4bc6c..20d9849 100644 --- a/scepter/methods/examples/classification/example.yaml +++ b/scepter/methods/examples/classification/example.yaml @@ -11,7 +11,7 @@ SOLVER: # NUM_FOLDS DESCRIPTION: Num folds for training. TYPE: int default: 0 NUM_FOLDS: 1 # WORK_DIR DESCRIPTION: Save dir of the training log or model. TYPE: str default: '' - WORK_DIR: ./exp12/ + WORK_DIR: ./cache/save_data/example/ LOG_FILE: std_log.txt # EVAL_INTERVAL DESCRIPTION: Eval the model interval. TYPE: int default: 1 EVAL_INTERVAL: 1 @@ -102,7 +102,7 @@ SOLVER: # DATASET DESCRIPTION: the public dataset name TYPE: str default: 'cifar10' DATASET: cifar10 # DATA_ROOT DESCRIPTION: the download data save path TYPE: str default: '' - DATA_ROOT: ./local_data/cifar10 + DATA_ROOT: ./cache/cache_data/cifar10 # MODE DESCRIPTION: test TYPE: str default: test MODE: test # PIN_MEMORY DESCRIPTION: pin_memory for data loader TYPE: bool default: False diff --git a/scepter/methods/examples/generation/dit_pixart_alpha_1024_lora.yaml b/scepter/methods/examples/generation/dit_pixart_alpha_1024_lora.yaml new file mode 100644 index 0000000..b8a4378 --- /dev/null +++ b/scepter/methods/examples/generation/dit_pixart_alpha_1024_lora.yaml @@ -0,0 +1,223 @@ +ENV: + BACKEND: nccl +# +SOLVER: + NAME: LatentDiffusionSolver + RESUME_FROM: + LOAD_MODEL_ONLY: True + USE_FSDP: False + SHARDING_STRATEGY: + USE_AMP: True + DTYPE: float16 + CHANNELS_LAST: True + MAX_STEPS: 1000 + MAX_EPOCHS: -1 + NUM_FOLDS: 1 + ACCU_STEP: 1 + EVAL_INTERVAL: 100 + RESCALE_LR: False + # + WORK_DIR: ./cache/save_data/dit_pixart_alpha_1024_lora + LOG_FILE: std_log.txt + # + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/cache_data" + # + FREEZE: + # + TUNER: + - NAME: SwiftLoRA + R: 128 + LORA_ALPHA: 128 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "model.*(.q|.k|.v|.o|mlp.fc1|mlp.fc2)$" + # + MODEL: + NAME: LatentDiffusionPixart + PARAMETERIZATION: eps + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: + IGNORE_KEYS: [ ] + SCALE_FACTOR: 0.18215 + SIZE_FACTOR: 8 + DECODER_BIAS: 0.5 + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "linear" + "BETA_MIN": 0.0001 + "BETA_MAX": 0.02 + USE_EMA: False + LOAD_REFINER: False + # + DIFFUSION_MODEL: + NAME: PixArt + PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@PixArt-XL-2-1024-MS.pth + INPUT_SIZE: 128 + PATCH_SIZE: 2 + IN_CHANNELS: 4 + HIDDEN_SIZE: 1152 + DEPTH: 28 + NUM_HEADS: 16 + MLP_RATIO: 4.0 + CLASS_DROPOUT_PROB: 0.1 + PRED_SIGMA: True + DROP_PATH: 0.0 + WINDOW_DIZE: 0 + USE_REL_POS: False + CAPTION_CHANNELS: 4096 + LEWEI_SCALE: 2 + MODEL_MAX_LENGTH: 120 + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-2-base@512-base-ema.safetensors + EMBED_DIM: 4 + IGNORE_KEYS: [ ] + BATCH_SIZE: 1 + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + COND_STAGE_MODEL: + NAME: T5EmbedderHF + PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/ + TOKENIZER_PATH: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/ + LENGTH: 120 + CLEAN: heavy + USE_GRAD: False + # + LOSS: + NAME: ReconstructLoss + LOSS_TYPE: l2 + # + SAMPLE_ARGS: + SAMPLER: ddim + SAMPLE_STEPS: 20 + SEED: 2024 + GUIDE_SCALE: 4.5 + GUIDE_RESCALE: 0.5 + DISCRETIZATION: trailing + RUN_TRAIN_N: False + # + OPTIMIZER: + NAME: AdamW + LEARNING_RATE: 0.0001 + BETAS: [ 0.9, 0.999 ] + EPS: 1e-8 + WEIGHT_DECAY: 1e-2 + AMSGRAD: False + # + TRAIN_DATA: + NAME: ImageTextPairMSDataset + MODE: train + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_DATASET_SPLIT: train + MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' } + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + SAMPLER: + NAME: LoopSampler + TRANSFORMS: + - NAME: LoadImageFromFile + RGB_ORDER: RGB + BACKEND: pillow + - NAME: FlexibleResize + INTERPOLATION: bilinear + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: FlexibleCenterCrop + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: ImageToTensor + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: Normalize + MEAN: [ 0.5, 0.5, 0.5 ] + STD: [ 0.5, 0.5, 0.5 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'image' ] + BACKEND: torchvision + - NAME: Select + KEYS: [ 'image', 'prompt' ] + META_KEYS: [ 'data_key' ] + # + EVAL_DATA: + NAME: Text2ImageDataset + MODE: eval + PROMPT_FILE: + PROMPT_DATA: [ "a boy wearing a jacket", "a dog running on the lawn" ] + IMAGE_SIZE: [ 1024, 1024 ] + FIELDS: [ "prompt" ] + DELIMITER: '#;#' + PROMPT_PREFIX: '' + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + TRANSFORMS: + - NAME: Select + KEYS: [ 'index', 'prompt' ] + META_KEYS: [ 'image_size' ] + # + TRAIN_HOOKS: + - + NAME: BackwardHook + PRIORITY: 0 + - + NAME: LogHook + LOG_INTERVAL: 10 + SHOW_GPU_MEM: True + - + NAME: TensorboardLogHook + - + NAME: CheckpointHook + INTERVAL: 10000 + PRIORITY: 200 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + DISABLE_SNAPSHOT: True + # + EVAL_HOOKS: + - + NAME: ProbeDataHook + PROB_INTERVAL: 100 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + SAVE_PROBE_PREFIX: 'image' diff --git a/scepter/methods/examples/generation/dit_sd3_1024.yaml b/scepter/methods/examples/generation/dit_sd3_1024.yaml new file mode 100644 index 0000000..26c4660 --- /dev/null +++ b/scepter/methods/examples/generation/dit_sd3_1024.yaml @@ -0,0 +1,233 @@ +ENV: + BACKEND: nccl +# +SOLVER: + NAME: LatentDiffusionSolver + RESUME_FROM: + LOAD_MODEL_ONLY: True + USE_FSDP: False + SHARDING_STRATEGY: + USE_AMP: True + DTYPE: float16 + CHANNELS_LAST: True + MAX_STEPS: 500 + MAX_EPOCHS: -1 + NUM_FOLDS: 1 + ACCU_STEP: 1 + EVAL_INTERVAL: 50 + RESCALE_LR: False + # + WORK_DIR: ./cache/save_data/dit_sd3_1024 + LOG_FILE: std_log.txt + # + FILE_SYSTEM: + - NAME: "ModelscopeFs" + TEMP_DIR: ./cache/cache_data + # + MODEL: + NAME: LatentDiffusionSD3 + PARAMETERIZATION: rf + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: + IGNORE_KEYS: [ ] + SCALE_FACTOR: 1.5305 + SHIFT_FACTOR: 0.0609 + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "shifted" + "SHIFT": 3 + USE_EMA: False + T_WEIGHT: uniform + # + DIFFUSION_MODEL: + NAME: MMDiT + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors + IGNORE_KEYS: '^first_stage_model.' + IN_CHANNELS: 16 + PATCH_SIZE: 2 + OUT_CHANNELS: 16 + DEPTH: 24 + INPUT_SIZE: + ADM_IN_CHANNELS: 2048 + CONTEXT_EMBEDDER_CONFIG: { 'target': 'torch.nn.Linear', 'params': { 'in_features': 4096, 'out_features': 1536 } } + NUM_PATCHES: 36864 + POS_EMBED_MAX_SIZE: 192 + POS_EMBED_SCALING_FACTOR: + USE_CHECKPOINT: True + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors + EMBED_DIM: 16 + IGNORE_KEYS: '^model.diffusion_model.' + BATCH_SIZE: 1 + USE_CONV: False + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 16 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 16 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + COND_STAGE_MODEL: + NAME: SD3TextEmbedder + P_ZERO: 0.0 + CLIP_L: + NAME: FrozenCLIPEmbedder2 + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer + MAX_LENGTH: 77 + FREEZE: True + LAYER: penultimate + RETURN_POOLED: True + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + CLIP_G: + NAME: FrozenCLIPEmbedder2 + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_2 + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_2 + MAX_LENGTH: 77 + FREEZE: True + LAYER: penultimate + RETURN_POOLED: True + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + T5_XXL: + NAME: T5EmbedderHF + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_3 + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_3 + LENGTH: 256 + CLEAN: whitespace + USE_GRAD: False + T5_DTYPE: float16 + # + LOSS: + NAME: ReconstructLoss + LOSS_TYPE: l2 + # + SAMPLE_ARGS: + SAMPLER: euler + SAMPLE_STEPS: 28 + SEED: 1749023094 + GUIDE_SCALE: 5.0 + GUIDE_RESCALE: 0.0 + DISCRETIZATION: trailing + RUN_TRAIN_N: False + # + OPTIMIZER: + NAME: AdamW + LEARNING_RATE: 1e-5 + BETAS: [ 0.9, 0.999 ] + EPS: 1e-8 + WEIGHT_DECAY: 1e-2 + AMSGRAD: False + # + TRAIN_DATA: + NAME: ImageTextPairMSDataset + MODE: train + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_DATASET_SPLIT: train + MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' } + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + SAMPLER: + NAME: LoopSampler + TRANSFORMS: + - NAME: LoadImageFromFile + RGB_ORDER: RGB + BACKEND: pillow + - NAME: FlexibleResize + INTERPOLATION: bilinear + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: FlexibleCenterCrop + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: ImageToTensor + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: Normalize + MEAN: [ 0.5, 0.5, 0.5 ] + STD: [ 0.5, 0.5, 0.5 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'image' ] + BACKEND: torchvision + - NAME: Select + KEYS: [ 'image', 'prompt' ] + META_KEYS: [ 'data_key' ] + # + EVAL_DATA: + NAME: Text2ImageDataset + MODE: eval + PROMPT_FILE: + PROMPT_DATA: [ "a cat holds a blackboard that writes \"hello world\"", "a dog running on the lawn" ] + IMAGE_SIZE: [ 1024, 1024 ] + FIELDS: [ "prompt" ] + DELIMITER: '#;#' + PROMPT_PREFIX: '' + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + TRANSFORMS: + - NAME: Select + KEYS: [ 'index', 'prompt' ] + META_KEYS: [ 'image_size' ] + # + TRAIN_HOOKS: + - + NAME: BackwardHook + PRIORITY: 10000 + - + NAME: LogHook + LOG_INTERVAL: 10 + SHOW_GPU_MEM: True + - + NAME: TensorboardLogHook + - + NAME: CheckpointHook + INTERVAL: 10000 + PRIORITY: 200 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + DISABLE_SNAPSHOT: True + # + EVAL_HOOKS: + - + NAME: ProbeDataHook + PROB_INTERVAL: 50 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + SAVE_PROBE_PREFIX: 'image' diff --git a/scepter/methods/examples/generation/dit_sd3_1024_lora.yaml b/scepter/methods/examples/generation/dit_sd3_1024_lora.yaml new file mode 100644 index 0000000..d79c633 --- /dev/null +++ b/scepter/methods/examples/generation/dit_sd3_1024_lora.yaml @@ -0,0 +1,241 @@ +ENV: + BACKEND: nccl +# +SOLVER: + NAME: LatentDiffusionSolver + RESUME_FROM: + LOAD_MODEL_ONLY: True + USE_FSDP: False + SHARDING_STRATEGY: + USE_AMP: True + DTYPE: float16 + CHANNELS_LAST: True + MAX_STEPS: 500 + MAX_EPOCHS: -1 + NUM_FOLDS: 1 + ACCU_STEP: 1 + EVAL_INTERVAL: 50 + RESCALE_LR: False + # + WORK_DIR: ./cache/save_data/dit_sd3_1024_lora + LOG_FILE: std_log.txt + # + FILE_SYSTEM: + - NAME: "ModelscopeFs" + TEMP_DIR: ./cache/cache_data + # + TUNER: + - NAME: SwiftLoRA + R: 128 + LORA_ALPHA: 128 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "model.*(.attn.qkv|.attn.proj|mlp.fc1|mlp.fc2)$" + # + MODEL: + NAME: LatentDiffusionSD3 + PARAMETERIZATION: rf + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: + IGNORE_KEYS: [ ] + SCALE_FACTOR: 1.5305 + SHIFT_FACTOR: 0.0609 + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "shifted" + "SHIFT": 3 + USE_EMA: False + T_WEIGHT: uniform + # + DIFFUSION_MODEL: + NAME: MMDiT + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors + IGNORE_KEYS: '^first_stage_model.' + IN_CHANNELS: 16 + PATCH_SIZE: 2 + OUT_CHANNELS: 16 + DEPTH: 24 + INPUT_SIZE: + ADM_IN_CHANNELS: 2048 + CONTEXT_EMBEDDER_CONFIG: { 'target': 'torch.nn.Linear', 'params': { 'in_features': 4096, 'out_features': 1536 } } + NUM_PATCHES: 36864 + POS_EMBED_MAX_SIZE: 192 + POS_EMBED_SCALING_FACTOR: + USE_CHECKPOINT: True + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors + EMBED_DIM: 16 + IGNORE_KEYS: '^model.diffusion_model.' + BATCH_SIZE: 1 + USE_CONV: False + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 16 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 16 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + COND_STAGE_MODEL: + NAME: SD3TextEmbedder + P_ZERO: 0.0 + CLIP_L: + NAME: FrozenCLIPEmbedder2 + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer + MAX_LENGTH: 77 + FREEZE: True + LAYER: penultimate + RETURN_POOLED: True + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + CLIP_G: + NAME: FrozenCLIPEmbedder2 + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_2 + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_2 + MAX_LENGTH: 77 + FREEZE: True + LAYER: penultimate + RETURN_POOLED: True + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + T5_XXL: + NAME: T5EmbedderHF + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_3 + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_3 + LENGTH: 256 + CLEAN: whitespace + USE_GRAD: False + T5_DTYPE: float16 + # + LOSS: + NAME: ReconstructLoss + LOSS_TYPE: l2 + # + SAMPLE_ARGS: + SAMPLER: euler + SAMPLE_STEPS: 28 + SEED: 1749023094 + GUIDE_SCALE: 5.0 + GUIDE_RESCALE: 0.0 + DISCRETIZATION: trailing + RUN_TRAIN_N: False + # + OPTIMIZER: + NAME: AdamW + LEARNING_RATE: 5e-5 + BETAS: [ 0.9, 0.999 ] + EPS: 1e-8 + WEIGHT_DECAY: 1e-2 + AMSGRAD: False + # + TRAIN_DATA: + NAME: ImageTextPairMSDataset + MODE: train + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_DATASET_SPLIT: train + MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' } + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + SAMPLER: + NAME: LoopSampler + TRANSFORMS: + - NAME: LoadImageFromFile + RGB_ORDER: RGB + BACKEND: pillow + - NAME: FlexibleResize + INTERPOLATION: bilinear + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: FlexibleCenterCrop + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: ImageToTensor + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: Normalize + MEAN: [ 0.5, 0.5, 0.5 ] + STD: [ 0.5, 0.5, 0.5 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'image' ] + BACKEND: torchvision + - NAME: Select + KEYS: [ 'image', 'prompt' ] + META_KEYS: [ 'data_key' ] + # + EVAL_DATA: + NAME: Text2ImageDataset + MODE: eval + PROMPT_FILE: + PROMPT_DATA: [ "a cat holds a blackboard that writes \"hello world\"", "a dog running on the lawn" ] + IMAGE_SIZE: [ 1024, 1024 ] + FIELDS: [ "prompt" ] + DELIMITER: '#;#' + PROMPT_PREFIX: '' + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + TRANSFORMS: + - NAME: Select + KEYS: [ 'index', 'prompt' ] + META_KEYS: [ 'image_size' ] + # + TRAIN_HOOKS: + - + NAME: BackwardHook + PRIORITY: 10000 + - + NAME: LogHook + LOG_INTERVAL: 10 + SHOW_GPU_MEM: True + - + NAME: TensorboardLogHook + - + NAME: CheckpointHook + INTERVAL: 10000 + PRIORITY: 200 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + DISABLE_SNAPSHOT: True + # + EVAL_HOOKS: + - + NAME: ProbeDataHook + PROB_INTERVAL: 50 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + SAVE_PROBE_PREFIX: 'image' diff --git a/scepter/methods/examples/generation/stable_diffusion_1.5_512.yaml b/scepter/methods/examples/generation/stable_diffusion_1.5_512.yaml index 207114d..70c49c4 100644 --- a/scepter/methods/examples/generation/stable_diffusion_1.5_512.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_1.5_512.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd15_512_full LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # MODEL: NAME: LatentDiffusion @@ -124,7 +125,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.0064 + LEARNING_RATE: 0.00001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -190,7 +191,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/examples/generation/stable_diffusion_1.5_512_lora.yaml b/scepter/methods/examples/generation/stable_diffusion_1.5_512_lora.yaml index 8e8af4d..b4b8ab4 100644 --- a/scepter/methods/examples/generation/stable_diffusion_1.5_512_lora.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_1.5_512_lora.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd15_512_lora LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" TUNER: - NAME: SwiftLoRA @@ -132,7 +133,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -198,7 +199,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/examples/generation/stable_diffusion_1.5_512_textlora.yaml b/scepter/methods/examples/generation/stable_diffusion_1.5_512_textlora.yaml index 8532812..c420fbb 100644 --- a/scepter/methods/examples/generation/stable_diffusion_1.5_512_textlora.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_1.5_512_textlora.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd15_512_textlora LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" TUNER: - NAME: SwiftLoRA @@ -140,7 +141,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -206,7 +207,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/examples/generation/stable_diffusion_2.1_512.yaml b/scepter/methods/examples/generation/stable_diffusion_2.1_512.yaml index f148649..22b888c 100644 --- a/scepter/methods/examples/generation/stable_diffusion_2.1_512.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_2.1_512.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd21_512_full LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # MODEL: NAME: LatentDiffusion @@ -120,7 +121,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.0064 + LEARNING_RATE: 0.00001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -186,7 +187,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/examples/generation/stable_diffusion_2.1_512_lora.yaml b/scepter/methods/examples/generation/stable_diffusion_2.1_512_lora.yaml index 2ea41b4..a2a800c 100644 --- a/scepter/methods/examples/generation/stable_diffusion_2.1_512_lora.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_2.1_512_lora.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd21_512_lora LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - @@ -129,7 +130,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.0064 + LEARNING_RATE: 0.00001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -195,7 +196,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/examples/generation/stable_diffusion_2.1_768.yaml b/scepter/methods/examples/generation/stable_diffusion_2.1_768.yaml index 08b3707..8910299 100644 --- a/scepter/methods/examples/generation/stable_diffusion_2.1_768.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_2.1_768.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd21_768_full LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # MODEL: NAME: LatentDiffusion @@ -120,7 +121,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.0064 + LEARNING_RATE: 0.00001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -186,7 +187,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/examples/generation/stable_diffusion_2.1_768_lora.yaml b/scepter/methods/examples/generation/stable_diffusion_2.1_768_lora.yaml index e3434c7..da935f0 100644 --- a/scepter/methods/examples/generation/stable_diffusion_2.1_768_lora.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_2.1_768_lora.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd21_768_lora LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - @@ -129,7 +130,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -195,7 +196,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/examples/generation/stable_diffusion_xl_1024.yaml b/scepter/methods/examples/generation/stable_diffusion_xl_1024.yaml index 48b4566..69985e4 100644 --- a/scepter/methods/examples/generation/stable_diffusion_xl_1024.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_xl_1024.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_full LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # MODEL: NAME: LatentDiffusionXL @@ -238,7 +239,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.0064 + LEARNING_RATE: 0.00001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -306,7 +307,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - NAME: Select diff --git a/scepter/methods/examples/generation/stable_diffusion_xl_1024_lora.yaml b/scepter/methods/examples/generation/stable_diffusion_xl_1024_lora.yaml index 5f6522e..8ead3be 100644 --- a/scepter/methods/examples/generation/stable_diffusion_xl_1024_lora.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_xl_1024_lora.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_lora LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - @@ -247,7 +248,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -315,7 +316,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - NAME: Select diff --git a/scepter/methods/examples/generation/stable_diffusion_xl_1024_textlora.yaml b/scepter/methods/examples/generation/stable_diffusion_xl_1024_textlora.yaml index 28ec143..719a33c 100644 --- a/scepter/methods/examples/generation/stable_diffusion_xl_1024_textlora.yaml +++ b/scepter/methods/examples/generation/stable_diffusion_xl_1024_textlora.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_textlora LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - @@ -255,7 +256,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -323,7 +324,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - NAME: Select diff --git a/scepter/methods/scedit/ctr/sd15_512_sce_ctr_hed.yaml b/scepter/methods/scedit/ctr/sd15_512_sce_ctr_hed.yaml index 4e727e5..c213a07 100644 --- a/scepter/methods/scedit/ctr/sd15_512_sce_ctr_hed.yaml +++ b/scepter/methods/scedit/ctr/sd15_512_sce_ctr_hed.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd15_512_sce_ctr_hed LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -141,7 +142,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/methods/scedit/ctr/sd21_768_sce_ctr_canny.yaml b/scepter/methods/scedit/ctr/sd21_768_sce_ctr_canny.yaml index af07087..5fb026c 100644 --- a/scepter/methods/scedit/ctr/sd21_768_sce_ctr_canny.yaml +++ b/scepter/methods/scedit/ctr/sd21_768_sce_ctr_canny.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd21_768_sce_ctr_canny LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -140,7 +141,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/methods/scedit/ctr/sd21_768_sce_ctr_pose.yaml b/scepter/methods/scedit/ctr/sd21_768_sce_ctr_pose.yaml index b3255bf..692eaa5 100644 --- a/scepter/methods/scedit/ctr/sd21_768_sce_ctr_pose.yaml +++ b/scepter/methods/scedit/ctr/sd21_768_sce_ctr_pose.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd21_768_sce_ctr_pose LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -140,7 +141,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_canny.yaml b/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_canny.yaml index d853348..87d01f3 100644 --- a/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_canny.yaml +++ b/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_canny.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_canny LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -254,7 +255,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_color.yaml b/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_color.yaml index 7a95e2d..db96bdd 100644 --- a/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_color.yaml +++ b/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_color.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_color LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -255,7 +256,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_color_datatxt.yaml b/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_color_datatxt.yaml index 40dddec..c8bfa8a 100644 --- a/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_color_datatxt.yaml +++ b/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_color_datatxt.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_color_datatxt LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -255,7 +256,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_depth.yaml b/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_depth.yaml index 815ffce..833d62c 100644 --- a/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_depth.yaml +++ b/scepter/methods/scedit/ctr/sdxl_1024_sce_ctr_depth.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_depth LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -255,7 +256,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/methods/scedit/t2i/sd15_512_sce_t2i.yaml b/scepter/methods/scedit/t2i/sd15_512_sce_t2i.yaml index dc4115b..66fc50f 100644 --- a/scepter/methods/scedit/t2i/sd15_512_sce_t2i.yaml +++ b/scepter/methods/scedit/t2i/sd15_512_sce_t2i.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd15_512_sce_t2i LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -134,7 +135,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -200,7 +201,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/scedit/t2i/sd15_512_sce_t2i_swift.yaml b/scepter/methods/scedit/t2i/sd15_512_sce_t2i_swift.yaml index 088a3f3..fa93ccb 100644 --- a/scepter/methods/scedit/t2i/sd15_512_sce_t2i_swift.yaml +++ b/scepter/methods/scedit/t2i/sd15_512_sce_t2i_swift.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd15_512_sce_t2i_swift LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - @@ -132,7 +133,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -198,7 +199,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/scedit/t2i/sd15_512_textsce_t2i_swift.yaml b/scepter/methods/scedit/t2i/sd15_512_textsce_t2i_swift.yaml index 77dcf20..4231a87 100644 --- a/scepter/methods/scedit/t2i/sd15_512_textsce_t2i_swift.yaml +++ b/scepter/methods/scedit/t2i/sd15_512_textsce_t2i_swift.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd15_512_textsce_t2i_swift LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - @@ -140,7 +141,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -206,7 +207,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/scedit/t2i/sd21_768_sce_t2i.yaml b/scepter/methods/scedit/t2i/sd21_768_sce_t2i.yaml index 3a30df4..d7bee5b 100644 --- a/scepter/methods/scedit/t2i/sd21_768_sce_t2i.yaml +++ b/scepter/methods/scedit/t2i/sd21_768_sce_t2i.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd21_768_sce_t2i LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -130,7 +131,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -196,7 +197,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/scedit/t2i/sd21_768_sce_t2i_swift.yaml b/scepter/methods/scedit/t2i/sd21_768_sce_t2i_swift.yaml index 304c83c..ad08f8c 100644 --- a/scepter/methods/scedit/t2i/sd21_768_sce_t2i_swift.yaml +++ b/scepter/methods/scedit/t2i/sd21_768_sce_t2i_swift.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sd21_768_sce_t2i_swift LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - @@ -128,7 +129,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -194,7 +195,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - diff --git a/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i.yaml b/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i.yaml index 2980236..2c2aef4 100644 --- a/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i.yaml +++ b/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_sce_t2i LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -247,7 +248,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -315,7 +316,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - NAME: Select diff --git a/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i_datatxt.yaml b/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i_datatxt.yaml index 49cca0b..6a869cb 100644 --- a/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i_datatxt.yaml +++ b/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i_datatxt.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_sce_t2i_datatxt LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ] @@ -247,7 +248,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i_swift.yaml b/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i_swift.yaml index dd38b0d..3abd1fb 100644 --- a/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i_swift.yaml +++ b/scepter/methods/scedit/t2i/sdxl_1024_sce_t2i_swift.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_sce_t2i_swift LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - @@ -245,7 +246,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -313,7 +314,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - NAME: Select diff --git a/scepter/methods/scedit/t2i/sdxl_1024_textsce_t2i_swift.yaml b/scepter/methods/scedit/t2i/sdxl_1024_textsce_t2i_swift.yaml index 26b8678..2db6006 100644 --- a/scepter/methods/scedit/t2i/sdxl_1024_textsce_t2i_swift.yaml +++ b/scepter/methods/scedit/t2i/sdxl_1024_textsce_t2i_swift.yaml @@ -14,13 +14,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: 100 + RESCALE_LR: False # WORK_DIR: ./cache/save_data/sdxl_1024_textsce_t2i_swift LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TUNER: - @@ -253,7 +254,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -321,7 +322,7 @@ SOLVER: NUM_WORKERS: 4 FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # TRANSFORMS: - NAME: Select diff --git a/scepter/methods/studio/extensions/mantra_book/mantra_book.yaml b/scepter/methods/studio/extensions/mantra_book/mantra_book.yaml index 109f639..c7aaa7e 100644 --- a/scepter/methods/studio/extensions/mantra_book/mantra_book.yaml +++ b/scepter/methods/studio/extensions/mantra_book/mantra_book.yaml @@ -838,10 +838,7 @@ MANTRAS: DESCRIPTION: SOURCE: fooocus BASE_MODEL: SD_XL1.0 - PROMPT: { - "prompt": null -} - + PROMPT: NEGATIVE_PROMPT: deformed, bad anatomy, disfigured, poorly drawn face, mutated, extra limb, ugly, poorly drawn hands, missing limb, floating limbs, disconnected limbs, disconnected head, malformed hands, long neck, mutated hands and fingers, bad hands, missing fingers, cropped, worst quality, low quality, mutation, poorly drawn, huge calf, bad hands, fused hand, missing hand, disappearing arms, disappearing thigh, disappearing calf, disappearing legs, missing fingers, fused fingers, abnormal eye proportion, Abnormal hands, abnormal legs, abnormal feet, abnormal fingers, drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly, anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/daaec2962291137532189b8a31012532.jpg PROMPT_EXAMPLE: a boy wearing green jacket @@ -6407,3 +6404,2136 @@ MANTRAS: NEGATIVE_PROMPT: blurry, noisy, deformed, flat, low contrast, unrealistic, oversaturated, underexposed IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD2.1/d058a157de848a5c03d9a3e1e0e560a2.jpg PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: cinematic-diva + NAME_ZH: 电影歌星画风 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: UHD, 8K, ultra detailed, a cinematic photograph of {prompt}, beautiful lighting, great composition + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, NSFW + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/967b1852ea26dcf41360fc5542a0df6b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Abstract Expressionism + NAME_ZH: 抽象表现主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Abstract Expressionism Art, {prompt}, High contrast, minimalistic, colorful, stark, dramatic, expressionism + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/11667c7a085b53a6ffbb76b83788f8b5.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Academia + NAME_ZH: 学院风 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Academia, {prompt}, preppy Ivy League style, stark, dramatic, chic boarding school, academia + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, grunge, sloppy, unkempt + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/c95197ef78312e2bbd45883bfbfa095a.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Action Figure + NAME_ZH: 动作人偶 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Action Figure, {prompt}, plastic collectable action figure, collectable toy action figure + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/24ac67fe21833fc1300fba09d5de6090.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Adorable 3D Character + NAME_ZH: 可爱的3D角色 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Adorable 3D Character, {prompt}, 3D render, adorable character, 3D art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, grunge, sloppy, unkempt, photograph, photo, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/31a0ffbfc93b5b336a3bedc0a9985a02.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Adorable Kawaii + NAME_ZH: 可爱卡哇伊风格 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Adorable Kawaii, {prompt}, pretty, cute, adorable, kawaii + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, gothic, dark, moody, monochromatic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/82624c429d504c8290d4c9147be64029.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Art Deco + NAME_ZH: 艺术装饰风格 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Art Deco, {prompt}, sleek, geometric forms, art deco style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/c60ef3e7ada8774c92bb6e73c570f819.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Art Nouveau + NAME_ZH: 新艺术风格 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Art Nouveau, beautiful art, {prompt}, sleek, organic forms, long, sinuous, art nouveau style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, industrial, mechanical + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/153d83e71fbf145d0aa5c41ecab0a505.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Astral Aura + NAME_ZH: 星体光环 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Astral Aura, {prompt}, astral, colorful aura, vibrant energy + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/e093c9d0f81037002e88f2a6c3f38a55.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Avant-garde + NAME_ZH: 先锋派 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Avant-garde, {prompt}, unusual, experimental, avant-garde art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9921d1944893466edfc6a060faf4070e.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Baroque + NAME_ZH: 巴洛克风格 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Baroque, {prompt}, dramatic, exuberant, grandeur, baroque art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/d8279be4bb34e5cb00a95919412154a4.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Bauhaus-Style Poster + NAME_ZH: 包豪斯风格海报 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Bauhaus-Style Poster, {prompt}, simple geometric shapes, clean lines, primary colors, Bauhaus-Style Poster + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/4bd18b32c23e13b9e4fc7f65a1916eab.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Blueprint Schematic Drawing + NAME_ZH: 蓝图原理图绘制 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Blueprint Schematic Drawing, {prompt}, technical drawing, blueprint, schematic + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/62392c3715267f7162a37a8ff1caa3a1.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Caricature + NAME_ZH: 漫画 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Caricature, {prompt}, exaggerated, comical, caricature + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/894f40ed44b37c3372e6a22b8ae577a4.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Cel Shaded Art + NAME_ZH: 单色阴影艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Cel Shaded Art, {prompt}, 2D, flat color, toon shading, cel shaded style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/71c2e8a2cd1031b2bfae1640f7b72c88.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Character Design Sheet + NAME_ZH: 角色设计图 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Character Design Sheet, {prompt}, character reference sheet, character turn around + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/eacf2134aa51d4b4a6f2f35cc170f315.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Classicism Art + NAME_ZH: 古典主义艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Classicism Art, {prompt}, inspired by Roman and Greek culture, clarity, harmonious, classicism art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/10025b4f3a09e6134086e3cdec4ef2c8.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Color Field Painting + NAME_ZH: 色域绘画 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Color Field Painting, {prompt}, abstract, simple, geometic, color field painting style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/80e5b4075c572c04cbb4e48c37b8366b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Colored Pencil Art + NAME_ZH: 彩色铅笔艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Colored Pencil Art, {prompt}, colored pencil strokes, light color, visible paper texture, colored pencil art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9ae235d7f1a7c2a4edab52a5e9f9cbae.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Conceptual Art + NAME_ZH: 概念艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Conceptual Art, {prompt}, concept art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/cf2e6781997c6842a16155fdff911ef8.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Constructivism + NAME_ZH: 结构主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Constructivism Art, {prompt}, minimalistic, geometric forms, constructivism art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/d07feccabcadfd3310464bedba858bd1.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Cubism + NAME_ZH: 立体主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Cubism Art, {prompt}, flat geometric forms, cubism art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/aa8313f6c3cb9cafb9a7ec07db78df1c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Dadaism + NAME_ZH: 达达主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Dadaism Art, {prompt}, satirical, nonsensical, dadaism art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/db27030369cb3ccb042b8aea6bf635e2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Dark Fantasy + NAME_ZH: 黑暗幻想 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Dark Fantasy Art, {prompt}, dark, moody, dark fantasy style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, bright, sunny + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/c88b6e641c707afc0c8d278ba7e1ac19.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Dark Moody Atmosphere + NAME_ZH: 暗色忧郁氛围 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Dark Moody Atmosphere, {prompt}, dramatic, mysterious, dark moody atmosphere + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, vibrant, colorful, bright + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/3da915da2f5cedaf243e57e08163f35b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: DMT Art Style + NAME_ZH: DMT艺术风格 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: DMT Art Style, {prompt}, bright colors, surreal visuals, swirling patterns, DMT art style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/d4a823d2bfa912ca4bb56d96f722f08f.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Doodle Art + NAME_ZH: 涂鸦艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Doodle Art Style, {prompt}, drawing, freeform, swirling patterns, doodle art style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8e1e21745c149b9634d3ce96fc7d505f.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Double Exposure + NAME_ZH: 双重曝光 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Double Exposure Style, {prompt}, double image ghost effect, image combination, double exposure style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/efd7aea4af1c4ede99fdfac17350e264.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Dripping Paint Splatter Art + NAME_ZH: 滴漆溅画艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Dripping Paint Splatter Art, {prompt}, dramatic, paint drips, splatters, dripping paint + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/69fd81f5983107acc3d334af62915851.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Expressionism + NAME_ZH: 表现主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Expressionism Art Style, {prompt}, movement, contrast, emotional, exaggerated forms, expressionism art style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/3dfa862bd1c80cec237e4a5717cea2bd.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Faded Polaroid Photo + NAME_ZH: 褪色的宝丽来照片 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Faded Polaroid Photo, {prompt}, analog, old faded photo, old polaroid + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, vibrant, colorful + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/f152edb4b3ca6248758b48115258ddfa.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Fauvism + NAME_ZH: 野兽派 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Fauvism Art, {prompt}, painterly, bold colors, textured brushwork, fauvism art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/0506b350cadcf8fca42da764cd6fd5bf.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Flat 2D Art + NAME_ZH: 扁平2D艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Flat 2D Art, {prompt}, simple flat color, 2-dimensional, Flat 2D Art Style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, 3D, photo, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/940cfd34155634cf051e1b2942cca426.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Fortnite Art Style + NAME_ZH: 堡垒之夜艺术风格 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Fortnite Art Style, {prompt}, 3D cartoon, colorful, Fortnite Art Style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, photo, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/156f8d943ff6d283f7a34f265daaa46c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Futurism + NAME_ZH: 未来主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Futurism Art Style, {prompt}, dynamic, dramatic, Futurism Art Style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/855283cc2ab6283b627ef32d06a1ae0f.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Glitchcore + NAME_ZH: 故障核心 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Glitchcore Art Style, {prompt}, dynamic, dramatic, distorted, vibrant colors, glitchcore art style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9af62376402a85774179e82cf7e0dc59.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Glo-fi + NAME_ZH: 光环音乐风格 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Glo-fi Art Style, {prompt}, dynamic, dramatic, vibrant colors, glo-fi art style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/275a6e2a3297cec57f9b5a4a17b0f749.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Googie Art Style + NAME_ZH: 古奇艺术风格 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Googie Art Style, {prompt}, dynamic, dramatic, 1950's futurism, bold boomerang angles, Googie art style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/08ee77586d7bec3f6bad7d60cee3c540.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Graffiti Art + NAME_ZH: 涂鸦艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Graffiti Art Style, {prompt}, dynamic, dramatic, vibrant colors, graffiti art style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/57b751b11564cb22cd49ef21f2004a5f.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Harlem Renaissance Art + NAME_ZH: 哈莱姆文艺复兴艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Harlem Renaissance Art Style, {prompt}, dynamic, dramatic, 1920s African American culture, Harlem Renaissance art style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/d09ea3dadcd0a15d8fa247ee74a75e6b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: High Fashion + NAME_ZH: 高级时装 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: High Fashion, {prompt}, dynamic, dramatic, haute couture, elegant, ornate clothing, High Fashion + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/868f752bbb5ef992a0be36d7d13de3be.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Idyllic + NAME_ZH: 田园诗般的 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Idyllic, {prompt}, peaceful, happy, pleasant, happy, harmonious, picturesque, charming + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/55bab29eeb628e7a9ae018e45e7e24db.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Impressionism + NAME_ZH: 印象主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Impressionism, {prompt}, painterly, small brushstrokes, visible brushstrokes, impressionistic style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/0312b673dc6858a9864d7f45f0c5c1fc.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Infographic Drawing + NAME_ZH: 信息图表绘制 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Infographic Drawing, {prompt}, diagram, infographic + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/b09151f6e0883d26056b80fcc9d398bb.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Ink Dripping Drawing + NAME_ZH: 墨水滴画 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Ink Dripping Drawing, {prompt}, ink drawing, dripping ink + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, colorful, vibrant + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/5f09912a07e915250a96b2023db15821.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Japanese Ink Drawing + NAME_ZH: 日本墨画 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Japanese Ink Drawing, {prompt}, ink drawing, inkwash, Japanese Ink Drawing + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, colorful, vibrant + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/633ad971ff30fe4717782fc6b5f92fda.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Knolling Photography + NAME_ZH: 秩序拍摄 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Knolling Photography, {prompt}, flat lay photography, object arrangment, knolling photography + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/89a205be3a276349ecddf0eeca43ed80.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Light Cheery Atmosphere + NAME_ZH: 轻快愉快的氛围 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Light Cheery Atmosphere, {prompt}, happy, joyful, cheerful, carefree, gleeful, lighthearted, pleasant atmosphere + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, monochromatic, dark, moody + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/0913ac9815f892411f8327a630f51ae4.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Logo Design + NAME_ZH: 标志设计 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Logo Design, {prompt}, dynamic graphic art, vector art, minimalist, professional logo design + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9aa040b0c60d289da9610c91ad9b7c7e.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Luxurious Elegance + NAME_ZH: 奢华优雅 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Luxurious Elegance, {prompt}, extravagant, ornate, designer, opulent, picturesque, lavish + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/e796e84ed745150f0d4e28e0ac99b4cf.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Macro Photography + NAME_ZH: 微距摄影 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Macro Photography, {prompt}, close-up, macro 100mm, macro photography + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/92e8b84379828f38e3d01c7272f41b0b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Mandola Art + NAME_ZH: 曼陀罗艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Mandola art style, {prompt}, complex, circular design, mandola + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/dbf5ca944d9213c3181348666cf337ac.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Marker Drawing + NAME_ZH: 马克笔绘图 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Marker Drawing, {prompt}, bold marker lines, visibile paper texture, marker drawing + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, photograph, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/c558745f7a4b77d7ca9a428d8874efe4.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Medievalism + NAME_ZH: 中世纪主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Medievalism, {prompt}, inspired by The Middle Ages, medieval art, elaborate patterns and decoration, Medievalism + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/26aa6244359a4fd87f6438a341057901.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Minimalism + NAME_ZH: 极简主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Minimalism, {prompt}, abstract, simple geometic shapes, hard edges, sleek contours, Minimalism + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/dce92a90da6299339cf8e9ebc6596ea2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Neo-Baroque + NAME_ZH: 新巴洛克 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Neo-Baroque, {prompt}, ornate and elaborate, dynaimc, Neo-Baroque + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9b6cd8751b18c65b259769b44cfac351.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Neo-Byzantine + NAME_ZH: 新拜占庭 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Neo-Byzantine, {prompt}, grand decorative religious style, Orthodox Christian inspired, Neo-Byzantine + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/99c67768869b546bf527ef0b1735c0d5.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Neo-Futurism + NAME_ZH: 新未来主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Neo-Futurism, {prompt}, high-tech, curves, spirals, flowing lines, idealistic future, Neo-Futurism + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/bada1534f7a60187f584febcc92f40d1.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Neo-Impressionism + NAME_ZH: 新印象主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Neo-Impressionism, {prompt}, tiny dabs of color, Pointillism, painterly, Neo-Impressionism + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, photograph, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/6bafeac67ce2679b64a86ee1023b53c9.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Neo-Rococo + NAME_ZH: 新洛可可 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Neo-Rococo, {prompt}, curved forms, naturalistic ornamentation, elaborate, decorative, gaudy, Neo-Rococo + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9ab0ad2b88e03933ea479357ff1e4435.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Neoclassicism + NAME_ZH: 新古典主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Neoclassicism, {prompt}, ancient Rome and Greece inspired, idealic, sober colors, Neoclassicism + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/83a4e3a2ca577400b05a81b5b2b8650e.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Op Art + NAME_ZH: 视觉艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Op Art, {prompt}, optical illusion, abstract, geometric pattern, impression of movement, Op Art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/5f11132803dff2de5213293b91d407d7.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Ornate and Intricate + NAME_ZH: 华丽复杂 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Ornate and Intricate, {prompt}, decorative, highly detailed, elaborate, ornate, intricate + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/24b895dca946c8765ad9fa38f720f671.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Pencil Sketch Drawing + NAME_ZH: 铅笔素描 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Pencil Sketch Drawing, {prompt}, black and white drawing, graphite drawing + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/a9056e1eac85e5e4fe96a93917d4cce4.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Pop Art 2 + NAME_ZH: 流行艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Pop Art, {prompt}, vivid colors, flat color, 2D, strong lines, Pop Art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, photo, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/83c4c06c70a48da6558ff970ece247b6.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Rococo + NAME_ZH: 洛可可 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Rococo, {prompt}, flamboyant, pastel colors, curved lines, elaborate detail, Rococo + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/4850170f410e10bc833b7d00324dff18.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Silhouette Art + NAME_ZH: 剪影艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Silhouette Art, {prompt}, high contrast, well defined, Silhouette Art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/568777f447fc02510b618152726d5002.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Simple Vector Art + NAME_ZH: 简单矢量艺术 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Simple Vector Art, {prompt}, 2D flat, simple shapes, minimalistic, professional graphic, flat color, high contrast, Simple Vector Art + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, 3D, photo, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/24d2ab1fd175bf4e9ef3a8327651dd4b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Sketchup + NAME_ZH: 草图大师 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Sketchup, {prompt}, CAD, professional design, Sketchup + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, photo, photograph + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/337dca7ac49cf7820c85ede096ebde9c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Steampunk 2 + NAME_ZH: 蒸汽朋克 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Steampunk, {prompt}, retrofuturistic science fantasy, steam-powered tech, vintage industry, gears, neo-victorian, steampunk + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/07d7b27cd73f2d43684003563511c15b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Surrealism + NAME_ZH: 超现实主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Surrealism, {prompt}, expressive, dramatic, organic lines and forms, dreamlike and mysterious, Surrealism + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/38bae70377b323d1a5f5756d7e04886d.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Suprematism + NAME_ZH: 至上主义 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Suprematism, {prompt}, abstract, limited color palette, geometric forms, Suprematism + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/c48fbd8cedee84d06b4d875d566f1bb8.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Terragen + NAME_ZH: 地形生成 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Terragen, {prompt}, beautiful massive landscape, epic scenery, Terragen + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8974cd8e0f38fb7aac117cf0983fc36d.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Tranquil Relaxing Atmosphere + NAME_ZH: 宁静放松的氛围 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Tranquil Relaxing Atmosphere, {prompt}, calming style, soothing colors, peaceful, idealic, Tranquil Relaxing Atmosphere + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, oversaturated + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/69170d8443a67be4210af8d5558779c2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Sticker Designs + NAME_ZH: 贴纸设计 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Vector Art Stickers, {prompt}, professional vector design, sticker designs, Sticker Sheet + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/2d1e9867058db2c57f2fe47530de3243.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Vibrant Rim Light + NAME_ZH: 生动的边缘光 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Vibrant Rim Light, {prompt}, bright rim light, high contrast, bold edge light + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/bb08936ca184daba2b30b4a9efb308f2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Volumetric Lighting + NAME_ZH: 体积光照明 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Volumetric Lighting, {prompt}, light depth, dramatic atmospheric lighting, Volumetric Lighting + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/bbad7964a2ae70b66c72804273f11e74.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Watercolor 2 + NAME_ZH: 水彩 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Watercolor style painting, {prompt}, visible paper texture, colorwash, watercolor + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, photo, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8859d532ae5901cc8457d6118fb9b7da.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Whimsical and Playful + NAME_ZH: 异想天开和俏皮 + DESCRIPTION: + SOURCE: diva + BASE_MODEL: PIXART + PROMPT: Whimsical and Playful, {prompt}, imaginative, fantastical, bight colors, stylized, happy, Whimsical and Playful + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, drab, boring, moody + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/af2267f1942e870be957c37cd73d4359.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Fooocus Enhance + NAME_ZH: 焦点增强 + DESCRIPTION: + SOURCE: fooocus + BASE_MODEL: PIXART + PROMPT: { + "prompt": null +} + + NEGATIVE_PROMPT: (worst quality, low quality, normal quality, lowres, low details, oversaturated, undersaturated, overexposed, underexposed, grayscale, bw, bad photo, bad photography, bad art:1.4), (watermark, signature, text font, username, error, logo, words, letters, digits, autograph, trademark, name:1.2), (blur, blurry, grainy), morbid, ugly, asymmetrical, mutated malformed, mutilated, poorly lit, bad shadow, draft, cropped, out of frame, cut off, censored, jpeg artifacts, out of focus, glitch, duplicate, (airbrushed, cartoon, anime, semi-realistic, cgi, render, blender, digital art, manga, amateur:1.3), (3D ,3D Game, 3D Game Scene, 3D Character:1.1), (bad hands, bad anatomy, bad body, bad face, bad teeth, bad arms, bad legs, deformities:1.3) + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/e501823a21bbda56592055c9613c1dbb.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Fooocus Sharp + NAME_ZH: 焦点锐化 + DESCRIPTION: + SOURCE: fooocus + BASE_MODEL: PIXART + PROMPT: cinematic still {prompt} . emotional, harmonious, vignette, 4k epic detailed, shot on kodak, 35mm photo, sharp focus, high budget, cinemascope, moody, epic, gorgeous, film grain, grainy + NEGATIVE_PROMPT: anime, cartoon, graphic, (blur, blurry, bokeh), text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/393fb7aab57992a1c3af7cbad01c1001.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Fooocus Masterpiece + NAME_ZH: 焦点杰作 + DESCRIPTION: + SOURCE: fooocus + BASE_MODEL: PIXART + PROMPT: (masterpiece), (best quality), (ultra-detailed), {prompt}, illustration, disheveled hair, detailed eyes, perfect composition, moist skin, intricate details, earrings, by wlop + NEGATIVE_PROMPT: longbody, lowres, bad anatomy, bad hands, missing fingers, pubic hair,extra digit, fewer digits, cropped, worst quality, low quality + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9e27c3d1475dacae0ee45cea783aaaf5.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Fooocus Photograph + NAME_ZH: 焦点摄影 + DESCRIPTION: + SOURCE: fooocus + BASE_MODEL: PIXART + PROMPT: photograph {prompt}, 50mm . cinematic 4k epic detailed 4k epic detailed photograph shot on kodak detailed cinematic hbo dark moody, 35mm photo, grainy, vignette, vintage, Kodachrome, Lomography, stained, highly detailed, found footage + NEGATIVE_PROMPT: Brad Pitt, bokeh, depth of field, blurry, cropped, regular face, saturated, contrast, deformed iris, deformed pupils, semi-realistic, cgi, 3d, render, sketch, cartoon, drawing, anime, text, cropped, out of frame, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8b2ea41ec15bf1a16d6a837d46dbc2ac.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Fooocus Negative + NAME_ZH: 焦点底片 + DESCRIPTION: + SOURCE: fooocus + BASE_MODEL: PIXART + PROMPT: + NEGATIVE_PROMPT: deformed, bad anatomy, disfigured, poorly drawn face, mutated, extra limb, ugly, poorly drawn hands, missing limb, floating limbs, disconnected limbs, disconnected head, malformed hands, long neck, mutated hands and fingers, bad hands, missing fingers, cropped, worst quality, low quality, mutation, poorly drawn, huge calf, bad hands, fused hand, missing hand, disappearing arms, disappearing thigh, disappearing calf, disappearing legs, missing fingers, fused fingers, abnormal eye proportion, Abnormal hands, abnormal legs, abnormal feet, abnormal fingers, drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly, anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/daaec2962291137532189b8a31012532.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: Fooocus Cinematic + NAME_ZH: 焦点电影 + DESCRIPTION: + SOURCE: fooocus + BASE_MODEL: PIXART + PROMPT: cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy + NEGATIVE_PROMPT: anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/adc4d52aa5b0afca0593a475ddc9d055.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-cinematic-dynamic + NAME_ZH: MRE电影动态 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: epic cinematic shot of dynamic {prompt} in motion. main subject of high budget action movie. raw photo, motion blur. best quality, high resolution + NEGATIVE_PROMPT: static, still, motionless, sluggish. drawing, painting, illustration, rendered. low budget. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/bd2470068e09d9f1b7d0a690c879b3ce.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-spontaneous-picture + NAME_ZH: MRE自发图片 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: spontaneous picture of {prompt}, taken by talented amateur. best quality, high resolution. magical moment, natural look. simple but good looking + NEGATIVE_PROMPT: overthinked. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/19ad6082cee5516ce330641b515467ef.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-artistic-vision + NAME_ZH: MRE艺术视觉 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: powerful artistic vision of {prompt}. breathtaking masterpiece made by great artist. best quality, high resolution + NEGATIVE_PROMPT: insignificant, flawed, made by bad artist. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/95af94e50bcb91ae8fd2914a3bf13f1e.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-dark-dream + NAME_ZH: MRE黑暗梦境 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: dark and unsettling dream showing {prompt}. best quality, high resolution. created by genius but depressed mad artist. grim beauty + NEGATIVE_PROMPT: naive, cheerful. comfortable, casual, boring, cliche. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/572c8c26cc20ac0ee66684e3c5ee4e8c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-gloomy-art + NAME_ZH: MRE忧郁艺术 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: astonishing gloomy art made mainly of shadows and lighting, forming {prompt}. masterful usage of lighting, shadows and chiaroscuro. made by black-hearted artist, drawing from darkness. best quality, high resolution + NEGATIVE_PROMPT: low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/66cbdf819e930f7580bd66a41bde7dfe.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-bad-dream + NAME_ZH: MRE恶梦 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: picture from really bad dream about terrifying {prompt}, true horror. bone-chilling vision. mad world that shouldn't exist. best quality, high resolution + NEGATIVE_PROMPT: nice dream, pleasant experience. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/b92b2bbfc400db9204fe8249354132b4.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-underground + NAME_ZH: MRE地下 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: uncanny caliginous vision of {prompt}, created by remarkable underground artist. best quality, high resolution. raw and brutal art, careless but impressive style. inspired by darkness and chaos + NEGATIVE_PROMPT: photography, mainstream, civilized. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8d46eca558d791a1f2b41b0ed7bba4d2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-surreal-painting + NAME_ZH: MRE超现实绘画 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: surreal painting representing strange vision of {prompt}. harmonious madness, synergy with chance. unique artstyle, mindbending art, magical surrealism. best quality, high resolution + NEGATIVE_PROMPT: photography, illustration, drawing. realistic, possible. logical, sane. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/58858425832b10d233f7887af4a2022f.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-dynamic-illustration + NAME_ZH: MRE动态插画 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: insanely dynamic illustration of {prompt}. best quality, high resolution. crazy artstyle, careless brushstrokes, emotional and fun + NEGATIVE_PROMPT: photography, realistic. static, still, slow, boring. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8bf9861d4d3fadcdb98ceec88a941582.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-undead-art + NAME_ZH: MRE不死艺术 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: long forgotten art created by undead artist illustrating {prompt}, tribute to the death and decay. miserable art of the damned. wretched and decaying world. best quality, high resolution + NEGATIVE_PROMPT: alive, playful, living. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/fada50979ca180006eba9a45a00f3675.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-elemental-art + NAME_ZH: MRE元素艺术 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: art illustrating insane amounts of raging elemental energy turning into {prompt}, avatar of elements. magical surrealism, wizardry. best quality, high resolution + NEGATIVE_PROMPT: photography, realistic, real. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/5895d78cf58c1ca05178991f37cc48ff.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-space-art + NAME_ZH: MRE太空艺术 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: winner of inter-galactic art contest illustrating {prompt}, symbol of the interstellar singularity. best quality, high resolution. artstyle previously unseen in the whole galaxy + NEGATIVE_PROMPT: created by human race, low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/e9815495587895a21d970728474c8be6.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-ancient-illustration + NAME_ZH: MRE古代插画 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: sublime ancient illustration of {prompt}, predating human civilization. crude and simple, but also surprisingly beautiful artwork, made by genius primeval artist. best quality, high resolution + NEGATIVE_PROMPT: low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/151e07c17a89ebaf7688905ea0199862.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-brave-art + NAME_ZH: MRE勇敢艺术 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: brave, shocking, and brutally true art showing {prompt}. inspired by courage and unlimited creativity. truth found in chaos. best quality, high resolution + NEGATIVE_PROMPT: low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/1a068f2728327d21ebb99285c2ce1370.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-heroic-fantasy + NAME_ZH: MRE英雄幻想 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: heroic fantasy painting of {prompt}, in the dangerous fantasy world. airbrush over oil on canvas. best quality, high resolution + NEGATIVE_PROMPT: low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/78d05472067e0b86c8270fa5476fcb8e.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-dark-cyberpunk + NAME_ZH: MRE黑暗赛博朋克 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: dark cyberpunk illustration of brutal {prompt} in a world without hope, ruled by ruthless criminal corporations. best quality, high resolution + NEGATIVE_PROMPT: low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/a9f50e3162958fd783872d6955ad7d0b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-lyrical-geometry + NAME_ZH: MRE抒情几何 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: geometric and lyrical abstraction painting presenting {prompt}. oil on metal. best quality, high resolution + NEGATIVE_PROMPT: photography, realistic, drawing, rendered. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/dfabd41d3042ced804bc97ae35e3a7cb.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-sumi-e-symbolic + NAME_ZH: MRE墨绘象征 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: big long brushstrokes of deep black sumi-e turning into symbolic painting of {prompt}. master level raw art. best quality, high resolution + NEGATIVE_PROMPT: photography, rendered. low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/695b1ba687544eaeec9fb5f871917aeb.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-sumi-e-detailed + NAME_ZH: MRE墨绘精细 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: highly detailed black sumi-e painting of {prompt}. in-depth study of perfection, created by a master. best quality, high resolution + NEGATIVE_PROMPT: low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/16eb95e180385b88794e368b28da1812.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-manga + NAME_ZH: MRE漫画 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: manga artwork presenting {prompt}. created by japanese manga artist. highly emotional. best quality, high resolution + NEGATIVE_PROMPT: low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/ebec631bf467937f82d05958ae59f9dc.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-anime + NAME_ZH: MRE动漫 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: anime artwork illustrating {prompt}. created by japanese anime studio. highly emotional. best quality, high resolution + NEGATIVE_PROMPT: low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/a08149bc8e50f6bc65c0010d4cd416f8.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: mre-comic + NAME_ZH: MRE漫画书 + DESCRIPTION: + SOURCE: mre + BASE_MODEL: PIXART + PROMPT: breathtaking illustration from adult comic book presenting {prompt}. fabulous artwork. best quality, high resolution + NEGATIVE_PROMPT: deformed, ugly, low quality, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/48c65cebf1fa4284d7b8feb619412e65.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-3d-model + NAME_ZH: SAI三维模型 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: professional 3d model {prompt} . octane render, highly detailed, volumetric, dramatic lighting + NEGATIVE_PROMPT: ugly, deformed, noisy, low poly, blurry, painting + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/c1a765ac089fdfb3c1d11b33c75d2afd.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-analog film + NAME_ZH: SAI模拟胶片 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: analog film photo {prompt} . faded film, desaturated, 35mm photo, grainy, vignette, vintage, Kodachrome, Lomography, stained, highly detailed, found footage + NEGATIVE_PROMPT: painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/3c46d19957efd7fb78f4ab2bdada5468.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-anime + NAME_ZH: SAI动漫 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: anime artwork {prompt} . anime style, key visual, vibrant, studio anime, highly detailed + NEGATIVE_PROMPT: photo, deformed, black and white, realism, disfigured, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/c030e72561eda96abcf738f1370d36ff.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-cinematic + NAME_ZH: SAI电影 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: cinematic film still {prompt} . shallow depth of field, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy + NEGATIVE_PROMPT: anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/d9161c0d5cbf2133b2bfc1021c0c5e2a.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-comic book + NAME_ZH: SAI漫画书 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: comic {prompt} . graphic illustration, comic art, graphic novel art, vibrant, highly detailed + NEGATIVE_PROMPT: photograph, deformed, glitch, noisy, realistic, stock photo + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/6759ecb831037367e64c4b36d802be87.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-craft clay + NAME_ZH: SAI手工粘土 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: play-doh style {prompt} . sculpture, clay art, centered composition, Claymation + NEGATIVE_PROMPT: sloppy, messy, grainy, highly detailed, ultra textured, photo + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8fc51113f725f27326c4398a7457cd6d.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-digital art + NAME_ZH: SAI数字艺术 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: concept art {prompt} . digital artwork, illustrative, painterly, matte painting, highly detailed + NEGATIVE_PROMPT: photo, photorealistic, realism, ugly + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/d29adb44458700c4a45ee6edaa04bfb6.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-enhance + NAME_ZH: SAI增强 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: breathtaking {prompt} . award-winning, professional, highly detailed + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, distorted, grainy + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/2be0de541ff65f8da80ddc24a65c98d8.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-fantasy art + NAME_ZH: SAI幻想艺术 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: ethereal fantasy concept art of {prompt} . magnificent, celestial, ethereal, painterly, epic, majestic, magical, fantasy art, cover art, dreamy + NEGATIVE_PROMPT: photographic, realistic, realism, 35mm film, dslr, cropped, frame, text, deformed, glitch, noise, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, sloppy, duplicate, mutated, black and white + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/a6f8d92afcd5803dfb2ebecbc92091b6.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-isometric + NAME_ZH: SAI等距 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: isometric style {prompt} . vibrant, beautiful, crisp, detailed, ultra detailed, intricate + NEGATIVE_PROMPT: deformed, mutated, ugly, disfigured, blur, blurry, noise, noisy, realistic, photographic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/d4e3fcbbfd7b1323bd89decf5d7b0006.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-line art + NAME_ZH: SAI线条艺术 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: line art drawing {prompt} . professional, sleek, modern, minimalist, graphic, line art, vector graphics + NEGATIVE_PROMPT: anime, photorealistic, 35mm film, deformed, glitch, blurry, noisy, off-center, deformed, cross-eyed, closed eyes, bad anatomy, ugly, disfigured, mutated, realism, realistic, impressionism, expressionism, oil, acrylic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/034a51b0dd34b018be8859bf45b4f7ed.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-lowpoly + NAME_ZH: SAI低多边形 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: low-poly style {prompt} . low-poly game art, polygon mesh, jagged, blocky, wireframe edges, centered composition + NEGATIVE_PROMPT: noisy, sloppy, messy, grainy, highly detailed, ultra textured, photo + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/76b9913e9fa5704b6d30adbde9e1f70f.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-neonpunk + NAME_ZH: SAI霓虹朋克 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: neonpunk style {prompt} . cyberpunk, vaporwave, neon, vibes, vibrant, stunningly beautiful, crisp, detailed, sleek, ultramodern, magenta highlights, dark purple shadows, high contrast, cinematic, ultra detailed, intricate, professional + NEGATIVE_PROMPT: painting, drawing, illustration, glitch, deformed, mutated, cross-eyed, ugly, disfigured + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/7e9ed25bb34008beb5f417df63c4b2fe.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-origami + NAME_ZH: SAI折纸 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: origami style {prompt} . paper art, pleated paper, folded, origami art, pleats, cut and fold, centered composition + NEGATIVE_PROMPT: noisy, sloppy, messy, grainy, highly detailed, ultra textured, photo + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/924f46a8f276011a0953d7988e90ee25.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-photographic + NAME_ZH: SAI摄影 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: cinematic photo {prompt} . 35mm photograph, film, bokeh, professional, 4k, highly detailed + NEGATIVE_PROMPT: drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/d6a2d8f3d37cc21c20c5dfc13d000b67.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-pixel art + NAME_ZH: SAI像素艺术 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: pixel-art {prompt} . low-res, blocky, pixel art style, 8-bit graphics + NEGATIVE_PROMPT: sloppy, messy, blurry, noisy, highly detailed, ultra textured, photo, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/a5ab89c0960be8c1216e65c98d92ae4a.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: sai-texture + NAME_ZH: SAI质地 + DESCRIPTION: + SOURCE: sai + BASE_MODEL: PIXART + PROMPT: texture {prompt} top down close-up + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/ebfeab574283fff2ac096e348a585e6d.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: ads-advertising + NAME_ZH: 广告 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: advertising poster style {prompt} . Professional, modern, product-focused, commercial, eye-catching, highly detailed + NEGATIVE_PROMPT: noisy, blurry, amateurish, sloppy, unattractive + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/1aa26a16126e2756bf4bf3fda29baa12.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: ads-automotive + NAME_ZH: 汽车广告 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: automotive advertisement style {prompt} . sleek, dynamic, professional, commercial, vehicle-focused, high-resolution, highly detailed + NEGATIVE_PROMPT: noisy, blurry, unattractive, sloppy, unprofessional + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/ddc833b36ca23c85a4f6e7bf0088d081.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: ads-corporate + NAME_ZH: 企业广告 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: corporate branding style {prompt} . professional, clean, modern, sleek, minimalist, business-oriented, highly detailed + NEGATIVE_PROMPT: noisy, blurry, grungy, sloppy, cluttered, disorganized + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/df97fa45c7842296c138aca4057db272.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: ads-fashion editorial + NAME_ZH: 时尚编辑 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: fashion editorial style {prompt} . high fashion, trendy, stylish, editorial, magazine style, professional, highly detailed + NEGATIVE_PROMPT: outdated, blurry, noisy, unattractive, sloppy + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/89259be723479cd0b81546b715bea04d.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: ads-food photography + NAME_ZH: 食品摄影 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: food photography style {prompt} . appetizing, professional, culinary, high-resolution, commercial, highly detailed + NEGATIVE_PROMPT: unappetizing, sloppy, unprofessional, noisy, blurry + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/33acc98f8615940c8cebbac8ce5297b2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: ads-gourmet food photography + NAME_ZH: 美食摄影 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: gourmet food photo of {prompt} . soft natural lighting, macro details, vibrant colors, fresh ingredients, glistening textures, bokeh background, styled plating, wooden tabletop, garnished, tantalizing, editorial quality + NEGATIVE_PROMPT: cartoon, anime, sketch, grayscale, dull, overexposed, cluttered, messy plate, deformed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/897bbd3f1d232de122796266f3845b50.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: ads-luxury + NAME_ZH: 奢华广告 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: luxury product style {prompt} . elegant, sophisticated, high-end, luxurious, professional, highly detailed + NEGATIVE_PROMPT: cheap, noisy, blurry, unattractive, amateurish + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/5b33ecc84dc5ff285a0f2e69f43162cb.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: ads-real estate + NAME_ZH: 房地产广告 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: real estate photography style {prompt} . professional, inviting, well-lit, high-resolution, property-focused, commercial, highly detailed + NEGATIVE_PROMPT: dark, blurry, unappealing, noisy, unprofessional + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/1b99b7bcd0476144d80d066e747d8dfc.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: ads-retail + NAME_ZH: 零售广告 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: retail packaging style {prompt} . vibrant, enticing, commercial, product-focused, eye-catching, professional, highly detailed + NEGATIVE_PROMPT: noisy, blurry, amateurish, sloppy, unattractive + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/0029e46ef36380dde2084257108f8a6f.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-abstract + NAME_ZH: 抽象艺术风格 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: abstract style {prompt} . non-representational, colors and shapes, expression of feelings, imaginative, highly detailed + NEGATIVE_PROMPT: realistic, photographic, figurative, concrete + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8c401cb0a6ea288230222c4985e78667.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-abstract expressionism + NAME_ZH: 抽象表现主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: abstract expressionist painting {prompt} . energetic brushwork, bold colors, abstract forms, expressive, emotional + NEGATIVE_PROMPT: realistic, photorealistic, low contrast, plain, simple, monochrome + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9df6635149c576b65d911c2e4cc6a86b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-art deco + NAME_ZH: 艺术装饰风格 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: art deco style {prompt} . geometric shapes, bold colors, luxurious, elegant, decorative, symmetrical, ornate, detailed + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, modernist, minimalist + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/82fff8a25de8075777caf14cb3eb5650.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-art nouveau + NAME_ZH: 新艺术风格 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: art nouveau style {prompt} . elegant, decorative, curvilinear forms, nature-inspired, ornate, detailed + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, modernist, minimalist + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/84075c5c0cb4b7a2541c6fbca9835cfd.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-constructivist + NAME_ZH: 构成主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: constructivist style {prompt} . geometric shapes, bold colors, dynamic composition, propaganda art style + NEGATIVE_PROMPT: realistic, photorealistic, low contrast, plain, simple, abstract expressionism + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/2a634ad0c89ecefbbea4ce5bcab3d5e5.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-cubist + NAME_ZH: 立体主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: cubist artwork {prompt} . geometric shapes, abstract, innovative, revolutionary + NEGATIVE_PROMPT: anime, photorealistic, 35mm film, deformed, glitch, low contrast, noisy + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/2a3d797008c08e12b485d61624741ea6.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-expressionist + NAME_ZH: 表现主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: expressionist {prompt} . raw, emotional, dynamic, distortion for emotional effect, vibrant, use of unusual colors, detailed + NEGATIVE_PROMPT: realism, symmetry, quiet, calm, photo + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/83914655659001716a1acede295289d0.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-graffiti + NAME_ZH: 涂鸦 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: graffiti style {prompt} . street art, vibrant, urban, detailed, tag, mural + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/f72a23c7623eb480737eb73c08bf8423.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-hyperrealism + NAME_ZH: 超现实主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: hyperrealistic art {prompt} . extremely high-resolution details, photographic, realism pushed to extreme, fine texture, incredibly lifelike + NEGATIVE_PROMPT: simplified, abstract, unrealistic, impressionistic, low resolution + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/db378d5a64a8e7ad1e29a4389bda5d1c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-impressionist + NAME_ZH: 印象主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: impressionist painting {prompt} . loose brushwork, vibrant color, light and shadow play, captures feeling over form + NEGATIVE_PROMPT: anime, photorealistic, 35mm film, deformed, glitch, low contrast, noisy + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/53b151aec4d5685dfc24511b6705b90e.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-pointillism + NAME_ZH: 点彩主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: pointillism style {prompt} . composed entirely of small, distinct dots of color, vibrant, highly detailed + NEGATIVE_PROMPT: line drawing, smooth shading, large color fields, simplistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/23b137a409ee8a8c6ee160c1ddf0659f.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-pop art + NAME_ZH: 波普艺术 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: pop Art style {prompt} . bright colors, bold outlines, popular culture themes, ironic or kitsch + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, minimalist + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/bd4faf0e2b7dbc2d0eb21f3ebee97d3d.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-psychedelic + NAME_ZH: 迷幻艺术 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: psychedelic style {prompt} . vibrant colors, swirling patterns, abstract forms, surreal, trippy + NEGATIVE_PROMPT: monochrome, black and white, low contrast, realistic, photorealistic, plain, simple + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/7dc0131817f4c31517581dd4a811067b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-renaissance + NAME_ZH: 文艺复兴 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: renaissance style {prompt} . realistic, perspective, light and shadow, religious or mythological themes, highly detailed + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, modernist, minimalist, abstract + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/69796484f61dc2e94d5853f5bbe27c05.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-steampunk + NAME_ZH: 蒸汽朋克 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: steampunk style {prompt} . antique, mechanical, brass and copper tones, gears, intricate, detailed + NEGATIVE_PROMPT: deformed, glitch, noisy, low contrast, anime, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/985005e28858b03c9500a6eb5bd9e201.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-surrealist + NAME_ZH: 超现实主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: surrealist art {prompt} . dreamlike, mysterious, provocative, symbolic, intricate, detailed + NEGATIVE_PROMPT: anime, photorealistic, realistic, deformed, glitch, noisy, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/892d19ec3c429b7148562796df0024db.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-typography + NAME_ZH: 排版艺术 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: typographic art {prompt} . stylized, intricate, detailed, artistic, text-based + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9adafe1bb169e1ee3f78ba2c1b1cf8d3.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: artstyle-watercolor + NAME_ZH: 水彩艺术 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: watercolor painting {prompt} . vibrant, beautiful, painterly, detailed, textural, artistic + NEGATIVE_PROMPT: anime, photorealistic, 35mm film, deformed, glitch, low contrast, noisy + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/29ca23d7a0397e9beaa72e3b63d17551.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-biomechanical + NAME_ZH: 未来生物力学 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: biomechanical style {prompt} . blend of organic and mechanical elements, futuristic, cybernetic, detailed, intricate + NEGATIVE_PROMPT: natural, rustic, primitive, organic, simplistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/5987795487a3fa74469499ae53963d5d.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-biomechanical cyberpunk + NAME_ZH: 未来生物力学赛博朋克 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: biomechanical cyberpunk {prompt} . cybernetics, human-machine fusion, dystopian, organic meets artificial, dark, intricate, highly detailed + NEGATIVE_PROMPT: natural, colorful, deformed, sketch, low contrast, watercolor + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/75b56d7cff9b3f011248b3092ae3b8d2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-cybernetic + NAME_ZH: 未来赛博 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: cybernetic style {prompt} . futuristic, technological, cybernetic enhancements, robotics, artificial intelligence themes + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, historical, medieval + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/47df6db04290f01aeee7381aaf12ec07.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-cybernetic robot + NAME_ZH: 未来机器人 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: cybernetic robot {prompt} . android, AI, machine, metal, wires, tech, futuristic, highly detailed + NEGATIVE_PROMPT: organic, natural, human, sketch, watercolor, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/6f18121b4ba22c8ba7834f404501fa4c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-cyberpunk cityscape + NAME_ZH: 未来赛博朋克城市景观 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: cyberpunk cityscape {prompt} . neon lights, dark alleys, skyscrapers, futuristic, vibrant colors, high contrast, highly detailed + NEGATIVE_PROMPT: natural, rural, deformed, low contrast, black and white, sketch, watercolor + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/5e4107dace2c9dd0217a9d1b18ae28f6.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-futuristic + NAME_ZH: 未来主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: futuristic style {prompt} . sleek, modern, ultramodern, high tech, detailed + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, vintage, antique + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8a61c0853ee4079bae96ad502a6ae840.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-retro cyberpunk + NAME_ZH: 未来复古赛博朋克 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: retro cyberpunk {prompt} . 80's inspired, synthwave, neon, vibrant, detailed, retro futurism + NEGATIVE_PROMPT: modern, desaturated, black and white, realism, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/460957bf9a07a58ddec08bf22d5b3698.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-retro futurism + NAME_ZH: 未来复古主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: retro-futuristic {prompt} . vintage sci-fi, 50s and 60s style, atomic age, vibrant, highly detailed + NEGATIVE_PROMPT: contemporary, realistic, rustic, primitive + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/0434a7b23a0f936db615d7a9b8e805f1.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-sci-fi + NAME_ZH: 科幻未来主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: sci-fi style {prompt} . futuristic, technological, alien worlds, space themes, advanced civilizations + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, historical, medieval + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/7c3bde651f426273758b68a2805d0c8a.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: futuristic-vaporwave + NAME_ZH: 未来波 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: vaporwave style {prompt} . retro aesthetic, cyberpunk, vibrant, neon colors, vintage 80s and 90s style, highly detailed + NEGATIVE_PROMPT: monochrome, muted colors, realism, rustic, minimalist, dark + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/1ebb0bf67ee3ef76288b4c0a2a65d557.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-bubble bobble + NAME_ZH: 游戏-泡泡龙 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: Bubble Bobble style {prompt} . 8-bit, cute, pixelated, fantasy, vibrant, reminiscent of Bubble Bobble game + NEGATIVE_PROMPT: realistic, modern, photorealistic, violent, horror + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/19257352df33228555cb350963d9432a.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-cyberpunk game + NAME_ZH: 赛博朋克游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: cyberpunk game style {prompt} . neon, dystopian, futuristic, digital, vibrant, detailed, high contrast, reminiscent of cyberpunk genre video games + NEGATIVE_PROMPT: historical, natural, rustic, low detailed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/6165ef47bc859e3dd0e3399ba2264c2b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-fighting game + NAME_ZH: 格斗游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: fighting game style {prompt} . dynamic, vibrant, action-packed, detailed character design, reminiscent of fighting video games + NEGATIVE_PROMPT: peaceful, calm, minimalist, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/aebda69ec1097dc61966995956ae34cc.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-gta + NAME_ZH: 侠盗猎车手游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: GTA-style artwork {prompt} . satirical, exaggerated, pop art style, vibrant colors, iconic characters, action-packed + NEGATIVE_PROMPT: realistic, black and white, low contrast, impressionist, cubist, noisy, blurry, deformed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/edff7ba5fc983468e79e64beb838b829.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-mario + NAME_ZH: 马里奥游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: Super Mario style {prompt} . vibrant, cute, cartoony, fantasy, playful, reminiscent of Super Mario series + NEGATIVE_PROMPT: realistic, modern, horror, dystopian, violent + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/2a229f623ce89160d1f67620599dfc7c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-minecraft + NAME_ZH: 我的世界游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: Minecraft style {prompt} . blocky, pixelated, vibrant colors, recognizable characters and objects, game assets + NEGATIVE_PROMPT: smooth, realistic, detailed, photorealistic, noise, blurry, deformed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/83e77332a9f234fd8c9cddd29e2ec3c7.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-pokemon + NAME_ZH: 宝可梦游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: Pokémon style {prompt} . vibrant, cute, anime, fantasy, reminiscent of Pokémon series + NEGATIVE_PROMPT: realistic, modern, horror, dystopian, violent + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/9bfe2805e6578875a8dfe2a0ea96c281.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-retro arcade + NAME_ZH: 复古街机 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: retro arcade style {prompt} . 8-bit, pixelated, vibrant, classic video game, old school gaming, reminiscent of 80s and 90s arcade games + NEGATIVE_PROMPT: modern, ultra-high resolution, photorealistic, 3D + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/76885e0b110f1fdf3eaee52fbaf27abf.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-retro game + NAME_ZH: 复古游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: retro game art {prompt} . 16-bit, vibrant colors, pixelated, nostalgic, charming, fun + NEGATIVE_PROMPT: realistic, photorealistic, 35mm film, deformed, glitch, low contrast, noisy + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/61fbae41c50f56e24dcf20fe3612d456.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-rpg fantasy game + NAME_ZH: 角色扮演幻想游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: role-playing game (RPG) style fantasy {prompt} . detailed, vibrant, immersive, reminiscent of high fantasy RPG games + NEGATIVE_PROMPT: sci-fi, modern, urban, futuristic, low detailed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/34797baf58e32b4e1a37753fe9dcff5c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-strategy game + NAME_ZH: 策略游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: strategy game style {prompt} . overhead view, detailed map, units, reminiscent of real-time strategy video games + NEGATIVE_PROMPT: first-person view, modern, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/0d276527d22510a1b5f8f74eac2790df.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-streetfighter + NAME_ZH: 街头霸王游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: Street Fighter style {prompt} . vibrant, dynamic, arcade, 2D fighting game, highly detailed, reminiscent of Street Fighter series + NEGATIVE_PROMPT: 3D, realistic, modern, photorealistic, turn-based strategy + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/f5927fc2b8d4a8212242bd97adcbdfa1.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: game-zelda + NAME_ZH: 塞尔达传说游戏 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: Legend of Zelda style {prompt} . vibrant, fantasy, detailed, epic, heroic, reminiscent of The Legend of Zelda series + NEGATIVE_PROMPT: sci-fi, modern, realistic, horror + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/4f9eadddbc196268258089b8cc4e5c5c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-architectural + NAME_ZH: 建筑 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: architectural style {prompt} . clean lines, geometric shapes, minimalist, modern, architectural drawing, highly detailed + NEGATIVE_PROMPT: curved lines, ornate, baroque, abstract, grunge + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/488545e9fc9417d62961f42d578547d2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-disco + NAME_ZH: 迪斯科 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: disco-themed {prompt} . vibrant, groovy, retro 70s style, shiny disco balls, neon lights, dance floor, highly detailed + NEGATIVE_PROMPT: minimalist, rustic, monochrome, contemporary, simplistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/784b0ac35e0c0fdb2df21a95d5ca1c55.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-dreamscape + NAME_ZH: 梦境 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: dreamscape {prompt} . surreal, ethereal, dreamy, mysterious, fantasy, highly detailed + NEGATIVE_PROMPT: realistic, concrete, ordinary, mundane + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/2bf77804b5bf352c4e97475b1e8eb29e.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-dystopian + NAME_ZH: 反乌托邦 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: dystopian style {prompt} . bleak, post-apocalyptic, somber, dramatic, highly detailed + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, cheerful, optimistic, vibrant, colorful + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/065db0385e126192cbd27e22ca4f154c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-fairy tale + NAME_ZH: 童话故事 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: fairy tale {prompt} . magical, fantastical, enchanting, storybook style, highly detailed + NEGATIVE_PROMPT: realistic, modern, ordinary, mundane + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/578dfe65d9f03e83c95820d4c45cd396.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-gothic + NAME_ZH: 哥特 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: gothic style {prompt} . dark, mysterious, haunting, dramatic, ornate, detailed + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, cheerful, optimistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/332a1f8cd655b724cb48bc5c91b764fc.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-grunge + NAME_ZH: 垃圾摇滚 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: grunge style {prompt} . textured, distressed, vintage, edgy, punk rock vibe, dirty, noisy + NEGATIVE_PROMPT: smooth, clean, minimalist, sleek, modern, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/0ccb4c03c6d983b9f6d05ba383f9d690.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-horror + NAME_ZH: 恐怖 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: horror-themed {prompt} . eerie, unsettling, dark, spooky, suspenseful, grim, highly detailed + NEGATIVE_PROMPT: cheerful, bright, vibrant, light-hearted, cute + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/fdca4780d099a39dc50e4553622fba72.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-kawaii + NAME_ZH: 卡哇伊 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: kawaii style {prompt} . cute, adorable, brightly colored, cheerful, anime influence, highly detailed + NEGATIVE_PROMPT: dark, scary, realistic, monochrome, abstract + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/99dab4185e7337189d8959878bfa308b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-lovecraftian + NAME_ZH: 克苏鲁神话 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: lovecraftian horror {prompt} . eldritch, cosmic horror, unknown, mysterious, surreal, highly detailed + NEGATIVE_PROMPT: light-hearted, mundane, familiar, simplistic, realistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/b7c9d084ed62c1c2e1ea080ad95af61e.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-macabre + NAME_ZH: 恐怖的 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: macabre style {prompt} . dark, gothic, grim, haunting, highly detailed + NEGATIVE_PROMPT: bright, cheerful, light-hearted, cartoonish, cute + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/fd7cf2315b27b0434b909020414e1173.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-manga + NAME_ZH: 漫画 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: manga style {prompt} . vibrant, high-energy, detailed, iconic, Japanese comic style + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, Western comic style + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/531aac90b321d39221ba7cf70a97b232.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-metropolis + NAME_ZH: 大都市 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: metropolis-themed {prompt} . urban, cityscape, skyscrapers, modern, futuristic, highly detailed + NEGATIVE_PROMPT: rural, natural, rustic, historical, simple + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/eaad79fe3d7160cec6e0656fc3b02d3c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-minimalist + NAME_ZH: 极简主义 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: minimalist style {prompt} . simple, clean, uncluttered, modern, elegant + NEGATIVE_PROMPT: ornate, complicated, highly detailed, cluttered, disordered, messy, noisy + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/e85dd1ccc7f22bdac054d74ba58521b2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-monochrome + NAME_ZH: 单色 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: monochrome {prompt} . black and white, contrast, tone, texture, detailed + NEGATIVE_PROMPT: colorful, vibrant, noisy, blurry, deformed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/cc793ada6b47561f4b63aa66dcd282ac.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-nautical + NAME_ZH: 航海 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: nautical-themed {prompt} . sea, ocean, ships, maritime, beach, marine life, highly detailed + NEGATIVE_PROMPT: landlocked, desert, mountains, urban, rustic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/3d9f6528180670804a2c5e85c7665c42.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-space + NAME_ZH: 太空 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: space-themed {prompt} . cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed + NEGATIVE_PROMPT: earthly, mundane, ground-based, realism + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/6f5742d23b43fc99cc01c8b68518df7c.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-stained glass + NAME_ZH: 彩色玻璃 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: stained glass style {prompt} . vibrant, beautiful, translucent, intricate, detailed + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/ab0512ced9ed572075c49ad4f9f9a45d.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-techwear fashion + NAME_ZH: 科技服饰 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: techwear fashion {prompt} . futuristic, cyberpunk, urban, tactical, sleek, dark, highly detailed + NEGATIVE_PROMPT: vintage, rural, colorful, low contrast, realism, sketch, watercolor + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/652a6bdb1b36860041222a458c7915d0.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-tribal + NAME_ZH: 部落 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: tribal style {prompt} . indigenous, ethnic, traditional patterns, bold, natural colors, highly detailed + NEGATIVE_PROMPT: modern, futuristic, minimalist, pastel + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/1ecdaf153ece5e16250b87020b7a209b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: misc-zentangle + NAME_ZH: 禅绕画 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: zentangle {prompt} . intricate, abstract, monochrome, patterns, meditative, highly detailed + NEGATIVE_PROMPT: colorful, representative, simplistic, large fields of color + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/80f3488cdb69f0ff13d83a4feac4499d.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: papercraft-collage + NAME_ZH: 纸艺拼贴 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: collage style {prompt} . mixed media, layered, textural, detailed, artistic + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/c9bbdaba28358faf1a8c1df44b3de950.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: papercraft-flat papercut + NAME_ZH: 平面剪纸 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: flat papercut style {prompt} . silhouette, clean cuts, paper, sharp edges, minimalist, color block + NEGATIVE_PROMPT: 3D, high detail, noise, grainy, blurry, painting, drawing, photo, disfigured + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/1bffcd9a4086b29c367b8301f26eed3b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: papercraft-kirigami + NAME_ZH: 剪纸 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: kirigami representation of {prompt} . 3D, paper folding, paper cutting, Japanese, intricate, symmetrical, precision, clean lines + NEGATIVE_PROMPT: painting, drawing, 2D, noisy, blurry, deformed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/2e5f95977b97b2a6248924e118a23076.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: papercraft-paper mache + NAME_ZH: 纸浆塑型 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: paper mache representation of {prompt} . 3D, sculptural, textured, handmade, vibrant, fun + NEGATIVE_PROMPT: 2D, flat, photo, sketch, digital art, deformed, noisy, blurry + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/68441f668fcf7a51db5fc9b9d4d45ce7.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: papercraft-paper quilling + NAME_ZH: 纸卷艺术 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: paper quilling art of {prompt} . intricate, delicate, curling, rolling, shaping, coiling, loops, 3D, dimensional, ornamental + NEGATIVE_PROMPT: photo, painting, drawing, 2D, flat, deformed, noisy, blurry + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8c0504eb14341b66796612c46bb3748b.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: papercraft-papercut collage + NAME_ZH: 剪纸拼贴 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: papercut collage of {prompt} . mixed media, textured paper, overlapping, asymmetrical, abstract, vibrant + NEGATIVE_PROMPT: photo, 3D, realistic, drawing, painting, high detail, disfigured + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/217e7c8b905b63c63c94bd6ff8fe3477.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: papercraft-papercut shadow box + NAME_ZH: 剪纸影箱 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: 3D papercut shadow box of {prompt} . layered, dimensional, depth, silhouette, shadow, papercut, handmade, high contrast + NEGATIVE_PROMPT: painting, drawing, photo, 2D, flat, high detail, blurry, noisy, disfigured + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/2af0d61de1b17a5a5257f512549f9616.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: papercraft-stacked papercut + NAME_ZH: 堆叠剪纸 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: stacked papercut art of {prompt} . 3D, layered, dimensional, depth, precision cut, stacked layers, papercut, high contrast + NEGATIVE_PROMPT: 2D, flat, noisy, blurry, painting, drawing, photo, deformed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/62618e32e90539827e0eb279a6ea170a.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: papercraft-thick layered papercut + NAME_ZH: 厚层剪纸 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: thick layered papercut art of {prompt} . deep 3D, volumetric, dimensional, depth, thick paper, high stack, heavy texture, tangible layers + NEGATIVE_PROMPT: 2D, flat, thin paper, low stack, smooth texture, painting, drawing, photo, deformed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/17008c40a6be8d41d3921ee88bac27ad.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: photo-alien + NAME_ZH: 异形 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: alien-themed {prompt} . extraterrestrial, cosmic, otherworldly, mysterious, sci-fi, highly detailed + NEGATIVE_PROMPT: earthly, mundane, common, realistic, simple + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/eae6043dee2b2d94cd9a22cb044db8b3.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: photo-film noir + NAME_ZH: 黑色电影 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: film noir style {prompt} . monochrome, high contrast, dramatic shadows, 1940s style, mysterious, cinematic + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, realism, photorealistic, vibrant, colorful + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/0b3629be5ebb7cda463877a1993a0298.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: photo-glamour + NAME_ZH: 魅力 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: glamorous photo {prompt} . high fashion, luxurious, extravagant, stylish, sensual, opulent, elegance, stunning beauty, professional, high contrast, detailed + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, distorted, grainy, sketch, low contrast, dull, plain, modest + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/f85dc6aa5d7f7d6d3663f2270e276796.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: photo-hdr + NAME_ZH: 高动态范围 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: HDR photo of {prompt} . High dynamic range, vivid, rich details, clear shadows and highlights, realistic, intense, enhanced contrast, highly detailed + NEGATIVE_PROMPT: flat, low contrast, oversaturated, underexposed, overexposed, blurred, noisy + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/269a98b284ae1345b0e5ac1f85c33179.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: photo-iphone photographic + NAME_ZH: iPhone摄影 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: iphone photo {prompt} . large depth of field, deep depth of field, highly detailed + NEGATIVE_PROMPT: drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly, shallow depth of field, bokeh + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/48c556a6b3c3a533847b80b031cd4962.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: photo-long exposure + NAME_ZH: 长曝光 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: long exposure photo of {prompt} . Blurred motion, streaks of light, surreal, dreamy, ghosting effect, highly detailed + NEGATIVE_PROMPT: static, noisy, deformed, shaky, abrupt, flat, low contrast + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/fda88c3fc43ea8cc04ce9b5c0261627f.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: photo-neon noir + NAME_ZH: 霓虹黑色 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: neon noir {prompt} . cyberpunk, dark, rainy streets, neon signs, high contrast, low light, vibrant, highly detailed + NEGATIVE_PROMPT: bright, sunny, daytime, low contrast, black and white, sketch, watercolor + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/8a5b71434592771e064af4b9017b59d6.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: photo-silhouette + NAME_ZH: 剪影 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: silhouette style {prompt} . high contrast, minimalistic, black and white, stark, dramatic + NEGATIVE_PROMPT: ugly, deformed, noisy, blurry, low contrast, color, realism, photorealistic + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/1892a37e66d33cdd9b713aaab6b8c597.jpg + PROMPT_EXAMPLE: a boy wearing green jacket + - + NAME: photo-tilt-shift + NAME_ZH: 倾斜移位 + DESCRIPTION: + SOURCE: twri + BASE_MODEL: PIXART + PROMPT: tilt-shift photo of {prompt} . selective focus, miniature effect, blurred background, highly detailed, vibrant, perspective control + NEGATIVE_PROMPT: blurry, noisy, deformed, flat, low contrast, unrealistic, oversaturated, underexposed + IMAGE_PATH: ms://iic/scepter@mantra_images_jpg/SD_XL1.0/d058a157de848a5c03d9a3e1e0e560a2.jpg + PROMPT_EXAMPLE: a boy wearing green jacket diff --git a/scepter/methods/studio/inference/dit/pixart_alpha_pro.yaml b/scepter/methods/studio/inference/dit/pixart_alpha_pro.yaml new file mode 100644 index 0000000..cfe2afc --- /dev/null +++ b/scepter/methods/studio/inference/dit/pixart_alpha_pro.yaml @@ -0,0 +1,120 @@ +NAME: PIXART_ALPHA +IS_DEFAULT: False +DEFAULT_PARAS: + PARAS: + RESOLUTIONS: [[1024, 1024]] + INPUT: + IMAGE: + ORIGINAL_SIZE_AS_TUPLE: [1024, 1024] + TARGET_SIZE_AS_TUPLE: [1024, 1024] + PROMPT: "" + NEGATIVE_PROMPT: "" + PROMPT_PREFIX: "" + SAMPLE: ddim + SAMPLE_STEPS: 20 + GUIDE_SCALE: 4.5 + GUIDE_RESCALE: 0.5 + DISCRETIZATION: trailing + OUTPUT: + LATENT: + IMAGES: + SEED: + MODULES_PARAS: + FIRST_STAGE_MODEL: + FUNCTION: + - + NAME: encode + DTYPE: float32 + INPUT: ["IMAGE"] + - + NAME: decode + DTYPE: float32 + INPUT: ["LATENT"] + PARAS: + SCALE_FACTOR: 0.18215 + SIZE_FACTOR: 8 + DIFFUSION_MODEL: + FUNCTION: + - + NAME: forward + DTYPE: float16 + INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE", "GUIDE_RESCALE", "DISCRETIZATION"] + COND_STAGE_MODEL: + FUNCTION: + - + NAME: encode + DTYPE: float32 + INPUT: ["PROMPT"] +# +MODEL: + PRETRAINED_MODEL: + DECODER_BIAS: 0.5 + SCHEDULE: + PARAMETERIZATION: "eps" + TIMESTEPS: 1000 + ZERO_TERMINAL_SNR: False + SCHEDULE_ARGS: + "NAME": "linear" + "BETA_MIN": 0.0001 + "BETA_MAX": 0.02 + # + DIFFUSION_MODEL: + NAME: PixArt + PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@PixArt-XL-2-1024-MS.pth + INPUT_SIZE: 128 + PATCH_SIZE: 2 + IN_CHANNELS: 4 + HIDDEN_SIZE: 1152 + DEPTH: 28 + NUM_HEADS: 16 + MLP_RATIO: 4.0 + CLASS_DROPOUT_PROB: 0.1 + PRED_SIGMA: True + DROP_PATH: 0.0 + WINDOW_DIZE: 0 + USE_REL_POS: False + CAPTION_CHANNELS: 4096 + LEWEI_SCALE: 2 + MODEL_MAX_LENGTH: 120 + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-2-base@512-base-ema.safetensors + EMBED_DIM: 4 + IGNORE_KEYS: [ ] + BATCH_SIZE: 1 + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + COND_STAGE_MODEL: + NAME: T5EmbedderHF + PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/ + TOKENIZER_PATH: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/ + LENGTH: 120 + CLEAN: heavy + USE_GRAD: False diff --git a/scepter/methods/studio/inference/dit/sd3_pro.yaml b/scepter/methods/studio/inference/dit/sd3_pro.yaml new file mode 100644 index 0000000..0d03de0 --- /dev/null +++ b/scepter/methods/studio/inference/dit/sd3_pro.yaml @@ -0,0 +1,148 @@ +NAME: SD3 +IS_DEFAULT: False +DEFAULT_PARAS: + PARAS: + RESOLUTIONS: [[1024, 1024]] + INPUT: + IMAGE: + ORIGINAL_SIZE_AS_TUPLE: [1024, 1024] + TARGET_SIZE_AS_TUPLE: [1024, 1024] + PROMPT: "" + NEGATIVE_PROMPT: "" + PROMPT_PREFIX: "" + SAMPLE: euler + SAMPLE_STEPS: 28 + GUIDE_SCALE: 5.0 + GUIDE_RESCALE: 0.0 + DISCRETIZATION: trailing + OUTPUT: + LATENT: + IMAGES: + SEED: + MODULES_PARAS: + FIRST_STAGE_MODEL: + FUNCTION: + - + NAME: encode + DTYPE: float32 + INPUT: ["IMAGE"] + - + NAME: decode + DTYPE: float32 + INPUT: ["LATENT"] + PARAS: + SCALE_FACTOR: 1.5305 + SHIFT_FACTOR: 0.0609 + SIZE_FACTOR: 8 + DIFFUSION_MODEL: + FUNCTION: + - + NAME: forward + DTYPE: float16 + INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE", "GUIDE_RESCALE", "DISCRETIZATION"] + COND_STAGE_MODEL: + FUNCTION: + - + NAME: encode + DTYPE: float32 + INPUT: ["PROMPT"] +# +MODEL: + PRETRAINED_MODEL: + SCHEDULE: + PARAMETERIZATION: rf + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: + IGNORE_KEYS: [ ] + SCALE_FACTOR: 1.5305 + SHIFT_FACTOR: 0.0609 + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "shifted" + "SHIFT": 3 + T_WEIGHT: uniform + # + DIFFUSION_MODEL: + NAME: MMDiT + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors + IGNORE_KEYS: '^first_stage_model.' + IN_CHANNELS: 16 + PATCH_SIZE: 2 + OUT_CHANNELS: 16 + DEPTH: 24 + INPUT_SIZE: + ADM_IN_CHANNELS: 2048 + CONTEXT_EMBEDDER_CONFIG: { 'target': 'torch.nn.Linear', 'params': { 'in_features': 4096, 'out_features': 1536 } } + NUM_PATCHES: 36864 + POS_EMBED_MAX_SIZE: 192 + POS_EMBED_SCALING_FACTOR: + USE_CHECKPOINT: True + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors + EMBED_DIM: 16 + IGNORE_KEYS: '^model.diffusion_model.' + BATCH_SIZE: 1 + USE_CONV: False + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 16 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 16 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + COND_STAGE_MODEL: + NAME: SD3TextEmbedder + P_ZERO: 0.0 + CLIP_L: + NAME: FrozenCLIPEmbedder2 + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer + MAX_LENGTH: 77 + FREEZE: True + LAYER: penultimate + RETURN_POOLED: True + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + CLIP_G: + NAME: FrozenCLIPEmbedder2 + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_2 + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_2 + MAX_LENGTH: 77 + FREEZE: True + LAYER: penultimate + RETURN_POOLED: True + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + T5_XXL: + NAME: T5EmbedderHF + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_3 + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_3 + LENGTH: 256 + CLEAN: whitespace + USE_GRAD: False + T5_DTYPE: float16 diff --git a/scepter/methods/studio/self_train/dit/pixart_alpha_pro.yaml b/scepter/methods/studio/self_train/dit/pixart_alpha_pro.yaml new file mode 100644 index 0000000..db38854 --- /dev/null +++ b/scepter/methods/studio/self_train/dit/pixart_alpha_pro.yaml @@ -0,0 +1,264 @@ +ENV: + BACKEND: nccl +META: + VERSION: 'PIXART_ALPHA' + DESCRIPTION: "PIXART ALPHA" + IS_DEFAULT: False + IS_SHARE: True + INFERENCE_PARAS: + INFERENCE_BATCH_SIZE: 1 + INFERENCE_PREFIX: "" + DEFAULT_SAMPLER: "ddim" + DEFAULT_SAMPLE_STEPS: 20 + INFERENCE_N_PROMPT: "" + RESOLUTION: [1024, 1024] + PARAS: + - + TRAIN_BATCH_SIZE: 2 + TRAIN_PREFIX: "" + TRAIN_N_PROMPT: "" + RESOLUTION: [1024, 1024] + MEMORY: 29000 + EPOCHS: 50 + SAVE_INTERVAL: 25 + EPSEC: 0.818 + LEARNING_RATE: 0.0001 + IS_DEFAULT: False + TUNER: FULL + - + TRAIN_BATCH_SIZE: 2 + TRAIN_PREFIX: "" + TRAIN_N_PROMPT: "" + RESOLUTION: [1024, 1024] + MEMORY: 29000 + EPOCHS: 50 + SAVE_INTERVAL: 25 + EPSEC: 0.818 + LEARNING_RATE: 0.0001 + IS_DEFAULT: False + TUNER: LORA + # + TUNERS: + LORA: + - + NAME: SwiftLoRA + R: 256 + LORA_ALPHA: 256 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "model.*(.q|.k|.v|.o|mlp.fc1|mlp.fc2)$" +# +SOLVER: + NAME: LatentDiffusionSolver + RESUME_FROM: + LOAD_MODEL_ONLY: True + USE_FSDP: False + SHARDING_STRATEGY: + USE_AMP: True + DTYPE: float16 + CHANNELS_LAST: True + MAX_STEPS: 1000 + MAX_EPOCHS: -1 + NUM_FOLDS: 1 + ACCU_STEP: 1 + EVAL_INTERVAL: -1 + RESCALE_LR: False + # + WORK_DIR: cache/scepter_ui/self_train/dit/pixart_alpha_pro + LOG_FILE: std_log.txt + # + FILE_SYSTEM: + NAME: "ModelscopeFs" + TEMP_DIR: "./cache/cache_data" + # + FREEZE: + # + TUNER: + # + MODEL: + NAME: LatentDiffusionPixart + PARAMETERIZATION: eps + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: + IGNORE_KEYS: [ ] + SCALE_FACTOR: 0.18215 + SIZE_FACTOR: 8 + DECODER_BIAS: 0.5 + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "linear" + "BETA_MIN": 0.0001 + "BETA_MAX": 0.02 + USE_EMA: False + LOAD_REFINER: False + # + DIFFUSION_MODEL: + NAME: PixArt + PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@PixArt-XL-2-1024-MS.pth + INPUT_SIZE: 128 + PATCH_SIZE: 2 + IN_CHANNELS: 4 + HIDDEN_SIZE: 1152 + DEPTH: 28 + NUM_HEADS: 16 + MLP_RATIO: 4.0 + CLASS_DROPOUT_PROB: 0.1 + PRED_SIGMA: True + DROP_PATH: 0.0 + WINDOW_DIZE: 0 + USE_REL_POS: False + CAPTION_CHANNELS: 4096 + LEWEI_SCALE: 2 + MODEL_MAX_LENGTH: 120 + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-2-base@512-base-ema.safetensors + EMBED_DIM: 4 + IGNORE_KEYS: [ ] + BATCH_SIZE: 1 + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 4 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + COND_STAGE_MODEL: + NAME: T5EmbedderHF + PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/ + TOKENIZER_PATH: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/ + LENGTH: 120 + CLEAN: heavy + USE_GRAD: False + # + LOSS: + NAME: ReconstructLoss + LOSS_TYPE: l2 + # + SAMPLE_ARGS: + SAMPLER: ddim + SAMPLE_STEPS: 20 + SEED: 2024 + GUIDE_SCALE: 4.5 + GUIDE_RESCALE: 0.5 + DISCRETIZATION: trailing + RUN_TRAIN_N: False + # + OPTIMIZER: + NAME: AdamW + LEARNING_RATE: 0.0001 + BETAS: [ 0.9, 0.999 ] + EPS: 1e-8 + WEIGHT_DECAY: 1e-2 + AMSGRAD: False + # + TRAIN_DATA: + NAME: ImageTextPairMSDataset + MODE: train + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_DATASET_SPLIT: train + MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' } + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + SAMPLER: + NAME: LoopSampler + TRANSFORMS: + - NAME: LoadImageFromFile + RGB_ORDER: RGB + BACKEND: pillow + - NAME: FlexibleResize + INTERPOLATION: bilinear + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: FlexibleCenterCrop + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: ImageToTensor + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: Normalize + MEAN: [ 0.5, 0.5, 0.5 ] + STD: [ 0.5, 0.5, 0.5 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'image' ] + BACKEND: torchvision + - NAME: Select + KEYS: [ 'image', 'prompt' ] + META_KEYS: [ 'data_key' ] + # + EVAL_DATA: + NAME: Text2ImageDataset + MODE: eval + PROMPT_FILE: + PROMPT_DATA: [ "a boy wearing a jacket", "a dog running on the lawn" ] + IMAGE_SIZE: [ 1024, 1024 ] + FIELDS: [ "prompt" ] + DELIMITER: '#;#' + PROMPT_PREFIX: '' + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + TRANSFORMS: + - NAME: Select + KEYS: [ 'index', 'prompt' ] + META_KEYS: [ 'image_size' ] + # + TRAIN_HOOKS: + - + NAME: BackwardHook + PRIORITY: 0 + - + NAME: LogHook + LOG_INTERVAL: 10 + SHOW_GPU_MEM: True + - + NAME: TensorboardLogHook + - + NAME: CheckpointHook + INTERVAL: 10000 + PRIORITY: 200 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + DISABLE_SNAPSHOT: True + # + EVAL_HOOKS: + - + NAME: ProbeDataHook + PROB_INTERVAL: 100 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + SAVE_PROBE_PREFIX: 'image' diff --git a/scepter/methods/studio/self_train/dit/sd3_pro.yaml b/scepter/methods/studio/self_train/dit/sd3_pro.yaml new file mode 100644 index 0000000..c6fe6d8 --- /dev/null +++ b/scepter/methods/studio/self_train/dit/sd3_pro.yaml @@ -0,0 +1,284 @@ +ENV: + BACKEND: nccl +META: + VERSION: 'SD3' + DESCRIPTION: "SD3 ALPHA" + IS_DEFAULT: False + IS_SHARE: True + INFERENCE_PARAS: + INFERENCE_BATCH_SIZE: 1 + INFERENCE_PREFIX: "" + DEFAULT_SAMPLER: "euler" + DEFAULT_SAMPLE_STEPS: 28 + INFERENCE_N_PROMPT: "" + RESOLUTION: [1024, 1024] + PARAS: + - + TRAIN_BATCH_SIZE: 2 + TRAIN_PREFIX: "" + TRAIN_N_PROMPT: "" + RESOLUTION: [1024, 1024] + MEMORY: 29000 + EPOCHS: 50 + SAVE_INTERVAL: 25 + EPSEC: 0.818 + LEARNING_RATE: 1e-5 + IS_DEFAULT: False + TUNER: FULL + - + TRAIN_BATCH_SIZE: 2 + TRAIN_PREFIX: "" + TRAIN_N_PROMPT: "" + RESOLUTION: [1024, 1024] + MEMORY: 29000 + EPOCHS: 50 + SAVE_INTERVAL: 25 + EPSEC: 0.818 + LEARNING_RATE: 5e-5 + IS_DEFAULT: True + TUNER: LORA + # + TUNERS: + LORA: + - + NAME: SwiftLoRA + R: 128 + LORA_ALPHA: 128 + LORA_DROPOUT: 0.0 + BIAS: "none" + TARGET_MODULES: "model.*(.attn.qkv|.attn.proj|mlp.fc1|mlp.fc2)$" +# +SOLVER: + NAME: LatentDiffusionSolver + RESUME_FROM: + LOAD_MODEL_ONLY: True + USE_FSDP: False + SHARDING_STRATEGY: + USE_AMP: True + DTYPE: float16 + CHANNELS_LAST: True + MAX_STEPS: 1000 + MAX_EPOCHS: -1 + NUM_FOLDS: 1 + ACCU_STEP: 1 + EVAL_INTERVAL: -1 + RESCALE_LR: False + # + WORK_DIR: cache/scepter_ui/self_train/dit/sd3 + LOG_FILE: std_log.txt + # + FILE_SYSTEM: + - NAME: "ModelscopeFs" + TEMP_DIR: "./cache/cache_data" + # + FREEZE: + # + TUNER: + # + MODEL: + NAME: LatentDiffusionSD3 + PARAMETERIZATION: rf + TIMESTEPS: 1000 + MIN_SNR_GAMMA: + ZERO_TERMINAL_SNR: False + PRETRAINED_MODEL: + IGNORE_KEYS: [ ] + SCALE_FACTOR: 1.5305 + SHIFT_FACTOR: 0.0609 + DEFAULT_N_PROMPT: + SCHEDULE_ARGS: + "NAME": "shifted" + "SHIFT": 3 + USE_EMA: False + T_WEIGHT: uniform + # + DIFFUSION_MODEL: + NAME: MMDiT + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors + IGNORE_KEYS: '^first_stage_model.' + IN_CHANNELS: 16 + PATCH_SIZE: 2 + OUT_CHANNELS: 16 + DEPTH: 24 + INPUT_SIZE: + ADM_IN_CHANNELS: 2048 + CONTEXT_EMBEDDER_CONFIG: { 'target': 'torch.nn.Linear', 'params': { 'in_features': 4096, 'out_features': 1536 } } + NUM_PATCHES: 36864 + POS_EMBED_MAX_SIZE: 192 + POS_EMBED_SCALING_FACTOR: + USE_CHECKPOINT: True + # + FIRST_STAGE_MODEL: + NAME: AutoencoderKL + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors + EMBED_DIM: 16 + IGNORE_KEYS: '^model.diffusion_model.' + BATCH_SIZE: 1 + USE_CONV: False + # + ENCODER: + NAME: Encoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 16 + DOUBLE_Z: True + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + # + DECODER: + NAME: Decoder + CH: 128 + OUT_CH: 3 + NUM_RES_BLOCKS: 2 + IN_CHANNELS: 3 + ATTN_RESOLUTIONS: [ ] + CH_MULT: [ 1, 2, 4, 4 ] + Z_CHANNELS: 16 + DROPOUT: 0.0 + RESAMP_WITH_CONV: True + GIVE_PRE_END: False + TANH_OUT: False + # + COND_STAGE_MODEL: + NAME: SD3TextEmbedder + P_ZERO: 0.0 + CLIP_L: + NAME: FrozenCLIPEmbedder2 + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer + MAX_LENGTH: 77 + FREEZE: True + LAYER: penultimate + RETURN_POOLED: True + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + CLIP_G: + NAME: FrozenCLIPEmbedder2 + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_2 + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_2 + MAX_LENGTH: 77 + FREEZE: True + LAYER: penultimate + RETURN_POOLED: True + USE_FINAL_LAYER_NORM: False + IS_TRAINABLE: False + T5_XXL: + NAME: T5EmbedderHF + PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_3 + TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_3 + LENGTH: 256 + CLEAN: whitespace + USE_GRAD: False + T5_DTYPE: float16 + # + LOSS: + NAME: ReconstructLoss + LOSS_TYPE: l2 + # + SAMPLE_ARGS: + SAMPLER: euler + SAMPLE_STEPS: 28 + SEED: 1749023094 + GUIDE_SCALE: 5.0 + GUIDE_RESCALE: 0.0 + DISCRETIZATION: trailing + RUN_TRAIN_N: False + # + OPTIMIZER: + NAME: AdamW + LEARNING_RATE: 5e-5 + BETAS: [ 0.9, 0.999 ] + EPS: 1e-8 + WEIGHT_DECAY: 1e-2 + AMSGRAD: False + # + TRAIN_DATA: + NAME: ImageTextPairMSDataset + MODE: train + MS_DATASET_NAME: style_custom_dataset + MS_DATASET_NAMESPACE: damo + MS_DATASET_SUBNAME: 3D + PROMPT_PREFIX: "" + MS_DATASET_SPLIT: train + MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' } + REPLACE_STYLE: False + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + SAMPLER: + NAME: LoopSampler + TRANSFORMS: + - NAME: LoadImageFromFile + RGB_ORDER: RGB + BACKEND: pillow + - NAME: FlexibleResize + INTERPOLATION: bilinear + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: FlexibleCenterCrop + SIZE: [ 1024, 1024 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: ImageToTensor + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'img' ] + BACKEND: pillow + - NAME: Normalize + MEAN: [ 0.5, 0.5, 0.5 ] + STD: [ 0.5, 0.5, 0.5 ] + INPUT_KEY: [ 'img' ] + OUTPUT_KEY: [ 'image' ] + BACKEND: torchvision + - NAME: Select + KEYS: [ 'image', 'prompt' ] + META_KEYS: [ 'data_key' ] + # + EVAL_DATA: + NAME: Text2ImageDataset + MODE: eval + PROMPT_FILE: + PROMPT_DATA: [ "a cat holds a blackboard that writes \"hello world\"", "a dog running on the lawn" ] + IMAGE_SIZE: [ 1024, 1024 ] + FIELDS: [ "prompt" ] + DELIMITER: '#;#' + PROMPT_PREFIX: '' + PIN_MEMORY: True + BATCH_SIZE: 1 + NUM_WORKERS: 4 + TRANSFORMS: + - NAME: Select + KEYS: [ 'index', 'prompt' ] + META_KEYS: [ 'image_size' ] + # + TRAIN_HOOKS: + - + NAME: BackwardHook + PRIORITY: 1 + - + NAME: LogHook + LOG_INTERVAL: 10 + SHOW_GPU_MEM: True + - + NAME: TensorboardLogHook + - + NAME: CheckpointHook + INTERVAL: 10000 + PRIORITY: 200 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + DISABLE_SNAPSHOT: True + # + EVAL_HOOKS: + - + NAME: ProbeDataHook + PROB_INTERVAL: 100 + SAVE_LAST: True + SAVE_NAME_PREFIX: 'step' + SAVE_PROBE_PREFIX: 'image' diff --git a/scepter/methods/studio/self_train/edit/stylebooth_tb_pro.yaml b/scepter/methods/studio/self_train/edit/stylebooth_tb_pro.yaml index 3b75264..11c27de 100644 --- a/scepter/methods/studio/self_train/edit/stylebooth_tb_pro.yaml +++ b/scepter/methods/studio/self_train/edit/stylebooth_tb_pro.yaml @@ -142,13 +142,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: -1 + RESCALE_LR: False # WORK_DIR: LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: # @@ -258,7 +259,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 @@ -267,7 +268,7 @@ SOLVER: TRAIN_DATA: NAME: ImageTextPairMSDataset MODE: train - MS_DATASET_NAME: cache/save_data/delogo/ + MS_DATASET_NAME: ./cache/cache_data/delogo/ MS_DATASET_NAMESPACE: "" MS_DATASET_SPLIT: "train" MS_DATASET_SUBNAME: "" diff --git a/scepter/methods/studio/self_train/sd_xl/sdxl_pro.yaml b/scepter/methods/studio/self_train/sd_xl/sdxl_pro.yaml index ef041ec..7101059 100644 --- a/scepter/methods/studio/self_train/sd_xl/sdxl_pro.yaml +++ b/scepter/methods/studio/self_train/sd_xl/sdxl_pro.yaml @@ -147,6 +147,7 @@ SOLVER: MAX_EPOCHS: -1 # NUM_FOLDS DESCRIPTION: Num folds for training. TYPE: int default: 1 NUM_FOLDS: 1 + RESCALE_LR: False # EVAL_INTERVAL: -1 # WORK_DIR DESCRIPTION: Save dir of the training log or model. TYPE: str default: '' @@ -155,7 +156,7 @@ SOLVER: LOG_FILE: std_log.txt FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" TUNER: # MODEL DESCRIPTION: TYPE: default: '' MODEL: @@ -538,7 +539,7 @@ SOLVER: OPTIMIZER: # NAME DESCRIPTION: TYPE: default: '' NAME: AdamW - LEARNING_RATE: 0.0064 + LEARNING_RATE: 0.00001 EPS: 1e-8 AMSGRAD: False # diff --git a/scepter/methods/studio/self_train/stable_diffusion/sd15_pro.yaml b/scepter/methods/studio/self_train/stable_diffusion/sd15_pro.yaml index c4a7880..a49ce83 100644 --- a/scepter/methods/studio/self_train/stable_diffusion/sd15_pro.yaml +++ b/scepter/methods/studio/self_train/stable_diffusion/sd15_pro.yaml @@ -137,13 +137,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: -1 + RESCALE_LR: False # WORK_DIR: LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: # @@ -249,7 +250,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/methods/studio/self_train/stable_diffusion/sd21_pro.yaml b/scepter/methods/studio/self_train/stable_diffusion/sd21_pro.yaml index e4c406a..db54dfa 100644 --- a/scepter/methods/studio/self_train/stable_diffusion/sd21_pro.yaml +++ b/scepter/methods/studio/self_train/stable_diffusion/sd21_pro.yaml @@ -80,13 +80,14 @@ SOLVER: NUM_FOLDS: 1 ACCU_STEP: 1 EVAL_INTERVAL: -1 + RESCALE_LR: False # WORK_DIR: LOG_FILE: std_log.txt # FILE_SYSTEM: NAME: "ModelscopeFs" - TEMP_DIR: "./cache/data" + TEMP_DIR: "./cache/cache_data" # FREEZE: # @@ -191,7 +192,7 @@ SOLVER: # OPTIMIZER: NAME: AdamW - LEARNING_RATE: 0.064 + LEARNING_RATE: 0.0001 BETAS: [ 0.9, 0.999 ] EPS: 1e-8 WEIGHT_DECAY: 1e-2 diff --git a/scepter/modules/inference/diffusion_inference.py b/scepter/modules/inference/diffusion_inference.py index 43ceac7..ff1a427 100644 --- a/scepter/modules/inference/diffusion_inference.py +++ b/scepter/modules/inference/diffusion_inference.py @@ -33,10 +33,12 @@ class DiffusionInference(): ''' def __init__(self, logger=None): self.logger = logger + self.is_redefine_paras = True self.loaded_model = {} self.loaded_model_name = [ 'diffusion_model', 'first_stage_model', 'cond_stage_model' ] + self.diffusion_insclass = GaussianDiffusion self.tuner_infer = TunerInference(self.logger) self.control_infer = ControlInference(self.logger) @@ -45,7 +47,8 @@ class DiffusionInference(): self.is_default = cfg.get('IS_DEFAULT', False) module_paras = self.load_default(cfg.get('DEFAULT_PARAS', None)) assert cfg.have('MODEL') - cfg.MODEL = self.redefine_paras(cfg.MODEL) + if self.is_redefine_paras: + cfg.MODEL = self.redefine_paras(cfg.MODEL) self.diffusion = self.load_schedule(cfg.MODEL.SCHEDULE) self.diffusion_model = self.infer_model( cfg.MODEL.DIFFUSION_MODEL, module_paras.get( @@ -316,8 +319,8 @@ class DiffusionInference(): def load_schedule(self, cfg): parameterization = cfg.get('PARAMETERIZATION', 'eps') assert parameterization in [ - 'eps', 'x0', 'v' - ], 'currently only supporting "eps" and "x0" and "v"' + 'eps', 'x0', 'v', 'rf' + ], 'currently only supporting "eps" and "x0" and "v" and "rf"' num_timesteps = cfg.get('TIMESTEPS', 1000) schedule_args = { @@ -336,8 +339,8 @@ class DiffusionInference(): n=num_timesteps, zero_terminal_snr=zero_terminal_snr, **schedule_args) - diffusion = GaussianDiffusion(sigmas=sigmas, - prediction_type=parameterization) + diffusion = self.diffusion_insclass(sigmas=sigmas, + prediction_type=parameterization) return diffusion def get_batch(self, value_dict, num_samples=1): diff --git a/scepter/modules/inference/largen_inference.py b/scepter/modules/inference/largen_inference.py index 811cb6b..7caf409 100644 --- a/scepter/modules/inference/largen_inference.py +++ b/scepter/modules/inference/largen_inference.py @@ -8,16 +8,12 @@ from collections import OrderedDict import gradio as gr import torch import torchvision.transforms.functional as TF +from scepter.modules.model.network.diffusion.diffusion import GaussianDiffusion from scepter.modules.model.utils.data_utils import crop_back from scepter.modules.utils.distribute import we from scepter.modules.utils.file_system import FS -from .diffusion_inference import DiffusionInference - - -def get_model(model_tuple): - assert 'model' in model_tuple - return model_tuple['model'] +from .diffusion_inference import DiffusionInference, get_model class LargenInference(DiffusionInference): @@ -28,10 +24,12 @@ class LargenInference(DiffusionInference): ''' def __init__(self, logger=None): self.logger = logger + self.is_redefine_paras = True self.loaded_model = {} self.loaded_model_name = [ 'diffusion_model', 'first_stage_model', 'cond_stage_model' ] + self.diffusion_insclass = GaussianDiffusion def redefine_paras(self, cfg): if cfg.get('PRETRAINED_MODEL', None): diff --git a/scepter/modules/inference/pixart_inference.py b/scepter/modules/inference/pixart_inference.py new file mode 100644 index 0000000..fc58e24 --- /dev/null +++ b/scepter/modules/inference/pixart_inference.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +import copy +import os.path +import random +from collections import OrderedDict + +import torch +import torch.nn.functional as F +from PIL.Image import Image +from scepter.modules.model.network.diffusion.diffusion import GaussianDiffusion +from scepter.modules.model.network.diffusion.schedules import noise_schedule +from scepter.modules.model.registry import (BACKBONES, EMBEDDERS, MODELS, + TOKENIZERS) +from scepter.modules.utils.distribute import we +from scepter.modules.utils.file_system import FS +from scepter.studio.utils.env import get_available_memory + +from .control_inference import ControlInference +from .diffusion_inference import DiffusionInference, get_model +from .tuner_inference import TunerInference + + +class PixArtInference(DiffusionInference): + ''' + define vae, unet, text-encoder, tuner, refiner components + support to load the components dynamicly. + create and load model when run this model at the first time. + ''' + def __init__(self, logger=None): + self.logger = logger + self.is_redefine_paras = False + self.loaded_model = {} + self.loaded_model_name = [ + 'diffusion_model', 'first_stage_model', 'cond_stage_model' + ] + self.diffusion_insclass = GaussianDiffusion + self.tuner_infer = TunerInference(self.logger) + self.control_infer = ControlInference(self.logger) + + @torch.no_grad() + def __call__(self, + input, + num_samples=1, + intermediate_callback=None, + refine_strength=0, + img_to_img_strength=0, + cat_uc=True, + tuner_model=None, + control_model=None, + **kwargs): + + value_input = copy.deepcopy(self.input) + value_input.update(input) + print(value_input) + height, width = value_input['target_size_as_tuple'] + value_output = copy.deepcopy(self.output) + + # register tuner + if tuner_model is not None and tuner_model != '' and len( + tuner_model) > 0: + if not isinstance(tuner_model, list): + tuner_model = [tuner_model] + self.dynamic_load(self.diffusion_model, 'diffusion_model') + self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + self.tuner_infer.register_tuner(tuner_model, self.diffusion_model, + self.cond_stage_model) + self.dynamic_unload(self.diffusion_model, + 'diffusion_model', + skip_loaded=True) + self.dynamic_unload(self.cond_stage_model, + 'cond_stage_model', + skip_loaded=True) + + # cond stage + self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + function_name, dtype = self.get_function_info(self.cond_stage_model) + with torch.autocast('cuda', + enabled=dtype == 'float16', + dtype=getattr(torch, dtype)): + context, null_context = {}, {} + cont_mask = None + cont, cont_mask = getattr(get_model(self.cond_stage_model), + function_name)(value_input['prompt'], + return_mask=True) + context['crossattn'] = cont.float() + null_context['crossattn'] = get_model( + self.diffusion_model).y_embedder.y_embedding[None].repeat( + num_samples, 1, 1) + self.dynamic_unload(self.cond_stage_model, + 'cond_stage_model', + skip_loaded=True) + + # get noise + seed = kwargs.pop('seed', -1) + g = torch.Generator(device=we.device_id) + seed = seed if seed >= 0 else random.randint(0, 2**32 - 1) + g.manual_seed(seed) + if 'seed' in value_output: + value_output['seed'] = seed + for sample_id in range(num_samples): + if self.diffusion_model is not None: + noise = torch.empty( + 1, + 4, + height // self.first_stage_model['paras']['size_factor'], + width // self.first_stage_model['paras']['size_factor'], + device=we.device_id).normal_(generator=g) + + self.dynamic_load(self.diffusion_model, 'diffusion_model') + # UNet use input n_prompt + function_name, dtype = self.get_function_info( + self.diffusion_model) + with torch.autocast('cuda', + enabled=dtype == 'float16', + dtype=getattr(torch, dtype)): + latent = self.diffusion.sample( + solver=value_input.get('sample', 'ddim'), + noise=noise, + model=get_model(self.diffusion_model), + model_kwargs=[{ + 'cond': context, + 'mask': cont_mask, + 'data_info': { + 'img_hw': + torch.tensor([[height, width]], + dtype=torch.float, + device=we.device_id).repeat( + num_samples, 1), + 'aspect_ratio': + torch.tensor([[1.]], + device=we.device_id).repeat( + num_samples, 1) + } + }, { + 'cond': null_context, + 'mask': cont_mask, + 'data_info': { + 'img_hw': + torch.tensor([[height, width]], + dtype=torch.float, + device=we.device_id).repeat( + num_samples, 1), + 'aspect_ratio': + torch.tensor([[1.]], + device=we.device_id).repeat( + num_samples, 1) + } + }], + cat_uc=False, + steps=value_input.get('sample_steps', 50), + guide_scale=value_input.get('guide_scale', 7.5), + guide_rescale=value_input.get('guide_rescale', 0.5), + discretization=value_input.get('discretization', + 'trailing'), + show_progress=True, + seed=seed, + condition_fn=None, + clamp=None, + percentile=None, + t_max=None, + t_min=None, + discard_penultimate_step=None, + return_intermediate=None, + **kwargs) + + self.dynamic_unload(self.diffusion_model, + 'diffusion_model', + skip_loaded=True) + + if 'latent' in value_output: + if value_output['latent'] is None or ( + isinstance(value_output['latent'], list) + and len(value_output['latent']) < 1): + value_output['latent'] = [] + value_output['latent'].append(latent) + + self.dynamic_load(self.first_stage_model, 'first_stage_model') + x_samples = self.decode_first_stage(latent).float() + self.dynamic_unload(self.first_stage_model, + 'first_stage_model', + skip_loaded=True) + images = torch.clamp((x_samples + 1.0) / 2.0, min=0.0, max=1.0) + if 'images' in value_output: + if value_output['images'] is None or ( + isinstance(value_output['images'], list) + and len(value_output['images']) < 1): + value_output['images'] = [] + value_output['images'].append(images) + + for k, v in value_output.items(): + if isinstance(v, list): + value_output[k] = torch.cat(v, dim=0) + if isinstance(v, torch.Tensor): + value_output[k] = v.cpu() + + # unregister tuner + if tuner_model is not None and tuner_model != '' and len( + tuner_model) > 0: + self.tuner_infer.unregister_tuner(tuner_model, + self.diffusion_model, + self.cond_stage_model) + + # unregister control + if control_model is not None and control_model != '': + self.control_infer.unregister_controllers(control_model, + self.diffusion_model) + + return value_output diff --git a/scepter/modules/inference/sd3_inference.py b/scepter/modules/inference/sd3_inference.py new file mode 100644 index 0000000..c7b8605 --- /dev/null +++ b/scepter/modules/inference/sd3_inference.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +import copy +import random + +import gradio as gr +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from scepter.modules.model.network.diffusion.diffusion import \ + GaussianDiffusionRF +from scepter.modules.utils.distribute import we + +from .control_inference import ControlInference +from .diffusion_inference import DiffusionInference, get_model +from .tuner_inference import TunerInference + + +class SD3Inference(DiffusionInference): + def __init__(self, logger=None): + self.logger = logger + self.is_redefine_paras = False + self.loaded_model = {} + self.loaded_model_name = [ + 'diffusion_model', 'first_stage_model', 'cond_stage_model' + ] + self.diffusion_insclass = GaussianDiffusionRF + self.tuner_infer = TunerInference(self.logger) + self.control_infer = ControlInference(self.logger) + + @torch.no_grad() + def decode_first_stage(self, z): + _, dtype = self.get_function_info(self.first_stage_model, 'decode') + # with torch.autocast('cuda', enabled=dtype == 'float16', dtype=getattr(torch, dtype)): + z = 1. / self.first_stage_model['paras'][ + 'scale_factor'] * z + self.first_stage_model['paras'][ + 'shift_factor'] + return get_model(self.first_stage_model).decode(z) + + @torch.no_grad() + def __call__(self, + input, + num_samples=1, + cat_uc=True, + tuner_model=None, + control_model=None, + **kwargs): + + value_input = copy.deepcopy(self.input) + value_input.update(input) + print(value_input) + height, width = value_input['target_size_as_tuple'] + value_output = copy.deepcopy(self.output) + + # register tuner + if tuner_model is not None and tuner_model != '' and len( + tuner_model) > 0: + if not isinstance(tuner_model, list): + tuner_model = [tuner_model] + self.dynamic_load(self.diffusion_model, 'diffusion_model') + self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + self.tuner_infer.register_tuner(tuner_model, self.diffusion_model, + self.cond_stage_model) + self.dynamic_unload(self.diffusion_model, + 'diffusion_model', + skip_loaded=True) + self.dynamic_unload(self.cond_stage_model, + 'cond_stage_model', + skip_loaded=True) + + # cond stage + self.dynamic_load(self.cond_stage_model, 'cond_stage_model') + function_name, dtype = self.get_function_info(self.cond_stage_model) + context, null_context = {}, {} + with torch.autocast('cuda', + enabled=dtype == 'float16', + dtype=getattr(torch, dtype)): + ctx, pooled = getattr(get_model(self.cond_stage_model), + function_name)(value_input['prompt']) + null_ctx, null_pooled = getattr(get_model(self.cond_stage_model), + function_name)([''] * num_samples) + context['crossattn'] = ctx.float() + context['y'] = pooled.float() + null_context['crossattn'] = null_ctx.float() + null_context['y'] = null_pooled.float() + self.dynamic_unload(self.cond_stage_model, + 'cond_stage_model', + skip_loaded=True) + + # get noise + seed = kwargs.pop('seed', -1) + g = torch.Generator(device=we.device_id) + seed = seed if seed >= 0 else random.randint(0, 2**32 - 1) + g.manual_seed(seed) + if 'seed' in value_output: + value_output['seed'] = seed + for sample_id in range(num_samples): + if self.diffusion_model is not None: + noise = torch.empty( + 1, + 16, + height // self.first_stage_model['paras']['size_factor'], + width // self.first_stage_model['paras']['size_factor'], + device=we.device_id).normal_(generator=g) + + self.dynamic_load(self.diffusion_model, 'diffusion_model') + # UNet use input n_prompt + function_name, dtype = self.get_function_info( + self.diffusion_model) + with torch.autocast('cuda', + enabled=dtype == 'float16', + dtype=getattr(torch, dtype)): + solver_sample = value_input.get('sample', 'ddim') + sample_steps = value_input.get('sample_steps', 50) + guide_scale = value_input.get('guide_scale', 7.5) + guide_rescale = value_input.get('guide_rescale', 0.5) + discretization = value_input.get('discretization', + 'trailing') + latent = self.diffusion.sample( + solver=solver_sample, + noise=noise, + model=get_model(self.diffusion_model), + model_kwargs=[{ + 'cond': context + }, { + 'cond': null_context + }] + if guide_scale is not None and guide_scale > 0 else { + 'cond': context, + }, + cat_uc=False, + steps=sample_steps, + guide_scale=guide_scale, + guide_rescale=guide_rescale, + discretization=discretization, + show_progress=True, + seed=seed, + condition_fn=None, + clamp=None, + percentile=None, + t_max=None, + t_min=None, + discard_penultimate_step=None, + return_intermediate=None, + **kwargs) + + self.dynamic_unload(self.diffusion_model, + 'diffusion_model', + skip_loaded=True) + + if 'latent' in value_output: + if value_output['latent'] is None or ( + isinstance(value_output['latent'], list) + and len(value_output['latent']) < 1): + value_output['latent'] = [] + value_output['latent'].append(latent) + + self.dynamic_load(self.first_stage_model, 'first_stage_model') + x_samples = self.decode_first_stage(latent).float() + self.dynamic_unload(self.first_stage_model, + 'first_stage_model', + skip_loaded=True) + images = torch.clamp((x_samples + 1.0) / 2.0, min=0.0, max=1.0) + if 'images' in value_output: + if value_output['images'] is None or ( + isinstance(value_output['images'], list) + and len(value_output['images']) < 1): + value_output['images'] = [] + value_output['images'].append(images) + + for k, v in value_output.items(): + if isinstance(v, list): + value_output[k] = torch.cat(v, dim=0) + if isinstance(v, torch.Tensor): + value_output[k] = v.cpu() + + # unregister tuner + if tuner_model is not None and tuner_model != '' and len( + tuner_model) > 0: + self.tuner_infer.unregister_tuner(tuner_model, + self.diffusion_model, + self.cond_stage_model) + + # unregister control + if control_model is not None and control_model != '': + self.control_infer.unregister_controllers(control_model, + self.diffusion_model) + + return value_output diff --git a/scepter/modules/inference/stylebooth_inference.py b/scepter/modules/inference/stylebooth_inference.py index 156bf02..8a4c892 100644 --- a/scepter/modules/inference/stylebooth_inference.py +++ b/scepter/modules/inference/stylebooth_inference.py @@ -7,18 +7,14 @@ import gradio as gr import torch import torch.nn.functional as F import torchvision.transforms.functional as TF +from scepter.modules.model.network.diffusion.diffusion import GaussianDiffusion from scepter.modules.utils.distribute import we from .control_inference import ControlInference -from .diffusion_inference import DiffusionInference +from .diffusion_inference import DiffusionInference, get_model from .tuner_inference import TunerInference -def get_model(model_tuple): - assert 'model' in model_tuple - return model_tuple['model'] - - class StyleboothInference(DiffusionInference): ''' define vae, unet, text-encoder, tuner, refiner components @@ -27,10 +23,12 @@ class StyleboothInference(DiffusionInference): ''' def __init__(self, logger=None): self.logger = logger + self.is_redefine_paras = True self.loaded_model = {} self.loaded_model_name = [ 'diffusion_model', 'first_stage_model', 'cond_stage_model' ] + self.diffusion_insclass = GaussianDiffusion self.tuner_infer = TunerInference(self.logger) self.control_infer = ControlInference(self.logger) diff --git a/scepter/modules/model/backbone/__init__.py b/scepter/modules/model/backbone/__init__.py index 03879b0..6f5c074 100644 --- a/scepter/modules/model/backbone/__init__.py +++ b/scepter/modules/model/backbone/__init__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -from scepter.modules.model.backbone import (autoencoder, image, unet, utils, - video) +from scepter.modules.model.backbone import (autoencoder, image, mmdit, pixart, + unet, utils, video) diff --git a/scepter/modules/model/backbone/mmdit/__init__.py b/scepter/modules/model/backbone/mmdit/__init__.py new file mode 100644 index 0000000..a588e26 --- /dev/null +++ b/scepter/modules/model/backbone/mmdit/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from .sd3 import MMDiT diff --git a/scepter/modules/model/backbone/mmdit/sd3.py b/scepter/modules/model/backbone/mmdit/sd3.py new file mode 100644 index 0000000..9296830 --- /dev/null +++ b/scepter/modules/model/backbone/mmdit/sd3.py @@ -0,0 +1,1330 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. + +# This file contains code that is adapted from +# diffusers: https://github.com/huggingface/diffusers +# ComfyUI: https://github.com/comfyanonymous/ComfyUI + +import logging +import math +import re +from collections import OrderedDict +from functools import partial +from typing import Dict, Optional + +import numpy as np +import torch +import torch.nn as nn +from einops import rearrange, repeat + +from scepter.modules.model.base_model import BaseModel +from scepter.modules.model.registry import BACKBONES +from scepter.modules.utils.config import Config +from scepter.modules.utils.file_system import FS + +try: + import xformers + import xformers.ops + XFORMERS_IS_AVAILBLE = True +except: + XFORMERS_IS_AVAILBLE = False + +BROKEN_XFORMERS = False +try: + x_vers = xformers.__version__ + # XFormers bug confirmed on all versions from 0.0.21 to 0.0.26 (q with bs bigger than 65535 gives CUDA error) + BROKEN_XFORMERS = x_vers.startswith( + '0.0.2') and not x_vers.startswith('0.0.20') +except: + pass + + +def attention_xformers(q, k, v, heads, mask=None, attn_precision=None): + b, _, dim_head = q.shape + dim_head //= heads + + disabled_xformers = False + + if BROKEN_XFORMERS: + if b * heads > 65535: + disabled_xformers = True + + if not disabled_xformers: + if torch.jit.is_tracing() or torch.jit.is_scripting(): + disabled_xformers = True + + if disabled_xformers: + return attention_pytorch(q, k, v, heads, mask) + + q, k, v = map( + lambda t: t.reshape(b, -1, heads, dim_head), + (q, k, v), + ) + + if mask is not None: + pad = 8 - q.shape[1] % 8 + mask_out = torch.empty([q.shape[0], q.shape[1], q.shape[1] + pad], + dtype=q.dtype, + device=q.device) + mask_out[:, :, :mask.shape[-1]] = mask + mask = mask_out[:, :, :mask.shape[-1]] + + out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=mask) + + out = (out.reshape(b, -1, heads * dim_head)) + return out + + +def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None): + b, _, dim_head = q.shape + dim_head //= heads + q, k, v = map( + lambda t: t.view(b, -1, heads, dim_head).transpose(1, 2), + (q, k, v), + ) + + out = torch.nn.functional.scaled_dot_product_attention(q, + k, + v, + attn_mask=mask, + dropout_p=0.0, + is_causal=False) + out = (out.transpose(1, 2).reshape(b, -1, heads * dim_head)) + return out + + +def default(x, y): + if x is not None: + return x + return y + + +class Mlp(nn.Module): + """ MLP as used in Vision Transformer, MLP-Mixer and related networks + """ + def __init__( + self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + norm_layer=None, + bias=True, + drop=0., + use_conv=False, + dtype=None, + device=None, + operations=None, + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + drop_probs = drop + linear_layer = partial( + operations.Conv2d, + kernel_size=1) if use_conv else operations.Linear + + self.fc1 = linear_layer(in_features, + hidden_features, + bias=bias, + dtype=dtype, + device=device) + self.act = act_layer() + self.drop1 = nn.Dropout(drop_probs) + self.norm = norm_layer( + hidden_features) if norm_layer is not None else nn.Identity() + self.fc2 = linear_layer(hidden_features, + out_features, + bias=bias, + dtype=dtype, + device=device) + self.drop2 = nn.Dropout(drop_probs) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop1(x) + x = self.norm(x) + x = self.fc2(x) + x = self.drop2(x) + return x + + +class PatchEmbed(nn.Module): + """ 2D Image to Patch Embedding + """ + dynamic_img_pad: torch.jit.Final[bool] + + def __init__( + self, + img_size: Optional[int] = 224, + patch_size: int = 16, + in_chans: int = 3, + embed_dim: int = 768, + norm_layer=None, + flatten: bool = True, + bias: bool = True, + strict_img_size: bool = True, + dynamic_img_pad: bool = True, + dtype=None, + device=None, + operations=None, + ): + super().__init__() + self.patch_size = (patch_size, patch_size) + if img_size is not None: + self.img_size = (img_size, img_size) + self.grid_size = tuple( + [s // p for s, p in zip(self.img_size, self.patch_size)]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + else: + self.img_size = None + self.grid_size = None + self.num_patches = None + + # flatten spatial dim and transpose to channels last, kept for bwd compat + self.flatten = flatten + self.strict_img_size = strict_img_size + self.dynamic_img_pad = dynamic_img_pad + + self.proj = operations.Conv2d(in_chans, + embed_dim, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + dtype=dtype, + device=device) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x): + B, C, H, W = x.shape + # if self.img_size is not None: + # if self.strict_img_size: + # _assert(H == self.img_size[0], f"Input height ({H}) doesn't match model ({self.img_size[0]}).") + # _assert(W == self.img_size[1], f"Input width ({W}) doesn't match model ({self.img_size[1]}).") + # elif not self.dynamic_img_pad: + # _assert( + # H % self.patch_size[0] == 0, + # f"Input height ({H}) should be divisible by patch size ({self.patch_size[0]})." + # ) + # _assert( + # W % self.patch_size[1] == 0, + # f"Input width ({W}) should be divisible by patch size ({self.patch_size[1]})." + # ) + if self.dynamic_img_pad: + pad_h = (self.patch_size[0] - + H % self.patch_size[0]) % self.patch_size[0] + pad_w = (self.patch_size[1] - + W % self.patch_size[1]) % self.patch_size[1] + x = torch.nn.functional.pad(x, (0, pad_w, 0, pad_h), + mode='reflect') + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # NCHW -> NLC + x = self.norm(x) + return x + + +def modulate(x, shift, scale): + if shift is None: + shift = torch.zeros_like(scale) + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +################################################################################# +# Sine/Cosine Positional Embedding Functions # +################################################################################# + + +def get_2d_sincos_pos_embed( + embed_dim, + grid_size, + cls_token=False, + extra_tokens=0, + scaling_factor=None, + offset=None, +): + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + if scaling_factor is not None: + grid = grid / scaling_factor + if offset is not None: + grid = grid - offset + + grid = grid.reshape([2, 1, grid_size, grid_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token and extra_tokens > 0: + pos_embed = np.concatenate( + [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, + grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, + grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid_torch(embed_dim, + pos, + device=None, + dtype=torch.float32): + omega = torch.arange(embed_dim // 2, device=device, dtype=dtype) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + pos = pos.reshape(-1) # (M,) + out = torch.einsum('m,d->md', pos, omega) # (M, D/2), outer product + emb_sin = torch.sin(out) # (M, D/2) + emb_cos = torch.cos(out) # (M, D/2) + emb = torch.cat([emb_sin, emb_cos], dim=1) # (M, D) + return emb + + +def get_2d_sincos_pos_embed_torch(embed_dim, + w, + h, + val_center=7.5, + val_magnitude=7.5, + device=None, + dtype=torch.float32): + small = min(h, w) + val_h = (h / small) * val_magnitude + val_w = (w / small) * val_magnitude + grid_h, grid_w = torch.meshgrid(torch.linspace(-val_h + val_center, + val_h + val_center, + h, + device=device, + dtype=dtype), + torch.linspace(-val_w + val_center, + val_w + val_center, + w, + device=device, + dtype=dtype), + indexing='ij') + emb_h = get_1d_sincos_pos_embed_from_grid_torch(embed_dim // 2, + grid_h, + device=device, + dtype=dtype) + emb_w = get_1d_sincos_pos_embed_from_grid_torch(embed_dim // 2, + grid_w, + device=device, + dtype=dtype) + emb = torch.cat([emb_w, emb_h], dim=1) # (H*W, D) + return emb + + +################################################################################# +# Embedding Layers for Timesteps and Class Labels # +################################################################################# + + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + def __init__(self, + hidden_size, + frequency_embedding_size=256, + dtype=None, + device=None, + operations=None): + super().__init__() + self.mlp = nn.Sequential( + operations.Linear(frequency_embedding_size, + hidden_size, + bias=True, + dtype=dtype, + device=device), + nn.SiLU(), + operations.Linear(hidden_size, + hidden_size, + bias=True, + dtype=dtype, + device=device), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + :param t: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an (N, D) Tensor of positional embeddings. + """ + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * + torch.arange(start=0, end=half, dtype=torch.float32) / + half).to(device=t.device) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat( + [embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + if torch.is_floating_point(t): + embedding = embedding.to(dtype=t.dtype) + return embedding + + def forward(self, t, dtype, **kwargs): + t_freq = self.timestep_embedding( + t, self.frequency_embedding_size).to(dtype) + t_emb = self.mlp(t_freq) + return t_emb + + +class VectorEmbedder(nn.Module): + """ + Embeds a flat vector of dimension input_dim + """ + def __init__(self, + input_dim: int, + hidden_size: int, + dtype=None, + device=None, + operations=None): + super().__init__() + self.mlp = nn.Sequential( + operations.Linear(input_dim, + hidden_size, + bias=True, + dtype=dtype, + device=device), + nn.SiLU(), + operations.Linear(hidden_size, + hidden_size, + bias=True, + dtype=dtype, + device=device), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + emb = self.mlp(x) + return emb + + +################################################################################# +# Core DiT Model # +################################################################################# + + +def split_qkv(qkv, head_dim): + qkv = qkv.reshape(qkv.shape[0], qkv.shape[1], 3, -1, + head_dim).movedim(2, 0) + return qkv[0], qkv[1], qkv[2] + + +def optimized_attention(qkv, num_heads): + if XFORMERS_IS_AVAILBLE: + optimized_attention_ops = attention_xformers + else: + optimized_attention_ops = attention_pytorch + return optimized_attention_ops(qkv[0], qkv[1], qkv[2], num_heads) + + +class SelfAttention(nn.Module): + ATTENTION_MODES = ('xformers', 'torch', 'torch-hb', 'math', 'debug') + + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = False, + qk_scale: Optional[float] = None, + proj_drop: float = 0.0, + attn_mode: str = 'xformers', + pre_only: bool = False, + qk_norm: Optional[str] = None, + rmsnorm: bool = False, + dtype=None, + device=None, + operations=None, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = dim // num_heads + + self.qkv = operations.Linear(dim, + dim * 3, + bias=qkv_bias, + dtype=dtype, + device=device) + if not pre_only: + self.proj = operations.Linear(dim, dim, dtype=dtype, device=device) + self.proj_drop = nn.Dropout(proj_drop) + assert attn_mode in self.ATTENTION_MODES + self.attn_mode = attn_mode + self.pre_only = pre_only + + if qk_norm == 'rms': + self.ln_q = RMSNorm(self.head_dim, + elementwise_affine=True, + eps=1.0e-6, + dtype=dtype, + device=device) + self.ln_k = RMSNorm(self.head_dim, + elementwise_affine=True, + eps=1.0e-6, + dtype=dtype, + device=device) + elif qk_norm == 'ln': + self.ln_q = operations.LayerNorm(self.head_dim, + elementwise_affine=True, + eps=1.0e-6, + dtype=dtype, + device=device) + self.ln_k = operations.LayerNorm(self.head_dim, + elementwise_affine=True, + eps=1.0e-6, + dtype=dtype, + device=device) + elif qk_norm is None: + self.ln_q = nn.Identity() + self.ln_k = nn.Identity() + else: + raise ValueError(qk_norm) + + def pre_attention(self, x: torch.Tensor) -> torch.Tensor: + B, L, C = x.shape + qkv = self.qkv(x) + q, k, v = split_qkv(qkv, self.head_dim) + q = self.ln_q(q).reshape(q.shape[0], q.shape[1], -1) + k = self.ln_k(k).reshape(q.shape[0], q.shape[1], -1) + return (q, k, v) + + def post_attention(self, x: torch.Tensor) -> torch.Tensor: + assert not self.pre_only + x = self.proj(x) + x = self.proj_drop(x) + return x + + def forward(self, x: torch.Tensor) -> torch.Tensor: + qkv = self.pre_attention(x) + x = optimized_attention(qkv, num_heads=self.num_heads) + x = self.post_attention(x) + return x + + +class RMSNorm(torch.nn.Module): + def __init__(self, + dim: int, + elementwise_affine: bool = False, + eps: float = 1e-6, + device=None, + dtype=None): + """ + Initialize the RMSNorm normalization layer. + Args: + dim (int): The dimension of the input tensor. + eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. + Attributes: + eps (float): A small value added to the denominator for numerical stability. + weight (nn.Parameter): Learnable scaling parameter. + """ + super().__init__() + self.eps = eps + self.learnable_scale = elementwise_affine + if self.learnable_scale: + self.weight = nn.Parameter( + torch.empty(dim, device=device, dtype=dtype)) + else: + self.register_parameter('weight', None) + + def _norm(self, x): + """ + Apply the RMSNorm normalization to the input tensor. + Args: + x (torch.Tensor): The input tensor. + Returns: + torch.Tensor: The normalized tensor. + """ + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + """ + Forward pass through the RMSNorm layer. + Args: + x (torch.Tensor): The input tensor. + Returns: + torch.Tensor: The output tensor after applying RMSNorm. + """ + x = self._norm(x) + if self.learnable_scale: + return x * self.weight.to(device=x.device, dtype=x.dtype) + else: + return x + + +class SwiGLUFeedForward(nn.Module): + def __init__( + self, + dim: int, + hidden_dim: int, + multiple_of: int, + ffn_dim_multiplier: Optional[float] = None, + ): + """ + Initialize the FeedForward module. + + Args: + dim (int): Input dimension. + hidden_dim (int): Hidden dimension of the feedforward layer. + multiple_of (int): Value to ensure hidden dimension is a multiple of this value. + ffn_dim_multiplier (float, optional): Custom multiplier for hidden dimension. Defaults to None. + + Attributes: + w1 (ColumnParallelLinear): Linear transformation for the first layer. + w2 (RowParallelLinear): Linear transformation for the second layer. + w3 (ColumnParallelLinear): Linear transformation for the third layer. + + """ + super().__init__() + hidden_dim = int(2 * hidden_dim / 3) + # custom dim factor multiplier + if ffn_dim_multiplier is not None: + hidden_dim = int(ffn_dim_multiplier * hidden_dim) + hidden_dim = multiple_of * ( + (hidden_dim + multiple_of - 1) // multiple_of) + + self.w1 = nn.Linear(dim, hidden_dim, bias=False) + self.w2 = nn.Linear(hidden_dim, dim, bias=False) + self.w3 = nn.Linear(dim, hidden_dim, bias=False) + + def forward(self, x): + return self.w2(nn.functional.silu(self.w1(x)) * self.w3(x)) + + +class DismantledBlock(nn.Module): + """ + A DiT block with gated adaptive layer norm (adaLN) conditioning. + """ + + ATTENTION_MODES = ('xformers', 'torch', 'torch-hb', 'math', 'debug') + + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 4.0, + attn_mode: str = 'xformers', + qkv_bias: bool = False, + pre_only: bool = False, + rmsnorm: bool = False, + scale_mod_only: bool = False, + swiglu: bool = False, + qk_norm: Optional[str] = None, + dtype=None, + device=None, + operations=None, + **block_kwargs, + ): + super().__init__() + assert attn_mode in self.ATTENTION_MODES + if not rmsnorm: + self.norm1 = operations.LayerNorm(hidden_size, + elementwise_affine=False, + eps=1e-6, + dtype=dtype, + device=device) + else: + self.norm1 = RMSNorm(hidden_size, + elementwise_affine=False, + eps=1e-6) + self.attn = SelfAttention(dim=hidden_size, + num_heads=num_heads, + qkv_bias=qkv_bias, + attn_mode=attn_mode, + pre_only=pre_only, + qk_norm=qk_norm, + rmsnorm=rmsnorm, + dtype=dtype, + device=device, + operations=operations) + if not pre_only: + if not rmsnorm: + self.norm2 = operations.LayerNorm(hidden_size, + elementwise_affine=False, + eps=1e-6, + dtype=dtype, + device=device) + else: + self.norm2 = RMSNorm(hidden_size, + elementwise_affine=False, + eps=1e-6) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + if not pre_only: + if not swiglu: + self.mlp = Mlp(in_features=hidden_size, + hidden_features=mlp_hidden_dim, + act_layer=lambda: nn.GELU(approximate='tanh'), + drop=0, + dtype=dtype, + device=device, + operations=operations) + else: + self.mlp = SwiGLUFeedForward( + dim=hidden_size, + hidden_dim=mlp_hidden_dim, + multiple_of=256, + ) + self.scale_mod_only = scale_mod_only + if not scale_mod_only: + n_mods = 6 if not pre_only else 2 + else: + n_mods = 4 if not pre_only else 1 + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + operations.Linear(hidden_size, + n_mods * hidden_size, + bias=True, + dtype=dtype, + device=device)) + self.pre_only = pre_only + + def pre_attention(self, x: torch.Tensor, c: torch.Tensor) -> torch.Tensor: + if not self.pre_only: + if not self.scale_mod_only: + ( + shift_msa, + scale_msa, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + ) = self.adaLN_modulation(c).chunk(6, dim=1) + else: + shift_msa = None + shift_mlp = None + ( + scale_msa, + gate_msa, + scale_mlp, + gate_mlp, + ) = self.adaLN_modulation(c).chunk(4, dim=1) + qkv = self.attn.pre_attention( + modulate(self.norm1(x), shift_msa, scale_msa)) + return qkv, ( + x, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + ) + else: + if not self.scale_mod_only: + ( + shift_msa, + scale_msa, + ) = self.adaLN_modulation(c).chunk(2, dim=1) + else: + shift_msa = None + scale_msa = self.adaLN_modulation(c) + qkv = self.attn.pre_attention( + modulate(self.norm1(x), shift_msa, scale_msa)) + return qkv, None + + def post_attention(self, attn, x, gate_msa, shift_mlp, scale_mlp, + gate_mlp): + assert not self.pre_only + x = x + gate_msa.unsqueeze(1) * self.attn.post_attention(attn) + x = x + gate_mlp.unsqueeze(1) * self.mlp( + modulate(self.norm2(x), shift_mlp, scale_mlp)) + return x + + def forward(self, x: torch.Tensor, c: torch.Tensor) -> torch.Tensor: + assert not self.pre_only + qkv, intermediates = self.pre_attention(x, c) + attn = optimized_attention( + qkv, + num_heads=self.attn.num_heads, + ) + return self.post_attention(attn, *intermediates) + + +def block_mixing(*args, use_checkpoint=True, **kwargs): + if use_checkpoint: + return torch.utils.checkpoint.checkpoint(_block_mixing, + *args, + use_reentrant=False, + **kwargs) + else: + return _block_mixing(*args, **kwargs) + + +def _block_mixing(context, x, context_block, x_block, c): + context_qkv, context_intermediates = context_block.pre_attention( + context, c) + + x_qkv, x_intermediates = x_block.pre_attention(x, c) + + o = [] + for t in range(3): + o.append(torch.cat((context_qkv[t], x_qkv[t]), dim=1)) + qkv = tuple(o) + + attn = optimized_attention( + qkv, + num_heads=x_block.attn.num_heads, + ) + context_attn, x_attn = ( + attn[:, :context_qkv[0].shape[1]], + attn[:, context_qkv[0].shape[1]:], + ) + + if not context_block.pre_only: + context = context_block.post_attention(context_attn, + *context_intermediates) + + else: + context = None + x = x_block.post_attention(x_attn, *x_intermediates) + return context, x + + +class JointBlock(nn.Module): + """just a small wrapper to serve as a fsdp unit""" + def __init__( + self, + *args, + **kwargs, + ): + super().__init__() + pre_only = kwargs.pop('pre_only') + qk_norm = kwargs.pop('qk_norm', None) + self.context_block = DismantledBlock(*args, + pre_only=pre_only, + qk_norm=qk_norm, + **kwargs) + self.x_block = DismantledBlock(*args, + pre_only=False, + qk_norm=qk_norm, + **kwargs) + + def forward(self, *args, **kwargs): + return block_mixing(*args, + context_block=self.context_block, + x_block=self.x_block, + **kwargs) + + +class FinalLayer(nn.Module): + """ + The final layer of DiT. + """ + def __init__( + self, + hidden_size: int, + patch_size: int, + out_channels: int, + total_out_channels: Optional[int] = None, + dtype=None, + device=None, + operations=None, + ): + super().__init__() + self.norm_final = operations.LayerNorm(hidden_size, + elementwise_affine=False, + eps=1e-6, + dtype=dtype, + device=device) + self.linear = (operations.Linear(hidden_size, + patch_size * patch_size * + out_channels, + bias=True, + dtype=dtype, + device=device) if + (total_out_channels is None) else operations.Linear( + hidden_size, + total_out_channels, + bias=True, + dtype=dtype, + device=device)) + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + operations.Linear(hidden_size, + 2 * hidden_size, + bias=True, + dtype=dtype, + device=device)) + + def forward(self, x: torch.Tensor, c: torch.Tensor) -> torch.Tensor: + shift, scale = self.adaLN_modulation(c).chunk(2, dim=1) + x = modulate(self.norm_final(x), shift, scale) + x = self.linear(x) + return x + + +class SelfAttentionContext(nn.Module): + def __init__(self, + dim, + heads=8, + dim_head=64, + dtype=None, + device=None, + operations=None): + super().__init__() + dim_head = dim // heads + inner_dim = dim + + self.heads = heads + self.dim_head = dim_head + + self.qkv = operations.Linear(dim, + dim * 3, + bias=True, + dtype=dtype, + device=device) + + self.proj = operations.Linear(inner_dim, + dim, + dtype=dtype, + device=device) + + def forward(self, x): + qkv = self.qkv(x) + q, k, v = split_qkv(qkv, self.dim_head) + x = optimized_attention((q.reshape(q.shape[0], q.shape[1], -1), k, v), + self.heads) + return self.proj(x) + + +class ContextProcessorBlock(nn.Module): + def __init__(self, context_size, dtype=None, device=None, operations=None): + super().__init__() + self.norm1 = operations.LayerNorm(context_size, + elementwise_affine=False, + eps=1e-6, + dtype=dtype, + device=device) + self.attn = SelfAttentionContext(context_size, + dtype=dtype, + device=device, + operations=operations) + self.norm2 = operations.LayerNorm(context_size, + elementwise_affine=False, + eps=1e-6, + dtype=dtype, + device=device) + self.mlp = Mlp(in_features=context_size, + hidden_features=(context_size * 4), + act_layer=lambda: nn.GELU(approximate='tanh'), + drop=0, + dtype=dtype, + device=device, + operations=operations) + + def forward(self, x): + x += self.attn(self.norm1(x)) + x += self.mlp(self.norm2(x)) + return x + + +class ContextProcessor(nn.Module): + def __init__(self, + context_size, + num_layers, + dtype=None, + device=None, + operations=None): + super().__init__() + self.layers = torch.nn.ModuleList([ + ContextProcessorBlock(context_size, + dtype=dtype, + device=device, + operations=operations) + for i in range(num_layers) + ]) + self.norm = operations.LayerNorm(context_size, + elementwise_affine=False, + eps=1e-6, + dtype=dtype, + device=device) + + def forward(self, x): + for i, l in enumerate(self.layers): + x = l(x) + return self.norm(x) + + +@BACKBONES.register_class() +class MMDiT(BaseModel): + """ + Diffusion model with a Transformer backbone. + """ + def __init__(self, cfg, logger=None): + super().__init__(cfg, logger=logger) + self.ignore_keys = cfg.get('IGNORE_KEYS', None) + self.input_size = cfg.get('INPUT_SIZE', 32) + self.patch_size = cfg.get('PATCH_SIZE', 2) + self.in_channels = cfg.get('IN_CHANNELS', 4) + self.depth = cfg.get('DEPTH', 28) + self.mlp_ratio = cfg.get('MLP_RATIO', 4.0) + self.learn_sigma = cfg.get('LEARN_SIGMA', False) + self.adm_in_channels = cfg.get('ADM_IN_CHANNELS', None) + self.context_embedder_config = cfg.get('CONTEXT_EMBEDDER_CONFIG', None) + self.compile_core = cfg.get('COMPILE_CORE', False) + self.use_checkpoint = cfg.get('USE_CHECKPOINT', False) + self.register_length = cfg.get('REGISTER_LENGTH', 0) + self.attn_mode = cfg.get('ATTN_MODE', 'torch') + self.rmsnorm = cfg.get('RMSNORM', False) + self.scale_mod_only = cfg.get('SCALE_MOD_ONLY', False) + self.swiglu = cfg.get('SWIGLU', False) + self.out_channels = cfg.get('OUT_CHANNELS', None) + self.pos_embed_scaling_factor = cfg.get('POS_EMBED_SCALING_FACTOR', + None) + self.pos_embed_offset = cfg.get('POS_EMBED_OFFSET', None) + self.pos_embed_max_size = cfg.get('POS_EMBED_MAX_SIZE', None) + self.num_patches = cfg.get('NUM_PATCHES', None) + self.qk_norm = cfg.get('QK_NORM', None) + self.qkv_bias = cfg.get('QKV_BIAS', True) + self.context_processor_layers = cfg.get('CONTEXT_PROCESSOR_LAYERS', + None) + self.context_size = cfg.get('CONTEXT_SIZE', 4096) + self.dtype = cfg.get('DTYPE', None) + self.device = cfg.get('DEVICE', None) + self.operations = cfg.get('OPERATIONS', nn) + + default_out_channels = self.in_channels * 2 if self.learn_sigma else self.in_channels + self.out_channels = default(self.out_channels, default_out_channels) + + # hidden_size = default(hidden_size, 64 * depth) + # num_heads = default(num_heads, hidden_size // 64) + + # apply magic --> this defines a head_size of 64 + self.hidden_size = 64 * self.depth + num_heads = self.depth + + self.num_heads = num_heads + + self.x_embedder = PatchEmbed( + self.input_size, + self.patch_size, + self.in_channels, + self.hidden_size, + bias=True, + strict_img_size=self.pos_embed_max_size is None, + dtype=self.dtype, + device=self.device, + operations=self.operations) + self.t_embedder = TimestepEmbedder(self.hidden_size, + dtype=self.dtype, + device=self.device, + operations=self.operations) + + self.y_embedder = None + if self.adm_in_channels is not None: + assert isinstance(self.adm_in_channels, int) + self.y_embedder = VectorEmbedder(self.adm_in_channels, + self.hidden_size, + dtype=self.dtype, + device=self.device, + operations=self.operations) + + if self.context_processor_layers is not None: + self.context_processor = ContextProcessor( + self.context_size, + self.context_processor_layers, + dtype=self.dtype, + device=self.device, + operations=self.operations) + else: + self.context_processor = None + + self.context_embedder = nn.Identity() + if self.context_embedder_config is not None: + self.context_embedder_config = Config.get_dict( + self.context_embedder_config) + if self.context_embedder_config['target'] == 'torch.nn.Linear': + self.context_embedder = self.operations.Linear( + **self.context_embedder_config['params'], + dtype=self.dtype, + device=self.device) + + self.register_length = self.register_length + if self.register_length > 0: + self.register = nn.Parameter( + torch.randn(1, + self.register_length, + self.hidden_size, + dtype=self.dtype, + device=self.device)) + + # num_patches = self.x_embedder.num_patches + # Will use fixed sin-cos embedding: + # just use a buffer already + if self.num_patches is not None: + self.register_buffer( + 'pos_embed', + torch.empty(1, + self.num_patches, + self.hidden_size, + dtype=self.dtype, + device=self.device), + ) + else: + self.pos_embed = None + + self.use_checkpoint = self.use_checkpoint + self.joint_blocks = nn.ModuleList([ + JointBlock(self.hidden_size, + num_heads, + mlp_ratio=self.mlp_ratio, + qkv_bias=self.qkv_bias, + attn_mode=self.attn_mode, + pre_only=i == self.depth - 1, + rmsnorm=self.rmsnorm, + scale_mod_only=self.scale_mod_only, + swiglu=self.swiglu, + qk_norm=self.qk_norm, + dtype=self.dtype, + device=self.device, + operations=self.operations) for i in range(self.depth) + ]) + + self.final_layer = FinalLayer(self.hidden_size, + self.patch_size, + self.out_channels, + dtype=self.dtype, + device=self.device, + operations=self.operations) + + if self.compile_core: + assert False + self.forward_core_with_concat = torch.compile( + self.forward_core_with_concat) + + def load_pretrained_model(self, pretrained_model): + if pretrained_model: + with FS.get_from(pretrained_model, wait_finish=True) as local_path: + if local_path.endswith('safetensors'): + from safetensors.torch import load_file as load_safetensors + model = load_safetensors(local_path) + else: + model = torch.load(local_path, map_location='cpu') + if 'state_dict' in model: + model = model['state_dict'] + new_ckpt = OrderedDict() + ignore_ckpt = OrderedDict() + for k, v in model.items(): + if self.ignore_keys is not None: + if (isinstance(self.ignore_keys, str) and re.match(self.ignore_keys, k)) or \ + (isinstance(self.ignore_keys, list) and k in self.ignore_keys): + ignore_ckpt[k] = v + continue + k = k.replace('model.diffusion_model.', '') + new_ckpt[k] = v + missing, unexpected = self.load_state_dict(new_ckpt, + strict=False) + print( + f'Restored from {pretrained_model} with {len(missing)} missing and {len(unexpected)} unexpected keys' + ) + if len(missing) > 0: + print(f'Missing Keys:\n {missing}') + if len(unexpected) > 0: + print(f'\nUnexpected Keys:\n {unexpected}') + + def cropped_pos_embed(self, hw, device=None): + p = self.x_embedder.patch_size[0] + h, w = hw + # patched size + h = (h + 1) // p + w = (w + 1) // p + if self.pos_embed is None: + return get_2d_sincos_pos_embed_torch(self.hidden_size, + w, + h, + device=device) + assert self.pos_embed_max_size is not None + assert h <= self.pos_embed_max_size, (h, self.pos_embed_max_size) + assert w <= self.pos_embed_max_size, (w, self.pos_embed_max_size) + top = (self.pos_embed_max_size - h) // 2 + left = (self.pos_embed_max_size - w) // 2 + spatial_pos_embed = rearrange( + self.pos_embed, + '1 (h w) c -> 1 h w c', + h=self.pos_embed_max_size, + w=self.pos_embed_max_size, + ) + spatial_pos_embed = spatial_pos_embed[:, top:top + h, left:left + w, :] + spatial_pos_embed = rearrange(spatial_pos_embed, + '1 h w c -> 1 (h w) c') + # print(spatial_pos_embed, top, left, h, w) + # # t = get_2d_sincos_pos_embed_torch(self.hidden_size, w, h, 7.875, 7.875, device=device) #matches exactly for 1024 res + # t = get_2d_sincos_pos_embed_torch(self.hidden_size, w, h, 7.5, 7.5, device=device) #scales better + # # print(t) + # return t + return spatial_pos_embed + + def unpatchify(self, x, hw=None): + """ + x: (N, T, patch_size**2 * C) + imgs: (N, H, W, C) + """ + c = self.out_channels + p = self.x_embedder.patch_size[0] + if hw is None: + h = w = int(x.shape[1]**0.5) + else: + h, w = hw + h = (h + 1) // p + w = (w + 1) // p + assert h * w == x.shape[1] + + x = x.reshape(shape=(x.shape[0], h, w, p, p, c)) + x = torch.einsum('nhwpqc->nchpwq', x) + imgs = x.reshape(shape=(x.shape[0], c, h * p, w * p)) + return imgs + + def forward_core_with_concat( + self, + x: torch.Tensor, + c_mod: torch.Tensor, + context: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if self.register_length > 0: + context = torch.cat( + ( + repeat(self.register, '1 ... -> b ...', b=x.shape[0]), + default(context, + torch.Tensor([]).type_as(x)), + ), + 1, + ) + + # context is B, L', D + # x is B, L, D + for block in self.joint_blocks: + context, x = block( + context, + x, + c=c_mod, + use_checkpoint=self.use_checkpoint, + ) + + x = self.final_layer(x, + c_mod) # (N, T, patch_size ** 2 * out_channels) + return x + + def forward(self, + x, + t=None, + cond=dict(), + mask=None, + data_info=None, + **kwargs): + """ + Forward pass of DiT. + x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images) + t: (N,) tensor of diffusion timesteps + y: (N,) tensor of class labels + """ + + if isinstance(cond, dict): + if 'label' in cond and cond['label'] is not None: + label = cond['label'] + if 'concat' in cond: + concat = cond['concat'] + x = torch.cat([x, concat], dim=1) + context = cond.get('crossattn', None) + y = cond.get('y', None) + else: + context = cond + y = None + + if self.context_processor is not None: + context = self.context_processor(context) + + hw = x.shape[-2:] + x = self.x_embedder(x) + self.cropped_pos_embed( + hw, device=x.device).to(dtype=x.dtype, device=x.device) + c = self.t_embedder(t, dtype=x.dtype) # (N, D) + if y is not None and self.y_embedder is not None: + y = self.y_embedder(y) # (N, D) + c = c + y # (N, D) + + if context is not None: + context = self.context_embedder(context) + + x = self.forward_core_with_concat(x, c, context) + + x = self.unpatchify(x, hw=hw) # (N, out_channels, H, W) + return x[:, :, :hw[-2], :hw[-1]] + + +if __name__ == '__main__': + config_dict = { + 'NMAE': + 'MMDiT', + 'IN_CHANNELS': + 16, + 'PATCH_SIZE': + 2, + 'OUT_CHANNELS': + 16, + 'DEPTH': + 24, + 'INPUT_SIZE': + None, + 'ADM_IN_CHANNELS': + 2048, + 'CONTEXT_EMBEDDER_CONFIG': { + 'target': 'torch.nn.Linear', + 'params': { + 'in_features': 4096, + 'out_features': 1536 + } + }, + 'NUM_PATCHES': + 36864, + 'POS_EMBED_MAX_SIZE': + 192, + 'POS_EMBED_SCALING_FACTOR': + None, + 'DTYPE': + torch.float16, + 'IGNORE_KEYS': + '^first_stage_model', + 'PRETRAINED_MODEL': + '/mnt/data/huggingface_repo/stabilityai/stable-diffusion-3-medium/sd3_medium.safetensors' + } + config = Config(load=False, cfg_dict=config_dict) + model = MMDiT(config) + model.load_pretrained_model(config_dict['PRETRAINED_MODEL']) + # print('=====') diff --git a/scepter/modules/model/backbone/pixart/__init__.py b/scepter/modules/model/backbone/pixart/__init__.py new file mode 100644 index 0000000..219ebb6 --- /dev/null +++ b/scepter/modules/model/backbone/pixart/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from .pixart_alpha import PixArt diff --git a/scepter/modules/model/backbone/pixart/pixart_alpha.py b/scepter/modules/model/backbone/pixart/pixart_alpha.py new file mode 100644 index 0000000..cbae899 --- /dev/null +++ b/scepter/modules/model/backbone/pixart/pixart_alpha.py @@ -0,0 +1,503 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +# All rights reserved. +# This file contains code that is adapted from +# timm: https://github.com/huggingface/pytorch-image-models +# pixart: https://github.com/PixArt-alpha/PixArt-alpha + +# This source code is also licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# -------------------------------------------------------- + +from collections import OrderedDict +# MAE: https://github.com/facebookresearch/mae/blob/main/models_mae.py +# -------------------------------------------------------- +from typing import Iterable + +import torch +import torch.nn as nn +# References: +# GLIDE: https://github.com/openai/glide-text2im +from torch.utils.checkpoint import checkpoint, checkpoint_sequential + +from scepter.modules.model.backbone.transformer.attention import \ + MultiHeadAttention +from scepter.modules.model.backbone.transformer.layers import ( + CaptionEmbedder, DropPath, LabelEmbedder, Mlp, SizeEmbedder, + TimestepEmbedder, modulate) +from scepter.modules.model.backbone.transformer.patchify import (PatchEmbed, + unpatchify) +from scepter.modules.model.backbone.transformer.pos_embed import \ + get_2d_sincos_pos_embed +from scepter.modules.model.base_model import BaseModel +from scepter.modules.model.registry import BACKBONES +from scepter.modules.utils.config import dict_to_yaml +from scepter.modules.utils.file_system import FS + + +def auto_grad_checkpoint(module, *args, use_grad_checkpoint=False, **kwargs): + if use_grad_checkpoint: + if not isinstance(module, Iterable): + return checkpoint(module, *args, use_reentrant=False, **kwargs) + gc_step = module[0].grad_checkpointing_step + return checkpoint_sequential(module, + gc_step, + *args, + use_reentrant=False, + **kwargs) + return module(*args, **kwargs) + + +class FinalLayer(nn.Module): + """ + The final layer of PixArt. + """ + def __init__(self, hidden_size, patch_size, out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, + elementwise_affine=False, + eps=1e-6) + self.linear = nn.Linear(hidden_size, + patch_size * patch_size * out_channels, + bias=True) + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)) + + def forward(self, x, c): + shift, scale = self.adaLN_modulation(c).chunk(2, dim=1) + # modulate x + x = modulate(self.norm_final(x), shift, scale, unsqueeze=True) + x = self.linear(x) + return x + + +class T2IFinalLayer(nn.Module): + """ + The final layer of PixArt. + """ + def __init__(self, hidden_size, patch_size, out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, + elementwise_affine=False, + eps=1e-6) + self.linear = nn.Linear(hidden_size, + patch_size * patch_size * out_channels, + bias=True) + self.scale_shift_table = nn.Parameter( + torch.randn(2, hidden_size) / hidden_size**0.5) + self.out_channels = out_channels + + def forward(self, x, t): + shift, scale = (self.scale_shift_table[None] + t[:, None]).chunk(2, + dim=1) + x = modulate(self.norm_final(x), shift, scale) + x = self.linear(x) + return x + + +class DitFinalLayer(nn.Module): + """ + The final layer of PixArt. + """ + def __init__(self, hidden_size, patch_size, out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, + elementwise_affine=False, + eps=1e-6) + self.linear = nn.Linear(hidden_size, + patch_size * patch_size * out_channels, + bias=True) + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)) + self.out_channels = out_channels + + def forward(self, x, t): + shift, scale = self.adaLN_modulation(t).chunk(2, dim=1) + x = modulate(self.norm_final(x), shift, scale, unsqueeze=True) + x = self.linear(x) + return x + + +class PixArtBlock(nn.Module): + """ + A PixArt block with adaptive layer norm zero (adaLN-Zero) conditioning. + """ + def __init__(self, + hidden_size, + num_heads, + mlp_ratio=4.0, + drop_path=0., + window_size=0, + use_rel_pos=False, + backend=None, + use_condition=True, + **block_kwargs): + super().__init__() + self.hidden_size = hidden_size + self.use_condition = use_condition + self.norm1 = nn.LayerNorm(hidden_size, + elementwise_affine=False, + eps=1e-6) + self.attn = MultiHeadAttention(hidden_size, + num_heads=num_heads, + qkv_bias=True, + backend=backend, + **block_kwargs) + if self.use_condition: + self.cross_attn = MultiHeadAttention(hidden_size, + context_dim=hidden_size, + num_heads=num_heads, + qkv_bias=True, + backend=backend, + **block_kwargs) + self.norm2 = nn.LayerNorm(hidden_size, + elementwise_affine=False, + eps=1e-6) + # to be compatible with lower version pytorch + approx_gelu = lambda: nn.GELU(approximate='tanh') + self.mlp = Mlp(in_features=hidden_size, + hidden_features=int(hidden_size * mlp_ratio), + act_layer=approx_gelu, + drop=0) + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + self.window_size = window_size + self.scale_shift_table = nn.Parameter( + torch.randn(6, hidden_size) / hidden_size**0.5) + + def forward(self, x, y, t, mask=None, **kwargs): + B, N, C = x.shape + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + self.scale_shift_table[None] + t.reshape(B, 6, -1)).chunk(6, dim=1) + x = x + self.drop_path(gate_msa * self.attn( + modulate(self.norm1(x), shift_msa, scale_msa, unsqueeze=False))) + if self.use_condition: + x = x + self.cross_attn(x, y, mask) + x = x + self.drop_path(gate_mlp * self.mlp( + modulate(self.norm2(x), shift_mlp, scale_mlp, unsqueeze=False))) + return x + + +@BACKBONES.register_class() +class PixArt(BaseModel): + """ + Diffusion model with a Transformer backbone. + """ + para_dict = BaseModel.para_dict + para_dict.update({ + 'PATCH_SIZE': { + 'value': 2, + 'description': '' + }, + 'IN_CHANNELS': { + 'value': 4, + 'description': '' + }, + 'HIDDEN_SIZE': { + 'value': 1152, + 'description': '' + }, + 'DEPTH': { + 'value': 28, + 'description': '' + }, + 'NUM_HEADS': { + 'value': 16, + 'description': '' + }, + 'MLP_RATIO': { + 'value': 4.0, + 'description': '' + }, + 'CLASS_DROPOUT_PROB': { + 'value': 0.1, + 'description': '' + }, + 'PRED_SIGMA': { + 'value': True, + 'description': '' + }, + 'DROP_PATH': { + 'value': 0., + 'description': '' + }, + 'WINDOW_DIZE': { + 'value': 0, + 'description': '' + }, + 'WINDOW_BLOCK_INDEXES': { + 'value': None, + 'description': '' + }, + 'USE_REL_POS': { + 'value': False, + 'description': '' + }, + 'CAPTION_CHANNELS': { + 'value': 4096, + 'description': '' + }, + 'USE_AR_SIZE': { + 'value': True, + 'description': '' + }, + 'DIT_FINAL_LAYER': { + 'value': False, + 'description': '' + }, + 'LEWEI_SCALE': { + 'value': 1.0, + 'description': '' + }, + 'MODEL_MAX_LENGTH': { + 'value': 120, + 'description': '' + }, + 'NUM_CLASSES': { + 'value': + None, + 'description': + 'The class num for class guided setting, also can be set as continuous.' + }, + 'ATTENTION_BACKEND': { + 'value': None, + 'description': '' + } + }) + + def __init__(self, cfg, logger): + super().__init__(cfg, logger=logger) + self.window_block_indexes = cfg.get('WINDOW_BLOCK_INDEXES', None) + if self.window_block_indexes is None: + self.window_block_indexes = [] + self.pred_sigma = cfg.get('PRED_SIGMA', True) + self.in_channels = cfg.get('IN_CHANNELS', 4) + self.out_channels = self.in_channels * 2 if self.pred_sigma else self.in_channels + self.patch_size = cfg.get('PATCH_SIZE', 2) + self.num_heads = cfg.get('NUM_HEADS', 16) + self.hidden_size = cfg.get('HIDDEN_SIZE', 1152) + self.lewei_scale = cfg.get('LEWEI_SCALE', 1.0), + self.caption_channels = cfg.get('CAPTION_CHANNELS', 4096) + self.class_dropout_prob = cfg.get('CLASS_DROPOUT_PROB', 0.1) + self.model_max_length = cfg.get('MODEL_MAX_LENGTH', 120) + self.drop_path = cfg.get('DROP_PATH', 0.) + self.depth = cfg.get('DEPTH', 28) + self.mlp_ratio = cfg.get('MLP_RATIO', 4.0) + self.num_classes = cfg.get('NUM_CLASSES', None) + self.use_grad_checkpoint = cfg.get('USE_GRAD_CHECKPOINT', False) + self.use_ar_size = cfg.get('USE_AR_SIZE', True) + self.use_dit_final_layer = cfg.get('DIT_FINAL_LAYER', False) + self.attention_backend = cfg.get('ATTENTION_BACKEND', None) + self.ignore_keys = cfg.get('IGNORE_KEYS', []) + + if self.num_classes is not None: + if isinstance(self.num_classes, int): + self.label_embedder = LabelEmbedder( + self.num_classes, + self.hidden_size, + dropout_prob=self.class_dropout_prob) + elif self.num_classes == 'continuous': + print('setting up linear c_adm embedding layer') + self.label_embedder = nn.Linear(1, self.hidden_size) + else: + raise ValueError() + + self.x_embedder = PatchEmbed(self.patch_size, + self.in_channels, + self.hidden_size, + bias=True) + self.t_embedder = TimestepEmbedder(self.hidden_size) + + # self.base_size = self.input_size // self.patch_size + approx_gelu = lambda: nn.GELU(approximate='tanh') + self.t_block = nn.Sequential( + nn.SiLU(), + nn.Linear(self.hidden_size, 6 * self.hidden_size, bias=True)) + if self.num_classes is None: + self.y_embedder = CaptionEmbedder( + in_channels=self.caption_channels, + hidden_size=self.hidden_size, + uncond_prob=self.class_dropout_prob, + act_layer=approx_gelu, + token_num=self.model_max_length) + if self.use_ar_size: + self.csize_embedder = SizeEmbedder(self.hidden_size // + 3) # c_size embed + self.ar_embedder = SizeEmbedder(self.hidden_size // + 3) # aspect ratio embed + + drop_path = [ + x.item() for x in torch.linspace(0, self.drop_path, self.depth) + ] # stochastic depth decay rule + self.blocks = nn.ModuleList([ + PixArtBlock(self.hidden_size, + self.num_heads, + mlp_ratio=self.mlp_ratio, + drop_path=drop_path[i], + window_size=self.window_size + if i in self.window_block_indexes else 0, + use_rel_pos=self.use_rel_pos + if i in self.window_block_indexes else False, + backend=self.attention_backend, + use_condition=self.num_classes is None) + for i in range(self.depth) + ]) + if self.use_dit_final_layer: + self.final_layer = DitFinalLayer(self.hidden_size, self.patch_size, + self.out_channels) + else: + self.final_layer = T2IFinalLayer(self.hidden_size, self.patch_size, + self.out_channels) + + self.initialize_weights() + + def load_pretrained_model(self, pretrained_model): + if pretrained_model: + with FS.get_from(pretrained_model, wait_finish=True) as local_path: + model = torch.load(local_path, map_location='cpu') + if 'state_dict' in model: + model = model['state_dict'] + new_ckpt = OrderedDict() + for k, v in model.items(): + k = k.replace('.cross_attn.q_linear.', '.cross_attn.q.') + k = k.replace('.cross_attn.proj.', + '.cross_attn.o.').replace( + '.attn.proj.', '.attn.o.') + if '.cross_attn.kv_linear.' in k: + k_p, v_p = torch.split(v, v.shape[0] // 2) + new_ckpt[k.replace('.cross_attn.kv_linear.', + '.cross_attn.k.')] = k_p + new_ckpt[k.replace('.cross_attn.kv_linear.', + '.cross_attn.v.')] = v_p + elif '.attn.qkv.' in k: + q_p, k_p, v_p = torch.split(v, v.shape[0] // 3) + new_ckpt[k.replace('.attn.qkv.', '.attn.q.')] = q_p + new_ckpt[k.replace('.attn.qkv.', '.attn.k.')] = k_p + new_ckpt[k.replace('.attn.qkv.', '.attn.v.')] = v_p + else: + new_ckpt[k] = v + missing, unexpected = self.load_state_dict(new_ckpt, + strict=False) + print( + f'Restored from {pretrained_model} with {len(missing)} missing and {len(unexpected)} unexpected keys' + ) + if len(missing) > 0: + print(f'Missing Keys:\n {missing}') + if len(unexpected) > 0: + print(f'\nUnexpected Keys:\n {unexpected}') + + def forward(self, + x, + t=None, + cond=dict(), + mask=None, + data_info=None, + **kwargs): + """ + Forward pass of PixArt. + x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images) + t: (N,) tensor of diffusion timesteps + y: (N, 1, 120, C) tensor of class labels + """ + label = None + if isinstance(cond, dict): + if 'label' in cond and cond['label'] is not None: + label = cond['label'] + if 'concat' in cond: + concat = cond['concat'] + x = torch.cat([x, concat], dim=1) + context = cond.get('crossattn', None) + else: + context = cond + + y = context + h, w = x.shape[-2] // self.patch_size, x.shape[-1] // self.patch_size + + x = self.x_embedder(x) # (N, T, D), where T = H * W / patch_size ** 2 + pos_embed = torch.from_numpy( + get_2d_sincos_pos_embed(self.hidden_size, (h, w), + lewei_scale=self.lewei_scale, + base_h_size=h, + base_w_size=w)).unsqueeze(0).float().to( + x.device) + x = x + pos_embed + + t = self.t_embedder(t) # (N, D) + if self.num_classes is not None and label is not None: + t = t + self.label_embedder(label, self.training) + if self.use_ar_size and data_info is not None: + bs = x.shape[0] + c_size, ar = data_info['img_hw'], data_info['aspect_ratio'] + csize = self.csize_embedder(c_size, bs) # (N, D) + ar = self.ar_embedder(ar, bs) # (N, D) + t = t + torch.cat([csize, ar], dim=1) + t0 = self.t_block(t) + if self.num_classes is not None: + y = None + else: + y = self.y_embedder(y, self.training) + for block in self.blocks: + x = auto_grad_checkpoint( + block, + x, + y, + t0, + mask, + use_grad_checkpoint=self.use_grad_checkpoint) + # (N, T, D) #support grad checkpoint + x = self.final_layer(x, t) # (N, T, patch_size ** 2 * out_channels) + x = unpatchify(x, h, w, self.out_channels, self.patch_size, + self.patch_size) # (N, out_channels, H, W) + if self.pred_sigma: + return x.chunk(2, dim=1)[0] + else: + return x + + def initialize_weights(self): + # Initialize transformer layers: + def _basic_init(module): + if isinstance(module, nn.Linear): + torch.nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + + self.apply(_basic_init) + + # Initialize patch_embed like nn.Linear (instead of nn.Conv2d): + w = self.x_embedder.proj.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + # Initialize timestep embedding MLP: + nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02) + nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02) + nn.init.normal_(self.t_block[1].weight, std=0.02) + if self.use_ar_size: + nn.init.normal_(self.csize_embedder.mlp[0].weight, std=0.02) + nn.init.normal_(self.csize_embedder.mlp[2].weight, std=0.02) + nn.init.normal_(self.ar_embedder.mlp[0].weight, std=0.02) + nn.init.normal_(self.ar_embedder.mlp[2].weight, std=0.02) + if self.num_classes is not None: + nn.init.normal_(self.label_embedder.embedding_table.weight, + std=0.02) + # Initialize caption embedding MLP: + if hasattr(self, 'y_embedder'): + nn.init.normal_(self.y_embedder.y_proj.fc1.weight, std=0.02) + nn.init.normal_(self.y_embedder.y_proj.fc2.weight, std=0.02) + # Zero-out adaLN modulation layers in PixArt blocks: + if self.num_classes is None: + for block in self.blocks: + nn.init.constant_(block.cross_attn.o.weight, 0) + nn.init.constant_(block.cross_attn.o.bias, 0) + # Zero-out output layers: + nn.init.constant_(self.final_layer.linear.weight, 0) + nn.init.constant_(self.final_layer.linear.bias, 0) + + @property + def dtype(self): + return next(self.parameters()).dtype + + @staticmethod + def get_config_template(): + return dict_to_yaml('BACKBONE', + __class__.__name__, + PixArt.para_dict, + set_name=True) diff --git a/scepter/modules/model/backbone/transformer/__init__.py b/scepter/modules/model/backbone/transformer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scepter/modules/model/backbone/transformer/attention.py b/scepter/modules/model/backbone/transformer/attention.py new file mode 100644 index 0000000..aca1353 --- /dev/null +++ b/scepter/modules/model/backbone/transformer/attention.py @@ -0,0 +1,760 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +# All rights reserved. +# This file contains code that is adapted from +# timm: https://github.com/huggingface/pytorch-image-models +# pixart: https://github.com/PixArt-alpha/PixArt-alpha +import math +import time +import warnings + +import torch +import torch.nn as nn +from torch.cuda import amp +from torch.nn import functional as F +from torch.nn.utils.rnn import pad_sequence +from tqdm import tqdm + +from scepter.modules.model.backbone.transformer.pos_embed import apply_2d_rope + +try: + import xformers + import xformers.ops + XFORMERS_IS_AVAILABLE = True +except Exception as e: + XFORMERS_IS_AVAILABLE = False + warnings.warn(f'{e}') +try: + from flash_attn import (flash_attn_varlen_func) + FLASHATTN_IS_AVAILABLE = True +except ImportError: + FLASHATTN_IS_AVAILABLE = False + flash_attn_varlen_func = None + + +def drop_path(x, drop_prob: float = 0., training: bool = False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class MultiHeadAttention(nn.Module): + def __init__(self, + dim, + context_dim=None, + num_heads=None, + head_dim=None, + attn_drop=0.0, + qkv_bias=False, + dropout=0.0, + backend=None, + **block_kwargs): + super().__init__() + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + context_dim = context_dim or dim + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + # layers + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.k = nn.Linear(context_dim, dim, bias=qkv_bias) + self.v = nn.Linear(context_dim, dim, bias=qkv_bias) + self.o = nn.Linear(dim, dim) + self.dropout = nn.Dropout(dropout) + self.attention_op = None + self.attn_drop = nn.Dropout(attn_drop) + self.backend = backend + assert self.backend in ('flash_attn', 'xformer_attn', 'pytorch_attn', + None) + if FLASHATTN_IS_AVAILABLE and self.backend in ('flash_attn', None): + self.backend = 'flash_attn' + self.softmax_scale = block_kwargs.get('softmax_scale', None) + self.causal = block_kwargs.get('causal', False) + self.window_size = block_kwargs.get('window_size', (-1, -1)) + self.deterministic = block_kwargs.get('deterministic', False) + elif XFORMERS_IS_AVAILABLE and self.backend in ('xformer_attn', None): + self.backend = 'xformer_attn' + else: + self.backend = 'pytorch_attn' + + def xformer_attn(self, x, context=None, mask=None, **kwargs): + context = x if context is None else context + b, n, d = x.size(0), self.num_heads, self.head_dim + # compute query, key, value + q = self.q(x).view(b, -1, n, d) + k = self.k(context).view(b, -1, n, d) + v = self.v(context).view(b, -1, n, d) + + attn_bias = None + if mask is not None: + assert mask.ndim in [2, 3] + mask = mask.view(b, 1, 1, + -1) if mask.ndim == 2 else mask.unsqueeze(1) + # To use an `attn_bias` with a sequence length that is not a multiple of 8, + # you need to ensure memory is aligned by slicing a bigger tensor. + # Example: use `attn_bias = torch.zeros([1, 1, 5, 8])[:,:,:,:5]` + # instead of `torch.zeros([1, 1, 5, 5]) + q_size = math.ceil(q.size(1) / 8) * 8 + k_size = math.ceil(k.size(1) / 8) * 8 + attn_bias = x.new_zeros(b, n, q_size, + k_size)[:, :, :q.size(1), :k.size(1)] + attn_bias = attn_bias.masked_fill_(mask == 0, + torch.finfo(x.dtype).min).to( + q.dtype) + x = xformers.ops.memory_efficient_attention(q, + k, + v, + p=self.attn_drop.p, + attn_bias=attn_bias) + x = x.reshape(b, -1, n * d) + x = self.o(x) + x = self.dropout(x) + return x + + def flash_attn(self, x, context=None, mask=None, **kwargs): + ''' + The implementation will be very slow when mask is not None, + because we need rearange the x/context features according to mask. + Args: + x: + context: + mask: + **kwargs: + Returns: x + ''' + context = x if context is None else context + dtype = kwargs.get('dtype', torch.float16) + q_lens = kwargs.get('q_lens', None) + + # if mask is not None or q_lens is not None: + # warnings.warn("Detected mask or q_lens is not None, " + # "which will be very slow because of the x/context features' rearrangement," + # "please use FlashMultiHeadAttention instead.") + def half(x): + return x if x.dtype in [torch.float16, torch.bfloat16 + ] else x.to(dtype) + + b, n, d = x.size(0), self.num_heads, self.head_dim + + q = self.q(x).view(b, -1, n, d) # [B, Lq, Nq, C1]. + k = self.k(context).view(b, -1, n, d) # [B, Lk, Nk, C1] + v = self.v(context).view( + b, -1, n, d) # [B, Lk, Nk, C2] Nq must be divisible by Nk. + + assert q.device.type == 'cuda' and q.size(-1) <= 256 + lq, lk, out_dtype = int(q.size(1)), int(k.size(1)), q.dtype + # preprocess query + if q_lens is None: + q_lens = torch.tensor([lq] * b, + dtype=torch.int32).to(q.device, + non_blocking=True) + # q_lens = (q.flatten(2, ).bool() + 1).sum(dim=-1).bool().sum(dim=-1) + q = half(q.flatten(0, 1)) + else: + q = half(torch.cat([q_v[:q_l] for q_v, q_l in zip(q, q_lens)])) + + # preprocess key, value + if mask is None: + k_lens = torch.tensor([lk] * b, + dtype=torch.int32).to(k.device, + non_blocking=True) + # k_lens = (k.flatten(2, ).bool() + 1).sum(dim=-1).bool().sum(dim=-1) + k = half(k.flatten(0, 1)) + v = half(v.flatten(0, 1)) + else: + assert mask.ndim in [1, 2, 3] + k_lens = mask if mask.ndim == 1 else mask.flatten(start_dim=1).sum( + dim=-1) + k = half(torch.cat([k_v[:k_l] for k_v, k_l in zip(k, k_lens)])) + v = half(torch.cat([v_v[:v_l] for v_v, v_l in zip(v, k_lens)])) + + x = flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q=torch.cat([q_lens.new_zeros([1]), + q_lens]).cumsum(0, dtype=torch.int32), + cu_seqlens_k=torch.cat([k_lens.new_zeros([1]), + k_lens]).cumsum(0, dtype=torch.int32), + max_seqlen_q=int(torch.max(q_lens).cpu().numpy()), + max_seqlen_k=int(torch.max(k_lens).cpu().numpy()), + dropout_p=self.attn_drop.p, + softmax_scale=self.softmax_scale, + causal=self.causal, + window_size=self.window_size, # -1 means infinite context window + deterministic=self.deterministic).unflatten(0, (b, lq)) + x = x.type(out_dtype) + x = x.flatten(2) + # output + x = self.o(x) + x = self.dropout(x) + return x + + def pytorch_attn(self, x, context=None, mask=None, **kwargs): + """x: [B, L, C]. + context: [B, L', C'] or None. + """ + context = x if context is None else context + b, n, d = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.q(x).view(b, -1, n, d) + k = self.k(context).view(b, -1, n, d) + v = self.v(context).view(b, -1, n, d) + # attention bias + attn_bias = x.new_zeros(b, n, q.size(1), k.size(1)) + if mask is not None: + assert mask.ndim in [2, 3] + mask = mask.view(b, 1, 1, + -1) if mask.ndim == 2 else mask.unsqueeze(1) + attn_bias = attn_bias.masked_fill_(mask == 0, + torch.finfo(x.dtype).min).to( + q.dtype) + + # compute attention (T5 does not use scaling) + attn = torch.einsum('binc,bjnc->bnij', q * self.scale, + k * self.scale) + attn_bias + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + x = torch.einsum('bnij,bjnc->binc', attn, v.float()) + # output + x = x.reshape(b, -1, n * d) + x = self.o(x) + x = self.dropout(x) + return x + + def forward(self, x, context=None, mask=None, **kwargs): + """x: [B, L, C]. + context: [B, L', C'] or None. + """ + x = getattr(self, self.backend)(x, + context=context, + mask=mask, + **kwargs) + return x + + +def flash_preprocess(x, context=None, q_mask=None, mask=None): + context = x if context is None else context + b, x_l, x_hidden_size = x.shape + x = x.flatten(0, 1) + if q_mask is None: + q_lens = torch.tensor([x_l] * b, + dtype=torch.int32).to(x.device, + non_blocking=True) + else: + assert q_mask.ndim in [1, 2, 3] + q_lens = q_mask if q_mask.ndim == 1 else q_mask.flatten( + start_dim=1).sum(dim=-1) + + mask_b, mask_l, mask_hidden_size = context.shape + + if mask is None: + mask_lens = torch.tensor([mask_l] * mask_b, + dtype=torch.int32).to(context.device, + non_blocking=True) + else: + assert mask.ndim in [1, 2, 3] + mask_lens = mask if mask.ndim == 1 else mask.flatten(start_dim=1).sum( + dim=-1) + + return_data = { + 'x': + x, + 'context': + torch.cat([u[:v] for u, v in zip(context, mask_lens)]), + 'cu_seqlens_q': + torch.cat([q_lens.new_zeros([1]), q_lens]).cumsum(0, + dtype=torch.int32), + 'max_seqlen_q': + int(torch.max(q_lens).cpu().numpy()), + 'cu_seqlens_k': + torch.cat([mask_lens.new_zeros([1]), + mask_lens]).cumsum(0, dtype=torch.int32), + 'max_seqlen_k': + int(torch.max(mask_lens).cpu().numpy()) + } + return return_data + + +class FlashMultiHeadAttention(nn.Module): + def __init__(self, + dim, + context_dim=None, + num_heads=None, + head_dim=None, + attn_drop=0.0, + qkv_bias=False, + dropout=0.0, + softmax_scale=None, + causal=False, + window_size=(-1, -1), + deterministic=False, + **block_kwargs): + super().__init__() + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + context_dim = context_dim or dim + self.dim = dim + self.context_dim = context_dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + # layers + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.k = nn.Linear(context_dim, dim, bias=qkv_bias) + self.v = nn.Linear(context_dim, dim, bias=qkv_bias) + self.o = nn.Linear(dim, dim) + self.dropout = nn.Dropout(dropout) + self.attention_op = None + self.attn_drop = nn.Dropout(attn_drop) + self.softmax_scale = softmax_scale + self.causal = causal + self.window_size = window_size + self.deterministic = deterministic + + def forward(self, + x, + context=None, + cu_seqlens_q=None, + max_seqlen_q=None, + cu_seqlens_k=None, + max_seqlen_k=None, + dtype=torch.float16, + **kwargs): + ''' + The implementation used the rearanaged x/context according to q_lens or k_lens. + Args: + x: [batch_size * max_seq_len or sum(q_lens) , heads, hidden_size]. + context: [batch_size * max_seq_len or sum(q_lens) , heads, hidden_size]. + cu_seqlens_q: cumsum of seq_q to index the postion of query in the batch. + max_seqlen_q: max length of query. + cu_seqlens_k: cumsum of seq_k to index the postion of key/value in the batch. + max_seqlen_k: max length of key/value. + dtype: the dtype for attention. + **kwargs: + Returns: x + ''' + context = x if context is None else context + + def half(x): + return x if x.dtype in [torch.float16, torch.bfloat16 + ] else x.to(dtype) + + n, d, out_dtype = self.num_heads, self.head_dim, x.dtype + q = self.q(x).view(-1, n, d) # [B * Lq, Nq, C1]. + k = self.k(context).view(-1, n, d) # [B * Lk, Nk, C1] + v = self.v(context).view( + -1, n, d) # [B * Lk, Nk, C2] Nq must be divisible by Nk. + q, k, v = half(q), half(k), half(v) + assert q.device.type == 'cuda' and d <= 256 + x = flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + dropout_p=self.attn_drop.p, + softmax_scale=self.softmax_scale, + causal=self.causal, + window_size=self.window_size, # -1 means infinite context window + deterministic=self.deterministic).unflatten(0, (x.shape[0], )) + x = x.flatten(1).type(out_dtype) + # output + x = self.o(x) + x = self.dropout(x) + return x + + +def multi_head_varlen_attention(q_img, + k_img, + v_img, + q_txt, + k_txt, + v_txt, + n, + d, + img_lens, + txt_lens, + dropout_p=0.0, + flash_dtype=torch.bfloat16): + ''' + q/k/v: b, s, n*d + q_lens/k_lens: b, + ''' + from flash_attn import flash_attn_varlen_func + q_lens = k_lens = img_lens + txt_lens + + cu_seqlens_q = torch.cat([q_lens.new_zeros([1]), + q_lens]).cumsum(0, dtype=torch.int32) + cu_seqlens_k = torch.cat([k_lens.new_zeros([1]), + k_lens]).cumsum(0, dtype=torch.int32) + max_seqlen_q = q_lens.max() + max_seqlen_k = k_lens.max() + + # concat img & txt for joint attention + q = torch.cat([ + torch.cat([i[:i_len], t[:t_len]], dim=0) + for i, i_len, t, t_len in zip(q_img, img_lens, q_txt, txt_lens) + ], + dim=0).view(-1, n, d) + + k = torch.cat([ + torch.cat([i[:i_len], t[:t_len]], dim=0) + for i, i_len, t, t_len in zip(k_img, img_lens, k_txt, txt_lens) + ], + dim=0).view(-1, n, d) + + v = torch.cat([ + torch.cat([i[:i_len], t[:t_len]], dim=0) + for i, i_len, t, t_len in zip(v_img, img_lens, v_txt, txt_lens) + ], + dim=0).view(-1, n, d) + + # attention + dtype = q.dtype + if dtype != flash_dtype: + q = q.type(flash_dtype) + k = k.type(flash_dtype) + v = v.type(flash_dtype) + + with amp.autocast(): + x = flash_attn_varlen_func(q=q, + k=k, + v=v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + dropout_p=dropout_p).type(dtype) + + return x, cu_seqlens_q + + +class RMSNorm(nn.Module): + def __init__(self, dim, eps=1e-6): + super().__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x): + return self._norm(x.float()).type_as(x) * self.weight + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) + + +class FullAttention(nn.Module): + def __init__(self, + dim, + num_heads=None, + head_dim=None, + dropout=0.0, + qkv_bias=False, + qk_norm=False, + eps=1e-6, + flash_dtype=torch.bfloat16): + + # consider head_dim first, then num_heads + num_heads = dim // head_dim if head_dim else num_heads + head_dim = dim // num_heads + assert num_heads * head_dim == dim + assert flash_dtype in (None, torch.float16, torch.bfloat16) + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = math.pow(head_dim, -0.25) + self.flash_dtype = flash_dtype + # layers + self.qkv_W = nn.Linear(dim, dim * 3, bias=qkv_bias) + + self.out_proj = nn.Linear(dim, dim) + self.dropout = nn.Dropout(dropout) + + if qk_norm: + from apex.normalization import FusedRMSNorm + self.q_img_norm = FusedRMSNorm(head_dim, eps=eps) + self.k_img_norm = FusedRMSNorm(head_dim, eps=eps) + self.q_txt_norm = FusedRMSNorm(head_dim, eps=eps) + self.k_txt_norm = FusedRMSNorm(head_dim, eps=eps) + else: + self.q_img_norm = nn.Identity() + self.k_img_norm = nn.Identity() + self.q_txt_norm = nn.Identity() + self.k_txt_norm = nn.Identity() + + def forward(self, + img, + txt, + img_lens=None, + txt_lens=None, + padded_pos_index=None): + ''' + img: B, L, C + txt: B, L', C + ''' + b, img_len, c = img.shape + txt_len, n, d = txt.shape[1], self.num_heads, self.head_dim + + # compute query, key, value + img_txt = torch.cat([img, txt], dim=1) + img_tokens, txt_tokens = torch.split(self.qkv_W(img_txt), + [img_len, txt_len], + dim=1) + + q_img, k_img, v_img = img_tokens.chunk(3, dim=-1) + q_txt, k_txt, v_txt = txt_tokens.chunk(3, dim=-1) + + # multi-head qk norm + q_img, k_img = q_img.view(b, -1, n, d), k_img.view(b, -1, n, d) + q_txt, k_txt = q_txt.view(b, -1, n, d), k_txt.view(b, -1, n, d) + q_img, q_txt = self.q_img_norm(q_img).view( + b, -1, n * d), self.q_txt_norm(q_txt).view(b, -1, n * d) + k_img, k_txt = self.k_img_norm(k_img).view( + b, -1, n * d), self.k_txt_norm(k_txt).view(b, -1, n * d) + + ### add position + q_img, k_img = apply_2d_rope(q_img, k_img, padded_pos_index, n, d) + + # support varying length + if img_lens is None: + img_lens = torch.tensor([img.size(1)] * b, + dtype=torch.int32, + device=img.device) + if txt_lens is None: + txt_lens = torch.tensor([txt.size(1)] * b, + dtype=torch.int32, + device=txt.device) + + # attention + x, cu_seqlens_q = multi_head_varlen_attention( + q_img, + k_img, + v_img, + q_txt, + k_txt, + v_txt, + n, + d, + img_lens, + txt_lens, + dropout_p=self.dropout.p if self.training else 0.0, + flash_dtype=self.flash_dtype) + + # output proj. + x = x.reshape(-1, n * d) + x = self.out_proj(x) + x = self.dropout(x) + + # split img & txt and padding to max_len + img = pad_sequence(tuple([ + x[s:s + img_len] for s, e, img_len in zip( + cu_seqlens_q[:-1], cu_seqlens_q[1:], img_lens) + ]), + batch_first=True) + txt = pad_sequence(tuple([ + x[s + img_len:e] for s, e, img_len in zip( + cu_seqlens_q[:-1], cu_seqlens_q[1:], img_lens) + ]), + batch_first=True) + + return img, txt + + +class FFNSwiGLU(nn.Module): + def __init__(self, in_features, hidden_features): + super().__init__() + self.W1 = nn.Linear(in_features, hidden_features, bias=False) + self.W2 = nn.Linear(in_features, hidden_features, bias=False) + self.W3 = nn.Linear(hidden_features, in_features, bias=False) + self.silu = nn.SiLU() + + def forward(self, x): + return self.W3(self.silu(self.W1(x)) * self.W2(x)) + + +if __name__ == '__main__': + # Align results for different attention implementation + torch.manual_seed(2023) + hidden_dim = 4096 + q_weight = torch.randn((hidden_dim, hidden_dim)) + q_bias = torch.zeros((hidden_dim)) + k_weight = torch.randn((hidden_dim, hidden_dim)) + k_bias = torch.zeros((hidden_dim)) + v_weight = torch.randn((hidden_dim, hidden_dim)) + v_bias = torch.zeros((hidden_dim)) + o_weight = torch.randn((hidden_dim, hidden_dim)) + o_bias = torch.randn((hidden_dim)) + pytorch_attn = MultiHeadAttention(hidden_dim, + context_dim=hidden_dim, + num_heads=32, + head_dim=None, + attn_drop=0.0, + dropout=0.0, + backend='pytorch_attn') + + pytorch_attn.load_state_dict({ + 'q.weight': q_weight, + 'k.weight': k_weight, + 'v.weight': v_weight, + 'o.weight': o_weight, + 'o.bias': o_bias + }) + pytorch_attn.to(0) + + xformer_attn = MultiHeadAttention(hidden_dim, + context_dim=hidden_dim, + num_heads=32, + head_dim=None, + attn_drop=0.0, + dropout=0.0, + backend='xformer_attn') + + xformer_attn.load_state_dict({ + 'q.weight': q_weight, + 'k.weight': k_weight, + 'v.weight': v_weight, + 'o.weight': o_weight, + 'o.bias': o_bias + }) + xformer_attn.to(0) + + flash_attn = MultiHeadAttention(hidden_dim, + context_dim=hidden_dim, + num_heads=32, + head_dim=None, + attn_drop=0.0, + dropout=0.0, + backend='flash_attn', + dtype=torch.float16) + flash_attn.load_state_dict({ + 'q.weight': q_weight, + 'k.weight': k_weight, + 'v.weight': v_weight, + 'o.weight': o_weight, + 'o.bias': o_bias + }) + flash_attn.to(0) + + improved_flash_attn = FlashMultiHeadAttention(hidden_dim, + context_dim=hidden_dim, + num_heads=32, + head_dim=None, + attn_drop=0.0, + dropout=0.0, + backend='flash_attn', + dtype=torch.float16) + + improved_flash_attn.load_state_dict({ + 'q.weight': q_weight, + 'k.weight': k_weight, + 'v.weight': v_weight, + 'o.weight': o_weight, + 'o.bias': o_bias + }) + improved_flash_attn.to(0) + + batch_size = 1 + query_length = 1024 + key_length = 1024 + # mask = None + run_num = 10 + torch.cuda.empty_cache() + x = torch.randn((batch_size, query_length, hidden_dim)).to(0) + context = torch.randn((batch_size, key_length, hidden_dim)).to(0) + # mask = torch.cat([torch.ones((batch_size, 80)), torch.zeros((batch_size, key_length - 80))], dim=1).long().to(0) + # mask = torch.randint(1, key_length, [batch_size]).to(0) + mask = None + st = time.time() + for i in tqdm(range(run_num)): + pytorch_res = pytorch_attn(x.clone(), context.clone(), + mask.clone() if mask is not None else mask) + if i == run_num - 1: + free_mem, total_mem = torch.cuda.mem_get_info(0) + free_mem, total_mem = free_mem / (1024**3), total_mem / (1024**3) + mem_msg = f'GPU {0}: free mem {free_mem:.3f}G, total mem {total_mem:.3f}G \n' + pytorch_res_data = pytorch_res.clone().detach().cpu() + print('pytorch attn ', mem_msg, + f'Cost time per time {(time.time() - st) / run_num}s') + # + torch.cuda.empty_cache() + st = time.time() + for i in tqdm(range(run_num)): + xformer_res = xformer_attn(x.clone(), context.clone(), + mask.clone() if mask is not None else mask) + if i == run_num - 1: + free_mem, total_mem = torch.cuda.mem_get_info(0) + free_mem, total_mem = free_mem / (1024**3), total_mem / (1024**3) + mem_msg = f'GPU {0}: free mem {free_mem:.3f}G, total mem {total_mem:.3f}G \n' + xformer_res_data = xformer_res.clone().detach().cpu() + print('xformer attn ', mem_msg, + f'Cost time per time {(time.time() - st) / run_num}s') + # + torch.cuda.empty_cache() + # mask = None + st = time.time() + for i in tqdm(range(run_num)): + flash_res = flash_attn(x.clone(), context.clone(), + mask.clone() if mask is not None else mask) + if i == run_num - 1: + free_mem, total_mem = torch.cuda.mem_get_info(0) + free_mem, total_mem = free_mem / (1024**3), total_mem / (1024**3) + mem_msg = f'GPU {0}: free mem {free_mem:.3f}G, total mem {total_mem:.3f}G \n' + flash_res_data = flash_res.clone().detach().cpu() + print('flash attn ', mem_msg, + f'Cost time per time {(time.time() - st) / run_num}s') + + # recommend this style for multi blocks to save the preprocess time. + + flash_input = flash_preprocess( + x.clone(), + context.clone(), + mask=mask.clone() if mask is not None else mask) + st = time.time() + for i in tqdm(range(run_num)): + improved_flash_res_v1 = improved_flash_attn(**flash_input).reshape( + (batch_size, -1, hidden_dim)) + if i == 0: + improved_flash_res_v1_data = improved_flash_res_v1.clone().detach( + ).cpu() + if i == run_num - 1: + free_mem, total_mem = torch.cuda.mem_get_info(0) + free_mem, total_mem = free_mem / (1024**3), total_mem / (1024**3) + mem_msg = f'GPU {0}: free mem {free_mem:.3f}G, total mem {total_mem:.3f}G \n' + print('improved flash attn ', mem_msg, + f'Cost time per time {(time.time() - st) / run_num}s') + # + print(pytorch_res_data, xformer_res_data, flash_res_data, + improved_flash_res_v1_data) + print(pytorch_res_data.shape, xformer_res_data.shape, flash_res_data.shape, + improved_flash_res_v1_data.shape) + print( + torch.sum(pytorch_res_data) / (batch_size * query_length * hidden_dim), + torch.sum(xformer_res_data) / (batch_size * query_length * hidden_dim), + torch.sum(flash_res_data) / (batch_size * query_length * hidden_dim), + torch.sum(improved_flash_res_v1_data) / + (batch_size * query_length * hidden_dim)) diff --git a/scepter/modules/model/backbone/transformer/layers.py b/scepter/modules/model/backbone/transformer/layers.py new file mode 100644 index 0000000..10875a9 --- /dev/null +++ b/scepter/modules/model/backbone/transformer/layers.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +# All rights reserved. +# This file contains code that is adapted from +# timm: https://github.com/huggingface/pytorch-image-models +# pixart: https://github.com/PixArt-alpha/PixArt-alpha +import math + +import torch +import torch.nn as nn +from einops import rearrange + +from scepter.modules.model.backbone.transformer.attention import drop_path + + +def modulate(x, shift, scale, unsqueeze=False): + if unsqueeze: + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + else: + return x * (1 + scale) + shift + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + +class MaskFinalLayer(nn.Module): + """ + The final layer of PixArt. + """ + def __init__(self, final_hidden_size, c_emb_size, patch_size, + out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(final_hidden_size, + elementwise_affine=False, + eps=1e-6) + self.linear = nn.Linear(final_hidden_size, + patch_size * patch_size * out_channels, + bias=True) + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(c_emb_size, 2 * final_hidden_size, bias=True)) + + def forward(self, x, t): + shift, scale = self.adaLN_modulation(t).chunk(2, dim=1) + x = modulate(self.norm_final(x), shift, scale, unsqueeze=True) + x = self.linear(x) + return x + + +class DecoderLayer(nn.Module): + """ + The final layer of PixArt. + """ + def __init__(self, hidden_size, decoder_hidden_size): + super().__init__() + self.norm_decoder = nn.LayerNorm(hidden_size, + elementwise_affine=False, + eps=1e-6) + self.linear = nn.Linear(hidden_size, decoder_hidden_size, bias=True) + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)) + + def forward(self, x, t): + shift, scale = self.adaLN_modulation(t).chunk(2, dim=1) + x = modulate(self.norm_decoder(x), shift, scale, unsqueeze=True) + x = self.linear(x) + return x + + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + def __init__(self, hidden_size, frequency_embedding_size=256): + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + :param t: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an (N, D) Tensor of positional embeddings. + """ + # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * + torch.arange(start=0, end=half, dtype=torch.float32) / + half).to(device=t.device) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat( + [embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t): + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + t_emb = self.mlp(t_freq) + return t_emb + + +class SizeEmbedder(TimestepEmbedder): + """ + Embeds scalar timesteps into vector representations. + """ + def __init__(self, hidden_size, frequency_embedding_size=256): + super().__init__(hidden_size=hidden_size, + frequency_embedding_size=frequency_embedding_size) + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + self.outdim = hidden_size + + def forward(self, s, bs): + if s.ndim == 1: + s = s[:, None] + assert s.ndim == 2 + if s.shape[0] != bs: + s = s.repeat(bs // s.shape[0], 1) + assert s.shape[0] == bs + b, dims = s.shape[0], s.shape[1] + s = rearrange(s, 'b d -> (b d)') + s_freq = self.timestep_embedding(s, self.frequency_embedding_size).to( + self.dtype) + s_emb = self.mlp(s_freq) + s_emb = rearrange(s_emb, + '(b d) d2 -> b (d d2)', + b=b, + d=dims, + d2=self.outdim) + return s_emb + + @property + def dtype(self): + # 返回模型参数的数据类型 + return next(self.parameters()).dtype + + +class LabelEmbedder(nn.Module): + """ + Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance. + """ + def __init__(self, num_classes, hidden_size, dropout_prob): + super().__init__() + use_cfg_embedding = dropout_prob > 0 + self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, + hidden_size) + self.num_classes = num_classes + self.dropout_prob = dropout_prob + + def token_drop(self, labels, force_drop_ids=None): + """ + Drops labels to enable classifier-free guidance. + """ + if force_drop_ids is None: + drop_ids = torch.rand(labels.shape[0]).cuda() < self.dropout_prob + else: + drop_ids = force_drop_ids == 1 + labels = torch.where(drop_ids, self.num_classes, labels) + return labels + + def forward(self, labels, train, force_drop_ids=None): + use_dropout = self.dropout_prob > 0 + if (train and use_dropout) or (force_drop_ids is not None): + labels = self.token_drop(labels, force_drop_ids) + return self.embedding_table(labels) + + +class CaptionEmbedder(nn.Module): + """ + Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance. + """ + def __init__(self, + in_channels, + hidden_size, + uncond_prob, + act_layer=nn.GELU(approximate='tanh'), + token_num=120): + super().__init__() + self.y_proj = Mlp(in_features=in_channels, + hidden_features=hidden_size, + out_features=hidden_size, + act_layer=act_layer, + drop=0) + self.register_buffer( + 'y_embedding', + nn.Parameter( + torch.randn(token_num, in_channels) / in_channels**0.5)) + self.uncond_prob = uncond_prob + + def token_drop(self, caption, force_drop_ids=None): + """ + Drops labels to enable classifier-free guidance. + """ + if force_drop_ids is None: + drop_ids = torch.rand(caption.shape[0]).cuda() < self.uncond_prob + else: + drop_ids = force_drop_ids == 1 + caption = torch.where(drop_ids[:, None, None], self.y_embedding, + caption) + return caption + + def forward(self, caption, train, force_drop_ids=None): + if train: + assert caption.shape[1:] == self.y_embedding.shape + use_dropout = self.uncond_prob > 0 + if (train and use_dropout) or (force_drop_ids is not None): + caption = self.token_drop(caption, force_drop_ids) + caption = self.y_proj(caption) + return caption + + +class CaptionEmbedderDoubleBr(nn.Module): + """ + Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance. + """ + def __init__(self, + in_channels, + hidden_size, + uncond_prob, + act_layer=nn.GELU(approximate='tanh'), + token_num=120): + super().__init__() + self.proj = Mlp(in_features=in_channels, + hidden_features=hidden_size, + out_features=hidden_size, + act_layer=act_layer, + drop=0) + self.embedding = nn.Parameter(torch.randn(1, in_channels) / 10**0.5) + self.y_embedding = nn.Parameter( + torch.randn(token_num, in_channels) / 10**0.5) + self.uncond_prob = uncond_prob + + def token_drop(self, global_caption, caption, force_drop_ids=None): + """ + Drops labels to enable classifier-free guidance. + """ + if force_drop_ids is None: + drop_ids = torch.rand( + global_caption.shape[0]).cuda() < self.uncond_prob + else: + drop_ids = force_drop_ids == 1 + global_caption = torch.where(drop_ids[:, None], self.embedding, + global_caption) + caption = torch.where(drop_ids[:, None, None, None], self.y_embedding, + caption) + return global_caption, caption + + def forward(self, caption, train, force_drop_ids=None): + assert caption.shape[2:] == self.y_embedding.shape + global_caption = caption.mean(dim=2).squeeze() + use_dropout = self.uncond_prob > 0 + if (train and use_dropout) or (force_drop_ids is not None): + global_caption, caption = self.token_drop(global_caption, caption, + force_drop_ids) + y_embed = self.proj(global_caption) + return y_embed, caption + + +class Mlp(nn.Module): + """ MLP as used in Vision Transformer, MLP-Mixer and related networks + """ + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x diff --git a/scepter/modules/model/backbone/transformer/patchify.py b/scepter/modules/model/backbone/transformer/patchify.py new file mode 100644 index 0000000..197476c --- /dev/null +++ b/scepter/modules/model/backbone/transformer/patchify.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +# All rights reserved. +# This file contains code that is adapted from +# timm: https://github.com/huggingface/pytorch-image-models +# pixart: https://github.com/PixArt-alpha/PixArt-alpha +import torch +import torch.nn as nn + + +class PatchEmbed(nn.Module): + """ 2D Image to Patch Embedding + """ + def __init__( + self, + patch_size=16, + in_chans=3, + embed_dim=768, + norm_layer=None, + flatten=True, + bias=True, + ): + super().__init__() + self.flatten = flatten + self.proj = nn.Conv2d(in_chans, + embed_dim, + kernel_size=patch_size, + stride=patch_size, + bias=bias) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x): + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + + +def unpatchify(x, h, w, c, p_h, p_w): + ''' + Args: + x: input tensor for unpatchified with shape as (N, T, patch_size**2 * C). + h: tokens' number align height + w: tokens' number align width + c: output channels + p_h: patch size for h + p_w: patch size for w + Returns: unpatchified imgs with shape as (N, H, W, C) + ''' + assert h * w == x.shape[1] + x = x.reshape(shape=(x.shape[0], h, w, p_h, p_w, c)) + x = torch.einsum('nhwpqc->nchpwq', x) + return x.reshape(shape=(x.shape[0], c, h * p_h, w * p_w)) diff --git a/scepter/modules/model/backbone/transformer/pos_embed.py b/scepter/modules/model/backbone/transformer/pos_embed.py new file mode 100644 index 0000000..d1b2f58 --- /dev/null +++ b/scepter/modules/model/backbone/transformer/pos_embed.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +# All rights reserved. +# This file contains code that is adapted from +# timm: https://github.com/huggingface/pytorch-image-models +# pixart: https://github.com/PixArt-alpha/PixArt-alpha +from itertools import repeat as iter_repeat +from typing import Iterable + +import numpy as np +import torch + + +def _ntuple(n): + def parse(x): + if isinstance(x, Iterable) and not isinstance(x, str): + return x + return tuple(iter_repeat(x, n)) + + return parse + + +to_1tuple = _ntuple(1) +to_2tuple = _ntuple(2) + + +def get_2d_sincos_pos_embed(embed_dim, + grid_size, + cls_token=False, + extra_tokens=0, + lewei_scale=1.0, + base_h_size=16., + base_w_size=16): + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + if isinstance(grid_size, int): + grid_size = to_2tuple(grid_size) + grid_h = np.arange(grid_size[0], dtype=np.float32) / ( + grid_size[0] / base_h_size) / lewei_scale + grid_w = np.arange(grid_size[1], dtype=np.float32) / ( + grid_size[1] / base_w_size) / lewei_scale + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + grid = grid.reshape([2, 1, grid_size[1], grid_size[0]]) + + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token and extra_tokens > 0: + pos_embed = np.concatenate( + [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, + grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, + grid[1]) # (H*W, D/2) + + return np.concatenate([emb_h, emb_w], axis=1) + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2. + omega = 1. / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + return np.concatenate([emb_sin, emb_cos], axis=1) + + +def apply_2d_rope(xq, + xk, + padded_pos_index, + num_head, + head_dim, + rotary_base=10000): + ''' + x query/key: [b, seq, num_head*head_dim] + padded_pos_index: [b, seq, 2] + ''' + b = xq.shape[0] + assert head_dim % 4 == 0, 'the 2d_rope dims should be divided by 4' + rope_dim = head_dim // 2 # 2d_rope_dim, 1d_rope_dim = head_dim + # 1. theta_d = b ** (-2d/D) + theta = 1.0 / (rotary_base**( + torch.arange(0, rope_dim, 2)[:(rope_dim // 2)].float() / rope_dim)) + # 2. [h * Theta || w * Theta] + theta = theta.to(xq.device).expand(b, 1, rope_dim // 2) + freqs_h = torch.bmm(padded_pos_index[:, :, :1], + theta).float() # h * \theta + freqs_w = torch.bmm(padded_pos_index[:, :, 1:], + theta).float() # w * \theta + freqs = torch.cat([freqs_h, freqs_w], dim=2).repeat(1, 1, + num_head) # multi-head + # 3. as_complex for complex multiply + # if freqs = [x, y] then freqs_cis = [cos(x) + sin(x)i, cos(y) + sin(y)i] + freqs_cis = torch.polar( + torch.ones_like(freqs), + freqs) # torch.polar(abs, angle)=> abs⋅cos(angle)+abs⋅sin(angle)⋅j + # xq.shape = [b, seq_len, dim] + # xq_.shape = [b, seq_len, dim // 2, 2] + xq_ = xq.float().reshape(*xq.shape[:-1], -1, 2) + xk_ = xk.float().reshape(*xk.shape[:-1], -1, 2) + # 转为复数域 + xq_ = torch.view_as_complex( + xq_) # [b, seq_len, dim // 2, 2]=>xq.shape = [b, seq_len, dim] + xk_ = torch.view_as_complex(xk_) + # 4. complex multiply and as real + # xq_out.shape = [b, seq_len, dim] + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten( + 2) # point_wise mul, then flatten eg[[1,2],[3,4],[5,6]]->[1,2,3,4,5,6] + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(2) + return xq_out.type_as(xq), xk_out.type_as(xk) diff --git a/scepter/modules/model/embedder/__init__.py b/scepter/modules/model/embedder/__init__.py index f7b27b9..cca51d2 100644 --- a/scepter/modules/model/embedder/__init__.py +++ b/scepter/modules/model/embedder/__init__.py @@ -2,6 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from scepter.modules.model.embedder.embedder import ( - ConcatTimestepEmbedderND, FrozenCLIPEmbedder, FrozenOpenCLIPEmbedder, - FrozenOpenCLIPEmbedder2, GeneralConditioner, IPAdapterPlusEmbedder, - RefCrossEmbedder) + ConcatTimestepEmbedderND, FrozenCLIPEmbedder, FrozenCLIPEmbedder2, + FrozenOpenCLIPEmbedder, FrozenOpenCLIPEmbedder2, GeneralConditioner, + IPAdapterPlusEmbedder, RefCrossEmbedder, SD3TextEmbedder, T5EmbedderHF) diff --git a/scepter/modules/model/embedder/embedder.py b/scepter/modules/model/embedder/embedder.py index 3000e35..8192c1a 100644 --- a/scepter/modules/model/embedder/embedder.py +++ b/scepter/modules/model/embedder/embedder.py @@ -11,20 +11,22 @@ import torch import torch.nn as nn import torch.utils.dlpack from einops import rearrange -# to check from scepter.modules.model.backbone.unet.unet_utils import Timestep +from scepter.modules.model.embedder.base_embedder import BaseEmbedder +from scepter.modules.model.embedder.resampler import Resampler from scepter.modules.model.registry import EMBEDDERS +from scepter.modules.model.tokenizer.tokenizer_component import ( + basic_clean, canonicalize, heavy_clean, whitespace_clean) from scepter.modules.model.utils.basic_utils import expand_dims_like from scepter.modules.utils.config import dict_to_yaml from scepter.modules.utils.distribute import we from scepter.modules.utils.file_system import FS from torch.utils.checkpoint import checkpoint -from .base_embedder import BaseEmbedder -from .resampler import Resampler - try: - from transformers import CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + from transformers import (CLIPTextModel, CLIPTokenizer, + CLIPVisionModelWithProjection, AutoTokenizer, + T5EncoderModel, CLIPTextModelWithProjection) except Exception as e: warnings.warn( f'Import transformers error, please deal with this problem: {e}') @@ -117,7 +119,6 @@ class FrozenCLIPEmbedder(BaseEmbedder): for param in self.parameters(): param.requires_grad = False - # @torch.no_grad() def _forward(self, text): batch_encoding = self.tokenizer(text, truncation=True, @@ -779,3 +780,296 @@ class GeneralConditioner(BaseEmbedder): __class__.__name__, GeneralConditioner.para_dict, set_name=True) + + +@EMBEDDERS.register_class() +class T5EmbedderHF(BaseEmbedder): + """ + Uses the OpenCLIP transformer encoder for text + """ + """ + Uses the OpenCLIP transformer encoder for text + """ + para_dict = { + 'PRETRAINED_MODEL': { + 'value': + 'google/umt5-small', + 'description': + 'Pretrained Model for umt5, modelcard path or local path.' + }, + 'TOKENIZER_PATH': { + 'value': 'google/umt5-small', + 'description': + 'Tokenizer Path for umt5, modelcard path or local path.' + }, + 'FREEZE': { + 'value': True, + 'description': '' + }, + 'USE_GRAD': { + 'value': False, + 'description': 'Compute grad or not.' + }, + 'CLEAN': { + 'value': + 'whitespace', + 'description': + 'Set the clean strtegy for tokenizer, used when TOKENIZER_PATH is not None.' + }, + 'LAYER': { + 'value': 'last', + 'description': '' + }, + 'LEGACY': { + 'value': + True, + 'description': + 'Whether use legacy returnd feature or not ,default True.' + } + } + + def __init__(self, cfg, logger=None): + super().__init__(cfg, logger=logger) + pretrained_path = cfg.get('PRETRAINED_MODEL', None) + t5_dtype = cfg.get('T5_DTYPE', None) + assert pretrained_path + with FS.get_dir_to_local_dir(pretrained_path, + wait_finish=True) as local_path: + if t5_dtype is not None: + self.model = T5EncoderModel.from_pretrained( + local_path, torch_dtype=getattr(torch, t5_dtype)) + else: + self.model = T5EncoderModel.from_pretrained(local_path) + tokenizer_path = cfg.get('TOKENIZER_PATH', None) + self.length = cfg.get('LENGTH', 77) + if tokenizer_path: + self.tokenize_kargs = {'return_tensors': 'pt'} + with FS.get_dir_to_local_dir(tokenizer_path, + wait_finish=True) as local_path: + self.tokenizer = AutoTokenizer.from_pretrained(local_path) + if self.length is not None: + self.tokenize_kargs.update({ + 'padding': 'max_length', + 'truncation': True, + 'max_length': self.length + }) + self.eos_token = self.tokenizer( + self.tokenizer.eos_token)['input_ids'][0] + else: + self.tokenizer = None + self.tokenize_kargs = {} + + self.use_grad = cfg.get('USE_GRAD', False) + self.clean = cfg.get('CLEAN', 'whitespace') + + def freeze(self): + self.model = self.model.eval() + for param in self.parameters(): + param.requires_grad = False + + # encode && encode_text + def forward(self, tokens, return_mask=False): + # tokenization + embedding_context = nullcontext if self.use_grad else torch.no_grad + with embedding_context(): + x = self.model(tokens.input_ids.to(we.device_id), + tokens.attention_mask.to(we.device_id)) + x = x.last_hidden_state + # if not self.return_pooled: + # return x.detach() + # else: + # return x.detach(), self.pool(x, tokens.input_ids) + if return_mask: + return x.detach() + 0.0, tokens.attention_mask.to(we.device_id) + else: + return x.detach() + 0.0 + + def pool(self, x, tokens): + # take features from the eot embedding (eot_token is the highest number in each sequence) + return x[torch.arange(x.shape[0]), + torch.argmax((tokens.input_ids == 1).float(), dim=-1)] + + def _clean(self, text): + if self.clean == 'whitespace': + text = whitespace_clean(basic_clean(text)) + elif self.clean == 'lower': + text = whitespace_clean(basic_clean(text)).lower() + elif self.clean == 'canonicalize': + text = canonicalize(basic_clean(text)) + elif self.clean == 'heavy': + text = heavy_clean(heavy_clean(text)) + return text + + def encode_text(self, + tokens, + tokenizer=None, + append_sentence_embedding=False, + return_mask=False): + return self(tokens, return_mask=return_mask) + + def encode(self, text, return_mask=False): + if isinstance(text, str): + text = [text] + if self.clean: + text = [self._clean(u) for u in text] + assert self.tokenizer is not None + tokens = self.tokenizer(text, **self.tokenize_kargs) + return self(tokens, return_mask=return_mask) + + @staticmethod + def get_config_template(): + return dict_to_yaml('MODELS', + __class__.__name__, + T5EmbedderHF.para_dict, + set_name=True) + + +@EMBEDDERS.register_class() +class FrozenCLIPEmbedder2(FrozenCLIPEmbedder): + """Uses the CLIP transformer encoder for text (from huggingface)""" + para_dict = { + 'RETURN_POOLED': { + 'value': False, + 'description': + 'Whether return pooled results or not, default False.' + } + } + para_dict.update(FrozenCLIPEmbedder.para_dict) + LAYERS = ['hidden', 'last', 'penultimate'] + + def __init__(self, cfg, logger=None): + super(FrozenCLIPEmbedder, self).__init__(cfg, logger=logger) + self.return_pooled = cfg.get('RETURN_POOLED', False) + tokenizer_path = cfg.get('TOKENIZER_PATH', None) + if tokenizer_path is not None: + with FS.get_dir_to_local_dir(tokenizer_path, + wait_finish=True) as local_path: + self.tokenizer = CLIPTokenizer.from_pretrained(local_path) + + pretrained_model = cfg.get('PRETRAINED_MODEL', None) + if pretrained_model is None: + raise 'You should set pretrained_model: modelcard.' + with FS.get_dir_to_local_dir(cfg.PRETRAINED_MODEL, + wait_finish=True) as local_path: + self.transformer = CLIPTextModelWithProjection.from_pretrained( + local_path) + + self.use_grad = cfg.get('USE_GRAD', False) + self.freeze_flag = cfg.get('FREEZE', True) + if self.freeze_flag: + self.freeze() + + self.max_length = cfg.get('MAX_LENGTH', 77) + self.layer = cfg.get('LAYER', 'last') + self.layer_idx = cfg.get('LAYER_IDX', None) + self.use_final_layer_norm = cfg.get('USE_FINAL_LAYER_NORM', False) + assert self.layer in self.LAYERS + if self.layer == 'hidden': + assert self.layer_idx is not None + assert 0 <= abs(self.layer_idx) <= 12 + + def _forward(self, text): + batch_encoding = self.tokenizer(text, + truncation=True, + max_length=self.max_length, + return_length=True, + return_overflowing_tokens=False, + padding='max_length', + return_tensors='pt') + tokens = batch_encoding['input_ids'].to(we.device_id) + outputs = self.transformer(input_ids=tokens, output_hidden_states=True) + if self.layer == 'last': + context = outputs.last_hidden_state + elif self.layer == 'penultimate': + context = outputs.hidden_states[-2] + else: + context = outputs.hidden_states[self.layer_idx] + + if self.return_pooled: + pooled = outputs[0] + return context, pooled + return context + + +@EMBEDDERS.register_class() +class SD3TextEmbedder(BaseEmbedder): + def __init__(self, cfg, logger=None): + super().__init__(cfg, logger=logger) + + clip_l_config = cfg.get('CLIP_L', None) + clip_g_config = cfg.get('CLIP_G', None) + t5_xxl_config = cfg.get('T5_XXL', None) + + self.clip_l = EMBEDDERS.build(clip_l_config) if clip_l_config else None + self.clip_g = EMBEDDERS.build(clip_g_config) if clip_g_config else None + self.t5_xxl = EMBEDDERS.build(t5_xxl_config) if t5_xxl_config else None + + self.p_zero = cfg.get('P_ZERO', 0.464) + + def encode(self, text): + return self(text) + + def forward(self, text): + l_ctx, g_ctx, t5_ctx = None, None, None + n = len(text) + l_pooled = torch.zeros((n, 768), device=we.device_id) + g_pooled = torch.zeros((n, 1280), device=we.device_id) + if self.clip_l: + with torch.autocast(device_type='cuda', + enabled=True, + dtype=torch.float16): + l_ctx, l_pooled = self.clip_l.encode(text) + if self.clip_g: + with torch.autocast(device_type='cuda', + enabled=True, + dtype=torch.float16): + g_ctx, g_pooled = self.clip_g.encode(text) + if self.t5_xxl: + with torch.autocast(device_type='cuda', + enabled=True, + dtype=torch.float16): + t5_ctx = self.t5_xxl.encode(text) + + pooled = torch.cat((l_pooled, g_pooled), dim=-1) + + if l_ctx is not None and g_ctx is not None: + lg_ctx = torch.cat([l_ctx, g_ctx], dim=-1) + lg_ctx = torch.nn.functional.pad(lg_ctx, + (0, 4096 - lg_ctx.shape[-1])) + elif l_ctx is not None: + lg_ctx = torch.nn.functional.pad(l_ctx, + (0, 4096 - l_ctx.shape[-1])) + elif g_ctx is not None: + lg_ctx = torch.nn.functional.pad(g_ctx, (768, 0)) + lg_ctx = torch.nn.functional.pad(lg_ctx, + (0, 4096 - lg_ctx.shape[-1])) + else: + lg_ctx = None + + if t5_ctx is not None and lg_ctx is not None: + ctx = torch.cat([lg_ctx, t5_ctx], dim=-2) + elif t5_ctx is not None: + ctx = t5_ctx + elif lg_ctx is not None: + ctx = lg_ctx + else: + ctx = torch.zeros((n, 77, 4096), device=we.device_id) + + return ctx, pooled + + +if __name__ == '__main__': + import argparse + from scepter.modules.utils.config import Config + from scepter.modules.utils.logger import get_logger + std_logger = get_logger(name='scepter') + parser = argparse.ArgumentParser(description='Argparser for Scepter:\n') + cfg = Config(load=True, parser_ins=parser) + + for file_sys in cfg.FILE_SYSTEM: + FS.init_fs_client(file_sys) + model = SD3TextEmbedder(cfg.COND_STAGE_MODEL, + logger=std_logger).to(we.device_id) + text = ['a dog is eating food.'] + ctx, pooled = model(text) + print(ctx.shape, pooled.shape) diff --git a/scepter/modules/model/network/__init__.py b/scepter/modules/model/network/__init__.py index 0e11597..c9e9b23 100644 --- a/scepter/modules/model/network/__init__.py +++ b/scepter/modules/model/network/__init__.py @@ -4,4 +4,5 @@ from scepter.modules.model.network.autoencoder import ae_kl from scepter.modules.model.network.classifier import Classifier from scepter.modules.model.network.diffusion import (diffusion, schedules, solvers) -from scepter.modules.model.network.ldm import ldm, ldm_sce, ldm_xl +from scepter.modules.model.network.ldm import (ldm, ldm_edit, ldm_pixart, + ldm_sce, ldm_sd3, ldm_xl) diff --git a/scepter/modules/model/network/autoencoder/ae_kl.py b/scepter/modules/model/network/autoencoder/ae_kl.py index 19a90a5..71bcbae 100644 --- a/scepter/modules/model/network/autoencoder/ae_kl.py +++ b/scepter/modules/model/network/autoencoder/ae_kl.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. - +import re from collections import OrderedDict import numpy as np import torch + from scepter.modules.model.network.train_module import TrainModule from scepter.modules.model.registry import BACKBONES, LOSSES, MODELS from scepter.modules.utils.config import dict_to_yaml @@ -87,6 +88,7 @@ class AutoencoderKL(TrainModule): self.pretrained_model = self.cfg.get('PRETRAINED_MODEL', None) self.ignore_keys = self.cfg.get('IGNORE_KEYS', []) self.batch_size = self.cfg.get('BATCH_SIZE', 16) + self.use_conv = self.cfg.get('USE_CONV', True) self.construct_network() self.init_network() @@ -95,8 +97,12 @@ class AutoencoderKL(TrainModule): z_channels = self.encoder_cfg.Z_CHANNELS self.encoder = BACKBONES.build(self.encoder_cfg, logger=self.logger) self.decoder = BACKBONES.build(self.decoder_cfg, logger=self.logger) - self.conv1 = torch.nn.Conv2d(2 * z_channels, 2 * self.embed_dim, 1) - self.conv2 = torch.nn.Conv2d(self.embed_dim, z_channels, 1) + self.conv1 = torch.nn.Conv2d( + 2 * z_channels, 2 * + self.embed_dim, 1) if self.use_conv else torch.nn.Identity() + self.conv2 = torch.nn.Conv2d( + self.embed_dim, z_channels, + 1) if self.use_conv else torch.nn.Identity() if self.loss_cfg is not None: self.loss = LOSSES.build(self.loss_cfg, logger=self.logger) @@ -107,7 +113,7 @@ class AutoencoderKL(TrainModule): wait_finish=True) as local_model: self.init_from_ckpt(local_model, ignore_keys=self.ignore_keys) - def init_from_ckpt(self, path, ignore_keys=list()): + def init_from_ckpt(self, path, ignore_keys): if path.find('.safetensors') > -1: from safetensors import safe_open sd = OrderedDict() @@ -122,20 +128,17 @@ class AutoencoderKL(TrainModule): sd = sd['state_dict'] new_sd = OrderedDict() + for k, v in sd.items(): - ignored = False - for ik in ignore_keys: - if ik in k: - if we.rank == 0: - self.logger.info( - 'ignore key {} from state_dict.'.format(k)) - ignored = True - break + if self.ignore_keys is not None: + if (isinstance(self.ignore_keys, str) and re.match(self.ignore_keys, k)) or \ + (isinstance(self.ignore_keys, list) and k in self.ignore_keys): + continue k = k.replace('post_quant_conv', 'conv2') if 'post_quant_conv' in k else k k = k.replace('quant_conv', 'conv1') if 'quant_conv' in k else k - if not ignored: - new_sd[k] = v + k = k.replace('first_stage_model.', '') + new_sd[k] = v missing, unexpected = self.load_state_dict(new_sd, strict=False) if we.rank == 0: @@ -264,3 +267,14 @@ class AutoencoderKL(TrainModule): __class__.__name__, AutoencoderKL.para_dict, set_name=True) + + +if __name__ == '__main__': + import argparse + from scepter.modules.utils.config import Config + from scepter.modules.utils.logger import get_logger + std_logger = get_logger(name='scepter') + parser = argparse.ArgumentParser(description='Argparser for Scepter:\n') + cfg = Config(load=True, parser_ins=parser) + model = AutoencoderKL(cfg, logger=std_logger) + model.load_pretrained_model(cfg.PRETRAINED_MODEL) diff --git a/scepter/modules/model/network/diffusion/diffusion.py b/scepter/modules/model/network/diffusion/diffusion.py index 48405fe..1ea84f0 100644 --- a/scepter/modules/model/network/diffusion/diffusion.py +++ b/scepter/modules/model/network/diffusion/diffusion.py @@ -170,6 +170,45 @@ def adaptive_anisotropic_filter(x, g=None): return y +def extract_into_tensor(a, t, x_shape): + b, *_ = t.shape + out = a.gather(-1, t) + return out.reshape(b, *((1, ) * (len(x_shape) - 1))) + + +def discretize_timesteps(t_max, t_min, steps, discretization): + """ + Implementation of timestep discretization methods. + """ + if discretization == 'leading': + steps = torch.arange(t_min, t_max + 1, + (t_max - t_min + 1) / steps).flip(0) + elif discretization == 'linspace': + steps = torch.linspace(t_max, t_min, steps) + elif discretization == 'trailing': + steps = torch.arange(t_max, t_min - 1, -((t_max - t_min + 1) / steps)) + else: + raise NotImplementedError( + f'{discretization} discretization not implemented') + return steps.clamp_(t_min, t_max) + + +def get_scalings_for_boundary_condition(sigma): + sigma_data = 0.5 + c_skip = (1 - + sigma**2)**0.5 * sigma_data**2 / (sigma**2 + + (1 - sigma**2) * sigma_data**2) + c_out = (sigma * sigma_data / (sigma**2 + + (1 - sigma**2) * sigma_data**2)**0.5) + return c_skip, c_out + + +def v_to_x0(v, t, x_t, diffusion): + sigmas = _i(diffusion.sigmas, t, v) + alphas = _i(diffusion.alphas, t, v) + return alphas * x_t - sigmas * v + + class GaussianDiffusion(object): def __init__(self, sigmas, prediction_type='eps'): assert prediction_type in {'x0', 'eps', 'v'} @@ -666,40 +705,212 @@ class GaussianDiffusion(object): noise) -def extract_into_tensor(a, t, x_shape): - b, *_ = t.shape - out = a.gather(-1, t) - return out.reshape(b, *((1, ) * (len(x_shape) - 1))) +class GaussianDiffusionRF(object): + def __init__(self, sigmas, prediction_type='rf'): + assert prediction_type in {'rf'} + self.sigmas = sigmas + self.num_timesteps = len(sigmas) + def diffuse(self, x0, t, noise, sigma): + """ + Add Gaussian noise to signal x0 according to: + q(x_t | x_0) = N(x_t | alpha_t x_0, sigma_t^2 I). + """ + shape = (x0.size(0), ) + (1, ) * (x0.ndim - 1) + sigma = sigma.view(shape) + alpha = 1 - sigma + xt = alpha * x0 + sigma * noise + return xt -def discretize_timesteps(t_max, t_min, steps, discretization): - """ - Implementation of timestep discretization methods. - """ - if discretization == 'leading': - steps = torch.arange(t_min, t_max + 1, - (t_max - t_min + 1) / steps).flip(0) - elif discretization == 'linspace': - steps = torch.linspace(t_max, t_min, steps) - elif discretization == 'trailing': - steps = torch.arange(t_max, t_min - 1, -((t_max - t_min + 1) / steps)) - else: - raise NotImplementedError( - f'{discretization} discretization not implemented') - return steps.clamp_(t_min, t_max) + def denoise(self, + xt, + t, + sigma, + model, + model_kwargs={}, + guide_scale=None, + guide_rescale=None, + cat_uc=False, + **kwargs): + assert sigma is not None + shape = (xt.size(0), ) + (1, ) * (xt.ndim - 1) + sigma = sigma.view(shape) -def get_scalings_for_boundary_condition(sigma): - sigma_data = 0.5 - c_skip = (1 - - sigma**2)**0.5 * sigma_data**2 / (sigma**2 + - (1 - sigma**2) * sigma_data**2) - c_out = (sigma * sigma_data / (sigma**2 + - (1 - sigma**2) * sigma_data**2)**0.5) - return c_skip, c_out + # prediction + if guide_scale is None: + if isinstance(model_kwargs, dict): + out = model(xt, t=t, **model_kwargs, **kwargs) + elif isinstance(model_kwargs, list) and len(model_kwargs) > 0: + out = model(xt, t=t, **model_kwargs[0], **kwargs) + else: + raise Exception('Error') + else: + # classifier-free guidance (arXiv:2207.12598) + # model_kwargs[0]: conditional kwargs + # model_kwargs[1]: non-conditional kwargs + assert isinstance(model_kwargs, list) and len(model_kwargs) >= 2 + if isinstance(guide_scale, float) or isinstance(guide_scale, int): + assert len(model_kwargs) == 2 + if guide_scale == 1.: + out = model(xt, t=t, **model_kwargs[0], **kwargs) + else: + if cat_uc: + def parse_model_kwargs(prev_value, value): + if isinstance(value, torch.Tensor): + prev_value = torch.cat([prev_value, value], + dim=0) + elif isinstance(value, dict): + for k, v in value.items(): + prev_value[k] = parse_model_kwargs( + prev_value[k], v) + elif isinstance(value, list): + for idx, v in enumerate(value): + prev_value[idx] = parse_model_kwargs( + prev_value[idx], v) + return prev_value -def v_to_x0(v, t, x_t, diffusion): - sigmas = _i(diffusion.sigmas, t, v) - alphas = _i(diffusion.alphas, t, v) - return alphas * x_t - sigmas * v + all_model_kwargs = copy.deepcopy(model_kwargs[0]) + for model_kwarg in model_kwargs[1:]: + for key, value in model_kwarg.items(): + all_model_kwargs[key] = parse_model_kwargs( + all_model_kwargs[key], value) + all_out = model(xt.repeat(2, 1, 1, 1), + t=t.repeat(2), + **all_model_kwargs, + **kwargs) + y_out, u_out = all_out.chunk(2) + else: + y_out = model(xt, t=t, **model_kwargs[0], **kwargs) + u_out = model(xt, t=t, **model_kwargs[1], **kwargs) + + out = u_out + guide_scale * (y_out - u_out) + if guide_rescale is not None and guide_rescale > 0.0: + assert guide_rescale >= 0 and guide_rescale <= 1 + ratio = ( + y_out.flatten(1).std(dim=1) / + (out.flatten(1).std(dim=1) + 1e-12)).view((-1, ) + (1, ) * + (y_out.ndim - 1)) + out *= guide_rescale * ratio + (1 - guide_rescale) * 1.0 + + x0 = xt - sigma * out + return x0 + + def loss(self, + x0, + t, + model, + model_kwargs={}, + reduction='mean', + noise=None, + **kwargs): + + sigma = t / self.num_timesteps + shape = (x0.size(0), ) + (1, ) * (x0.ndim - 1) + sigma = sigma.view(shape) + if noise is None: + noise = torch.randn_like(x0) + xt = self.diffuse(x0, t, noise, sigma=sigma) + out = model(xt, t=t, **model_kwargs, **kwargs) + loss = ((xt - sigma * out) - x0)**2 + # loss = (out - (x0 - noise)) ** 2 + if reduction == 'mean': + loss = loss.flatten(1).mean(dim=1) + return loss + + @torch.no_grad() + def sample(self, + noise, + model, + model_kwargs={}, + guide_scale=None, + guide_rescale=None, + solver='euler', + steps=20, + shift=3, + discretization=None, + return_intermediate=None, + show_progress=False, + seed=-1, + intermediate_callback=None, + cat_uc=False, + **kwargs): + # sanity check + assert isinstance(steps, (int, torch.LongTensor)) + assert return_intermediate in (None, 'x0', 'xt') + + # function of diffusion solver + solver_fn = { + 'ddim': sample_ddim, + 'euler_ancestral': sample_euler_ancestral, + 'euler': sample_euler, + 'heun': sample_heun, + 'dpm2': sample_dpm_2, + 'dpm2_ancestral': sample_dpm_2_ancestral, + 'dpmpp_2s_ancestral': sample_dpmpp_2s_ancestral, + 'dpmpp_2m': sample_dpmpp_2m, + 'dpmpp_sde': sample_dpmpp_sde, + 'dpmpp_2m_sde': sample_dpmpp_2m_sde, + 'dpm2_karras': sample_dpm_2, + 'dpm2_ancestral_karras': sample_dpm_2_ancestral, + 'dpmpp_2s_ancestral_karras': sample_dpmpp_2s_ancestral, + 'dpmpp_2m_karras': sample_dpmpp_2m, + 'dpmpp_sde_karras': sample_dpmpp_sde, + 'dpmpp_2m_sde_karras': sample_dpmpp_2m_sde, + 'onestep': sample_onestep, + 'multistep': stochastic_iterative_sampler, + 'multistep2': stochastic_iterative_sampler2, + 'multistep3': stochastic_iterative_sampler3, + 'dpmpp_2m_sde_lcm': sample_dpmpp_2m_sde_lcm, + }[solver] + + seed = seed if seed >= 0 else random.randint(0, 2**31) + intermediates = [] + + def model_fn(xt, sigma): + # denoising + sigma = sigma.repeat(len(xt)).to(xt.device) + t = self._sigma_to_t(sigma).round().long() + x0 = self.denoise(xt, + t, + sigma, + model, + model_kwargs, + guide_scale, + guide_rescale, + cat_uc=cat_uc, + **kwargs) + + # collect intermediate outputs + if return_intermediate == 'xt': + intermediates.append(xt) + elif return_intermediate == 'x0': + intermediates.append(x0) + if intermediate_callback is not None: + intermediate_callback(intermediates[-1]) + return x0 + + # get timesteps + device = self.sigmas.device + sigma_max = self.sigmas[0] + sigma_min = self.sigmas[-1] + t_max = sigma_max * self.num_timesteps + t_min = sigma_min * self.num_timesteps + steps = torch.linspace(t_max, t_min, steps).to(device) + sigmas = steps / self.num_timesteps + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + sigmas = sigmas.to(torch.float32).to(device) + sigmas = torch.cat([sigmas, sigmas.new_zeros([1])]) + + kwargs['seed'] = seed + # sampling + x0 = solver_fn(noise, + model_fn, + sigmas, + show_progress=show_progress, + **kwargs) + return (x0, intermediates) if return_intermediate is not None else x0 + + def _sigma_to_t(self, sigma): + return sigma * self.num_timesteps diff --git a/scepter/modules/model/network/diffusion/schedules.py b/scepter/modules/model/network/diffusion/schedules.py index 72a1fb3..cb954f0 100644 --- a/scepter/modules/model/network/diffusion/schedules.py +++ b/scepter/modules/model/network/diffusion/schedules.py @@ -13,6 +13,7 @@ where alpha_t^2 = 1 - sigma_t^2. """ import math +import numpy as np import torch __all__ = [ @@ -154,6 +155,15 @@ def logsnr_cosine_interp_schedule(n, _logsnr_cosine_interp(n, logsnr_min, logsnr_max, scale_min, scale_max)) +def shifted_schedule(n, shift=3): + timesteps = np.linspace(1, n, n, dtype=np.float32)[::-1].copy() + timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32) + + sigmas = timesteps / n + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + return sigmas + + def noise_schedule(schedule='logsnr_cosine_interp', n=1000, zero_terminal_snr=False, @@ -171,7 +181,8 @@ def noise_schedule(schedule='logsnr_cosine_interp', 'vp': vp_schedule, 'logsnr_cosine': logsnr_cosine_schedule, 'logsnr_cosine_shifted': logsnr_cosine_shifted_schedule, - 'logsnr_cosine_interp': logsnr_cosine_interp_schedule + 'logsnr_cosine_interp': logsnr_cosine_interp_schedule, + 'shifted': shifted_schedule, }[schedule](n, **kwargs) # post-processing diff --git a/scepter/modules/model/network/diffusion/solvers.py b/scepter/modules/model/network/diffusion/solvers.py index 437da91..99d22ab 100644 --- a/scepter/modules/model/network/diffusion/solvers.py +++ b/scepter/modules/model/network/diffusion/solvers.py @@ -12,9 +12,8 @@ q(x_t | x_0) = N(x_t | alpha_t x_0, sigma_t^2 I), where 0 <= sigma_t <= 1 and alpha_t^2 = 1 - sigma_t^2. """ -from tqdm.auto import trange - import torch +from tqdm.auto import trange __all__ = [ 'sample_euler', 'sample_euler_ancestral', 'sample_heun', 'sample_dpm_2', @@ -77,8 +76,9 @@ def sample_euler(noise, denoised = model(noise, sigma_hat) x = denoised + sigmas[i + 1] * (gamma + 1) * noise else: - _, c_in = get_scalings(sigma_hat) - denoised = model(x * c_in, sigma_hat) + # _, c_in = get_scalings(sigma_hat) + # denoised = model(x * c_in, sigma_hat) + denoised = model(x, sigmas[i]) d = (x - denoised) / sigma_hat dt = sigmas[i + 1] - sigma_hat x = x + d * dt diff --git a/scepter/modules/model/network/ldm/__init__.py b/scepter/modules/model/network/ldm/__init__.py index 9f8f971..b1f3610 100644 --- a/scepter/modules/model/network/ldm/__init__.py +++ b/scepter/modules/model/network/ldm/__init__.py @@ -2,7 +2,9 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from scepter.modules.model.network.ldm.ldm import LatentDiffusion from scepter.modules.model.network.ldm.ldm_edit import LatentDiffusionEdit +from scepter.modules.model.network.ldm.ldm_pixart import LatentDiffusionPixart from scepter.modules.model.network.ldm.ldm_sce import ( LatentDiffusionSCEControl, LatentDiffusionSCETuning, LatentDiffusionXLSCEControl, LatentDiffusionXLSCETuning) +from scepter.modules.model.network.ldm.ldm_sd3 import LatentDiffusionSD3 from scepter.modules.model.network.ldm.ldm_xl import LatentDiffusionXL diff --git a/scepter/modules/model/network/ldm/ldm.py b/scepter/modules/model/network/ldm/ldm.py index 492039a..2a519dd 100644 --- a/scepter/modules/model/network/ldm/ldm.py +++ b/scepter/modules/model/network/ldm/ldm.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. +import copy import numbers import random from collections import OrderedDict import torch + from scepter.modules.model.network.diffusion.diffusion import GaussianDiffusion from scepter.modules.model.network.diffusion.schedules import noise_schedule from scepter.modules.model.network.train_module import TrainModule @@ -91,8 +93,8 @@ class LatentDiffusion(TrainModule): def init_params(self): self.parameterization = self.cfg.get('PARAMETERIZATION', 'eps') assert self.parameterization in [ - 'eps', 'x0', 'v' - ], 'currently only supporting "eps" and "x0" and "v"' + 'eps', 'x0', 'v', 'rf' + ], 'currently only supporting "eps" and "x0" and "v" and "rf"' self.num_timesteps = self.cfg.get('TIMESTEPS', 1000) self.schedule_args = { @@ -137,15 +139,18 @@ class LatentDiffusion(TrainModule): self.default_n_prompt = '' if self.train_n_prompt is None: self.train_n_prompt = '' - self.use_ema = self.cfg.get('USE_EMA', True) + self.use_ema = self.cfg.get('USE_EMA', False) self.model_ema_config = self.cfg.get('DIFFUSION_MODEL_EMA', None) def construct_network(self): self.model = BACKBONES.build(self.model_config, logger=self.logger) self.logger.info('all parameters:{}'.format(count_params(self.model))) - if self.use_ema and self.model_ema_config: - self.model_ema = BACKBONES.build(self.model_ema_config, - logger=self.logger) + if self.use_ema: + if self.model_ema_config: + self.model_ema = BACKBONES.build(self.model_ema_config, + logger=self.logger) + else: + self.model_ema = copy.deepcopy(self.model) self.model_ema = self.model_ema.eval() for param in self.model_ema.parameters(): param.requires_grad = False @@ -154,13 +159,15 @@ class LatentDiffusion(TrainModule): if self.tokenizer_config is not None: self.tokenizer = TOKENIZERS.build(self.tokenizer_config, logger=self.logger) - - self.first_stage_model = MODELS.build(self.first_stage_config, - logger=self.logger) - self.first_stage_model = self.first_stage_model.eval() - self.first_stage_model.train = disabled_train - for param in self.first_stage_model.parameters(): - param.requires_grad = False + if self.first_stage_config: + self.first_stage_model = MODELS.build(self.first_stage_config, + logger=self.logger) + self.first_stage_model = self.first_stage_model.eval() + self.first_stage_model.train = disabled_train + for param in self.first_stage_model.parameters(): + param.requires_grad = False + else: + self.first_stage_model = None if self.tokenizer_config is not None: self.cond_stage_config.KWARGS = { 'vocab_size': self.tokenizer.vocab_size @@ -269,8 +276,8 @@ class LatentDiffusion(TrainModule): ret = {'loss': loss, 'probe_data': {'prompt': prompt}} return ret - def noise_sample(self, batch_size, h, w, g): - noise = torch.empty(batch_size, 4, h, w, + def noise_sample(self, batch_size, h, w, g, c=4): + noise = torch.empty(batch_size, c, h, w, device=we.device_id).normal_(generator=g) return noise diff --git a/scepter/modules/model/network/ldm/ldm_pixart.py b/scepter/modules/model/network/ldm/ldm_pixart.py new file mode 100644 index 0000000..e338b73 --- /dev/null +++ b/scepter/modules/model/network/ldm/ldm_pixart.py @@ -0,0 +1,272 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +import copy +import numbers +import random +from collections import OrderedDict + +import torch + +from scepter.modules.model.network.diffusion.diffusion import GaussianDiffusion +from scepter.modules.model.network.diffusion.schedules import noise_schedule +from scepter.modules.model.network.ldm import LatentDiffusion +from scepter.modules.model.network.train_module import TrainModule +from scepter.modules.model.registry import (BACKBONES, EMBEDDERS, LOSSES, + MODELS, TOKENIZERS) +from scepter.modules.model.utils.basic_utils import count_params, default +from scepter.modules.utils.config import dict_to_yaml +from scepter.modules.utils.distribute import we +from scepter.modules.utils.file_system import FS + + +def disabled_train(self, mode=True): + """Overwrite model.train with this function to make sure train/eval mode + does not change anymore.""" + return self + + +@MODELS.register_class() +class LatentDiffusionPixart(LatentDiffusion): + para_dict = LatentDiffusion.para_dict + para_dict['DECODER_BIAS'] = {'value': 0, 'description': ''} + + def __init__(self, cfg, logger=None): + super().__init__(cfg, logger=logger) + self.decoder_bias = cfg.get('DECODER_BIAS', 0.5) + + def construct_network(self): + self.model = BACKBONES.build(self.model_config, logger=self.logger) + self.logger.info('all parameters:{}'.format(count_params(self.model))) + if self.use_ema: + self.model_ema = copy.deepcopy(self.model).eval() + for param in self.model_ema.parameters(): + param.requires_grad = False + if self.loss_config: + self.loss = LOSSES.build(self.loss_config, logger=self.logger) + if self.tokenizer_config is not None: + self.tokenizer = TOKENIZERS.build(self.tokenizer_config, + logger=self.logger) + + if self.first_stage_config: + self.first_stage_model = MODELS.build(self.first_stage_config, + logger=self.logger) + self.first_stage_model = self.first_stage_model.eval() + self.first_stage_model.train = disabled_train + for param in self.first_stage_model.parameters(): + param.requires_grad = False + else: + self.first_stage_model = None + if self.tokenizer_config is not None: + self.cond_stage_config.KWARGS = { + 'vocab_size': self.tokenizer.vocab_size + } + if self.cond_stage_config == '__is_unconditional__': + print( + f'Training {self.__class__.__name__} as an unconditional model.' + ) + self.cond_stage_model = None + else: + model = EMBEDDERS.build(self.cond_stage_config, logger=self.logger) + self.cond_stage_model = model.eval().requires_grad_(False) + self.cond_stage_model.train = disabled_train + + def forward_train(self, + image=None, + noise=None, + prompt=None, + label=None, + **kwargs): + n, c, h, w = image.shape + x_start = self.encode_first_stage(image, **kwargs) + t = torch.randint(0, self.num_timesteps, (n, ), + device=x_start.device).long() + ar = torch.tensor([[h / w]], device=we.device_id).repeat(n, 1) + hw = torch.tensor([[h, w]], dtype=torch.float, + device=we.device_id).repeat(n, 1) + context = {} + cont_mask = None + if prompt and self.cond_stage_model: + with torch.autocast(device_type='cuda', + enabled=True, + dtype=torch.bfloat16): + cont, cont_mask = getattr(self.cond_stage_model, + 'encode')(prompt, return_mask=True) + context['crossattn'] = cont.float() + else: + assert label is not None + context['label'] = label + + if 'hint' in kwargs and kwargs['hint'] is not None: + hint = kwargs.pop('hint') + if isinstance(context, dict): + context['hint'] = hint + else: + context = {'crossattn': context, 'hint': hint} + else: + hint = None + if self.min_snr_gamma is not None: + alphas = self.diffusion.alphas.to(we.device_id)[t] + sigmas = self.diffusion.sigmas.pow(2).to(we.device_id)[t] + snrs = (alphas / sigmas).clamp(min=1e-20) + min_snrs = snrs.clamp(max=self.min_snr_gamma) + weights = min_snrs / snrs + else: + weights = 1 + self.register_probe({'snrs_weights': weights}) + + loss = self.diffusion.loss(x0=x_start, + t=t, + model=self.model, + model_kwargs={ + 'cond': context, + 'mask': cont_mask, + 'data_info': { + 'img_hw': hw, + 'aspect_ratio': ar + } + }, + noise=noise, + **kwargs) + loss = loss * weights + loss = loss.mean() + ret = {'loss': loss, 'probe_data': {'prompt': prompt}} + return ret + + @torch.no_grad() + def forward_test(self, + prompt=None, + label=None, + sampler='ddim', + sample_steps=20, + seed=2023, + guide_scale=4.5, + guide_rescale=0.5, + discretization='trailing', + **kwargs): + g = torch.Generator(device=we.device_id) + seed = seed if seed >= 0 else random.randint(0, 2**32 - 1) + g.manual_seed(seed) + num_samples = label.shape[0] if label is not None else len(prompt) + context = {} + null_context = {} + cont_mask = None + if prompt and self.cond_stage_model: + with torch.autocast(device_type='cuda', + enabled=True, + dtype=torch.bfloat16): + cont, cont_mask = getattr(self.cond_stage_model, + 'encode')(prompt, return_mask=True) + context['crossattn'] = cont.float() + null_context['crossattn'] = self.model.y_embedder.y_embedding[ + None].repeat(len(prompt), 1, 1) + else: + assert label is not None + context['label'] = label + null_context['label'] = torch.tensor( + [self.model.num_classes]).repeat(num_samples).to(we.device_id) + + if 'hint' in kwargs and kwargs['hint'] is not None: + hint = kwargs.pop('hint') + if isinstance(context, dict): + context['hint'] = hint + if isinstance(null_context, dict): + null_context['hint'] = hint + else: + hint = None + if 'index' in kwargs: + kwargs.pop('index') + image_size = None + if 'meta' in kwargs: + meta = kwargs.pop('meta') + if 'image_size' in meta: + h = int(meta['image_size'][0][0]) + w = int(meta['image_size'][1][0]) + image_size = [h, w] + if 'image_size' in kwargs: + image_size = kwargs.pop('image_size') + if isinstance(image_size, numbers.Number): + image_size = [image_size, image_size] + if image_size is None: + image_size = [1024, 1024] + height, width = image_size + noise = self.noise_sample(num_samples, height // self.size_factor, + width // self.size_factor, g) + # UNet use input n_prompt + samples = self.diffusion.sample( + solver=sampler, + noise=noise, + model=self.model, + model_kwargs=[{ + 'cond': context, + 'mask': cont_mask, + 'data_info': { + 'img_hw': + torch.tensor([image_size], + dtype=torch.float, + device=we.device_id).repeat(num_samples, 1), + 'aspect_ratio': + torch.tensor([[1.]], device=we.device_id).repeat( + num_samples, 1) + } + }, { + 'cond': null_context, + 'mask': cont_mask, + 'data_info': { + 'img_hw': + torch.tensor([image_size], + dtype=torch.float, + device=we.device_id).repeat(num_samples, 1), + 'aspect_ratio': + torch.tensor([[1.]], device=we.device_id).repeat( + num_samples, 1) + } + }] if guide_scale is not None and guide_scale > 0 else { + 'cond': context, + 'mask': cont_mask, + 'data_info': { + 'img_hw': + torch.tensor([image_size], + dtype=torch.float, + device=we.device_id).repeat(num_samples, 1), + 'aspect_ratio': + torch.tensor([[1.]], device=we.device_id).repeat( + num_samples, 1) + } + }, + cat_uc=False, + steps=sample_steps, + guide_scale=guide_scale, + guide_rescale=guide_rescale, + discretization=discretization, + show_progress=True, + seed=seed, + condition_fn=None, + clamp=None, + percentile=None, + t_max=None, + t_min=None, + discard_penultimate_step=None, + return_intermediate=None, + **kwargs) + x_samples = self.decode_first_stage(samples).float() + x_samples = torch.clamp( + (x_samples + 1.0) / 2.0 + self.decoder_bias / 255, + min=0.0, + max=1.0) + outputs = list() + prompt = label.detach().cpu().numpy().tolist( + ) if prompt is None else prompt + for i, (p, img) in enumerate(zip(prompt, x_samples)): + one_tup = {'prompt': str(p), 'n_prompt': '', 'image': img} + if hint is not None: + one_tup.update({'hint': hint[i]}) + outputs.append(one_tup) + + return outputs + + @staticmethod + def get_config_template(): + return dict_to_yaml('MODEL', + __class__.__name__, + LatentDiffusionPixart.para_dict, + set_name=True) diff --git a/scepter/modules/model/network/ldm/ldm_sd3.py b/scepter/modules/model/network/ldm/ldm_sd3.py new file mode 100644 index 0000000..c711485 --- /dev/null +++ b/scepter/modules/model/network/ldm/ldm_sd3.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +import numbers +import random + +import torch +import torch.nn.functional as F +from scepter.modules.model.network.diffusion.diffusion import \ + GaussianDiffusionRF +from scepter.modules.model.network.diffusion.schedules import noise_schedule +from scepter.modules.model.network.ldm import LatentDiffusion +from scepter.modules.model.registry import MODELS +from scepter.modules.utils.config import dict_to_yaml +from scepter.modules.utils.distribute import we + + +@MODELS.register_class() +class LatentDiffusionSD3(LatentDiffusion): + para_dict = LatentDiffusion.para_dict + + def __init__(self, cfg, logger=None): + super().__init__(cfg, logger=logger) + + self.shift_factor = cfg.get('SHIFT_FACTOR', 0) + self.t_weight_type = cfg.get('T_WEIGHT', 'logit_normal') + self.logit_mean = cfg.get('LOGIT_MEAN', 0.0) + self.logit_std = cfg.get('LOGIT_STD', 1.0) + + def init_params(self): + self.parameterization = self.cfg.get('PARAMETERIZATION', 'rf') + assert self.parameterization in [ + 'eps', 'x0', 'v', 'rf' + ], 'currently only supporting "eps" and "x0" and "v" and "rf"' + self.num_timesteps = self.cfg.get('TIMESTEPS', 1000) + + self.schedule_args = { + k.lower(): v + for k, v in self.cfg.get('SCHEDULE_ARGS', { + 'NAME': 'logsnr_cosine_interp', + 'SCALE_MIN': 2.0, + 'SCALE_MAX': 4.0 + }).items() + } + + self.min_snr_gamma = self.cfg.get('MIN_SNR_GAMMA', None) + + self.zero_terminal_snr = self.cfg.get('ZERO_TERMINAL_SNR', False) + if self.zero_terminal_snr: + assert self.parameterization == 'v', 'Now zero_terminal_snr only support v-prediction mode.' + + self.sigmas = noise_schedule(schedule=self.schedule_args.pop('name'), + n=self.num_timesteps, + zero_terminal_snr=self.zero_terminal_snr, + **self.schedule_args) + + self.diffusion = GaussianDiffusionRF( + sigmas=self.sigmas, prediction_type=self.parameterization) + + self.pretrained_model = self.cfg.get('PRETRAINED_MODEL', None) + self.ignore_keys = self.cfg.get('IGNORE_KEYS', []) + + self.model_config = self.cfg.DIFFUSION_MODEL + self.first_stage_config = self.cfg.FIRST_STAGE_MODEL + self.cond_stage_config = self.cfg.COND_STAGE_MODEL + self.tokenizer_config = self.cfg.get('TOKENIZER', None) + self.loss_config = self.cfg.get('LOSS', None) + + self.scale_factor = self.cfg.get('SCALE_FACTOR', 0.18215) + self.size_factor = self.cfg.get('SIZE_FACTOR', 8) + self.default_n_prompt = self.cfg.get('DEFAULT_N_PROMPT', '') + self.default_n_prompt = '' if self.default_n_prompt is None else self.default_n_prompt + self.p_zero = self.cfg.get('P_ZERO', 0.0) + self.train_n_prompt = self.cfg.get('TRAIN_N_PROMPT', '') + if self.default_n_prompt is None: + self.default_n_prompt = '' + if self.train_n_prompt is None: + self.train_n_prompt = '' + self.use_ema = self.cfg.get('USE_EMA', False) + self.model_ema_config = self.cfg.get('DIFFUSION_MODEL_EMA', None) + + def noise_sample(self, batch_size, h, w, g, c=4): + noise = torch.empty(batch_size, c, h, w, + device=we.device_id).normal_(generator=g) + return noise + + def forward_train(self, image=None, noise=None, prompt=None, **kwargs): + n, c, h, w = image.shape + x_start = self.encode_first_stage(image, **kwargs) + if self.t_weight_type == 'uniform': + t = torch.randint(0, + self.num_timesteps, (n, ), + device=x_start.device).long() + elif self.t_weight_type == 'logit_normal': + density = F.sigmoid( + torch.normal(mean=self.logit_mean, + std=self.logit_std, + size=(n, ), + device=x_start.device)) + t = (density * (self.num_timesteps - 1)).round().long() + sigma = (t + 1) / self.num_timesteps + shift = self.schedule_args['shift'] + if shift > 1.: + sigma = shift * sigma / (1 + (shift - 1) * sigma) + t = sigma * self.num_timesteps + + context = {} + if prompt and self.cond_stage_model: + ctx, pooled = getattr(self.cond_stage_model, 'encode')(prompt) + context['crossattn'] = ctx.float() + context['y'] = pooled + else: + assert False + + if self.min_snr_gamma is not None: + alphas = self.diffusion.alphas.to(we.device_id)[t] + sigmas = self.diffusion.sigmas.pow(2).to(we.device_id)[t] + snrs = (alphas / sigmas).clamp(min=1e-20) + min_snrs = snrs.clamp(max=self.min_snr_gamma) + weights = min_snrs / snrs + else: + weights = 1 + self.register_probe({'snrs_weights': weights}) + + loss = self.diffusion.loss(x0=x_start, + t=t, + model=self.model, + model_kwargs={ + 'cond': context, + }, + noise=noise, + **kwargs) + loss = loss * weights + loss = loss.mean() + ret = {'loss': loss, 'probe_data': {'prompt': prompt}} + return ret + + @torch.no_grad() + def forward_test(self, + prompt=None, + sampler='ddim', + sample_steps=20, + seed=2023, + guide_scale=4.5, + guide_rescale=0.0, + **kwargs): + g = torch.Generator(device=we.device_id) + seed = seed if seed >= 0 else random.randint(0, 2**32 - 1) + g.manual_seed(seed) + num_samples = len(prompt) + context = {} + null_context = {} + + if prompt and self.cond_stage_model: + ctx, pooled = getattr(self.cond_stage_model, 'encode')(prompt) + null_ctx, null_pooled = getattr(self.cond_stage_model, + 'encode')([''] * len(prompt)) + context['crossattn'] = ctx.float() + context['y'] = pooled.float() + null_context['crossattn'] = null_ctx.float() + null_context['y'] = null_pooled.float() + else: + assert False + + if 'index' in kwargs: + kwargs.pop('index') + image_size = None + if 'meta' in kwargs: + meta = kwargs.pop('meta') + if 'image_size' in meta: + h = int(meta['image_size'][0][0]) + w = int(meta['image_size'][1][0]) + image_size = [h, w] + if 'image_size' in kwargs: + image_size = kwargs.pop('image_size') + if isinstance(image_size, numbers.Number): + image_size = [image_size, image_size] + if image_size is None: + image_size = [1024, 1024] + height, width = image_size + noise = self.noise_sample(num_samples, + height // self.size_factor, + width // self.size_factor, + g, + c=16) + # UNet use input n_prompt + samples = self.diffusion.sample( + solver=sampler, + noise=noise, + model=self.model, + model_kwargs=[{ + 'cond': context + }, { + 'cond': null_context + }] if guide_scale is not None and guide_scale > 0 else { + 'cond': context, + }, + cat_uc=False, + steps=sample_steps, + guide_scale=guide_scale, + guide_rescale=guide_rescale, + show_progress=True, + seed=seed, + condition_fn=None, + clamp=None, + percentile=None, + t_max=None, + t_min=None, + discard_penultimate_step=None, + return_intermediate=None, + **kwargs) + x_samples = self.decode_first_stage(samples).float() + x_samples = torch.clamp((x_samples + 1.0) / 2.0, min=0.0, max=1.0) + outputs = list() + + for i, (p, img) in enumerate(zip(prompt, x_samples)): + one_tup = {'prompt': str(p), 'n_prompt': '', 'image': img} + outputs.append(one_tup) + + return outputs + + @staticmethod + def get_config_template(): + return dict_to_yaml('MODEL', + __class__.__name__, + LatentDiffusionSD3.para_dict, + set_name=True) + + @torch.no_grad() + def encode_first_stage(self, x, **kwargs): + z = self.first_stage_model.encode(x) + return self.scale_factor * (z - self.shift_factor) + + @torch.no_grad() + def decode_first_stage(self, z): + z = 1. / self.scale_factor * z + self.shift_factor + return self.first_stage_model.decode(z) diff --git a/scepter/modules/model/tokenizer/tokenizer.py b/scepter/modules/model/tokenizer/tokenizer.py index 92c8d86..b6c9efd 100644 --- a/scepter/modules/model/tokenizer/tokenizer.py +++ b/scepter/modules/model/tokenizer/tokenizer.py @@ -4,7 +4,7 @@ import open_clip from scepter.modules.model.registry import TOKENIZERS from scepter.modules.model.tokenizer import BaseTokenizer from scepter.modules.model.tokenizer.tokenizer_component import ( - basic_clean, whitespace_clean) + basic_clean, canonicalize, heavy_clean, whitespace_clean) from scepter.modules.utils.config import dict_to_yaml from scepter.modules.utils.file_system import FS from transformers import CLIPTokenizer as transformer_clip_tokenizer diff --git a/scepter/modules/model/tokenizer/tokenizer_component.py b/scepter/modules/model/tokenizer/tokenizer_component.py index 75c26eb..c8be2bb 100644 --- a/scepter/modules/model/tokenizer/tokenizer_component.py +++ b/scepter/modules/model/tokenizer/tokenizer_component.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. import html +import string from functools import lru_cache +from urllib import parse import ftfy import regex as re +from bs4 import BeautifulSoup @lru_cache() @@ -59,3 +62,132 @@ def whitespace_clean(text): text = re.sub(r'\s+', ' ', text) text = text.strip() return text + + +def canonicalize(text, keep_punctuation_exact_string=None): + text = text.replace('_', ' ') + if keep_punctuation_exact_string: + text = keep_punctuation_exact_string.join( + part.translate(str.maketrans('', '', string.punctuation)) + for part in text.split(keep_punctuation_exact_string)) + else: + text = text.translate(str.maketrans('', '', string.punctuation)) + text = text.lower() + text = re.sub(r'\s+', ' ', text) + return text.strip() + + +def heavy_clean(text): + text = str(text) + text = parse.unquote_plus(text) + text = text.strip().lower() + text = re.sub('', 'person', text) + + # urls: + text = re.sub( + r'\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))', # noqa: E501 + '', + text) + text = re.sub( + r'\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))', # noqa: E501 + '', + text) + + # html: + text = BeautifulSoup(text, features='html.parser').text + + # @ + text = re.sub(r'@[\w\d]+\b', '', text) + + # 31C0—31EF CJK Strokes + # 31F0—31FF Katakana Phonetic Extensions + # 3200—32FF Enclosed CJK Letters and Months + # 3300—33FF CJK Compatibility + # 3400—4DBF CJK Unified Ideographs Extension A + # 4DC0—4DFF Yijing Hexagram Symbols + # 4E00—9FFF CJK Unified Ideographs + text = re.sub(r'[\u31c0-\u31ef]+', '', text) + text = re.sub(r'[\u31f0-\u31ff]+', '', text) + text = re.sub(r'[\u3200-\u32ff]+', '', text) + text = re.sub(r'[\u3300-\u33ff]+', '', text) + text = re.sub(r'[\u3400-\u4dbf]+', '', text) + text = re.sub(r'[\u4dc0-\u4dff]+', '', text) + text = re.sub(r'[\u4e00-\u9fff]+', '', text) + ####################################################### + + # все виды тире / all types of dash --> "-" + text = re.sub( + r'[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+', # noqa: E501 + '-', + text) + + # кавычки к одному стандарту + text = re.sub(r'[`´«»“”¨]', '"', text) + text = re.sub(r'[‘’]', "'", text) + + # " + text = re.sub(r'"?', '', text) + # & + text = re.sub(r'&', '', text) + + # ip adresses: + text = re.sub(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ' ', text) + + # article ids: + text = re.sub(r'\d:\d\d\s+$', '', text) + + # \n + text = re.sub(r'\\n', ' ', text) + + # "#123" + text = re.sub(r'#\d{1,3}\b', '', text) + # "#12345.." + text = re.sub(r'#\d{5,}\b', '', text) + # "123456.." + text = re.sub(r'\b\d{6,}\b', '', text) + # filenames: + text = re.sub(r'[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)', '', + text) + + text = re.sub(r'[\"\']{2,}', r'"', text) # """AUSVERKAUFT""" + text = re.sub(r'[\.]{2,}', r' ', text) # """AUSVERKAUFT""" + text = re.sub( + re.compile(r'[' + '#®•©™&@·º½¾¿¡§~' + '\)' + '\(' + '\]' + # noqa + '\[' + # noqa + '\}' + '\{' + '\|' + '\\' + '\/' + '\*' + # noqa + r']{1,}'), # noqa + r' ', + text) # ***AUSVERKAUFT***, #AUSVERKAUFT + text = re.sub(r'\s+\.\s+', r' ', text) # " . " + + # this-is-my-cute-cat / this_is_my_cute_cat + regex2 = re.compile(r'(?:\-|\_)') + if len(re.findall(regex2, text)) > 3: + text = re.sub(regex2, ' ', text) + + text = basic_clean(text) + + text = re.sub(r'\b[a-zA-Z]{1,3}\d{3,15}\b', '', text) # jc6640 + text = re.sub(r'\b[a-zA-Z]+\d+[a-zA-Z]+\b', '', text) # jc6640vc + text = re.sub(r'\b\d+[a-zA-Z]+\d+\b', '', text) # 6640vc231 + + text = re.sub(r'(worldwide\s+)?(free\s+)?shipping', '', text) + text = re.sub(r'(free\s)?download(\sfree)?', '', text) + text = re.sub(r'\bclick\b\s(?:for|on)\s\w+', '', text) + text = re.sub(r'\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?', + '', text) + text = re.sub(r'\bpage\s+\d+\b', '', text) + + # j2d1a2a... + text = re.sub(r'\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b', r' ', text) + + text = re.sub(r'\b\d+\.?\d*[xх×]\d+\.?\d*\b', '', text) + text = re.sub(r'\b\s+\:\s+', r': ', text) + text = re.sub(r'(\D[,\./])\b', r'\1 ', text) + text = re.sub(r'\s+', ' ', text) + + text = re.sub(r'^[\"\']([\w\W]+)[\"\']$', r'\1', text) + text = re.sub(r'^[\'\_,\-\:;]', r'', text) + text = re.sub(r'[\'\_,\-\:\-\+]$', r'', text) + text = re.sub(r'^\.\S+$', '', text) + return text.strip() diff --git a/scepter/modules/solver/diffusion_solver.py b/scepter/modules/solver/diffusion_solver.py index 03fba64..8fccecd 100644 --- a/scepter/modules/solver/diffusion_solver.py +++ b/scepter/modules/solver/diffusion_solver.py @@ -4,8 +4,6 @@ import copy import os from collections import OrderedDict, defaultdict -from tqdm import tqdm - import numpy as np import torch import torch.cuda.amp as amp @@ -23,6 +21,7 @@ from torch.distributed.fsdp import (BackwardPrefetch, CPUOffload, FullyShardedDataParallel, MixedPrecision, ShardingStrategy, StateDictType) from torch.nn.parallel import DistributedDataParallel +from tqdm import tqdm sharding_strategy_map = { 'full_shard': ShardingStrategy.FULL_SHARD, @@ -343,9 +342,7 @@ class LatentDiffusionSolver(BaseSolver): self.train_mode() batch_data = next(data_iter) self.before_iter(self.hooks_dict[self._mode]) - if self.sample_args: - batch_data.update(self.sample_args.get_lowercase_dict()) - if 'meta' in batch_data: + if 'meta' in batch_data and isinstance(batch_data['meta'], dict): self.register_probe({ 'data_key': ProbeData(batch_data['meta'].get('data_key', []), @@ -356,6 +353,8 @@ class LatentDiffusionSolver(BaseSolver): 'batch_size': len(batch_data['prompt']) }) self.current_batch_data[self.mode] = batch_data + if self.sample_args: + batch_data.update(self.sample_args.get_lowercase_dict()) with torch.autocast(device_type='cuda', enabled=self.use_amp, dtype=self.dtype): @@ -524,6 +523,8 @@ class LatentDiffusionSolver(BaseSolver): if len(swift_cfg_dict) > 0: from swift import Swift model = Swift.prepare_model(self.model, config=swift_cfg_dict) + + # self.logger.info([(key, param.shape) for key, param in self.model.named_parameters() if param.requires_grad]) return model def freeze(self, freeze_cfg, model=None): diff --git a/scepter/modules/solver/hooks/checkpoint.py b/scepter/modules/solver/hooks/checkpoint.py index 856ceb7..451c01c 100644 --- a/scepter/modules/solver/hooks/checkpoint.py +++ b/scepter/modules/solver/hooks/checkpoint.py @@ -132,8 +132,9 @@ class CheckpointHook(Hook): else: if hasattr(solver, 'save_pretrained'): save_path = osp.join( - solver.work_dir, 'checkpoints/{}-{}-bin'.format( - self.save_name_prefix, solver.total_iter + 1)) + solver.work_dir, + 'checkpoints/{}-{}'.format(self.save_name_prefix, + solver.total_iter + 1)) local_folder, _ = FS.map_to_local(save_path) FS.make_dir(local_folder) ckpt, cfg = solver.save_pretrained() diff --git a/scepter/modules/utils/model.py b/scepter/modules/utils/model.py index 053955d..9ba924f 100644 --- a/scepter/modules/utils/model.py +++ b/scepter/modules/utils/model.py @@ -134,11 +134,11 @@ def load_pretrained_dict(model: torch.nn.Module, missing_keys = load_status.missing_keys err_msgs = [] if unexpected_keys: - err_msgs.append('unexpected key in source ' - f'state_dict: {", ".join(unexpected_keys)}\n') + err_msgs.append('unexpected key in source state_dict: {}\n'.format( + ', '.join(unexpected_keys))) if missing_keys: - err_msgs.append('missing key in source ' - f'state_dict: {", ".join(missing_keys)}\n') + err_msgs.append('missing key in source state_dict: {}\n'.format( + ', '.join(missing_keys))) err_msgs = '\n'.join(err_msgs) if len(err_msgs) > 0: diff --git a/scepter/studio/inference/inference_manager/infer_runer.py b/scepter/studio/inference/inference_manager/infer_runer.py index 6aa4112..769a043 100644 --- a/scepter/studio/inference/inference_manager/infer_runer.py +++ b/scepter/studio/inference/inference_manager/infer_runer.py @@ -2,6 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from scepter.modules.inference.diffusion_inference import DiffusionInference from scepter.modules.inference.largen_inference import LargenInference +from scepter.modules.inference.pixart_inference import PixArtInference +from scepter.modules.inference.sd3_inference import SD3Inference from scepter.modules.inference.stylebooth_inference import StyleboothInference from scepter.modules.utils.logger import get_logger @@ -102,6 +104,10 @@ class PipelineManager(): PipelineBuilder = LargenInference elif pipeline_name.startswith('EDIT'): PipelineBuilder = StyleboothInference + elif pipeline_name.startswith('PIXART'): + PipelineBuilder = PixArtInference + elif pipeline_name.startswith('SD3'): + PipelineBuilder = SD3Inference else: PipelineBuilder = DiffusionInference new_inference = PipelineBuilder(logger=self.logger) diff --git a/scepter/studio/inference/inference_ui/component_names.py b/scepter/studio/inference/inference_ui/component_names.py index 98dab99..7812c1a 100644 --- a/scepter/studio/inference/inference_ui/component_names.py +++ b/scepter/studio/inference/inference_ui/component_names.py @@ -6,9 +6,15 @@ from scepter.modules.utils.file_system import FS def download_image(image): + # return None if image is not None: - name = get_md5(image) - local_path = FS.get_from(image, f'/tmp/gradio/scepter_examples/{name}') + client = FS.get_fs_client(image) + if client.tmp_dir.startswith('/home'): + name = get_md5(image) + local_path = FS.get_from(image, + f'/tmp/gradio/scepter_examples/{name}') + else: + local_path = FS.get_from(image) return local_path else: return image @@ -48,20 +54,20 @@ class ModelManageUIName(): if language == 'en': self.model_block_name = 'Model Management' self.postprocess_model_name = 'Refiners and Tuners' - self.diffusion_model = 'Unet' + self.diffusion_model = 'Model' self.first_stage_model = 'Vae' self.cond_stage_model = 'Condition Model' - self.refine_diffusion_model = 'Refine Unet' + self.refine_diffusion_model = 'Refine Model' self.refine_cond_model = 'Refine Condition Model' self.load_lora_tuner = 'Load Lora Tuner' self.load_swift_tuner = 'Load swift Tuner' elif language == 'zh': self.model_block_name = '模型管理' self.postprocess_model_name = 'Refiners and Tuners' - self.diffusion_model = 'Unet' + self.diffusion_model = 'Model' self.first_stage_model = 'Vae' self.cond_stage_model = 'Condition Model' - self.refine_diffusion_model = 'Refine Unet' + self.refine_diffusion_model = 'Refine Model' self.refine_cond_model = 'Refine Condition Model' self.load_lora_tuner = '加载 Lora 微调模型' self.load_swift_tuner = '加载 Swift 微调模型' @@ -371,6 +377,7 @@ class LargenUIName(): self.masked_image = 'Masked Image' self.preprocess = 'Input Preprocess' self.button_name = 'Data Preprocess' + self.value = 'CenterAround' self.direction = ( 'Instruction: \n\n' 'For customized data: \n\n' @@ -404,6 +411,7 @@ class LargenUIName(): self.masked_image = '掩码图片' self.preprocess = '输入图片预处理' self.button_name = '数据预处理' + self.value = '中心向外' self.direction = ('使用说明：\n' '针对自定义数据：\n' 'a.1）上传背景图片（待编辑）；\n' @@ -435,7 +443,7 @@ class LargenUIName(): None, 1.0, 0.75, - 'CenterAround', + self.value, 1024, 1024 ], @@ -456,7 +464,7 @@ class LargenUIName(): ), 1.0, 0.0, - '', + None, 1024, 1024 ], @@ -473,7 +481,7 @@ class LargenUIName(): None, 1.0, 0.0, - '', + None, 1024, 1024 ], @@ -494,7 +502,7 @@ class LargenUIName(): ), 0.45, 0.0, - '', + None, 1024, 1024 ], diff --git a/scepter/studio/inference/inference_ui/control_ui.py b/scepter/studio/inference/inference_ui/control_ui.py index a065c54..81c7884 100644 --- a/scepter/studio/inference/inference_ui/control_ui.py +++ b/scepter/studio/inference/inference_ui/control_ui.py @@ -75,13 +75,13 @@ class ControlUI(UIBase): self.source_image = gr.Image( label=self.component_names.source_image, type='pil', - tool='editor', + sources=['upload'], interactive=True) with gr.Column(scale=1, min_width=0): self.cond_image = gr.Image( label=self.component_names.cond_image, type='pil', - tool='editor', + sources=['upload'], interactive=True) with gr.Row(): with gr.Column(scale=1, min_width=0): diff --git a/scepter/studio/inference/inference_ui/gallery_ui.py b/scepter/studio/inference/inference_ui/gallery_ui.py index 6eb3c3a..7d2cc6d 100644 --- a/scepter/studio/inference/inference_ui/gallery_ui.py +++ b/scepter/studio/inference/inference_ui/gallery_ui.py @@ -47,12 +47,10 @@ class GalleryUI(UIBase): container=False, autofocus=True, elem_classes='type_row', - submit_on_enter=True, lines=1) with gr.Column(scale=3, min_width=0): self.generate_button = gr.Button( - label='Generate', value=self.component_names.generate, elem_classes='type_row', elem_id='generate_button', @@ -280,7 +278,7 @@ class GalleryUI(UIBase): **kwargs): self.manager = manager self.gen_inputs = list(self.component_mapping.values()) - print(self.gen_inputs, len(self.gen_inputs)) + # print(self.gen_inputs, len(self.gen_inputs)) self.gen_outputs = [ self.before_refine_panel, self.before_refine_gallery, diff --git a/scepter/studio/inference/inference_ui/largen_ui.py b/scepter/studio/inference/inference_ui/largen_ui.py index 4845499..a242abd 100644 --- a/scepter/studio/inference/inference_ui/largen_ui.py +++ b/scepter/studio/inference/inference_ui/largen_ui.py @@ -68,30 +68,29 @@ class LargenUI(UIBase): with gr.Column(scale=2, min_width=0): with gr.Row(): with gr.Column(scale=1, min_width=0): - self.scene_image = gr.Image( + self.scene_image = gr.ImageMask( label=self.component_names.scene_image, type='pil', - tool='sketch', - source='upload', - height=400, + sources=['upload'], + layers=False, interactive=True) self.cache_button = gr.Button( value='Use Last Generated Image', visible=True) with gr.Column(scale=1, min_width=0): - self.subject_image = gr.Image( + self.subject_image = gr.ImageMask( label=self.component_names.subject_image, type='pil', - tool='sketch', - source='upload', - interactive=True, - height=400, - visible=False) + sources=['upload'], + layers=False, + visible=False, + interactive=True) self.gallery = gr.Gallery(label='Image History', value=[], columns=1, rows=1, - height=500) + height=500, + interactive=False) self.clear_button = gr.Button(value='Clear History', visible=True) @@ -255,7 +254,7 @@ class LargenUI(UIBase): queue=False) def read_gallery_image(gallery): - if len(gallery) == 0: + if gallery is None: last_image = None else: last_image = gallery[-1]['name'] @@ -267,7 +266,7 @@ class LargenUI(UIBase): def clear_gallery(image_history, gallery): image_history.clear() - gallery.clear() + gallery = [] return image_history, gallery self.clear_button.click(fn=clear_gallery, @@ -276,8 +275,8 @@ class LargenUI(UIBase): def data_process(scene_image, subject_image, task, image_ratio, out_direction, output_height, output_width): - tar_image = scene_image['image'].convert('RGB') - tar_mask = scene_image['mask'].convert('L') + tar_image = scene_image['background'].convert('RGB') + tar_mask = scene_image['layers'][0].split()[-1].convert('L') tar_image = np.asarray(tar_image) tar_mask = np.asarray(tar_mask) tar_mask = np.where(tar_mask > 128, 1, 0).astype(np.uint8) @@ -288,8 +287,8 @@ class LargenUI(UIBase): output_height, output_width) elif task == 'Subject_Guided_Inpainting': - ref_image = subject_image['image'].convert('RGB') - ref_mask = subject_image['mask'].convert('L') + ref_image = subject_image['background'].convert('RGB') + ref_mask = subject_image['layers'][0].split()[-1].convert('L') ref_image = np.asarray(ref_image) ref_mask = np.asarray(ref_mask) ref_mask = np.where(ref_mask > 128, 1, 0).astype(np.uint8) @@ -298,8 +297,8 @@ class LargenUI(UIBase): 1.3, output_height, output_width) elif task == 'Text_Subject_Guided_Inpainting': - ref_image = subject_image['image'].convert('RGB') - ref_mask = subject_image['mask'].convert('L') + ref_image = subject_image['background'].convert('RGB') + ref_mask = subject_image['layers'][0].split()[-1].convert('L') ref_image = np.asarray(ref_image) ref_mask = np.asarray(ref_mask) ref_mask = np.where(ref_mask > 128, 1, 0).astype(np.uint8) diff --git a/scepter/studio/inference/inference_ui/mantra_ui.py b/scepter/studio/inference/inference_ui/mantra_ui.py index 23bb5a3..dac2b21 100644 --- a/scepter/studio/inference/inference_ui/mantra_ui.py +++ b/scepter/studio/inference/inference_ui/mantra_ui.py @@ -47,8 +47,8 @@ class MantraUI(UIBase): def create_ui(self, *args, **kwargs): self.state = gr.State(value=False) - with gr.Column(equal_height=True, visible=False) as self.tab: - with gr.Row(scale=1): + with gr.Column(visible=False) as self.tab: + with gr.Row(): with gr.Column(scale=1): with gr.Group(visible=True): with gr.Row(equal_height=True): @@ -99,7 +99,7 @@ class MantraUI(UIBase): with gr.Row(equal_height=True): self.style_example = gr.Image( label=self.component_names.style_example, - source='upload', + sources=['upload'], value=None, interactive=False) with gr.Row(equal_height=True): diff --git a/scepter/studio/inference/inference_ui/model_manage_ui.py b/scepter/studio/inference/inference_ui/model_manage_ui.py index 4de6fb5..adbfe9b 100644 --- a/scepter/studio/inference/inference_ui/model_manage_ui.py +++ b/scepter/studio/inference/inference_ui/model_manage_ui.py @@ -142,6 +142,7 @@ class ModelManageUI(UIBase): last_pipeline_ins = self.pipe_manager.pipeline_level_modules[ last_pipline] last_pipeline_ins.dynamic_unload(name='all') + now_pipeline = self.pipe_manager.model_level_info[diffusion_model][ 'pipeline'][0] pipeline_ins = self.pipe_manager.pipeline_level_modules[ @@ -202,9 +203,11 @@ class ModelManageUI(UIBase): value=controller_default), gr.Dropdown(choices=mantra_ui.all_styles.get(now_pipeline, []), value=[]), - gr.Textbox(choices=cur_paras.NEGATIVE_PROMPT.get('VALUES', []), + gr.Textbox(elem_classes=cur_paras.NEGATIVE_PROMPT.get( + 'VALUES', []), value=cur_paras.NEGATIVE_PROMPT.get('DEFAULT', '')), - gr.Textbox(choices=cur_paras.PROMPT_PREFIX.get('VALUES', []), + gr.Textbox(elem_classes=cur_paras.PROMPT_PREFIX.get( + 'VALUES', []), value=cur_paras.PROMPT_PREFIX.get('DEFAULT', '')), gr.Dropdown(choices=[key for key in h_level_dict.keys()], value=default_res[0]), diff --git a/scepter/studio/inference/inference_ui/stylebooth_ui.py b/scepter/studio/inference/inference_ui/stylebooth_ui.py index 334b3a6..141beb4 100644 --- a/scepter/studio/inference/inference_ui/stylebooth_ui.py +++ b/scepter/studio/inference/inference_ui/stylebooth_ui.py @@ -15,7 +15,7 @@ class StyleboothUI(UIBase): def create_ui(self, *args, **kwargs): self.state = gr.State(value=False) - with gr.Column(equal_height=False, visible=False) as self.tab: + with gr.Column(visible=False) as self.tab: with gr.Row(): self.selected_app = gr.Dropdown( label=self.component_names.dropdown_name, @@ -30,7 +30,7 @@ class StyleboothUI(UIBase): self.edit_image = gr.Image( label=self.component_names.source_image, type='pil', - tool='editor', + sources=['upload'], interactive=True) with gr.Column(variant='panel', scale=1, @@ -58,7 +58,6 @@ class StyleboothUI(UIBase): interactive=True, allow_custom_value=True) self.compose_instruction = gr.Button( - label=self.component_names.compose_button, value=self.component_names.compose_button, elem_classes='type_row', elem_id='push') diff --git a/scepter/studio/inference/inference_ui/tuner_ui.py b/scepter/studio/inference/inference_ui/tuner_ui.py index cbe646a..d74c269 100644 --- a/scepter/studio/inference/inference_ui/tuner_ui.py +++ b/scepter/studio/inference/inference_ui/tuner_ui.py @@ -51,8 +51,8 @@ class TunerUI(UIBase): def create_ui(self, *args, **kwargs): self.state = gr.State(value=False) - with gr.Column(equal_height=True, visible=False) as self.tab: - with gr.Row(scale=1): + with gr.Column(visible=False) as self.tab: + with gr.Row(): with gr.Column(variant='panel', scale=1, min_width=0): with gr.Group(visible=True): with gr.Row(equal_height=True): @@ -72,7 +72,6 @@ class TunerUI(UIBase): multiselect=True, interactive=True) self.save_button = gr.Button( - label=self.component_names.save_button, value=self.component_names.save_button, elem_classes='type_row', elem_id='save_button', @@ -96,7 +95,7 @@ class TunerUI(UIBase): with gr.Row(equal_height=True): self.tuner_example = gr.Image( label=self.component_names.tuner_example, - source='upload', + sources=['upload'], value=None, interactive=False) with gr.Row(equal_height=True): diff --git a/scepter/studio/preprocess/caption_editor_ui/create_dataset_ui.py b/scepter/studio/preprocess/caption_editor_ui/create_dataset_ui.py index 91bfee9..fb6ba45 100644 --- a/scepter/studio/preprocess/caption_editor_ui/create_dataset_ui.py +++ b/scepter/studio/preprocess/caption_editor_ui/create_dataset_ui.py @@ -3,12 +3,11 @@ from __future__ import annotations import datetime +import json import os.path from collections import OrderedDict import gradio as gr -from tqdm import tqdm - from scepter.modules.utils.file_system import FS from scepter.studio.preprocess.caption_editor_ui.component_names import \ CreateDatasetUIName @@ -17,6 +16,7 @@ from scepter.studio.preprocess.utils.img2img_data_card import \ from scepter.studio.preprocess.utils.txt2img_data_card import \ Text2ImageDataCard from scepter.studio.utils.uibase import UIBase +from tqdm import tqdm refresh_symbol = '\U0001f504' # 🔄 @@ -144,10 +144,17 @@ class CreateDatasetUI(UIBase): meta_file = os.path.join(one_dir, 'meta.json') if not FS.exists(meta_file): continue + with FS.get_from(meta_file) as local_meta: + meta = json.load(open(local_meta, 'r')) + dataset_type_name = 'scepter_' + meta.get('task_type', '') dataset_type = self.default_dataset_type dataset_cls = self.default_dataset_cls for key, value in self.dataset_type_dict.items(): - if one_dir.split('/')[-1].startswith(key): + if one_dir.endswith('/'): + folder_name = one_dir[:-1].split('/')[-1] + else: + folder_name = one_dir.split('/')[-1] + if folder_name.startswith(key) or dataset_type_name == key: dataset_type = key dataset_cls = value break @@ -164,9 +171,9 @@ class CreateDatasetUI(UIBase): return dataset_list def create_ui(self): - with gr.Box(): + with gr.Group(): gr.Markdown(self.components_name.user_direction) - with gr.Box(): + with gr.Group(): with gr.Row(): self.sys_log = gr.Markdown( self.components_name.system_log.format('')) @@ -518,8 +525,8 @@ class CreateDatasetUI(UIBase): if prev_data_name in dataset_list: dataset_list.remove(prev_data_name) dataset_list.append(user_dataset_name) - self.user_level_dataset_list[ - login_user_name] = dataset_list + self.user_level_dataset_list[login_user_name][ + dataset_type] = dataset_list self.dataset_dict[dataset_type].pop(prev_data_name) self.dataset_dict[dataset_type][ user_dataset_name] = ori_dataset_ins @@ -582,8 +589,11 @@ class CreateDatasetUI(UIBase): user_level_dataset_list = self.load_history( login_user_name=login_user_name) self.user_level_dataset_list.update(user_level_dataset_list) - dataset_list = user_level_dataset_list[login_user_name].get( - trans_dataset_type, []) + if user_level_dataset_list.get(login_user_name) is not None: + dataset_list = user_level_dataset_list[login_user_name].get( + trans_dataset_type, []) + else: + dataset_list = [] return gr.Dropdown( value=dataset_list[-1] if len(dataset_list) > 0 else '', choices=dataset_list), self.components_name.system_log.format( diff --git a/scepter/studio/preprocess/caption_editor_ui/dataset_gallery_ui.py b/scepter/studio/preprocess/caption_editor_ui/dataset_gallery_ui.py index f32cea5..65a093c 100644 --- a/scepter/studio/preprocess/caption_editor_ui/dataset_gallery_ui.py +++ b/scepter/studio/preprocess/caption_editor_ui/dataset_gallery_ui.py @@ -39,13 +39,13 @@ class DatasetGalleryUI(UIBase): self.default_image_format = os.path.splitext( current_info.get('relative_path', ''))[-1] - self.default_edit_image_width = gr.Text(value=current_info.get( - 'edit_width', current_info.get('width', -1))) - self.default_edit_image_height = gr.Text(value=current_info.get( - 'edit_height', current_info.get('height', -1))) - self.default_edit_image_format = gr.Text(value=os.path.splitext( + self.default_edit_image_width = current_info.get( + 'edit_width', current_info.get('width', -1)) + self.default_edit_image_height = current_info.get( + 'edit_height', current_info.get('height', -1)) + self.default_edit_image_format = os.path.splitext( current_info.get('edit_relative_path', - current_info.get('relative_path', '')))[-1]) + current_info.get('relative_path', '')))[-1] self.default_select_index = self.default_dataset.cursor self.default_info = f'{self.default_dataset.cursor + 1}/{len(self.default_dataset)}' @@ -120,6 +120,8 @@ class DatasetGalleryUI(UIBase): elem_id='dataset_tag_editor_dataset_gallery', value=self.default_image_list, selected_index=self.default_select_index, + object_fit='fill', + preview=True, columns=4, visible=False, interactive=False) @@ -162,9 +164,7 @@ class DatasetGalleryUI(UIBase): preview=True, allow_preview=True, object_fit='fill') - # with gr.Column(variant='panel', scale=2, min_width=0): - # with gr.Row(visible=False) as : with gr.Row(visible=False) as self.edit_setting_panel: self.sys_log = gr.Markdown( self.component_names.system_log.format('')) @@ -173,7 +173,7 @@ class DatasetGalleryUI(UIBase): visible=False, scale=1, min_width=0) as self.edit_confirm_panel: - with gr.Box(): + with gr.Group(): with gr.Row(): gr.Markdown(self.component_names.confirm_direction) with gr.Row(): @@ -235,6 +235,7 @@ class DatasetGalleryUI(UIBase): with gr.Column(variant='panel', scale=1, min_width=0): self.upload_image = gr.Image( label=self.component_names.upload_image, + sources=['upload'], type='pil') with gr.Row(): with gr.Column(min_width=0): @@ -251,7 +252,7 @@ class DatasetGalleryUI(UIBase): variant='panel', visible=False, ) as self.image_preprocess_panel: - with gr.Box(): + with gr.Group(): with gr.Column(variant='panel', min_width=0): with gr.Row(): self.image_preprocess_method = gr.Dropdown( @@ -294,7 +295,7 @@ class DatasetGalleryUI(UIBase): image_preprocess_btn) with gr.Row(variant='panel', visible=False) as self.caption_preprocess_panel: - with gr.Box(): + with gr.Group(): with gr.Column(variant='panel', min_width=0): with gr.Row(): self.caption_preprocess_method = gr.Dropdown( diff --git a/scepter/studio/self_train/self_train_ui/component_names.py b/scepter/studio/self_train/self_train_ui/component_names.py index 7b34118..d809d4b 100644 --- a/scepter/studio/self_train/self_train_ui/component_names.py +++ b/scepter/studio/self_train/self_train_ui/component_names.py @@ -112,7 +112,11 @@ class TrainerUIName(): self.train_data = 'Training Data' self.eval_prompts = 'Eval Prompts' self.eval_image = 'Eval Image' - self.training_block = 'Training Parameters' + self.training_block = ''' + ### Training Parameters + ''' + self.model_param = 'Model Parameters' + self.base_param = 'Base Parameters' self.base_model = 'Base Model' self.tuner_name = 'Tuner Method' self.base_model_revision = 'Model Version Number' @@ -175,7 +179,11 @@ class TrainerUIName(): self.train_data = '训练数据' self.eval_prompts = '评测文本' self.eval_image = '评测图片' - self.training_block = '训练参数' + self.training_block = ''' + ### 训练参数 + ''' + self.model_param = '模型参数' + self.base_param = '基本参数' self.base_model = '基础模型' self.tuner_name = '微调方法' self.base_model_revision = '模型版本号' diff --git a/scepter/studio/self_train/self_train_ui/model_ui.py b/scepter/studio/self_train/self_train_ui/model_ui.py index 0852661..871de5b 100644 --- a/scepter/studio/self_train/self_train_ui/model_ui.py +++ b/scepter/studio/self_train/self_train_ui/model_ui.py @@ -89,21 +89,21 @@ class ModelUI(UIBase): return all_gallery_list def create_ui(self, *args, **kwargs): - with gr.Box(): + with gr.Group(): gr.Markdown(self.component_names.output_model_block) with gr.Row(variant='panel', equal_height=True): with gr.Column(scale=7, min_width=0): self.output_model_name = gr.Dropdown( label=self.component_names.output_model_name, choices=self.model_list, - value=self.base_model_info.get('model_default', ''), + value=None, show_label=False, container=False, interactive=True) with gr.Column(scale=2, min_width=0): self.output_ckpt_name = gr.Dropdown( label=self.component_names.output_ckpt_name, - value='', + value=None, choices=[], show_label=False, container=False, @@ -127,7 +127,7 @@ class ModelUI(UIBase): self.confirm_add = gr.Button(value=confirm_symbol) with gr.Row(variant='panel', equal_height=True): with gr.Column(scale=3, min_width=0): - with gr.Box(): + with gr.Group(): self.log_message = gr.Text( placeholder='Please select model' 'or press export button.', diff --git a/scepter/studio/self_train/self_train_ui/trainer_ui.py b/scepter/studio/self_train/self_train_ui/trainer_ui.py index f9488cd..b2ff280 100644 --- a/scepter/studio/self_train/self_train_ui/trainer_ui.py +++ b/scepter/studio/self_train/self_train_ui/trainer_ui.py @@ -8,10 +8,9 @@ import random from collections import OrderedDict import gradio as gr -import yaml - import scepter import torch +import yaml from scepter.modules.utils.file_system import FS from scepter.studio.self_train.scripts.trainer import TrainManager from scepter.studio.self_train.self_train_ui.component_names import \ @@ -98,27 +97,25 @@ class TrainerUI(UIBase): reverse=False)) def create_ui(self): - with gr.Box(): + with gr.Tabs(): with gr.Row(variant='panel', equal_height=True): with gr.Column(scale=1, min_width=0, variant='panel'): gr.Markdown(self.component_names.user_direction) - with gr.Box(): + with gr.Group(): gr.Markdown(self.component_names.train_data) self.data_source = gr.Dropdown( choices=self.component_names.data_source_choices, value=self.component_names.data_source_value, label=self.component_names.data_source_name, - placeholder=self.component_names.data_source_name, interactive=False) self.data_type = gr.Dropdown( choices=[ - self.component_names.data_type_map[key] - for key in self.component_names.data_type_choices + self.component_names.data_type_map[key] for key + in self.component_names.data_type_choices ], value=self.component_names.data_type_map[ self.component_names.data_type_value], label=self.component_names.data_type_name, - placeholder=self.component_names.data_type_name, interactive=False) self.ori_data_name = gr.Textbox( label=self.component_names.ori_data_name, @@ -133,7 +130,7 @@ class TrainerUI(UIBase): ms_data_name_place_hold, visible=False, interactive=False) - with gr.Box(visible=False) as self.ms_data_box: + with gr.Group(visible=False) as self.ms_data_box: with gr.Row(): self.ms_data_space = gr.Textbox( label=self.component_names.ms_data_space, @@ -142,7 +139,7 @@ class TrainerUI(UIBase): label=self.component_names.ms_data_subname, value='default', max_lines=1) - with gr.Box(): + with gr.Group(): gr.Markdown(self.component_names.eval_data) self.eval_prompts = gr.Dropdown( value=None, @@ -161,18 +158,18 @@ class TrainerUI(UIBase): self.eval_image = gr.Image( label=self.component_names.eval_image, type='pil', - tool='editor', + sources=['upload'], interactive=False, visible=False) with gr.Column(scale=2, min_width=0, variant='panel'): - with gr.Box(): - gr.Markdown(self.component_names.training_block) + gr.Markdown(self.component_names.training_block) + with gr.Group(): + gr.Markdown(self.component_names.model_param) with gr.Row(): self.base_model = gr.Dropdown( choices=self.para_data.get( 'model_choices', []), - value=self.para_data.get( - 'model_default', ''), + value=self.para_data.get('model_default', ''), label=self.component_names.base_model, interactive=True) self.base_model_revision = gr.Dropdown( @@ -180,114 +177,114 @@ class TrainerUI(UIBase): 'version_choices', []), value=self.para_data.get( 'version_default', ''), - label=self.component_names. - base_model_revision, + label=self.component_names.base_model_revision, interactive=True) - with gr.Box(): - gr.Markdown(self.component_names.tuner_param) - # with gr.Column(scale=1, min_width=0): - if 'TUNER' in self.para_data: - lora_visible, text_lora_visible, sce_visible = judge_tuner_visible( - self.para_data['TUNER']) - else: - lora_visible, text_lora_visible, sce_visible = False, False, False - with gr.Row(): - self.tuner_name = gr.Dropdown( - choices=self.para_data.get( - 'tuner_choices', []), + with gr.Group(): + gr.Markdown(self.component_names.tuner_param) + # with gr.Column(scale=1, min_width=0): + if 'TUNER' in self.para_data: + lora_visible, text_lora_visible, sce_visible = judge_tuner_visible( + self.para_data['TUNER']) + else: + lora_visible, text_lora_visible, sce_visible = False, False, False + with gr.Row(): + self.tuner_name = gr.Dropdown( + choices=self.para_data.get( + 'tuner_choices', []), + value=self.para_data.get('tuner_default', ''), + label=self.component_names.tuner_name, + interactive=True) + with gr.Row(): + with gr.Row( + visible=lora_visible) as self.lora_param: + self.lora_alpha = gr.Number( + label='LoRA Alpha', value=self.para_data.get( - 'tuner_default', ''), - label=self.component_names.tuner_name, + 'lora_alpha', 256), + interactive=True) + self.lora_rank = gr.Number( + label='LoRA Rank', + value=self.para_data.get('lora_rank', 256), + interactive=True) + with gr.Row(visible=text_lora_visible + ) as self.text_lora_param: + self.text_lora_alpha = gr.Number( + label='Text LoRA Alpha', + value=self.para_data.get( + 'text_lora_alpha', 256), + interactive=True) + self.text_lora_rank = gr.Number( + label='Text LoRA Rank', + value=self.para_data.get( + 'text_lora_rank', 256), interactive=True) - with gr.Row(): - with gr.Row(visible=lora_visible) as self.lora_param: - self.lora_alpha = gr.Number( - label='LoRA Alpha', - value=self.para_data.get( - 'lora_alpha', 256), - interactive=True) - self.lora_rank = gr.Number( - label='LoRA Rank', - value=self.para_data.get( - 'lora_rank', 256), - interactive=True) - with gr.Row(visible=text_lora_visible) as self.text_lora_param: - self.text_lora_alpha = gr.Number( - label='Text LoRA Alpha', - value=self.para_data.get( - 'text_lora_alpha', 256), - interactive=True) - self.text_lora_rank = gr.Number( - label='Text LoRA Rank', - value=self.para_data.get( - 'text_lora_rank', 256), - interactive=True) self.sce_ratio = gr.Slider( label='SCE Ratio', minimum=0.2, maximum=2.0, step=0.1, - value=self.para_data.get( - 'sce_ratio', 1.0), + value=self.para_data.get('sce_ratio', 1.0), interactive=True, visible=sce_visible) - with gr.Box(): - gr.Markdown(self.component_names.resolution_param) + with gr.Group(): + gr.Markdown(self.component_names.resolution_param) + with gr.Row(equal_height=True): + self.resolution_height = gr.Dropdown( + choices=list(self.h_level_dict.keys()), + value=self.para_data.get('RESOLUTION', + 1024)[0], + label=self.component_names.resolution_height, + allow_custom_value=True, + interactive=True) + self.resolution_width = gr.Dropdown( + choices=self.h_level_dict[ + self.resolution_height.value], + value=self.para_data.get('RESOLUTION', + 1024)[1], + label=self.component_names.resolution_width, + allow_custom_value=True, + interactive=True) + self.enable_resolution_bucket = gr.Checkbox( + value=False, + container=True, + interactive=True, + label=self.component_names. + enable_resolution_bucket, + info=self.component_names. + enable_resolution_bucket_ins) + with gr.Column(visible=self.enable_resolution_bucket. + value) as self.resolution_bucket_param: + with gr.Row(): + self.min_bucket_resolution = gr.Number( + label=self.component_names. + min_bucket_resolution, + value=self.para_data.get( + 'min_bucket_resolution', 256), + interactive=True) + self.max_bucket_resolution = gr.Number( + label=self.component_names. + max_bucket_resolution, + value=self.para_data.get( + 'max_bucket_resolution', 1024), + interactive=True) with gr.Row(equal_height=True): - self.resolution_height = gr.Dropdown( - choices=list(self.h_level_dict.keys()), - value=self.para_data.get( - 'RESOLUTION', 1024)[0], + self.bucket_resolution_steps = gr.Number( label=self.component_names. - resolution_height, - allow_custom_value=True, - interactive=True) - self.resolution_width = gr.Dropdown( - choices=self.h_level_dict[ - self.resolution_height.value], + bucket_resolution_steps, value=self.para_data.get( - 'RESOLUTION', 1024)[1], - label=self.component_names. - resolution_width, - allow_custom_value=True, + 'bucket_resolution_steps', 64), interactive=True) - with gr.Box(): - self.enable_resolution_bucket = gr.Checkbox( + with gr.Group(): + self.bucket_no_upscale = gr.Checkbox( value=False, container=True, interactive=True, - label=self.component_names.enable_resolution_bucket, - info=self.component_names.enable_resolution_bucket_ins) - with gr.Column(visible=self.enable_resolution_bucket. - value) as self.resolution_bucket_param: - with gr.Row(): - self.min_bucket_resolution = gr.Number( label=self.component_names. - min_bucket_resolution, - value=self.para_data.get( - 'min_bucket_resolution', 256), - interactive=True) - self.max_bucket_resolution = gr.Number( - label=self.component_names. - max_bucket_resolution, - value=self.para_data.get( - 'max_bucket_resolution', 1024), - interactive=True) - with gr.Row(equal_height=True): - self.bucket_resolution_steps = gr.Number( - label=self.component_names. - bucket_resolution_steps, - value=self.para_data.get( - 'bucket_resolution_steps', 64), - interactive=True) - with gr.Box(): - self.bucket_no_upscale = gr.Checkbox( - value=False, - container=True, - interactive=True, - label=self.component_names.bucket_no_upscale, - info=self.component_names.bucket_no_upscale_ins) - + bucket_no_upscale, + info=self.component_names. + bucket_no_upscale_ins) + with gr.Group(): + gr.Markdown(self.component_names.base_param) with gr.Row(): self.train_epoch = gr.Number( label=self.component_names.train_epoch, @@ -303,13 +300,11 @@ class TrainerUI(UIBase): with gr.Row(): self.save_interval = gr.Number( label=self.component_names.save_interval, - value=self.para_data.get( - 'SAVE_INTERVAL', 10), + value=self.para_data.get('SAVE_INTERVAL', 10), precision=0, interactive=True) self.train_batch_size = gr.Number( - label=self.component_names. - train_batch_size, + label=self.component_names.train_batch_size, value=self.para_data.get( 'TRAIN_BATCH_SIZE', 4), precision=0, @@ -318,14 +313,12 @@ class TrainerUI(UIBase): with gr.Row(): self.prompt_prefix = gr.Text( label=self.component_names.prompt_prefix, - value=self.para_data.get( - 'TRAIN_PREFIX', '')) + value=self.para_data.get('TRAIN_PREFIX', '')) self.replace_keywords = gr.Text( - label=self.component_names. - replace_keywords, + label=self.component_names.replace_keywords, value='') with gr.Row(variant='panel', equal_height=True): - with gr.Box(): + with gr.Group(): with gr.Row(variant='panel', equal_height=True): gr.Markdown(self.component_names.work_name) with gr.Row(variant='panel', equal_height=True): @@ -386,9 +379,9 @@ class TrainerUI(UIBase): self.component_names.data_source_choices[0], self.component_names.data_source_choices[2] ]: - return gr.Box(visible=False) + return gr.Group(visible=False) elif data_source == self.component_names.data_source_choices[1]: - return gr.Box(visible=True) + return gr.Group(visible=True) self.data_source.change(fn=change_data_source, inputs=[self.data_source], @@ -826,8 +819,10 @@ class TrainerUI(UIBase): cfg = current_val # update config cfg['SOLVER']['WORK_DIR'] = work_dir + # cfg['SOLVER']['OPTIMIZER']['LEARNING_RATE'] = float( + # learning_rate * 640 / int(train_batch_size)) cfg['SOLVER']['OPTIMIZER']['LEARNING_RATE'] = float( - learning_rate * 640 / int(train_batch_size)) + learning_rate) cfg['SOLVER']['MAX_EPOCHS'] = int(train_epoch) cfg['SOLVER']['TRAIN_DATA']['BATCH_SIZE'] = int( train_batch_size) @@ -903,8 +898,8 @@ class TrainerUI(UIBase): if work_name not in inference_ui.model_list: inference_ui.model_list.append(work_name) gr.Info('Start Training!' + message) - return gr.Dropdown.update(choices=inference_ui.model_list, - value=work_name) + return gr.Dropdown(choices=inference_ui.model_list, + value=work_name) self.training_button.click( run_train, diff --git a/scepter/studio/tuner_manager/manager_ui/browser_ui.py b/scepter/studio/tuner_manager/manager_ui/browser_ui.py index d485e67..c88cdf4 100644 --- a/scepter/studio/tuner_manager/manager_ui/browser_ui.py +++ b/scepter/studio/tuner_manager/manager_ui/browser_ui.py @@ -5,9 +5,8 @@ import shutil from collections import OrderedDict import gradio as gr -from huggingface_hub import HfApi, snapshot_download - import scepter +from huggingface_hub import HfApi, snapshot_download from scepter.modules.utils.config import Config from scepter.modules.utils.file_system import FS from scepter.modules.utils.module_transform import \ @@ -95,7 +94,7 @@ class BrowserUI(UIBase): diffusion_models_choice, diffusion_model, tuner_models_choice, tuner_model = self.get_choices_and_values( ) with gr.Column(): - with gr.Box(): + with gr.Group(): gr.Markdown(self.component_names.browser_block_name) with gr.Row(variant='panel', equal_height=True): with gr.Column(scale=4, min_width=0): @@ -114,38 +113,33 @@ class BrowserUI(UIBase): interactive=True) with gr.Column(scale=1, min_width=0): self.save_button = gr.Button( - label='Save', value=self.component_names.save_symbol, elem_classes='type_row', elem_id='save_button', visible=True) self.delete_button = gr.Button( - label='Delete', value=self.component_names.delete_symbol, elem_classes='type_row', elem_id='delete_button', visible=False) self.model_export = gr.Button( - label='Model Export', value=self.component_names.model_export, elem_classes='type_row', elem_id='save_button', visible=True) with gr.Column(scale=1, min_width=0): self.refresh_button = gr.Button( - label='Refresh', value=self.component_names.refresh_symbol, elem_classes='type_row', elem_id='refresh_button', visible=True) self.model_import = gr.Button( - label='Model Import', value=self.component_names.model_import, elem_classes='type_row', elem_id='save_button', visible=True) - with gr.Box(visible=False) as self.export_setting: + with gr.Group(visible=False) as self.export_setting: gr.Markdown(self.component_names.export_desc) with gr.Column(variant='panel'): with gr.Row(equal_height=True): @@ -157,7 +151,6 @@ class BrowserUI(UIBase): show_label=False) with gr.Column(scale=1, min_width=0): self.export_close = gr.Button( - label='Close Export', value=self.component_names.close, elem_classes='type_row', elem_id='save_button') @@ -185,7 +178,6 @@ class BrowserUI(UIBase): value=False) with gr.Column(scale=1, min_width=0): self.ms_export_submit = gr.Button( - label='Submit MS', value=self.component_names.submit, elem_classes='type_row', elem_id='save_button') @@ -213,12 +205,11 @@ class BrowserUI(UIBase): value=False) with gr.Column(scale=1, min_width=0): self.hf_export_submit = gr.Button( - label='Submit HF', value=self.component_names.submit, elem_classes='type_row', elem_id='save_button') - with gr.Box(visible=False) as self.import_setting: + with gr.Group(visible=False) as self.import_setting: gr.Markdown(self.component_names.import_desc) with gr.Column(variant='panel'): with gr.Row(equal_height=True): @@ -232,21 +223,20 @@ class BrowserUI(UIBase): show_label=False) with gr.Column(scale=1, min_width=0): self.import_close = gr.Button( - label='Close Download', value=self.component_names.close, elem_classes='type_row', elem_id='save_button') with gr.Row( equal_height=True) as self.ms_import_setting: - with gr.Column(scale=4.5, min_width=0): + with gr.Column(scale=4, min_width=0): self.ms_modelid = gr.Text( label=self.component_names.ms_modelid, show_label=False, container=False, placeholder='ModelScope Model Path', value='') - with gr.Column(scale=4.5, min_width=0): + with gr.Column(scale=4, min_width=0): self.ms_import_username = gr.Text( label=self.component_names.ms_username, show_label=False, @@ -255,7 +245,6 @@ class BrowserUI(UIBase): value='') with gr.Column(scale=1, min_width=0): self.ms_import_submit = gr.Button( - label='Submit MS', value=self.component_names.submit, elem_classes='type_row', elem_id='save_button') @@ -285,7 +274,6 @@ class BrowserUI(UIBase): value='') with gr.Column(scale=1, min_width=0): self.hf_import_submit = gr.Button( - label='Submit HF', value=self.component_names.submit, elem_classes='type_row', elem_id='save_button') @@ -333,7 +321,6 @@ class BrowserUI(UIBase): placeholder='Upload Tuner Name', value='') self.local_upload_bt = gr.Button( - label='Submit Local Model', value=self.component_names.submit, elem_classes='type_row', elem_id='upload_button') @@ -358,7 +345,8 @@ class BrowserUI(UIBase): tar_path = os.path.join(tar_path, tuner_name) if FS.exists(tar_path): raise gr.Error(self.component_names.same_name) - local_model_dir, _ = FS.map_to_local(tar_path) + # local_model_dir, _ = FS.map_to_local(tar_path) + local_model_dir = src_path FS.put_dir_from_local_dir(src_path, tar_path) # save image tuner_example_path = None diff --git a/scepter/studio/tuner_manager/manager_ui/info_ui.py b/scepter/studio/tuner_manager/manager_ui/info_ui.py index bcbcd84..0e92315 100644 --- a/scepter/studio/tuner_manager/manager_ui/info_ui.py +++ b/scepter/studio/tuner_manager/manager_ui/info_ui.py @@ -43,7 +43,7 @@ class InfoUI(UIBase): def create_ui(self, *args, **kwargs): with gr.Column(): - with gr.Box(): + with gr.Group(): gr.Markdown(self.component_names.info_block_name) with gr.Row(variant='panel', equal_height=True): with gr.Column(variant='panel', scale=1, min_width=0): @@ -80,7 +80,6 @@ class InfoUI(UIBase): with gr.Column(scale=1): self.save_bt2 = gr.Button( value=self.component_names.save_symbol, - label='Save', elem_classes='type_row', elem_id='save_button', visible=True, @@ -91,7 +90,7 @@ class InfoUI(UIBase): with gr.Row(equal_height=True): self.tuner_example = gr.Image( label=self.component_names.tuner_example, - source='upload', + sources=['upload'], value=None, interactive=True) with gr.Row(equal_height=True): @@ -116,7 +115,6 @@ class InfoUI(UIBase): self.component_names.go_to_inference) with gr.Column(scale=1, min_width=0): self.local_download_bt = gr.Button( - label='Download to Local Dir', value=self.component_names. download_to_local, # elem_classes='type_row', @@ -134,7 +132,6 @@ class InfoUI(UIBase): show_label=False) with gr.Column(scale=1, min_width=0): self.download_confirm = gr.Button( - label='Confirm Download Format', value=self.component_names.submit, elem_classes='type_row', elem_id='save_button') diff --git a/scepter/tools/convert_tuner.py b/scepter/tools/convert_tuner.py index 7b91a81..59a42b3 100644 --- a/scepter/tools/convert_tuner.py +++ b/scepter/tools/convert_tuner.py @@ -2,8 +2,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import argparse import importlib -# from safetensors.torch import save_file -# from safetensors import safe_open import json import os import sys diff --git a/scepter/tools/process_watcher.py b/scepter/tools/process_watcher.py index b802c43..7bc42b7 100644 --- a/scepter/tools/process_watcher.py +++ b/scepter/tools/process_watcher.py @@ -16,7 +16,7 @@ parser.add_argument( '--python_engine', dest='python_engine', help='the engine path of python interpreter!', - default='', + default='python', type=str, ) @@ -24,7 +24,7 @@ parser.add_argument( '--script', dest='script', help='the script to run!', - default='', + default='main_mmpose.py', type=str, ) parser.add_argument( diff --git a/scepter/tools/webui.py b/scepter/tools/webui.py index 086ed5d..b055240 100644 --- a/scepter/tools/webui.py +++ b/scepter/tools/webui.py @@ -171,5 +171,4 @@ if __name__ == '__main__': root_path=config['ROOT'], show_error=True, debug=True, - enable_queue=True, auth=check_auth if len(auth_info) > 0 else None) diff --git a/scepter/version.py b/scepter/version.py index 5a93d5e..5205e57 100644 --- a/scepter/version.py +++ b/scepter/version.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -__version__ = '1.0.2' +__version__ = '1.0.3' version_info = tuple(int(x) for x in __version__.split('.')[0:3]) diff --git a/tests/modules/test_diffusion_inference.py b/tests/modules/test_diffusion_inference.py index fc95c08..1396f2d 100644 --- a/tests/modules/test_diffusion_inference.py +++ b/tests/modules/test_diffusion_inference.py @@ -10,6 +10,7 @@ import torchvision.transforms.functional as TF from PIL import Image from scepter.modules.annotator.registry import ANNOTATORS from scepter.modules.inference.diffusion_inference import DiffusionInference +from scepter.modules.inference.sd3_inference import SD3Inference from scepter.modules.inference.stylebooth_inference import StyleboothInference from scepter.modules.utils.config import Config from scepter.modules.utils.distribute import we @@ -98,7 +99,7 @@ class DiffusionInferenceTest(unittest.TestCase): save_path = os.path.join(self.tmp_dir, 'sd15_flower_2d.png') save_image(output['images'], save_path) - # @unittest.skip('') + @unittest.skip('') def test_sd21_scedit_ctr_canny(self): # init model config_file = 'scepter/methods/studio/inference/stable_diffusion/sd21_pro.yaml' @@ -225,6 +226,20 @@ class DiffusionInferenceTest(unittest.TestCase): 'stylebooth_test_lowpoly_cute_dog.png') save_image(output['images'], save_path) + # @unittest.skip('') + def test_sd3(self): + config_file = 'scepter/methods/studio/inference/dit/sd3_pro.yaml' + cfg = Config(cfg_file=config_file) + diff_infer = SD3Inference(logger=self.logger) + diff_infer.init_from_cfg(cfg) + output = diff_infer({ + 'prompt': 'a cat holds a blackboard that writes "hello world"', + 'seed': 2024 + }) + save_path = os.path.join(self.tmp_dir, 'sd3_cat.png') + save_image(output['images'], save_path) + print(save_path) + if __name__ == '__main__': unittest.main()