v1.0.3 update

2024-07-18 14:12:42 +08:00
parent 7a9f90efb2
commit 01fd8335af
94 changed files with 8776 additions and 417 deletions
@@ -4,7 +4,7 @@ import open_clip
 from scepter.modules.model.registry import TOKENIZERS
 from scepter.modules.model.tokenizer import BaseTokenizer
 from scepter.modules.model.tokenizer.tokenizer_component import (
-    basic_clean, whitespace_clean)
+    basic_clean, canonicalize, heavy_clean, whitespace_clean)
 from scepter.modules.utils.config import dict_to_yaml
 from scepter.modules.utils.file_system import FS
 from transformers import CLIPTokenizer as transformer_clip_tokenizer