91 lines
4.7 KiB
Python
91 lines
4.7 KiB
Python
# These codes are copied from modelscope revision c58451baead80d83281f063d12fb377fad415257
|
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
|
|
import os
|
|
|
|
from transformers import XLMRobertaTokenizer
|
|
|
|
SPIECE_UNDERLINE = '▁'
|
|
|
|
|
|
class VLDocXLMTokenizer(XLMRobertaTokenizer):
|
|
"""
|
|
Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
|
|
[SentencePiece](https://github.com/google/sentencepiece).
|
|
|
|
This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
|
|
this superclass for more information regarding those methods.
|
|
|
|
Args:
|
|
vocab_file (`str`):
|
|
Path to the vocabulary file.
|
|
bos_token (`str`, *optional*, defaults to `"<s>"`):
|
|
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
|
|
|
|
<Tip>
|
|
|
|
When building a sequence using special tokens, this is not the token that is used for the beginning of
|
|
sequence. The token used is the `cls_token`.
|
|
|
|
</Tip>
|
|
|
|
eos_token (`str`, *optional*, defaults to `"</s>"`):
|
|
The end of sequence token.
|
|
|
|
<Tip>
|
|
|
|
When building a sequence using special tokens, this is not the token that is used for the end of sequence.
|
|
The token used is the `sep_token`.
|
|
|
|
</Tip>
|
|
|
|
sep_token (`str`, *optional*, defaults to `"</s>"`):
|
|
The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
|
|
sequence classification or for a text and a question for question answering. It is also used as the last
|
|
token of a sequence built with special tokens.
|
|
cls_token (`str`, *optional*, defaults to `"<s>"`):
|
|
The classifier token which is used when doing sequence classification (classification of the whole sequence
|
|
instead of per-token classification). It is the first token of the sequence when built with special tokens.
|
|
unk_token (`str`, *optional*, defaults to `"<unk>"`):
|
|
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
|
|
token instead.
|
|
pad_token (`str`, *optional*, defaults to `"<pad>"`):
|
|
The token used for padding, for example when batching sequences of different lengths.
|
|
mask_token (`str`, *optional*, defaults to `"<mask>"`):
|
|
The token used for masking values. This is the token used when training this model with masked language
|
|
modeling. This is the token which the model will try to predict.
|
|
cls_token_box (`List[int]`, *optional*, defaults to `[0, 0, 0, 0]`):
|
|
The bounding box to use for the special [CLS] token.
|
|
sep_token_box (`List[int]`, *optional*, defaults to `[1000, 1000, 1000, 1000]`):
|
|
The bounding box to use for the special [SEP] token.
|
|
pad_token_box (`List[int]`, *optional*, defaults to `[0, 0, 0, 0]`):
|
|
The bounding box to use for the special [PAD] token.
|
|
pad_token_label (`int`, *optional*, defaults to -100):
|
|
The label to use for padding tokens. Defaults to -100, which is the `ignore_index` of PyTorch's
|
|
CrossEntropyLoss.
|
|
only_label_first_subword (`bool`, *optional*, defaults to `True`):
|
|
Whether or not to only label the first subword, in case word labels are provided.
|
|
additional_special_tokens (`List[str]`, *optional*, defaults to `["<s>NOTUSED", "</s>NOTUSED"]`):
|
|
Additional special tokens used by the tokenizer.
|
|
sp_model_kwargs (`dict`, *optional*):
|
|
Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
|
|
SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
|
|
to set:
|
|
|
|
- `enable_sampling`: Enable subword regularization.
|
|
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
|
|
|
|
- `nbest_size = {0,1}`: No sampling is performed.
|
|
- `nbest_size > 1`: samples from the nbest_size results.
|
|
- `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
|
|
using forward-filtering-and-backward-sampling algorithm.
|
|
|
|
- `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
|
|
BPE-dropout.
|
|
|
|
Attributes:
|
|
sp_model (`SentencePieceProcessor`):
|
|
The *SentencePiece* processor that is used for every conversion (string, tokens and IDs).
|
|
"""
|
|
model_input_names = ['input_ids', 'attention_mask']
|