Merge pull request #43 from githubcto/main

Update tokenization_chatglm.py Fix TypeError: ChatGLMTokenizer._pad()…
This commit is contained in:
Jukka Seppänen
2024-10-18 11:47:45 +03:00
committed by GitHub
+1
View File
@@ -243,6 +243,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
pad_to_multiple_of: Optional[int] = None, pad_to_multiple_of: Optional[int] = None,
return_attention_mask: Optional[bool] = None, return_attention_mask: Optional[bool] = None,
padding_side: Optional[str] = None,
) -> dict: ) -> dict:
""" """
Pad encoded inputs (on left/right and up to predefined length or max length in the batch) Pad encoded inputs (on left/right and up to predefined length or max length in the batch)