pre tokenizer configurations are removed

2024-11-17 22:37:50 +03:00
parent ca07640563
commit 917898ec50
1 changed files with 0 additions and 3 deletions
@@ -133,9 +133,6 @@ def get_tokenizer(model_name: str):
    except:
        tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)

-    tokenizer.pad_token_id = tokenizer.eos_token_id
-    tokenizer.padding_side = "left"
-
    return tokenizer