tokenizer init except bug is fixed

2024-04-07 19:01:59 +03:00
parent 0754aa840c
commit efd833ea97
1 changed files with 1 additions and 1 deletions
@@ -34,7 +34,7 @@ def get_tokenizer(model_name: str):
    try:
        tokenizer = AutoTokenizer.from_pretrained(model_name)
    except:
-        tokenizer = AutoModelForCausalLM.from_pretrained(model_name, use_fast=False)
+        tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)

    return tokenizer