From aa016d91de532c25a00ee66296ff62f7bd0220e4 Mon Sep 17 00:00:00 2001 From: Gang Wen Date: Tue, 7 Apr 2026 19:59:26 +0200 Subject: [PATCH] Add slow tokenizer fallback for models with incompatible added_tokens Some Florence2 variants (e.g. Florence-2-Flux-Large) have added_tokens in a format that fast tokenizers in transformers 5.x cannot parse. Fall back to slow tokenizer on TypeError. Co-Authored-By: Claude Opus 4.6 --- py/florence2_ultra.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/py/florence2_ultra.py b/py/florence2_ultra.py index bc302fd..bba8498 100644 --- a/py/florence2_ultra.py +++ b/py/florence2_ultra.py @@ -108,7 +108,11 @@ def _load_model_v5(model_path, attention, dtype): image_processor.image_seq_length = 577 log(f"[DEBUG] Loading tokenizer from {model_path}") - tokenizer = AutoTokenizer.from_pretrained(model_path) + try: + tokenizer = AutoTokenizer.from_pretrained(model_path) + except TypeError as e: + log(f"[DEBUG] Fast tokenizer failed ({e}), trying slow tokenizer", message_type='warning') + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) log(f"[DEBUG] Creating Florence2Processor") processor = Florence2Processor(image_processor=image_processor, tokenizer=tokenizer) log(f"[DEBUG] _load_model_v5 completed successfully")