fix(studio): right tokenizer family, and never the network first

MiniMax H3 encodes with Qwen3-VL; the stand-in path fetched a 12B Gemma
tokenizer regardless. Local HF cache is now tried before any download,
and a download announces itself — it blocks the run with no timeout.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

CHANGELOG.md

@@ -3,6 +3,13 @@
 ## [Unreleased]
 
 ### Fixed
+- **The stand-in tokenizer no longer downloads Gemma for a Qwen model, or reaches the
+  network mid-run.** With no CLIP wired, Studio fetched a 12B *Gemma* tokenizer whatever the
+  model was — wrong vocabulary for MiniMax H3, which encodes with Qwen3-VL. There is now an
+  H3 entry, and every source is tried from the local HuggingFace cache FIRST. Only if nothing
+  is cached does it go online, and it says so: `from_pretrained` has no token, no timeout and
+  no progress, and it runs on ComfyUI's execution thread, so an uncached fetch is an
+  unbounded stall in the middle of a generation with nothing in the log to explain it.
 - **A LoRA weight that cannot fit is now dropped, not attempted.** FunPack already detected
   and reported shape mismatches, then handed them to ComfyUI anyway — which builds the full
   `lora_A @ lora_B` delta and only then discovers it will not reshape into the weight. On a

conditioning.py

@@ -1191,6 +1191,13 @@ class FunPackVideoRefiner:
                 "use_fast": True,
             }),
         ],
+        # MiniMax H3 encodes with Qwen3-VL, not Gemma. Falling through to the ltx2 entry
+        # downloaded a 12B Gemma tokenizer for a Qwen model: every token span it produced was
+        # measured with the wrong vocabulary, and it fetched ~50 MB over the network to do it.
+        "minimax_h3": [
+            ("Qwen/Qwen3-VL-32B-Instruct", {"use_fast": True}),
+            ("Qwen/Qwen3-32B", {"use_fast": True}),
+        ],
         "wan": [
             ("Wan-AI/Wan2.2-T2V-A14B", {
                 "subfolder": "google/umt5-xxl",
@@ -1287,12 +1294,25 @@ class FunPackVideoRefiner:
             return cls._tokenizers[mode]
 
         sources = cls._get_tokenizer_sources(mode)
-        for model_id, kwargs in sources:
-            try:
-                cls._tokenizers[mode] = AutoTokenizer.from_pretrained(model_id, **kwargs)
-                return cls._tokenizers[mode]
-            except Exception as e:
-                print(f"[FunPackVideoRefiner] Tokenizer load failed for mode '{mode}' from '{model_id}': {e}")
+        # Local cache FIRST, every source, before any network call. `from_pretrained` reaches
+        # HuggingFace with no token, no timeout and no progress, on ComfyUI's execution
+        # thread — mid-run that is an unbounded stall with nothing in the log to explain it,
+        # and on a rate-limited unauthenticated request it can be a long one. A tokenizer
+        # that is already on disk should never pay for that.
+        for local_only in (True, False):
+            for model_id, kwargs in sources:
+                if not local_only:
+                    print(f"[FunPackVideoRefiner] tokenizer for '{mode}' is not in the local "
+                          f"HuggingFace cache — DOWNLOADING '{model_id}'. This blocks the run "
+                          f"until it finishes; set HF_TOKEN to avoid unauthenticated rate limits.")
+                try:
+                    cls._tokenizers[mode] = AutoTokenizer.from_pretrained(
+                        model_id, **dict(kwargs, local_files_only=local_only))
+                    return cls._tokenizers[mode]
+                except Exception as e:
+                    if not local_only:
+                        print(f"[FunPackVideoRefiner] Tokenizer load failed for mode '{mode}' "
+                              f"from '{model_id}': {e}")
 
         return None
 
@@ -7057,11 +7077,15 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner):
             encode_cache[cache_key] = result
         return result
 
-    def _v2_text_tokenizer_status(self):
+    def _v2_text_tokenizer_status(self, mode="ltx2"):
         """Only reached with no CLIP connected, so the encoder's own tokenizer is out of reach
         and a downloaded stand-in is all there is. Says which one, because on LTX-2.5 it is the
-        wrong family and the concept spans it produces are approximate."""
-        tokenizer = self._get_tokenizer("ltx2")
+        wrong family and the concept spans it produces are approximate.
+
+        `mode` is the model family. It used to be hardcoded to "ltx2", which fetched a 12B
+        Gemma tokenizer for a MiniMax H3 run — wrong vocabulary, and a network round trip in
+        the middle of a generation to get it."""
+        tokenizer = self._get_tokenizer(mode)
         if tokenizer is None:
             return "no text tokenizer available"
         source = str(getattr(tokenizer, "name_or_path", "") or "").strip()

tests/test_encoder_family.py

@@ -189,3 +189,54 @@ def test_no_vision_when_the_encoder_has_none():
     outer = types.SimpleNamespace(gemma3_12b=types.SimpleNamespace(transformer=types.SimpleNamespace()))
     assert FunPackVideoRefinerV2._encoder_has_vision(types.SimpleNamespace(cond_stage_model=outer)) is False
     assert FunPackVideoRefinerV2._encoder_has_vision(types.SimpleNamespace()) is False
+
+
+# ── the tokenizer must not reach the network mid-run ──────────────────────────
+# `from_pretrained` calls HuggingFace with no token, no timeout and no progress, on
+# ComfyUI's execution thread. Mid-generation that is an unbounded stall with nothing in the
+# log to explain it — and unauthenticated requests are rate limited.
+
+
+def _tok_calls(monkeypatch, results):
+    """Record every from_pretrained call; `results` maps (id, local_only) -> value or Exception."""
+    calls = []
+
+    def fake(model_id, **kw):
+        local = kw.get("local_files_only", False)
+        calls.append((model_id, local))
+        out = results.get((model_id, local), FileNotFoundError("not cached"))
+        if isinstance(out, Exception):
+            raise out
+        return out
+
+    monkeypatch.setattr("conditioning.AutoTokenizer.from_pretrained", fake)
+    FunPackVideoRefinerV2._tokenizers.clear()
+    return calls
+
+
+def test_the_local_cache_is_tried_before_any_network_call(monkeypatch):
+    sentinel = object()
+    calls = _tok_calls(monkeypatch, {("DreamFast/gemma-3-12b-it-heretic-v2", True): sentinel})
+    assert FunPackVideoRefinerV2._get_tokenizer("ltx2") is sentinel
+    assert calls == [("DreamFast/gemma-3-12b-it-heretic-v2", True)]   # never went online
+
+
+def test_it_falls_back_to_the_network_only_when_nothing_is_cached(monkeypatch, capsys):
+    sentinel = object()
+    calls = _tok_calls(monkeypatch, {("DreamFast/gemma-3-12b-it-heretic-v2", False): sentinel})
+    assert FunPackVideoRefinerV2._get_tokenizer("ltx2") is sentinel
+    assert [c[1] for c in calls] == [True, False]        # cache first, then network
+    assert "DOWNLOADING" in capsys.readouterr().out      # and it says so
+
+
+def test_minimax_h3_asks_for_a_qwen_tokenizer_not_gemma(monkeypatch):
+    """H3 encodes with Qwen3-VL. Falling through to the ltx2 entry measured every token span
+    with a Gemma vocabulary, and fetched a 12B model's tokenizer to do it."""
+    sources = FunPackVideoRefinerV2._get_tokenizer_sources("minimax_h3")
+    assert sources and all("qwen" in mid.lower() for mid, _ in sources)
+    assert not any("gemma" in mid.lower() for mid, _ in sources)
+
+
+def test_an_unknown_family_still_falls_back_to_ltx2():
+    sources = FunPackVideoRefinerV2._get_tokenizer_sources("nonesuch")
+    assert any("gemma" in mid.lower() for mid, _ in sources)