From 5cf13f45924cc284e9aa0193519b0e542e330c37 Mon Sep 17 00:00:00 2001 From: Jason Hoku Date: Mon, 18 May 2026 12:00:08 -0700 Subject: [PATCH] =?UTF-8?q?fix:=20florence2=20=E2=80=94=20load=5Floras=5Ff?= =?UTF-8?q?or=5Fpreencoding=20+=20per-clip=20conditioning=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs surfacing when 'From manifest (per-image)' mode loads a different model: 1. load_loras() requires 5 positional args including target_model_name and incompatible_loras dict. I was calling with 3 — caused 'load_loras() missing 2 required positional arguments' and silent LoRA skip. Switched to load_loras_for_preencoding(base_model, base_clip, lora_string) which has exactly the signature my code assumed and returns the same shape. 2. CLIP architecture mismatch: the upscale_runner's conditioning_cache was encoded with the SESSION's clip. When from_manifest loaded a model with a different clip arch (e.g. session=SDXL 2048-dim, per-item=SD1.5 768-dim), reusing that cache fed wrong-shape conditioning into KSampler's cross-attention. Error: 'mat1 and mat2 shapes cannot be multiplied (154x2048 and 768x320)'. Fix: detect via identity check (clip_handle is fallback_handles[1]). If same clip, reuse shared cache. If different, use a per-clip-id sub-cache stored inside ckpt_cache (so multiple images with the same per-item model still benefit from caching after the first encode). --- florence2_hires.py | 62 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/florence2_hires.py b/florence2_hires.py index 667fdbf..1139f9f 100644 --- a/florence2_hires.py +++ b/florence2_hires.py @@ -327,15 +327,21 @@ _FH_LORA_LOADER = None def _resolve_loaders(): """Lazy-import the real loaders. Only called from runtime path; tests - monkeypatch _FH_CKPT_LOADER / _FH_LORA_LOADER directly.""" + monkeypatch _FH_CKPT_LOADER / _FH_LORA_LOADER directly. + + Uses load_loras_for_preencoding (NOT load_loras) — its signature is + (base_model, base_clip, lora_string) and returns (model, clip), exactly + matching what we need. The full load_loras() requires target_model_name + and incompatible_loras dict tracking which is only meaningful when + iterating across multiple grid configs.""" global _FH_CKPT_LOADER, _FH_LORA_LOADER if _FH_CKPT_LOADER is None or _FH_LORA_LOADER is None: try: - from .model_loader import load_checkpoint, load_loras + from .model_loader import load_checkpoint, load_loras_for_preencoding except ImportError: - from model_loader import load_checkpoint, load_loras + from model_loader import load_checkpoint, load_loras_for_preencoding _FH_CKPT_LOADER = _FH_CKPT_LOADER or load_checkpoint - _FH_LORA_LOADER = _FH_LORA_LOADER or load_loras + _FH_LORA_LOADER = _FH_LORA_LOADER or load_loras_for_preencoding return _FH_CKPT_LOADER, _FH_LORA_LOADER @@ -559,21 +565,47 @@ def run_florence2_step( cache=ckpt_cache, ) - # 9. Conditioning (reuse the upscale_runner's cache structure) + # 9. Conditioning. When the per-item clip is the SAME identity as the session's + # clip (fallback_handles[1]), we can reuse the upscale_runner's conditioning + # cache. When from_manifest loaded a DIFFERENT model, the per-item clip can + # have a different architecture (e.g. SDXL 2048-dim vs SD1.5 768-dim); reusing + # session-cached conditioning would feed wrong-shape tensors into cross-attn + # and crash with "mat1 and mat2 shapes cannot be multiplied". So we maintain + # a per-clip-identity conditioning cache inside ckpt_cache. try: from .batch_encoding import encode_prompt_with_combinators except ImportError: from batch_encoding import encode_prompt_with_combinators - if positive_prompt not in conditioning_cache["positive"]: - conditioning_cache["positive"][positive_prompt] = encode_prompt_with_combinators( - clip_handle, positive_prompt, clip_skip - ) - if negative_prompt not in conditioning_cache["negative"]: - conditioning_cache["negative"][negative_prompt] = encode_prompt_with_combinators( - clip_handle, negative_prompt, clip_skip - ) - pos_cond = conditioning_cache["positive"][positive_prompt] - neg_cond = conditioning_cache["negative"][negative_prompt] + + session_clip = fallback_handles[1] + if clip_handle is session_clip: + # Same clip as session — reuse the upscale_runner's shared cache. + if positive_prompt not in conditioning_cache["positive"]: + conditioning_cache["positive"][positive_prompt] = encode_prompt_with_combinators( + clip_handle, positive_prompt, clip_skip + ) + if negative_prompt not in conditioning_cache["negative"]: + conditioning_cache["negative"][negative_prompt] = encode_prompt_with_combinators( + clip_handle, negative_prompt, clip_skip + ) + pos_cond = conditioning_cache["positive"][positive_prompt] + neg_cond = conditioning_cache["negative"][negative_prompt] + else: + # Different clip (per-manifest model) — use a per-clip-id cache inside + # ckpt_cache so multiple images with the same per-item model still share. + per_clip = ckpt_cache.setdefault("_cond_cache", {}) + clip_id = id(clip_handle) + bucket = per_clip.setdefault(clip_id, {"positive": {}, "negative": {}}) + if positive_prompt not in bucket["positive"]: + bucket["positive"][positive_prompt] = encode_prompt_with_combinators( + clip_handle, positive_prompt, clip_skip + ) + if negative_prompt not in bucket["negative"]: + bucket["negative"][negative_prompt] = encode_prompt_with_combinators( + clip_handle, negative_prompt, clip_skip + ) + pos_cond = bucket["positive"][positive_prompt] + neg_cond = bucket["negative"][negative_prompt] # 10. VAE encode -> generate_image (project's known-working sampling path) try: