From 467ed9c6fa746e6a6a4c3df4c7ad65939aafde29 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 22 Nov 2025 03:06:29 +0000 Subject: [PATCH] Simplify CLIP encoding - remove broken Flux detection CRITICAL FIX: hasattr(clip, 'encode_from_tokens_scheduled') is useless - This method exists in base CLIP class inherited by ALL models - Always returns True (SD/SDXL/Flux/Chroma/SD3/etc.) - Cannot be used for model detection Simplified to universal encoding for ALL models: - Removed guidance parameter from RegionalPrompterFlux - Removed all is_flux detection branching - Use encode_from_tokens(return_pooled=True) universally - ComfyUI's CLIP object handles 1/2/3/4 encoder complexity internally Benefits: - Simpler code (removed 30+ lines of complex branching) - More maintainable (no model-specific special cases) - Actually works correctly (previous logic was broken) Documented in CONTEXT.md with encoder architecture table. --- CONTEXT.md | 45 ++++++++++++++++++++++++---------------- RegionalPrompting.py | 49 +++++++------------------------------------- 2 files changed, 34 insertions(+), 60 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 741473a..5b08de9 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -38,6 +38,7 @@ - **Format:** Must wrap in list: `[[cond, {"pooled_output": pooled}]]` - **Works:** Identical to external CLIPTextEncode nodes - **Benefit:** Massive UX improvement - no external nodes needed! +- **Universal:** Same method works for ALL models (SD/SDXL/Flux/Chroma/SD3/etc.) - ComfyUI handles multi-encoder complexity internally #### Text Encoder Architectures by Model **CRITICAL:** Different models use different numbers and types of text encoders. ComfyUI's CLIP object handles the complexity internally, but understanding the architecture helps with debugging. @@ -47,34 +48,42 @@ | **SD 1.5** | 1 | CLIP-L | Standard `encode_from_tokens` | | **SD 2.x** | 1 | OpenCLIP-H | Standard `encode_from_tokens` | | **SDXL** | 2 | CLIP-L + OpenCLIP-G | Standard `encode_from_tokens` (handled internally) | -| **Flux** ✅ | 2 | T5-XXL + CLIP-L | **`encode_from_tokens_scheduled`** with guidance | +| **Flux** ✅ | 2 | T5-XXL + CLIP-L | Standard `encode_from_tokens` (handled internally) | | **Chroma** ✅ | 1 | T5-XXL only | Standard `encode_from_tokens` (Flux-based but single encoder!) | | **SD3** | 3 | CLIP-L + CLIP-G + T5-XXL | Standard `encode_from_tokens` (multi-encoder internal) | | **SD3.5** | 3 | CLIP-L + CLIP-G + T5-XXL | Standard `encode_from_tokens` (multi-encoder internal) | -| **HiDream** | 4 | CLIP-L + CLIP-G + 2× T5 variants | ComfyUI uses 4 encoder files | -| **WAN 2.1** | 1 | UMT5-XXL | Standard encoding (fp16/fp8/bf16 variants) | -| **WAN 2.2** | 1 | UMT5-XXL + MoE | Standard encoding (nf4 4-bit quantization) | -| **Qwen-Image** | 1 | Qwen 2.5 VL (qwen_2.5_vl_7b.safetensors) | Standard encoding (vision-language LLM encoder) | +| **HiDream** | 4 | CLIP-L + CLIP-G + 2× T5 variants | Standard `encode_from_tokens` (multi-encoder internal) | +| **WAN 2.1** | 1 | UMT5-XXL | Standard `encode_from_tokens` (fp16/fp8/bf16 variants) | +| **WAN 2.2** | 1 | UMT5-XXL + MoE | Standard `encode_from_tokens` (nf4 4-bit quantization) | +| **Qwen-Image** | 1 | Qwen 2.5 VL (qwen_2.5_vl_7b.safetensors) | Standard `encode_from_tokens` (vision-language LLM) | -**Detection Strategy:** +**Implementation (Simplified - November 2025):** ```python -# Current implementation (correct for most models) -is_flux = hasattr(clip, 'encode_from_tokens_scheduled') - -if is_flux: - # Flux-specific: dual encoder with guidance scheduling - cond = clip.encode_from_tokens_scheduled(tokens, add_dict={"guidance": guidance}) -else: - # Standard encoding: SD/SDXL/Chroma/SD3/SD3.5/HiDream/WAN/Qwen - # ComfyUI's CLIP object handles multi-encoder complexity internally - cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) +# Universal encoding for ALL models (1, 2, 3, or 4 encoders) +# ComfyUI's CLIP object handles multi-encoder complexity internally +for prompt in prompts: + if prompt and prompt.strip(): + tokens = clip.tokenize(prompt) + cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) + encoded_conditionings.append([[cond, {"pooled_output": pooled}]]) ``` +**Critical Discovery (November 2025):** +- ❌ **WRONG:** Using `hasattr(clip, 'encode_from_tokens_scheduled')` to detect Flux + - This method exists in the **base CLIP class** that ALL models inherit from + - Returns `True` for SD1.5, SDXL, SD3, Flux, Chroma - everything! + - Completely useless for model detection +- ✅ **CORRECT:** Use standard `encode_from_tokens(return_pooled=True)` for ALL models + - Works universally: SD (1 encoder), SDXL (2), Flux (2), SD3 (3), HiDream (4) + - ComfyUI's CLIP object abstracts encoder complexity + - No model-specific branching needed +- 📝 **Note:** `encode_from_tokens_scheduled` is what CLIPTextEncode uses internally, but for regional prompting without temporal effects, `encode_from_tokens` works perfectly + **Key Insights:** - ✅ **Chroma is Flux-based architecture but uses ONLY 1 encoder** (T5-XXL, not CLIP-L+T5-XXL) -- ✅ **SD3/SD3.5 use 3 encoders** but ComfyUI's CLIP handles it transparently via standard method +- ✅ **SD3/SD3.5 use 3 encoders** but ComfyUI's CLIP handles it transparently - ✅ **HiDream uses 4 encoders** in ComfyUI (CLIP-L, CLIP-G, T5 variants) -- ✅ **Flux is the ONLY model** currently requiring `encode_from_tokens_scheduled` method +- ✅ **ALL models** use the same encoding method - no special cases needed - ⚠️ **Don't assume encoder count from architecture** - Chroma proves Flux-based ≠ dual encoder! #### JavaScript UI diff --git a/RegionalPrompting.py b/RegionalPrompting.py index 812d557..10141d0 100644 --- a/RegionalPrompting.py +++ b/RegionalPrompting.py @@ -207,13 +207,6 @@ class RegionalPrompterFlux: "default": True, "tooltip": "✅ RECOMMENDED: Softer masks (0.8 strength + gentle edge blend) work better than harsh full-strength (1.0) masks. Confirmed better for Flux, works well with other mask-based models. Disable if you prefer sharper region edges." }), - "guidance": ("FLOAT", { - "default": 3.5, - "min": 0.0, - "max": 100.0, - "step": 0.1, - "tooltip": "Guidance strength for Flux (default 3.5). Higher values = stronger prompt adherence. Try 5-7 for better regional control." - }), }, "optional": { "region2_prompt": ("STRING", { @@ -267,15 +260,15 @@ How to use: • Works well with other mask-based models - try it first! Model-Specific Tips: -• FLUX: Use 3-4 regions max (quality degrades with more), increase CFG to 5-7 +• FLUX: Use 3-4 regions max (quality degrades with more) • FLUX-BASED (Chroma, etc.): Likely 3-4 region limit too (untested), try and see! -• SD3/SD3.5: No known region limit, standard CFG values +• SD3/SD3.5: No known region limit • QWEN-IMAGE: Experimental - try mask-based conditioning and report results! Compatible: Flux (all variants), Chroma, SD3, SD3.5, and other mask-based models""" def encode_regions_flux(self, clip, width, height, background_prompt, region1_prompt, - soften_masks, guidance, extra_pnginfo, unique_id, + soften_masks, extra_pnginfo, unique_id, region2_prompt="", region3_prompt="", region4_prompt=""): """Encode all prompts and apply mask-based regional conditioning for Flux/Chroma/SD3.""" @@ -314,43 +307,15 @@ Compatible: Flux (all variants), Chroma, SD3, SD3.5, and other mask-based models print(f"⚠️ Warning: {num_regions} regions detected. Flux works best with 3-4 regions maximum.") # Encode each prompt using CLIP - # Flux uses dual-encoder (CLIP-L + T5-XXL), Chroma/Qwen use single encoder + # ComfyUI's CLIP object handles multi-encoder complexity internally + # Works for all models: Flux (2 encoders), SD3 (3 encoders), Chroma/Qwen (1 encoder), etc. encoded_conditionings = [] - # Detect if this is Flux by checking for encode_from_tokens_scheduled method - is_flux = hasattr(clip, 'encode_from_tokens_scheduled') - for prompt in prompts: if prompt and prompt.strip(): tokens = clip.tokenize(prompt) - - if is_flux: - # Flux-specific: Dual encoder (CLIP-L + T5-XXL) with guidance - # Check if t5xxl tokens are available - if "t5xxl" in tokens: - # Already has both encoders in single tokenize call - pass - else: - # Need to merge t5xxl tokens manually - try: - t5_result = clip.tokenize(prompt) - if "t5xxl" in t5_result: - tokens["t5xxl"] = t5_result["t5xxl"] - except: - pass - - # Use Flux's scheduled encoding with guidance - try: - cond = clip.encode_from_tokens_scheduled(tokens, add_dict={"guidance": guidance}) - encoded_conditionings.append(cond) - except Exception as e: - print(f"⚠️ Flux encoding failed: {e}, using fallback") - cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) - encoded_conditionings.append([[cond, {"pooled_output": pooled}]]) - else: - # Standard encoding for Chroma, Qwen, SD3, etc. (single encoder) - cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) - encoded_conditionings.append([[cond, {"pooled_output": pooled}]]) + cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) + encoded_conditionings.append([[cond, {"pooled_output": pooled}]]) else: encoded_conditionings.append(None)