From 0ca8dd2ce645e56db7789a570ffcc1633a6b88dc Mon Sep 17 00:00:00 2001 From: Ezequiel Tojo Date: Thu, 5 Mar 2026 21:35:50 -0300 Subject: [PATCH] v1.0.0: Robust model loading with GPU->CPU fallback Major improvements: - Automatic GPU to CPU fallback with VRAM management - Free PyTorch CUDA cache before each load attempt - CPU-pure mode: op_offload=False, flash_attn=disabled, offload_kqv=False (prevents llama.cpp from allocating ~606 MiB CUDA compute buffers) - Progressive context reduction (4096 -> 2048 -> 512) - Vision-Language model detection and filtering (Qwen3VL, LLaVA, etc.) - Proactive CUDA DLL registration for Windows - Always-verbose mode for native llama.cpp diagnostics --- .gitignore | 5 ++ README.md | 35 ++++++-- __init__.py | 2 +- node.zip | Bin 6899 -> 0 bytes nodes.py | 224 ++++++++++++++++++++++++++++++++++++++++++++----- pyproject.toml | 4 +- 6 files changed, 239 insertions(+), 31 deletions(-) delete mode 100644 node.zip diff --git a/.gitignore b/.gitignore index 978e7ab..0d35b66 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ __pycache__/ *.py[cod] *$py.class *.so +*.whl # ComfyUI models/ @@ -18,3 +19,7 @@ input/ # OS .DS_Store Thumbs.db + +# Misc +.tracking +.gemini/ diff --git a/README.md b/README.md index 16e08a3..b19952c 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ # ComfyUI-PromptTranslator-Enhancer -**Autor**: EzeTojo | **Versión**: 0.0.1 (Nightly) +**Autor**: EzeTojo | **Versión**: 1.0.0 Custom node para ComfyUI que traduce y mejora prompts desde distintos idiomas al inglés usando un modelo de lenguaje local (GGUF). Los idiomas soportados dependen exclusivamente del modelo que elijas usar. ## Características -- 🌐 **Multi**: Traduce automáticamente desde los idiomas soportados por el modelo al inglés +- 🌐 **Multi-idioma**: Traduce automáticamente desde los idiomas soportados por el modelo al inglés - ✨ **Mejora de prompts** con 3 niveles: basic, detailed, creative - 🖥️ **100% local** — usa modelos GGUF vía `llama-cpp-python` - 🔄 **Modelo reutilizable** — carga una vez, usa en múltiples nodos +- 🛡️ **Carga robusta** — fallback automático GPU → CPU con gestión inteligente de VRAM +- 🚫 **Filtro de modelos VL** — excluye automáticamente modelos Vision-Language no compatibles ## Ejemplos en acción @@ -36,7 +38,7 @@ Aquí tienes cómo se ve el nodo en ComfyUI en sus diferentes estados: ## Instalación -1. Clonar o copiar este directorio en `ComfyUI/custom_nodes/ComfyUI-PromptEnhancer/` +1. Clonar o copiar este directorio en `ComfyUI/custom_nodes/comfyui-prompt-translator-enhancer/` 2. Instalar dependencias: ```bash pip install llama-cpp-python @@ -49,13 +51,22 @@ Aquí tienes cómo se ve el nodo en ComfyUI en sus diferentes estados: ## Modelos recomendados -> **IMPORTANTE**: Recomendamos **fuertemente** usar modelos **Instruct** o **Chat** (que suelen tener "Instruct" o "Chat" en su nombre, ej. `Qwen3-VL-4B-Instruct-Q4.gguf`) en lugar de modelos base. Los modelos Instruct siguen nuestras reglas mucho mejor y no requieren del *fallback* de completado raw que es más lento. -> *Nota: Los idiomas de entrada que el nodo puede entender dependen de si el modelo GGUF fue entrenado en esos idiomas.* +> **IMPORTANTE**: Usa modelos **Instruct** o **Chat** (que suelen tener "Instruct" o "Chat" en su nombre). Los modelos Vision-Language (VL) como `Qwen3VL`, `LLaVA`, etc. **no son compatibles** con este nodo y serán filtrados automáticamente de la lista. - **Qwen3-4B-Instruct-Q4_K_M.gguf** (~2.5 GB) — Buen balance velocidad/calidad, excelente soporte multi-idioma. - **Qwen2.5-3B-Instruct-Q4_K_M.gguf** (~2 GB) — Más ligero. - **Phi-3-mini-4k-instruct-Q4_K_M.gguf** (~2.3 GB) — Alternativa sólida. +## Gestión de VRAM + +Este nodo está diseñado para funcionar dentro de ComfyUI, donde la GPU suele estar ocupada con modelos de difusión. La estrategia de carga es: + +1. **Intenta GPU** con el contexto solicitado, luego con contextos más pequeños (2048, 512) +2. **Libera cache VRAM** de PyTorch antes de cada intento (`torch.cuda.empty_cache()`) +3. **Fallback CPU puro** — si la GPU no tiene VRAM suficiente, carga el modelo 100% en CPU sin usar CUDA para compute, KQV offload, ni Flash Attention. Esto garantiza que funcione sin importar cuánta VRAM esté ocupada. + +> El modo CPU es más lento pero siempre funcional. Para mejor rendimiento, descarga otros modelos de la GPU antes de ejecutar el nodo. + ## Uso ### Opción 1: Todo-en-uno @@ -73,3 +84,17 @@ Aquí tienes cómo se ve el nodo en ComfyUI en sus diferentes estados: - **basic**: Traduce y añade tags mínimos de calidad - **detailed**: Traduce y añade iluminación, composición, calidad - **creative**: Traduce con interpretación artística, estilos, efectos + +## Changelog + +### v1.0.0 +- ✅ Carga robusta con fallback automático GPU → CPU +- ✅ Gestión inteligente de VRAM (libera cache de PyTorch) +- ✅ Modo CPU puro: `op_offload=False`, `flash_attn=disabled`, `offload_kqv=False` +- ✅ Filtro automático de modelos Vision-Language (VL) +- ✅ Registro proactivo de DLLs CUDA para Windows +- ✅ Verbose mode siempre activo para diagnóstico +- ✅ Soporte para prompts en cualquier idioma (depende del modelo) + +### v0.0.1 +- Versión inicial (nightly) diff --git a/__init__.py b/__init__.py index 8fe7ca3..0d182d5 100644 --- a/__init__.py +++ b/__init__.py @@ -5,7 +5,7 @@ Translates and enhances prompts from multiple languages to English using local G from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS -__version__ = "0.0.1-nightly" +__version__ = "1.0.0" __author__ = "EzeTojo" __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "__version__", "__author__"] diff --git a/node.zip b/node.zip deleted file mode 100644 index bc53016cbea0137c7c5717ee7fe34501a77b92cf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6899 zcmZ{J1yCGavo^B$;sjgVf-de3!QI{69TMDmg9UeYTY|g0y9Wp)xCFOgm+zOG)UEf< zRM(lQnrEu3yHB6)r4{dh=*CsGt&!%r^^(sKsalH9+epo3tzV8bL0>6sS@m~Fx*$5;dQ|EgA zt{t+dGCRu13caSyYQ6D8gx=H{5phK<|hgkjHZQJV4;^P;3_WZs&J833M1~ds__)Tw#|zEcHhziMP;9fJ;`r z0IEGo8y`<)2K9WdtpZB(ZU+4(>T#T7ss~0ix`q zFH={{Jkdp)eI)lXNb^=I`GLomML=}I`s5y!`z@?A&esdYkGFq!+T{$wv*I!^yP@VL z_yl-tE>6c8FfZLX{YNehZ|oH!1je$B)r3ss*r3gtRxWa@Tp*)Yg;|p#pM;e!P8m-( zSk4jUX&PS!eihS%yT60%IiM6@$=f%@Y-vD~k6UUXwGPcsiO`2g+J|nxi-uv{tOYnB zIrMwzu-mL8T8HxDd}_jy#5$6-VX;r=Dp==~H*25QTXbd75v8Zwkwt0U1A!+gLqpZK zzPE5#+bwYnBwN{A*%r%C$TkXE*H)w2E~@J{fBaz$@Ugv*kt0#&lD9`zx9?vnQW8(k zI2I^u%@jGYQ{*%?$ExBGlW|q0x8Yz=fG%u!=xoPhV(w|Mrb|rL%%qs6RckCDAh%Ju z1OazDL$4nhWtm^wJwJi}DdVvV)o*U-Xz|VTpQtzb6RXL@y7~kf6sWInaccdxwUFLo zRaHXtgMtLu!8}`Y(hH~`c3wgpZM_sUZQ^{&1*W*7#$T><^ zV8V zRoO(Qi@_ay{n7Gd(lX0JywY<=IFvcZ&XN#Z_t|{lj-`{IGL|6o`}4D}!cPFPYZPLf zde*YKeu@?Q+i)g3s42w&fr@oLZ9t858YVd4Asbq@I&CHaOfI4e`^|WPz~+*lMeM6N zNENw^qu3!XmZglH{^!#}bHK}Q%+QtfmHUbvS;Us7U!3oC$bF12x9o3mP0Bbe(?$f8 zF0WV`Ew$S|Jqt>y!5@_&%2wj>%&GGx9?&3$iSiWUogQQY&Fi~8lv`=0B}^#2Ah<-A z{IrxF`4x*%Er@u|Q628|srr`Av-bK_7g~Z9q+dfY1f%G#z5V`i(V!*+=%42gI@{Ws z`4eGSEYUGmy5)pNiI9C*#jLSLx+gp;30(ZRlFDp!N}NIlo4FP$uZrKUhN;}wD9m^} z4c&5TuUUF2v{Y-PxKAY4v5=CUbSDHljDn{Z@OPl5iLe-a)F7}1aU+s1cV1C2hz z_G9;g?azCG9|L2C8{IK_nqLbQ&^{-59$D;qZh=P@I@W=~PC(W>e z(R<&qAwuJZ9w^2eY3Z!_ie>--Y&*uqT>8)`4E9ZZqxUe;lajG(BBjLmn?913{GV3j zsyF11h(|l^vPX;X(U(6)fN3s9_F1E6)XMvAWIHiCGH}d$aCSn(75xfk#WhFDnT~O% zdTTlam~eR6RI&&08%gqgDexuN7?%f*kkE3IGfv4fJ1+zj#)I_zH-CVo5~?7AHGXGQxF3r5C4~ z`+O(g^`rVeCDIY(?`a0J&uLcR%zJ#!$aTok@^uW_>x`@AprW?7f^J=G&9M(ac1K(N zVqcELt9L4#QDM$Mnxrz?`2O?n*OumeMNbbhtB+G;_pAJ|O*v`9n<-9AEqX`~dsB6~ zT%g)Y62GQ(q44q4uUjh3Ud{kmZxZ8a=}gr|-+gFTm6qY;@0Bxp{O9qt8WGMOq3USe zABwxF2Krq?hi^aE8Kf!`9gZ(Yy7jE)reOiHoLjp&BNqcf1&m3qMt%tQ2b#rE^69Qv z4Q7-&UylOD4UZoC>+rhVXt76jW5o_(XTpwD7xoAW8g3l&Sfg^BxL#oYWJwQFU43+r zr0ULFcC>+pfVtZ}-S%4yE(61rqtw~GpiT|DBzrzglH#;#B~*N#E1?3u3EGg2NSkzMn^1R9v8V~H_E743?WRpy zB2@aoTI>hJ)5k8XcJbA_AKIP<^GnY(y?*mv5trf(yS2aF#@~^5C8LLDVVooT>e)x7 zt`i7Kc3eq}^TzHZLi`s#J2vcv#r?zZvZK`I3yUPo#C&VS8}kM0)pYf*@+R}`$Pg)z z5V=Q%f}&&pPl*Nv<>+K?;r3>UQ(XgB%GP*)tPERp-dKGPS6Xkq@vJZFJyw+!)oba%T+my@g`t?vl_F(flA1B zV#9*cH3_ozi+Dl~iB1o=7_a)^@|)zDNK6rC8{%u*E(2|)M8;9 zF7B$M;Mmn)J|K^SD5oo8^Vef%won?ydsZ#BO~K8Fs~7kElXKrER>Biv5)!2n4!f$I z;e$~N6P7f(>@5*988&=rPHFAB9?>xmvs$4~eJJ?8t?0Fn*pWny4?nU#Zxc+l)|Q#D z@uCH7`+=f^F-n~&7qg)~A;>7}j*jv)!}Fbm?D!#U;*mkycah_U?6l#rsp%#9!x#!Q z!zux8A=6oQNr~q-SVO`FxI`oaR{%i$Dh~*$N411rMDVRasSGc z>js5e{0x{V2J(+GMPdi5` zbQkLEf@QZ37AyaGX|s^_$`}Yu`~7YhoDDowVy=l4O?6Gtoy)>=?tyc_Ab3pyRZ~%S zp0U=-B0TsQ4)s=cjmMG?a3WhiFo}(*4on^+nzeLXGFuJ+TBr~^l&mk`=nFMw_i$2tDw|K!IDGKLS-;%B^2=dQa#bzLu@t? z%DAF{b`fOJ{9hxF$RT{vP;|>_+O+`iG4j4=oXi$MCgp@mL5V6~sHAS>Z4!z=O3Ar+ zK4S*H>g!kyb;lF7ZBd7UTxxML!-Fg0s|LPYPAXm#XQWMImxUWUFzV)Ko=;?oOVFYB z&4rMZgWRI>RO^QYak2z{FhBw};m*?o1zMzj#9-p5f_qYC z={>Cz4)xlMK2xM24aBqxQ7FE^dVD^p8ms8@>JA%RNhX1)X>A=JyH{S>vv zgUt$!dv;nFBg4^FRO4utM+H?P_eHIHp{b=deU2c8x75nmp1io8H(iTSM$wT`$&|Y| z?kOS|tFu-v$OfaFsqc!bVE6kuy3R}DG;No8aT;#x7E(PAwWJ`kX`$ii#6BVFJD8E= z+7SIpKumqgkbFFM3Q@G$ns#!Cvg(w5Td9v$Pcn^b`6?7|L@JKX%HF<*QwW&PD~n3_`3fF1@MY%t02w zcwv(0g39m-Urk4w!k&*q8%J0jE<_|?xUtAfTkER5Pt&4E6rLvB;c|LtJfE|fQ;&Cp zzZv34JuCB}QY;39q^!(<&U_uU_^Fpz3+_+}P8Sws@k1aLB}4i(0H4`7AO@%56MaY7 z4tTXAP_Q9QTRP2R$TpezBaB$N?}w!}apx3@vJb7uRjP3+zI?37@Z#@Y6b}~~CiEls zVORxxjL$er@66}I=lt%`x>k?uVKLElGHziK!w-lvlP%Vpj=AZ6&1$%n^X0jdmx`Is zOFI0T_1cefS?Qu1G+(d!uBO#(HnQ@(PytG!fsfKr9IcwcPu#M>mcX4$7J(Vlgz&~< ziJqs@KShUN&W{m)3pC-8`ld&&nR!%KU8a>hZD|e6SN>f&!Np`8^Tx?N4`PI zWyRs^h#!N`ul@K<0@VU4b#jgxLX*RK{olPdA1l}k((@7mB8D5hh43*l>iANiiz;5>TTIVsp$c?k+7_;GEzZ7-$O3jHJe-^maW;CFc_K z4DIb^$bHL{NEHf8b;w@!`DiflwIgKVaX%8igz{sL(k#8GpWKkoH)!l18PwcUnJ9dJ@LS(3AI zffXJXI}p+F7`sEh&Aas$Z-4ve(0gzCp`3SCjtZ5R7Z%s?YYMC%HaD-;*X}FW%hjfj z4s2K2K_(3j8)@E(`ht{AQ!57S4QVX42ELHbwN{KGoaaEdTB6>NA^kgn%q}w{$)jo! zn*0$dYTHH>u#-LyV)A+#A;)jn68-?ona6SBvF7tN6$G0HV8PjXl<)_DNbmyA?5x*U zQdAC6+!zIw5~1(!`p`IU0=;iKILm7>ZtHTMK@lE=i5VDk4%ymF=He-M&Q$$N0JD#B z1I0lH(X~ZEt!KONj(I!8cD?o0S%rDJGgV0!m}8KwPP&E?O9ZA}ei#xRHB=-iXrYiA zh(g&jo3{>E`Z~r~1Id9J#gq2uR#We5HeJDN_zbA7%I#b0nrG>3C=Ale!}DHb*FL))UE zKC+q*3}$2up}Vaj0kuC^EB1d&H5-O|=BvJj*WApG9zI4L4o1vBUl~%!`$S}2h&NY0 zeE?{*cxLBL#eu5F&^XC*4mO9*F^YN2!pNY01Ge7+uC%UwR#!%G{}byw`_F4Lrn9r7 z5(}36E3xEbEmb6|ejLH45C}en8EYvsgZ9_w51Mp!%=rh26zwI|r@#(G^_N9^nh}Sp z0=p=3V3Wy7#%zu`K4hM%_RdcI{&eZA8H^!@mAPYaMSnd^oqmgs7CJK zEXUiFO~cFh-slh)!PkN`SZQH03t2gF85QrxfI1(S>tCUf`03oh$KH+XsONq!D^lw?zQ?YTUVpzW zcfiK|akWXa?%-m!4DJH8(@f+2AZdOt>@ym77-R6nvyJv?zUC3UXs zW)m3nuEul_a+W;qsh&WS_!Cn=y{}n$-0eQF9rrpM)GIz8n`?^bcnXi@^F4n4`$Vjk z{+=fzx|z~9L4Qqn{N*p%#&M`YLlA>sbJVx~Coi6ojw(hBAwOYhqUn)?e_3Q40umU#5*3eW?yQ{%cN#QyC!>svKeLbHY0`*{z6urmqx^?Ls z`?qyV+8B*$Ku05*-QtK+#cXnMMc_~FpnaG2s2y90)z`ZTX53%8f`fT%H|a2*DS?bS zP@86|EwI#$hJ88+YHcC~B=}PYvCYslTinE9XbofQBg}!w9r^m7`MY`e`FPzmrh?w} zQhO91NOFWrZbudDYweBPDC7O|&M5gc-#Nt@%ycAxX6<_0{aOeh9?@Ff&P8t*I6|0< zOx>28O(*Ao{W%mfrQx%YaZvfAaB<=ozK3urR7!T!$!?*p^JcG^^QzfN)6K~pmFO{K zv_1P5%v+k(Kv8C>(%+8pK|`@eZSKA z%EuH}OWJb-rO@|u3a`cR|>Y#MKyoyqKJv|_elmRJhS*luuYz>H? zL-h*xRGHsOYaPgQYX^z@nJ?`271xOY;zPx|r&avcHqWq#IYQ!LIH-}(G!O@7e%U4dpP`}$MeL(`o~ zo2RErUhnz$_ve3R&(9Yo_V1l(Wd0&}K#9GydpPd(V2Uea{Prb6Qwd^wSeU*jP2|tL zAG*oUC!Bra>nyWPP}hETa|*mZSeRu-nO3t6t+gCKT;{|*Bv!L{?CyTPdl}_(RG7@M zTmo=oh#?fO|T z!o7lC`t$QMh1Y81z~vwyb!bz1G=hZCH{=9GY1dw+=d;1ja&6sr>ps(*LNf#!w!Ch0 z6!s#tbP?}| z+qB||Z!n!FQw7#r5qHL4`9Jq3BXQHN_X;~gswDwd-GA)*rCLW{?jT|U^kz?jH^F!l zR|^*p8&?Yl3rBZ1u=_{%@llxxg#`sL?F>^H83!+CQkTc>gno|Hl5CXZ~Oj-kkM6{iG}h2mhZh Q5Z>;Xw>_f#ABRHy7uI-#WdHyG diff --git a/nodes.py b/nodes.py index 19f3c4f..6f2e407 100644 --- a/nodes.py +++ b/nodes.py @@ -1,16 +1,88 @@ """ ComfyUI-PromptTranslator-Enhancer -Custom nodes that enhance and translate prompts from Spanish to English +Custom nodes that enhance and translate prompts from any language to English using a local GGUF model via llama-cpp-python. """ import os +import sys import glob +import ctypes import folder_paths +# --------------------------------------------------------------------------- +# CUDA DLL Pre-loading (critical for running inside ComfyUI on Windows) +# --------------------------------------------------------------------------- +_cuda_dlls_loaded = False + + +def _ensure_cuda_dlls(): + """Pre-register DLL directories so llama.cpp can find CUDA runtime libs. + + Inside ComfyUI, PyTorch loads its own copy of CUDA DLLs from + ``site-packages/torch/lib``. llama-cpp-python ships its own + ``ggml-cuda.dll`` that depends on ``cudart64_*.dll`` and + ``cublas64_*.dll`` from the *same* CUDA toolkit. If those DLLs + are not on PATH or registered via ``os.add_dll_directory``, + ``llama_model_load_from_file`` silently returns NULL. + + This function registers every directory that might contain the + needed CUDA runtime DLLs. + """ + global _cuda_dlls_loaded + if _cuda_dlls_loaded: + return + _cuda_dlls_loaded = True + + if sys.platform != "win32": + return + + dll_dirs_to_register = set() + + # 1. llama_cpp's own bin/ and lib/ directories + try: + import llama_cpp as _lc + pkg_dir = os.path.dirname(_lc.__file__) + for subdir in ("bin", "lib"): + d = os.path.join(pkg_dir, subdir) + if os.path.isdir(d): + dll_dirs_to_register.add(d) + except ImportError: + pass + + # 2. PyTorch's CUDA DLLs (cudart, cublas) + try: + import torch + torch_lib = os.path.join(os.path.dirname(torch.__file__), "lib") + if os.path.isdir(torch_lib): + dll_dirs_to_register.add(torch_lib) + except ImportError: + pass + + # 3. NVIDIA CUDA toolkit on PATH (if installed) + cuda_path = os.environ.get("CUDA_PATH", "") + if cuda_path: + cuda_bin = os.path.join(cuda_path, "bin") + if os.path.isdir(cuda_bin): + dll_dirs_to_register.add(cuda_bin) + + # Register all directories + for d in dll_dirs_to_register: + try: + os.add_dll_directory(d) + except (OSError, AttributeError): + pass + # Also prepend to PATH as legacy fallback + if d not in os.environ.get("PATH", ""): + os.environ["PATH"] = d + os.pathsep + os.environ.get("PATH", "") + + if dll_dirs_to_register: + print(f"[PromptTranslator-Enhancer] Registered {len(dll_dirs_to_register)} CUDA DLL directories") + def _get_llama_class(): """Lazy import of Llama to avoid errors when llama-cpp-python is not installed.""" + _ensure_cuda_dlls() try: from llama_cpp import Llama return Llama @@ -19,9 +91,124 @@ def _get_llama_class(): "llama-cpp-python is not installed. Please install it with:\n" " pip install llama-cpp-python\n" "For GPU (CUDA) support:\n" - " CMAKE_ARGS=\"-DGGML_CUDA=on\" pip install llama-cpp-python --force-reinstall --no-cache-dir" + ' CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python --force-reinstall --no-cache-dir' ) + +# Vision-Language model filename patterns (these need special multimodal handlers) +_VL_PATTERNS = ("qwen3vl", "qwen2vl", "qwen2.5vl", "llava", "minicpm-v", "internvl") + + +def _is_vl_model(model_path): + """Check if a GGUF file is a Vision-Language model (not suitable for text-only use).""" + name_lower = os.path.basename(model_path).lower() + # Also check parent folder name + parent_lower = os.path.basename(os.path.dirname(model_path)).lower() + combined = f"{parent_lower}/{name_lower}" + return any(p in combined for p in _VL_PATTERNS) + + +def _free_torch_vram(): + """Ask PyTorch to release cached VRAM so llama.cpp can use it.""" + try: + import torch + if torch.cuda.is_available(): + before = torch.cuda.memory_reserved(0) / 1024**2 + torch.cuda.empty_cache() + after = torch.cuda.memory_reserved(0) / 1024**2 + freed = before - after + if freed > 1: + print(f"[PromptTranslator-Enhancer] Freed {freed:.0f} MiB of PyTorch VRAM cache") + except Exception: + pass + + +def _load_model_with_fallback(Llama, model_path, n_gpu_layers, n_ctx): + """Try to load a GGUF model with an aggressive fallback strategy. + + Fallback order (tries freeing PyTorch VRAM cache before each): + 1. GPU + requested n_ctx (flash_attn=auto) + 2. GPU + 2048 (flash_attn=auto) + 3. GPU + 512 (flash_attn=auto) + 4. CPU + requested n_ctx (flash_attn=DISABLED, offload_kqv=False) + 5. CPU + 2048 (flash_attn=DISABLED, offload_kqv=False) + 6. CPU + 512 (flash_attn=DISABLED, offload_kqv=False) + + The CPU fallbacks explicitly disable Flash Attention and KQV offloading + so that llama.cpp does NOT try to allocate CUDA compute buffers. + + Returns the loaded Llama model instance. + Raises RuntimeError if all attempts fail. + """ + # Warn about Vision-Language models + if _is_vl_model(model_path): + print( + f"[PromptTranslator-Enhancer] ⚠ WARNING: '{os.path.basename(model_path)}' appears to be a " + f"Vision-Language model (VL). These models require a multimodal handler and are NOT " + f"supported by this text-only node. Please select a text/instruct model instead " + f"(e.g. Qwen3-4B-Q4_K_M.gguf)." + ) + raise RuntimeError( + f"Model '{os.path.basename(model_path)}' is a Vision-Language model and cannot be used " + f"with this text-only prompt enhancement node. Please choose a regular text model." + ) + + # Flash Attention type values (llama.cpp enum) + FLASH_ATTN_DISABLED = 0 # force off – no CUDA compute buffers + + # Build list of attempts: (label, kwargs_dict) + ctx_sizes = sorted(set([n_ctx, 2048, 512]), reverse=True) # dedupe & descending + + attempts = [] + # GPU attempts (flash_attn left at default/auto) + if n_gpu_layers != 0: + for ctx in ctx_sizes: + attempts.append((f"GPU/ctx={ctx}", dict( + n_gpu_layers=n_gpu_layers, n_ctx=ctx, + ))) + # CPU attempts – disable flash_attn, kqv offload, AND op_offload + # op_offload=False is critical: it prevents llama.cpp from allocating + # ~606 MiB CUDA compute buffers even when the model is on CPU + for ctx in ctx_sizes: + attempts.append((f"CPU-pure/ctx={ctx}", dict( + n_gpu_layers=0, n_ctx=ctx, + flash_attn_type=FLASH_ATTN_DISABLED, + offload_kqv=False, + op_offload=False, + ))) + + last_error = None + for label, kwargs in attempts: + try: + _free_torch_vram() + gpu_layers = kwargs.get("n_gpu_layers", 0) + ctx = kwargs.get("n_ctx", n_ctx) + print(f"[PromptTranslator-Enhancer] Trying {label} (n_gpu_layers={gpu_layers}) ...") + model = Llama( + model_path=model_path, + verbose=True, # ALWAYS verbose so native errors show in console + **kwargs, + ) + if ctx < n_ctx: + print(f"[PromptTranslator-Enhancer] ⚠ Context reduced from {n_ctx} to {ctx} to fit in memory") + print(f"[PromptTranslator-Enhancer] ✓ Model loaded successfully ({label})") + return model + except Exception as e: + last_error = e + print(f"[PromptTranslator-Enhancer] ✗ {label} failed: {e}") + + raise RuntimeError( + f"Failed to load model after {len(attempts)} attempts.\n" + f" Model: {model_path}\n" + f" Last error: {last_error}\n" + f" Troubleshooting:\n" + f" 1. Make sure the .gguf file is a TEXT model (not Vision-Language)\n" + f" 2. Make sure the .gguf file is valid and not corrupted\n" + f" 3. Try freeing GPU VRAM by unloading other models first\n" + f" 4. Check the console output above for native llama.cpp error messages" + ) + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -33,15 +220,17 @@ if not os.path.exists(LLM_DIR): def get_available_gguf_models(): - """Recursively find all .gguf files under models/LLM/, excluding mmproj files.""" + """Recursively find all .gguf files under models/LLM/, excluding mmproj and VL files.""" pattern = os.path.join(LLM_DIR, "**", "*.gguf") all_files = glob.glob(pattern, recursive=True) - # Filter out vision projector files (mmproj-*) and return relative paths + # Filter out vision projector files (mmproj-*), VL models, and return relative paths models = [] for f in all_files: basename = os.path.basename(f) if basename.lower().startswith("mmproj-"): continue + if _is_vl_model(f): + continue rel_path = os.path.relpath(f, LLM_DIR) models.append(rel_path) return sorted(models) if models else ["no_models_found"] @@ -187,14 +376,8 @@ class LoadLLMModel: print(f"[PromptTranslator-Enhancer] GPU layers: {n_gpu_layers}, Context: {n_ctx}") Llama = _get_llama_class() - model = Llama( - model_path=model_path, - n_gpu_layers=n_gpu_layers, - n_ctx=n_ctx, - verbose=False, - ) + model = _load_model_with_fallback(Llama, model_path, n_gpu_layers, n_ctx) - print(f"[PromptTranslator-Enhancer] Model loaded successfully!") return (model,) @@ -203,7 +386,7 @@ class LoadLLMModel: # --------------------------------------------------------------------------- class PromptEnhancerTranslator: - """Takes a Spanish prompt, enhances it for image generation, and translates to English. + """Takes a prompt in any language, enhances it for image generation, and translates to English. This node loads the model on each execution. For repeated use, prefer LoadLLMModel + PromptEnhancerFromModel.""" @classmethod @@ -211,7 +394,7 @@ class PromptEnhancerTranslator: return { "required": { "prompt": ("STRING", {"multiline": True, "default": "", - "placeholder": "Escribe tu prompt en español..."}), + "placeholder": "Escribe tu prompt en cualquier idioma..."}), "llm_model": (get_available_gguf_models(), {"default": get_available_gguf_models()[0]}), "enhancement_level": (["basic", "detailed", "creative"], {"default": "detailed"}), "n_gpu_layers": ("INT", {"default": -1, "min": -1, "max": 100, "step": 1}), @@ -225,7 +408,7 @@ class PromptEnhancerTranslator: RETURN_NAMES = ("enhanced_prompt", "original_prompt",) FUNCTION = "enhance" CATEGORY = "LLM/Prompt Enhancement" - DESCRIPTION = "Enhances and translates a Spanish prompt to English using a local GGUF model." + DESCRIPTION = "Enhances and translates a prompt to English using a local GGUF model." def enhance(self, prompt, llm_model, enhancement_level, n_gpu_layers, max_tokens, temperature, seed): if not prompt.strip(): @@ -237,12 +420,7 @@ class PromptEnhancerTranslator: print(f"[PromptTranslator-Enhancer] Loading model: {llm_model}") Llama = _get_llama_class() - model = Llama( - model_path=model_path, - n_gpu_layers=n_gpu_layers, - n_ctx=4096, - verbose=False, - ) + model = _load_model_with_fallback(Llama, model_path, n_gpu_layers, n_ctx=4096) enhanced = run_llm_enhancement(model, prompt, enhancement_level, max_tokens, temperature, seed) @@ -260,7 +438,7 @@ class PromptEnhancerTranslator: # --------------------------------------------------------------------------- class PromptEnhancerFromModel: - """Takes a Spanish prompt and a pre-loaded LLM model, enhances the prompt and translates to English.""" + """Takes a prompt in any language and a pre-loaded LLM model, enhances the prompt and translates to English.""" @classmethod def INPUT_TYPES(cls): @@ -268,7 +446,7 @@ class PromptEnhancerFromModel: "required": { "llm_model": ("LLM_MODEL",), "prompt": ("STRING", {"multiline": True, "default": "", - "placeholder": "Escribe tu prompt en español..."}), + "placeholder": "Escribe tu prompt en cualquier idioma..."}), "enhancement_level": (["basic", "detailed", "creative"], {"default": "detailed"}), "max_tokens": ("INT", {"default": 256, "min": 32, "max": 1024, "step": 32}), "temperature": ("FLOAT", {"default": 0.7, "min": 0.0, "max": 2.0, "step": 0.05}), @@ -280,7 +458,7 @@ class PromptEnhancerFromModel: RETURN_NAMES = ("enhanced_prompt", "original_prompt",) FUNCTION = "enhance" CATEGORY = "LLM/Prompt Enhancement" - DESCRIPTION = "Enhances and translates a Spanish prompt using a pre-loaded LLM model from LoadLLMModel node." + DESCRIPTION = "Enhances and translates a prompt using a pre-loaded LLM model from LoadLLMModel node." def enhance(self, llm_model, prompt, enhancement_level, max_tokens, temperature, seed): if not prompt.strip(): diff --git a/pyproject.toml b/pyproject.toml index 73aaa19..0513229 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-prompt-translator-enhancer" description = "Translates and enhances prompts from multiple languages to English using local GGUF models." -version = "0.0.1" +version = "1.0.0" license = { file = "LICENSE" } dependencies = ["llama-cpp-python>=0.2.0"] @@ -11,4 +11,4 @@ Repository = "https://github.com/ezetojo/ComfyUI-PromptTranslatorEnhancer" [tool.comfy] PublisherId = "ezetojo" DisplayName = "Prompt Translator & Enhancer" -Icon = "" +Icon = "https://raw.githubusercontent.com/ezetojo/ComfyUI-PromptTranslatorEnhancer/refs/heads/master/assets/prompt_translator_enhancer.png" \ No newline at end of file