From 364a6fc7054ec87b58686d59fa78d1f577b259dd Mon Sep 17 00:00:00 2001 From: Nakano Kenji Date: Tue, 9 Dec 2025 23:39:00 +0800 Subject: [PATCH] Initial commit of ComfyUI-SimpleChat --- README.md | 98 ++++++ README_CN.md | Bin 0 -> 2765 bytes __init__.py | 47 +++ __pycache__/__init__.cpython-311.pyc | Bin 0 -> 1071 bytes api/__init__.py | 6 + api/__pycache__/__init__.cpython-311.pyc | Bin 0 -> 337 bytes api/__pycache__/routes.cpython-311.pyc | Bin 0 -> 11905 bytes api/routes.py | 201 +++++++++++ core/__init__.py | 51 +++ core/__pycache__/__init__.cpython-311.pyc | Bin 0 -> 1130 bytes core/__pycache__/image_utils.cpython-311.pyc | Bin 0 -> 4992 bytes core/__pycache__/noass.cpython-311.pyc | Bin 0 -> 4201 bytes core/image_utils.py | 122 +++++++ core/noass.py | 174 ++++++++++ core/providers/__init__.py | 34 ++ .../__pycache__/__init__.cpython-311.pyc | Bin 0 -> 1244 bytes .../__pycache__/base.cpython-311.pyc | Bin 0 -> 3544 bytes .../__pycache__/claude.cpython-311.pyc | Bin 0 -> 5580 bytes .../__pycache__/gemini.cpython-311.pyc | Bin 0 -> 8856 bytes .../__pycache__/openai.cpython-311.pyc | Bin 0 -> 5158 bytes core/providers/base.py | 72 ++++ core/providers/claude.py | 121 +++++++ core/providers/gemini.py | 199 +++++++++++ core/providers/openai.py | 94 ++++++ docs/architecture.md | 312 ++++++++++++++++++ docs/research_comfyui_api_patterns.md | 153 +++++++++ docs/research_gemini_image.md | 127 +++++++ docs/research_noass.md | 69 ++++ nodes/__init__.py | 18 + nodes/__pycache__/__init__.cpython-311.pyc | Bin 0 -> 694 bytes nodes/__pycache__/chat.cpython-311.pyc | Bin 0 -> 2289 bytes nodes/__pycache__/chat_image.cpython-311.pyc | Bin 0 -> 2486 bytes nodes/__pycache__/chat_noass.cpython-311.pyc | Bin 0 -> 3491 bytes nodes/__pycache__/config.cpython-311.pyc | Bin 0 -> 2540 bytes nodes/__pycache__/gemini_edit.cpython-311.pyc | Bin 0 -> 2270 bytes nodes/__pycache__/gemini_gen.cpython-311.pyc | Bin 0 -> 2495 bytes nodes/chat.py | 55 +++ nodes/chat_image.py | 59 ++++ nodes/chat_noass.py | 132 ++++++++ nodes/config.py | 79 +++++ nodes/gemini_edit.py | 58 ++++ nodes/gemini_gen.py | 59 ++++ requirements.txt | 1 + web/simple_chat_config.js | 88 +++++ 44 files changed, 2429 insertions(+) create mode 100644 README.md create mode 100644 README_CN.md create mode 100644 __init__.py create mode 100644 __pycache__/__init__.cpython-311.pyc create mode 100644 api/__init__.py create mode 100644 api/__pycache__/__init__.cpython-311.pyc create mode 100644 api/__pycache__/routes.cpython-311.pyc create mode 100644 api/routes.py create mode 100644 core/__init__.py create mode 100644 core/__pycache__/__init__.cpython-311.pyc create mode 100644 core/__pycache__/image_utils.cpython-311.pyc create mode 100644 core/__pycache__/noass.cpython-311.pyc create mode 100644 core/image_utils.py create mode 100644 core/noass.py create mode 100644 core/providers/__init__.py create mode 100644 core/providers/__pycache__/__init__.cpython-311.pyc create mode 100644 core/providers/__pycache__/base.cpython-311.pyc create mode 100644 core/providers/__pycache__/claude.cpython-311.pyc create mode 100644 core/providers/__pycache__/gemini.cpython-311.pyc create mode 100644 core/providers/__pycache__/openai.cpython-311.pyc create mode 100644 core/providers/base.py create mode 100644 core/providers/claude.py create mode 100644 core/providers/gemini.py create mode 100644 core/providers/openai.py create mode 100644 docs/architecture.md create mode 100644 docs/research_comfyui_api_patterns.md create mode 100644 docs/research_gemini_image.md create mode 100644 docs/research_noass.md create mode 100644 nodes/__init__.py create mode 100644 nodes/__pycache__/__init__.cpython-311.pyc create mode 100644 nodes/__pycache__/chat.cpython-311.pyc create mode 100644 nodes/__pycache__/chat_image.cpython-311.pyc create mode 100644 nodes/__pycache__/chat_noass.cpython-311.pyc create mode 100644 nodes/__pycache__/config.cpython-311.pyc create mode 100644 nodes/__pycache__/gemini_edit.cpython-311.pyc create mode 100644 nodes/__pycache__/gemini_gen.cpython-311.pyc create mode 100644 nodes/chat.py create mode 100644 nodes/chat_image.py create mode 100644 nodes/chat_noass.py create mode 100644 nodes/config.py create mode 100644 nodes/gemini_edit.py create mode 100644 nodes/gemini_gen.py create mode 100644 requirements.txt create mode 100644 web/simple_chat_config.js diff --git a/README.md b/README.md new file mode 100644 index 0000000..1198c15 --- /dev/null +++ b/README.md @@ -0,0 +1,98 @@ +# ComfyUI-SimpleChat + +Simple, no-nonsense LLM chat nodes for ComfyUI. Connect to OpenAI, Claude, Gemini, and local LLMs with ease. + +[English](README.md) | [中文](README_CN.md) + +## Features + +- **All-in-One Configuration**: Single, powerful config node for all providers. +- **Dynamic Model Fetching**: Click one button to fetch the *actual* list of models available to your API key. No hardcoded lists! +- **Universal Compatibility**: Works with official APIs and any OpenAI-compatible proxy/local endpoint. +- **Multimodal Support**: Chat with images using vision-capable models. +- **NoASS Roleplay Mode (Exp)**: Experimental Hardcore roleplay node with Assistant Prefill support. +- **Gemini Native Features**: Dedicated nodes for Gemini's image generation and editing capabilities. + +## Installation + +1. Clone this repository into your `ComfyUI/custom_nodes` folder: +```bash +git clone https://github.com/yourname/ComfyUI-SimpleChat.git +``` +2. Restart ComfyUI. + +## Usage Guide + +### 1. API Configuration (The Core) + +Everything starts with the **API Config** node. + +1. **Add Node**: Right-click -> `SimpleChat` -> `API Config`. +2. **Select Provider**: Choose `openai`, `claude`, or `gemini`. +3. **Enter API Key**: Paste your secret key. + * *OpenAI*: `sk-...` + * *Claude*: `sk-ant-...` + * *Gemini*: Your Google AI Studio key. +4. **Base URL (Optional)**: + * Leave empty for official APIs. + * Enter a custom URL if you use a proxy, a local LLM (like LM Studio/Ollama), or a third-party service. + * *Example*: `https://api.deepseek.com/v1` or `http://localhost:1234/v1`. +5. **Select Model**: + * **Click the "Refresh Models" button**. + * The node will connect to the API and fetch the *real-time list* of models you can use. + * Select your desired model from the dropdown. + +### 2. Basic Chat +Use this for standard text generation or Q&A. + +* **Node**: `Chat` +* **Connection**: Connect `config` from **API Config** to `config` on **Chat**. +* **Input**: Enter your prompt in `text`. +* **Output**: Returns the AI's response as a string. + +### 3. Chat with Images +Use this to have the AI analyze or describe an image. + +* **Node**: `Chat with Image` +* **Connection**: + * Connect `config` from **API Config**. + * Connect an image (from `Load Image` node) to the `image` input. +* **Input**: Enter a prompt (e.g., "Describe this image"). + +### 4. NoASS Chat (Experimental) +The "No Assistant" (NoASS) node is an experimental feature designed for **hardcore roleplay and creative writing**. It forces the AI to break character restrictions by pre-filling its response start. + +* **Node**: `Chat NoASS (Exp)` +* **Inputs**: + * **Scenario Instructions**: (User Message) Define the world, characters, and rules. This sets the stage. + * **User Action**: Your current turn or dialogue. + * **Prefill Start** (Optional): Force the first few words of the AI's response. The AI *must* continue from here. Great for steering the tone. + * **History**: Connect output `history` to the next node's input `history` to maintain conversation context. + +### 5. Gemini Image Generation +Generate images using Google's Gemini models. + +* **Node**: `Gemini Image Gen` +* **Requirement**: Must use `gemini` provider in **API Config**. +* **Output**: Generates a standard ComfyUI IMAGE. + +### 6. Gemini Image Editing +Modify existing images using text instructions. + +* **Node**: `Gemini Image Edit` +* **Input**: Source image + Mask image + Prompt. + +## FAQ + +**Q: I don't see my model in the list.** +A: Make sure you entered your API Key and Base URL correctly, then click **Refresh Models**. If the API supports listing models, they will appear. If not, you can try typing the model name directly (if the UI allows) or checking your provider's documentation. + +**Q: Can I use local models?** +A: Yes! Select `openai` as the provider, set the **Base URL** to your local server address (e.g., `http://127.0.0.1:1234/v1`), and click Refresh. + +**Q: My dropdown is empty.** +A: Check your internet connection and API Key. If the API Key is invalid, the list will not populate. + +## License + +MIT diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 0000000000000000000000000000000000000000..be72cff9ee082e8db860250479fe0e4d87904278 GIT binary patch literal 2765 zcmai0ZD?Cn7;f^THEHT5N{SWjn-=WJS(2O1T4xYhtX+*uX17>|IQ7QlthqF~x8&YV z>xSLp#VRSH#2@=&b6_z35&zBn8GdatKc>tN7_MP3`(vW(P}x|A#rNEgq^p80h4j4d zdC&8HJkOh67*g^(r^jN!q?Dh`v7u~AbGxlG(D&_ycsv1*`-?T_?w9Kocw&;t(O3Y6 za;d2d3&04=OR@wDUk1Y5)9H3UJuHvsBsKdC*)|+~G%*~?XZ)}So(|61C(jLS;fHQ_ zZ!b*u0olY-+Ejt5coGCkTh6UX$dpE)g+}`Hgj}qti}xJd==$!^Ca2+0bsYc6SK=PE z8?F!3#p6F!b%>6}U^APBBg2bCr}W5GU#T-osq^u52X^b7S%-16i|ncO@r#0Ef6chE z@a2uMJvA#Wcxa7&u~i6!D;I$j0TWMZQyEDKz!SM#DxdP3?|1nef`dnyAuqI@I{o({ z-B-GGdF1E#^?B1_ixN#H;cP*^!?y;=k-0uU_r5)n+rZCG9Mg+6kKll)s!FPsk~J7D zu$@va2SlIn!(PFRVwLu2_tC%2>vpj*z-;&O-13O`Eo_TX#@9~*iOCoYacYtxV=4PX z@H4~0h*mR&^pvV8`R8OM!&DJwuRgeI^Q(dH%Uy0!6rWG2S@*c4K{}_%46>RwsYV0= zPqI_bhtf)3m{z6=aw^Y+f6q$@-FOi9`Vf30*aKw(n;~u6fsCoCaRwt(Qid6A?*&B1 zJC=}vZ1{}5i>EDZXR!JFa7dANO5>D5vDWexhCyGn+<2#0<8}Bm1i^umF{NKDN zIyn%$;ktw)17x(I?81i&MiX3@KvZysQ<4|}F>OpSe)57iZcH*>zR$Q^X7)(&NB0j@ z^-~q68}NtnKa&S&5!H!cC=?Q#CX*VL3A?+>Y_=86*qaeW8PDNEF>BCZf=xT!0|tOc zQYwS7ZE+xrE)Ko%zznfw_v*E$UTQ)r9;1e&TDefbB$y$3k=b8qNO4SX7@KW zDTOkOO{$Dd7+v1gFY;P&JGMWkq*J-9qH2-;_4f@3Xmq;ow@T|P0ZFnPOKXt8iOSss zkIrY#E+lX=aWicjZfDw0pC1dlE>-^|NHb%o zdI9^pwDI~p5G<0E1}<8DH+iG&ls+5iUhBR5B>GX_CTh-%$UQ{ss9xb6ZCyQV$dt4( zcp{XS>Xk6s`7Ip7l@{i-qLdkrLO|5miyDq#8M*ddMJUG071cO)}*@5c!K6h??)BEj}<&pnNV1@Thxio5p^<2fECiG_E>y;LS0EGPt zVq8gO%=blPn;*g;N*oQ=U{$7NhS>or6}x)Ay>bo%^q?A|g7*TBYg6#7yv&0j48J(Z z3R0fQs7!u$&laUAdObi4FF!tfdxTO`9Ov#@b6^LhCyqH^@sp-=Lm~GHo=E)jgS=Ap z4!vc-bHCH&q#o z<+uK>ha*eO zSv=e!I6T$Gg}Z!t`*>v|*txo{%HeK)Bp|Ljl*FO*n<8OasZTL<67JBwWd9^+N>U7Zc`7 zo*k}ZdiVe^FaYK-Z*vZB@Qz>zj%bLEWJpfV$T_khb4)MTd8c3$IKTr^_)KqpoBIPC zAUqL>NFIBK~;;tD-F*%yL_eZ_ASS^ zN#op%_Tcmk+wBfKr37?6OhS0(1#o6vQ!f3<-R7*oNWsc1?Ln%xDx)?_ICn??Z=1$^r$RO?=zDgq|N+ zjuo4c<+;ngEyo-ZI3z9!GF?si&Cz;PbIIx=m4@e@S;M5XV2;R}C|Oy^I&-s3Wba5y z(+o0gi@3?klH9~rROJ#iB+L*(nxv%$o?fNn5);rCGPJ7*a zf%f&oX5;AK@VHwoB{D+TJ4GmwvStPXtyMth5(E;190N9%L7qY)n7$Phm{??>GSHZU zx-o4Vp|9YYir}W(+8fmMR%0+54El3CcP{OAOv@cidylfXe>IB^3=pET5+PLcFK_bC z=md;x@;taj-Lr;s4~Z=;Qzq};I^;8-Q_(2>R5eSdRH2k(mdF# zPRjQum4|F+hg6d*>;Lke>`V-#}= zOE7~bYn2L@V}PeZQGRJjYOz9EevyK6er{T2sHbjlW^O@Fs&htSi5}NWMxa5OjJJ4- zQ%gz<;vuSwn1O(--n{_PCeB%#3mj#WAj17QW0Aup8#@X1&ym6`Ob{?&;1J-lzi8M2 zjQqN)9zKSoWZB*vaKm9uPj_`scXfC5*H!(Y!(k(EB|CQo|J*>xf8$H;GG_y~{|bS( zi9p7QzzBvAGtSVbVcY;uHe{G#$61D0G1V9{&T!)#Lj-Qz6n3-{(Y%#N_--0Ey+p#s zGvqA$5;?<65yAXEQVXTl_Zh)gXUU84~DoD5A zBfT2Z9cASjgc^vi6r696@hZVEMm(US(D~5s7g$CFa0KU)Ml3))wjUMkVodP&^YeLm2aUOtDRq zp}?#ldgVwsEJDHFUSIEkZ=k0?wmF;9=N-tULRx>1ueUc=uS=T_1Ye!SY+6!XJ}DBN zm1#WW*q#4SpT01f;?CzN6y@vlJqC^4G9_X?y}i(klc9h-ooj;EHvmP)s`KeSEj{)~ zX^dAq8;wED6C`avuJ-{{*p55e8;ToLSN_ z6oLYCnIYt3_-og@L@>N(m@^2hV2qP@$$N$%Gsjwq!sa*#IhFs`Wr+MZpU_XvGXnP- z`vQ5J{RiWmF>YMa<;t^>xFOC8rccb5A@$=tM@eoj&Yd#m@(7k!={Kc#oLk~@vHE9m zP0qty&)35wSl=LV)7o_rY*XYil>KqOO@2~b)|@$RmZ}s-;cIbovCP7DrLvGJR1`GC zq(R{DSNw(Z5SyzVL+*!X3^mz?C53BRNe|JNE*sAnG~LFKmtY*A4&10d_ftyP(@K=; zLYqOKHqKcTS0P5Q$1ON|iY0}$5V$$Z>z1%Zd!Au(+IKw}g7$O7I!964=~SJj>5joP z89L;h1YH(Tf+s>E4@$D@N2dEnb2n9sa@sL8t4v3vUf@ssPeJt`(oSZVlbg;jR~#gU3){3$v{Zn z2Z0>pE~t#$2xOiln*7?m+_D&0GG3g@N8E;Vk~RtfB_Np~80-95p%Sm~qk&*Z6nG_~ ztw6li2U^$2dw7oxRFbSn!6;~}U}RcRqG`u)C@6+uwUK4$gETuODru9f1e94hZ9D}_ z1iVEgMHJFC&m14|kBlBVI^v%=a?C&S^jO+Da&}UTV&|o~s1yt<9;;-9YDw6EX%=*# zVg#f%M+lP{s(^ZET0Ti$-{x34Yrt(d=XUU)6(_N#7xhHwrnU}O5NG39c<_4rcjs;e#GYD=}WeOUi~ z{qGw-Xh<}4mfRQoDLMp}mCw%p;p`RD7lvf_u-ZKg;~?4OSDXA7MsIri5)IG2w{xle zcl$2xTio}T-G{&3rS5(%Q?YhcZqX{=7qoyCSq?|d{M1vPZ-pQk)=6x@h?S1CTB-($Y_H$Ydj(u_S5-RwzD{fr}_EoH# zViz36rI)-F!DWWL`Fu$&A1PF#IHy#*el1I3(wpjhZog*{=cCXoAx7dG{Xw2n%M(WM$BI4mhBk~%R| z2PAC_2WG@{G1$fZ7zMk?kp;UZ1iPOZ;|zdY^B9qE_-O>YBnxpVrjig`=HRHddA$c5 zoy6QtKoE`&X!p|4N)Ln<1Nj&UIV9ZEdMc%64D(1@kUWZ{70Fg0Xze#*eX+o00CHO+srwN zQDvRAD?{!@)`gR`4#;^gqQ{ZR4RK_~M7Ho3M&U8=_!VL0=*rPW|D}WP99()~`FOI? zr#AW$jlOS<|7H2JCGqS_iJ!faJot0<;Lk6NeCKXiYWwt&q`OOXLyfrFo>WVFh8XL6 zC|Ml3(d=1n`SiJDbFbRm3zBH+N%8F;I^K6I2R;-3Ag=hYwESn=AKSh#eQQYWc}m^$ zRI=-7wd?65e?;YvB+jXP4ZRzCiUcdNyW=FZqZlN6S+*6o3QM2XUbql-j&OQ6Fw3iJ<)p)QP z+;2^b`6xIn`l{Bb;`Tg5Qy!?@$V_WeE=2&AT@<1kI7f!$$W4oV!KnQp-`xxc3Gn>E zqQ{^jOv~TbAE+UJY#tqCzpVG$M;p0oZBBT(_LvjmzqVoc>qgsXyY=gK7Q-F}!yPp2 zr96YS16K1lRu=fbu`?KU&~T0IKpXoF&jBq$pLr9V9~8oaqfcs@W9JMHV3=cbCZ{gWxasAIjY_7+sk#IvEY9!o{D{tP*m%fIP@FyvS z8VPl1m>LPKdh9AV;?{MHgzTL4bt~XH`qbgNbz}&*t}@o5gLE2*qVA9b)YeZegA_)^ z>{$#q2&m}^4I_CN2|TlQN^%u7qvZ@);m-&ktve)n#Tb2Q_nTO!c_LfJUx9)eTFtof^3|rYp*W8M&G- zjHa4eF1`59i%ZihhGbKp+5}%S)%qw|WKzxirP+68DOzdvsm;DcS^e5g zyG0v*=f0bt6%@O+oNp0~ze8!}0Vy?_l~(d0tM{PIDdVRLeQ9OZYy02Q>IapQo}X57 z?T!GL(fEI7H2atOkxugOTShjsU)JvL@IS_VHRyzwt2IuDzuHE_kJ&~T>otaj_-jT6 z!yFA;kmuTF+sIb)wXH0Mw=o#rPQ#tHQ73zCr*qW7e(m5OJPCRse;FZ!3-;|05ZoQk z5hb5U$dcY|l8?U(p@NqX7IHW2C&^OYm`b4IW}; z0+|c@@+{i~?klBXsOvw%UmgSkE`l1@{NWo7o!1-sRvY^AT7%mGT7uiLP?fScuUqO@ zE%l4tNlS-n=}6E!RZ*KT*3u1bk(g1$_`mH3fxcZNVi^uNj1fgAyn;LS6y~AV5_VMC zC6u`11on+mXXnF~Y%E4Q0#{Zt^N@8e!BK*_tYm|N8}aNVlwmWJm&Y38$G}k@MHrwz zMN#ltGMCW}CpjTG}QS;;h*{hg+%2)e+PN94RQS4;%roIF}xRY zR^)T)ar#C%n<1w?pA%7H_S*2922hXpEfsKbuTWDT0ye7YPJ{U`fIe+atIk|mT}Od*mGFhhKq>Sdof-{^)r$6-{*bv1&+5MvwQ+(Le@O!B#$(x zuD0tg&#KF_+_7>Z>DsNjc3*c5uDS-3t|8Smv|vd&>)-NTcebuNTbB+kOG#(1>g-)G zr(B!gdg8ik>#A$(()fxq>FQTq{RLc(QhvTDxmu z^oFZWg$yqwU7f0{Ghy${SctnfRo9=YJD4#WH`%}{x5>6(&)7-zCRo_4J5%n~>+Wr< z?rqDKl}D5A0o6UQa1g#?f8@g1)r#gsMe~h@uH`dVEXjr;wP7exTY4{yE{vuewTqip z!LROU`N_F&*18u)zpHYpRs2$2vTD0pwLM|kP8Zymvs~M+t>UrcH5ISu6xFGGSkak2 zE(31?>l81F6vdp+&mKC-*TOG+09*$;oUUrtWBbrs309X>7{1EQB!eNI|sa52_>Z<&>4 zxQa!ik0C2Ln6sMXdXrUsYE@st(x=Jn(V}BHI$R@(Qvgv#a43QLvK<^6n;088{=&#{ z3B{7O>&;tU1 zbSoE3-*Gi6w^`*{u5)dxTw9`jB*~4c+-Szo#(6RX$Sszb4luc>AC5A_40RZ}~Ec>eI+l-UXDngP|HBdMA?OqoB7pL63j)^0&x zQ=T}@_?jox`2@<+)=+E65FiOsli6F%Y+I5u1Rg7s^l?S_qVMw{4VIGj4viTe$PAAh zGX&|oj*K&P8aF(U8y-1s2-0^Q>E)QZW#jVLCsr6OOkF9d-ot>!r<}-m5ahR(2J)`C Kjs1XWWB(WWKEql7 literal 0 HcmV?d00001 diff --git a/api/routes.py b/api/routes.py new file mode 100644 index 0000000..1509914 --- /dev/null +++ b/api/routes.py @@ -0,0 +1,201 @@ +""" +API routes for fetching model lists from various providers. +""" +import json +import aiohttp +from aiohttp import web + +try: + from server import PromptServer + HAS_SERVER = True +except ImportError: + HAS_SERVER = False + + +# Predefined model lists for each provider +PREDEFINED_MODELS = { + "openai": [ + "gpt-4o", + "gpt-4o-mini", + "gpt-4-turbo", + "gpt-4", + "gpt-3.5-turbo", + "o1-preview", + "o1-mini", + ], + "claude": [ + "claude-sonnet-4-20250514", + "claude-3-5-sonnet-20241022", + "claude-3-5-haiku-20241022", + "claude-3-opus-20240229", + "claude-3-sonnet-20240229", + "claude-3-haiku-20240307", + ], + "gemini": [ + "gemini-2.5-flash-preview-05-20", + "gemini-2.0-flash", + "gemini-2.0-flash-exp", + "gemini-1.5-pro", + "gemini-1.5-flash", + "gemini-2.5-flash-image", + "gemini-3-pro-image-preview", + ], +} + + +async def fetch_openai_models(api_key: str, base_url: str) -> list[str]: + """Fetch model list from OpenAI-compatible API.""" + url = f"{base_url.rstrip('/')}/models" + headers = {"Authorization": f"Bearer {api_key}"} + + try: + async with aiohttp.ClientSession() as session: + async with session.get(url, headers=headers, timeout=10) as resp: + if resp.status == 200: + data = await resp.json() + models = [m["id"] for m in data.get("data", [])] + # Return all models, sorted alphabetically + return sorted(models) if models else PREDEFINED_MODELS["openai"] + except Exception as e: + print(f"[SimpleChat] Failed to fetch OpenAI models: {e}") + + return PREDEFINED_MODELS["openai"] + + +async def fetch_claude_models(api_key: str, base_url: str) -> list[str]: + """ + Fetch models for Claude provider. + Tries to hit the /models endpoint (supported by some proxies and newer Anthropic API). + Falls back to predefined list if fails. + """ + # Anthropic API format for models list is /v1/models + # Note: Official Anthropic API recently added models endpoint support + url = f"{base_url.rstrip('/')}/models" + headers = { + "x-api-key": api_key, + "anthropic-version": "2023-06-01" + } + + try: + async with aiohttp.ClientSession() as session: + async with session.get(url, headers=headers, timeout=10) as resp: + if resp.status == 200: + data = await resp.json() + # Anthropic format: {"data": [{"id": "...", ...}]} similar to OpenAI + # Some proxies might return OpenAI format + models = [] + + # Handle both standard formats just in case + items = data.get("data", []) if "data" in data else data.get("models", []) + + if isinstance(items, list): + for m in items: + if isinstance(m, dict): + models.append(m.get("id", m.get("name", ""))) + elif isinstance(m, str): + models.append(m) + + # Filter out empty strings + models = [m for m in models if m] + + if models: + return sorted(models) + except Exception as e: + # Silently fail for Claude since it's common for this endpoint to not exist/fail + pass + + return PREDEFINED_MODELS["claude"] + + +async def fetch_gemini_models(api_key: str, base_url: str) -> list[str]: + """Fetch model list from Gemini API.""" + url = f"{base_url.rstrip('/')}/models?key={api_key}" + + try: + async with aiohttp.ClientSession() as session: + async with session.get(url, timeout=10) as resp: + if resp.status == 200: + data = await resp.json() + models = [] + for m in data.get("models", []): + name = m.get("name", "") + # Extract model name from full path (models/gemini-xxx) + if name.startswith("models/"): + name = name[7:] + if name: + models.append(name) + # Return all models + return sorted(models) if models else PREDEFINED_MODELS["gemini"] + except Exception as e: + print(f"[SimpleChat] Failed to fetch Gemini models: {e}") + + return PREDEFINED_MODELS["gemini"] + + +def get_all_predefined_models_list() -> list[str]: + """Get a flat list of all predefined models.""" + all_models = [] + for provider_models in PREDEFINED_MODELS.values(): + all_models.extend(provider_models) + return all_models + + +def setup_routes(): + """Setup API routes for SimpleChat.""" + if not HAS_SERVER: + return + + @PromptServer.instance.routes.get("/simplechat/models/{provider}") + async def get_models(request): + """ + Get available models for a provider. + + Query params: + - api_key: API key for the provider + - base_url: (optional) Custom base URL + """ + provider = request.match_info.get("provider", "").lower() + api_key = request.query.get("api_key", "") + base_url = request.query.get("base_url", "") + + # Return predefined models if no API key provided + if not api_key: + models = PREDEFINED_MODELS.get(provider, []) + return web.json_response(models) + + # Default base URLs + default_urls = { + "openai": "https://api.openai.com/v1", + "claude": "https://api.anthropic.com/v1", + "gemini": "https://generativelanguage.googleapis.com/v1beta", + } + + if not base_url: + base_url = default_urls.get(provider, "") + + # Fetch models based on provider + if provider == "openai": + models = await fetch_openai_models(api_key, base_url) + elif provider == "claude": + models = await fetch_claude_models(api_key, base_url) + elif provider == "gemini": + models = await fetch_gemini_models(api_key, base_url) + else: + models = [] + + return web.json_response(models) + + @PromptServer.instance.routes.get("/simplechat/models") + async def get_all_models_list(request): + """ + Get all predefined models as a flat list (for COMBO remote input). + Returns a simple array of model names. + """ + return web.json_response(get_all_predefined_models_list()) + + @PromptServer.instance.routes.get("/simplechat/models_dict") + async def get_all_predefined_models(request): + """Get all predefined models organized by provider.""" + return web.json_response(PREDEFINED_MODELS) + + print("[SimpleChat] API routes registered") diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..0d17d73 --- /dev/null +++ b/core/__init__.py @@ -0,0 +1,51 @@ +""" +SimpleChat core module. +""" +from .providers import ( + BaseProvider, + ChatResponse, + ChatConfig, + OpenAIProvider, + ClaudeProvider, + GeminiProvider, + PROVIDERS, + get_provider, +) +from .image_utils import ( + tensor_to_pil, + pil_to_tensor, + tensor_to_base64, + base64_to_tensor, + create_data_uri, +) +from .noass import ( + format_noass_prompt, + build_noass_messages, + extract_noass_response, + build_full_history, + get_stop_sequences, +) + +__all__ = [ + # Providers + "BaseProvider", + "ChatResponse", + "ChatConfig", + "OpenAIProvider", + "ClaudeProvider", + "GeminiProvider", + "PROVIDERS", + "get_provider", + # Image utils + "tensor_to_pil", + "pil_to_tensor", + "tensor_to_base64", + "base64_to_tensor", + "create_data_uri", + # NoASS + "format_noass_prompt", + "build_noass_messages", + "extract_noass_response", + "build_full_history", + "get_stop_sequences", +] diff --git a/core/__pycache__/__init__.cpython-311.pyc b/core/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88fc3d6434f702996b4159ba6b8b2065a85e26e9 GIT binary patch literal 1130 zcmcJNJC74F5XZfn_hUED3qlC#+7w)cu0ROs2q;PccS;0>70KGmIyp-_>&4zhgoNk}KW($8bhtmp6XeD%C4vE%#iQ}HWD$PZlXhPyScl{G$- zkWdmjp&L2iQWtVG2YH%@0xdw17NJB-P^M+5(29d`xu^;?T7x>RgGW8^sSgd>So`u( z6I!$dZQ6zo?Le1y;fNkNWI@8h7l-!3;)L`|AMaNu0+NUiu7dPDEVH5T$wrsj=4@`thuS;Y9QmeSQ!62;qlpMAL$RGAPWzF`a=#P7JGl? zVtoE~^y1}sV(JT?vSgd0Uoc+EV=Wa+B}+tPJPWipyQX>YURr6Vk4W;*U{|ObqK@zoKB9qWB3g(xqJ!vK zyzbvKCB{NIW6V@`HPio^(9{IjE3zyVk-jy%DXfchs|ZvO*cz~JDg#F zm<}Y&mv2UP0vYdCk6r~Lp3X8`C9tIqx!!vY%{Rs7n=)owBxUR*S>BYNZd&n~dT4WC xH|nnCD(5)P%5|OmiXe&=$6b;Ci*6Bf+!g*^4_GT`_G(tk4xdW+sRywTyJn5q7H!aH7THyMO`%{BZL7B0TZ9#?o9rSd*cw|wreOrg z*g?i2c$+jiL?=i(BNoBAX*F7j?1*#;xqgpQ_>?fOU$jC2d81qtNQx%nd`gb4Mdhd{@sR}LrxLNq`pr3B7UNO^Nhi2HY{$HhR;`!TWl@^D230dr zqFOW|)q+Ghh2mklW-^p!co=wI_nYvhW~e;8>`%|2>4n}b(04(aqL3{^38r4XeFKkg z)UNj!$LGmty|Y>|$_Pvv)|SU>+m~eh&?ssZTR_q?nE8c1`v#q%S1q+ZfvvAnFjrSU znxXSGr>Y5(Qec~P$)37E-GMT+V0meM2G+dLEn1*j&TFgA+uc%|{r}KbJA>BRxc%DX zuDTt9qt!~kqW)kq?!bN8=S=^d;~`BEmz0}4ke3tu^|>oN@wgKlk-UhOr3q5Q<1{Cn z;1^axNs%AE9DIX91JY0M6^Y=n)De&@>fm52A8^miHH0ks3!;vVGWIjr; z5{>Y8LUMRT6ntdTJr-<5_+a+llI-oRQd{lgN-EwaS!{K%G{W}q_U7*O!wW4 zOQ~csfn-tO!z-b9TwDVbrsDDh?{lb(5RIu!IGIw-@l-6iu3AEpyq*+QQ#`4fQ_;A5 zPBlx|Le(mLni9pdi10F0djut7A%sHfDt8MTIE_#O`Ai7AugXA=gjdS*Z*I*_EM1(N zom#3w9Ew;XUV95%4MpQi;glpNV!=3AAn82|(O7a#oLULVOW_0(m&mRJF>OdEV5KhY z0D^KNn2fHW0hrAJk5qvyOI124TiciE`{^$-o0;uTOV*x(wWsVkl%s#!`@mAMP)_gT ziANI;C$}bd90MiCK+!R%I0o}SDmsRCxIl>u{O!)ml_K}P!o8oJ0(qNbbA8*pxsjtw z*4~1(x7_A=eD~4a+_8s=t;7Rn*WJ1G%bfJQqu>q{-2ufNDA)pJyK}2|`@OA!+|iP~ zuVC-1Np%(6V@3Cv;vOs5##%~s7u+X{?vsl9WWjdw`?j7!@60RdoAqzK-)uk?{>8Rg zrERugn|%#7Q>vNlkbw@dI}kcV;3$|5SFt2XI_c4I-q3EkOGpZJwIV;E2}~Q5tdkZH z7#WSH$WTVm<+11NOL9YNEAj*KgtOhCWmkRps!nwo+BnaKDPzhvP-=Zl6D6}%0_y^K zJ&M+)DTuEJXg*ldvR@J+pEdn!1Ig5(4*X(_c?XgVInjwa6SZKFk4V(y2~hdrQ*|mb zun%S})DHF`H)PAi(C-pI-@`yf_?|+@#m6DM%c@n2)iS~qtC|VcQ77q%tR+J7>GxE# zoIv3fmBU;RB>aJJTv6?*c$F-C3>t(M)Q2xt_(uAEqGKd!VNe_;N>LOEPp z!`o*cj%|(YaD0j5|LpzxaFO#Xod0>M$eqbf?Yg{*Yakacx{fKXW7!#u?1Ok>$8)sg zIhsFK^b9GUp$C?-t7FGCP=cYMqKj8te0JtLhbMa#9l>@m*XL(d3*2L3K-hZdgdXhM z1bC72p~Sw|qut|x(M_E5ROla0vOU;q&fnAVk0 z874!&WQ|t@nreYd=3P)|T-Ceb5Rhd_|I>ghBbadbz7vwf@iV+6qiB5jKy=kYcx|i} zcePeM&TFSSq1^=VT@}ulABF?>^OvtrL;msfrR%;Hz`UfbL5znXvMr?$!sYnDa;@Cd{ja4|dYo=pFYP!)`=%0c|`W0kZ>W6O1b0ilj zwht-oL)keEJem8MFFxP=Jl9>a4i>C~`MIy>UqwpCCkw|X%bf!|okOM0p}bh^^edhI z2lnq>oeyts-OkRGS=X2I_vg3A^FPkc7g?Xe`UIo24xcf%y(J4iKUn&KpQEuH?O!D|gZik-IRirPsW=(Tl|C-^4+B8S_;6>FLs zo!k&B$ijRUGzQBjPNlh!cHd(YNij~mkFYfG`ZyNjh_;{*l0$^(jr88Gnb*MZ=V8!o zc%%;?1ApqIygdr=>`2iQP&|R`+snw0_9;Vd2++1a!6y)HWVa1QZs5}l zFkn+ZE0{C%SCnAMu<)V5lBi$MP8o?IS>!`J-XH$vjX54i%IcQ``Ru)TeU3*WT&yJ- zzIn-@gTFEkIz1we(W_fO!C#%bI?dya;D`0?@oOIxW2;GVxmN#7ypwE2TED2RRs$B9 z2KivGkRLLYC4|SINq9+X4nS1b>R5H!j7^?M+s$OxU$g-C! z9M#tI_>)JU{5JS+5I&em=N(@}`G|*T+BKI;v7)V6|SD1o?lPw*iLq`B%zjUz>_%Uuq%-@5abv(F%iiMCkS~1qt0}V-DIYF zsP0K(GZqnwz#t@i%Ss`VPs^gc_`o4Yj-1eBWobuqK$bY|tx-7T#P_OtIx|i-^mM;^ zRrRXs)qC%IRr8lpslf1DG{5COJjB?)sWN!RW9Hevz&vCQTVZ+poJq$NhLgr;ZDbj9 zGT^fKa`^HhB@AKIQ^y9gu=xDkIdE0aAu5h?#ONqgE1i!l`q~La=!5!Cb_^WMEa06cRo3~sKg7G-gpaHRgg1iZ> z>{T(#@4Dd{Eg|h%C?pgXb@!eKc+-}*N$sE|?At;HbA^8TMj&KApX{=q3D$#9H2c{# zY}=RXSNnxFM5?aWYKQ&2;yt@5`gvNQs@RMaVOx4ND{(Yqhz4fjd-f9$M2yifA2BEO z*xaHldY%VDa-z+!B~Ogd0+dc z=6iRrG35i;iEBRw;d(>BB_y~YSA%)QQB7}lo-eCSlWUu;kbjfk^<~51fe1s`E0K;( zGS5Gd;*RUL$tOeW#oZJ(6g@gzD#_>J8885NR}oQ^k%#X!b(wbFqBnGSDsL)Kryr+YJxYOhV*uWZZ5L>Z(-&_b z@u;{OhEa@O-2`;r4*0Eg4%`qxRjQ;Y90{)kj$e`vb%-NL#3Ms-FPx={hn%YNBz363 z<_AMnj~U<+7}Zf1Yl&n;spO99AQwRtxjtyOTE3*i1GyN9eP*#?w;fSIvWD8WN10$K zmB(Up3KBG(4_z?V%(a&A7M5oDCDC*}R}WW1bt)OyP;Xg-Ymf{R)aqFks4A&`nhvmj zHpYEFO^DjqC=_NE2iPPtND;JvcK{X2P(zZG=J@x0?)h|_)f(8+XfI4dNh`L`EdK?d z#qUZt)N;wRjw?kiY^={!Dn>tJx3Ep8Vrt-1@b+`@)`Eb-7D$#;)I5EQVkgGlKtvu# z*ExJa0|e-MlTA$Czr3FR}3sHetrQcoq z&9&ad(caOoMvP7WJT*cs^GGIQApc>emHjUfR6z=CZ2JB+d4!ZI4YpOKn5vP&gNOJb z+yPf~_|dUwcFYdD!{k>wY-b=nQg+Ny^f~5>3(O|Ns05e8!#+mk)1(2oOK<#EMb!`P z;X+aiUvk$#ca@Uk1JrMafLa{g*Ar9(Ykr%K$@5-Bu+nPmn;8?BxWi0UZTuv~vGZ*q3$@2f#pg z`87!Wki@TmAONNhre8|8Q`^mM_7vV;`Sj=Iv!9jEesbk$`TSn_{FB3bUV4aa052btM+Mj}X&5ei2i4GgO@#xzRZan(-^EkiE;#<$b zo-C!!jbAJ4>70o>Ar2*g>6Mw!uG96;PQV*G@TF_P`)y>TSI&@Jq4XJdz;CjIol5*y zV3PhIRI)>7^eJi`^_@J7wt(8mfp|28CpCoYP@)>=YbA0Y+4-pQR zjNat2?)VM7AN@*~pRDNe$%nf9$9Y|Lb2s)UPjtsGw(+DFPgDeVshBf1MD4p+P*zdvWXDALsV0V^Nlx*(kXX zE0~=y-ijDVZ|Z1dqEz!xV}=w_-8q)YNBM!ElGbEyu*)f9$%q)yUc5h(D*KqhQhgZ) w^|Ff9TUD&yt75U)d{k1bnn%wT)7h-Ci1~@`oYLj)yHCu&y{GxtQ{}w>0ri54e*gdg literal 0 HcmV?d00001 diff --git a/core/image_utils.py b/core/image_utils.py new file mode 100644 index 0000000..e558b9c --- /dev/null +++ b/core/image_utils.py @@ -0,0 +1,122 @@ +""" +Image conversion utilities for ComfyUI tensors. +""" +import base64 +from io import BytesIO +import torch +import numpy as np +from PIL import Image + + +def tensor_to_pil(tensor: torch.Tensor) -> Image.Image: + """ + Convert ComfyUI tensor to PIL Image. + + Args: + tensor: Shape (B, H, W, C) or (H, W, C), values 0-1 + + Returns: + PIL Image (first image if batched) + """ + if tensor.dim() == 4: + tensor = tensor[0] # Take first image from batch + + # Convert to numpy and scale to 0-255 + arr = (tensor.cpu().numpy() * 255).astype(np.uint8) + + # Handle different channel counts + if arr.shape[-1] == 1: + arr = arr.squeeze(-1) + return Image.fromarray(arr, mode='L') + elif arr.shape[-1] == 3: + return Image.fromarray(arr, mode='RGB') + elif arr.shape[-1] == 4: + return Image.fromarray(arr, mode='RGBA') + else: + raise ValueError(f"Unsupported channel count: {arr.shape[-1]}") + + +def pil_to_tensor(image: Image.Image) -> torch.Tensor: + """ + Convert PIL Image to ComfyUI tensor. + + Args: + image: PIL Image + + Returns: + Tensor shape (1, H, W, C), values 0-1 + """ + # Convert to RGB if needed + if image.mode == 'RGBA': + pass # Keep alpha + elif image.mode != 'RGB': + image = image.convert('RGB') + + # Convert to tensor + arr = np.array(image).astype(np.float32) / 255.0 + tensor = torch.from_numpy(arr) + + # Add batch dimension + if tensor.dim() == 2: # Grayscale + tensor = tensor.unsqueeze(-1) + tensor = tensor.unsqueeze(0) + + return tensor + + +def tensor_to_base64(tensor: torch.Tensor, format: str = "PNG") -> str: + """ + Convert ComfyUI tensor to base64 string. + + Args: + tensor: ComfyUI image tensor + format: Image format (PNG, JPEG, WEBP) + + Returns: + Base64 encoded string + """ + pil_image = tensor_to_pil(tensor) + + # Convert RGBA to RGB for JPEG + if format.upper() == "JPEG" and pil_image.mode == "RGBA": + pil_image = pil_image.convert("RGB") + + buffer = BytesIO() + pil_image.save(buffer, format=format) + return base64.b64encode(buffer.getvalue()).decode('utf-8') + + +def base64_to_tensor(b64_string: str) -> torch.Tensor: + """ + Convert base64 string to ComfyUI tensor. + + Args: + b64_string: Base64 encoded image + + Returns: + ComfyUI image tensor (1, H, W, C) + """ + image_data = base64.b64decode(b64_string) + image = Image.open(BytesIO(image_data)) + return pil_to_tensor(image) + + +def create_data_uri(tensor: torch.Tensor, mime_type: str = "image/png") -> str: + """ + Create a data URI from tensor for API requests. + + Args: + tensor: ComfyUI image tensor + mime_type: MIME type (image/png, image/jpeg, image/webp) + + Returns: + Data URI string + """ + format_map = { + "image/png": "PNG", + "image/jpeg": "JPEG", + "image/webp": "WEBP", + } + format = format_map.get(mime_type, "PNG") + b64 = tensor_to_base64(tensor, format) + return f"data:{mime_type};base64,{b64}" diff --git a/core/noass.py b/core/noass.py new file mode 100644 index 0000000..f1db63b --- /dev/null +++ b/core/noass.py @@ -0,0 +1,174 @@ +""" +NoASS (No Assistant) format processing for roleplay optimization. + +NoASS transforms traditional chat into a unified narrative where all dialogue +is combined into a single assistant message, with character prefixes marking speakers. +""" + + +def format_noass_prompt( + system: str, + history: str | None, + user_input: str, + prefill_start: str = "", + user_name: str = "User", + char_name: str = "Assistant", +) -> tuple[str, str]: + """ + Format conversation in NoASS style. + + Args: + system: System prompt / world setting + history: Previous conversation history (NoASS format) + user_input: Current user input + prefill_start: Optional starting text for assistant response + user_name: User's character name + char_name: AI character name + + Returns: + Tuple of (system_prompt, prefilled_content) + - system_prompt: Instructions for the model + - prefilled_content: The conversation to continue from + """ + user_prefix = f"**{user_name}:**" + char_prefix = f"**{char_name}:**" + + # Build the conversation history structure + content_parts = [] + + # 1. Add existing history if any + if history: + content_parts.append(history) + + # 2. Add current user input + content_parts.append(f"{user_prefix} {user_input}") + + # 3. Add character prefix to prompt response + if prefill_start.strip(): + # If we have a forced start, include it + content_parts.append(f"{char_prefix} {prefill_start}") + else: + # Otherwise just the prefix + content_parts.append(f"{char_prefix}") + + prefilled = "\n\n".join(content_parts) + + return system, prefilled + + +def build_noass_messages( + system: str, + history: str | None, + user_input: str, + prefill_start: str = "", + user_name: str = "User", + char_name: str = "Assistant", +) -> list[dict[str, str]]: + """ + Build message list for NoASS format using proper Assistant Prefill where possible. + + Structure: + [ + {"role": "user", "content": system}, # System/Scenario as first user message + {"role": "assistant", "content": [History] + [User Input] + [Prefill]} # The entire story so far + ] + + The API is expected to continue directly from the end of the assistant message. + """ + messages = [] + + # 1. System/Scenario (as User message to ensure it's read as instructions) + # Using 'system' role is also fine but 'user' often works better for "Instruction" in NoASS context + # However, standard practice is still system role for actual instructions. + # Let's stick to 'user' for the "Scenario Description" as requested by the user's mental model. + if system: + messages.append({"role": "user", "content": system}) + else: + # Fallback if system is empty + messages.append({"role": "user", "content": "Narrative Roleplay."}) + + # 2. Build the giant Assistant Prefill block + user_prefix = f"**{user_name}:**" + char_prefix = f"**{char_name}:**" + + story_blocks = [] + + if history: + story_blocks.append(history) + + # Append current user action + story_blocks.append(f"{user_prefix} {user_input}") + + # Append the start of the assistant's turn + if prefill_start: + story_blocks.append(f"{char_prefix} {prefill_start}") + else: + # Just the prefix to cue the AI + story_blocks.append(f"{char_prefix}") + + full_prefill = "\n\n".join(story_blocks) + + messages.append({ + "role": "assistant", + "content": full_prefill + }) + + return messages + + +def extract_noass_response( + response_text: str, + user_name: str = "User", + char_name: str = "Assistant", +) -> str: + """ + Extract the character's response from NoASS output. + + Since we are using prefill, the API might return JUST the continuation, + or (depending on provider behavior) might repeat some text. + Standard OpenAI/Claude behavior for prefill is to return just the NEW tokens. + """ + user_prefix = f"**{user_name}:**" + + # If response contains user prefix (hallucinated next turn), truncate there + if user_prefix in response_text: + response_text = response_text.split(user_prefix)[0] + + return response_text.strip() + + +def build_full_history( + history: str | None, + user_input: str, + prefill_start: str, + response: str, + user_name: str = "User", + char_name: str = "Assistant", +) -> str: + """ + Build complete conversation history after a response. + Reconstructs the full text including the prefill and the new response. + """ + user_prefix = f"**{user_name}:**" + char_prefix = f"**{char_name}:**" + + # Construct the assistant's full turn + if prefill_start: + full_response = f"{char_prefix} {prefill_start} {response}" + else: + full_response = f"{char_prefix} {response}" + + # Append to history + if history: + return f"{history}\n\n{user_prefix} {user_input}\n\n{full_response}" + else: + return f"{user_prefix} {user_input}\n\n{full_response}" + + +def get_stop_sequences(user_name: str = "User") -> list[str]: + """ + Get stop sequences for NoASS mode. + The model should stop when it tries to write for the user. + """ + user_prefix = f"**{user_name}:**" + return [user_prefix, f"\n{user_prefix}", f"\n\n{user_prefix}"] diff --git a/core/providers/__init__.py b/core/providers/__init__.py new file mode 100644 index 0000000..39c30d8 --- /dev/null +++ b/core/providers/__init__.py @@ -0,0 +1,34 @@ +""" +Provider implementations for SimpleChat. +""" +from .base import BaseProvider, ChatResponse, ChatConfig +from .openai import OpenAIProvider +from .claude import ClaudeProvider +from .gemini import GeminiProvider + + +PROVIDERS = { + "openai": OpenAIProvider, + "claude": ClaudeProvider, + "gemini": GeminiProvider, +} + + +def get_provider(config: ChatConfig) -> BaseProvider: + """Get provider instance from config.""" + provider_cls = PROVIDERS.get(config.provider) + if not provider_cls: + raise ValueError(f"Unknown provider: {config.provider}") + return provider_cls(api_key=config.api_key, base_url=config.base_url or None) + + +__all__ = [ + "BaseProvider", + "ChatResponse", + "ChatConfig", + "OpenAIProvider", + "ClaudeProvider", + "GeminiProvider", + "PROVIDERS", + "get_provider", +] diff --git a/core/providers/__pycache__/__init__.cpython-311.pyc b/core/providers/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..47408eb569f26da1b4585287c9532129050c134a GIT binary patch literal 1244 zcma)5&1=*^6rV}5*=+Ws)k0gPrJ=VKEc7C_wA#8YOF{cddk8BbZf0w2K9)&VX{k`e zgJ&-lR47tvrTqgG^zLyBJ(K}Y;=x<6x1M~Hnza=LXD7e+F~9fn^4{#*d|pMcKK;06 zFNz3#2q-F7ZM$ zw0$qoYChE#VrS{T84jr5F%(u5S?-n@kR*!b_~ASW>JUPh%B_;`)$HXWW`&tL@y4eT zF87oivthNOxu;BA+p}97k;y(pnKqNFF_+29F$3&4FOw+=8`P`va+1ZH2MtBX+W?y= zO#C3X&hlFRb@V_;j2&R?wOEhwIu5$RbaINIE?c$=h;`xuf7H?RaE($Dd)@@^j ztTN>u%))3;rzo?`?EK8q)Wqcc0uz@>$dn{EQj&CHHT(DPc*Z02idv9>|9ftJBGxl zs8D`ID4!DLOcx6*Z5WncH4G;47XKNU;nX;s|DjKFWVv=e5zk^e zj$yn*w3{Dy0-Pu#u5UFQa)b7Q$uC~;0U#1F#*rXkDMB1%gom2w`oH!O>TjOk|Ko`i t)Yf}c=z$%jrMwU!fQ?L4Na1<>;&Ozz+|8ZJzhHYN3o!|7r1>xYw4d$dKgpoca$I`YYX zTHuLFIZM?Wq?BvEpZV1Em|o}ZR~BVluhdM>)2lAmS6A2crRyuY<@juqo4(~b-hAc< zoL`cI#L}nBLBYK3`P{7dHuJY!T9Se^HGNa)fJ$P?*$eUMLZqO7sxom4!PCYs=J=v6wsX_o1>sZkqPqXoI;Ez|!4^Xjly);xF1 zv8ooMI`6dTRqool>1+$*Vw>j!#b@96>w#+7<|Yer-2B>zrF2P^;oU zBZp||>pxw+w6U~ub$MgiwX55=R^Vm3PF#C$&9t12O2hMA+i+aUyl&48%dXcL+P+b7 zIopUW^EPfnaOdmW99<~@cjB!Aw2u5t+ZC|ue4|8TEBbn@wb zGEMF%1WUW8nA9MP4Gu2Xodl+K;C7m zGVUTOJqb8ep*fbLd6uUI7+bi5eiY@%D6tb*y;#lBfgSbOdV}5dCg~99L!HA%HAR-t z5ojNUC(<)^LeCJDA+@H09Fq6-4tF=tE_LWw$Mil%ZK$|*@0$lt1co?v#xFsRoPos8FCtt*_yFLfCkmULl_MLPr}})- z-}!4)^AUnz*>fPlxj2sv7XaW0t@Dqu_6hb3znQp8&@V1FNELsZD>w%;GEiz3e1Ql` z$3zL#YRxtMAOX3}k;eHj0^}XxA0ptjdz|<2g5d~f;PLRL>+ybDS=d(=LPb_Dg#=(P zAzt5k|GfFEDa7NDB3hxX4u`U==J(EEZ@vdF6GR)?`2t^soj;=5JRB8mB+QWMnfBCl z`@%w)&)^Jz7eiwY{oUog;oZ+3{pI=4*-*tEjTDE&BuXhVb}CGxlp&)NVHTwv8616h z>(R#Z;yZgc4~nHoTajeO!a?juy#hKr31w`Uh_n@Yr6i(wEWEE%m!pr^-|#Ci1S!P(5g<%Rk_1tb6bKK1Mx@z&a^~c}O=kA{|7|k1 z-~VruTub~wjhqc-MVbrANqC8EuMoifa&~7j8d(r?@)CS4oP?Ly_6ngp2M6hM5(M%j QJpAw_w!Zp_F5&C{0eN~g$p8QV literal 0 HcmV?d00001 diff --git a/core/providers/__pycache__/claude.cpython-311.pyc b/core/providers/__pycache__/claude.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b2f22d394c81422f63c4ea70796273a4f0f81d6 GIT binary patch literal 5580 zcmb_gU2NOd6}}XIB+8OWTekd@#HQ^uwp`1OQ#WmrqMNHUPTI}unzT(^Z<->~juJ^! zNh*mIItZF=jIrrRq&;8E1`q=OHB1nJ!_s+!MS`qp?F4PiT&D=c-nafB-b4Vma zSCW_`;7cZ5DRPb^oH}Zb5-E{R(kW(+NxA3TP|t|&q$kDBu_^DImq0EwgGA2-B(f57 zjYQ}={G2q`BDxNukT>_ZHtsPV3Od>916!~OAloTP6$|R+fl-6QeLQao+ zZ)4us5Mh$j>80BQ6hbc3_gqYsUaGQ~zR0nd_OhgA64m7BQRuvL z%H-mT6w{=r7}H|WtdfXh$+hIKX728XAub~c!Q|1LOC;tcpkv`{%WLLl8I(W0H-SW=_>1z7eA^qikX z_z{EuQwuMQ(IT~qK8gDd8u1H0(GT+5s`9%;9{910w#Knxzi#{zjUC!`{O2~NLp4&$MHtj63$~Iy2}6o10_JRYc{v}n5Jda z-4heBOkzA%v!wC3oSHZ}`8xzAL#a}7LB;ICx3ACao}WH26Pb_5sfDFi4!|cRr)#C~ z4`Ye+d_1daaw?jZMMv)tcZ<{se14EZ>YiotVjtAd z)tWQ9OIgxstwEplV~I=-V5ZtX4w|vL3&syGd$k(KfNb5{tl@Ht&K4V_;_;m%xMf%8Dk-*Hbv5%5=;HVzIeofE^gqa*Oj!TuLNd=!W7h7lgdaZ@LCY5@Zs zm;1R_sHW$#iKHkvMiE|+l~haP`&%CNhxwtGTm`G4XCJ@jKgA5SyfUL3KrX>?NLp} z!O9$K7bi05#Sm#y+EPX`y(vjd#G-bd!WbHIDVs4Xqsmz&E^OkyzDYefjCUY6OCXlb817_Zq>TE1^s2{EWpQzN2vZ z7S~_m`i&l;z!-eDQ+e|3v-z{dZTYh+KmOy;jbpzL{~`S8KzT>xRya}$M+{-G;5Rl6 zScq-wvPmKGb?@lP6Xo8CO7BD=Vss3A)iH9bW5f`KSGSHCqfZ)Jhb=cX)N3J3X!mH) zLO|{ycW2iTW!)Q zfYqeg0=AmLR*+S&O~p=whdu$WI*!4;mau8O1HZff0yKIN5rFzSI$momR=jR%kqFhg(o#M2xWt>gCV{^Rwv z5ZMzpGAdKLzu2J3n#^WR&Z7oBTD7*dexkQgF05dS-ugcJfNI9nHwhRAdzlB)2IFvm z)9^H}?RnRlCIm{i4WUWYSf^5_T&Kw+=oCSqBe25|NdAAM_?CH{E85}i`bmiXMbxY% zc&Dl5Hoff;{ac$?N93x$tw-k{$-{}h<}q>*TQ2h#pVTl8Ruk}ZpYYB7z+<%KgoA*F zLL8zEL6P3bN~$Jka@B9(P-#5JPSionyJP!~C&JrzhPO}VcmUF5B5p^`6US9K9U^n= z=`h4b;Sm$XoSVP|qwp*Q-0ovi45D+@ zq_F7QUqW@R&4ExmfCD!oD?-k-TQD8Gw_QWc3kcBN^eM4ZQNVLMN)P!=4|X*;Rx;h7 zWE7NCMsYuPEFt5_&h$r;3Gh}2As_;EO)4X+*mc3hAgkJvP2bDew3bLo&*RaR9hjS3 zlr#k&pHO>)N3o~2HY@C4OyYl=MXJSi9IVPHJwWbN4 zh3U2D9K}vg>BH;-mP#K@%JCB_#=k}{@vFUD{I|x)pe=6nM-Se*EL5 z-z@$5%tvQR13T8eg%`f=-BucY_2$%H_P__;FGrnkx%ah7?`x%w*H*U-e0b!&Bfon6 z>g%Phk#%o@vRaySxr0@^x1oL;YyTZu@87@w&|YNIdfvgydvC)v-gpPTL#wSx)2I$p zJL22(l+oF9`S81kuOwF5%bjDD&M^zQ*m0ZWpEtOUUoO49bR}5kgbF7V_ub+~OWdfz zcU+-AWZz?p+g1k4-Qh}ixXgzO)ctR`O~sBIdrRD|GPkS3?J9Y8sZW7Xf7aclc?F7zG;`|qkcQ5b$B0vEBmYm!@X}Ts~G#MwlH919E zkQ6B$m!eL<>WirvDXv8!ph(DY;Y;MCrbq&NPdk#>3K=JoXuK_n>uyVL5+PS^qzY`J zEUD>XO;EF$jI3x*U@k05X&k=bKPPer8+r<9d4sOPE2<>SNU20RQNQg9a1VG;DBN2E zv$A%e_CJUu+VB=~J9u^g=HWo~3pn>8CO^Q$1`iw}RB^`vo7Sqj)%D74SqD4K%}HQa z`++zB^q&sI4O%Y3|MVS_-&k0fEA-GJ{L|frJ94CmK!1 zQc^T(dZSSY{<2A&_eG;`WMj!{i#r+>E+&;s0OC_(BCaWT zA5B`5mH07}Ib{E}14VoXa=Zq;02r;oJs|DCzp@W&@*_;(|1M-JY_EvYf{U2on}Yg| z9pj$^w8f7>jg+X7RoV}erpJpzfQ~erg&Mg-BBu4iVY?A9H#}*%Lv){ofULF$3)+=P zF>q!7rL&dxZ5EAN7}VBTc;(W3g&!;)D)S+yzXy4_yjnQ)R_@AFkt!Zs87dzBsI{_b zqQp&9*zFcu6KYF#$WIB>H(>Pk8DrZm{}{d7LO>oI6t}>oBVf64&I8bIu{h^No!wRo z&iU4>WJ}|bH)yrtb}YAnhy>a`Wo#ZWI=U^tpN8!RvI+yxxZn(ha~{;&e|7T;Rqh$H zSX|*nUAZs_RUhkQLMAs%?9wT=NZQspH+SaUGN1${VfoSBnZNCxd;lt4+w(@ w?p!N=3=}LizXtN1v!7KoR(jZLpgpCBy%v)q2$=c9X<`13-E!{{?e^;Y7yCSy)C{q!S_+X=DLNo*4^0g^y){7Qzzfy6*!%=WfjWv4?w%&qDq zZmJP8pivJSi57|!?@mVBJDO#TU?o=Cl`QT2Kr`Qt-E;3b=iYnnx!=9zKRTUu3aiq8XA}16bb2`AzATK%4emxxK@PsSt>__|t+9|!U0MQJ6SlEm*#MiGxJrld59%IW9=6h1Pl zd5kCK7`#OkUBBG5#Qho2af=cu=ss%BB+zq=$Ov=g~Mho6a~_GkrVXv&XQW?FTucl$jv!zl<^Zff6@*I zXyHYd;1u0IX_HIzfL$F+F2N02yfqzNf)`|<9A9gT4BG`i+wPm>iGl~T&7!d z8Lm82N5i_4cEbri|)rD4rXL^(rJi|7jV3)ASQ_su|H4$z>~G?RVu zvMjGiM@L5L4tG^d#8S%{@WJ6_-GyT-acOulogBG(ctMn7pHbjwtRyCuBorBb_~!J{ z`LWZ}fqa-Vk_o+?ySjjzhAJWu}W=)Ec1g(-Z zvp&P5#m1}!K333sN6}qdpmWSJm1E_W9#rU0;qCKA&II+=KhMKV%5gcfVisE5I%k1* zrs={tt6~w@w{g}{gi)+H=EiQw;c7W1flJYXIcEcn_Llr9#Y83)PHwSts||uB3sJ8o zXIL?TO$T!hxgPI8pW#&aVv9ty$+!foU{iR(e%ln^(o&kbapUryw-R(y+Zz?7Z&n z($Pi+R-es4pbe*&!uzFFNfwjQcuJB zmjvHU5ElAMbHr1LcuI^4F*)|orhC)KN^1EtlV(fClVVg}T@f`Fxf&~r*X4+b1hF2s z5`#z|VKtkW$|NzoiyD&xUyEQtlF-d7z7fC|M za~Sg_iuRL!%wgimWsO-lGAcRYwbq-)gWv?w+|dQ>+h|>CcK4QPwS|v{NMZ2(_o-@- z@&@x})fdQHzYF#it>s{(5{y)-9@{u=(BDku$MTZu3l^^4JeHqOgI#Npo2mSC)l7N( zh3j|SeI<9F+7&LCRZk~m?u)EX{eGr6vo^3cuwlu+3|WCte$ps@^`rOl?-d8~@2$W3 z?|Z+z{I5g*HuP10`Kj@{L*u2PakZzvN?AHWIw_3b>prmlbh&$^(mhfbSN&bL&U|u4 z?cAqEM%5<G^u3mBVOpK|AIfbswck3z%oSukH9E_ z6RzN(XNaiTldex#irdsnP_L}KBbBVe})6^yKAON;MkCb2z zH~lW($mz5L%|%EzS89ANg4E37yTce@xx3_?EhlDQAB~hhOo-ayXw0fd2(x zHtQoz1RtZTM+mmJr`W))t!)=}wU0Cj$W8H-5)^uy zuk9ss-WzYpJxy6)IrQ!qyW71O7R$bD_yVjM#^!;D?_@+t4$JA*?v{GOV$)lNV<-?)2HQ=}Gb& z7V?tX9_4IWh$Z53T$E;DUA0^mW3YRYG#0J+{tgs<*kP<#*vPO*NScHnnvRAwO0zG* zHYyIwuP6}=QN|)^Hx}&)T$Qc3OzSN)u;2*kgL0oShM}Auh&7(Xu0`wSL|mE`_eKao zH48xED0CM&h$dTN@ic;H%{iWk12VY)I1@02#;&9#8Bhf5^fHnmnLnRN$)I`?%_S$W zHnS|snjH!OGC)*MFcc7sMv+hp5Wp2g?|L$VDIFI)2`SC|_O%#UmNW(oBQGKk;es(5 zmX+9QA{`Sni&VD>+oRd_?t&fOby;tC%{bs8($-r~CQyowC0Il)0Gf5oBO%~u!xPOH z16i=Rk!^_J`NAK%X}n1o*&=bE_>70U)uXj$$KPk+3e~2wrAvS&&Ze7je&B{SfyvIZ0*dO z3s#Wb)xX(yK;1h8aG>7>Z~zEr_Svfxki4bpp?GhVqHU*X_0ZE_UfsA_Ix^o5L9RRU507pBK z^<_1*`v~jJVlta^%&@`YDC)C`D+!s=sq>q8Pk= zrtIHem?%uBf&SubDG-5M?d>lfs`QQ&rqodHt?Vb+;(_&ra_Dd+bokrQ@w=hp8yCx= zsY+-Hyf$!(zBl;nhI!-apIqg^>B`{r8vo!scTds3K7WTTxzCo}XDjZrCF@z~S(veZ zw)RaNVg5Wg);;0lzNWqK^0m)C(QEsZt z?CX;&vgRh@l1RkVVh!nPd`wyq7v-pa%mqgZ@vOMmfMKAy0eNry1mxWUW4W9~U<4K* ztQB_DHih|Rq-{scY*}pWif#MdDgt5HuL7LiW{3N@IyMD>Zo38mTA%|Vu;37Q!MP0b z1Q!5ghr&uGg(HXKuup#!2rDe9{7M5p3;qVEb{275AGz8s0Z`qv!zKHg#Q{XO z$OTXxR9xGFYUlRgZ3jJWfx}-rv;zR$u%rc4H}bXt)jyAvP|hQ8xemqs7;xG72;lW9 zUI5izp{wa3kEz@J2&mp3z&;A9ZHi3?)jl{Y^68-312Ep!)To1M4}j_(oj(QKo+dX3 zp!_j5?Gi%mJzwOTquxvjz3p=gKy@on_6xh4+S*IzI&QqBbiD6w^QOM`GQXnb-li_a z1Hjw;MY!G10e1IipO`kz=i!8jghe4P*G^+IQarU>gWtpCDUg*7Y(?EV(k7$8BhLVt zA=m)23kfRM(U^goapIDrNRA=FaBRS2hYr#Vh^;w{IC>sox@H_lU4U3hUcllnAwlFt zP9hmcqGP)TGA5IlH-%&x2}0rQOEqu}I69n4%VFa{8NN>#?HmrzlGSi5EX0@em@ebd zGz59<4SlyUOiqEAh?~5OBF`YfX-}R&au&%9l5_gL98{a48RbHV7pLgxP;$h_$Yl^lKdyt_+%v9foe;$0~53p?QG=ldoHnZI<7 zbx*R~*H3xj9T#}Vldr7W7?~y2kn!GE#HjNKqK@_ge8YCG7SNlnTjwoH{!|;kZVAp4Wca+0f*K3ybV5I=Aa9{P zmyx)T;1ReUC<(#}o#1B(=`S>XKdApycD%%nZ*tD6iRMnyMch%tvz7k9P-fSM7mWf{ zo*Ft_wH)NaRSL+aH&~F@#*4nSQ#aqMcn7K+av)v?3a{OquXy$pXUm?5QQk_~-Fc}n z`mSl?T`_y<>|XQbpFso0*Z+Ui1e$p%@Vsoni*_ipvjlj{DVs{2K525dw6 zsAio9-!^;hZG!x@ux*78W`nR*H7N@4}x-1xxL>O7ZXOzfI~; gX}epcj+eH(tLChU#=(C4qcH!2!ToWELV7>_FG`tUx&QzG literal 0 HcmV?d00001 diff --git a/core/providers/__pycache__/openai.cpython-311.pyc b/core/providers/__pycache__/openai.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24beb848ca11d68441926634970dabb765b647b4 GIT binary patch literal 5158 zcmb_gO>7&-6`uX!E`LQzlqFfR>{a|rS)!v%`7|LdXkuD~azPPuel3s=Xu)()3#CI0V$o5g_%9=q7{R2XG^;Ra4)Y3&eKV}WUFJWAYPT9~P+ABe zWs~B=BKBh+4$QEdo3b@d5kMY16KaTBBz#ds2()k5eg+oqvM(JDAo?gh|I=O&`;-ul zC}G_3J#0s3!b$}6$Ljifl^C?AIyzf>t$2Oyg7P^0SWMe>L%U9e2S@31I=ppntW$Bg z?l-Xb%_Wxh$hsn}^Nd8&5eT*lM93OiNYJZD*X^T57+a$Zdu z;F_|U1cm#~jeUy1Z+H{uW=z^v`tF^h2c{<`kEW(mhBmYC#w5HnLvMVI9+y>pI$JO; zL(Axfg3YGnlf zv>F{QM~5E(kER^IhD}qR#ik>u8hvM*yAb37__@`ViqlOR?BqR%*XI` zt66dc{q#4}usIg_MW5|cm=>pF&(ckU$gIsz#Tll=? zh*?7ioUs6|cxxe#37sk@;I^XP;c!3_EE#}evIPk6y_D~#JnsasUeGWBd~&!P)*V64 zgKsN@9tR0Op}s@9DcJ;s?iUk$&tX)D)67{{iNmSdtm|J=J&{gYH=22ujxy62S8?k) zKV2uMR?L3`xrl0E6zi!WCa|R%A6R;|CZMkFCE@FyZ6*IoPomP3sG(3`yGwrc^3ueT zS&eorKX>KA^_jcT9p&hbo1LF$Z}0h3`%JqduOtuMO&lsG4z2DOzMZ-g`14eGAVqWk zk!7(uFjPZ)XRk|^Q`PQ)Yg6w}Ri(YvU3)*z+&Q%}I#C&&DDOO6^Km<%M?mf)Zd30= zO5n3W{HU|{TIBsm>D-;&<*kPyuf$T7SgP#i)mRS|UeB(?o~p#2D!X|#x@jpw;xOKS z!VgmI&^-jZ)bVM62{|WZVGWj327JO!P?OKYiEsp8G(3$4A75`Zm zKFGKoDz$9!HMQ+Mxdb6dvmarLI%hp8fmXdRTR}Svv%|M<++Q$Yhr;0_X#+bE2z%gZ zpTSG)T2C0HcWy^xsI|=G=h^ezX*AC;=rpXR8$A6VwAc|P)B>U57CzF#cdYS?9Twg2 z^<;P0kQ+%e|mC2n=_*^>2< zT^@WEa&=1*z%d>KG>iwAM1by|fok8DzxR#Y?HjrI_U&^keJ@t}UaSu6s`l@w1$?w7 zkR|b(4%9tRiSJp74_D&DH6#Q8);A(IXREziui5Y0rPS@-JM+u-O7GE1?@>?}ftDMQ zug0tK-m5Q^w%?8KEXQ|NcklgV;ZF;HyztS5^3XHu^72ot3F`^7HHORiZQp{1X3lbfDCI z`$##ueC$0^1EIB zVDN|_{w=@&{f-+uFy^phuQ&ok&syGcq6E)ig7qxUxKWW4md!lQS{XQ7RRaQdwTRme zReHF%hw2VsyhF7$;#Jl~yuV-)Z1F@LY!m~V`ff`y3;Dc3EJ+6&=dc&VNwZj|2RcB6 zr0OneCZ#t_EFGCRc5L(|$-N7J#Iyj+lEn*km2Rio+L}kEu0xJmH zlmiGJ#hR+CYd0f^J|F2`5-v3HDZ`p<+_Pj{ITgU1hv~Ek^$f>n86-RB2(P<0fA1vl%>X1@5DDv+i|gb^P?JnjLIK$A zsT&m&_;UrhgkJnj8-lWn&8Nzbm-+EkUj&Bh+gpMF7hY@RK8J*k%Wu0vfZ5Tpn%L)q zpcKezSI@F_JynWd|MAtIRl0W7d{iQ!&P~g2T%CrPv-J8(EaBDnqhQC9viyVV-0}~v z=#{`=EznTo>hj1zv~{T3zqLB@bS*OEJ5WPF9<@Rx0*b|JBIW!5{j~t)f@o7;EkwES z`aZbYVkp>C>!fn3w}nhG=;`t5mZ55QUo8^x!9oLB1p_|%;hCbGAN6m$vE?SevU#Ky zpantH+g}S&E{vjGUR%!!Eu&fy*BlT!6rcfH9;TEf#1fY1Qe zU0)|uzhGHv&LlJnCp18GZ%y73w=es5{)3+Hy+}?&JsOIep8~0I48zn|j^V+hf%p*< wz1TsYD(We>=PC+cbe~l;Qhr>nqT}T!=bCVWVPFy-PnW0fyOM`bkh>H0Ur&AtYXATM literal 0 HcmV?d00001 diff --git a/core/providers/base.py b/core/providers/base.py new file mode 100644 index 0000000..f770a9d --- /dev/null +++ b/core/providers/base.py @@ -0,0 +1,72 @@ +""" +Base provider class for LLM API integrations. +""" +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any +import torch + + +@dataclass +class ChatResponse: + """Unified response from any LLM provider.""" + text: str + image: torch.Tensor | None = None + raw_response: dict | None = None + + +@dataclass +class ChatConfig: + """Configuration for API connection.""" + provider: str + api_key: str + base_url: str + model: str + + def to_dict(self) -> dict: + return { + "provider": self.provider, + "api_key": self.api_key, + "base_url": self.base_url, + "model": self.model, + } + + +class BaseProvider(ABC): + """Abstract base class for LLM providers.""" + + def __init__(self, api_key: str, base_url: str | None = None): + self.api_key = api_key + self.base_url = base_url or self.default_base_url + + @property + @abstractmethod + def default_base_url(self) -> str: + """Default API endpoint for this provider.""" + pass + + @abstractmethod + async def chat( + self, + messages: list[dict[str, Any]], + model: str, + temperature: float = 1.0, + max_tokens: int = 2048, + images: list[torch.Tensor] | None = None, + **kwargs, + ) -> ChatResponse: + """Send a chat request to the API.""" + pass + + @abstractmethod + async def generate_image( + self, + prompt: str, + model: str, + reference_image: torch.Tensor | None = None, + aspect_ratio: str = "1:1", + size: str = "1K", + **kwargs, + ) -> ChatResponse: + """Generate or edit an image. Only supported by some providers.""" + pass diff --git a/core/providers/claude.py b/core/providers/claude.py new file mode 100644 index 0000000..20c1155 --- /dev/null +++ b/core/providers/claude.py @@ -0,0 +1,121 @@ +""" +Claude/Anthropic provider implementation. +""" +import aiohttp +from typing import Any +import torch + +from .base import BaseProvider, ChatResponse +from ..image_utils import tensor_to_base64, create_data_uri + + +class ClaudeProvider(BaseProvider): + """Anthropic Claude API provider.""" + + @property + def default_base_url(self) -> str: + return "https://api.anthropic.com/v1" + + def _build_messages( + self, + messages: list[dict[str, Any]], + images: list[torch.Tensor] | None = None, + ) -> tuple[str | None, list[dict[str, Any]]]: + """ + Build Claude-format messages with optional images. + + Returns: + Tuple of (system_prompt, messages) + """ + system = None + result = [] + + for i, msg in enumerate(messages): + role = msg["role"] + content = msg["content"] + + # Extract system message + if role == "system": + system = content + continue + + # Handle user message with images + if role == "user" and images and i == len(messages) - 1: + content_parts = [] + for img in images: + b64 = tensor_to_base64(img) + content_parts.append({ + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": b64, + } + }) + content_parts.append({"type": "text", "text": content}) + result.append({"role": "user", "content": content_parts}) + else: + result.append({"role": role, "content": content}) + + return system, result + + async def chat( + self, + messages: list[dict[str, Any]], + model: str, + temperature: float = 1.0, + max_tokens: int = 2048, + images: list[torch.Tensor] | None = None, + **kwargs, + ) -> ChatResponse: + """Send chat request to Claude API.""" + + url = f"{self.base_url}/messages" + headers = { + "x-api-key": self.api_key, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + } + + system, claude_messages = self._build_messages(messages, images) + + payload = { + "model": model, + "messages": claude_messages, + "temperature": temperature, + "max_tokens": max_tokens, + } + + if system: + payload["system"] = system + + async with aiohttp.ClientSession() as session: + async with session.post(url, headers=headers, json=payload) as resp: + if resp.status != 200: + error_text = await resp.text() + raise RuntimeError(f"Claude API error {resp.status}: {error_text}") + + data = await resp.json() + + # Extract text from response + text = "" + for block in data.get("content", []): + if block.get("type") == "text": + text += block.get("text", "") + + return ChatResponse(text=text, raw_response=data) + + async def generate_image( + self, + prompt: str, + model: str, + reference_image: torch.Tensor | None = None, + aspect_ratio: str = "1:1", + size: str = "1K", + **kwargs, + ) -> ChatResponse: + """Claude doesn't support image generation.""" + raise NotImplementedError( + "Claude does not support image generation. " + "Use Gemini provider for image generation." + ) diff --git a/core/providers/gemini.py b/core/providers/gemini.py new file mode 100644 index 0000000..b615653 --- /dev/null +++ b/core/providers/gemini.py @@ -0,0 +1,199 @@ +""" +Google Gemini provider implementation. +Supports chat and image generation (Nano Banana / Nano Banana Pro). +""" +import aiohttp +from typing import Any +import torch + +from .base import BaseProvider, ChatResponse +from ..image_utils import tensor_to_base64, base64_to_tensor + + +class GeminiProvider(BaseProvider): + """Google Gemini API provider with image generation support.""" + + @property + def default_base_url(self) -> str: + return "https://generativelanguage.googleapis.com/v1beta" + + def _build_contents( + self, + messages: list[dict[str, Any]], + images: list[torch.Tensor] | None = None, + ) -> tuple[str | None, list[dict[str, Any]]]: + """ + Build Gemini-format contents. + + Returns: + Tuple of (system_instruction, contents) + """ + system = None + contents = [] + + for i, msg in enumerate(messages): + role = msg["role"] + content = msg["content"] + + # Extract system message + if role == "system": + system = content + continue + + # Map roles + gemini_role = "user" if role == "user" else "model" + + # Build parts + parts = [] + + # Add images to last user message + if role == "user" and images and i == len(messages) - 1: + for img in images: + b64 = tensor_to_base64(img) + parts.append({ + "inline_data": { + "mime_type": "image/png", + "data": b64, + } + }) + + parts.append({"text": content}) + contents.append({"role": gemini_role, "parts": parts}) + + return system, contents + + async def chat( + self, + messages: list[dict[str, Any]], + model: str, + temperature: float = 1.0, + max_tokens: int = 2048, + images: list[torch.Tensor] | None = None, + enable_image_generation: bool = False, + **kwargs, + ) -> ChatResponse: + """Send chat request to Gemini API.""" + + url = f"{self.base_url}/models/{model}:generateContent" + + # Add API key as query parameter + if "?" in url: + url += f"&key={self.api_key}" + else: + url += f"?key={self.api_key}" + + headers = {"Content-Type": "application/json"} + + system, contents = self._build_contents(messages, images) + + payload = { + "contents": contents, + "generationConfig": { + "temperature": temperature, + "maxOutputTokens": max_tokens, + } + } + + if system: + payload["systemInstruction"] = {"parts": [{"text": system}]} + + # Enable image generation if requested + if enable_image_generation: + payload["generationConfig"]["responseModalities"] = ["TEXT", "IMAGE"] + + async with aiohttp.ClientSession() as session: + async with session.post(url, headers=headers, json=payload) as resp: + if resp.status != 200: + error_text = await resp.text() + raise RuntimeError(f"Gemini API error {resp.status}: {error_text}") + + data = await resp.json() + + # Extract text and images from response + text = "" + result_image = None + + candidates = data.get("candidates", []) + if candidates: + parts = candidates[0].get("content", {}).get("parts", []) + for part in parts: + if "text" in part: + text += part["text"] + elif "inlineData" in part: + inline = part["inlineData"] + if inline.get("mimeType", "").startswith("image/"): + result_image = base64_to_tensor(inline["data"]) + + return ChatResponse(text=text, image=result_image, raw_response=data) + + async def generate_image( + self, + prompt: str, + model: str, + reference_image: torch.Tensor | None = None, + aspect_ratio: str = "1:1", + size: str = "1K", + **kwargs, + ) -> ChatResponse: + """Generate or edit image using Gemini.""" + + url = f"{self.base_url}/models/{model}:generateContent?key={self.api_key}" + headers = {"Content-Type": "application/json"} + + # Build parts + parts = [] + + # Add reference image if provided + if reference_image is not None: + b64 = tensor_to_base64(reference_image) + parts.append({ + "inline_data": { + "mime_type": "image/png", + "data": b64, + } + }) + + parts.append({"text": prompt}) + + payload = { + "contents": [{"parts": parts}], + "generationConfig": { + "responseModalities": ["TEXT", "IMAGE"], + } + } + + # Add image config if supported + if aspect_ratio or size: + payload["generationConfig"]["imageConfig"] = {} + if aspect_ratio: + payload["generationConfig"]["imageConfig"]["aspectRatio"] = aspect_ratio + if size: + payload["generationConfig"]["imageConfig"]["imageSize"] = size + + async with aiohttp.ClientSession() as session: + async with session.post(url, headers=headers, json=payload) as resp: + if resp.status != 200: + error_text = await resp.text() + raise RuntimeError(f"Gemini API error {resp.status}: {error_text}") + + data = await resp.json() + + # Extract text and image + text = "" + result_image = None + + candidates = data.get("candidates", []) + if candidates: + parts = candidates[0].get("content", {}).get("parts", []) + for part in parts: + if "text" in part: + text += part["text"] + elif "inlineData" in part: + inline = part["inlineData"] + if inline.get("mimeType", "").startswith("image/"): + result_image = base64_to_tensor(inline["data"]) + + if result_image is None: + raise RuntimeError("Gemini did not return an image. Try a different prompt or model.") + + return ChatResponse(text=text, image=result_image, raw_response=data) diff --git a/core/providers/openai.py b/core/providers/openai.py new file mode 100644 index 0000000..9a2cfb5 --- /dev/null +++ b/core/providers/openai.py @@ -0,0 +1,94 @@ +""" +OpenAI provider implementation. +Also works with OpenAI-compatible APIs (e.g., local LLMs, other providers). +""" +import aiohttp +from typing import Any +import torch + +from .base import BaseProvider, ChatResponse +from ..image_utils import tensor_to_base64, base64_to_tensor, create_data_uri + + +class OpenAIProvider(BaseProvider): + """OpenAI API provider (and compatible APIs).""" + + @property + def default_base_url(self) -> str: + return "https://api.openai.com/v1" + + def _build_messages( + self, + messages: list[dict[str, Any]], + images: list[torch.Tensor] | None = None, + ) -> list[dict[str, Any]]: + """Build OpenAI-format messages with optional images.""" + if not images: + return messages + + # Find the last user message and add images to it + result = [] + for i, msg in enumerate(messages): + if msg["role"] == "user" and i == len(messages) - 1: + # Convert content to multimodal format + content = [{"type": "text", "text": msg["content"]}] + for img in images: + content.append({ + "type": "image_url", + "image_url": {"url": create_data_uri(img)} + }) + result.append({"role": "user", "content": content}) + else: + result.append(msg) + + return result + + async def chat( + self, + messages: list[dict[str, Any]], + model: str, + temperature: float = 1.0, + max_tokens: int = 2048, + images: list[torch.Tensor] | None = None, + **kwargs, + ) -> ChatResponse: + """Send chat request to OpenAI API.""" + + url = f"{self.base_url}/chat/completions" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + payload = { + "model": model, + "messages": self._build_messages(messages, images), + "temperature": temperature, + "max_tokens": max_tokens, + } + + async with aiohttp.ClientSession() as session: + async with session.post(url, headers=headers, json=payload) as resp: + if resp.status != 200: + error_text = await resp.text() + raise RuntimeError(f"OpenAI API error {resp.status}: {error_text}") + + data = await resp.json() + + text = data["choices"][0]["message"]["content"] + return ChatResponse(text=text, raw_response=data) + + async def generate_image( + self, + prompt: str, + model: str, + reference_image: torch.Tensor | None = None, + aspect_ratio: str = "1:1", + size: str = "1K", + **kwargs, + ) -> ChatResponse: + """OpenAI doesn't support native image generation in chat. Use DALL-E nodes instead.""" + raise NotImplementedError( + "OpenAI image generation is not supported in SimpleChat. " + "Use dedicated DALL-E nodes or Gemini provider for image generation." + ) diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..ecfa95b --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,312 @@ +# ComfyUI-SimpleChat Architecture Design + +## Design Philosophy + +**Do one thing, do best** - 每个节点只做一件事，做到极致。 + +## 节点设计原则 + +1. **单一职责**: 每个节点只负责一个功能 +2. **清晰命名**: 节点名称直接表达功能 +3. **最少配置**: 合理默认值，减少必填项 +4. **美观布局**: 输入输出顺序符合直觉 + +--- + +## 节点清单 + +### 配置节点 + +| 节点 | 功能 | 设计理念 | +|------|------|----------| +| `API Config` | 配置 API 连接 | 一次配置，到处使用 | + +### 对话节点 + +| 节点 | 功能 | 设计理念 | +|------|------|----------| +| `Chat` | 基础对话 | 最简单的文本对话 | +| `Chat with Image` | 发送图片给 LLM | 视觉理解 | +| `Chat NoASS` | NoASS 格式对话 | 角色扮演优化 | + +### 图片生成节点 (Gemini Only) + +| 节点 | 功能 | 设计理念 | +|------|------|----------| +| `Gemini Image Gen` | 文生图 | 纯文本生成图片 | +| `Gemini Image Edit` | 图片编辑 | 基于参考图修改 | + +--- + +## 节点详细设计 + +### 1. API Config + +**功能**: 配置 API 连接参数 + +``` +┌─────────────────────────────────────┐ +│ 🔧 API Config │ +├─────────────────────────────────────┤ +│ provider [openai ▼] │ +│ api_key [••••••••••] │ +│ base_url [ ] │ +│ model [gpt-4o ▼] │ +├─────────────────────────────────────┤ +│ CONFIG ○───│ +└─────────────────────────────────────┘ +``` + +**输入:** +- `provider`: 下拉 ["openai", "claude", "gemini"] +- `api_key`: 密码输入 +- `base_url`: 文本 (可选，留空使用默认) +- `model`: 可编辑下拉 + +**输出:** +- `CONFIG`: 配置对象 + +--- + +### 2. Chat + +**功能**: 基础文本对话 + +``` +┌─────────────────────────────────────┐ +│ 💬 Chat │ +├─────────────────────────────────────┤ +│ ───○ config │ +├─────────────────────────────────────┤ +│ prompt [ ] │ +│ [ ] │ +│ system [ ] │ +│ temperature [1.0 ] │ +│ max_tokens [2048 ] │ +├─────────────────────────────────────┤ +│ text ○───│ +└─────────────────────────────────────┘ +``` + +**输入:** +- `config`: CONFIG (必需) +- `prompt`: 多行文本 +- `system`: 系统提示 (可选) +- `temperature`: 0-2, 默认 1.0 +- `max_tokens`: 默认 2048 + +**输出:** +- `text`: STRING + +--- + +### 3. Chat with Image + +**功能**: 发送图片给 LLM 分析 + +``` +┌─────────────────────────────────────┐ +│ 🖼️ Chat with Image │ +├─────────────────────────────────────┤ +│ ───○ config │ +│ ───○ image │ +├─────────────────────────────────────┤ +│ prompt [Describe this image ] │ +│ [ ] │ +│ system [ ] │ +│ temperature [1.0 ] │ +│ max_tokens [2048 ] │ +├─────────────────────────────────────┤ +│ text ○───│ +└─────────────────────────────────────┘ +``` + +**输入:** +- `config`: CONFIG +- `image`: IMAGE +- `prompt`: 多行文本 +- `system`: 系统提示 (可选) +- `temperature`: 0-2 +- `max_tokens`: 默认 2048 + +**输出:** +- `text`: STRING + +--- + +### 4. Chat NoASS + +**功能**: NoASS 格式的角色扮演对话 + +``` +┌─────────────────────────────────────┐ +│ 🎭 Chat NoASS │ +├─────────────────────────────────────┤ +│ ───○ config │ +│ ───○ history (optional) │ +│ ───○ image (optional) │ +├─────────────────────────────────────┤ +│ system [You are... ] │ +│ [ ] │ +│ user_input [ ] │ +│ user_name [User ] │ +│ char_name [Assistant] │ +│ temperature [1.0 ] │ +│ max_tokens [2048 ] │ +├─────────────────────────────────────┤ +│ text ○───│ +│ history ○───│ +└─────────────────────────────────────┘ +``` + +**输入:** +- `config`: CONFIG +- `history`: STRING (可选，上一轮输出) +- `image`: IMAGE (可选) +- `system`: 系统提示/世界观设定 +- `user_input`: 用户本轮输入 +- `user_name`: 用户角色名 +- `char_name`: AI 角色名 +- `temperature`: 0-2 +- `max_tokens`: 默认 2048 + +**输出:** +- `text`: STRING (本轮 AI 回复) +- `history`: STRING (完整对话历史，可连接到下一个节点的 history 输入) + +--- + +### 5. Gemini Image Gen + +**功能**: Gemini 文生图 + +``` +┌─────────────────────────────────────┐ +│ 🎨 Gemini Image Gen │ +├─────────────────────────────────────┤ +│ ───○ config (Gemini only) │ +├─────────────────────────────────────┤ +│ prompt [A cat wearing a hat ] │ +│ [ ] │ +│ aspect [1:1 ▼] │ +│ size [1K ▼] │ +├─────────────────────────────────────┤ +│ IMAGE ○───│ +│ text ○───│ +└─────────────────────────────────────┘ +``` + +**输入:** +- `config`: CONFIG (必须是 Gemini provider) +- `prompt`: 多行文本 +- `aspect`: 下拉 ["1:1", "16:9", "9:16", "4:3", "3:4"] +- `size`: 下拉 ["1K", "2K", "4K"] + +**输出:** +- `IMAGE`: IMAGE +- `text`: STRING (Gemini 的文字描述) + +--- + +### 6. Gemini Image Edit + +**功能**: Gemini 图片编辑 + +``` +┌─────────────────────────────────────┐ +│ ✏️ Gemini Image Edit │ +├─────────────────────────────────────┤ +│ ───○ config (Gemini only) │ +│ ───○ image │ +├─────────────────────────────────────┤ +│ prompt [Add a red hat ] │ +│ [ ] │ +│ size [1K ▼] │ +├─────────────────────────────────────┤ +│ IMAGE ○───│ +│ text ○───│ +└─────────────────────────────────────┘ +``` + +**输入:** +- `config`: CONFIG (必须是 Gemini provider) +- `image`: IMAGE (要编辑的图片) +- `prompt`: 编辑指令 +- `size`: 下拉 ["1K", "2K", "4K"] + +**输出:** +- `IMAGE`: IMAGE +- `text`: STRING + +--- + +## 数据流示例 + +### 基础对话 + +``` +[API Config] ──CONFIG──> [Chat] ──text──> [Display Text] +``` + +### 图片分析 + +``` +[Load Image] ──IMAGE──┐ + ├──> [Chat with Image] ──text──> [Display Text] +[API Config] ──CONFIG─┘ +``` + +### 多轮 NoASS 对话 + +``` +[API Config] ──CONFIG──> [Chat NoASS] ──history──> [Chat NoASS] ──history──> ... + │ │ + └──text──> └──text──> +``` + +### Gemini 图片生成 + +``` +[API Config] ──CONFIG──> [Gemini Image Gen] ──IMAGE──> [Preview Image] + │ │ + │ ──text──> [Display Text] +``` + +--- + +## 文件结构 + +``` +custom_nodes/ComfyUI-SimpleChat/ +├── __init__.py # 节点注册 +├── nodes/ +│ ├── __init__.py +│ ├── config.py # API Config +│ ├── chat.py # Chat +│ ├── chat_image.py # Chat with Image +│ ├── chat_noass.py # Chat NoASS +│ ├── gemini_gen.py # Gemini Image Gen +│ └── gemini_edit.py # Gemini Image Edit +├── core/ +│ ├── __init__.py +│ ├── providers/ +│ │ ├── __init__.py +│ │ ├── base.py # Provider 抽象基类 +│ │ ├── openai.py # OpenAI (+ 兼容 API) +│ │ ├── claude.py # Claude/Anthropic +│ │ └── gemini.py # Google Gemini +│ ├── noass.py # NoASS 格式处理 +│ └── image_utils.py # 图片转换 +├── docs/ # 文档 +├── requirements.txt +└── README.md +``` + +--- + +## 实现优先级 + +1. **Phase 1**: API Config + Chat (最小可用) +2. **Phase 2**: Chat with Image +3. **Phase 3**: Chat NoASS +4. **Phase 4**: Gemini Image Gen/Edit diff --git a/docs/research_comfyui_api_patterns.md b/docs/research_comfyui_api_patterns.md new file mode 100644 index 0000000..f53cc37 --- /dev/null +++ b/docs/research_comfyui_api_patterns.md @@ -0,0 +1,153 @@ +# ComfyUI API Nodes Patterns Research + +## 目录结构 + +ComfyUI 官方 API 节点位于 `/comfy_api_nodes/`: + +``` +comfy_api_nodes/ +├── apis/ # Pydantic request/response models +├── util/ # 共享工具 +│ ├── client.py # HTTP 客户端 (sync_op, poll_op) +│ ├── _helpers.py # 认证, base URL +│ ├── conversions.py # 图片/音频/视频转换 +│ └── ... +├── nodes_openai.py # OpenAI 节点 +├── nodes_gemini.py # Gemini 节点 +└── ... +``` + +## 节点定义模式 + +### 新版 API 节点模式 (IO.ComfyNode) + +```python +class MyApiNode(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="MyApiNode", + display_name="My API Node", + category="api node/category", + inputs=[ + IO.String.Input("prompt", multiline=True), + IO.Image.Input("image", optional=True), + ], + outputs=[ + IO.Image.Output("IMAGE"), + IO.String.Output("TEXT"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + ], + is_api_node=True, + ) + + @classmethod + async def execute(cls, prompt: str, image=None): + # Implementation + return IO.NodeOutput(result_image, result_text) +``` + +### 传统节点模式 (ComfyNodeABC) + +```python +class MyNode(ComfyNodeABC): + @classmethod + def INPUT_TYPES(s) -> InputTypeDict: + return { + "required": { + "prompt": ("STRING", {"multiline": True}), + }, + "optional": { + "image": ("IMAGE",), + } + } + + RETURN_TYPES = ("IMAGE", "STRING") + RETURN_NAMES = ("image", "text") + FUNCTION = "execute" + CATEGORY = "my_category" + + def execute(self, prompt, image=None): + return (result_image, result_text) +``` + +## 图片处理 + +### Tensor 转 Base64 + +```python +from comfy_api_nodes.util import tensor_to_base64_string + +base64_str = tensor_to_base64_string(image_tensor) +data_uri = f"data:image/png;base64,{base64_str}" +``` + +### Base64 转 Tensor + +```python +import base64 +from io import BytesIO +from comfy_api_nodes.util import bytesio_to_image_tensor + +image_data = base64.b64decode(base64_string) +tensor = bytesio_to_image_tensor(BytesIO(image_data)) +``` + +### Tensor 格式 + +ComfyUI 图片 tensor: `(B, H, W, C)` - Batch, Height, Width, Channels (RGBA) + +## HTTP 请求 + +### 同步请求 + +```python +from comfy_api_nodes.util import sync_op + +response = await sync_op( + cls, + ApiEndpoint(path="/api/endpoint", method="POST"), + response_model=ResponseModel, + data=RequestModel(...), + timeout=3600.0, +) +``` + +### 轮询请求 + +```python +from comfy_api_nodes.util import poll_op + +response = await poll_op( + cls, + ApiEndpoint(path="/api/status/{id}", method="GET"), + response_model=ResponseModel, + status_extractor=lambda r: r.status, + completed_statuses=["completed"], + failed_statuses=["failed"], +) +``` + +## 认证 + +官方节点使用 ComfyUI 代理认证。我们的自定义节点将支持： +1. 直接 API Key 输入 +2. 自定义 base_url + +## 重要发现 + +1. **无 Claude/Anthropic 集成** - 官方没有 Claude 节点 +2. **代理模式** - 官方节点通过 ComfyUI 代理访问 API +3. **异步执行** - 所有 API 调用使用 async/await +4. **Pydantic 验证** - 请求/响应使用 Pydantic 模型 + +## 对我们的启示 + +由于我们要支持自定义 base_url 和直接 API 访问，我们将： +1. 使用传统节点模式（更简单，更灵活） +2. 直接使用 aiohttp/httpx 发请求 +3. 手动处理认证 +4. 复用 ComfyUI 的图片转换工具 diff --git a/docs/research_gemini_image.md b/docs/research_gemini_image.md new file mode 100644 index 0000000..0933b4e --- /dev/null +++ b/docs/research_gemini_image.md @@ -0,0 +1,127 @@ +# Gemini Image Generation Research Notes + +## Available Models + +| Model | Use Case | Features | +|-------|----------|----------| +| `gemini-2.5-flash-image` | Fast, general image generation | Lower cost, faster | +| `gemini-3-pro-image-preview` | Professional asset production | Higher quality, 1K/2K/4K output | + +## API Endpoint + +``` +POST https://generativelanguage.googleapis.com/v1beta/models/{model-id}:generateContent +``` + +## Request Format + +### 关键：启用图像生成 + +必须在 `generationConfig` 中指定 `responseModalities`: + +```json +{ + "contents": [{ + "parts": [ + {"text": "Generate an image of a cat"} + ] + }], + "generationConfig": { + "responseModalities": ["TEXT", "IMAGE"] + } +} +``` + +### 带图片输入的请求（图片编辑/参考） + +```json +{ + "contents": [{ + "parts": [ + {"text": "Edit this image to add a hat"}, + { + "inline_data": { + "mime_type": "image/png", + "data": "base64_encoded_image_data" + } + } + ] + }], + "generationConfig": { + "responseModalities": ["TEXT", "IMAGE"] + } +} +``` + +### Image Configuration + +```json +{ + "generationConfig": { + "responseModalities": ["TEXT", "IMAGE"], + "imageConfig": { + "aspectRatio": "16:9", + "imageSize": "2K" + } + } +} +``` + +支持的 aspectRatio: "1:1", "16:9", "9:16", "5:4", "4:5", "3:4", "4:3" +支持的 imageSize: "1K", "2K", "4K" + +## Response Format + +```json +{ + "candidates": [{ + "content": { + "parts": [ + {"text": "Here is your generated image:"}, + { + "inlineData": { + "mimeType": "image/png", + "data": "base64_encoded_image_data" + } + } + ] + } + }] +} +``` + +## Input Modalities + +1. **Text-to-Image**: 纯文本提示词 +2. **Text-and-Image-to-Image**: 文本 + 参考图片 +3. **Multi-Reference Images**: 最多 14 张参考图片 + +## Authentication + +使用 API Key: +``` +x-goog-api-key: YOUR_API_KEY +``` + +或 Bearer Token: +``` +Authorization: Bearer YOUR_TOKEN +``` + +## Special Features + +- **Thought Signatures**: 多轮对话保持推理上下文 +- **SynthID**: 自动添加 AI 生成图片水印 +- **Grounding with Google Search**: 可使用搜索验证事实 + +## Implementation Notes for ComfyUI-SimpleChat + +1. 需要支持自定义 base_url（用户可能使用代理） +2. 需要处理 base64 图片编解码 +3. 需要支持图片输入作为参考 +4. 返回的图片需要转换为 ComfyUI tensor 格式 + +## Reference + +- https://ai.google.dev/gemini-api/docs/image-generation +- https://ai.google.dev/gemini-api/docs/imagen diff --git a/docs/research_noass.md b/docs/research_noass.md new file mode 100644 index 0000000..9de9374 --- /dev/null +++ b/docs/research_noass.md @@ -0,0 +1,69 @@ +# NoASS Research Notes + +## What is NoASS? + +NoASS (No Assistant) 是一种用于角色扮演和创意写作的提示词方法论。它将基于聊天的助手交互转换为统一的叙事格式。 + +## Core Concept + +传统聊天是 human/assistant 轮流对话，NoASS 将整个 roleplay 合并成一个连续的故事，用户通过编辑来贡献角色动作。 + +## Key Motivation + +1. **消除角色偏见**: 传统聊天模型对 human 角色有内在偏见，往往给予不合理的优势 +2. **解决统一响应长度问题**: 标准聊天训练鼓励一致的回答长度，而动态叙事需要变化的响应规模 +3. **消除闭合模式**: 传统响应以隐式呼吁用户结束（反问句），破坏沉浸感 +4. **平等对待角色**: 所有角色被平等对待，因为没有角色正式"属于"任何一方 + +## Technical Implementation + +### Context Transformation + +**传统 ChatML 格式:** +``` +<|im_start|>user +[Instructions] +<|im_end|> +<|im_start|>assistant +[Greeting] +<|im_end|> +<|im_start|>user +[User action] +<|im_end|> +``` + +**NoASS 格式:** +``` +<|im_start|>user +[Instructions for story with all characters] +<|im_start|>assistant +[Greeting] +{{user}}: [user's action] +{{char}}: [character's action] +... +``` + +关键机制：所有 roleplay 变成单个预填充的 assistant 消息，用户输入被"拼接"到叙事中，使用角色前缀标记。 + +## Formatting Standards + +- 一段 = 一个角色前缀 (`**char name:**`) +- 前缀下：*动作*、"对话"、'思想'，顺序任意 +- 最小叙事描述；主要动作使用前缀 +- 用户响应省略自己的前缀（扩展自动插入） + +## Stop Token Mechanism + +使用用户角色的前缀作为停止令牌，防止模型为玩家角色编写对话。这是防止模型为用户生成动作的关键。 + +## Implementation for ComfyUI-SimpleChat + +在我们的实现中，NoASS 功能将： +1. 允许用户选择是否启用 NoASS 模式 +2. 自动处理消息格式转换 +3. 支持自定义角色前缀 +4. 使用用户前缀作为停止字符串 + +## Reference + +- https://rentry.org/noass_ext diff --git a/nodes/__init__.py b/nodes/__init__.py new file mode 100644 index 0000000..9b6e614 --- /dev/null +++ b/nodes/__init__.py @@ -0,0 +1,18 @@ +""" +SimpleChat nodes module. +""" +from .config import SimpleChatConfig +from .chat import SimpleChatText +from .chat_image import SimpleChatImage +from .chat_noass import SimpleChatNoASS +from .gemini_gen import GeminiImageGen +from .gemini_edit import GeminiImageEdit + +__all__ = [ + "SimpleChatConfig", + "SimpleChatText", + "SimpleChatImage", + "SimpleChatNoASS", + "GeminiImageGen", + "GeminiImageEdit", +] diff --git a/nodes/__pycache__/__init__.cpython-311.pyc b/nodes/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..48aacd9f2955ef36552b8f19fcc522d45a900278 GIT binary patch literal 694 zcmZutJx{|h5Iv`95>g5XF)?-ED+A03AtY3xssk)716Cy~s!fPw$EZmqVB$Bhu(KdW z{z=9vD-&BJwocflr9{FxJwHF+`Cjg|+4MoJtKcwxM*tuCFkjY;d|^eNfdU4KsKt@6 zu!S_Q@tUx)ZMeB`lFoDI18LnP4ykw8tNJx4Xy@H!9}oAxP(wSB&2YL#*nEf?4kMxQ&;$! zl34pNZ_I>5hG}ZvEBwe9ArVW=+eKq0Fy+BC(p^H%33-Dr)fcAy&e7ns+i4HlQe59( zcGV_iGC%E~hjB6(j?+vEVg~tpvzW}_=YkM5gN%@a+xw|^{4?*9(S~ZGN1i?@C~F9z x(z1|Uf(8fBb`D#AV*;(b8WY&ftMQ+*lS3sYu$Nb(wB3eP!WmgxEpU4J<3qICZhT?(U|K z;7XAip;Cc*sZ_O!M8qKog+mS;%B{y9U8x6aPL()7Z(F5YdO-Z+O`L|1*_m(ung8E^ z|L6Q13aJRj5C6PhxF#d?2ixonahupW1H^rVQ5Ini^F^-2XZezl6*$DBafF2xgvE6p zp$%Bx$ck7>qqw|uLQPyU9Ib3(qMg=847-ri9CF>!a%TA|v24RBnB_tB1^Y72)8GPe z^onI(Ens5N#Nh=}FsB`joR zEM|RJ$|_jS`mryo`Vo6@D-MtV4w4`a!D=BP940M!6-QR2Y#2wui8k7jE|e-o!aT`< z?Mp`v@I$*+a4u<+lT(8^uK9O_6rcrITN3~fM~ER7FhppT&+}Mb<@fA;Siyd*VlZk| z1bzs&;4qHh=&As_)>Q$wt@2=2dn(S+XgWDHGdY@=7|!U4>C{+qoXY9UY%(>Tp)8*M z(qhpm6bofSeVF78*ukj`!H1Yq86sV(I8+*&oF2~n>6z@FTkr^kc8HG`9zI~>_j7U4 z3Idl3r9v6NxK2geAr&elQ<)dc(HG3|0(!-vxX=)_qHtif0>lL>+c!YGL<0a7Vj0e& zMW_mDbjSRflx?t0vB;Ihf<-V@%nBt9w1{0X%QhLbm`7HGK?j2ngBAvd7<4h{2FStP&>W1u z1#3%ze4Ixd)bs+nDOZKHrpTMb-e$DAs`A>K-O&`qTvfpQD!&fgjiv;L4wM(F&^i0% z9jU1jRk^*|Zu+XeHP(y!MpdqImv?SFR28ezui`rFH=1X+;0F9g_zc|_z7=mORpoN? zKY0%NQA#yp1|=+7W)b=gn#>_($kC!rthh*(60vP#f!I_kL0J}eqZ^m0WII-&LS>^; zf&5v_MC&MnID;bqaX<7LDdrE%k@Yr6QKfl%LMM4ywpqz&JHtB6G&0>wZM${TsB~6L zjBEb}@E1H#=B=o`^R`kCMDGsW8T#(*_h((i`4e0{*nW5V&h*;nk51HjhhaVqjywyF z)Pf^*C2}`>C%o3VIacfX5av^5=$SH9Q-=PQQMoEaUTp1nW=pNysxv&k8DY_STW0wkD9`JxH^Diq3&m)%}q zHH`MM!aB=h;DcrV1hCwE>f*_oc=EXvaruA*5dnB9AUV2n(L0Et_WtF`dRs3*xMO*u z9_|5X9bP`~2GODJdY@L09d<+AlI9|S=aHBzv5kzP?XHhe3ZwjtQc-7*8(>rrb#(21 z)cEp0;F-^iPQ@kbJ$S>dWLm_lVdi_oBWtg#uazM$3ump5K!mlseG$MFIF569o)cZf cKt|lD8alauy7D_5c(NZ}va?tJ!)O@#FFqwYM*si- literal 0 HcmV?d00001 diff --git a/nodes/__pycache__/chat_image.cpython-311.pyc b/nodes/__pycache__/chat_image.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..42f349b6160068d578e80e3d6f5cdb14acf4c234 GIT binary patch literal 2486 zcmaJ?OKclO7@qY$yne+_X=+HE5IYsPftB%$lP&hHJG-H6 zaHUAC5LA`Q0jUB-kW&!~LOGTj#~xj&ht-@aae&^oO1W@A{Ii=l4JpjdeEZMLuYdpd z{*XvS5sdHte$_Y{M(8)b*&fmsu{jFFZG=$?VF8C~LP@|PmTIB8SQ6_}NfJ;9O&}~^ zM_9QNLTDXUFiJ8G6;LL;{CYIISa!5ahO?;U>g5HZnHDD6ajig1tQr1^V`-C5`Dx4r(?0%6yqw%d8p;mEd~;Cbl-4CJ?s~L7*@y zg|JW(iG)Mf(H%Hn-(o@tloU$}mP%nPmsG5jA~;-%Mi75+7mkq_j+6L>ZK((+NC)nK zl_W`=BB^Q|cV3rEop>LZnr6v@QE$`;x7W9H`RKE4{M4|Q$~A4t1XFCMY+}P)7^w(t zwGdK)7GZ6^0{{U4=X1wn87r%yY6yo{MSl-E`}+uvV$c;QaL1|y_n;jHt#H?>2)i_J zcdznr^yD)F>n!A^W+un8XUB?qb~=A1HvxxoZfa~|oGFFkY%V`hTuz@Rc7+-r6U|vP z>>y4f5Qj*8spc3p!z3(>Nwo~SjL4LV9}6b?@ni}Fs@`xIV)B{E>9OMPfoa~!g@8b4 zn|OQS+i&>z9huKe)k*6+WpoV=3U3NO-Y#RPXRJZ#YGsX!(kQVT zmT8j_+6iBvT^xEjbZ|&=NO9=rFu-9DpaKo5Jve<0)@B@H_AU~jJ{Hi`a8p`qvph}g zu6Wwp46pH6?-^KVLWful-GSTnHiv-zlV5C#&*FFF9nno;x3sHjQ(fbAy=OGTP2s|J z+eJ;esWd|i!cWp2khb2Icnw{RXKnQL6?83RUS4 z69Two6Pl5kS|_#*E!bvq9n!b9<6)-wxoT$=Cfg1*8cZoS8qlFBSBf6uaE!x2fJ_u> zlhmsJA2fOtSee@H*-(B#>}~$z(b|K)z~%BUET?x;ET{~gcpBW;zX1FJ50ph3rF*_m z-B{#{8TG`%2iXhk~flTJsW3Q zy>G(2uZ})YM_cOXp9)HAYs&quBM-Wce4%;@>Ob)P{%`lY-TU2ryG0;ou=9{oeqhQmC`<>qIL#)KZ7&Fle8LuC+VoGy9mT>t?x5be%n5!0*s8#BtWqo6cAYqj7N!S4B#Y0E7w2a4L0@U_@?%ru1*V>v z%V&$Z={!@jW5w}_>Ddoh?DTjcJDZ!~$CT#|lO2mz7MW6n7HLr?LhPAbtyxg-qG38g z;D>1>FF?wH|E27o0an_NE4|W^UU?{|yii;Q2LU`5kwf+ZqW_G*) literal 0 HcmV?d00001 diff --git a/nodes/__pycache__/chat_noass.cpython-311.pyc b/nodes/__pycache__/chat_noass.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0795134e1de547a02981d882a0bb47df15d24d44 GIT binary patch literal 3491 zcmcInO>7&-6`tjCm;e9N4<$?5%CcnEmeL>BDbm0RAxko0n^r|Cg&Cy;!&yq}?sDnf zWfCbAq9R72Lk>CwaAOFF4~6Q&Ip|chw;X+>0Us87C<64*`nHDcOAbZf+a)O}sSi1H zcIV@pH$QLQ&%9awIT8sHD1ZIWMd=@5LjH-3Uh#H`=gUCcC6ts2WvEAH$_!cZON4TF2;~nvggk`mmU57KiX`jXS`5x_R1Kk^ z=88q3p;A$p6;@PPY|7QmS+!|MinJwCp{6!I5H-DONNPj)P%<{UGwQ0QRE@ddQ(U(2 z6xYxCtnj*MRGOOlfkZ{kawN>Fjk>gMrQ8XXhFaBi9HKN0EB$^;l4+Nxh`L@~7jqw6hMu53uUp=z5}5(NU;tmxwHme{C?dd*eT#ZxkC>^N2e{TbX>H&M zTGt~&*CUP}=4wn&9Q=l$vQVGP01zk;Bp4u++xOHx4$XII-#*)Iv(&%OLT>=tK^l@6 zXJn9u9V$YjG)ChzL6iGl=tZ5VS|j}=f_vLVS*L$rKUtXH9B zkt}QR5Oo+dyuPU$qGAPGx~Ns8MzdvPSuGBFY6$=<2qvnDd56zfT>g4)X~FIBuEGo} zkn2gaxW#KXa;1O4EGYI{>n=e^k9d3i-WOQ@{#MrOF2O2N!(x@{N0t{h(X`lnq4bpa z33-avm&gkSw(1PkP60z31IY50uNRt}q(C@Z0cY=xssdzfqpDRLG6ZThMXhSIG|>uz zkcy%HPHgC46u$)^!vueEZJQ8*91z()`-Is?;}E^YEjn zjb3Z^dR1zy)mm^Y6gWdv)cZY0(YNMyz>d1-Y_7R!h4O{v)l#K&b9te-HF`AIiP5=Z zop`MDKLP9z+fPP+YL1>hywJJ)e&y^%L-tA1U;XC@CnK^BaPd~^UE_TlTsx$Vc+ql{}`*vq6X9_ci#jjJ3iks>=LE=W4Y6hjGhKhr2x`lK02K^XXu#r5PMy(@^*iy4s|rro z&w}q80+p(=y517!v?Oe%pbg_DGYF#y0s>lFOCt;*;I6gP2vZ1W5T+4k0BX*S0t6Na zR`My-=l3CQt`Y{)gK;c%v^{Mv_I$sb%!(R}Ye3#?vu(cZ`-(Z>AZJ2#%NpeV?cjFE z=yM2Z2iu{O_s(|M@bzXuKXU+bANFN$N7~%)**3S%pq%Zf5jZxM|IXN0J97V+tYag? zZRQr=Yq$OFKs(xwedRd--orlsQ@Be%X5S%q**|!<;|4}Wx3uH8&|*$;G-VGUu|9t@Hpj<5S3hRovc-;kUFr&>O8y2U%W zM#oI0HX+oHrg3h#;9{6r+^h6he^5iiSe$NXQd2`0XUEgF3GY;wV%V9mcRGip#1`y(UZ5j_Bt{nJq;>&eyvbL{IK^a zprdPq?-%Vn@EuLI(nn|9&1iV?&q25TGl;~0!?QzPj*zL>%~ZymnErD6-u4%t?0?cp z|8j3>Z|U%|IrJlQZ~|UC#7_mE5r7v6kNv`KWN-6jlBCZ*h#YP{PUW7Ya=ZSQJV^`p z`Nyf5C#jiTzs-}e$caGa_ruiIWT4> zhd)1W4xdJuu$#n$9U&t#&xmgXWCB2;0^A)O#r@$kJubGc#=rl zKBW9)VAu{I6(q?4JA_o2#0NfW+YzLrBt2xukcuOfKq^TFN9`0+X`J0(-jVSgaFs?? zvmtFBezdCqa3`vo=q3U*qLU-s=khVK9X literal 0 HcmV?d00001 diff --git a/nodes/__pycache__/config.cpython-311.pyc b/nodes/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..51afd185c2a1ddd65c5d49c079dec825aa73784f GIT binary patch literal 2540 zcmZ`*OK;mo5MGiZMM-`;vF!N8+DH`@i1KI>1Zfb|Q2bD1D~4qOiRyx&xhshXMJgoa z)`A`Q;Dgaad*~rW;X~0w+8{mjcl4AL0V?oRpy;7D2F@v`&MqZC(qt`uoZXq7otd2( z`seQMID(P5eBJmxhS1-1(ixF9ac~ic#|Wbm!W`yJuEv*m4)JInVPO~H$P*r+XYhoh zBw(?C(o*xY_{?%n$=X)gs3?|=iE_R@YdA!qJl(b|qI-sIDPJ01Rl%ED6Hr;8-YGVb0LOI^4JTTA7hNgH`qD-xjvP0aevP6U8PATgR z&$F#iiOwDRgisV{0iJ_7fXAo;xAha;7Zz}2pD*)R#1f7kqvbXo+ZS-0alvKWy)Tq` zgTp=hJhyjF4NQEgLTZL(I5M2mOcvNRFm~^|xS6!z%N?Wqs8%;rIMEkbj`{Ms<`T8xn9b2eqHPd` z)aqVnPuqBtSebZ0m^2^0lb@l%8Y@s=J zBpBv#<`}mdY;eB=*g=7eCT>0-{{()m$y96ddLRn71_Bhq_#p7d&$cE%Xid%oCq+37 z#s-5fG&;>V>%cinISkGW2Fc!$o&ZspLj9xP-=k8~pSC8y2%?d5AQ%ED7bK9G9`1_= zD1=!20L-Gn>M!J$mKSHU3p1-~_HKSIHxG+a=&j$aJ4Ew{+MdRboNYc0-Ti;rX5y?w zi>_87HyC*-qqGxoxt-r^`Eo{?wLB6g@*T3t7C+7?S<}!TD5*}4hNYWom+t#>m2JOr zcof^l)|?*j)aj+rN1>m>0Dvw6bcO^3YXIK>0S&uT{KB@l75T9vb>e-pFSlyLMsYQgiG$OS8PHW7iG6!@bzyR)|F$%`-NLsacf<+)k!q+ZB_5Mz<}!PCU(_ zt&qhm9XS>?8ZZ_*XbGb0Ina=V3h^8ovvkyt&;mXtei&rG>-Oy2OmT5lg|aU=V<7I* z6KLm$;S5u~;{eTxqwH#z(uwT5Mt7P3P)q|TdAT)or8TqwnLZ3z4Y1c0^q{`sZy$aA z=tb}3%ihTVaj|Qy--{L??$Iz8Sx#c&`Q#KeL4D+KVG z!kz?*`jdN`&*fp~3>=tGb74HXeOXoED@#<>kE<$l!G=lcE>(Tl(9AX`s;b!5VL1UU z-*s!mtJ>HZ0Y!e-%Is=!B_H;V!&!c2X|~|YbH#jiHFr1f%h{RL+4;LG_x;4}*+OsnL3!s+Fos4(gT#n9LidgQHE=o*=|n<6MJXktzQG_y zsW=)OIud2_Z$W4_H8v0Fm4dh5quUP2biOh!%P8i~TYkQgZ-x=S<Kp=1& j7w|k62@nMdap#_+)bU>+EpQyz|89K6)?dGe%Pjs6%Du5Z literal 0 HcmV?d00001 diff --git a/nodes/__pycache__/gemini_edit.cpython-311.pyc b/nodes/__pycache__/gemini_edit.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc0efabbc68a0acf4be8c4552e6a5f2c482e76c4 GIT binary patch literal 2270 zcmZ`(%Wo4$7@vK5{fZsOQR5_3%cD>WFpr|XM1j;b#tJb};!xp6T5UaJXQ}Nqvrb!F zBSj^udf? zijSGLH19-eq^-56`Kf^kbwb%Cy`43iRijq%&+;+YpKN{%#2rKs7>mk0=E?#QF~5q| zfWO{fgb)}glw~ZI16V36SS|-~pd1P!9K%5p#vu~98XYXgVKRgxWT>j(=&DqX;URD> z?!um|)kz zRIe`-;MXvlgW1s;-DoaWmaVqg&{&OFTatOB(W(=c`9)t)^NLC%5cW*#x)UxGXBSFZ z>B?+wzB92SFPUw^FxX+u0<0icK@%r>sr|i){Xd`j^?m!bJp0BkxB)RW)YXv9se0h8ZLPDiXk; z1%S8-WAjgdX~aQsb-AkpDHPf#tn3htwY|SD!2B8{Z+)vkzvSB6bauIyr8lImbbs6K zE#;^yz*&Qr1$B9hy5dj58t7Wz(svWynlFUQ=#KD}cth^WSGOc9;Vutzb~U`#;;>kS zg5SPoyPV0=Qqht9Lh0oFaBqKw`sHf%UmX@o?}f>zvvs0d1TsKWeIWJkL?$hHv2`Rb z7fy_lDxsuVA)0?#BD&Qgm9|E8xB*@UI*Fp$&tHu+2?z*I2OMQ^oD;gN*Oy6-Qj<~^ zA7^N=&euuz0@F|H9-v^aXpwq#w`|h`Y(HE?t7SGVVzGp)exu_|%c}sL% z`+iRyf0#Pn8%gyNW4Dj=CinJIdwRp8ZWxW9c!K2d&{GC3l9T@rpelhP$y;aa@ZODN z>h9b3-m@pp*+jGB6({^bZazC#m}P9r3e}O?Cao+va)~r8lRARcre4J97)U*9{VcGw zPzDMN>o&m3R_}>fTg+}qahDHC5GjBs0+NSUKk^x4D0yh*VsGRCKx}m7LNB%tAaP}7 z+8sipDK|VR9di-DMtt0r*hEJ0(OX?Nz-R>xC*2^ULMWDavDn+O1AK#-nQ56GfggPX zz!57ZC0?O=PJ4draPXOO+7JI3`W9?qPlts8T!G^_m*+XrMGR!bMOPxM_t0T`x6hSN Ra^TK?;R)kC-9_M8{V&vPO(g&T literal 0 HcmV?d00001 diff --git a/nodes/__pycache__/gemini_gen.cpython-311.pyc b/nodes/__pycache__/gemini_gen.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db99796fd8d2c769a4cf95d4b76ffc400aabde05 GIT binary patch literal 2495 zcmZ`*OKcNI7@mE4{m9EPA`YZ1X?a)>gGoRoqCjwsu>vL}PAXiat?L;lOKoqM-9T|e z6_u#sz>x|Lk&3E{wxWj~tDbu7aY0D6R;q+lRVuedp%)HS|Jk*Jp)kAq?fm~gJM+Ko z@4C7|2*z*!9@4)KAoM3A+Cywj){nsCHo~ZgFo*dnSLCpO#VTJDib72+iX7rm7GY@> zVfh}9(0zFPQIs%0gA#$pu~3%O^t!I*YNdIiW=UPG8zGg{lwK%VX`^1z=ga=FJ_cLFdJ!hK z5ka6gD)N{s3Pi;GD!K>!`>jO?fv!SP#$qvmrJ{o6Vh{(4p&$w(+>1jbjKd^?Bk**Q zNJYWjt5UHG_keYMPPeb7XB(K-$O14ki0WBosugO~R0}LsmrT7r-!eU1W3YNDW}&*o3&vW`4cn9k+11&4V!SX-)EdR4CzCxA(% z1Qc-Pvgc(-2H)2ftwi9tkQz@l#MGhj!wvE9cK_J&;KLol=s z2Qha^xib^l3}tnj;OP*&9Fe7#qqK^LH2Pi`xX)L=aq;Dpd{ajHRagUWRZVrb@0O7R z$%7-6YROz^6{$hhmSl7UGQsDX&RX zA$f6dBrmp34EdAoPB3!-=4yw9j@z zbEWDM$xv!g%6#f{x7W1}7-me#JIs+RgO(Q@sX?e=(zig8P@wQhweq@*p}W|p&=1T- zqizyMBG7xTn#`kWYtTU;=r(}HX5U{q_HzO`pKPnaPBDK0_!}-L*j^Ng-aK|^;bC~E z9p3pzbgMl?9z-jTq7^$*`A0@E^{)J&f7he_T{je0MuWq@$M-*s?{D_Uo6&(=dz)L; zW_(ApZ<8BFgQ=%T9t=HW;37HpW`M1tC=$DQ)DCa|D;B>sX{-Am#8Qu9sT-MQEPm(R zwfF2T$L+nx?fwZnGV!=~;8yot<=Za1Cuu85x)s#VL(#Kd>UYm>N71x;sYW!-327Px zcd5$e5ly?aRI0Wl0Zqe38Q9@6l(AVO)`EfQ7TCp!OlJzS(|NzyZBY4%GnpAjIXRn8 z7jkFwj*^}zWU^WjGnHdD~%K{v=|A=DFnvD91t3iui`Jrgn=g%{le;D}{|5-*iK zGd)9IvGBQa#CP~n%Kjp;vN17$D{vg=@;oQHh=Gi_u9XP8nrM&xy4#gUIWXq`@RaeM Jy++_^{U46miHZOK literal 0 HcmV?d00001 diff --git a/nodes/chat.py b/nodes/chat.py new file mode 100644 index 0000000..8f0ca4d --- /dev/null +++ b/nodes/chat.py @@ -0,0 +1,55 @@ +""" +Chat node - Basic text conversation. +""" +from ..core import get_provider, ChatConfig + + +class SimpleChatText: + """Basic text conversation with LLM.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "config": ("SIMPLECHAT_CONFIG",), + "prompt": ("STRING", {"multiline": True, "default": ""}), + }, + "optional": { + "system": ("STRING", {"multiline": True, "default": ""}), + "temperature": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.1}), + "max_tokens": ("INT", {"default": 2048, "min": 1, "max": 128000}), + } + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("text",) + FUNCTION = "chat" + CATEGORY = "SimpleChat" + DESCRIPTION = "Send a text prompt to LLM and get a response." + + async def chat( + self, + config: ChatConfig, + prompt: str, + system: str = "", + temperature: float = 1.0, + max_tokens: int = 2048, + ): + # Build messages + messages = [] + if system.strip(): + messages.append({"role": "system", "content": system}) + messages.append({"role": "user", "content": prompt}) + + # Get provider and send request + provider = get_provider(config) + + # Directly await the async provider method + response = await provider.chat( + messages=messages, + model=config.model, + temperature=temperature, + max_tokens=max_tokens, + ) + + return (response.text,) diff --git a/nodes/chat_image.py b/nodes/chat_image.py new file mode 100644 index 0000000..5dbcaa0 --- /dev/null +++ b/nodes/chat_image.py @@ -0,0 +1,59 @@ +""" +Chat with Image node - Send image to LLM for analysis. +""" +import torch +from ..core import get_provider, ChatConfig + + +class SimpleChatImage: + """Send image to LLM for visual understanding.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "config": ("SIMPLECHAT_CONFIG",), + "image": ("IMAGE",), + "prompt": ("STRING", {"multiline": True, "default": "Describe this image."}), + }, + "optional": { + "system": ("STRING", {"multiline": True, "default": ""}), + "temperature": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.1}), + "max_tokens": ("INT", {"default": 2048, "min": 1, "max": 128000}), + } + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("text",) + FUNCTION = "chat" + CATEGORY = "SimpleChat" + DESCRIPTION = "Send an image to LLM for visual analysis and get a text response." + + async def chat( + self, + config: ChatConfig, + image: torch.Tensor, + prompt: str, + system: str = "", + temperature: float = 1.0, + max_tokens: int = 2048, + ): + # Build messages + messages = [] + if system.strip(): + messages.append({"role": "system", "content": system}) + messages.append({"role": "user", "content": prompt}) + + # Get provider and send request + provider = get_provider(config) + + # Directly await the async provider method + response = await provider.chat( + messages=messages, + model=config.model, + temperature=temperature, + max_tokens=max_tokens, + images=[image], + ) + + return (response.text,) diff --git a/nodes/chat_noass.py b/nodes/chat_noass.py new file mode 100644 index 0000000..d6415ee --- /dev/null +++ b/nodes/chat_noass.py @@ -0,0 +1,132 @@ +""" +Chat NoASS node - Roleplay-optimized conversation with NoASS format. +""" +import torch +from ..core import ( + get_provider, + ChatConfig, + format_noass_prompt, + build_noass_messages, + extract_noass_response, + build_full_history, + get_stop_sequences, +) + + +class SimpleChatNoASS: + """NoASS format conversation for roleplay optimization.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "config": ("SIMPLECHAT_CONFIG",), + "scenario_instructions": ("STRING", { + "multiline": True, + "default": "", + "placeholder": "Define world setting, characters, and instructions here (The Scenario)." + }), + "user_action": ("STRING", { + "multiline": True, + "default": "", + "placeholder": "Your current action or dialogue." + }), + }, + "optional": { + "prefill_start": ("STRING", { + "multiline": True, + "default": "", + "placeholder": "Force the start of the assistant's response (Prefill)." + }), + "history": ("STRING", {"multiline": True, "default": "", "forceInput": True}), + "image": ("IMAGE",), + "user_name": ("STRING", {"default": "User"}), + "char_name": ("STRING", {"default": "Assistant"}), + "temperature": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.1}), + "max_tokens": ("INT", {"default": 2048, "min": 1, "max": 128000}), + } + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("text", "history") + FUNCTION = "chat" + CATEGORY = "SimpleChat" + DESCRIPTION = "NoASS (Experimental) - Hardcore Roleplay with Assistant Prefill. Uses single-turn context with giant assistant prefill to force formatting." + + async def chat( + self, + config: ChatConfig, + scenario_instructions: str, + user_action: str, + prefill_start: str = "", + history: str = "", + image: torch.Tensor = None, + user_name: str = "User", + char_name: str = "Assistant", + temperature: float = 1.0, + max_tokens: int = 2048, + ): + # Format in NoASS style + system_prompt, prefilled = format_noass_prompt( + system=scenario_instructions, + history=history if history.strip() else None, + user_input=user_action, + prefill_start=prefill_start, + user_name=user_name, + char_name=char_name, + ) + + # Build messages using the new prefill logic + messages = build_noass_messages( + system=scenario_instructions, + history=history if history.strip() else None, + user_input=user_action, + prefill_start=prefill_start, + user_name=user_name, + char_name=char_name, + ) + + # Get provider + provider = get_provider(config) + + # Prepare images + images = [image] if image is not None else None + + # Get stop sequences + stop_sequences = get_stop_sequences(user_name) + + # Directly await the async provider method + response = await provider.chat( + messages=messages, + model=config.model, + temperature=temperature, + max_tokens=max_tokens, + images=images, + stop=stop_sequences, # Some providers support this + ) + + # Extract response and build history + response_text = extract_noass_response( + response.text, + user_name=user_name, + char_name=char_name, + ) + + # Reconstruct the full text for the history output + new_history = build_full_history( + history=history if history.strip() else None, + user_input=user_action, + prefill_start=prefill_start, + response=response_text, + user_name=user_name, + char_name=char_name, + ) + + # If we had a prefill, the returned 'text' should probably include it + # so the user sees the full response in the text output node + if prefill_start.strip(): + final_text_output = f"{prefill_start} {response_text}" + else: + final_text_output = response_text + + return (final_text_output, new_history) diff --git a/nodes/config.py b/nodes/config.py new file mode 100644 index 0000000..8b99b1a --- /dev/null +++ b/nodes/config.py @@ -0,0 +1,79 @@ +""" +API Config node - Configure API connection with dynamic model selection. +""" +from ..core.providers import ChatConfig + + +class SimpleChatConfig: + """ + Configure API connection for SimpleChat nodes. + Supports dynamic model fetching via the "Refresh Models" button. + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "provider": (["openai", "claude", "gemini"], { + "default": "openai", + "tooltip": "Select the LLM provider", + }), + "api_key": ("STRING", { + "default": "", + "tooltip": "Your API key for the selected provider", + }), + # Defined as STRING to bypass ComfyUI's strict list validation. + # Frontend JS will convert this to a COMBO widget for dropdown functionality. + "model": ("STRING", { + "default": "", + "tooltip": "Select a model. Use the Refresh button to fetch available models.", + }), + }, + "optional": { + "base_url": ("STRING", { + "default": "", + "placeholder": "Leave empty for default URL", + "tooltip": "Custom API base URL (for proxies or self-hosted deployments)", + }), + } + } + + RETURN_TYPES = ("SIMPLECHAT_CONFIG",) + RETURN_NAMES = ("config",) + FUNCTION = "create_config" + CATEGORY = "SimpleChat" + DESCRIPTION = """Configure API connection. + + Usage: + 1. Select Provider + 2. Enter API Key + 3. Click 'Refresh Models' button to fetch available models + 4. Select Model + """ + + def create_config( + self, + provider: str, + api_key: str, + model: str, + base_url: str = "", + ): + # Default URLs map + DEFAULT_URLS = { + "openai": "https://api.openai.com/v1", + "claude": "https://api.anthropic.com/v1", + "gemini": "https://generativelanguage.googleapis.com/v1beta", + } + + # Use default URL if not provided + if not base_url.strip(): + base_url = DEFAULT_URLS.get(provider, "") + + config = ChatConfig( + provider=provider, + api_key=api_key, + base_url=base_url, + model=model, + ) + + return (config,) diff --git a/nodes/gemini_edit.py b/nodes/gemini_edit.py new file mode 100644 index 0000000..42ff0b1 --- /dev/null +++ b/nodes/gemini_edit.py @@ -0,0 +1,58 @@ +""" +Gemini Image Edit node - Edit images using Gemini. +""" +import torch +from ..core import get_provider, ChatConfig + + +class GeminiImageEdit: + """Edit images using Gemini (Nano Banana / Nano Banana Pro).""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "config": ("SIMPLECHAT_CONFIG",), + "image": ("IMAGE",), + "prompt": ("STRING", {"multiline": True, "default": ""}), + }, + "optional": { + "size": (["1K", "2K", "4K"], {"default": "1K"}), + } + } + + RETURN_TYPES = ("IMAGE", "STRING") + RETURN_NAMES = ("IMAGE", "text") + FUNCTION = "edit" + CATEGORY = "SimpleChat/Gemini" + DESCRIPTION = "Edit or modify an image using Gemini image models with a text prompt." + + async def edit( + self, + config: ChatConfig, + image: torch.Tensor, + prompt: str, + size: str = "1K", + ): + # Validate provider + if config.provider != "gemini": + raise ValueError("Gemini Image Edit requires Gemini provider. Please use a Gemini config.") + + # Get provider + provider = get_provider(config) + + # Directly await the async provider method + response = await provider.generate_image( + prompt=prompt, + model=config.model, + reference_image=image, + aspect_ratio=None, # Preserve original aspect ratio for edits + size=size, + ) + + # Ensure we have an image + if response.image is None: + # Return original image if edit failed + return (image, response.text or "No image generated") + + return (response.image, response.text) diff --git a/nodes/gemini_gen.py b/nodes/gemini_gen.py new file mode 100644 index 0000000..e8b7dac --- /dev/null +++ b/nodes/gemini_gen.py @@ -0,0 +1,59 @@ +""" +Gemini Image Gen node - Text-to-image generation with Gemini. +""" +import torch +from ..core import get_provider, ChatConfig + + +class GeminiImageGen: + """Generate images from text using Gemini (Nano Banana / Nano Banana Pro).""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "config": ("SIMPLECHAT_CONFIG",), + "prompt": ("STRING", {"multiline": True, "default": ""}), + }, + "optional": { + "aspect_ratio": (["1:1", "16:9", "9:16", "4:3", "3:4", "5:4", "4:5"], {"default": "1:1"}), + "size": (["1K", "2K", "4K"], {"default": "1K"}), + } + } + + RETURN_TYPES = ("IMAGE", "STRING") + RETURN_NAMES = ("IMAGE", "text") + FUNCTION = "generate" + CATEGORY = "SimpleChat/Gemini" + DESCRIPTION = "Generate image from text using Gemini image models (gemini-2.5-flash-image or gemini-3-pro-image-preview)." + + async def generate( + self, + config: ChatConfig, + prompt: str, + aspect_ratio: str = "1:1", + size: str = "1K", + ): + # Validate provider + if config.provider != "gemini": + raise ValueError("Gemini Image Gen requires Gemini provider. Please use a Gemini config.") + + # Get provider + provider = get_provider(config) + + # Directly await the async provider method + response = await provider.generate_image( + prompt=prompt, + model=config.model, + reference_image=None, + aspect_ratio=aspect_ratio, + size=size, + ) + + # Ensure we have an image + if response.image is None: + # Return empty image if generation failed + empty = torch.zeros((1, 512, 512, 3)) + return (empty, response.text or "No image generated") + + return (response.image, response.text) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6cea0c0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +aiohttp>=3.8.0 diff --git a/web/simple_chat_config.js b/web/simple_chat_config.js new file mode 100644 index 0000000..5f8db88 --- /dev/null +++ b/web/simple_chat_config.js @@ -0,0 +1,88 @@ +import { app } from "../../scripts/app.js"; +import { api } from "../../scripts/api.js"; + +app.registerExtension({ + name: "ComfyUI.SimpleChat.Config.V3", // Use a new name to avoid browser caching issues + async nodeCreated(node, app) { + if (node.comfyClass === "SimpleChatConfig") { + // --- WIDGET REPLACEMENT AND REORDERING --- + + // Find the original string widget for 'model' + const modelWidgetIndex = node.widgets.findIndex((w) => w.name === "model"); + if (modelWidgetIndex === -1) return; + const originalModelWidget = node.widgets[modelWidgetIndex]; + const originalModelValue = originalModelWidget.value; + + // Step 1: Create new widgets using the addWidget API. This appends them temporarily. + const newModelCombo = node.addWidget("combo", "model", originalModelValue, () => {}, { + values: [originalModelValue || ""] + }); + const refreshButton = node.addWidget("button", "Refresh Models", "refresh", () => {}); + + // Step 2: Remove the newly created widgets from the end of the array. + // We now have fully constructed widget objects, detached from the node's widget list. + node.widgets.pop(); // removes refreshButton + node.widgets.pop(); // removes newModelCombo + + // Step 3: Remove the original string widget from its position. + node.widgets.splice(modelWidgetIndex, 1); + + // Step 4: Insert the new, constructed widgets back into the array at the correct positions. + node.widgets.splice(modelWidgetIndex, 0, newModelCombo); // Insert combo where the old one was. + node.widgets.splice(modelWidgetIndex + 1, 0, refreshButton); // Insert button right after it. + + // --- END WIDGET MANIPULATION --- + + // Now, find all widgets again to be safe and attach logic + const providerWidget = node.widgets.find((w) => w.name === "provider"); + const apiKeyWidget = node.widgets.find((w) => w.name === "api_key"); + const baseUrlWidget = node.widgets.find((w) => w.name === "base_url"); + + // Attach the callback logic to the refresh button + refreshButton.callback = () => { + const provider = providerWidget.value; + const apiKey = apiKeyWidget.value; + const baseUrl = baseUrlWidget ? baseUrlWidget.value : ""; + + if (!apiKey) { + alert("Please enter an API Key first."); + return; + } + + refreshButton.label = "Loading..."; + refreshButton.disabled = true; + node.setDirtyCanvas(true, true); + + const url = `/simplechat/models/${provider}?api_key=${encodeURIComponent(apiKey)}&base_url=${encodeURIComponent(baseUrl)}`; + + api.fetchApi(url) + .then(response => response.json()) + .then(models => { + if (Array.isArray(models) && models.length > 0) { + newModelCombo.options.values = models; + if (!newModelCombo.value || !models.includes(newModelCombo.value)) { + newModelCombo.value = models[0]; + } + } else { + newModelCombo.options.values = []; + newModelCombo.value = ""; + alert("No models found."); + } + }) + .catch(err => { + console.error("Error fetching models:", err); + alert("Error fetching models: " + err.message); + }) + .finally(() => { + refreshButton.label = "Refresh Models"; + refreshButton.disabled = false; + node.setDirtyCanvas(true, true); + }); + }; + + // Trigger a redraw to ensure the node resizes correctly after manipulation + node.computeSize(); + node.setDirtyCanvas(true, true); + } + } +});