Compare commits
528
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
77a6b5cd41 | ||
|
|
1287555f3c | ||
|
|
d578a62677 | ||
|
|
20c69ebf16 | ||
|
|
a2560ea727 | ||
|
|
8b2a36c185 | ||
|
|
d05d7021c3 | ||
|
|
06b7f5129f | ||
|
|
d99f8a7331 | ||
|
|
d0ecbd4131 | ||
|
|
4178b4facc | ||
|
|
4371e74fad | ||
|
|
da2061a8a5 | ||
|
|
6c22ae9136 | ||
|
|
87abc2aa6b | ||
|
|
2b4df4c83a | ||
|
|
1f5a28035b | ||
|
|
c36f0d1c4f | ||
|
|
84a3414d89 | ||
|
|
4a2d33166b | ||
|
|
433a1e337f | ||
|
|
b79c8b9dea | ||
|
|
1ac2d581ac | ||
|
|
4b48b74f47 | ||
|
|
3899931560 | ||
|
|
7d86208531 | ||
|
|
7b2f872f32 | ||
|
|
b3325897a3 | ||
|
|
db160d2360 | ||
|
|
08f63ca950 | ||
|
|
ccb9525201 | ||
|
|
f3f1740b6c | ||
|
|
2d4ca16768 | ||
|
|
551ecade95 | ||
|
|
4e741f367e | ||
|
|
a9a25f247d | ||
|
|
c5d78bf682 | ||
|
|
fb3cb8b6c0 | ||
|
|
a0c1b0d863 | ||
|
|
645e473cb7 | ||
|
|
a9a04bafe9 | ||
|
|
d481343cbb | ||
|
|
d54067e736 | ||
|
|
2f9b574a82 | ||
|
|
fa0fb562d9 | ||
|
|
c5aef4d036 | ||
|
|
a1eb7957ba | ||
|
|
5651f53bfc | ||
|
|
c933dc1ccc | ||
|
|
44110e6b2d | ||
|
|
df5191de9c | ||
|
|
65131b2024 | ||
|
|
da823ccb57 | ||
|
|
872f5d5942 | ||
|
|
6a23293f2f | ||
|
|
21e22d89b4 | ||
|
|
83ef80cc4b | ||
|
|
4cb8e0658c | ||
|
|
d830624b67 | ||
|
|
f92c3b9a04 | ||
|
|
e679a562f2 | ||
|
|
17d0ff9d25 | ||
|
|
80a8082e17 | ||
|
|
953325b699 | ||
|
|
2b9f3d49ef | ||
|
|
f9b5cbad90 | ||
|
|
aa5cadf527 | ||
|
|
64f7f3da7f | ||
|
|
df08c95044 | ||
|
|
8ecafa242c | ||
|
|
13007792d2 | ||
|
|
162e0e89c0 | ||
|
|
318b988c7f | ||
|
|
99a8ba2709 | ||
|
|
56c7d28577 | ||
|
|
8fa9013b81 | ||
|
|
aa90f948b7 | ||
|
|
9ca156d07e | ||
|
|
c2ca220053 | ||
|
|
36485f0907 | ||
|
|
416ec9f844 | ||
|
|
d9a41c45d7 | ||
|
|
71b66de4ac | ||
|
|
be6e31c1c9 | ||
|
|
f5254347ac | ||
|
|
fb57c1d1fb | ||
|
|
64d0f3f3c2 | ||
|
|
be36b0b547 | ||
|
|
af2e29e08b | ||
|
|
fd16cf274e | ||
|
|
35b49ad00c | ||
|
|
6aab2c6d3e | ||
|
|
7c9c9b29ce | ||
|
|
183e8a37aa | ||
|
|
b42baa0309 | ||
|
|
099f8f198f | ||
|
|
aaa256983d | ||
|
|
4d41b5a510 | ||
|
|
674506c88c | ||
|
|
b8840e66d9 | ||
|
|
8ab9e56657 | ||
|
|
9bf4f190ae | ||
|
|
405e672587 | ||
|
|
f00028cfbe | ||
|
|
970b624fcf | ||
|
|
15965ca387 | ||
|
|
1f3cfb746d | ||
|
|
c2a2c7f8d8 | ||
|
|
29a448493b | ||
|
|
5d37f9bd69 | ||
|
|
0a5298bc56 | ||
|
|
08cf3e4dd8 | ||
|
|
7cc8b5617a | ||
|
|
9e5aa39110 | ||
|
|
3fce50fd85 | ||
|
|
a7ed5de1eb | ||
|
|
9a31bd8f32 | ||
|
|
ed8c9c1261 | ||
|
|
2b15c62f6b | ||
|
|
16c87d76ed | ||
|
|
81d783d904 | ||
|
|
262f6e655c | ||
|
|
8297e94c4e | ||
|
|
2f2d2ef76d | ||
|
|
92ab1fe982 | ||
|
|
26b485d438 | ||
|
|
7896561ae5 | ||
|
|
9e26966d7b | ||
|
|
0432f6c86a | ||
|
|
3853cc675a | ||
|
|
7cb0baa06e | ||
|
|
6df454810c | ||
|
|
a766c3079e | ||
|
|
902d10350b | ||
|
|
f61cfb55fc | ||
|
|
053cb69cf5 | ||
|
|
0a140fd60e | ||
|
|
ca54ff9753 | ||
|
|
4f5fef9fa3 | ||
|
|
1713d1a924 | ||
|
|
f499dee1c7 | ||
|
|
148c6ec512 | ||
|
|
b16199b19b | ||
|
|
9c61bf5afa | ||
|
|
6731f75aa1 | ||
|
|
0d73e61b34 | ||
|
|
c30913007c | ||
|
|
2c451fb5fe | ||
|
|
2e2e2a5f45 | ||
|
|
0d91f542e0 | ||
|
|
7897132738 | ||
|
|
ddddcd6bb3 | ||
|
|
1fc4e8f7fd | ||
|
|
b172feff99 | ||
|
|
e9e5dac74f | ||
|
|
797a126d02 | ||
|
|
cb8f0d8844 | ||
|
|
595e4ce4b7 | ||
|
|
c17343c3bc | ||
|
|
d4df7dcbb5 | ||
|
|
c1bfb80561 | ||
|
|
38edacfec3 | ||
|
|
8d88559c21 | ||
|
|
b37138eff0 | ||
|
|
c89e678813 | ||
|
|
21588e3cd8 | ||
|
|
6e885ce98d | ||
|
|
bad8947410 | ||
|
|
69bbb23aa8 | ||
|
|
ea09ea6b53 | ||
|
|
badfe3597c | ||
|
|
1c229b1c59 | ||
|
|
c9738f904e | ||
|
|
3b484514df | ||
|
|
01c642f4c9 | ||
|
|
4efea4a7f2 | ||
|
|
dc0ed1586a | ||
|
|
fa32e0cac5 | ||
|
|
0b67b14c2d | ||
|
|
c265a8e30f | ||
|
|
39d8e8dda5 | ||
|
|
244e298fb8 | ||
|
|
8aaf970c3a | ||
|
|
271106b633 | ||
|
|
39e506c85e | ||
|
|
d295bcf0d6 | ||
|
|
4384e6ca43 | ||
|
|
37c07c294f | ||
|
|
0f9d749f51 | ||
|
|
536f111487 | ||
|
|
561ac08edb | ||
|
|
da316e461c | ||
|
|
c2dd8d9df6 | ||
|
|
b52daeff69 | ||
|
|
606985e3c8 | ||
|
|
058abdf79e | ||
|
|
4476ec8347 | ||
|
|
f422083360 | ||
|
|
64ce6956ef | ||
|
|
f5f276927f | ||
|
|
48fe00f427 | ||
|
|
226ada0148 | ||
|
|
e6f88055a9 | ||
|
|
c636c48fe8 | ||
|
|
91f28af322 | ||
|
|
b5ff30b1c3 | ||
|
|
ef8de26bfb | ||
|
|
b478719a59 | ||
|
|
f20d281d5d | ||
|
|
0359bc8cff | ||
|
|
b12ab8e52b | ||
|
|
7cbce4ea75 | ||
|
|
665119a770 | ||
|
|
21e52b37a4 | ||
|
|
4b3606214e | ||
|
|
bd53d7087b | ||
|
|
8615809506 | ||
|
|
fd9bcc4181 | ||
|
|
ce979c10d1 | ||
|
|
c082acaae9 | ||
|
|
952c93d072 | ||
|
|
a4c3797fc2 | ||
|
|
ce8047b95d | ||
|
|
be39df191a | ||
|
|
23a93e9452 | ||
|
|
8096b4437f | ||
|
|
c81ce9d873 | ||
|
|
39c30db67c | ||
|
|
93c5543a62 | ||
|
|
6063f49359 | ||
|
|
3f891ae03b | ||
|
|
fa2a921990 | ||
|
|
b6b67e009b | ||
|
|
8f0646a504 | ||
|
|
8e9350162c | ||
|
|
e1c9969392 | ||
|
|
209ff283ce | ||
|
|
21f879064a | ||
|
|
e41f1747a2 | ||
|
|
816fc31bc3 | ||
|
|
80ad9bd0ce | ||
|
|
f1198266a9 | ||
|
|
ea637ebab2 | ||
|
|
53df9f029a | ||
|
|
c21822908a | ||
|
|
ceb3333ded | ||
|
|
ace443142b | ||
|
|
1f6b5a5c77 | ||
|
|
9c908c6ce0 | ||
|
|
b60aa3cc1d | ||
|
|
e6752829b8 | ||
|
|
bfb06a75b9 | ||
|
|
a9ba81b12b | ||
|
|
f30974f287 | ||
|
|
afea4d832a | ||
|
|
fb41d35803 | ||
|
|
e87a0c66c3 | ||
|
|
2ea3800bc2 | ||
|
|
6295b19ecb | ||
|
|
f2ab3302ab | ||
|
|
fbf7d828ec | ||
|
|
159478fe9e | ||
|
|
6908339d60 | ||
|
|
9a840a0e60 | ||
|
|
9ce5aaee5e | ||
|
|
97b8c4f139 | ||
|
|
cad4d98c49 | ||
|
|
20300ddb78 | ||
|
|
357a699f25 | ||
|
|
08f41e2ef7 | ||
|
|
ca71aabc41 | ||
|
|
9a937edea8 | ||
|
|
99b4f27938 | ||
|
|
05abe57102 | ||
|
|
403ae8d7df | ||
|
|
0c707f80a6 | ||
|
|
7da880f0db | ||
|
|
6d96bb162b | ||
|
|
70b35ac75f | ||
|
|
bf5a3dfe4c | ||
|
|
17843dea5e | ||
|
|
c76ae2fef6 | ||
|
|
f8b7e9717c | ||
|
|
db2a30b9b4 | ||
|
|
6ebf0579e6 | ||
|
|
6b217ec6c7 | ||
|
|
83e3936ed2 | ||
|
|
67acb8eecc | ||
|
|
a867b9cf85 | ||
|
|
3725de6dc8 | ||
|
|
a0e9888aa4 | ||
|
|
79d02907f1 | ||
|
|
ebe6bbac65 | ||
|
|
3573be39f1 | ||
|
|
088cb7777e | ||
|
|
2dec3c7c40 | ||
|
|
ed6c9aad4a | ||
|
|
703e40f4c6 | ||
|
|
3950dcf3af | ||
|
|
afd6d1ae50 | ||
|
|
2a70e4f872 | ||
|
|
e6bb91a854 | ||
|
|
818fcaaab1 | ||
|
|
39a39098c9 | ||
|
|
2991f7ec1e | ||
|
|
bd5bce3b78 | ||
|
|
366f710bd5 | ||
|
|
dc6e8cd2a3 | ||
|
|
233bfa897f | ||
|
|
3704ee423f | ||
|
|
c1bc9e31fe | ||
|
|
7a1820a95b | ||
|
|
55732bfe14 | ||
|
|
91798defc7 | ||
|
|
c29b45a34b | ||
|
|
7e75cb64df | ||
|
|
b2651a51a8 | ||
|
|
290e19007a | ||
|
|
db74ea6394 | ||
|
|
6494929ca5 | ||
|
|
02e19fcebe | ||
|
|
3ca61dacf9 | ||
|
|
644b230410 | ||
|
|
f817e4eb81 | ||
|
|
e622e045de | ||
|
|
68908817c0 | ||
|
|
685dc3cbcc | ||
|
|
d6588708c5 | ||
|
|
9643af90f6 | ||
|
|
addd0d2154 | ||
|
|
59afef47ff | ||
|
|
4486525250 | ||
|
|
8af7aebc6c | ||
|
|
858960bf9b | ||
|
|
c8ab9ba7d2 | ||
|
|
cbbe9695c6 | ||
|
|
7d37e6b0cf | ||
|
|
4cf0c6a609 | ||
|
|
fbb56e2ffd | ||
|
|
4ce237d8a0 | ||
|
|
fbb06bc15e | ||
|
|
eb2d1f9bde | ||
|
|
235b2ccb7e | ||
|
|
00dd008d07 | ||
|
|
cd268b6289 | ||
|
|
df2a309146 | ||
|
|
455ef1358b | ||
|
|
87525a2cbb | ||
|
|
af026764dc | ||
|
|
e41bd104a0 | ||
|
|
399fe1d801 | ||
|
|
7b49bbd213 | ||
|
|
a6d98c9244 | ||
|
|
b7133a5521 | ||
|
|
50bc33178d | ||
|
|
2773f44cc1 | ||
|
|
e8041c1210 | ||
|
|
37b0428572 | ||
|
|
f83087d494 | ||
|
|
7039eb4bc2 | ||
|
|
8fccf74ff5 | ||
|
|
48cfc51f50 | ||
|
|
79693a337e | ||
|
|
6619d19a32 | ||
|
|
5f5af1be77 | ||
|
|
67b94d2360 | ||
|
|
baefe92076 | ||
|
|
0b8e8450cf | ||
|
|
dbec3ac4d2 | ||
|
|
d820d09dce | ||
|
|
fd278bf006 | ||
|
|
40478a049c | ||
|
|
fa208d2b3a | ||
|
|
8fcf20904f | ||
|
|
94e21e3fe6 | ||
|
|
95015ce8a2 | ||
|
|
5ffcbc8e05 | ||
|
|
89ce688eba | ||
|
|
77c43895c1 | ||
|
|
5f7dc3b914 | ||
|
|
2837d2e940 | ||
|
|
056c2de198 | ||
|
|
b1951b8ca6 | ||
|
|
ef1cd3df9f | ||
|
|
f15ad58e7e | ||
|
|
44fe570b21 | ||
|
|
b85a23a404 | ||
|
|
e01e376323 | ||
|
|
baed844040 | ||
|
|
63d36b00fb | ||
|
|
b1c6315b95 | ||
|
|
dbdf3fd664 | ||
|
|
e199051340 | ||
|
|
b6673541e5 | ||
|
|
5f8ca110c2 | ||
|
|
efc98134c6 | ||
|
|
b7a0b9b66e | ||
|
|
584128128a | ||
|
|
0d6b629826 | ||
|
|
d2ba0ab46c | ||
|
|
a16a7b12d8 | ||
|
|
33c5787776 | ||
|
|
090f16bfb8 | ||
|
|
f0b2012622 | ||
|
|
b8881dbf85 | ||
|
|
c646b21d07 | ||
|
|
1503fbeec1 | ||
|
|
8967e175a6 | ||
|
|
9be76a9fae | ||
|
|
4c5f163b45 | ||
|
|
7263c98d5e | ||
|
|
71e0e0573f | ||
|
|
1903ac9f96 | ||
|
|
13ce62c844 | ||
|
|
9d9865cc97 | ||
|
|
4557e3bbb5 | ||
|
|
872f508a86 | ||
|
|
ed01da8d36 | ||
|
|
a6c9eb4e7a | ||
|
|
fd837874e2 | ||
|
|
f092b7ba68 | ||
|
|
2ed2eda500 | ||
|
|
5da08d13ed | ||
|
|
e76cacfd4a | ||
|
|
c757b1dae8 | ||
|
|
759254a24e | ||
|
|
acdeff88bf | ||
|
|
3d313d7204 | ||
|
|
b072f1374e | ||
|
|
99fd6e2206 | ||
|
|
0bfca4c67d | ||
|
|
b215d91f24 | ||
|
|
cc99c24c81 | ||
|
|
87fea25596 | ||
|
|
12a200baf9 | ||
|
|
deb213f8f5 | ||
|
|
bbc8f977a7 | ||
|
|
31478f8ab4 | ||
|
|
41796cdc79 | ||
|
|
4cc05feecf | ||
|
|
66931f74fa | ||
|
|
ac9c62615d | ||
|
|
a4b4c58897 | ||
|
|
04f79708da | ||
|
|
1e5c31f9b2 | ||
|
|
e47ca3f362 | ||
|
|
d6ef6141e4 | ||
|
|
05919690d8 | ||
|
|
a70b2a30ba | ||
|
|
4ab75e023d | ||
|
|
42f5468553 | ||
|
|
16e846b3a4 | ||
|
|
bc944a0747 | ||
|
|
7546e62417 | ||
|
|
80663a78b8 | ||
|
|
4e937d11f7 | ||
|
|
4eb08dbfb6 | ||
|
|
6a379c8073 | ||
|
|
8ce785fe26 | ||
|
|
c61021bc9a | ||
|
|
cbf3054cd4 | ||
|
|
a6b395bbf5 | ||
|
|
b246e276a0 | ||
|
|
3a1cf8029f | ||
|
|
10036dc092 | ||
|
|
78c7e66fd7 | ||
|
|
d46a6ad658 | ||
|
|
69be46f0da | ||
|
|
479fac5f4e | ||
|
|
a210707f02 | ||
|
|
12fca1825e | ||
|
|
91658f400e | ||
|
|
b8b535dbbf | ||
|
|
937ae462df | ||
|
|
dff87f5bd4 | ||
|
|
7cff17bac7 | ||
|
|
89a074f9dd | ||
|
|
7443abf25c | ||
|
|
d4ae02ab53 | ||
|
|
91f3015a70 | ||
|
|
b1057fa817 | ||
|
|
e5a36eb6e8 | ||
|
|
d37afca1f0 | ||
|
|
bcd60ad4ab | ||
|
|
447a589cc2 | ||
|
|
d05a619bb1 | ||
|
|
f8c9c73e3a | ||
|
|
fb02f92745 | ||
|
|
9025225b6d | ||
|
|
a5eafdf2f4 | ||
|
|
cde44c1c46 | ||
|
|
fa88e26b10 | ||
|
|
5d820f3395 | ||
|
|
b0f1c5ffa9 | ||
|
|
ead63de184 | ||
|
|
ac3f520670 | ||
|
|
f212e92530 | ||
|
|
5453da1cc8 | ||
|
|
0770636cc7 | ||
|
|
2855c355d1 | ||
|
|
2edda521f7 | ||
|
|
5a8a92f972 | ||
|
|
317f2fe51e | ||
|
|
17d662aca9 | ||
|
|
1c32723e74 | ||
|
|
c7ab111a94 | ||
|
|
7ab7391cca | ||
|
|
1df3d343c9 | ||
|
|
6aa9b8e86d | ||
|
|
b1bba53c53 | ||
|
|
a15dfe46d0 | ||
|
|
3ac6859b11 | ||
|
|
1e151ce829 | ||
|
|
50d20f34e9 | ||
|
|
573f2a26d9 | ||
|
|
6ed2bf1e55 | ||
|
|
0592d1c29d | ||
|
|
a936b602af | ||
|
|
4085577688 | ||
|
|
c12dfad592 | ||
|
|
2b11745597 | ||
|
|
8349b5ba61 | ||
|
|
a73347b89e | ||
|
|
fd8ead1eae | ||
|
|
13cc525d87 | ||
|
|
25a6bcef21 | ||
|
|
03bc8c13e1 | ||
|
|
71a3df9859 |
@@ -1,38 +0,0 @@
|
||||
name: Publish to Comfy registry
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
publish-node:
|
||||
name: Publish Custom Node to registry
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.repository_owner == 'huchukato' }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
- name: Extract changelog
|
||||
run: |
|
||||
VERSION=$(sed -n 's/^version = "\([^"]*\)"/\1/p' pyproject.toml | head -1)
|
||||
ESCAPED=$(printf '%s' "$VERSION" | sed 's/\./\\./g')
|
||||
for f in update.md UPDATE.md CHANGELOG.md; do
|
||||
[ -f "$f" ] || continue
|
||||
awk -v ver="$ESCAPED" '
|
||||
/^## / {
|
||||
if (found) exit
|
||||
if ($0 ~ ("(^|[^0-9.])" ver "([^0-9.]|$)")) { found = 1 }
|
||||
next
|
||||
}
|
||||
found { print }
|
||||
' "$f" > changelog.txt
|
||||
[ -s changelog.txt ] && break
|
||||
done
|
||||
[ -s changelog.txt ] || git log -1 --pretty=%B > changelog.txt
|
||||
head -c 4000 changelog.txt > changelog.tmp && mv changelog.tmp changelog.txt
|
||||
- name: Publish Custom Node
|
||||
run: |
|
||||
pip install comfy-cli
|
||||
comfy --skip-prompt --no-enable-telemetry env
|
||||
comfy node publish --token "${{ secrets.REGISTRY_ACCESS_TOKEN }}" --changelog-file changelog.txt
|
||||
+11
-20
@@ -1,4 +1,4 @@
|
||||
# ComfyUI-QwenVL-Mod Git Ignore File
|
||||
.DS_S# ComfyUI-QwenVL-Mod Git Ignore File
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
@@ -35,10 +35,6 @@ ENV/
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Kilo Code agent worktrees/state
|
||||
.kilo/
|
||||
.kilocode/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
@@ -57,6 +53,7 @@ models/
|
||||
input/
|
||||
output/
|
||||
temp/
|
||||
*.png
|
||||
*.jpg
|
||||
*.jpeg
|
||||
*.gif
|
||||
@@ -82,19 +79,13 @@ npm-debug.log*
|
||||
prompt_cache.json
|
||||
|
||||
# Personal build scripts and backups (don't commit to shared repo)
|
||||
runpod/
|
||||
example_workflows/
|
||||
vastai/build-and-push-*.sh
|
||||
build-and-push-4090.sh
|
||||
build-and-push-5090.sh
|
||||
vastai/
|
||||
|
||||
# VastAI README files (keep local, not for shared repo)
|
||||
vastai/READMECivitai.md
|
||||
vastai/READMEVast4090.md
|
||||
vastai/READMEVast5090.md
|
||||
|
||||
# Local caches / Devin local config
|
||||
.ruff_cache/
|
||||
.devin/
|
||||
node.zip
|
||||
|
||||
# Garage-synced preset mirror
|
||||
presets_remote/
|
||||
# API consulting tools (keep local for client work)
|
||||
API_GUIDE.md
|
||||
debug_api.py
|
||||
api_client_examples/
|
||||
consulting/
|
||||
client_work/
|
||||
|
||||
@@ -155,12 +155,3 @@ def _strip_planning_paragraphs(text: str) -> str:
|
||||
if not kept:
|
||||
return text.strip()
|
||||
return "\n\n".join(kept).strip()
|
||||
|
||||
|
||||
def prompt_output_guard() -> str:
|
||||
return (
|
||||
"Output rules that override all style preferences: return only the final usable prompt text. "
|
||||
"Do not include analysis, planning, bullet points, headings, markdown, JSON, explanations, alternatives, self-corrections, or notes. "
|
||||
"Do not write phrases such as Final Plan, Final Check, Wait, Okay, First, Next, Then, I will, or I need. "
|
||||
"Start directly with the required prompt text."
|
||||
)
|
||||
|
||||
+213
-669
File diff suppressed because it is too large
Load Diff
+120
-466
@@ -1,5 +1,5 @@
|
||||
# ComfyUI-QwenVL (GGUF)
|
||||
# GGUF nodes powered by llama.cpp for Qwen-VL models, including the latest Qwen3-VL.
|
||||
# GGUF nodes powered by llama.cpp for Qwen-VL models, including Qwen3-VL and Qwen2.5-VL.
|
||||
# Provides vision-capable GGUF inference and prompt execution.
|
||||
#
|
||||
# Models are loaded via llama-cpp-python and configured through gguf_models.json.
|
||||
@@ -16,7 +16,6 @@ import io
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
@@ -25,90 +24,19 @@ from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from huggingface_hub import hf_hub_download, snapshot_download
|
||||
try:
|
||||
from llama_cpp import Llama
|
||||
except ImportError:
|
||||
Llama = None
|
||||
print("[QwenVL] llama-cpp-python is not installed — GGUF nodes register but need a "
|
||||
"vision-capable build to run. Local install: `pip install llama-cpp-python`; "
|
||||
"for CUDA/vision wheels see github.com/1038lab/ComfyUI-QwenVL/blob/main/docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md")
|
||||
from llama_cpp import Llama
|
||||
from PIL import Image
|
||||
|
||||
# Import cache functions from main module
|
||||
sys.path.append(str(Path(__file__).parent))
|
||||
from AILab_QwenVL import PROMPT_CACHE, ensure_cuda_vram_headroom, get_cache_key, get_alternative_cache_key, get_image_hash, get_video_hash, save_prompt_cache, CAMERA_TAG_OPTIONS, CAMERA_TAG_TOOLTIP, CAMERA_TAG_DESCRIPTIONS, add_danbooru_guidance, camera_directive_location
|
||||
from qwenvl_presets import (
|
||||
VL_PRESET_NAMES, VL_PROMPTS, VL_DURATIONS,
|
||||
DURATION_OPTIONS, DEFAULT_DURATION, resolve_vl_preset,
|
||||
)
|
||||
from AILab_QwenVL import PROMPT_CACHE, get_cache_key, get_alternative_cache_key, get_image_hash, get_video_hash, save_prompt_cache
|
||||
|
||||
import folder_paths
|
||||
from AILab_OutputCleaner import OutputCleanConfig, clean_model_output, prompt_output_guard
|
||||
from chat_service import normalize_minimax_output
|
||||
from wildcard_util import expand_wildcard_tokens
|
||||
from AILab_OutputCleaner import OutputCleanConfig, clean_model_output
|
||||
|
||||
# Simple global variable to store last generated prompt
|
||||
LAST_SAVED_PROMPT = None
|
||||
|
||||
|
||||
def read_gguf_architecture(filepath: Path) -> str | None:
|
||||
"""Read general.architecture from a GGUF file header without loading the model.
|
||||
|
||||
Returns the architecture string (e.g. 'qwen3', 'qwen2vl', 'llama') or None on failure.
|
||||
"""
|
||||
# GGUF value type enum
|
||||
_VTYPE_SIZE = {
|
||||
0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8,
|
||||
}
|
||||
_VTYPE_STRING = 8
|
||||
_VTYPE_ARRAY = 9
|
||||
|
||||
def _read_string(f):
|
||||
length = struct.unpack("<Q", f.read(8))[0]
|
||||
return f.read(length).decode("utf-8", errors="replace")
|
||||
|
||||
def _skip_value(f, vtype):
|
||||
if vtype in _VTYPE_SIZE:
|
||||
f.seek(_VTYPE_SIZE[vtype], 1)
|
||||
elif vtype == _VTYPE_STRING:
|
||||
length = struct.unpack("<Q", f.read(8))[0]
|
||||
f.seek(length, 1)
|
||||
elif vtype == _VTYPE_ARRAY:
|
||||
arr_type = struct.unpack("<I", f.read(4))[0]
|
||||
arr_len = struct.unpack("<Q", f.read(8))[0]
|
||||
for _ in range(arr_len):
|
||||
_skip_value(f, arr_type)
|
||||
else:
|
||||
return False # unknown type, bail
|
||||
return True
|
||||
|
||||
try:
|
||||
with open(filepath, "rb") as f:
|
||||
magic = f.read(4)
|
||||
if magic != b"GGUF":
|
||||
return None
|
||||
version = struct.unpack("<I", f.read(4))[0]
|
||||
if version not in (2, 3):
|
||||
return None
|
||||
_tensor_count = struct.unpack("<Q", f.read(8))[0]
|
||||
kv_count = struct.unpack("<Q", f.read(8))[0]
|
||||
|
||||
for _ in range(kv_count):
|
||||
key = _read_string(f)
|
||||
vtype = struct.unpack("<I", f.read(4))[0]
|
||||
if key == "general.architecture":
|
||||
if vtype == _VTYPE_STRING:
|
||||
return _read_string(f)
|
||||
else:
|
||||
return None
|
||||
# Skip this value and continue searching
|
||||
if not _skip_value(f, vtype):
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
NODE_DIR = Path(__file__).parent
|
||||
CONFIG_PATH = NODE_DIR / "hf_models.json"
|
||||
SYSTEM_PROMPTS_PATH = NODE_DIR / "AILab_System_Prompts.json"
|
||||
@@ -116,14 +44,14 @@ GGUF_CONFIG_PATH = NODE_DIR / "gguf_models.json"
|
||||
|
||||
|
||||
def _load_prompt_config():
|
||||
preset_prompts = list(VL_PRESET_NAMES) or ["IMG › Detailed"]
|
||||
system_prompts: dict[str, str] = dict(VL_PROMPTS)
|
||||
preset_prompts = ["🖼️ Detailed Description"]
|
||||
system_prompts: dict[str, str] = {}
|
||||
|
||||
try:
|
||||
with open(CONFIG_PATH, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh) or {}
|
||||
preset_prompts = data.get("_preset_prompts") or preset_prompts
|
||||
system_prompts.update(data.get("_system_prompts") or {})
|
||||
system_prompts = data.get("_system_prompts") or system_prompts
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL] Config load failed: {exc}")
|
||||
|
||||
@@ -133,7 +61,7 @@ def _load_prompt_config():
|
||||
qwenvl_prompts = data.get("qwenvl") or {}
|
||||
preset_override = data.get("_preset_prompts") or []
|
||||
if isinstance(qwenvl_prompts, dict) and qwenvl_prompts:
|
||||
system_prompts.update(qwenvl_prompts)
|
||||
system_prompts = qwenvl_prompts
|
||||
if isinstance(preset_override, list) and preset_override:
|
||||
preset_prompts = preset_override
|
||||
except FileNotFoundError:
|
||||
@@ -178,49 +106,6 @@ def _safe_dirname(value: str) -> str:
|
||||
return "".join(ch for ch in value if ch.isalnum() or ch in "._- ").strip() or "unknown"
|
||||
|
||||
|
||||
def find_in_llm_paths(filename: str, author: str = "", repo_dirname: str = "") -> Path | None:
|
||||
"""Search ComfyUI's registered LLM folder paths for an existing GGUF file.
|
||||
|
||||
Honours alternate paths declared via extra_model_paths.yaml so catalog entries
|
||||
whose default location is empty can still be located on a secondary drive.
|
||||
"""
|
||||
bare = Path(filename).name
|
||||
try:
|
||||
if "LLM" not in folder_paths.folder_names_and_paths:
|
||||
return None
|
||||
llm_roots = folder_paths.get_folder_paths("LLM")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
author_dir = _safe_dirname(author) if author else ""
|
||||
repo_dir = _safe_dirname(repo_dirname) if repo_dirname else ""
|
||||
|
||||
for llm_root in llm_roots:
|
||||
root = Path(llm_root)
|
||||
candidates: list[Path] = []
|
||||
if author_dir and author_dir != "unknown" and repo_dir:
|
||||
candidates.append(root / "GGUF" / author_dir / repo_dir / bare)
|
||||
candidates.append(root / author_dir / repo_dir / bare)
|
||||
if repo_dir:
|
||||
candidates.append(root / "GGUF" / repo_dir / bare)
|
||||
candidates.append(root / repo_dir / bare)
|
||||
candidates.append(root / "GGUF" / bare)
|
||||
candidates.append(root / bare)
|
||||
for c in candidates:
|
||||
if c.exists():
|
||||
return c
|
||||
|
||||
for llm_root in llm_roots:
|
||||
try:
|
||||
matches = list(Path(llm_root).rglob(bare))
|
||||
if matches:
|
||||
return matches[0]
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _model_name_to_filename_candidates(model_name: str) -> set[str]:
|
||||
raw = (model_name or "").strip()
|
||||
if not raw:
|
||||
@@ -235,69 +120,15 @@ def _model_name_to_filename_candidates(model_name: str) -> set[str]:
|
||||
return candidates
|
||||
|
||||
|
||||
def _scan_local_gguf_models(base_dir: Path, existing_filenames: set[str]) -> dict[str, dict]:
|
||||
"""Scan the GGUF base directory for locally available .gguf files not already in the JSON catalog."""
|
||||
local_models: dict[str, dict] = {}
|
||||
if not base_dir.exists() or not base_dir.is_dir():
|
||||
return local_models
|
||||
|
||||
# Walk all subdirectories and collect .gguf files grouped by parent directory
|
||||
dirs_with_gguf: dict[Path, list[Path]] = {}
|
||||
try:
|
||||
for gguf_file in base_dir.rglob("*.gguf"):
|
||||
if gguf_file.is_file():
|
||||
parent = gguf_file.parent
|
||||
dirs_with_gguf.setdefault(parent, []).append(gguf_file)
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
for dir_path, gguf_files in dirs_with_gguf.items():
|
||||
# Separate mmproj files from model files
|
||||
mmproj_files = [f for f in gguf_files if "mmproj" in f.name.lower()]
|
||||
model_files = [f for f in gguf_files if "mmproj" not in f.name.lower()]
|
||||
|
||||
# Vision-only node: skip directories without a paired mmproj
|
||||
if not mmproj_files:
|
||||
continue
|
||||
|
||||
mmproj_path = mmproj_files[0]
|
||||
|
||||
for model_file in model_files:
|
||||
# Skip if this filename is already in the JSON catalog
|
||||
if model_file.name in existing_filenames:
|
||||
continue
|
||||
|
||||
display = f"[local] {model_file.name}"
|
||||
local_models[display] = {
|
||||
"filename": str(model_file),
|
||||
"mmproj_filename": str(mmproj_path),
|
||||
"is_local": True,
|
||||
"repo_id": None,
|
||||
"alt_repo_ids": [],
|
||||
"author": None,
|
||||
"repo_dirname": dir_path.name,
|
||||
"context_length": 32768,
|
||||
"image_max_tokens": 4096,
|
||||
"n_batch": 512,
|
||||
"gpu_layers": -1,
|
||||
"top_k": 20,
|
||||
"pool_size": 4194304,
|
||||
}
|
||||
|
||||
if local_models:
|
||||
print(f"[QwenVL] Discovered {len(local_models)} local GGUF model(s) on disk")
|
||||
|
||||
return local_models
|
||||
|
||||
|
||||
def _load_gguf_vl_catalog():
|
||||
data = {}
|
||||
if GGUF_CONFIG_PATH.exists():
|
||||
try:
|
||||
with open(GGUF_CONFIG_PATH, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh) or {}
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL] gguf_models.json load failed: {exc}")
|
||||
if not GGUF_CONFIG_PATH.exists():
|
||||
return {"base_dir": "LLM/GGUF", "models": {}}
|
||||
try:
|
||||
with open(GGUF_CONFIG_PATH, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh) or {}
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL] gguf_models.json load failed: {exc}")
|
||||
return {"base_dir": "LLM/GGUF", "models": {}}
|
||||
|
||||
base_dir = data.get("base_dir") or "LLM/GGUF"
|
||||
|
||||
@@ -317,10 +148,6 @@ def _load_gguf_vl_catalog():
|
||||
mmproj_file = repo.get("mmproj_file")
|
||||
model_files = repo.get("model_files") or []
|
||||
|
||||
# Vision-only node: skip catalog repos without an mmproj projector
|
||||
if not mmproj_file:
|
||||
continue
|
||||
|
||||
for model_file in model_files:
|
||||
display = Path(model_file).name
|
||||
if display in seen_display_names:
|
||||
@@ -341,28 +168,6 @@ def _load_gguf_vl_catalog():
|
||||
if isinstance(entry, dict):
|
||||
flattened[name] = entry
|
||||
|
||||
# Scan filesystem for locally available models not in JSON config
|
||||
# Collect all directories to scan: the configured base_dir + any extra LLM paths from ComfyUI
|
||||
existing_filenames = {Path(e.get("filename", "")).name for e in flattened.values() if e.get("filename")}
|
||||
scan_dirs: list[Path] = [_resolve_base_dir(base_dir)]
|
||||
try:
|
||||
if "LLM" in folder_paths.folder_names_and_paths:
|
||||
for llm_path in folder_paths.get_folder_paths("LLM"):
|
||||
gguf_dir = Path(llm_path) / "GGUF"
|
||||
if gguf_dir not in scan_dirs:
|
||||
scan_dirs.append(gguf_dir)
|
||||
# Also scan the LLM path itself (users may put GGUFs directly there)
|
||||
llm_p = Path(llm_path)
|
||||
if llm_p not in scan_dirs:
|
||||
scan_dirs.append(llm_p)
|
||||
except Exception:
|
||||
pass
|
||||
for scan_dir in scan_dirs:
|
||||
local_models = _scan_local_gguf_models(scan_dir, existing_filenames)
|
||||
flattened.update(local_models)
|
||||
# Update existing_filenames so we don't add duplicates across directories
|
||||
existing_filenames.update(Path(e.get("filename", "")).name for e in local_models.values() if e.get("filename"))
|
||||
|
||||
return {"base_dir": base_dir, "models": flattened}
|
||||
|
||||
|
||||
@@ -392,12 +197,7 @@ def _tensor_to_base64_png(tensor) -> str | None:
|
||||
if tensor.ndim == 4:
|
||||
tensor = tensor[0]
|
||||
array = (tensor * 255).clamp(0, 255).to(torch.uint8).cpu().numpy()
|
||||
if array.ndim == 2:
|
||||
pil_img = Image.fromarray(array, mode="L")
|
||||
elif array.shape[-1] == 4:
|
||||
pil_img = Image.fromarray(array, mode="RGBA")
|
||||
else:
|
||||
pil_img = Image.fromarray(array[..., :3], mode="RGB")
|
||||
pil_img = Image.fromarray(array, mode="RGB")
|
||||
buf = io.BytesIO()
|
||||
pil_img.save(buf, format="PNG")
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
@@ -505,7 +305,7 @@ def _resolve_model_entry(model_name: str) -> GGUFVLResolved:
|
||||
image_max_tokens=_int("image_max_tokens", 4096),
|
||||
n_batch=_int("n_batch", 512),
|
||||
gpu_layers=_int("gpu_layers", -1),
|
||||
top_k=_int("top_k", 20),
|
||||
top_k=_int("top_k", 0),
|
||||
pool_size=_int("pool_size", 4194304),
|
||||
)
|
||||
|
||||
@@ -518,7 +318,7 @@ class QwenVLGGUFBase:
|
||||
|
||||
def clear(self):
|
||||
print(f"[QwenVL GGUF DEBUG] Starting VRAM cleanup...")
|
||||
|
||||
|
||||
# Force cleanup of chat handler first
|
||||
if self.chat_handler is not None:
|
||||
try:
|
||||
@@ -531,7 +331,7 @@ class QwenVLGGUFBase:
|
||||
print(f"[QwenVL GGUF DEBUG] Error closing chat_handler: {e}")
|
||||
finally:
|
||||
self.chat_handler = None
|
||||
|
||||
|
||||
# Force cleanup of LLM model
|
||||
if self.llm is not None:
|
||||
try:
|
||||
@@ -546,25 +346,21 @@ class QwenVLGGUFBase:
|
||||
print(f"[QwenVL GGUF DEBUG] Error closing LLM: {e}")
|
||||
finally:
|
||||
self.llm = None
|
||||
|
||||
|
||||
# Clear signature
|
||||
self.current_signature = None
|
||||
|
||||
|
||||
# Aggressive garbage collection
|
||||
gc.collect()
|
||||
|
||||
|
||||
# Force CUDA cache cleanup multiple times
|
||||
if torch.cuda.is_available():
|
||||
print(f"[QwenVL GGUF DEBUG] Clearing CUDA cache...")
|
||||
torch.cuda.empty_cache()
|
||||
try:
|
||||
torch.cuda.ipc_collect()
|
||||
except Exception:
|
||||
pass
|
||||
torch.cuda.synchronize()
|
||||
# Additional cleanup
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
|
||||
print(f"[QwenVL GGUF DEBUG] VRAM cleanup completed")
|
||||
|
||||
def _load_backend(self):
|
||||
@@ -572,7 +368,7 @@ class QwenVLGGUFBase:
|
||||
from llama_cpp import Llama # noqa: F401
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"[QwenVL] llama_cpp is not available. Install the GGUF vision dependency first. See github.com/abetlen/llama-cpp-python"
|
||||
"[QwenVL] llama_cpp is not available. Install the GGUF vision dependency first. See docs/GGUF_MANUAL_INSTALL.md"
|
||||
) from exc
|
||||
|
||||
def _load_model(
|
||||
@@ -589,49 +385,29 @@ class QwenVLGGUFBase:
|
||||
self._load_backend()
|
||||
|
||||
resolved = _resolve_model_entry(model_name)
|
||||
base_dir = _resolve_base_dir(GGUF_VL_CATALOG.get("base_dir") or "llm/GGUF")
|
||||
|
||||
# Local models store absolute paths — use them directly, skip download logic
|
||||
if Path(resolved.model_filename).is_absolute():
|
||||
model_path = Path(resolved.model_filename)
|
||||
mmproj_path = Path(resolved.mmproj_filename) if resolved.mmproj_filename else None
|
||||
if not model_path.exists():
|
||||
raise FileNotFoundError(f"[QwenVL] Local GGUF model not found: {model_path}")
|
||||
if mmproj_path is not None and not mmproj_path.exists():
|
||||
raise FileNotFoundError(f"[QwenVL] Local mmproj not found: {mmproj_path}")
|
||||
else:
|
||||
base_dir = _resolve_base_dir(GGUF_VL_CATALOG.get("base_dir") or "llm/GGUF")
|
||||
author_dir = _safe_dirname(resolved.author or "")
|
||||
repo_dir = _safe_dirname(resolved.repo_dirname)
|
||||
target_dir = base_dir / author_dir / repo_dir
|
||||
|
||||
author_dir = _safe_dirname(resolved.author or "")
|
||||
repo_dir = _safe_dirname(resolved.repo_dirname)
|
||||
target_dir = base_dir / author_dir / repo_dir
|
||||
model_path = target_dir / Path(resolved.model_filename).name
|
||||
mmproj_path = target_dir / Path(resolved.mmproj_filename).name if resolved.mmproj_filename else None
|
||||
|
||||
model_path = target_dir / Path(resolved.model_filename).name
|
||||
mmproj_path = target_dir / Path(resolved.mmproj_filename).name if resolved.mmproj_filename else None
|
||||
repo_ids: list[str] = []
|
||||
if resolved.repo_id:
|
||||
repo_ids.append(resolved.repo_id)
|
||||
repo_ids.extend(resolved.alt_repo_ids)
|
||||
|
||||
repo_ids: list[str] = []
|
||||
if resolved.repo_id:
|
||||
repo_ids.append(resolved.repo_id)
|
||||
repo_ids.extend(resolved.alt_repo_ids)
|
||||
if not model_path.exists():
|
||||
if not repo_ids:
|
||||
raise FileNotFoundError(f"[QwenVL] GGUF model not found locally and no repo_id provided: {model_path}")
|
||||
_download_single_file(repo_ids, resolved.model_filename, model_path)
|
||||
|
||||
if not model_path.exists():
|
||||
found = find_in_llm_paths(resolved.model_filename, resolved.author or "", resolved.repo_dirname or "")
|
||||
if found is not None:
|
||||
print(f"[QwenVL] Using model from alternate LLM path: {found}")
|
||||
model_path = found
|
||||
else:
|
||||
if not repo_ids:
|
||||
raise FileNotFoundError(f"[QwenVL] GGUF model not found locally and no repo_id provided: {model_path}")
|
||||
_download_single_file(repo_ids, resolved.model_filename, model_path)
|
||||
|
||||
if mmproj_path is not None and not mmproj_path.exists():
|
||||
found_mm = find_in_llm_paths(resolved.mmproj_filename, resolved.author or "", resolved.repo_dirname or "")
|
||||
if found_mm is not None:
|
||||
print(f"[QwenVL] Using mmproj from alternate LLM path: {found_mm}")
|
||||
mmproj_path = found_mm
|
||||
else:
|
||||
if not repo_ids:
|
||||
raise FileNotFoundError(f"[QwenVL] mmproj not found locally and no repo_id provided: {mmproj_path}")
|
||||
_download_single_file(repo_ids, resolved.mmproj_filename, mmproj_path)
|
||||
if mmproj_path is not None and not mmproj_path.exists():
|
||||
if not repo_ids:
|
||||
raise FileNotFoundError(f"[QwenVL] mmproj not found locally and no repo_id provided: {mmproj_path}")
|
||||
_download_single_file(repo_ids, resolved.mmproj_filename, mmproj_path)
|
||||
|
||||
device_kind = _pick_device(device)
|
||||
|
||||
@@ -661,13 +437,12 @@ class QwenVLGGUFBase:
|
||||
pool_size_val,
|
||||
)
|
||||
if self.llm is not None and self.current_signature == signature:
|
||||
ensure_cuda_vram_headroom("QwenVL GGUF", min_free_gb=1.0, min_free_ratio=0.08)
|
||||
return
|
||||
|
||||
# Force aggressive cleanup before loading new model (especially for same model conflicts)
|
||||
print(f"[QwenVL GGUF DEBUG] Forcing cleanup before model loading...")
|
||||
self.clear()
|
||||
|
||||
|
||||
# Additional wait for CUDA cleanup
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.synchronize()
|
||||
@@ -689,7 +464,7 @@ class QwenVLGGUFBase:
|
||||
handler_cls = Qwen25VLChatHandler
|
||||
except ImportError:
|
||||
raise RuntimeError(
|
||||
"[QwenVL] Missing Qwen VL chat handler in llama_cpp. Install the correct fork/wheel. See github.com/abetlen/llama-cpp-python"
|
||||
"[QwenVL] Missing Qwen VL chat handler in llama_cpp. Install the correct fork/wheel. See docs/GGUF_MANUAL_INSTALL.md"
|
||||
)
|
||||
|
||||
mmproj_kwargs = {
|
||||
@@ -716,27 +491,12 @@ class QwenVLGGUFBase:
|
||||
"pool_size": pool_size_val,
|
||||
"top_k": top_k_val,
|
||||
}
|
||||
|
||||
# Detect Qwen3.x family from GGUF metadata instead of relying on model name
|
||||
arch = read_gguf_architecture(model_path)
|
||||
self.is_qwen35 = bool(
|
||||
(arch and "qwen3" in arch)
|
||||
or "qwen3" in model_name.lower()
|
||||
)
|
||||
if self.is_qwen35:
|
||||
print(f"[QwenVL] Qwen3 family detected (arch={arch}): Will disable thinking via /no_think + chat_template_kwargs/reasoning.")
|
||||
|
||||
if has_mmproj and self.chat_handler is not None:
|
||||
llm_kwargs["chat_handler"] = self.chat_handler
|
||||
llm_kwargs["image_min_tokens"] = 1024
|
||||
llm_kwargs["image_max_tokens"] = img_max
|
||||
|
||||
print(f"[QwenVL] Loading GGUF: {model_path.name} (device={device_kind}, gpu_layers={n_gpu_layers}, ctx={n_ctx})")
|
||||
if Llama is None:
|
||||
raise RuntimeError(
|
||||
"llama-cpp-python is not installed — GGUF vision cannot run. "
|
||||
"Install a vision-capable build: local `pip install llama-cpp-python`, "
|
||||
"CUDA/vision wheels: github.com/1038lab/ComfyUI-QwenVL/blob/main/docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md")
|
||||
llm_kwargs_filtered = _filter_kwargs_for_callable(getattr(Llama, "__init__", Llama), llm_kwargs)
|
||||
if has_mmproj and self.chat_handler is not None and "chat_handler" not in llm_kwargs_filtered:
|
||||
print(
|
||||
@@ -745,7 +505,6 @@ class QwenVLGGUFBase:
|
||||
)
|
||||
if device_kind == "cuda" and n_gpu_layers == 0:
|
||||
print("[QwenVL] Warning: device=cuda selected but n_gpu_layers=0; model will run on CPU.")
|
||||
|
||||
self.llm = Llama(**llm_kwargs_filtered)
|
||||
self.current_signature = signature
|
||||
|
||||
@@ -759,17 +518,7 @@ class QwenVLGGUFBase:
|
||||
top_p: float,
|
||||
repetition_penalty: float,
|
||||
seed: int,
|
||||
model_name: str = "",
|
||||
enable_thinking: bool = False,
|
||||
response_format: dict | None = None,
|
||||
) -> str:
|
||||
ensure_cuda_vram_headroom("QwenVL GGUF", min_free_gb=1.0, min_free_ratio=0.08)
|
||||
if self.llm is not None and hasattr(self.llm, "reset"):
|
||||
try:
|
||||
self.llm.reset()
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL GGUF DEBUG] llama context reset skipped: {exc}")
|
||||
|
||||
if images_b64:
|
||||
content = [{"type": "text", "text": user_prompt}]
|
||||
for img in images_b64:
|
||||
@@ -787,13 +536,6 @@ class QwenVLGGUFBase:
|
||||
]
|
||||
|
||||
start = time.perf_counter()
|
||||
extra_kwargs = {}
|
||||
if getattr(self, "is_qwen35", False):
|
||||
extra_kwargs["chat_template_kwargs"] = {"enable_thinking": bool(enable_thinking)}
|
||||
extra_kwargs["reasoning"] = bool(enable_thinking)
|
||||
if response_format is not None:
|
||||
extra_kwargs["response_format"] = response_format
|
||||
extra_kwargs = _filter_kwargs_for_callable(self.llm.create_chat_completion, extra_kwargs)
|
||||
result = self.llm.create_chat_completion(
|
||||
messages=messages,
|
||||
max_tokens=int(max_tokens),
|
||||
@@ -802,7 +544,6 @@ class QwenVLGGUFBase:
|
||||
repeat_penalty=float(repetition_penalty),
|
||||
seed=int(seed),
|
||||
stop=["<|im_end|>", "<|im_start|>"],
|
||||
**extra_kwargs
|
||||
)
|
||||
elapsed = max(time.perf_counter() - start, 1e-6)
|
||||
|
||||
@@ -827,7 +568,7 @@ class QwenVLGGUFBase:
|
||||
self,
|
||||
model_name: str,
|
||||
preset_prompt: str,
|
||||
prompt: str,
|
||||
custom_prompt: str,
|
||||
image,
|
||||
video,
|
||||
frame_count,
|
||||
@@ -845,25 +586,11 @@ class QwenVLGGUFBase:
|
||||
top_k=None,
|
||||
pool_size=None,
|
||||
keep_last_prompt=False,
|
||||
camera_tag="None",
|
||||
passthrough=False,
|
||||
image2=None,
|
||||
image3=None,
|
||||
duration=DEFAULT_DURATION,
|
||||
):
|
||||
print(f"[QwenVL GGUF DEBUG] Starting run with seed={seed}, keep_last_prompt={keep_last_prompt}")
|
||||
|
||||
|
||||
global LAST_SAVED_PROMPT
|
||||
|
||||
# Expand TagForge __wildcard__ tokens first — also in passthrough mode,
|
||||
# so raw tokens never reach the downstream prompt.
|
||||
prompt = expand_wildcard_tokens(prompt or "")
|
||||
|
||||
# Passthrough mode: skip model loading entirely, return prompt as-is.
|
||||
if passthrough:
|
||||
print(f"[QwenVL GGUF] Passthrough mode ON — skipping model load, returning prompt directly ({len(prompt or '')} chars)")
|
||||
return (prompt or "",)
|
||||
|
||||
|
||||
# Simple keep last prompt logic
|
||||
if keep_last_prompt:
|
||||
print(f"[QwenVL GGUF] Keep last prompt enabled - using last saved prompt")
|
||||
@@ -873,24 +600,17 @@ class QwenVLGGUFBase:
|
||||
else:
|
||||
print(f"[QwenVL GGUF] No previous prompt found, returning empty")
|
||||
return ("",)
|
||||
|
||||
|
||||
# Always generate when keep last prompt is disabled
|
||||
print(f"[QwenVL GGUF] Keep last prompt disabled - generating new prompt")
|
||||
|
||||
# Resolve preset aliases (legacy dropdown names) and the duration
|
||||
# widget to a flat prompt key like "MiniMax › NSFW (10s)".
|
||||
preset_prompt, preset_key = resolve_vl_preset(preset_prompt, duration)
|
||||
prompt_template = SYSTEM_PROMPTS.get(preset_key, preset_key)
|
||||
|
||||
|
||||
prompt_template = SYSTEM_PROMPTS.get(preset_prompt, preset_prompt)
|
||||
|
||||
# Generate cache key with all inputs including seed
|
||||
image_hash = get_image_hash(image)
|
||||
image2_hash = get_image_hash(image2)
|
||||
image3_hash = get_image_hash(image3)
|
||||
video_hash = get_video_hash(video)
|
||||
# Combine image2/image3 and video hashes for backward-compatible cache key
|
||||
combined_hash = f"{image2_hash or ''}/{image3_hash or ''}/{video_hash or ''}" if (image2_hash or image3_hash or video_hash) else None
|
||||
cache_key = get_cache_key(model_name, preset_key, prompt, image_hash, combined_hash, int(seed))
|
||||
|
||||
cache_key = get_cache_key(model_name, preset_prompt, custom_prompt, image_hash, video_hash, int(seed))
|
||||
|
||||
# TEMPORARILY DISABLED CACHE FOR DEBUGGING
|
||||
# Check cache first (only for random mode)
|
||||
# if cache_key in PROMPT_CACHE:
|
||||
@@ -898,62 +618,40 @@ class QwenVLGGUFBase:
|
||||
# if cached_text:
|
||||
# print(f"[QwenVL GGUF] Using cached prompt for seed {seed}: {cache_key[:8]}...")
|
||||
# return cached_text.strip()
|
||||
|
||||
|
||||
print(f"[QwenVL GGUF DEBUG] Cache disabled - proceeding with generation")
|
||||
|
||||
prompt_template = add_danbooru_guidance(prompt_template, preset_prompt)
|
||||
# User content stays separate from the preset template: the template
|
||||
# rides in the system message, the request in the user message.
|
||||
full_prompt = prompt.strip() or "Describe a scene vividly."
|
||||
|
||||
# ── Camera tag injection (same logic as AILab_QwenVL) ─────────────
|
||||
CAMERA_TAGS = list(CAMERA_TAG_DESCRIPTIONS.keys())
|
||||
found_cam_tag = None
|
||||
if camera_tag and camera_tag.strip() and camera_tag.strip().upper() != "NONE":
|
||||
tag_clean = camera_tag.strip().upper().strip("[]")
|
||||
if tag_clean in CAMERA_TAGS:
|
||||
found_cam_tag = tag_clean
|
||||
if not found_cam_tag and prompt and prompt.strip():
|
||||
upper = prompt.upper()
|
||||
for tag in CAMERA_TAGS:
|
||||
if f"[{tag}]" in upper:
|
||||
found_cam_tag = tag
|
||||
break
|
||||
if found_cam_tag:
|
||||
desc = CAMERA_TAG_DESCRIPTIONS.get(found_cam_tag, "")
|
||||
tag_str = f"[{found_cam_tag}]"
|
||||
prefix = f"{tag_str}\n\n"
|
||||
reminder = (
|
||||
f"\n\n═══ FINAL CAMERA DIRECTIVE (HIGHEST PRIORITY) ═══\n"
|
||||
f"Camera: {tag_str} — {desc}\n"
|
||||
f"You MUST use this camera movement and NO other. "
|
||||
f"{camera_directive_location(preset_prompt, full_prompt)}\n"
|
||||
f"IMPORTANT: the camera tag controls ONLY the camera. "
|
||||
f"The subject MUST still have natural, lively action and "
|
||||
f"movement throughout the clip — breathing, gestures, "
|
||||
f"expression changes, body motion, interaction with the "
|
||||
f"environment. Do NOT freeze the subject just because the "
|
||||
f"camera is moving. The subject is alive and active while "
|
||||
f"the camera performs {tag_str}.\n"
|
||||
f"═══ END DIRECTIVE ═══"
|
||||
)
|
||||
full_prompt = prefix + full_prompt + reminder
|
||||
|
||||
print(f"[QwenVL GGUF DEBUG] Final prompt: {full_prompt[:100]}...")
|
||||
|
||||
if custom_prompt and custom_prompt.strip():
|
||||
# Combine user input with template - custom prompt first for priority
|
||||
prompt = f"{custom_prompt.strip()}\n\n{prompt_template}"
|
||||
else:
|
||||
prompt = prompt_template
|
||||
|
||||
print(f"[QwenVL GGUF DEBUG] Final prompt: {prompt[:100]}...")
|
||||
|
||||
images_b64: list[str] = []
|
||||
if image is not None:
|
||||
print(f"[QwenVL GGUF DEBUG] Processing image...")
|
||||
print(f"[QwenVL GGUF DEBUG] Image shape before processing: {image.shape}")
|
||||
|
||||
|
||||
# Handle batch images from T2V
|
||||
if len(image.shape) == 4: # [batch, height, width, channels]
|
||||
print(f"[QwenVL GGUF DEBUG] Detected batch image with shape: {image.shape}")
|
||||
frame_img = image[0]
|
||||
# Take first frame from batch or process all frames
|
||||
if image.shape[0] > 1:
|
||||
print(f"[QwenVL GGUF DEBUG] IMAGE input contains {image.shape[0]} items; using the first item only. Use the video input for multi-frame analysis.")
|
||||
print(f"[QwenVL GGUF DEBUG] Single image from batch, shape: {frame_img.shape}")
|
||||
img = _tensor_to_base64_png(frame_img)
|
||||
if img:
|
||||
print(f"[QwenVL GGUF DEBUG] Processing {image.shape[0]} frames from batch")
|
||||
for i in range(image.shape[0]):
|
||||
frame_img = image[i] # Take each frame from batch
|
||||
print(f"[QwenVL GGUF DEBUG] Processing batch frame {i} with shape: {frame_img.shape}")
|
||||
img = _tensor_to_base64_png(frame_img)
|
||||
if img:
|
||||
images_b64.append(img)
|
||||
else:
|
||||
# Single frame in batch format
|
||||
frame_img = image[0]
|
||||
print(f"[QwenVL GGUF DEBUG] Single frame in batch, shape: {frame_img.shape}")
|
||||
img = _tensor_to_base64_png(frame_img)
|
||||
if img:
|
||||
images_b64.append(img)
|
||||
else:
|
||||
# Regular single image [height, width, channels]
|
||||
@@ -961,16 +659,6 @@ class QwenVLGGUFBase:
|
||||
img = _tensor_to_base64_png(image)
|
||||
if img:
|
||||
images_b64.append(img)
|
||||
if image2 is not None:
|
||||
frame_img = image2[0] if len(image2.shape) == 4 else image2
|
||||
img = _tensor_to_base64_png(frame_img)
|
||||
if img:
|
||||
images_b64.append(img)
|
||||
if image3 is not None:
|
||||
frame_img = image3[0] if len(image3.shape) == 4 else image3
|
||||
img = _tensor_to_base64_png(frame_img)
|
||||
if img:
|
||||
images_b64.append(img)
|
||||
if video is not None:
|
||||
for frame in _sample_video_frames(video, int(frame_count)):
|
||||
img = _tensor_to_base64_png(frame)
|
||||
@@ -978,14 +666,14 @@ class QwenVLGGUFBase:
|
||||
images_b64.append(img)
|
||||
|
||||
print(f"[QwenVL GGUF DEBUG] Images processed: {len(images_b64)} images/videos")
|
||||
|
||||
|
||||
# Debug video/image info
|
||||
if video is not None:
|
||||
print(f"[QwenVL GGUF DEBUG] Video shape: {video.shape}")
|
||||
print(f"[QwenVL GGUF DEBUG] Frame count requested: {frame_count}")
|
||||
if image is not None:
|
||||
print(f"[QwenVL GGUF DEBUG] Image shape: {image.shape}")
|
||||
|
||||
|
||||
# Debug VRAM before model loading
|
||||
if torch.cuda.is_available():
|
||||
allocated = torch.cuda.memory_allocated()
|
||||
@@ -1008,30 +696,23 @@ class QwenVLGGUFBase:
|
||||
if images_b64 and self.chat_handler is None:
|
||||
print("[QwenVL] Warning: images provided but this model entry has no mmproj_file; images will be ignored")
|
||||
print(f"[QwenVL GGUF DEBUG] Starting generation...")
|
||||
# Prepend /no_think for Qwen3.5 models (enable_thinking is deprecated in recent llama.cpp)
|
||||
effective_prompt = ("/no_think\n" + full_prompt) if getattr(self, "is_qwen35", False) else full_prompt
|
||||
# The preset template is the INSTRUCTION: it belongs in the system
|
||||
# message, with the user's request as the user message. Putting it
|
||||
# after the user text made imperative prompts echo back unchanged.
|
||||
text = self._invoke(
|
||||
system_prompt=(
|
||||
f"{prompt_template}\n\n{prompt_output_guard()}"
|
||||
"You are a helpful vision-language assistant. "
|
||||
"Answer directly with the final answer only. No <think> and no reasoning."
|
||||
),
|
||||
user_prompt=effective_prompt,
|
||||
user_prompt=prompt,
|
||||
images_b64=images_b64 if self.chat_handler is not None else [],
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
repetition_penalty=repetition_penalty,
|
||||
seed=seed,
|
||||
model_name=model_name,
|
||||
)
|
||||
|
||||
text = normalize_minimax_output(text, preset_prompt, has_image=image is not None, duration=duration)
|
||||
|
||||
|
||||
print(f"[QwenVL GGUF DEBUG] Generation completed. Text length: {len(text) if text else 0}")
|
||||
print(f"[QwenVL GGUF DEBUG] Generated text: {text[:100] if text else 'EMPTY'}...")
|
||||
|
||||
|
||||
# Cache the generated text
|
||||
PROMPT_CACHE[cache_key] = {
|
||||
"text": text,
|
||||
@@ -1040,20 +721,18 @@ class QwenVLGGUFBase:
|
||||
"preset": preset_prompt,
|
||||
"seed": int(seed),
|
||||
"image_hash": image_hash,
|
||||
"image2_hash": image2_hash,
|
||||
"image3_hash": image3_hash,
|
||||
"video_hash": video_hash
|
||||
}
|
||||
save_prompt_cache() # Save cache to file
|
||||
|
||||
|
||||
print(f"[QwenVL GGUF] Cached new prompt for seed {seed}: {cache_key[:8]}...")
|
||||
|
||||
|
||||
print(f"[QwenVL GGUF DEBUG] Returning tuple with text...")
|
||||
|
||||
|
||||
# Save the generated prompt for future bypass mode
|
||||
LAST_SAVED_PROMPT = text
|
||||
print(f"[QwenVL GGUF] Saved prompt for bypass mode: {text[:50]}...")
|
||||
|
||||
|
||||
return (text,)
|
||||
finally:
|
||||
if not keep_model_loaded:
|
||||
@@ -1064,70 +743,61 @@ class AILab_QwenVL_GGUF(QwenVLGGUFBase):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
all_models = GGUF_VL_CATALOG.get("models") or {}
|
||||
model_keys = sorted([key for key, entry in all_models.items() if (entry or {}).get("mmproj_filename")]) or ["(no GGUF VL models found)"]
|
||||
model_keys = sorted([key for key, entry in all_models.items() if (entry or {}).get("mmproj_filename")]) or ["(edit gguf_models.json)"]
|
||||
default_model = model_keys[0]
|
||||
|
||||
prompts = PRESET_PROMPTS or ["IMG › Detailed"]
|
||||
preferred_prompt = "IMG › Detailed"
|
||||
prompts = PRESET_PROMPTS or ["🖼️ Detailed Description"]
|
||||
preferred_prompt = "🖼️ Detailed Description"
|
||||
default_prompt = preferred_prompt if preferred_prompt in prompts else prompts[0]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"model_name": (model_keys, {"default": default_model}),
|
||||
"preset_prompt": (prompts, {"default": default_prompt}),
|
||||
"prompt": ("STRING", {"default": "", "multiline": True, "tooltip": "Additional user input that gets combined with the preset template. Leave empty to use only the template."}),
|
||||
"max_tokens": ("INT", {"default": 8192, "min": 64, "max": 8192}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": False}),
|
||||
"custom_prompt": ("STRING", {"default": "", "multiline": True, "tooltip": "Additional user input that gets combined with the preset template. Leave empty to use only the template."}),
|
||||
"max_tokens": ("INT", {"default": 512, "min": 64, "max": 2048}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": True}),
|
||||
"seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1}),
|
||||
"keep_last_prompt": ("BOOLEAN", {"default": False, "tooltip": "Keep the last generated prompt instead of creating a new one"}),
|
||||
"passthrough": ("BOOLEAN", {"default": False, "tooltip": "Skip Qwen model loading and return prompt directly. Use when the chat already generated the final prompt — saves VRAM and inference time."}),
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE", {"tooltip": "First reference image (single image). For R2VA this is Picture 1."}),
|
||||
"image2": ("IMAGE", {"tooltip": "Second reference image (single image). For R2VA this is Picture 2."}),
|
||||
"image3": ("IMAGE", {"tooltip": "Third reference image (single image). For R2VA this is Picture 3."}),
|
||||
"video": ("IMAGE", {"tooltip": "Video frames input. Use frame_count to control how many frames are sampled."}),
|
||||
"duration": (DURATION_OPTIONS, {"default": DEFAULT_DURATION, "tooltip": "Clip length for duration-aware presets (MiniMax/LTX/Wan). Ignored by image presets."}),
|
||||
"image": ("IMAGE",),
|
||||
"video": ("IMAGE",),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("RESPONSE",)
|
||||
FUNCTION = "process"
|
||||
CATEGORY = "🔮 QwenVL-Mod"
|
||||
CATEGORY = "🔷 QwenVL-Mod/QwenVL"
|
||||
|
||||
def process(
|
||||
self,
|
||||
model_name,
|
||||
preset_prompt,
|
||||
prompt,
|
||||
custom_prompt,
|
||||
max_tokens,
|
||||
keep_model_loaded,
|
||||
seed,
|
||||
keep_last_prompt,
|
||||
passthrough=False,
|
||||
unload_after_run,
|
||||
image=None,
|
||||
image2=None,
|
||||
image3=None,
|
||||
video=None,
|
||||
duration=DEFAULT_DURATION,
|
||||
):
|
||||
return self.run(
|
||||
model_name=model_name,
|
||||
preset_prompt=preset_prompt,
|
||||
prompt=prompt,
|
||||
custom_prompt=custom_prompt,
|
||||
image=image,
|
||||
image2=image2,
|
||||
image3=image3,
|
||||
video=video,
|
||||
frame_count=16,
|
||||
duration=duration,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0.6,
|
||||
top_p=0.9,
|
||||
repetition_penalty=1.0,
|
||||
repetition_penalty=1.05,
|
||||
seed=seed,
|
||||
keep_model_loaded=keep_model_loaded,
|
||||
unload_after_run=unload_after_run,
|
||||
device="auto",
|
||||
ctx=None,
|
||||
n_batch=None,
|
||||
@@ -1136,7 +806,6 @@ class AILab_QwenVL_GGUF(QwenVLGGUFBase):
|
||||
top_k=None,
|
||||
pool_size=None,
|
||||
keep_last_prompt=keep_last_prompt,
|
||||
passthrough=passthrough,
|
||||
)
|
||||
|
||||
|
||||
@@ -1144,11 +813,11 @@ class AILab_QwenVL_GGUF_Advanced(QwenVLGGUFBase):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
all_models = GGUF_VL_CATALOG.get("models") or {}
|
||||
model_keys = sorted([key for key, entry in all_models.items() if (entry or {}).get("mmproj_filename")]) or ["(no GGUF VL models found)"]
|
||||
model_keys = sorted([key for key, entry in all_models.items() if (entry or {}).get("mmproj_filename")]) or ["(edit gguf_models.json)"]
|
||||
default_model = model_keys[0]
|
||||
|
||||
prompts = PRESET_PROMPTS or ["IMG › Detailed"]
|
||||
preferred_prompt = "IMG › Detailed"
|
||||
prompts = PRESET_PROMPTS or ["🖼️ Detailed Description"]
|
||||
preferred_prompt = "🖼️ Detailed Description"
|
||||
default_prompt = preferred_prompt if preferred_prompt in prompts else prompts[0]
|
||||
|
||||
num_gpus = torch.cuda.device_count()
|
||||
@@ -1160,45 +829,39 @@ class AILab_QwenVL_GGUF_Advanced(QwenVLGGUFBase):
|
||||
"model_name": (model_keys, {"default": default_model}),
|
||||
"device": (device_options, {"default": "auto"}),
|
||||
"preset_prompt": (prompts, {"default": default_prompt}),
|
||||
"camera_tag": (CAMERA_TAG_OPTIONS, {"default": "None", "tooltip": CAMERA_TAG_TOOLTIP}),
|
||||
"prompt": ("STRING", {"default": "", "multiline": True, "tooltip": "Additional user input that gets combined with the preset template. Leave empty to use only the template."}),
|
||||
"max_tokens": ("INT", {"default": 8192, "min": 64, "max": 8192}),
|
||||
"custom_prompt": ("STRING", {"default": "", "multiline": True, "tooltip": "Additional user input that gets combined with the preset template. Leave empty to use only the template."}),
|
||||
"max_tokens": ("INT", {"default": 512, "min": 64, "max": 4096}),
|
||||
"temperature": ("FLOAT", {"default": 0.6, "min": 0.0, "max": 2.0}),
|
||||
"top_p": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0}),
|
||||
"repetition_penalty": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0}),
|
||||
"repetition_penalty": ("FLOAT", {"default": 1.2, "min": 0.5, "max": 2.0}),
|
||||
"frame_count": ("INT", {"default": 16, "min": 1, "max": 64}),
|
||||
"ctx": ("INT", {"default": 32768, "min": 1024, "max": 262144, "step": 512}),
|
||||
"n_batch": ("INT", {"default": 512, "min": 64, "max": 32768, "step": 64}),
|
||||
"gpu_layers": ("INT", {"default": -1, "min": -1, "max": 200}),
|
||||
"image_max_tokens": ("INT", {"default": 4096, "min": 256, "max": 1024000, "step": 256}),
|
||||
"top_k": ("INT", {"default": 20, "min": 0, "max": 32768}),
|
||||
"top_k": ("INT", {"default": 0, "min": 0, "max": 32768}),
|
||||
"pool_size": ("INT", {"default": 4194304, "min": 1048576, "max": 10485760, "step": 524288}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": False}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": True}),
|
||||
"seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1}),
|
||||
"keep_last_prompt": ("BOOLEAN", {"default": False, "tooltip": "Keep the last generated prompt instead of creating a new one"}),
|
||||
"passthrough": ("BOOLEAN", {"default": False, "tooltip": "Skip Qwen model loading and return prompt directly. Use when the chat already generated the final prompt — saves VRAM and inference time."}),
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE", {"tooltip": "First reference image (single image). For R2VA this is Picture 1."}),
|
||||
"image2": ("IMAGE", {"tooltip": "Second reference image (single image). For R2VA this is Picture 2."}),
|
||||
"image3": ("IMAGE", {"tooltip": "Third reference image (single image). For R2VA this is Picture 3."}),
|
||||
"video": ("IMAGE", {"tooltip": "Video frames input. Use frame_count to control how many frames are sampled."}),
|
||||
"duration": (DURATION_OPTIONS, {"default": DEFAULT_DURATION, "tooltip": "Clip length for duration-aware presets (MiniMax/LTX/Wan). Ignored by image presets."}),
|
||||
"image": ("IMAGE",),
|
||||
"video": ("IMAGE",),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("RESPONSE",)
|
||||
FUNCTION = "process"
|
||||
CATEGORY = "🔮 QwenVL-Mod"
|
||||
CATEGORY = "🔷 QwenVL-Mod/QwenVL"
|
||||
|
||||
def process(
|
||||
self,
|
||||
model_name,
|
||||
device,
|
||||
preset_prompt,
|
||||
camera_tag,
|
||||
prompt,
|
||||
custom_prompt,
|
||||
max_tokens,
|
||||
temperature,
|
||||
top_p,
|
||||
@@ -1213,23 +876,16 @@ class AILab_QwenVL_GGUF_Advanced(QwenVLGGUFBase):
|
||||
keep_model_loaded,
|
||||
seed,
|
||||
keep_last_prompt,
|
||||
passthrough=False,
|
||||
image=None,
|
||||
image2=None,
|
||||
image3=None,
|
||||
video=None,
|
||||
duration=DEFAULT_DURATION,
|
||||
):
|
||||
return self.run(
|
||||
model_name=model_name,
|
||||
preset_prompt=preset_prompt,
|
||||
prompt=prompt,
|
||||
custom_prompt=custom_prompt,
|
||||
image=image,
|
||||
image2=image2,
|
||||
image3=image3,
|
||||
video=video,
|
||||
frame_count=frame_count,
|
||||
duration=duration,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
@@ -1243,9 +899,7 @@ class AILab_QwenVL_GGUF_Advanced(QwenVLGGUFBase):
|
||||
image_max_tokens=image_max_tokens,
|
||||
top_k=top_k,
|
||||
pool_size=pool_size,
|
||||
keep_last_prompt=keep_last_prompt,
|
||||
camera_tag=camera_tag,
|
||||
passthrough=passthrough,
|
||||
keep_last_prompt=keep_last_prompt
|
||||
)
|
||||
|
||||
|
||||
@@ -1255,6 +909,6 @@ NODE_CLASS_MAPPINGS = {
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"AILab_QwenVL_GGUF": "🧠 QwenVL-Mod (GGUF)",
|
||||
"AILab_QwenVL_GGUF_Advanced": "🧠 QwenVL-Mod Advanced (GGUF)",
|
||||
"AILab_QwenVL_GGUF": "QwenVL-Mod (GGUF)",
|
||||
"AILab_QwenVL_GGUF_Advanced": "QwenVL-Mod Advanced (GGUF)",
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# ComfyUI-QwenVL GGUF prompt enhancer
|
||||
#
|
||||
# GGUF nodes powered by llama.cpp for Qwen-VL models, including the latest Qwen3-VL.
|
||||
# GGUF nodes powered by llama.cpp for Qwen-VL models, including Qwen3-VL and Qwen2.5-VL.
|
||||
# Provides vision-capable GGUF inference and prompt execution.
|
||||
#
|
||||
# Models are loaded via llama-cpp-python and configured through gguf_models.json.
|
||||
@@ -21,27 +21,15 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import torch
|
||||
from huggingface_hub import hf_hub_download, snapshot_download
|
||||
try:
|
||||
from llama_cpp import Llama
|
||||
except ImportError:
|
||||
Llama = None
|
||||
print("[QwenVL] llama-cpp-python is not installed — GGUF nodes register but need a "
|
||||
"vision-capable build to run. Local install: `pip install llama-cpp-python`; "
|
||||
"for CUDA/vision wheels see github.com/1038lab/ComfyUI-QwenVL/blob/main/docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md")
|
||||
from llama_cpp import Llama
|
||||
|
||||
import folder_paths
|
||||
from AILab_OutputCleaner import OutputCleanConfig, clean_model_output, prompt_output_guard
|
||||
from wildcard_util import expand_wildcard_tokens
|
||||
from AILab_OutputCleaner import OutputCleanConfig, clean_model_output
|
||||
|
||||
# Import cache functions from main module
|
||||
import sys
|
||||
sys.path.append(str(Path(__file__).parent))
|
||||
from AILab_QwenVL import PROMPT_CACHE, ensure_cuda_vram_headroom, get_cache_key, get_alternative_cache_key, save_prompt_cache, CAMERA_TAG_OPTIONS, CAMERA_TAG_TOOLTIP, CAMERA_TAG_DESCRIPTIONS, STYLE_TAG_OPTIONS, STYLE_TAG_TOOLTIP, STYLE_TAG_DESCRIPTIONS, add_danbooru_guidance, camera_directive_location, style_reference_guard
|
||||
from AILab_QwenVL_GGUF import read_gguf_architecture, find_in_llm_paths, _filter_kwargs_for_callable
|
||||
from qwenvl_presets import (
|
||||
TEXT_STYLE_NAMES, TEXT_PROMPTS, TEXT_DURATIONS, TRANSLATION_PROMPT,
|
||||
DURATION_OPTIONS, DEFAULT_DURATION, resolve_text_style,
|
||||
)
|
||||
from AILab_QwenVL import PROMPT_CACHE, get_cache_key, get_alternative_cache_key, save_prompt_cache
|
||||
|
||||
# Simple global variable to store last generated prompt
|
||||
LAST_SAVED_PROMPT = None
|
||||
@@ -52,28 +40,19 @@ PROMPT_CONFIG_PATH = NODE_DIR / "AILab_System_Prompts.json"
|
||||
|
||||
|
||||
def load_prompt_config():
|
||||
styles = dict(TEXT_PROMPTS)
|
||||
translation_prompt = TRANSLATION_PROMPT
|
||||
# Legacy user file still wins over the bundled presets when present.
|
||||
if PROMPT_CONFIG_PATH.exists():
|
||||
try:
|
||||
with open(PROMPT_CONFIG_PATH, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh) or {}
|
||||
qwen_text = data.get("qwen_text") or {}
|
||||
legacy_styles = qwen_text.get("styles") or {}
|
||||
if isinstance(legacy_styles, dict):
|
||||
styles.update({
|
||||
name: entry.get("system_prompt", "")
|
||||
for name, entry in legacy_styles.items()
|
||||
if isinstance(entry, dict) and entry.get("system_prompt")
|
||||
})
|
||||
if isinstance(qwen_text.get("translation_prompt"), str):
|
||||
translation_prompt = qwen_text["translation_prompt"]
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL] Legacy prompt config skipped: {exc}")
|
||||
if not styles:
|
||||
raise RuntimeError("[QwenVL] No text styles found in presets/*.json")
|
||||
return {"styles": styles, "translation_prompt": translation_prompt}
|
||||
if not PROMPT_CONFIG_PATH.exists():
|
||||
raise FileNotFoundError(f"[QwenVL] Missing AILab_System_Prompts.json at {PROMPT_CONFIG_PATH}")
|
||||
try:
|
||||
with open(PROMPT_CONFIG_PATH, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh) or {}
|
||||
qwen_text = data.get("qwen_text") or {}
|
||||
styles = qwen_text.get("styles")
|
||||
translation_prompt = qwen_text.get("translation_prompt")
|
||||
if not styles or not translation_prompt:
|
||||
raise ValueError("AILab_System_Prompts.json must include qwen_text.styles and qwen_text.translation_prompt")
|
||||
return {"styles": styles, "translation_prompt": translation_prompt}
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"[QwenVL] Failed to load AILab_System_Prompts.json: {exc}") from exc
|
||||
|
||||
|
||||
PROMPT_CONFIG = load_prompt_config()
|
||||
@@ -112,7 +91,7 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("ENHANCED_OUTPUT",)
|
||||
FUNCTION = "process"
|
||||
CATEGORY = "🔮 QwenVL-Mod"
|
||||
CATEGORY = "🔷 QwenVL-Mod/QwenVL"
|
||||
|
||||
def __init__(self):
|
||||
self.llm = None
|
||||
@@ -120,50 +99,20 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
self.gguf_models = self.load_gguf_models()
|
||||
self.styles = STYLES
|
||||
|
||||
@staticmethod
|
||||
def _scan_local_gguf_text_models(base_dir: Path, existing_filenames: set[str]) -> dict[str, dict]:
|
||||
"""Scan the GGUF base directory for locally available text-only .gguf files."""
|
||||
local_models: dict[str, dict] = {}
|
||||
if not base_dir.exists() or not base_dir.is_dir():
|
||||
return local_models
|
||||
try:
|
||||
for gguf_file in base_dir.rglob("*.gguf"):
|
||||
if not gguf_file.is_file():
|
||||
continue
|
||||
# Skip mmproj files — they are vision projectors, not text models
|
||||
if "mmproj" in gguf_file.name.lower():
|
||||
continue
|
||||
if gguf_file.name in existing_filenames:
|
||||
continue
|
||||
display = f"[local] {gguf_file.name}"
|
||||
local_models[display] = {
|
||||
"filename": str(gguf_file),
|
||||
"is_local": True,
|
||||
"repo_id": None,
|
||||
"alt_repo_ids": [],
|
||||
"author": None,
|
||||
"repo_dirname": gguf_file.parent.name,
|
||||
"context_length": 32768,
|
||||
}
|
||||
except PermissionError:
|
||||
pass
|
||||
if local_models:
|
||||
print(f"[QwenVL] Discovered {len(local_models)} local GGUF text model(s) on disk")
|
||||
return local_models
|
||||
|
||||
@staticmethod
|
||||
def load_gguf_models():
|
||||
fallback = {
|
||||
"base_dir": "LLM/GGUF",
|
||||
"models": {},
|
||||
}
|
||||
data = {}
|
||||
if GGUF_CONFIG_PATH.exists():
|
||||
try:
|
||||
with open(GGUF_CONFIG_PATH, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh) or {}
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL] gguf_models.json load failed: {exc}")
|
||||
if not GGUF_CONFIG_PATH.exists():
|
||||
return fallback
|
||||
try:
|
||||
with open(GGUF_CONFIG_PATH, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh) or {}
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL] gguf_models.json load failed: {exc}")
|
||||
return fallback
|
||||
|
||||
base_dir = data.get("base_dir") or fallback["base_dir"]
|
||||
|
||||
@@ -176,13 +125,10 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
if isinstance(entry, dict):
|
||||
models[name] = entry
|
||||
|
||||
# Text catalog: Qwen_model first, then qwenVL_model — the 3.x VL models
|
||||
# are multimodal but work fine text-only, so they belong here too.
|
||||
seen_display_names: set[str] = set()
|
||||
for section in ("Qwen_model", "qwenVL_model"):
|
||||
qwen_repos = data.get(section) or {}
|
||||
if not isinstance(qwen_repos, dict):
|
||||
continue
|
||||
# Text-only catalog (use Qwen_model; do not use qwenVL_model here)
|
||||
qwen_repos = data.get("Qwen_model") or {}
|
||||
if isinstance(qwen_repos, dict):
|
||||
seen_display_names: set[str] = set()
|
||||
for repo_key, repo in qwen_repos.items():
|
||||
if not isinstance(repo, dict):
|
||||
continue
|
||||
@@ -210,54 +156,31 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
)
|
||||
models[display] = entry
|
||||
|
||||
# Scan filesystem for locally available models not in JSON config
|
||||
# Collect all directories to scan: the configured base_dir + any extra LLM paths from ComfyUI
|
||||
existing_filenames = {Path(e.get("filename", "")).name for e in models.values() if e.get("filename")}
|
||||
scan_dirs: list[Path] = [_resolve_base_dir(base_dir)]
|
||||
try:
|
||||
if "LLM" in folder_paths.folder_names_and_paths:
|
||||
for llm_path in folder_paths.get_folder_paths("LLM"):
|
||||
gguf_dir = Path(llm_path) / "GGUF"
|
||||
if gguf_dir not in scan_dirs:
|
||||
scan_dirs.append(gguf_dir)
|
||||
llm_p = Path(llm_path)
|
||||
if llm_p not in scan_dirs:
|
||||
scan_dirs.append(llm_p)
|
||||
except Exception:
|
||||
pass
|
||||
for scan_dir in scan_dirs:
|
||||
local_models = AILab_QwenVL_GGUF_PromptEnhancer._scan_local_gguf_text_models(scan_dir, existing_filenames)
|
||||
models.update(local_models)
|
||||
existing_filenames.update(Path(e.get("filename", "")).name for e in local_models.values() if e.get("filename"))
|
||||
|
||||
return {"base_dir": base_dir, "models": models}
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
styles = list(TEXT_STYLE_NAMES) or list(STYLES.keys())
|
||||
preferred_style = "Enhance"
|
||||
default_style = preferred_style if preferred_style in styles else (styles[0] if styles else "Enhance")
|
||||
styles = list(STYLES.keys())
|
||||
preferred_style = "📝 Enhance"
|
||||
default_style = preferred_style if preferred_style in styles else (styles[0] if styles else "📝 Enhance")
|
||||
temp = cls.load_gguf_models()
|
||||
model_keys = sorted(list((temp.get("models") or {}).keys())) or ["(no GGUF models found)"]
|
||||
model_keys = sorted(list((temp.get("models") or {}).keys())) or ["(edit gguf_models.json)"]
|
||||
default_model = model_keys[0]
|
||||
return {
|
||||
"required": {
|
||||
"model_name": (model_keys, {"default": default_model, "tooltip": "GGUF model from config or auto-detected from models/LLM/GGUF directory. [local] prefix = found on disk."}),
|
||||
"model_name": (model_keys, {"default": default_model, "tooltip": "GGUF model entry defined in gguf_models.json."}),
|
||||
"prompt_text": ("STRING", {"default": "", "multiline": True, "tooltip": "Prompt text to enhance. Leave blank to just emit the preset instruction."}),
|
||||
"preset_system_prompt": (styles, {"default": default_style}),
|
||||
"camera_tag": (CAMERA_TAG_OPTIONS, {"default": "None", "tooltip": CAMERA_TAG_TOOLTIP}),
|
||||
"style_tag": (STYLE_TAG_OPTIONS, {"default": "None", "tooltip": STYLE_TAG_TOOLTIP}),
|
||||
"max_tokens": ("INT", {"default": 8192, "min": 32, "max": 16384}),
|
||||
"custom_system_prompt": ("STRING", {"default": "", "multiline": True}),
|
||||
"max_tokens": ("INT", {"default": 256, "min": 32, "max": 1024}),
|
||||
"temperature": ("FLOAT", {"default": 0.7, "min": 0.1, "max": 1.0}),
|
||||
"top_p": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0}),
|
||||
"repetition_penalty": ("FLOAT", {"default": 1.1, "min": 0.5, "max": 2.0}),
|
||||
"english_output": ("BOOLEAN", {"default": False, "tooltip": "Force final output in English using translation prompt."}),
|
||||
"device": (["auto", "cuda", "cpu", "mps"], {"default": "auto", "tooltip": "Select device; auto prefers GPU when available."}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": False, "tooltip": "Keep model loaded in memory for faster repeated inference (uses more VRAM)."}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": True, "tooltip": "Keep model loaded in memory for faster repeated inference (uses more VRAM)."}),
|
||||
"seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1}),
|
||||
"keep_last_prompt": ("BOOLEAN", {"default": False, "tooltip": "Keep the last generated prompt instead of creating a new one"}),
|
||||
"passthrough": ("BOOLEAN", {"default": False, "tooltip": "Skip Qwen model loading and return prompt_text directly. Use when the chat already generated the final prompt — saves VRAM and inference time."}),
|
||||
"duration": (DURATION_OPTIONS, {"default": DEFAULT_DURATION, "tooltip": "Clip length for duration-aware styles (MiniMax/LTX/Wan). Ignored by generic styles."}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -289,14 +212,10 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
if torch.cuda.is_available():
|
||||
print(f"[QwenVL PromptEnhancer DEBUG] Clearing CUDA cache...")
|
||||
torch.cuda.empty_cache()
|
||||
try:
|
||||
torch.cuda.ipc_collect()
|
||||
except Exception:
|
||||
pass
|
||||
torch.cuda.synchronize()
|
||||
# Additional cleanup
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
|
||||
print(f"[QwenVL PromptEnhancer DEBUG] VRAM cleanup completed")
|
||||
|
||||
def _resolve_model_path(self, model_name):
|
||||
@@ -312,50 +231,27 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
entry = candidate
|
||||
break
|
||||
|
||||
# Local models store absolute paths — use directly
|
||||
filename = entry.get("filename")
|
||||
if filename and Path(filename).is_absolute():
|
||||
return Path(filename)
|
||||
|
||||
base_dir = _resolve_base_dir(self.gguf_models.get("base_dir") or "LLM/GGUF")
|
||||
|
||||
path = entry.get("path")
|
||||
if path:
|
||||
return Path(path).expanduser()
|
||||
|
||||
filename = entry.get("filename")
|
||||
if filename:
|
||||
author = _safe_dirname(str(entry.get("author") or entry.get("publisher") or ""))
|
||||
repo_dir = _safe_dirname(str(entry.get("repo_dirname") or model_name))
|
||||
bare = Path(filename).name
|
||||
if author and author != "unknown":
|
||||
default_path = base_dir / author / repo_dir / bare
|
||||
else:
|
||||
default_path = base_dir / repo_dir / bare
|
||||
|
||||
if default_path.exists():
|
||||
return default_path
|
||||
|
||||
found = find_in_llm_paths(
|
||||
filename,
|
||||
str(entry.get("author") or entry.get("publisher") or ""),
|
||||
str(entry.get("repo_dirname") or model_name),
|
||||
)
|
||||
if found is not None:
|
||||
print(f"[QwenVL] Using model from alternate LLM path: {found}")
|
||||
return found
|
||||
|
||||
return default_path
|
||||
return base_dir / author / repo_dir / Path(filename).name
|
||||
return base_dir / repo_dir / Path(filename).name
|
||||
|
||||
return base_dir / model_name
|
||||
|
||||
def _maybe_download_model(self, model_name, resolved):
|
||||
if resolved.exists():
|
||||
return
|
||||
# Local models should already exist on disk — don't attempt download
|
||||
models = self.gguf_models.get("models") or {}
|
||||
entry = models.get(model_name) or {}
|
||||
if entry.get("is_local"):
|
||||
raise FileNotFoundError(f"[QwenVL] Local GGUF model not found: {resolved}")
|
||||
if not entry:
|
||||
wanted = _model_name_to_filename_candidates(model_name)
|
||||
for candidate in models.values():
|
||||
@@ -415,9 +311,8 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
context_length = model_cfg.get("context_length", 32768)
|
||||
signature = (resolved, context_length, device)
|
||||
if self.llm is not None and self.current_signature == signature:
|
||||
ensure_cuda_vram_headroom("QwenVL PromptEnhancer GGUF", min_free_gb=1.0, min_free_ratio=0.08)
|
||||
return
|
||||
|
||||
|
||||
# Force aggressive cleanup before loading new model (especially for same model conflicts)
|
||||
print(f"[QwenVL PromptEnhancer DEBUG] Forcing cleanup before model loading...")
|
||||
self.clear()
|
||||
@@ -447,21 +342,6 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
"verbose": False,
|
||||
"chat_format": "qwen",
|
||||
}
|
||||
|
||||
# Detect Qwen3.x family from GGUF metadata instead of relying on model name
|
||||
arch = read_gguf_architecture(resolved)
|
||||
self.is_qwen35 = bool(
|
||||
(arch and "qwen3" in arch)
|
||||
or "qwen3" in model_name.lower()
|
||||
)
|
||||
if self.is_qwen35:
|
||||
print(f"[QwenVL] Qwen3 family detected (arch={arch}): Will disable thinking via /no_think + chat_template_kwargs/reasoning.")
|
||||
|
||||
if Llama is None:
|
||||
raise RuntimeError(
|
||||
"llama-cpp-python is not installed — GGUF prompt enhancement cannot run. "
|
||||
"Install a vision-capable build: local `pip install llama-cpp-python`, "
|
||||
"CUDA/vision wheels: github.com/1038lab/ComfyUI-QwenVL/blob/main/docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md")
|
||||
self.llm = Llama(**kwargs)
|
||||
self.current_signature = signature
|
||||
|
||||
@@ -480,39 +360,23 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
return False
|
||||
return bool(
|
||||
re.search(
|
||||
r"(?im)^\s*(?:[-*]\s*)?(?:\*\*)?\s*(okay[,.:]?|first[,.:]?|next[,.:]?|then[,.:]?|wait[,.:]?|final\s+plan|final\s+check)\b",
|
||||
r"(?im)^\s*(okay[,.:]?|first[,.:]?|next[,.:]?|then[,.:]?|wait[,.:]?)\b",
|
||||
text,
|
||||
)
|
||||
or re.search(r"(?i)\b(i\s+(should|need|must|will|am\s+going\s+to|have\s+to))\b", text)
|
||||
)
|
||||
|
||||
def _call(system: str, user: str, temp: float, seed_val: int) -> str:
|
||||
ensure_cuda_vram_headroom("QwenVL PromptEnhancer GGUF", min_free_gb=1.0, min_free_ratio=0.08)
|
||||
if self.llm is not None and hasattr(self.llm, "reset"):
|
||||
try:
|
||||
self.llm.reset()
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL PromptEnhancer DEBUG] llama context reset skipped: {exc}")
|
||||
|
||||
effective_user = user
|
||||
extra_kwargs = {}
|
||||
if getattr(self, "is_qwen35", False):
|
||||
effective_user = "/no_think\n" + user
|
||||
extra_kwargs["chat_template_kwargs"] = {"enable_thinking": False}
|
||||
extra_kwargs["reasoning"] = False
|
||||
extra_kwargs = _filter_kwargs_for_callable(self.llm.create_chat_completion, extra_kwargs)
|
||||
|
||||
response = self.llm.create_chat_completion(
|
||||
messages=[
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": effective_user},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
max_tokens=max_tokens,
|
||||
temperature=temp,
|
||||
top_p=top_p,
|
||||
repeat_penalty=repetition_penalty,
|
||||
seed=seed_val,
|
||||
**extra_kwargs
|
||||
)
|
||||
if not response or "choices" not in response or not response["choices"]:
|
||||
raise RuntimeError("[QwenVL] llama_cpp returned empty response")
|
||||
@@ -546,8 +410,7 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
model_name,
|
||||
prompt_text,
|
||||
preset_system_prompt,
|
||||
camera_tag,
|
||||
style_tag,
|
||||
custom_system_prompt,
|
||||
max_tokens,
|
||||
temperature,
|
||||
top_p,
|
||||
@@ -557,20 +420,9 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
keep_model_loaded,
|
||||
seed,
|
||||
keep_last_prompt,
|
||||
passthrough=False,
|
||||
duration=DEFAULT_DURATION,
|
||||
):
|
||||
global LAST_SAVED_PROMPT
|
||||
|
||||
# Expand TagForge __wildcard__ tokens before anything else — also in
|
||||
# passthrough mode, so raw tokens never reach the downstream prompt.
|
||||
prompt_text = expand_wildcard_tokens(prompt_text or "")
|
||||
|
||||
# Passthrough mode: skip model loading entirely, return prompt_text as-is.
|
||||
if passthrough:
|
||||
print(f"[QwenVL PromptEnhancer GGUF] Passthrough mode ON — skipping model load, returning prompt_text directly ({len(prompt_text or '')} chars)")
|
||||
return (prompt_text or "",)
|
||||
|
||||
|
||||
# Simple keep last prompt logic
|
||||
if keep_last_prompt: # Keep last prompt enabled
|
||||
print(f"[QwenVL PromptEnhancer GGUF] Keep last prompt enabled - using last saved prompt")
|
||||
@@ -580,93 +432,31 @@ class AILab_QwenVL_GGUF_PromptEnhancer:
|
||||
else:
|
||||
print(f"[QwenVL PromptEnhancer GGUF] No previous prompt found, returning empty")
|
||||
return ("",)
|
||||
|
||||
|
||||
# Always generate when keep last prompt is disabled
|
||||
print(f"[QwenVL PromptEnhancer GGUF] Keep last prompt disabled - generating new prompt")
|
||||
|
||||
|
||||
# Generate cache key with all inputs including seed
|
||||
cache_prompt = "\n\n".join(
|
||||
part for part in (
|
||||
prompt_text.strip(),
|
||||
f"english_output={bool(english_output)}",
|
||||
) if part
|
||||
)
|
||||
preset_system_prompt, style_key = resolve_text_style(preset_system_prompt, duration)
|
||||
cache_key = get_cache_key(model_name, style_key, cache_prompt, seed=seed)
|
||||
|
||||
cache_key = get_cache_key(model_name, preset_system_prompt, prompt_text, seed=seed)
|
||||
|
||||
# Check cache first (only for random mode)
|
||||
if cache_key in PROMPT_CACHE:
|
||||
cached_text = PROMPT_CACHE[cache_key].get("text", "")
|
||||
if cached_text:
|
||||
print(f"[QwenVL PromptEnhancer GGUF] Using cached prompt for seed {seed}: {cache_key[:8]}...")
|
||||
return (cached_text.strip(),)
|
||||
|
||||
system_prompt = (self.styles.get(style_key) or "").strip()
|
||||
|
||||
style_entry = self.styles.get(preset_system_prompt, {})
|
||||
system_prompt = (custom_system_prompt.strip() or style_entry.get("system_prompt") or "").strip()
|
||||
if not system_prompt:
|
||||
raise ValueError("system_prompt is empty; check presets/*.json or preset selection.")
|
||||
system_prompt = add_danbooru_guidance(system_prompt, preset_system_prompt)
|
||||
system_prompt = f"{system_prompt}\n\n{prompt_output_guard()}"
|
||||
merged_prompt = prompt_text.strip() or "Describe a scene vividly."
|
||||
|
||||
# ── Camera tag injection ───────────────────────────────────────────
|
||||
CAMERA_TAGS = list(CAMERA_TAG_DESCRIPTIONS.keys())
|
||||
found_cam_tag = None
|
||||
if camera_tag and camera_tag.strip() and camera_tag.strip().upper() != "NONE":
|
||||
tag_clean = camera_tag.strip().upper().strip("[]")
|
||||
if tag_clean in CAMERA_TAGS:
|
||||
found_cam_tag = tag_clean
|
||||
if not found_cam_tag and prompt_text and prompt_text.strip():
|
||||
upper = prompt_text.upper()
|
||||
for tag in CAMERA_TAGS:
|
||||
if f"[{tag}]" in upper:
|
||||
found_cam_tag = tag
|
||||
break
|
||||
if found_cam_tag:
|
||||
desc = CAMERA_TAG_DESCRIPTIONS.get(found_cam_tag, "")
|
||||
tag_str = f"[{found_cam_tag}]"
|
||||
merged_prompt = f"{tag_str}\n\n{merged_prompt}"
|
||||
merged_prompt += (
|
||||
f"\n\n═══ FINAL CAMERA DIRECTIVE (HIGHEST PRIORITY) ═══\n"
|
||||
f"Camera: {tag_str} — {desc}\n"
|
||||
f"You MUST use this camera movement and NO other. "
|
||||
f"{camera_directive_location(preset_system_prompt, system_prompt)}\n"
|
||||
f"IMPORTANT: the camera tag controls ONLY the camera. "
|
||||
f"The subject MUST still have natural, lively action and "
|
||||
f"movement throughout the clip — breathing, gestures, "
|
||||
f"expression changes, body motion, interaction with the "
|
||||
f"environment. Do NOT freeze the subject just because the "
|
||||
f"camera is moving. The subject is alive and active while "
|
||||
f"the camera performs {tag_str}.\n"
|
||||
f"═══ END DIRECTIVE ═══"
|
||||
)
|
||||
|
||||
# ── Style tag injection ────────────────────────────────────────
|
||||
# Only for T2V (no image): inject visual style as prefix + reminder
|
||||
STYLE_TAGS = list(STYLE_TAG_DESCRIPTIONS.keys())
|
||||
found_style_tag = None
|
||||
if style_tag and style_tag.strip() and style_tag.strip().upper() != "NONE":
|
||||
tag_clean = style_tag.strip().upper().strip("[]")
|
||||
if tag_clean in STYLE_TAGS:
|
||||
found_style_tag = tag_clean
|
||||
if not found_style_tag and prompt_text and prompt_text.strip():
|
||||
upper = prompt_text.upper()
|
||||
for tag in STYLE_TAGS:
|
||||
if f"[{tag}]" in upper:
|
||||
found_style_tag = tag
|
||||
break
|
||||
if found_style_tag:
|
||||
desc = STYLE_TAG_DESCRIPTIONS.get(found_style_tag, "")
|
||||
tag_str = f"[{found_style_tag}]"
|
||||
merged_prompt = f"{tag_str}\n\n{merged_prompt}"
|
||||
merged_prompt += (
|
||||
f"\n\n═══ FINAL STYLE DIRECTIVE (HIGHEST PRIORITY) ═══\n"
|
||||
f"Visual style: {tag_str} — {desc}\n"
|
||||
f"You MUST use this visual style for the ENTIRE clip. "
|
||||
f"State it explicitly in the first sentence and "
|
||||
f"maintain it consistently.{style_reference_guard(preset_system_prompt, system_prompt)}\n"
|
||||
f"═══ END DIRECTIVE ═══"
|
||||
)
|
||||
|
||||
raise ValueError("system_prompt is empty; check AILab_System_Prompts.json or preset selection.")
|
||||
system_prompt = (
|
||||
f"{system_prompt}\n\n"
|
||||
"Return only the final prompt text. No preface, no explanations, no analysis, no JSON, no markdown fences, and no </think>.\n"
|
||||
"Do not write planning steps (no 'First', 'Next', 'Then') and do not use first-person ('I', 'we')."
|
||||
)
|
||||
user_prompt = prompt_text.strip() or "Describe a scene vividly."
|
||||
merged_prompt = user_prompt
|
||||
self._load_model(model_name, device)
|
||||
enhanced = self._invoke_llama(
|
||||
system_prompt=system_prompt,
|
||||
@@ -736,5 +526,5 @@ NODE_CLASS_MAPPINGS = {
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"AILab_QwenVL_GGUF_PromptEnhancer": "✍🏻 QwenVL-Mod Prompt Enhancer (GGUF)",
|
||||
"AILab_QwenVL_GGUF_PromptEnhancer": "QwenVL-Mod Prompt Enhancer (GGUF)",
|
||||
}
|
||||
|
||||
+46
-202
@@ -8,33 +8,17 @@ import gc
|
||||
import hashlib
|
||||
import json
|
||||
import platform
|
||||
import re
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
|
||||
|
||||
from AILab_OutputCleaner import OutputCleanConfig, clean_model_output, prompt_output_guard
|
||||
from wildcard_util import expand_wildcard_tokens
|
||||
from qwenvl_presets import (
|
||||
TEXT_STYLE_NAMES, TEXT_PROMPTS, TEXT_DURATIONS,
|
||||
DURATION_OPTIONS, DEFAULT_DURATION, resolve_text_style,
|
||||
)
|
||||
|
||||
from AILab_QwenVL import (
|
||||
ATTENTION_MODES,
|
||||
CAMERA_TAG_OPTIONS,
|
||||
CAMERA_TAG_TOOLTIP,
|
||||
STYLE_TAG_OPTIONS,
|
||||
STYLE_TAG_TOOLTIP,
|
||||
STYLE_TAG_DESCRIPTIONS,
|
||||
add_danbooru_guidance,
|
||||
style_reference_guard,
|
||||
HF_TEXT_MODELS,
|
||||
HF_VL_MODELS,
|
||||
PROMPT_CACHE,
|
||||
ensure_cuda_vram_headroom,
|
||||
get_cache_key,
|
||||
get_alternative_cache_key,
|
||||
save_prompt_cache,
|
||||
@@ -50,34 +34,34 @@ NODE_DIR = Path(__file__).parent
|
||||
SYSTEM_PROMPTS_PATH = NODE_DIR / "AILab_System_Prompts.json"
|
||||
|
||||
DEFAULT_STYLES = {
|
||||
"📝 Enhance": "Write one production-ready prompt paragraph in the same language as the user. Expand the idea with concrete subject, action, environment, lighting, camera, composition, color, texture, mood, and style details. Output only the final prompt paragraph.",
|
||||
"📝 Refine": "Write one polished prompt paragraph in the same language as the user. Preserve the core intent, remove redundancy and contradiction, and add useful visual specificity for subject, scene, lighting, camera perspective, composition, palette, texture, and atmosphere. Output only the final prompt paragraph.",
|
||||
"📝 Creative Rewrite": "Write one fresh, imaginative prompt paragraph in the same language as the user. Preserve the core intent while adding cohesive cinematic atmosphere, gesture, micro-details, color relationships, light interaction, materials, depth, and composition. Output only the final prompt paragraph.",
|
||||
"📝 Detailed Visual": "Write one highly detailed visual prompt paragraph in the same language as the user. Include subject traits, pose, expression, wardrobe, materials, foreground, midground, background, lighting source and direction, palette, contrast, lens feel, depth of field, framing, and final aesthetic style. Output only the final prompt paragraph.",
|
||||
"📝 Artistic Style": "Write one artistic prompt paragraph in the same language as the user. Build a coherent visual direction with mood, palette, shape language, contrast, material feel, camera perspective, composition rhythm, and fitting style references. Output only the final prompt paragraph.",
|
||||
"📝 Technical Specs": "Write one clear technical photography or cinematography prompt paragraph in the same language as the user. Include camera distance, angle, lens feel, aperture or depth of field, focus target, lighting type and direction, color temperature, contrast, framing, background separation, texture rendering, and final image style. Output only the final prompt paragraph.",
|
||||
"📝 Enhance": "Expand and enrich this prompt with vivid visual context:",
|
||||
"📝 Refine": "Polish this prompt for clarity and precise AI interpretation:",
|
||||
"📝 Creative Rewrite": "Rewrite this prompt imaginatively while preserving intent:",
|
||||
"📝 Detailed Visual": "Turn this prompt into a highly detailed visual description:",
|
||||
"📝 Artistic Style": "Describe this prompt in artistic language suitable for image generation:",
|
||||
"📝 Technical Specs": "Convert this prompt into clear technical parameters:",
|
||||
}
|
||||
|
||||
|
||||
def _load_prompt_styles() -> dict[str, str]:
|
||||
styles = dict(TEXT_PROMPTS)
|
||||
try:
|
||||
with open(SYSTEM_PROMPTS_PATH, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh) or {}
|
||||
qwen_text = data.get("qwen_text") or {}
|
||||
legacy_styles = qwen_text.get("styles") or {}
|
||||
if isinstance(legacy_styles, dict) and legacy_styles:
|
||||
styles.update({
|
||||
styles = qwen_text.get("styles") or {}
|
||||
if isinstance(styles, dict) and styles:
|
||||
resolved = {
|
||||
name: entry.get("system_prompt", "")
|
||||
for name, entry in legacy_styles.items()
|
||||
for name, entry in styles.items()
|
||||
if isinstance(entry, dict) and entry.get("system_prompt")
|
||||
})
|
||||
return styles
|
||||
}
|
||||
if resolved:
|
||||
return resolved
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL] Prompt style load failed: {exc}")
|
||||
return styles or DEFAULT_STYLES
|
||||
return DEFAULT_STYLES
|
||||
|
||||
|
||||
PROMPT_STYLES = _load_prompt_styles()
|
||||
@@ -89,7 +73,7 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("ENHANCED_OUTPUT",)
|
||||
FUNCTION = "process"
|
||||
CATEGORY = "🔮 QwenVL-Mod"
|
||||
CATEGORY = "🔷 QwenVL-Mod/QwenVL"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -101,9 +85,9 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
def INPUT_TYPES(cls):
|
||||
models = list(HF_TEXT_MODELS.keys()) + [name for name in HF_VL_MODELS.keys() if name not in HF_TEXT_MODELS]
|
||||
default_model = models[0] if models else "Qwen3-VL-4B-Instruct"
|
||||
styles = list(TEXT_STYLE_NAMES) or list(cls.STYLES.keys())
|
||||
preferred_style = "Enhance"
|
||||
default_style = preferred_style if preferred_style in styles else (styles[0] if styles else "Enhance")
|
||||
styles = list(cls.STYLES.keys())
|
||||
preferred_style = "📝 Enhance"
|
||||
default_style = preferred_style if preferred_style in styles else (styles[0] if styles else "📝 Enhance")
|
||||
return {
|
||||
"required": {
|
||||
"model_name": (models, {"default": default_model, "tooltip": TOOLTIPS["model_name"]}),
|
||||
@@ -113,17 +97,14 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
"device": (["auto", "cuda", "cpu", "mps"], {"default": "auto", "tooltip": TOOLTIPS["device"]}),
|
||||
"prompt_text": ("STRING", {"default": "", "multiline": True, "tooltip": "Prompt text to enhance. Leave blank to just emit the preset instruction."}),
|
||||
"enhancement_style": (styles, {"default": default_style}),
|
||||
"camera_tag": (CAMERA_TAG_OPTIONS, {"default": "None", "tooltip": CAMERA_TAG_TOOLTIP}),
|
||||
"style_tag": (STYLE_TAG_OPTIONS, {"default": "None", "tooltip": STYLE_TAG_TOOLTIP}),
|
||||
"max_tokens": ("INT", {"default": 8192, "min": 32, "max": 16384}),
|
||||
"custom_system_prompt": ("STRING", {"default": "", "multiline": True}),
|
||||
"max_tokens": ("INT", {"default": 256, "min": 32, "max": 1024}),
|
||||
"temperature": ("FLOAT", {"default": 0.7, "min": 0.1, "max": 1.0}),
|
||||
"top_p": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0}),
|
||||
"repetition_penalty": ("FLOAT", {"default": 1.1, "min": 0.5, "max": 2.0}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": False}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": True}),
|
||||
"seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1}),
|
||||
"keep_last_prompt": ("BOOLEAN", {"default": False, "tooltip": "Keep the last generated prompt instead of creating a new one"}),
|
||||
"passthrough": ("BOOLEAN", {"default": False, "tooltip": "Skip Qwen model loading and return prompt_text directly. Use when the chat already generated the final prompt — saves VRAM and inference time."}),
|
||||
"duration": (DURATION_OPTIONS, {"default": DEFAULT_DURATION, "tooltip": "Clip length for duration-aware styles (MiniMax/LTX/Wan). Ignored by generic styles."}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,8 +117,7 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
device,
|
||||
prompt_text,
|
||||
enhancement_style,
|
||||
camera_tag,
|
||||
style_tag,
|
||||
custom_system_prompt,
|
||||
max_tokens,
|
||||
temperature,
|
||||
top_p,
|
||||
@@ -145,20 +125,9 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
keep_model_loaded,
|
||||
seed,
|
||||
keep_last_prompt=False,
|
||||
passthrough=False,
|
||||
duration=DEFAULT_DURATION,
|
||||
):
|
||||
global LAST_SAVED_PROMPT
|
||||
|
||||
# Expand TagForge __wildcard__ tokens before anything else — also in
|
||||
# passthrough mode, so raw tokens never reach the downstream prompt.
|
||||
prompt_text = expand_wildcard_tokens(prompt_text or "")
|
||||
|
||||
# Passthrough mode: skip model loading entirely, return prompt_text as-is.
|
||||
if passthrough:
|
||||
print(f"[QwenVL PromptEnhancer] Passthrough mode ON — skipping model load, returning prompt_text directly ({len(prompt_text or '')} chars)")
|
||||
return (prompt_text or "",)
|
||||
|
||||
|
||||
# Simple keep last prompt logic
|
||||
if keep_last_prompt:
|
||||
print(f"[QwenVL PromptEnhancer HF] Keep last prompt enabled - using last saved prompt")
|
||||
@@ -168,47 +137,16 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
else:
|
||||
print(f"[QwenVL PromptEnhancer HF] No previous prompt found, returning empty")
|
||||
return ("",)
|
||||
|
||||
|
||||
# Always generate when keep last prompt is disabled
|
||||
print(f"[QwenVL PromptEnhancer HF] Keep last prompt disabled - generating new prompt")
|
||||
|
||||
enhancement_style, style_key = resolve_text_style(enhancement_style, duration)
|
||||
style_instruction = self.STYLES.get(
|
||||
style_key,
|
||||
self.STYLES.get(enhancement_style, next(iter(self.STYLES.values()), "")),
|
||||
).strip()
|
||||
style_instruction = add_danbooru_guidance(style_instruction, enhancement_style)
|
||||
base_instruction = "\n\n".join(part for part in (style_instruction, prompt_output_guard()) if part)
|
||||
|
||||
base_instruction = custom_system_prompt.strip() or self.STYLES.get(
|
||||
enhancement_style,
|
||||
next(iter(self.STYLES.values()), ""),
|
||||
)
|
||||
user_prompt = prompt_text.strip() or "Describe a scene vividly."
|
||||
merged_prompt = f"{user_prompt}\n\n{base_instruction}".strip()
|
||||
|
||||
# ── Style tag injection ────────────────────────────────────────
|
||||
# Only for T2V (no image): inject visual style as prefix + reminder
|
||||
STYLE_TAGS = list(STYLE_TAG_DESCRIPTIONS.keys())
|
||||
found_style_tag = None
|
||||
if style_tag and style_tag.strip() and style_tag.strip().upper() != "NONE":
|
||||
tag_clean = style_tag.strip().upper().strip("[]")
|
||||
if tag_clean in STYLE_TAGS:
|
||||
found_style_tag = tag_clean
|
||||
if not found_style_tag and prompt_text and prompt_text.strip():
|
||||
upper = prompt_text.upper()
|
||||
for tag in STYLE_TAGS:
|
||||
if f"[{tag}]" in upper:
|
||||
found_style_tag = tag
|
||||
break
|
||||
if found_style_tag:
|
||||
desc = STYLE_TAG_DESCRIPTIONS.get(found_style_tag, "")
|
||||
tag_str = f"[{found_style_tag}]"
|
||||
merged_prompt = (
|
||||
f"{tag_str}\n\n{merged_prompt}\n\n"
|
||||
f"═══ FINAL STYLE DIRECTIVE (HIGHEST PRIORITY) ═══\n"
|
||||
f"Visual style: {tag_str} — {desc}\n"
|
||||
f"You MUST use this visual style for the ENTIRE clip. "
|
||||
f"State it explicitly in the first sentence and "
|
||||
f"maintain it consistently.{style_reference_guard(enhancement_style, style_instruction)}\n"
|
||||
f"═══ END DIRECTIVE ═══"
|
||||
)
|
||||
|
||||
if model_name in HF_TEXT_MODELS:
|
||||
enhanced = self._invoke_text(
|
||||
model_name,
|
||||
@@ -236,13 +174,12 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
repetition_penalty,
|
||||
keep_model_loaded,
|
||||
seed,
|
||||
camera_tag=camera_tag,
|
||||
)
|
||||
|
||||
|
||||
# Save the generated prompt for future bypass mode
|
||||
LAST_SAVED_PROMPT = enhanced.strip()
|
||||
print(f"[QwenVL PromptEnhancer HF] Saved prompt for bypass mode: {LAST_SAVED_PROMPT[:50]}...")
|
||||
|
||||
|
||||
return (enhanced.strip(),)
|
||||
|
||||
def _invoke_qwen(
|
||||
@@ -259,15 +196,14 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
repetition_penalty,
|
||||
keep_model_loaded,
|
||||
seed,
|
||||
camera_tag="None",
|
||||
):
|
||||
output = self.run(
|
||||
model_name=model_name,
|
||||
quantization=quantization,
|
||||
preset_prompt="🪄 Prompt Refine & Expand",
|
||||
prompt=prompt,
|
||||
custom_prompt=prompt,
|
||||
image=None,
|
||||
image2=None,
|
||||
video=None,
|
||||
frame_count=1,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
@@ -279,7 +215,6 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
attention_mode=attention_mode,
|
||||
use_torch_compile=use_torch_compile,
|
||||
device=device,
|
||||
camera_tag=camera_tag,
|
||||
)
|
||||
return output[0]
|
||||
|
||||
@@ -306,43 +241,24 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
else:
|
||||
quant_cfg = None
|
||||
|
||||
# BnB needs a GPU and cannot tolerate a post-load .to(device).
|
||||
if quant_cfg is not None and device == "cpu":
|
||||
print("[QwenVL] ⚠️ BitsAndBytes requires a CUDA/ROCm GPU — falling back to FP32 on CPU")
|
||||
quant_cfg = None
|
||||
|
||||
signature = (repo_id, quantization, device)
|
||||
if self.text_model is not None and self.text_signature == signature:
|
||||
ensure_cuda_vram_headroom("QwenVL PromptEnhancer HF", min_free_gb=1.0, min_free_ratio=0.08)
|
||||
return
|
||||
|
||||
self.text_model = None
|
||||
self.text_tokenizer = None
|
||||
self.text_signature = None
|
||||
|
||||
load_kwargs = {"trust_remote_code": True}
|
||||
if quant_cfg is not None:
|
||||
load_kwargs = {}
|
||||
if quant_cfg:
|
||||
load_kwargs["quantization_config"] = quant_cfg
|
||||
# accelerate must dispatch BnB weights straight to the target GPU.
|
||||
load_kwargs["device_map"] = device if device.startswith("cuda") else "auto"
|
||||
else:
|
||||
load_kwargs["dtype"] = torch.float16 if device == "cuda" else torch.float32
|
||||
load_kwargs["torch_dtype"] = torch.float16 if device == "cuda" else torch.float32
|
||||
|
||||
print(f"[QwenVL] Loading text model {model_name} ({quantization})")
|
||||
self.text_tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
|
||||
self.text_model = AutoModelForCausalLM.from_pretrained(repo_id, **load_kwargs).eval()
|
||||
# Only move non-BnB models — BnB is already dispatched via device_map.
|
||||
if quant_cfg is None:
|
||||
self.text_model.to(device)
|
||||
ensure_cuda_vram_headroom("QwenVL PromptEnhancer HF", min_free_gb=1.0, min_free_ratio=0.08)
|
||||
# Detect Qwen3.x family from loaded model config
|
||||
hf_model_type = getattr(self.text_model.config, "model_type", None)
|
||||
self.is_qwen35 = bool(
|
||||
(hf_model_type and "qwen3" in hf_model_type)
|
||||
or "qwen3" in model_name.lower()
|
||||
)
|
||||
if self.is_qwen35:
|
||||
print(f"[QwenVL] Qwen3 family detected (model_type={hf_model_type}): Will disable thinking in chat template.")
|
||||
self.text_model = AutoModelForCausalLM.from_pretrained(repo_id, trust_remote_code=True, **load_kwargs).eval()
|
||||
self.text_model.to(device)
|
||||
self.text_signature = signature
|
||||
|
||||
def _invoke_text(
|
||||
@@ -359,37 +275,13 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
seed,
|
||||
):
|
||||
self._load_text_model(model_name, quantization, device)
|
||||
ensure_cuda_vram_headroom("QwenVL PromptEnhancer HF", min_free_gb=1.0, min_free_ratio=0.08)
|
||||
|
||||
if device == "auto":
|
||||
device_choice = "cuda" if torch.cuda.is_available() else ("mps" if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available() else "cpu")
|
||||
else:
|
||||
device_choice = device
|
||||
|
||||
is_qwen35 = getattr(self, "is_qwen35", False)
|
||||
if is_qwen35:
|
||||
prompt = "/no_think\n" + prompt
|
||||
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
template_kwargs = {"tokenize": False, "add_generation_prompt": True}
|
||||
|
||||
# Inject the disable thinking kwargs for HF Transformers correctly
|
||||
if is_qwen35:
|
||||
template_kwargs["chat_template_kwargs"] = {"enable_thinking": False}
|
||||
|
||||
try:
|
||||
formatted_prompt = self.text_tokenizer.apply_chat_template(messages, **template_kwargs)
|
||||
except TypeError:
|
||||
# Older chat templates may not accept enable_thinking; retry without it
|
||||
template_kwargs.pop("chat_template_kwargs", None)
|
||||
try:
|
||||
formatted_prompt = self.text_tokenizer.apply_chat_template(messages, **template_kwargs)
|
||||
except Exception:
|
||||
formatted_prompt = prompt
|
||||
except Exception:
|
||||
# Fallback to raw prompt if the tokenizer lacks a chat template
|
||||
formatted_prompt = prompt
|
||||
|
||||
inputs = self.text_tokenizer(prompt, return_tensors="pt").to(device_choice)
|
||||
kwargs = {
|
||||
"max_new_tokens": max_tokens,
|
||||
"repetition_penalty": repetition_penalty,
|
||||
@@ -399,59 +291,11 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
"eos_token_id": self.text_tokenizer.eos_token_id,
|
||||
"pad_token_id": self.text_tokenizer.eos_token_id,
|
||||
}
|
||||
|
||||
def _looks_like_planning(text: str) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
return bool(
|
||||
re.search(
|
||||
r"(?im)^\s*(?:[-*]\s*)?(?:\*\*)?\s*(okay[,.:]?|first[,.:]?|next[,.:]?|then[,.:]?|wait[,.:]?|final\s+plan|final\s+check)\b",
|
||||
text,
|
||||
)
|
||||
or re.search(r"(?i)\b(i\s+(should|need|must|will|am\s+going\s+to|have\s+to))\b", text)
|
||||
)
|
||||
|
||||
def _generate(user_prompt, temp):
|
||||
if seed is not None:
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
gen_inputs = self.text_tokenizer(user_prompt, return_tensors="pt").to(device_choice)
|
||||
gen_kwargs = dict(kwargs)
|
||||
gen_kwargs["temperature"] = temp
|
||||
out = self.text_model.generate(**gen_inputs, **gen_kwargs)
|
||||
gen_tokens = out[0][gen_inputs["input_ids"].shape[1]:]
|
||||
return self.text_tokenizer.decode(gen_tokens, skip_special_tokens=True).strip()
|
||||
|
||||
raw = _generate(formatted_prompt, temperature)
|
||||
result = clean_model_output(raw, OutputCleanConfig(mode="prompt")) or raw
|
||||
|
||||
# Some Qwen3.x instruct models emit <think> blocks regardless of the
|
||||
# /no_think hint — if the output is only planning, do one constrained
|
||||
# retry asking for the final prompt text only (same as GGUF path).
|
||||
if "<think" in raw.lower() or _looks_like_planning(result):
|
||||
tail = re.split(r"</think>", raw, flags=re.IGNORECASE)[-1].strip()
|
||||
tail = clean_model_output(tail, OutputCleanConfig(mode="prompt"))
|
||||
if tail and not _looks_like_planning(tail):
|
||||
result = tail
|
||||
else:
|
||||
retry_system = (
|
||||
"You are a professional photography prompt writer.\n"
|
||||
"Output ONLY ONE final photography prompt paragraph.\n"
|
||||
"No analysis, no planning steps, no first-person, and no <think>.\n"
|
||||
"No bullet points, no headings, no JSON, no markdown, no quotes."
|
||||
)
|
||||
retry_msgs = [{"role": "system", "content": retry_system},
|
||||
{"role": "user", "content": f"Rewrite the following into the final prompt paragraph:\n\n{raw}\n"}]
|
||||
try:
|
||||
retry_prompt = self.text_tokenizer.apply_chat_template(retry_msgs, tokenize=False, add_generation_prompt=True)
|
||||
except Exception:
|
||||
retry_prompt = retry_msgs[1]["content"]
|
||||
raw_retry = _generate(retry_prompt, 0.4)
|
||||
cleaned_retry = clean_model_output(raw_retry, OutputCleanConfig(mode="prompt"))
|
||||
if cleaned_retry and not _looks_like_planning(cleaned_retry):
|
||||
result = cleaned_retry
|
||||
|
||||
outputs = self.text_model.generate(**inputs, **kwargs)
|
||||
decoded = self.text_tokenizer.decode(outputs[0], skip_special_tokens=True)
|
||||
prefix = self.text_tokenizer.decode(inputs["input_ids"][0], skip_special_tokens=True)
|
||||
result = decoded[len(prefix) :].strip()
|
||||
|
||||
# Cache the generated text
|
||||
# PROMPT_CACHE[cache_key] = {
|
||||
# "text": result,
|
||||
@@ -463,7 +307,7 @@ class AILab_QwenVL_PromptEnhancer(QwenVLBase):
|
||||
# "video_hash": None # PromptEnhancer doesn't use videos
|
||||
# }
|
||||
# save_prompt_cache() # Save cache to file
|
||||
|
||||
|
||||
# print(f"[QwenVL PromptEnhancer HF] Cached new prompt for seed {seed}: {cache_key[:8]}...")
|
||||
|
||||
if not keep_model_loaded:
|
||||
@@ -480,5 +324,5 @@ NODE_CLASS_MAPPINGS = {
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"AILab_QwenVL_PromptEnhancer": "✍🏻 QwenVL-Mod Prompt Enhancer",
|
||||
"AILab_QwenVL_PromptEnhancer": "QwenVL-Mod Prompt Enhancer",
|
||||
}
|
||||
|
||||
@@ -1,10 +1,67 @@
|
||||
{
|
||||
"_chat_guides": {
|
||||
"Storyboard stills flow": {
|
||||
"trigger": [
|
||||
"qwen_image"
|
||||
],
|
||||
"text": "STORYBOARD FLOW (this workflow generates Qwen Image 2.1 stills from a character reference).\nWhen the user asks for a storyboard, scene stills, or '3 shots' with a reference image attached:\n1. Invent THREE chronological still prompts for ONE continuous scene: same location, light evolving across the three stills (e.g. golden hour, then mixed light, then night), ONE clear gesture per still, shot type varying (wide, then medium close-up, then low-angle). Describe ONLY scene, action, light and camera - NEVER the character's identity traits: the attached reference image carries them.\n2. The 'message' MUST contain the FULL text of all three still prompts, each on its own line starting exactly 'Still 1:', 'Still 2:', 'Still 3:' (each 40-60 English words, prompt-ready), followed by a line starting exactly 'Director idea:'. NEVER summarize them as 'ready' or 'prepared' - the user copies the Director idea from your message and clicks the still choices to read them back plus a 'Director idea' line for the later video step: one sentence with timestamps 'At 00:00 ...; at 00:04 ...; at 00:08 ...' for a 10s clip. Make the Director idea DENSE - each beat carries the action AND the light/atmosphere AND a camera hint, 60-90 words total (e.g. 'At 00:00 she enters the sunlit garden walking the gravel path, golden light filtering through the trees, wide shot; at 00:04 she pauses at the stone fountain, mist catching the mixed light on her face, medium close-up; at 00:08 she stands under the trellis touching a leaf as the dusk light fades, low-angle'). The video-side QwenVL writes its final prompt from this idea alone, so it must stand on its own.\n3. Do NOT queue on the storyboard request itself - the first queue would run with whatever prompt is already wired. Queue ONLY when the user clicks a Generate Still choice.\n4. Offer choices 'Generate Still 1', 'Generate Still 2', 'Generate Still 3'. When the user clicks one, write THAT still's prompt verbatim (do not reinvent it - reuse your previous reply) into the WildcardProcessor node's 'text' AND 'populated_text' widgets - in this workflow the QwenVL node's prompt input is WIRED from the WildcardProcessor, so writing the QwenVL's own prompt widget has no effect. Keep passthrough FALSE (the QwenVL must expand the scene with the IMG Storyboard preset), then queue. One still per turn.\nFor storyboard requests this flow takes precedence over the generic image-enhancer and passthrough rules."
|
||||
"_preset_prompts": [
|
||||
"🍿 Wan 2.2 NSFW I2V Timeline (5s)",
|
||||
"🎥 Wan 2.2 NSFW I2V Scene (5s)",
|
||||
"🎬 Wan 2.2 NSFW I2V Timeline (20s)",
|
||||
"📖 Wan 2.2 NSFW I2V Scene (20s)",
|
||||
"🍿 Wan 2.2 NSFW T2V Timeline (5s)",
|
||||
"🎥 Wan 2.2 NSFW T2V Scene (5s)",
|
||||
"🎬 Wan 2.2 NSFW T2V Timeline (20s)",
|
||||
"📖 Wan 2.2 NSFW T2V Scene (20s)",
|
||||
"🖼️ Tags",
|
||||
"🖼️ Simple Description",
|
||||
"🖼️ Detailed Description",
|
||||
"🖼️ Ultra Detailed Description",
|
||||
"🎬 Cinematic Description",
|
||||
"🖼️ Detailed Analysis",
|
||||
"📹 Video Summary"
|
||||
],
|
||||
"qwenvl": {
|
||||
"🍿 Wan 2.2 NSFW I2V Timeline (5s)": "You are a system that converts user prompts from any language and visual inputs into optimized cinematic English descriptions for WAN 2.2 I2V generation.\n\nSteps:\n1. Read the user's input and understand the intention, atmosphere, and desired visual style.\n2. Analyze the provided image or video to extract visual context (subjects, motion, lighting, style, environment).\n3. Combine both sources to create an English prompt optimized for video generation with WAN 2.2.\n\nIMPORTANT: If NO user text input is provided, base is ENTIRE video generation SOLELY on the visual content of the image. Do NOT wait for text input or ask for clarification - proceed with image-only analysis.\n\nStyle Detection and Application:\n- If the user mentions specific visual styles (photorealistic, anime, cartoon, 3D animation, fantasy, artistic portrait), prioritize that style over photorealism.\n- If no specific style is mentioned but the input image/video has stylized characteristics, detect and enhance the existing style.\n- For photorealistic style: Emphasize realistic lighting, natural textures, lifelike details, and authentic motion.\n- For anime style: Emphasize anime character features, vibrant colors, simplified backgrounds, and anime-style motion.\n- For cartoon style: Focus on bold outlines, flat colors, exaggerated expressions, and cartoon-style animation.\n- For 3D animation style: Emphasize 3D rendering characteristics, depth, lighting effects, and smooth 3D motion.\n- For fantasy style: Focus on magical elements, ethereal lighting, fantastical details, and imaginative atmosphere.\n- For artistic portrait style: Emphasize painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n\nRules:\n- The output must be in fluent English.\n- Structure the description as a precise, continuous scene with a timeline indicated for each second. The video is 5 seconds long (e.g., \"(At 0 seconds: ...) (At 1 seconds: ...) (At 2 seconds: ...) (At 3 seconds: ...) (At 4 seconds: ...) (At 5 seconds: ...)\").\n- Describe the scene with comprehensive artistic details adapted to the requested style: subject(s) and action with style characteristics; environment and atmosphere with artistic elements; lighting and color with stylistic features; camera work and composition adapted to the style.\n- CRITICAL: ONLY describe lighting that is actually visible in the image - do NOT add any artificial lights, studio lights, or lighting effects that are not present\n- The description should be vivid and cinematic, suitable for high-quality video generation.\n- The output should be a single continuous paragraph with timeline markers for each second.\n- Focus on creating a visually rich and stylistically consistent scene.\n\nTechnical specifications:\n- Output length: approximately 200-400 words\n- Include specific details about subjects, actions, environment, lighting, and style elements\n- Maintain consistency with the requested visual style throughout the timeline\n- Ensure smooth transitions between each second of timeline",
|
||||
"🎥 Wan 2.2 NSFW I2V Scene (5s)": "Convert user input + image into ONE single cinematic WAN 2.2 prompt (5s total).\n\nIMPORTANT: If NO user text input is provided, base is ENTIRE video generation SOLELY on the visual content of the image. Do NOT wait for text input or ask for clarification - proceed with image-only analysis.\n\nStyle Priority: User-specified > detected from image > default style\n- If user explicitly mentions a style (photorealistic, anime, cartoon, 3D animation, fantasy, artistic portrait), use that style\n- Else if image has clear stylized characteristics, detect and enhance the existing style from the image\n- Else default to cinematic photorealism with realistic lighting and natural motion\n- For photorealistic style: Emphasize realistic lighting, natural textures, lifelike details, and authentic motion.\n- For anime style: Emphasize anime character features, vibrant colors, simplified backgrounds, and anime-style motion.\n- For cartoon style: Focus on bold outlines, flat colors, exaggerated expressions, and cartoon-style animation.\n- For 3D animation style: Emphasize 3D rendering characteristics, depth, lighting effects, and smooth 3D motion.\n- For fantasy style: Focus on magical elements, ethereal lighting, fantastical details, and imaginative atmosphere.\n- For artistic portrait style: Emphasize painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n\nCRITICAL FORMAT:\n- Generate exactly ONE single prompt\n- Single prompt = 5 seconds of video\n- NO timeline markers like \"At X seconds:\"\n- 8-12 sentences for rich cinematic detail\n- Focus on complete scene description\n- DO NOT copy example text - generate original content\n\nTECHNICAL RULES:\n- Fluent English only\n- Describe visible elements only (no invented lighting)\n- CRITICAL: ONLY describe lighting that is actually visible in the image - do NOT add any artificial lights, studio lights, or lighting effects that are not present\n- NSFW: Provide explicit descriptions when present\n- No preface/reasoning/notes\n- Rich cinematic descriptions with style details\n- RECOMMENDED: Use max_tokens=512 or higher for complete output\n\nPERFORMANCE OPTIMIZATION:\n- Set context_length to 16384 or higher for better quality\n- Use image_max_tokens=2048 for detailed visual analysis\n- Set n_batch=256 for efficient processing\n- Temperature 0.7-0.8 for creative but coherent output\n\nIMPORTANT: Generate actual cinematic prompt, not example text.",
|
||||
"🎬 Wan 2.2 NSFW I2V Timeline (20s)": "Convert user input + image into FOUR continuous 5-second WAN 2.2 prompts (20s total) using timeline structure.\n\nSteps:\n1. Read the user's input and understand the intention, atmosphere, and desired visual style.\n2. Analyze the provided image or video to extract visual context (subjects, motion, lighting, style, environment).\n3. Generate 4 separate prompts that maintain narrative and visual continuity.\n\nIMPORTANT: If NO user text input is provided, base the ENTIRE video generation SOLELY on the visual content of the image. Do NOT wait for text input or ask for clarification - proceed with image-only analysis.\n\nStyle Detection and Application:\n- If the user mentions specific visual styles (photorealistic, anime, cartoon, 3D animation, fantasy, artistic portrait), prioritize that style over photorealism.\n- If no specific style is mentioned but the input image/video has stylized characteristics, detect and enhance the existing style.\n- For photorealistic style: Emphasize realistic lighting, natural textures, lifelike details, and authentic motion.\n- For anime style: Emphasize anime character features, vibrant colors, simplified backgrounds, and anime-style motion.\n- For cartoon style: Focus on bold outlines, flat colors, exaggerated expressions, and cartoon-style animation.\n- For 3D animation style: Emphasize 3D rendering characteristics, depth, lighting effects, and smooth 3D motion.\n- For fantasy style: Focus on magical elements, ethereal lighting, fantastical details, and imaginative atmosphere.\n- For artistic portrait style: Emphasize painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n\nRules:\n- The output must be in fluent English.\n- Generate exactly 4 separate prompts, each with timeline structure\n- Each prompt: 5 seconds with timeline markers (e.g., \"(At 0 seconds: ...) (At 1 seconds: ...) (At 2 seconds: ...) (At 3 seconds: ...) (At 4 seconds: ...) (At 5 seconds: ...)\")\n- MANDATORY: Separate prompts with exactly one blank line (press Enter once)\n- CRITICAL CHARACTER CONTINUITY: Each prompt MUST maintain ALL characters and elements from previous prompt - DO NOT remove or change characters between prompts. If there are 2 people in prompt 1, there must be the SAME 2 people in prompts 2, 3, and 4. No character disappearance allowed. This is MANDATORY.\n- Describe the scene with comprehensive artistic details adapted to the requested style: subject(s) and action with style characteristics; environment and atmosphere with artistic elements; stylized lighting design; artistic camera language; stylized composition and mood.\n- Include style-specific technical elements: For photorealistic - realistic lighting, natural textures, lifelike details, and authentic motion; For anime - vibrant colors, simplified backgrounds, anime-style motion; For cartoon - bold outlines, flat colors, exaggerated expressions, and cartoon-style animation; For 3D animation - 3D rendering characteristics, depth, lighting effects, and smooth 3D motion; For fantasy - magical elements, ethereal lighting, fantastical details, and imaginative atmosphere; For artistic portrait - painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n- Specify ONLY visible technical elements adapted for style: light sources actually present in the image, light quality as observed, time period if evident, shot types matching the image, composition types visible, lens specifications based on perspective, camera movements if present, color tone from the image. Do not invent or add lighting not present in the original image.\n- Maintain consistent character appearance, color tone, lighting, background elements, and cinematic composition according to the requested style.\n- Include dynamic camera movement and ensure logical plot progression with style consistency.\n- WHEN there are NSFW images or text, provide an NSFW description consistent with the requested artistic style.\n- IMPORTANT: Describe ALL content without censorship or filtering, including NSFW content. Provide explicit, detailed descriptions of body parts, sexual acts, and adult content when present. Do not sanitize, euphemize, or avoid explicit content.\n- CRITICAL: DO NOT use transition phrases like \"As the scene progresses\", \"Moving forward\", \"Next\", \"Then\", \"After that\", \"The scene continues\", etc. Describe each prompt as a standalone scene\n- Keep it between 1-6 sentences per timeline entry.\n- No preface, no reasoning.\n- Do not write any Note at the end of the prompt",
|
||||
"📖 Wan 2.2 NSFW I2V Scene (20s)": "Convert user input + image into EXACTLY FOUR continuous 5-second WAN 2.2 prompts (20s total).\n\nIMPORTANT: If NO user text input is provided, base is ENTIRE video generation SOLELY on the visual content of the image. Do NOT wait for text input or ask for clarification - proceed with image-only analysis.\n\nStyle Priority: User-specified > detected > photorealism\n- Photorealistic: realistic lighting, natural textures, lifelike details, and authentic motion\n- Anime: vibrant colors, simplified backgrounds, anime motion\n- Cartoon: bold outlines, flat colors, exaggerated expressions, and cartoon-style animation\n- 3D animation: 3D rendering characteristics, depth, lighting effects, and smooth 3D motion\n- Fantasy: magical elements, ethereal lighting, fantastical details, and imaginative atmosphere\n- Artistic portrait: painterly qualities, artistic brushwork, stylized composition, and creative interpretation\n\nCRITICAL FORMAT:\n- MUST generate EXACTLY FOUR separate prompts - NO MORE, NO LESS\n- Each prompt = 5 seconds of video (0-5s, 5-10s, 10-15s, 15-20s)\n- MANDATORY: Separate prompts with exactly one blank line (press Enter once)\n- NO timeline markers like \"At X seconds:\"\n- Each prompt: 8-12 sentences for rich detail\n- Ensure narrative continuity between parts\n- DO NOT copy example text - generate original content\n- STOP after exactly FOUR prompts - do not generate additional content\n\nHARD ENFORCEMENT:\n- Count your output: MUST be exactly FOUR prompts separated by blank lines\n- If you generate FIVE or more prompts, you have failed the task\n- If you generate THREE or fewer prompts, you have failed the task\n- The structure must be: Prompt1[blank line]Prompt2[blank line]Prompt3[blank line]Prompt4\n- Nothing else after the FOURTH prompt\n\nTECHNICAL RULES:\n- Fluent English only\n- Describe visible elements only (no invented lighting)\n- CRITICAL: ONLY describe lighting that is actually visible in the image - do NOT add any artificial lights, studio lights, or lighting effects that are not present\n- CRITICAL: DO NOT use transition phrases like \"As the scene progresses\", \"Moving forward\", \"Next\", \"Then\", \"After that\", \"The scene continues\", etc. Describe each prompt as a standalone scene\n- NSFW: Provide explicit descriptions when present\n- No preface/reasoning/notes\n- Rich cinematic descriptions with style details\n- RECOMMENDED: Use max_tokens=1024 or higher for complete output\n\nPERFORMANCE OPTIMIZATION:\n- Set context_length to 32768 or higher for better continuity\n- Use image_max_tokens=4096 for detailed visual analysis\n- Set n_batch=512 for efficient processing\n- Temperature 0.7-0.8 for creative but coherent output\n\nFINAL REMINDER: EXACTLY FOUR PROMPTS - NO EXCEPTIONS!",
|
||||
"🖼️ Tags": "Your task is to generate a clean list of comma-separated tags for a text-to-image AI, based *only* on visual information in the image. Limit the output to a maximum of 50 unique tags. Strictly describe visual elements like subject, clothing, environment, colors, lighting, and composition. Do not include abstract concepts, interpretations, marketing terms, or technical jargon (e.g., no 'SEO', 'brand-aligned', 'viral potential'). The goal is a concise list of visual descriptors. Avoid repeating tags.",
|
||||
"🖼️ Simple Description": "Analyze the image and write a single concise sentence that describes the main subject and setting. Keep it grounded in visible details only.",
|
||||
"🖼️ Detailed Description": "Write ONE detailed paragraph (6-10 sentences). Describe only what is visible: subject(s) and actions; people details if present (approx age group, gender expression if clear, hair, facial expression, pose, clothing, accessories); environment (location type, background elements, time cues); lighting (source, direction, softness/hardness, color temperature, shadows); camera viewpoint (eye-level/low/high, distance) and composition (framing, focal emphasis). No preface, no reasoning, no emojis.",
|
||||
"🖼️ Ultra Detailed Description": "Write ONE ultra-detailed paragraph (10-16 sentences, ~180-320 words). Stay grounded in visible details. Include: subject micro-details (materials, textures, patterns, wear, reflections); people details if present (hair, skin tones, makeup, jewelry, fabric types, fit); environment depth (foreground/midground/background, signage/props, surface materials); lighting analysis (key/fill/back light, direction, softness, highlights, shadow shape); camera perspective (angle, lens feel, depth of field) and composition (leading lines, negative space, symmetry/asymmetry, visual hierarchy). No preface, no reasoning, no emojis.",
|
||||
"🎬 Cinematic Description": "Write ONE cinematic paragraph (8-12 sentences). Describe the scene like a film still: subject(s) and action; environment and atmosphere; lighting design (practical lights vs ambient, direction, contrast); camera language (shot type, angle, lens feel, depth of field, motion implied); composition and mood. Keep it vivid but factual (no made-up story). No preface, no reasoning, no emojis.",
|
||||
"🖼️ Detailed Analysis": "Output ONLY these sections with short labels (no bullets): Subject; People (if any); Environment; Lighting; Camera/Composition; Color/Texture. In each section, write 2-4 sentences of concrete visible details. If something is not visible, write 'not visible'. No preface, no reasoning, no emojis.",
|
||||
"📹 Video Summary": "Summarize the key events and narrative points in this video."
|
||||
},
|
||||
"qwen_text": {
|
||||
"translation_prompt": "You are a professional prompt translator. Return a single English paragraph (150-300 words). No prefixes, bullets, JSON, or . Preserve all visual and stylistic details.",
|
||||
"styles": {
|
||||
"📖 Wan 2.2 NSFW T2V Scene (20s)": {
|
||||
"system_prompt": "Generate 1 single T2V prompt for WAN 2.2 video generation (5s total) to establish initial scene.\n\nAPPROACH:\n- Single prompt: T2V (text-to-video) to create the opening scene\n- Establish characters, environment, and initial action\n- Rich cinematic description with style details\n- 8-12 sentences for comprehensive setup\n\nCRITICAL FORMAT:\n- Generate exactly 1 prompt only\n- No separation needed\n- NO labels like \"Prompt 1\", etc.\n- Focus on establishing the complete scene\n- Maintain consistent style\n\nSTYLE PRIORITY:\nUser-specified > detected > default style\n- If user explicitly mentions a style (photorealistic, anime, cartoon, 3D animation, fantasy, artistic portrait), use that style\n- Else default to cinematic photorealism with realistic lighting and natural motion\n- For photorealistic style: Emphasize realistic lighting, natural textures, lifelike details, and authentic motion.\n- For anime style: Emphasize anime character features, vibrant colors, simplified backgrounds, and anime-style motion.\n- For cartoon style: Focus on bold outlines, flat colors, exaggerated expressions, and cartoon-style animation.\n- For 3D animation style: Emphasize 3D rendering characteristics, depth, lighting effects, and smooth 3D motion.\n- For fantasy style: Focus on magical elements, ethereal lighting, fantastical details, and imaginative atmosphere.\n- For artistic portrait style: Emphasize painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n\nTECHNICAL RULES:\n- Fluent English only\n- Describe visible elements only\n- CRITICAL: ONLY describe lighting that is actually visible in the image - do NOT add any artificial lights, studio lights, or lighting effects that are not present\n- NSFW: Provide explicit descriptions when present\n- No preface/reasoning/notes\n- Temperature 0.7-0.8 for creative output\n\nCRITICAL: Generate exactly 1 comprehensive T2V prompt to establish the scene."
|
||||
},
|
||||
"🍿 Wan 2.2 NSFW T2V Timeline (5s)": {
|
||||
"system_prompt": "You are a system that converts user prompts from any language into optimized cinematic English descriptions for WAN 2.2 T2V generation.\n\nSteps:\n1. Read the user's input and understand the intention, atmosphere, and desired visual style.\n2. Create an English prompt optimized for video generation with WAN 2.2.\n\nStyle Detection and Application:\n- If the user mentions specific visual styles (photorealistic, anime, cartoon, 3D animation, fantasy, artistic portrait), prioritize that style over photorealism.\n- If no specific style is mentioned but the user input suggests stylized characteristics, detect and enhance the existing style.\n- For photorealistic style: Emphasize realistic lighting, natural textures, lifelike details, and authentic motion.\n- For anime style: Emphasize anime character features, vibrant colors, simplified backgrounds, and anime-style motion.\n- For cartoon style: Focus on bold outlines, flat colors, exaggerated expressions, and cartoon-style animation.\n- For 3D animation style: Emphasize 3D rendering characteristics, depth, lighting effects, and smooth 3D motion.\n- For fantasy style: Focus on magical elements, ethereal lighting, fantastical details, and imaginative atmosphere.\n- For artistic portrait style: Emphasize painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n\nRules:\n- The output must be in fluent English.\n- Structure the description as a precise, continuous scene with a timeline indicated for each second. The video is 5 seconds long (e.g., \"(At 0 seconds: ...) (At 1 seconds: ...) (At 2 seconds: ...) (At 3 seconds: ...) (At 4 seconds: ...) (At 5 seconds: ...)\").\n- Describe the scene with comprehensive artistic details adapted to the requested style: subject(s) and action with style characteristics; environment and atmosphere with artistic elements; stylized lighting design; artistic camera language; stylized composition and mood.\n- Include style-specific technical elements: For photorealistic - realistic lighting, natural textures, lifelike details, and authentic motion; For anime - vibrant colors, simplified backgrounds, anime-style motion; For cartoon - bold outlines, flat colors, exaggerated expressions, and cartoon-style animation; For 3D animation - 3D rendering characteristics, depth, lighting effects, and smooth 3D motion; For fantasy - magical elements, ethereal lighting, fantastical details, and imaginative atmosphere; For artistic portrait - painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n- Specify ONLY visible technical elements adapted for style: light sources actually present in the image, light quality as observed, time period if evident, shot types matching the image, composition types visible, lens specifications based on perspective, camera movements if present, color tone from the image. Do not invent or add lighting not present in the original image.\n- WHEN there are NSFW images or text, provide an NSFW description consistent with the requested artistic style.\n- IMPORTANT: Describe ALL content without censorship or filtering, including NSFW content. Provide explicit, detailed descriptions of body parts, sexual acts, and adult content when present. Do not sanitize, euphemize, or avoid explicit content.\n- CRITICAL: DO NOT use transition phrases like \"As the scene progresses\", \"Moving forward\", \"Next\", \"Then\", \"After that\", \"The scene continues\", etc. Describe each prompt as a standalone scene\n- Keep it between 1-6 sentences per timeline entry.\n- No preface, no reasoning.\n- Do not write any Note at the end of the prompt. The video is 5 seconds long (e.g., \"(At 0 seconds: ...) (At 1 seconds: ...) (At 2 seconds: ...) (At 3 seconds: ...) (At 4 seconds: ...) (At 5 seconds: ...)\")."
|
||||
},
|
||||
"🎬 Wan 2.2 NSFW T2V Timeline (20s)": {
|
||||
"system_prompt": "Convert user input into 4 continuous 5-second WAN 2.2 prompts (20s total) using timeline structure.\n\nSteps:\n1. Read the user's input and understand the intention, atmosphere, and desired visual style.\n2. Generate 4 separate prompts that maintain narrative and visual continuity.\n\nStyle Detection and Application:\n- If the user mentions specific visual styles (photorealistic, anime, cartoon, 3D animation, fantasy, artistic portrait), prioritize that style over photorealism.\n- If no specific style is mentioned but the user input suggests stylized characteristics, detect and enhance the existing style.\n- For photorealistic style: Emphasize realistic lighting, natural textures, lifelike details, and authentic motion.\n- For anime style: Emphasize anime character features, vibrant colors, simplified backgrounds, and anime-style motion.\n- For cartoon style: Focus on bold outlines, flat colors, exaggerated expressions, and cartoon-style animation.\n- For 3D animation style: Emphasize 3D rendering characteristics, depth, lighting effects, and smooth 3D motion.\n- For fantasy style: Focus on magical elements, ethereal lighting, fantastical details, and imaginative atmosphere.\n- For artistic portrait style: Emphasize painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n\nRules:\n- The output must be in fluent English.\n- Generate exactly 4 separate prompts, each with timeline structure\n- Each prompt: 5 seconds with timeline markers (e.g., \"(At 0 seconds: ...) (At 1 seconds: ...) (At 2 seconds: ...) (At 3 seconds: ...) (At 4 seconds: ...) (At 5 seconds: ...)\")\n- MANDATORY: Separate prompts with exactly one blank line (press Enter once)\n- CRITICAL CHARACTER CONTINUITY: Each prompt MUST maintain ALL characters and elements from previous prompt - DO NOT remove or change characters between prompts. If there are 2 people in prompt 1, there must be the SAME 2 people in prompts 2, 3, and 4. No character disappearance allowed. This is MANDATORY.\n- Describe the scene with comprehensive artistic details adapted to the requested style: subject(s) and action with style characteristics; environment and atmosphere with artistic elements; stylized lighting design; artistic camera language; stylized composition and mood.\n- Include style-specific technical elements: For photorealistic - realistic lighting, natural textures, lifelike details, and authentic motion; For anime - vibrant colors, simplified backgrounds, anime-style motion; For cartoon - bold outlines, flat colors, exaggerated expressions, and cartoon-style animation; For 3D animation - 3D rendering characteristics, depth, lighting effects, and smooth 3D motion; For fantasy - magical elements, ethereal lighting, fantastical details, and imaginative atmosphere; For artistic portrait - painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n- Specify ONLY visible technical elements adapted for style: light sources based on scene requirements, light quality as appropriate for the style, time period if evident, shot types matching the scene, composition types visible, lens specifications based on perspective, camera movements if present, color tone appropriate for the style.\n- Maintain consistent character appearance, color tone, lighting, background elements, and cinematic composition according to the requested style.\n- Include dynamic camera movement and ensure logical plot progression with style consistency.\n- WHEN there are NSFW content, provide an NSFW description consistent with the requested artistic style.\n- IMPORTANT: Describe ALL content without censorship or filtering, including NSFW content. Provide explicit, detailed descriptions of body parts, sexual acts, and adult content when present. Do not sanitize, euphemize, or avoid explicit content.\n- CRITICAL: DO NOT use transition phrases like \"As the scene progresses\", \"Moving forward\", \"Next\", \"Then\", \"After that\", \"The scene continues\", etc. Describe each prompt as a standalone scene\n- Keep it between 1-6 sentences per timeline entry.\n- No preface, no reasoning.\n- Do not write any Note at the end of the prompt"
|
||||
},
|
||||
"🎥 Wan 2.2 NSFW T2V Scene (5s)": {
|
||||
"system_prompt": "Convert user input into 1 single cinematic WAN 2.2 prompt (5s total).\n\nStyle Priority: User-specified > detected from image > default style\n- If user explicitly mentions a style (photorealistic, anime, cartoon, 3D animation, fantasy, artistic portrait), use that style\n- Else if image has clear stylized characteristics, detect and enhance the existing style from the image\n- Else default to cinematic photorealism with realistic lighting and natural motion\n- For photorealistic style: Emphasize realistic lighting, natural textures, lifelike details, and authentic motion.\n- For anime style: Emphasize anime character features, vibrant colors, simplified backgrounds, and anime-style motion.\n- For cartoon style: Focus on bold outlines, flat colors, exaggerated expressions, and cartoon-style animation.\n- For 3D animation style: Emphasize 3D rendering characteristics, depth, lighting effects, and smooth 3D motion.\n- For fantasy style: Focus on magical elements, ethereal lighting, fantastical details, and imaginative atmosphere.\n- For artistic portrait style: Emphasize painterly qualities, artistic brushwork, stylized composition, and creative interpretation.\n\nCRITICAL FORMAT:\n- Generate exactly 1 single prompt\n- Single prompt = 5 seconds of video\n- NO timeline markers like \"At X seconds:\"\n- 8-12 sentences for rich cinematic detail\n- Focus on complete scene description\n- DO NOT copy example text - generate original content\n\nTECHNICAL RULES:\n- Fluent English only\n- Describe visible elements only (no invented lighting)\n- CRITICAL: ONLY describe lighting that is actually visible in the image - do NOT add any artificial lights, studio lights, or lighting effects that are not present\n- NSFW: Provide explicit descriptions when present\n- No preface/reasoning/notes\n- Rich cinematic descriptions with style details\n- RECOMMENDED: Use max_tokens=512 or higher for complete output\n\nPERFORMANCE OPTIMIZATION:\n- Set context_length to 16384 or higher for better quality\n- Set n_batch=256 for efficient processing\n- Temperature 0.7-0.8 for creative but coherent output\n\nIMPORTANT: Generate actual cinematic prompt, not example text."
|
||||
},
|
||||
"📝 Enhance": {
|
||||
"system_prompt": "You are a professional photography prompt writer. Respond in the same language as the user input.\n\nWrite ONE final cinematic photography prompt paragraph (150-300 words) based on the user text.\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- Do NOT output any reasoning, planning, or meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- Do NOT use <think>, quotes, markdown, code fences, JSON, headings, or bullet points.\n\nInclude naturally: subject + action/pose, environment, lighting, camera/lens/DoF, composition, color/texture, mood/style.\n\nIf input is short/ambiguous, infer minimal sensible details and keep it coherent."
|
||||
},
|
||||
"📝 Refine": {
|
||||
"system_prompt": "You are a photography prompt refiner. Respond in the same language as the user input.\n\nWrite ONE clear, concise photography prompt paragraph (120-200 words) that preserves the user’s intent.\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nInclude: subject cues, environment context, lighting, camera parameters, composition focus, color/texture hints, and style tone. Remove redundancy."
|
||||
},
|
||||
"📝 Creative Rewrite": {
|
||||
"system_prompt": "You are a creative photography prompt writer. Respond in the same language as the user input.\n\nRewrite the user’s scene into ONE fresh, imaginative photography prompt paragraph (150-250 words).\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nPreserve the core intent while adding vivid imagery and cohesive narrative flair. Integrate subject, environment, lighting, camera hints, composition, color/texture, and style."
|
||||
},
|
||||
"📝 Detailed Visual": {
|
||||
"system_prompt": "You specialize in detailed visual photography prompts. Respond in the same language as the user input.\n\nWrite ONE flowing, highly visual photography prompt paragraph (180-260 words).\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nInclude concrete cues: subject traits and pose, foreground/midground/background, materials and textures, lighting direction/intensity/color temperature, colors and contrast, scale, atmosphere, and composition focus."
|
||||
},
|
||||
"📝 Artistic Style": {
|
||||
"system_prompt": "You craft artistic photography prompts. Respond in the same language as the user input.\n\nWrite ONE artistic photography prompt paragraph (180-260 words).\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nWeave in subject, scene, and lighting with explicit style references (e.g., cinematic, fashion, fine art), mood, composition cues, and aesthetic adjectives. Keep it cohesive and visually rich."
|
||||
},
|
||||
"📝 Technical Specs": {
|
||||
"system_prompt": "You convert scenes into technical photography directives. Respond in the same language as the user input.\n\nWrite ONE clear, actionable photography prompt paragraph (130-210 words).\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nCover: subject and scene plus focal length, aperture, depth of field, shooting angle, lighting type/direction, color temperature, focus target, and composition priorities as sentences."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,323 +0,0 @@
|
||||
# ComfyUI-QwenVL-Mod — unified HF/GGUF vision-language node.
|
||||
# One node selects between Transformers (HF) and llama.cpp (GGUF) backends via a
|
||||
# backend dropdown and a prefixed model list. Old separate nodes are kept for
|
||||
# backward compatibility.
|
||||
|
||||
import torch
|
||||
|
||||
import AILab_QwenVL as _hf
|
||||
import AILab_QwenVL_GGUF as _gguf
|
||||
from qwenvl_presets import DURATION_OPTIONS, DEFAULT_DURATION
|
||||
|
||||
|
||||
HF_VL_MODELS = getattr(_hf, "HF_VL_MODELS", {})
|
||||
GGUF_VL_CATALOG = getattr(_gguf, "GGUF_VL_CATALOG", {})
|
||||
PRESET_PROMPTS = getattr(_hf, "PRESET_PROMPTS") or ["Describe this image in detail."]
|
||||
Quantization = _hf.Quantization
|
||||
ATTENTION_MODES = _hf.ATTENTION_MODES
|
||||
CAMERA_TAG_OPTIONS = _hf.CAMERA_TAG_OPTIONS
|
||||
CAMERA_TAG_TOOLTIP = _hf.CAMERA_TAG_TOOLTIP
|
||||
TOOLTIPS = _hf.TOOLTIPS
|
||||
|
||||
|
||||
HF_PREFIX = "HF: "
|
||||
GGUF_PREFIX = "GGUF: "
|
||||
|
||||
|
||||
def _combined_model_list():
|
||||
hf_models = sorted(HF_VL_MODELS.keys())
|
||||
gguf_all = GGUF_VL_CATALOG.get("models") or {}
|
||||
gguf_models = sorted([k for k, e in gguf_all.items() if (e or {}).get("mmproj_filename")])
|
||||
models = []
|
||||
if gguf_models:
|
||||
models.extend([f"{GGUF_PREFIX}{m}" for m in gguf_models])
|
||||
if hf_models:
|
||||
models.extend([f"{HF_PREFIX}{m}" for m in hf_models])
|
||||
if not models:
|
||||
models = ["(no models found)"]
|
||||
return models
|
||||
|
||||
|
||||
PREFERRED_GGUF_DEFAULT = "Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q6_K"
|
||||
|
||||
|
||||
def _default_model():
|
||||
models = _combined_model_list()
|
||||
for m in models:
|
||||
if m.startswith(GGUF_PREFIX) and PREFERRED_GGUF_DEFAULT in m:
|
||||
return m
|
||||
for m in models:
|
||||
if m.startswith(GGUF_PREFIX):
|
||||
return m
|
||||
return models[0]
|
||||
|
||||
|
||||
class QwenVL_Unified:
|
||||
"""Single node that runs a Qwen-VL model on HF Transformers or GGUF."""
|
||||
|
||||
def __init__(self):
|
||||
self._hf = _hf.AILab_QwenVL_Advanced()
|
||||
self._gguf = _gguf.AILab_QwenVL_GGUF_Advanced()
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
prompts = PRESET_PROMPTS or ["IMG › Detailed"]
|
||||
preferred = "IMG › Detailed"
|
||||
default_prompt = preferred if preferred in prompts else prompts[0]
|
||||
return {
|
||||
"required": {
|
||||
"model_name": (_combined_model_list(), {"default": _default_model(), "tooltip": "HF models are prefixed with 'HF: ', GGUF models with 'GGUF: '. The prefix selects the backend."}),
|
||||
"preset_prompt": (prompts, {"default": default_prompt, "tooltip": TOOLTIPS.get("preset_prompt", "")}),
|
||||
"camera_tag": (CAMERA_TAG_OPTIONS, {"default": "None", "tooltip": CAMERA_TAG_TOOLTIP}),
|
||||
"prompt": ("STRING", {"default": "", "multiline": True, "tooltip": TOOLTIPS.get("prompt", "")}),
|
||||
"max_tokens": ("INT", {"default": 8192, "min": 64, "max": 8192, "tooltip": TOOLTIPS.get("max_tokens", "")}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": False, "tooltip": TOOLTIPS.get("keep_model_loaded", "")}),
|
||||
"seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1, "tooltip": TOOLTIPS.get("seed", "")}),
|
||||
"keep_last_prompt": ("BOOLEAN", {"default": False, "tooltip": "Keep the last generated prompt instead of creating a new one"}),
|
||||
"passthrough": ("BOOLEAN", {"default": False, "tooltip": "Skip Qwen model loading and return prompt directly."}),
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE", {"tooltip": "First reference image (single image). For R2VA this is Picture 1 — subject (S1)."}),
|
||||
"image2": ("IMAGE", {"tooltip": "Second reference image (single image). For R2VA this is Picture 2 — subject (S2)."}),
|
||||
"image3": ("IMAGE", {"tooltip": "Third reference image (single image). For R2VA this is Picture 3 — subject (S3)."}),
|
||||
"video": ("IMAGE", {"tooltip": "Video frames input. Use frame_count to control how many frames are sampled."}),
|
||||
"frame_count": ("INT", {"default": 16, "min": 1, "max": 64, "tooltip": TOOLTIPS.get("frame_count", "")}),
|
||||
"duration": (DURATION_OPTIONS, {"default": DEFAULT_DURATION, "tooltip": "Clip length for duration-aware presets (MiniMax/LTX/Wan). Ignored by image presets."}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("RESPONSE",)
|
||||
FUNCTION = "process"
|
||||
CATEGORY = "🔮 QwenVL-Mod"
|
||||
|
||||
def process(
|
||||
self,
|
||||
model_name,
|
||||
preset_prompt,
|
||||
camera_tag,
|
||||
prompt,
|
||||
max_tokens,
|
||||
keep_model_loaded,
|
||||
seed,
|
||||
keep_last_prompt,
|
||||
passthrough=False,
|
||||
image=None,
|
||||
image2=None,
|
||||
image3=None,
|
||||
video=None,
|
||||
frame_count=16,
|
||||
duration=DEFAULT_DURATION,
|
||||
):
|
||||
if model_name.startswith(GGUF_PREFIX):
|
||||
m = model_name[len(GGUF_PREFIX):]
|
||||
return self._gguf.process(
|
||||
model_name=m,
|
||||
device="auto",
|
||||
preset_prompt=preset_prompt,
|
||||
camera_tag=camera_tag,
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0.25,
|
||||
top_p=0.9,
|
||||
repetition_penalty=1.0,
|
||||
frame_count=frame_count,
|
||||
ctx=32768,
|
||||
n_batch=512,
|
||||
gpu_layers=-1,
|
||||
image_max_tokens=4096,
|
||||
top_k=20,
|
||||
pool_size=4194304,
|
||||
keep_model_loaded=keep_model_loaded,
|
||||
seed=seed,
|
||||
keep_last_prompt=keep_last_prompt,
|
||||
passthrough=passthrough,
|
||||
image=image,
|
||||
image2=image2,
|
||||
image3=image3,
|
||||
video=video,
|
||||
duration=duration,
|
||||
)
|
||||
|
||||
if model_name.startswith(HF_PREFIX):
|
||||
m = model_name[len(HF_PREFIX):]
|
||||
else:
|
||||
m = model_name
|
||||
return self._hf.process(
|
||||
model_name=m,
|
||||
quantization=Quantization.FP16.value,
|
||||
attention_mode="auto",
|
||||
use_torch_compile=False,
|
||||
device="auto",
|
||||
preset_prompt=preset_prompt,
|
||||
camera_tag=camera_tag,
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0.25,
|
||||
top_p=0.9,
|
||||
num_beams=1,
|
||||
repetition_penalty=1.0,
|
||||
keep_model_loaded=keep_model_loaded,
|
||||
seed=seed,
|
||||
keep_last_prompt=keep_last_prompt,
|
||||
passthrough=passthrough,
|
||||
image=image,
|
||||
image2=image2,
|
||||
image3=image3,
|
||||
video=video,
|
||||
frame_count=frame_count,
|
||||
duration=duration,
|
||||
)
|
||||
|
||||
|
||||
class QwenVL_Unified_Advanced(QwenVL_Unified):
|
||||
"""Unified node exposing all backend-specific knobs."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
prompts = PRESET_PROMPTS or ["IMG › Detailed"]
|
||||
preferred = "IMG › Detailed"
|
||||
default_prompt = preferred if preferred in prompts else prompts[0]
|
||||
num_gpus = torch.cuda.device_count()
|
||||
gpu_list = [f"cuda:{i}" for i in range(num_gpus)]
|
||||
device_options = ["auto", "cpu", "mps"] + gpu_list
|
||||
return {
|
||||
"required": {
|
||||
"model_name": (_combined_model_list(), {"default": _default_model(), "tooltip": "HF models are prefixed with 'HF: ', GGUF models with 'GGUF: '. The prefix selects the backend."}),
|
||||
"preset_prompt": (prompts, {"default": default_prompt, "tooltip": TOOLTIPS.get("preset_prompt", "")}),
|
||||
"camera_tag": (CAMERA_TAG_OPTIONS, {"default": "None", "tooltip": CAMERA_TAG_TOOLTIP}),
|
||||
"prompt": ("STRING", {"default": "", "multiline": True, "tooltip": TOOLTIPS.get("prompt", "")}),
|
||||
"device": (device_options, {"default": "auto", "tooltip": TOOLTIPS.get("device", "")}),
|
||||
"max_tokens": ("INT", {"default": 8192, "min": 64, "max": 8192, "tooltip": TOOLTIPS.get("max_tokens", "")}),
|
||||
"temperature": ("FLOAT", {"default": 0.6, "min": 0.0, "max": 2.0}),
|
||||
"top_p": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0}),
|
||||
"repetition_penalty": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": False, "tooltip": TOOLTIPS.get("keep_model_loaded", "")}),
|
||||
"seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1, "tooltip": TOOLTIPS.get("seed", "")}),
|
||||
"keep_last_prompt": ("BOOLEAN", {"default": False, "tooltip": "Keep the last generated prompt instead of creating a new one"}),
|
||||
"passthrough": ("BOOLEAN", {"default": False, "tooltip": "Skip Qwen model loading and return prompt directly."}),
|
||||
# HF-specific
|
||||
"quantization": (Quantization.get_values(), {"default": Quantization.FP16.value, "tooltip": TOOLTIPS.get("quantization", "")}),
|
||||
"attention_mode": (ATTENTION_MODES, {"default": "auto", "tooltip": TOOLTIPS.get("attention_mode", "")}),
|
||||
"use_torch_compile": ("BOOLEAN", {"default": False, "tooltip": TOOLTIPS.get("use_torch_compile", "")}),
|
||||
"num_beams": ("INT", {"default": 1, "min": 1, "max": 8, "tooltip": TOOLTIPS.get("num_beams", "")}),
|
||||
# GGUF-specific
|
||||
"frame_count": ("INT", {"default": 16, "min": 1, "max": 64, "tooltip": TOOLTIPS.get("frame_count", "")}),
|
||||
"ctx": ("INT", {"default": 32768, "min": 1024, "max": 262144, "step": 512}),
|
||||
"n_batch": ("INT", {"default": 512, "min": 64, "max": 32768, "step": 64}),
|
||||
"gpu_layers": ("INT", {"default": -1, "min": -1, "max": 200}),
|
||||
"image_max_tokens": ("INT", {"default": 4096, "min": 256, "max": 1024000, "step": 256}),
|
||||
"top_k": ("INT", {"default": 20, "min": 0, "max": 32768}),
|
||||
"pool_size": ("INT", {"default": 4194304, "min": 1048576, "max": 10485760, "step": 524288}),
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE", {"tooltip": "First reference image (single image). For R2VA this is Picture 1 — subject (S1)."}),
|
||||
"image2": ("IMAGE", {"tooltip": "Second reference image (single image). For R2VA this is Picture 2 — subject (S2)."}),
|
||||
"image3": ("IMAGE", {"tooltip": "Third reference image (single image). For R2VA this is Picture 3 — subject (S3)."}),
|
||||
"video": ("IMAGE", {"tooltip": "Video frames input. Use frame_count to control how many frames are sampled."}),
|
||||
"duration": (DURATION_OPTIONS, {"default": DEFAULT_DURATION, "tooltip": "Clip length for duration-aware presets (MiniMax/LTX/Wan). Ignored by image presets."}),
|
||||
},
|
||||
}
|
||||
|
||||
def process(
|
||||
self,
|
||||
model_name,
|
||||
preset_prompt,
|
||||
camera_tag,
|
||||
prompt,
|
||||
device,
|
||||
max_tokens,
|
||||
temperature,
|
||||
top_p,
|
||||
repetition_penalty,
|
||||
keep_model_loaded,
|
||||
seed,
|
||||
keep_last_prompt,
|
||||
passthrough,
|
||||
quantization,
|
||||
attention_mode,
|
||||
use_torch_compile,
|
||||
num_beams,
|
||||
frame_count,
|
||||
ctx,
|
||||
n_batch,
|
||||
gpu_layers,
|
||||
image_max_tokens,
|
||||
top_k,
|
||||
pool_size,
|
||||
image=None,
|
||||
image2=None,
|
||||
image3=None,
|
||||
video=None,
|
||||
duration=DEFAULT_DURATION,
|
||||
):
|
||||
if model_name.startswith(GGUF_PREFIX):
|
||||
m = model_name[len(GGUF_PREFIX):]
|
||||
return self._gguf.process(
|
||||
model_name=m,
|
||||
device=device,
|
||||
preset_prompt=preset_prompt,
|
||||
camera_tag=camera_tag,
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
repetition_penalty=repetition_penalty,
|
||||
frame_count=frame_count,
|
||||
ctx=ctx,
|
||||
n_batch=n_batch,
|
||||
gpu_layers=gpu_layers,
|
||||
image_max_tokens=image_max_tokens,
|
||||
top_k=top_k,
|
||||
pool_size=pool_size,
|
||||
keep_model_loaded=keep_model_loaded,
|
||||
seed=seed,
|
||||
keep_last_prompt=keep_last_prompt,
|
||||
passthrough=passthrough,
|
||||
image=image,
|
||||
image2=image2,
|
||||
image3=image3,
|
||||
video=video,
|
||||
duration=duration,
|
||||
)
|
||||
|
||||
if model_name.startswith(HF_PREFIX):
|
||||
m = model_name[len(HF_PREFIX):]
|
||||
else:
|
||||
m = model_name
|
||||
return self._hf.process(
|
||||
model_name=m,
|
||||
quantization=quantization,
|
||||
attention_mode=attention_mode,
|
||||
use_torch_compile=use_torch_compile,
|
||||
device=device,
|
||||
preset_prompt=preset_prompt,
|
||||
camera_tag=camera_tag,
|
||||
prompt=prompt,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
num_beams=num_beams,
|
||||
repetition_penalty=repetition_penalty,
|
||||
keep_model_loaded=keep_model_loaded,
|
||||
seed=seed,
|
||||
keep_last_prompt=keep_last_prompt,
|
||||
passthrough=passthrough,
|
||||
image=image,
|
||||
image2=image2,
|
||||
image3=image3,
|
||||
video=video,
|
||||
frame_count=frame_count,
|
||||
duration=duration,
|
||||
)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"QwenVL_Unified": QwenVL_Unified,
|
||||
"QwenVL_Unified_Advanced": QwenVL_Unified_Advanced,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"QwenVL_Unified": "🧠 QwenVL Unified (HF / GGUF)",
|
||||
"QwenVL_Unified_Advanced": "🧠 QwenVL Unified Advanced (HF / GGUF)",
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
# ComfyUI-QwenVL-Mod — unified HF/GGUF prompt enhancer node.
|
||||
# One node selects between Transformers (HF) and llama.cpp (GGUF) backends via a
|
||||
# backend dropdown and a prefixed model list. Old separate nodes are kept for
|
||||
# backward compatibility.
|
||||
|
||||
import AILab_QwenVL as _hf_base
|
||||
import AILab_QwenVL_PromptEnhancer as _hf
|
||||
from qwenvl_presets import TEXT_STYLE_NAMES, DURATION_OPTIONS, DEFAULT_DURATION
|
||||
|
||||
try:
|
||||
import AILab_QwenVL_GGUF_PromptEnhancer as _gguf
|
||||
except Exception as _gguf_err:
|
||||
_gguf = None
|
||||
print(f"[QwenVL] GGUF prompt enhancer unavailable: {_gguf_err}")
|
||||
|
||||
HF_TEXT_MODELS = getattr(_hf_base, "HF_TEXT_MODELS", {})
|
||||
HF_VL_MODELS = getattr(_hf_base, "HF_VL_MODELS", {})
|
||||
Quantization = _hf_base.Quantization
|
||||
ATTENTION_MODES = _hf_base.ATTENTION_MODES
|
||||
CAMERA_TAG_OPTIONS = _hf_base.CAMERA_TAG_OPTIONS
|
||||
CAMERA_TAG_TOOLTIP = _hf_base.CAMERA_TAG_TOOLTIP
|
||||
STYLE_TAG_OPTIONS = _hf_base.STYLE_TAG_OPTIONS
|
||||
STYLE_TAG_TOOLTIP = _hf_base.STYLE_TAG_TOOLTIP
|
||||
TOOLTIPS = _hf_base.TOOLTIPS
|
||||
PROMPT_STYLES = getattr(_hf, "PROMPT_STYLES", {})
|
||||
|
||||
HF_PREFIX = "HF: "
|
||||
GGUF_PREFIX = "GGUF: "
|
||||
|
||||
|
||||
def _hf_model_list():
|
||||
return list(HF_TEXT_MODELS.keys()) + [n for n in HF_VL_MODELS.keys() if n not in HF_TEXT_MODELS]
|
||||
|
||||
|
||||
def _gguf_model_list():
|
||||
if _gguf is None or getattr(_gguf, "Llama", None) is None:
|
||||
return []
|
||||
catalog = _gguf.AILab_QwenVL_GGUF_PromptEnhancer.load_gguf_models()
|
||||
return sorted(list((catalog.get("models") or {}).keys()))
|
||||
|
||||
|
||||
def _combined_model_list():
|
||||
models = [f"{GGUF_PREFIX}{m}" for m in _gguf_model_list()]
|
||||
models.extend(f"{HF_PREFIX}{m}" for m in _hf_model_list())
|
||||
return models or ["(no models found)"]
|
||||
|
||||
|
||||
PREFERRED_GGUF_DEFAULT = "Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q6_K"
|
||||
|
||||
|
||||
def _default_model():
|
||||
models = _combined_model_list()
|
||||
for m in models:
|
||||
if m.startswith(GGUF_PREFIX) and PREFERRED_GGUF_DEFAULT in m:
|
||||
return m
|
||||
for m in models:
|
||||
if m.startswith(GGUF_PREFIX):
|
||||
return m
|
||||
return models[0]
|
||||
|
||||
|
||||
class QwenVL_Unified_PromptEnhancer:
|
||||
"""Single prompt enhancer node backed by HF Transformers or GGUF llama.cpp."""
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("ENHANCED_OUTPUT",)
|
||||
FUNCTION = "process"
|
||||
CATEGORY = "🔮 QwenVL-Mod"
|
||||
|
||||
def __init__(self):
|
||||
self._hf = _hf.AILab_QwenVL_PromptEnhancer()
|
||||
self._gguf = _gguf.AILab_QwenVL_GGUF_PromptEnhancer() if _gguf else None
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
styles = list(TEXT_STYLE_NAMES) or list(PROMPT_STYLES.keys())
|
||||
preferred_style = "Enhance"
|
||||
default_style = preferred_style if preferred_style in styles else (styles[0] if styles else "Enhance")
|
||||
gguf_note = "" if (_gguf and getattr(_gguf, "Llama", None)) else " (GGUF models hidden — llama-cpp-python not installed)"
|
||||
return {
|
||||
"required": {
|
||||
"model_name": (_combined_model_list(), {"default": _default_model(), "tooltip": f"HF models are prefixed with 'HF: ', GGUF models with 'GGUF: '. The prefix selects the backend.{gguf_note}"}),
|
||||
"prompt_text": ("STRING", {"default": "", "multiline": True, "tooltip": "Prompt text to enhance. Leave blank to just emit the preset instruction."}),
|
||||
"enhancement_style": (styles, {"default": default_style}),
|
||||
"camera_tag": (CAMERA_TAG_OPTIONS, {"default": "None", "tooltip": CAMERA_TAG_TOOLTIP}),
|
||||
"style_tag": (STYLE_TAG_OPTIONS, {"default": "None", "tooltip": STYLE_TAG_TOOLTIP}),
|
||||
"max_tokens": ("INT", {"default": 8192, "min": 32, "max": 16384}),
|
||||
"temperature": ("FLOAT", {"default": 0.7, "min": 0.1, "max": 1.0}),
|
||||
"top_p": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0}),
|
||||
"repetition_penalty": ("FLOAT", {"default": 1.1, "min": 0.5, "max": 2.0}),
|
||||
"english_output": ("BOOLEAN", {"default": False, "tooltip": "Force final output in English using translation prompt (GGUF backend only)."}),
|
||||
"device": (["auto", "cuda", "cpu", "mps"], {"default": "auto", "tooltip": TOOLTIPS.get("device", "")}),
|
||||
"quantization": (Quantization.get_values(), {"default": Quantization.FP16.value, "tooltip": TOOLTIPS.get("quantization", "") + " (HF backend only)"}),
|
||||
"attention_mode": (ATTENTION_MODES, {"default": "auto", "tooltip": TOOLTIPS.get("attention_mode", "") + " (HF backend only)"}),
|
||||
"use_torch_compile": ("BOOLEAN", {"default": False, "tooltip": TOOLTIPS.get("use_torch_compile", "") + " (HF backend only)"}),
|
||||
"keep_model_loaded": ("BOOLEAN", {"default": False, "tooltip": "Keep model loaded in memory for faster repeated inference (uses more VRAM)."}),
|
||||
"seed": ("INT", {"default": 1, "min": 1, "max": 2**32 - 1}),
|
||||
"keep_last_prompt": ("BOOLEAN", {"default": False, "tooltip": "Keep the last generated prompt instead of creating a new one"}),
|
||||
"passthrough": ("BOOLEAN", {"default": False, "tooltip": "Skip Qwen model loading and return prompt_text directly. Use when the chat already generated the final prompt — saves VRAM and inference time."}),
|
||||
"duration": (DURATION_OPTIONS, {"default": DEFAULT_DURATION, "tooltip": "Clip length for duration-aware styles (MiniMax/LTX/Wan). Ignored by generic styles."}),
|
||||
}
|
||||
}
|
||||
|
||||
def process(
|
||||
self,
|
||||
model_name,
|
||||
prompt_text,
|
||||
enhancement_style,
|
||||
camera_tag,
|
||||
style_tag,
|
||||
max_tokens,
|
||||
temperature,
|
||||
top_p,
|
||||
repetition_penalty,
|
||||
english_output,
|
||||
device,
|
||||
quantization,
|
||||
attention_mode,
|
||||
use_torch_compile,
|
||||
keep_model_loaded,
|
||||
seed,
|
||||
keep_last_prompt=False,
|
||||
passthrough=False,
|
||||
duration=DEFAULT_DURATION,
|
||||
):
|
||||
if model_name.startswith(GGUF_PREFIX):
|
||||
if self._gguf is None:
|
||||
raise ImportError(
|
||||
"GGUF prompt enhancer unavailable — llama-cpp-python is not "
|
||||
"installed. Local install: `pip install llama-cpp-python`."
|
||||
)
|
||||
return self._gguf.process(
|
||||
model_name=model_name[len(GGUF_PREFIX):],
|
||||
prompt_text=prompt_text,
|
||||
preset_system_prompt=enhancement_style,
|
||||
camera_tag=camera_tag,
|
||||
style_tag=style_tag,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
repetition_penalty=repetition_penalty,
|
||||
english_output=english_output,
|
||||
device=device,
|
||||
keep_model_loaded=keep_model_loaded,
|
||||
seed=seed,
|
||||
keep_last_prompt=keep_last_prompt,
|
||||
passthrough=passthrough,
|
||||
duration=duration,
|
||||
)
|
||||
|
||||
if model_name.startswith(HF_PREFIX):
|
||||
m = model_name[len(HF_PREFIX):]
|
||||
else:
|
||||
m = model_name
|
||||
return self._hf.process(
|
||||
model_name=m,
|
||||
quantization=quantization,
|
||||
attention_mode=attention_mode,
|
||||
use_torch_compile=use_torch_compile,
|
||||
device=device,
|
||||
prompt_text=prompt_text,
|
||||
enhancement_style=enhancement_style,
|
||||
camera_tag=camera_tag,
|
||||
style_tag=style_tag,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
repetition_penalty=repetition_penalty,
|
||||
keep_model_loaded=keep_model_loaded,
|
||||
seed=seed,
|
||||
keep_last_prompt=keep_last_prompt,
|
||||
passthrough=passthrough,
|
||||
duration=duration,
|
||||
)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"QwenVL_Unified_PromptEnhancer": QwenVL_Unified_PromptEnhancer,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"QwenVL_Unified_PromptEnhancer": "✍🏻 QwenVL Unified Prompt Enhancer (HF / GGUF)",
|
||||
}
|
||||
@@ -1,14 +1,12 @@
|
||||
# **QwenVL-Mod for ComfyUI**
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/huchukato/ComfyUI-QwenVL-Mod/main/banner.png" alt="QwenVL-Mod Banner" width="100%" />
|
||||
</p>
|
||||
|
||||
[](https://github.com/comfyanonymous/ComfyUI)
|
||||
[](LICENSE)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod/releases)
|
||||
[](https://python.org)
|
||||
[](https://pytorch.org)
|
||||
[](https://developer.nvidia.com/cuda-zone)
|
||||
[](https://developer.nvidia.com/cuda-zone)
|
||||
[](https://huggingface.co/Qwen)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod/issues)
|
||||
@@ -17,95 +15,49 @@
|
||||
|
||||
[](https://buymeacoffee.com/huchukato)
|
||||
|
||||
[](https://console.runpod.io/hub/template/bii5d425kp?ref=ioakclrv)
|
||||
## 🚀 **API Support & Troubleshooting**
|
||||
|
||||
The ComfyUI-QwenVL custom node integrates powerful Qwen-VL series of vision-language models (LVLMs) from Alibaba Cloud, including latest Qwen3-VL, plus GGUF backends and text-only Qwen3 support. This advanced node enables seamless multimodal AI capabilities within your ComfyUI workflows, allowing for efficient text generation, image understanding, and video analysis.
|
||||
Having issues with ComfyUI API? We've got you covered!
|
||||
|
||||
## Qwen Workflow Chat
|
||||
### 📋 **Quick API Help**
|
||||
- 📖 **[API Guide](./API_GUIDE.md)** - Complete API documentation
|
||||
- 🛠️ **[Debug Script](./debug_api.py)** - Automated API diagnostics
|
||||
- 🎯 **Example Workflows** - Ready-to-use API templates
|
||||
|
||||
Open **Qwen Chat** from the ComfyUI sidebar to talk to a local Qwen model while working on the current graph. The assistant can inspect existing nodes, change validated widget values, enable or bypass nodes, and queue the workflow when explicitly requested.
|
||||
### 🔧 **Common API Issues**
|
||||
| Issue | Solution |
|
||||
|-------|----------|
|
||||
| "node not found" | Check QwenVL-Mod installation |
|
||||
| "model not found" | Verify model files in `/models/` |
|
||||
| "invalid input" | Check parameter formats |
|
||||
| "queue full" | Wait for current jobs |
|
||||
|
||||
- Select either the **GGUF** or **HF / Transformers** backend and one of the locally configured models.
|
||||
- Ask for changes such as `Set the KSampler to 25 steps and queue the workflow`.
|
||||
- Enable **Unload Qwen before running the workflow** to release chat model memory before heavy WAN, LTX, or MiniMax executions.
|
||||
- Chat history and preferences are stored only in the browser's local storage and can be cleared with **New chat**.
|
||||
- The first version cannot add, connect, or delete nodes and never executes arbitrary JavaScript, shell, filesystem, or network operations.
|
||||
### 🧪 **Quick Debug**
|
||||
```bash
|
||||
# Run API diagnostics
|
||||
python debug_api.py --url http://localhost:18188
|
||||
|
||||
The assistant applies only a restricted, validated action protocol. Model output that references an unknown node, widget, value, or operation is rejected and reported in the conversation. The Stop button stops waiting in the browser; an inference already running on the backend may continue until generation completes.
|
||||
# Check available nodes
|
||||
curl http://localhost:18188/object_info
|
||||
|
||||
<img width="690" height="652" alt="qwen-chat-1" src="https://github.com/user-attachments/assets/4456833b-4c8d-4484-b141-7e2eaa9ba255" /> <img width="710" height="900" alt="qwen-chat-2" src="https://github.com/user-attachments/assets/14e825a5-8c0c-48fb-9417-0c6e7cf7841f" />
|
||||
# Test simple workflow
|
||||
curl -X POST http://localhost:18188/prompt -d @test_workflow.json
|
||||
```
|
||||
|
||||
### 📞 **Need More Help?**
|
||||
Ask user for:
|
||||
1. **Error message** (exact text)
|
||||
2. **Workflow JSON** (sanitized)
|
||||
3. **Debug output** (from debug_api.py)
|
||||
4. **ComfyUI logs** (API section)
|
||||
|
||||
---
|
||||
|
||||
The ComfyUI-QwenVL custom node integrates powerful Qwen-VL series of vision-language models (LVLMs) from Alibaba Cloud, including latest Qwen3-VL and Qwen2.5-VL, plus GGUF backends and text-only Qwen3 support. This advanced node enables seamless multimodal AI capabilities within your ComfyUI workflows, allowing for efficient text generation, image understanding, and video analysis.
|
||||
|
||||
<img width="749" height="513" alt="Qwen3-VL-Mod" src="https://github.com/user-attachments/assets/0f10b887-1953-4923-b813-37ccacb8a9aa" />
|
||||
|
||||
## **📰 News & Updates**
|
||||
* **2026/10/01**: **v2.10.2 — Livepeer Agent Integration Removed**. [[Update](update.md#version-2102-20261001)]
|
||||
> Removed the hackathon-specific render node and external Agent integration after the event ended.
|
||||
* **2026/09/22**: **v2.8.14 — Removed Duplicate Livepeer Node**. [[Update](update.md#version-2814-20260922)]
|
||||
> 🌐 `AILab_LivepeerRender` is no longer shown in the node menu; old workflows still load via `legacy_aliases.js`.
|
||||
|
||||
* **2026/09/22**: **v2.8.13 — Pixaroma-Style Node Branding & Chat Styling**. [[Update](update.md#version-2813-20260922)]
|
||||
> 🎨 **Node colours**: each QwenVL-Mod family (Vision, Vision GGUF, Prompt Enhancer, Livepeer, Utils) now has distinct title/body colours, with user colour choices always preserved.
|
||||
> 💬 **Chat sidebar**: wrapped composer in a styled panel and added a shared stylesheet for a cleaner, more modern look.
|
||||
|
||||
* **2026/09/22**: **v2.8.12 — Pony→Natural Language in QwenVL Presets Too**. [[Update](update.md#version-2812-20260922)]
|
||||
> 🐴 **Pony→Natural Language**: now available both as a **PromptEnhancer style** (text-only) and as a **QwenVL preset** (with image input). Converts Danbooru/Pony tags into clean natural-language prompts for Flux, Krea, Qwen-Image 2.1, and other image models.
|
||||
|
||||
* **2026/09/22**: **v2.8.11 — Removed Redundant MiniMax Chat Guide**. [[Update](update.md#version-2811-20260922)]
|
||||
> 🧹 Removed the `mmh3_acceleration` `_chat_guides` entry: acceleration-mode switching is handled deterministically by Qwen Chat selectors and `use native/10eros/turbo` shortcuts, so the extra system-prompt guidance was no longer needed.
|
||||
|
||||
* **2026/09/22**: **v2.8.10 — Pony→Natural Language Preset Moved to PromptEnhancer**. [[Update](update.md#version-2810-20260922)]
|
||||
> 🐴 **Pony→Natural Language**: The generic Danbooru/Pony tag-to-natural-language converter is now a **PromptEnhancer style** (HF + GGUF), replacing the old `Pony→Z-Image` and `Pony→Flux` styles. Use it to turn tag soups into clean prompts for Flux, Krea, Qwen-Image 2.1, and other image models.
|
||||
|
||||
* **2026/09/22**: **v2.8.9 — Single Pony→Natural Language Preset**. [[Update](update.md#version-289-20260922)]
|
||||
> 🐴 **Pony→Natural Language**: Replaced the model-specific `Pony→Z-Image` and `Pony→Flux` presets with a single generic converter.
|
||||
|
||||
* **2026/09/21**: **v2.8.0 — Livepeer Agent Render Node, Unified Media Loader, and Chat-Driven Generation**. [[Update](update.md#version-280-20260921)]
|
||||
> 🌐 **Livepeer Agent Render**: New `QwenVL_LivepeerRender` node generates images and videos through the Livepeer Agent network via MCP — capability dropdown with image and video models, IMAGE output for stills, `source_video` frame picker for i2v references.
|
||||
> 🗂️ **Unified media loader**: New `QwenVL_LoadMedia` node picks images or videos from input/output folders with preview and an upload button.
|
||||
> 💬 **Chat-driven generation**: Qwen Chat routes requests through the render node, accepts video attachments with browser frame sampling, syncs selections with LoadMedia, and switches MiniMax H3 configs (10Eros/Native/Turbo) on demand.
|
||||
> 🧠 **Unified HF/GGUF nodes** with backend dropdown, rewritten system prompt, and MCP error handling fixes.
|
||||
|
||||
* **2026/09/16**: **v2.7.0 — Qwen Workflow Chat, Image Attachments, Guided Actions, and Qwen 3.8 Models**. [[Update](update.md#version-270-20260916)]
|
||||
> **Qwen Workflow Chat**: Native ComfyUI sidebar assistant with validated workflow actions, clickable choices, malformed-JSON recovery, preset-aware prompt routing, and autonomous queue execution.
|
||||
> **Image-aware chat**: Analyze workflow images automatically or attach a priority reference image directly in chat with preview, resizing, and removal controls.
|
||||
> **Refined interface**: Repeat-last-message control, busy spinner, improved message layout, and persistent English/Italian interface switch.
|
||||
> **Models and memory**: Qwen3.8 9B Q8 GGUF is the default chat model, thinking receives a larger token budget, and the chat model unloads before workflow execution.
|
||||
> **Diagnostics**: Reduced repeated cache lookup output while preserving useful model loading, image processing, generation, and failure logs.
|
||||
|
||||
* **2026/09/03**: **v2.6** 🎬 **Camera & Style Tag Dropdowns + Pony Converters + Qwen 3.8 + LTX 2.3 FL2VA + MiniMax H3 Loop Mode**. [[Update](update.md#version-26-20260903)]
|
||||
> 🎥 **Camera Tag Dropdown**: New `camera_tag` parameter in all nodes with 19 camera movements (STATIC_CAMERA, SLOW/FAST ZOOM, PAN, TILT, DOLLY, TRACKING, CRANE, ORBIT, HANDHELD, ROLL). Tags injected as prefix + re-injected at end for recency bias. STATIC_CAMERA overrides RICHNESS RULES.
|
||||
> 🎨 **Style Tag Dropdown**: New `style_tag` parameter in PromptEnhancer (T2V only) with 12 visual styles (ANIME, PHOTOREALISTIC, 3DCG, CARTOON, CLAYMATION, WATERCOLOR, VINTAGE, NOIR, CYBERPUNK, FANTASY, SOFTFOCUS, HENTAI).
|
||||
> 🐴 **Pony Prompt Converter**: Presets for converting Danbooru/Pony tags into natural-language image prompts. (Consolidated into a single generic `🎨 Pony→Natural Language` preset in v2.8.9.) Rewritten `🖼️ Tags` preset generates Pony/SDXL tags with `score_9` prefix.
|
||||
> 🧠 **Qwen 3.8 Models**: Added Qwen3.8-27B-Uncensored-Heretic-Abliterated (VL), Qwen3.8-4B-Distill-heretic and Qwen3.8-9B-heretic-uncensored (text). Switched to armand0e repaired Qwen3.5-9B-heretic weights.
|
||||
> 🎬 **LTX 2.3 FL2VA**: New FL2VA presets (5s/10s/20s) + 10s/20s I2V/T2V presets. Dropdown reordered I2V → FL2VA → T2V. Danbooru tag support in all LTX 2.3 presets.
|
||||
> 🎬 **MiniMax H3**: LOOP MODE for FL2VA, anatomical coherence, [DIALOGUE] trigger, [P3]/[P4] R2VA reference tags, I2VA user-text priority, mandatory sexual sounds, camera control tags.
|
||||
> ⚡ **PromptEnhancer**: max_tokens default raised to 8192 (was 1024). Removed `custom_system_prompt`.
|
||||
> 🐛 **Fixes**: Python 3.12 compat + rope_scaling=None crash, PromptEnhancer TypeError, SageAttention 2.x import, qwen3_5→qwen3_vl config fallback, model list cleanup.
|
||||
|
||||
* **2026/08/11**: **v2.5.1** 🚀 **MiniMax-H3 Turbo LoRA + Official Diffusion + Uncensored Text Encoder**. [[Update](update.md#version-251-20260811)]
|
||||
> ⚡ **Turbo LoRA**: New 4–8 step workflows (`MiniMaxH3-Turbo-*-Qwen3VL.json`) using `Larryvrh/ComfyUI-MiniMax-H3-Turbo` and `minimax_h3_turbo_v4_step600_ema.safetensors`.
|
||||
> 📦 **Models**: T2VA/I2VA/FL2VA use the official `minimax_h3_fl2va_pruned_int8_convrot.safetensors` (Comfy-Org). R2VA keeps the official `ref2va`. Text encoder remains `qwen3vl_32b_h3_ultra_uncensored_heretic_int8_convrot.safetensors` (ethanfel) for uncensored prompting.
|
||||
> 🏷️ **Explicit Tag Handling**: MiniMax H3 system prompts now expand short explicit tags (e.g., `blowjob`, `deepthroat`) into full cinematic NSFW descriptions instead of refusing or asking for clarification.
|
||||
> 📝 **Docs Updated**: `minimax/Civitai_MiniMaxH3.md` and `runpod/README_MiniMaxH3.md` now point to the current model links.
|
||||
|
||||
* **2026/08/04**: **v2.4** 🚀 **Local Model Discovery + Qwen3.5 + SageAttention**. [[Update](update.md#version-24-20260804)]
|
||||
> 🔍 **Local Model Discovery**: Drop GGUF/HF models in your models directory — they appear in the dropdown automatically, no JSON editing needed.
|
||||
> 🧠 **Qwen3.5 Support**: Architecture detection from file metadata, auto-disable thinking mode, top_k=20 tuning.
|
||||
> ⚡ **SageAttention Restored**: Architecture-aware kernels (Blackwell/Hopper/Ada/Ampere) with graceful SDPA fallback.
|
||||
> 🛠️ **VRAM headroom, image fixes, max_tokens 8192, BitsAndBytes loading fix**.
|
||||
|
||||
* **2026/07/09**: **v2.3** 🎬 **New LTX 2.3 Presets + Multilingual Support**. [[Update](update.md#version-23-20260709)]
|
||||
> 🎬 **LTX 2.3 Presets**: New specialized presets for LTX 2.3 I2V and T2V with official prompting guides.
|
||||
> 🌐 **Multilingual Support**: Complete multilingual support for all LTX 2.3 presets.
|
||||
> 📝 **Prompting Guide**: Core Actions, Visual Details, Audio for optimized video generation.
|
||||
> 🎯 **Simplified Format**: No timeline, single fluid paragraph max 200 words.
|
||||
> 🔥 **NSFW Support**: Full NSFW support for both LTX 2.3 presets.
|
||||
|
||||
* **2026/03/06**: **v2.2.4** 🔧 **Critical OOM Fix + Quantization Removal**. [[Update](update.md#version-224-20260306)]
|
||||
> 🚨 **BitsAndBytes Disabled**: Removed problematic quantization causing OOM on RTX 5090.
|
||||
> ✅ **FP16 Only**: All HF nodes now use stable FP16 (~6GB VRAM).
|
||||
> 🎯 **Cleaner Interface**: Removed quantization dropdown - use GGUF nodes for quantized models.
|
||||
> 🔧 **Both Nodes Fixed**: Applied fixes to Standard and Advanced nodes with consistent parameters.
|
||||
> 💡 **User Guidance**: HF nodes for quality, GGUF nodes for quantization - clear separation.
|
||||
|
||||
* **2026/02/27**: **v2.2.3** 🔧 **CUDA 13 Compatibility Fix + Redundancy Removal**. [[Update](update.md#version-223-20260227)]
|
||||
> 🔧 **Removed unload_after_run**: Eliminated redundant checkbox from all QwenVL nodes to prevent CUDA 13 conflicts.
|
||||
> 🐛 **Fixed Parameter Errors**: Resolved "missing 1 required positional argument: unload_after_run" errors in all nodes.
|
||||
@@ -180,16 +132,6 @@ The assistant applies only a restricted, validated action protocol. Model output
|
||||
> 📹 **Technical Details**: Light sources, shot types, lens specs, camera movements, color tone requirements.
|
||||
> 🎯 **Consistent Branding**: Updated preset names with WAN family branding for better organization.
|
||||
|
||||
* **2026/03/13**: **v2.2.4** 🎬 Critical I2V Timeline Fixes & NSFW Presets Optimization. [[Update](https://github.com/huchukato/ComfyUI-QwenVL-Mod/blob/main/update.md#version-240-20260313)]
|
||||
> 🎬 **I2V Timeline (20s) Critical Fixes**:
|
||||
> - ✅ **Style Coherence**: Fixed AI changing anime→realism mid-sequence
|
||||
> - ✅ **Character Stability**: Fixed characters disappearing/appearing incorrectly
|
||||
> - ✅ **Natural Lighting**: Fixed AI adding artificial lights not in image
|
||||
> - ✅ **Timeline Structure**: Fixed continuous numbering (6,7,8...) instead of 0-5 restart
|
||||
> - ✅ **Format Consistency**: Fixed missing parentheses and unwanted labels
|
||||
> - 🔧 **All 8 NSFW Presets**: Complete specifications + emoji display restored
|
||||
> - 📋 **Token Settings Guide**: Comprehensive workflow note for optimal parameters
|
||||
|
||||
* **2026/02/01**: **v2.0.5** Extended Storyboard preset added for WAN 2.2 format continuity. [[Update](https://github.com/huchukato/ComfyUI-QwenVL-Mod/blob/main/update.md#version-205-20260201)]
|
||||
> [!NOTE]
|
||||
> 🎬 **Extended Storyboard**: New preset for seamless storyboard-to-storyboard generation with timeline format.
|
||||
@@ -220,14 +162,14 @@ The assistant applies only a restricted, validated action protocol. Model output
|
||||
>
|
||||
* **2025/12/22**: **v2.0.0** Added GGUF supported nodes and Prompt Enhancer nodes. [[Update](https://github.com/1038lab/ComfyUI-QwenVL/blob/main/update.md#version-200-20251222)]
|
||||
> [!IMPORTANT]
|
||||
> Install llama-cpp-python before running GGUF nodes [instructions](https://github.com/abetlen/llama-cpp-python)
|
||||
> Install llama-cpp-python before running GGUF nodes [instruction](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md)
|
||||
>
|
||||

|
||||
* **2025/11/10**: **v1.1.0** Runtime overhaul with attention-mode selector, flash-attn auto detection, smarter caching, and quantization/torch.compile controls in both nodes. [[Update](https://github.com/1038lab/ComfyUI-QwenVL/blob/main/update.md#version-110-20251110)]
|
||||
* **2025/10/31**: **v1.0.4** Custom Models Supported [[Update](https://github.com/1038lab/ComfyUI-QwenVL/blob/main/update.md#version-104-20251031)]
|
||||
* **2025/10/22**: **v1.0.3** Models list updated [[Update](https://github.com/1038lab/ComfyUI-QwenVL/blob/main/update.md#version-103-20251022)]
|
||||
* **2025/10/17**: **v1.0.0** Initial Release
|
||||
* Support for Qwen3-VL series models.
|
||||
* Support for Qwen3-VL and Qwen2.5-VL series models.
|
||||
* Automatic model downloading from Hugging Face.
|
||||
* On-the-fly quantization (4-bit, 8-bit, FP16).
|
||||
* Preset and Custom Prompt system for flexible and easy use.
|
||||
@@ -242,7 +184,7 @@ The assistant applies only a restricted, validated action protocol. Model output
|
||||
## **✨ Features**
|
||||
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://huggingface.co/Qwen)
|
||||
[](https://huggingface.co/Qwen)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
@@ -326,7 +268,7 @@ This repo includes **GGUF** nodes powered by `llama-cpp-python` (separate from t
|
||||
- **Nodes**: `QwenVL (GGUF)`, `QwenVL (GGUF Advanced)`, `QwenVL Prompt Enhancer (GGUF)`
|
||||
- **Model folder** (default): `ComfyUI/models/llm/GGUF/` (configurable via `gguf_models.json`)
|
||||
- **Vision requirement**: install a vision-capable `llama-cpp-python` wheel that provides `Qwen3VLChatHandler` / `Qwen25VLChatHandler`
|
||||
See [llama-cpp-python install guide](https://github.com/abetlen/llama-cpp-python)
|
||||
See [docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md)
|
||||
|
||||
## **🗂️ Config Files**
|
||||
|
||||
@@ -334,7 +276,7 @@ This repo includes **GGUF** nodes powered by `llama-cpp-python` (separate from t
|
||||
- `hf_vl_models`: vision-language models (used by QwenVL nodes).
|
||||
- `hf_text_models`: text-only models (used by Prompt Enhancer).
|
||||
- **GGUF models**: `gguf_models.json`
|
||||
- **System prompts**: `presets/*.json` — one file per family (VL presets + prompt-enhancer styles); a user-created `AILab_System_Prompts.json` is still honored as an overlay.
|
||||
- **System prompts**: `AILab_System_Prompts.json` (includes both VL prompts and prompt-enhancer styles).
|
||||
|
||||
## **📥 Download Models**
|
||||
|
||||
@@ -359,6 +301,8 @@ The models will be automatically downloaded on first use. If you prefer to downl
|
||||
| Qwen3-VL-32B-Thinking | [Download](https://huggingface.co/Qwen/Qwen3-VL-32B-Thinking) |
|
||||
| Qwen3-VL-32B-Instruct-FP8 | [Download](https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct-FP8) |
|
||||
| Qwen3-VL-32B-Thinking-FP8 | [Download](https://huggingface.co/Qwen/Qwen3-VL-32B-Thinking-FP8) |
|
||||
| Qwen2.5-VL-3B-Instruct | [Download](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct) |
|
||||
| Qwen2.5-VL-7B-Instruct | [Download](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) |
|
||||
|
||||
### **HF Text Models (Qwen3)**
|
||||
| Model | Link |
|
||||
@@ -455,6 +399,7 @@ This integration includes support for **Flash Attention 2**, a cutting-edge atte
|
||||
|
||||
| Model | Flash Attention 2 | Speedup |
|
||||
|-------|----------------|---------|
|
||||
| Qwen2.5-VL-3B | 100% | 200-300% | 2-3x |
|
||||
| Qwen3-VL-4B | 100% | 150-250% | 1.5-2.5x |
|
||||
|
||||
### **🎯 How to Use**
|
||||
@@ -543,51 +488,11 @@ This enhanced version includes specialized prompts for **WAN 2.2** video generat
|
||||
- Leverage professional cinematography specs for optimal video quality
|
||||
- The system automatically handles timeline optimization for WAN 2.2 presets
|
||||
|
||||
## **🎬 LTX 2.3 Integration**
|
||||
|
||||
This version includes new specialized presets for **LTX 2.3** video generation, supporting both I2V (image-to-video) and T2V (text-to-video) workflows with official prompting guides.
|
||||
|
||||
### **🎯 Available LTX 2.3 Prompts**
|
||||
|
||||
| Prompt Type | Use Case | Input | Output | Location |
|
||||
|:---|:---|:---|:---|:---|
|
||||
| **🎥 LTX 2.3 NSFW I2V** | Image-to-Video | Image + Text | Description of changes | QwenVL nodes |
|
||||
| **🎥 LTX 2.3 NSFW T2V** | Text-to-Video | Text only | Complete scene description | Prompt Enhancer nodes |
|
||||
|
||||
### **⚡ Features**
|
||||
|
||||
- **Official Prompting Guide**: Core Actions, Visual Details, Audio for optimized generation
|
||||
- **Multilingual Support**: Input in any language → English optimized output
|
||||
- **Simplified Format**: No timeline, single fluid paragraph max 200 words
|
||||
- **Professional Scene Description**: Film-style direction including lighting, camera, composition
|
||||
- **NSFW Handling**: Full NSFW support for both presets
|
||||
- **LTX 2.3 Optimization**: Specifically formatted for best video generation results
|
||||
|
||||
### **📝 Output Format Example**
|
||||
|
||||
```
|
||||
A woman in a red dress walks slowly through a sunlit garden, her hands gently brushing against blooming roses as golden afternoon light filters through the leaves, creating dappled patterns on her flowing fabric while the camera follows her graceful movement with a smooth tracking shot.
|
||||
```
|
||||
|
||||
### **🔧 Usage**
|
||||
|
||||
1. **For I2V**: Use "🎥 LTX 2.3 NSFW I2V" preset in QwenVL nodes with image input
|
||||
2. **For T2V**: Use "🎥 LTX 2.3 NSFW T2V" style in Prompt Enhancer nodes with text only
|
||||
3. **Prompting Guide**: Follow Core Actions, Visual Details, Audio for optimal results
|
||||
|
||||
### **🎨 Best Practices**
|
||||
|
||||
- Describe events and actions as they occur over time
|
||||
- Include all visual details you want to appear in the video
|
||||
- Describe sounds and dialogue needed for the scene
|
||||
- Keep descriptions literal and precise
|
||||
- Think like a film director describing a shot list
|
||||
|
||||
## **🗺️ Roadmap**
|
||||
|
||||
### **✅ Completed (v2.0.7)**
|
||||
|
||||
* ✅ Support for Qwen3-VL models.
|
||||
* ✅ Support for Qwen3-VL and Qwen2.5-VL models.
|
||||
* ✅ GGUF backend support for faster inference.
|
||||
* ✅ Prompt Enhancer nodes for text-only workflows.
|
||||
* ✅ Flash Attention 2 integration for 2-3x performance boost.
|
||||
@@ -612,8 +517,7 @@ A woman in a red dress walks slowly through a sunlit garden, her hands gently br
|
||||
* **Qwen Team**: [Alibaba Cloud](https://github.com/QwenLM) - For development and open-source powerful Qwen-VL models.
|
||||
* **ComfyUI**: [comfyanonymous](https://github.com/comfyanonymous/ComfyUI) - For incredible and extensible ComfyUI platform.
|
||||
* **llama-cpp-python**: [JamePeng/llama-cpp-python](https://github.com/JamePeng/llama-cpp-python) - GGUF backend with vision support used by GGUF nodes.
|
||||
* **GenorTG**: [GenorTG/ComfyUI-Genor-QwenVL-Mod](https://github.com/GenorTG/ComfyUI-Genor-QwenVL-Mod) - For innovative memory management improvements including `unload_after_run` parameter and prompt cache optimizations that prevent OOM errors in multi-node workflows.
|
||||
* **Deaquay**: [Deaquay/ComfyUI-Qwen3.5-Uncensored](https://github.com/Deaquay/ComfyUI-Qwen3.5-Uncensored) - For automatic local model discovery (GGUF and HF auto-scan with extra_model_paths.yaml support), Qwen3.5 architecture detection, and the VRAM headroom and image-handling fixes ported in v2.4.
|
||||
* **GenorTG**: [GenorTG/ComfyUI-Genor-QwenVL-Mod](https://github.com/GenorTG/ComfyUI-Genor-QwenVL-Mod) - For innovative memory management improvements including `unload_after_run` parameter and prompt cache optimizations that prevent OOM errors in multi-node workflows.
|
||||
* **ComfyUI Integration**: [1038lab](https://github.com/1038lab) - Developer of this custom node.
|
||||
|
||||
## **👥 Author**
|
||||
|
||||
+14
-134
@@ -1,33 +1,19 @@
|
||||
# **QwenVL-Mod per ComfyUI**
|
||||
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/huchukato/ComfyUI-QwenVL-Mod/main/banner.png" alt="QwenVL-Mod Banner" width="100%" />
|
||||
</p>
|
||||
|
||||
[](https://github.com/comfyanonymous/ComfyUI)
|
||||
[](LICENSE)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod/releases)
|
||||
[](https://python.org)
|
||||
[](https://pytorch.org)
|
||||
[](https://developer.nvidia.com/cuda-zone)
|
||||
[](https://developer.nvidia.com/cuda-zone)
|
||||
[](https://huggingface.co/Qwen)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod/issues)
|
||||
|
||||
[](https://lightning.ai/huchukato/environments/comfyui-v0-14-2-qwen3-vl-autoprompt)
|
||||
|
||||
Il custom node ComfyUI-QwenVL integra la potente serie di modelli vision-linguaggio (LVLM) Qwen-VL di Alibaba Cloud, inclusi i più recenti Qwen3-VL, oltre a backend GGUF e supporto solo testo Qwen3. Questo nodo avanzato abilita funzionalità multimodali AI senza soluzione di continuità nei tuoi workflow ComfyUI, permettendo efficiente generazione di testo, comprensione di immagini e analisi video.
|
||||
|
||||
## Chat Qwen per il workflow
|
||||
|
||||
Apri **Qwen Chat** dalla barra laterale di ComfyUI per conversare con un modello Qwen locale mentre lavori sul graph corrente. L'assistente può analizzare i nodi esistenti, modificare valori dei widget validati, attivare o bypassare nodi e accodare il workflow quando richiesto esplicitamente.
|
||||
|
||||
- Seleziona il backend **GGUF** oppure **HF / Transformers** e uno dei modelli configurati localmente.
|
||||
- Puoi chiedere, ad esempio: `Imposta il KSampler a 25 step e avvia il workflow`.
|
||||
- Attiva **Scarica Qwen prima di avviare il workflow** per liberare la memoria del modello chat prima di esecuzioni WAN, LTX o MiniMax pesanti.
|
||||
- Cronologia e preferenze vengono conservate soltanto nel local storage del browser e si cancellano con **Nuova chat**.
|
||||
- La prima versione non può aggiungere, collegare o eliminare nodi e non esegue JavaScript, shell, operazioni filesystem o rete arbitrarie.
|
||||
|
||||
L'assistente applica esclusivamente un protocollo ristretto e validato. Output del modello che indicano nodi, widget, valori od operazioni sconosciuti vengono rifiutati e riportati nella conversazione. Il pulsante Stop interrompe l'attesa nel browser; un'inferenza già partita nel backend potrebbe continuare fino al completamento.
|
||||
Il custom node ComfyUI-QwenVL integra la potente serie di modelli vision-linguaggio (LVLM) Qwen-VL di Alibaba Cloud, inclusi i più recenti Qwen3-VL e Qwen2.5-VL, oltre a backend GGUF e supporto solo testo Qwen3. Questo nodo avanzato abilita funzionalità multimodali AI senza soluzione di continuità nei tuoi workflow ComfyUI, permettendo efficiente generazione di testo, comprensione di immagini e analisi video.
|
||||
|
||||
<img width="749" height="513" alt="Qwen3-VL-Mod" src="https://github.com/user-attachments/assets/0f10b887-1953-4923-b813-37ccacb8a9aa" />
|
||||
|
||||
@@ -88,61 +74,6 @@ T2V → VRAM Cleanup (Full Cleanup) → I2V-1 → VRAM Cleanup (Full Cleanup)
|
||||
---
|
||||
|
||||
## **📰 Notizie & Aggiornamenti**
|
||||
* **2026/09/22**: **v2.8.14 — Rimosso nodo Livepeer duplicato**. [[Aggiornamenti](update.md#version-2814-20260922)]
|
||||
> 🌐 `AILab_LivepeerRender` non compare più nel menu; i vecchi workflow si caricano comunque tramite `legacy_aliases.js`.
|
||||
|
||||
* **2026/09/22**: **v2.8.13 — Branding nodi in stile Pixaroma e chat styling**. [[Aggiornamenti](update.md#version-2813-20260922)]
|
||||
> 🎨 **Colori nodi**: ogni famiglia QwenVL-Mod (Vision, Vision GGUF, Prompt Enhancer, Livepeer, Utils) ha ora colori distinti per titolo/corpo, preservando sempre le scelte dell’utente.
|
||||
> 💬 **Chat sidebar**: composer racchiuso in un pannello stilizzato con un foglio di stile condiviso per un aspetto più pulito e moderno.
|
||||
|
||||
* **2026/09/22**: **v2.8.12 — Pony→Natural Language anche nei preset QwenVL**. [[Aggiornamenti](update.md#version-2812-20260922)]
|
||||
> 🐴 **Pony→Natural Language**: ora disponibile sia come **stile PromptEnhancer** (solo testo) che come **preset QwenVL** (con immagine). Converte tag Danbooru/Pony in prompt natural-language puliti per Flux, Krea, Qwen-Image 2.1 e altri modelli.
|
||||
|
||||
* **2026/09/22**: **v2.8.11 — Rimossa guida chat MiniMax ridondante**. [[Aggiornamenti](update.md#version-2811-20260922)]
|
||||
> 🧹 Rimossa la guida `_chat_guides.mmh3_acceleration`: lo switching delle modalità di accelerazione MiniMax è gestito deterministicamente dai selettori di Qwen Chat e dagli shortcut `use native/10eros/turbo`, quindi la guida aggiuntiva non serviva più.
|
||||
|
||||
* **2026/09/22**: **v2.8.10 — Pony→Natural Language spostato su PromptEnhancer**. [[Aggiornamenti](update.md#version-2810-20260922)]
|
||||
> 🐴 **Pony→Natural Language**: Il converter generico da tag Danbooru/Pony a natural-language è ora uno **stile del PromptEnhancer** (HF + GGUF), sostituendo i vecchi stili `Pony→Z-Image` e `Pony→Flux`. Per Flux, Krea, Qwen-Image 2.1 e altri modelli d'immagine.
|
||||
|
||||
* **2026/09/22**: **v2.8.9 — Preset unico Pony→Natural Language**. [[Aggiornamenti](update.md#version-289-20260922)]
|
||||
> 🐴 **Pony→Natural Language**: Sostituiti i preset model-specifici `Pony→Z-Image` e `Pony→Flux` con un unico converter generico.
|
||||
|
||||
* **2026/09/16**: **v2.7.0 — Qwen Workflow Chat, allegati immagine, azioni guidate e modelli Qwen 3.8**. [[Aggiornamenti](update.md#version-270-20260916)]
|
||||
> **Qwen Workflow Chat**: Assistente nativo nella sidebar ComfyUI con azioni validate sul workflow, choices cliccabili, recupero del JSON malformato, routing basato sui preset ed esecuzione autonoma della coda.
|
||||
> **Chat con immagini**: Analisi automatica delle immagini del workflow oppure allegato prioritario direttamente in chat, con anteprima, ridimensionamento e rimozione.
|
||||
> **Interfaccia rifinita**: Ripetizione dell’ultimo messaggio, indicatore di attività, layout dei messaggi migliorato e selettore persistente inglese/italiano.
|
||||
> **Modelli e memoria**: Qwen3.8 9B Q8 GGUF è il modello chat predefinito, il thinking riceve un budget token maggiore e il modello chat viene scaricato dalla memoria prima del workflow.
|
||||
> **Diagnostica**: Ridotto lo spam delle ricerche in cache mantenendo visibili caricamento modelli, elaborazione immagini, generazione ed errori.
|
||||
|
||||
* **2026/09/03**: **v2.6** 🎬 **Dropdown Camera & Style Tag + Converter Pony + Qwen 3.8 + LTX 2.3 FL2VA + MiniMax H3 Loop Mode**. [[Aggiornamenti](update.md#version-26-20260903)]
|
||||
> 🎥 **Dropdown Camera Tag**: Nuovo parametro `camera_tag` in tutti i nodi con 19 movimenti camera (STATIC_CAMERA, SLOW/FAST ZOOM, PAN, TILT, DOLLY, TRACKING, CRANE, ORBIT, HANDHELD, ROLL). Tag iniettati come prefisso + re-iniettati alla fine per recency bias. STATIC_CAMERA sovrascrive RICHNESS RULES.
|
||||
> 🎨 **Dropdown Style Tag**: Nuovo parametro `style_tag` nel PromptEnhancer (solo T2V) con 12 stili visivi (ANIME, PHOTOREALISTIC, 3DCG, CARTOON, CLAYMATION, WATERCOLOR, VINTAGE, NOIR, CYBERPUNK, FANTASY, SOFTFOCUS, HENTAI).
|
||||
> 🐴 **Converter Pony**: Preset per convertire tag Danbooru in prompt natural-language per modelli d'immagine. (Unificato nel preset generico `🎨 Pony→Natural Language` in v2.8.9.) Preset `🖼️ Tags` riscritto genera tag Pony/SDXL con prefisso `score_9`.
|
||||
> 🧠 **Modelli Qwen 3.8**: Aggiunti Qwen3.8-27B-Uncensored-Heretic-Abliterated (VL), Qwen3.8-4B-Distill-heretic e Qwen3.8-9B-heretic-uncensored (testo). Passati a pesi armand0e riparati Qwen3.5-9B-heretic.
|
||||
> 🎬 **LTX 2.3 FL2VA**: Nuovi preset FL2VA (5s/10s/20s) + preset 10s/20s I2V/T2V. Dropdown riordinato I2V → FL2VA → T2V. Supporto tag Danbooru in tutti i preset LTX 2.3.
|
||||
> 🎬 **MiniMax H3**: LOOP MODE per FL2VA, coerenza anatomica, trigger [DIALOGUE], tag riferimento [P3]/[P4] per R2VA, priorità testo utente in I2VA, suoni sessuali obbligatori, tag controllo camera.
|
||||
> ⚡ **PromptEnhancer**: max_tokens predefinito aumentato a 8192 (era 1024). Rimosso `custom_system_prompt`.
|
||||
> 🐛 **Fix**: Compatibilità Python 3.12 + crash rope_scaling=None, TypeError PromptEnhancer, import SageAttention 2.x, fallback config qwen3_5→qwen3_vl, pulizia lista modelli.
|
||||
|
||||
* **2026/08/11**: **v2.5.1** 🚀 **MiniMax-H3 Turbo LoRA + Diffusion Ufficiale + Text Encoder Uncensored**. [[Aggiornamenti](update.md#version-251-20260811)]
|
||||
> ⚡ **Turbo LoRA**: Nuovi workflow 4–8 step (`MiniMaxH3-Turbo-*-Qwen3VL.json`) con `Larryvrh/ComfyUI-MiniMax-H3-Turbo` e `minimax_h3_turbo_v4_step600_ema.safetensors`.
|
||||
> 📦 **Modelli**: T2VA/I2VA/FL2VA usano il diffusion model ufficiale `minimax_h3_fl2va_pruned_int8_convrot.safetensors` (Comfy-Org). R2VA mantiene il ref2va ufficiale. Il text encoder rimane `qwen3vl_32b_h3_ultra_uncensored_heretic_int8_convrot.safetensors` (ethanfel).
|
||||
> 🏷️ **Gestione Tag Espliciti**: I system prompt MiniMax H3 espandono ora tag brevi espliciti (es. `blowjob`, `deepthroat`) in descrizioni cinematografiche NSFW complete, senza rifiuti o richieste di chiarimento.
|
||||
> 📝 **Docs Aggiornate**: `minimax/Civitai_MiniMaxH3.md` e `runpod/README_MiniMaxH3.md` puntano ai link dei modelli attuali.
|
||||
|
||||
* **2026/08/04**: **v2.4** 🚀 **Scoperta Modelli Locali + Qwen3.5 + SageAttention**. [[Aggiornamenti](update.md#version-24-20260804)]
|
||||
> 🔍 **Scoperta Modelli Locali**: Droppa modelli GGUF/HF nella directory models — appaiono nel dropdown automaticamente, senza editare JSON.
|
||||
> 🧠 **Supporto Qwen3.5**: Rilevamento architettura da metadati file, disabilitazione thinking mode, top_k=20.
|
||||
> ⚡ **SageAttention Ripristinato**: Kernel specifici per architettura (Blackwell/Hopper/Ada/Ampere) con fallback SDPA.
|
||||
> 🛠️ **Gestione VRAM, fix immagini, max_tokens 8192, fix caricamento BitsAndBytes**.
|
||||
|
||||
* **2026/07/09**: **v2.3** 🎬 **Nuovi Preset LTX 2.3 + Supporto Multilingua**. [[Aggiornamenti](update.md#version-23-20260709)]
|
||||
> 🎬 **LTX 2.3 Preset**: Nuovi preset specializzati per LTX 2.3 I2V e T2V con prompting guide ufficiali.
|
||||
> 🌐 **Supporto Multilingua**: Supporto completo multilingua per tutti i preset LTX 2.3.
|
||||
> 📝 **Prompting Guide**: Core Actions, Visual Details, Audio per generazione video ottimizzata.
|
||||
> 🎯 **Formato Semplificato**: Nessuna timeline, singolo paragrafo fluido max 200 parole.
|
||||
> 🔥 **Supporto NSFW**: Supporto completo NSFW per entrambi i preset LTX 2.3.
|
||||
|
||||
* **2026/02/27**: **v2.2.3** 🔧 **Fix Compatibilità CUDA 13 + Rimozione Ridondanze**. [[Aggiornamenti](update.md#version-223-20260227)]
|
||||
> 🔧 **Rimozione unload_after_run**: Eliminata checkbox ridondante da tutti i nodi QwenVL per prevenire conflitti su CUDA 13.
|
||||
> 🐛 **Fix Errori Parametri**: Risolti errori "missing 1 required positional argument: unload_after_run" in tutti i nodi.
|
||||
@@ -226,19 +157,6 @@ T2V → VRAM Cleanup (Full Cleanup) → I2V-1 → VRAM Cleanup (Full Cleanup)
|
||||
> 🔄 **Focus Continuità**: Ogni paragrafo ripete contenuto precedente per transizioni fluide.
|
||||
> 🎯 **Compatibile WAN 2.2**: Stessa struttura timeline e supporto NSFW del preset I2V.
|
||||
|
||||
* **2026/03/06**: **v2.2.4** 🔧 **Fix OOM Critico + Rimozione Quantizzazione**. [[Aggiornamenti](update.md#version-224-20260306)]
|
||||
> 🚨 **BitsAndBytes Disabilitato**: Rimosso quantizzazione problematica che causava OOM su RTX 5090.
|
||||
> ✅ **Solo FP16**: Tutti i nodi HF ora usano FP16 stabile (~6GB VRAM).
|
||||
> 🎯 **Interfaccia Pulita**: Rimosso dropdown quantizzazione - usa nodi GGUF per modelli quantizzati.
|
||||
> 🔧 **Entrambi i Nodi Fixati**: Applicati fix a nodi Standard e Advanced con parametri consistenti.
|
||||
> 💡 **Guida Utente**: Nodi HF per qualità, nodi GGUF per quantizzazione - separazione chiara.
|
||||
|
||||
* **2026/02/27**: **v2.2.3** 🔧 **Compatibilità CUDA 13 + Rimozione Ridondanze**. [[Aggiornamenti](update.md#version-223-20260227)]
|
||||
> 🔧 **Rimosso unload_after_run**: Eliminato checkbox ridondante da tutti i nodi QwenVL per prevenire conflitti CUDA 13.
|
||||
> 🐛 **Fix Errori Parametri**: Risolti errori "missing 1 required positional argument: unload_after_run" in tutti i nodi.
|
||||
> 🎯 **Interfaccia Semplificata**: Interfaccia nodo più pulita senza parametri ridondanti.
|
||||
> 🧠 **Nodo Cleanup VRAM**: Mantenuto per cleanup manuale quando necessario.
|
||||
|
||||
* **2026/02/01**: **v2.0.4** Aggiornamento stabilità - rimosso SageAttention per migliore compatibilità e affidabilità output modello. [[Aggiornamenti](https://github.com/huchukato/ComfyUI-QwenVL-Mod/blob/main/update.md#version-204-20260201)]
|
||||
> [!NOTE]
|
||||
> 🔧 **Flash Attention 2 Rimosso**: Eliminata complessità e problemi interferenza per performance modello stabili.
|
||||
@@ -264,14 +182,14 @@ T2V → VRAM Cleanup (Full Cleanup) → I2V-1 → VRAM Cleanup (Full Cleanup)
|
||||
|
||||
* **2025/12/22**: **v2.0.0** Aggiunti nodi supportati GGUF e nodi Prompt Enhancer. [[Aggiornamenti](https://github.com/1038lab/ComfyUI-QwenVL/blob/main/update.md#version-200-20251222)]
|
||||
> [!IMPORTANT]
|
||||
> Installa llama-cpp-python prima di eseguire nodi GGUF [istruzioni](https://github.com/abetlen/llama-cpp-python)
|
||||
> Installa llama-cpp-python prima di eseguire nodi GGUF [istruzioni](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md)
|
||||
|
||||

|
||||
* **2025/11/10**: **v1.1.0** Revisione runtime con selettore modalità attention, rilevamento automatico flash-attn, cache più intelligente, e controlli quantizzazione/torch.compile in entrambi i nodi. [[Aggiornamenti](https://github.com/1038lab/ComfyUI-QwenVL/blob/main/update.md#version-110-20251110)]
|
||||
* **2025/10/31**: **v1.0.4** Modelli Personalizzati Supportati [[Aggiornamenti](https://github.com/1038lab/ComfyUI-QwenVL/blob/main/update.md#version-104-20251031)]
|
||||
* **2025/10/22**: **v1.0.3** Elenco modelli aggiornato [[Aggiornamenti](https://github.com/1038lab/ComfyUI-QwenVL/blob/main/update.md#version-103-20251022)]
|
||||
* **2025/10/17**: **v1.0.0** Rilascio Iniziale
|
||||
* Supporto per serie modelli Qwen3-VL.
|
||||
* Supporto per serie modelli Qwen3-VL e Qwen2.5-VL.
|
||||
* Download automatico modelli da Hugging Face.
|
||||
* Quantizzazione al volo (4-bit, 8-bit, FP16).
|
||||
* Sistema Preset e Prompt Personalizzati per uso flessibile e facile.
|
||||
@@ -286,7 +204,7 @@ T2V → VRAM Cleanup (Full Cleanup) → I2V-1 → VRAM Cleanup (Full Cleanup)
|
||||
## **✨ Funzionalità**
|
||||
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://huggingface.co/Qwen)
|
||||
[](https://huggingface.co/Qwen)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
[](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
@@ -370,7 +288,7 @@ Questo repository include nodi **GGUF** powered by `llama-cpp-python` (separati
|
||||
- **Nodi**: `QwenVL (GGUF)`, `QwenVL (GGUF Avanzato)`, `QwenVL Prompt Enhancer (GGUF)`
|
||||
- **Cartella modelli** (predefinito): `ComfyUI/models/llm/GGUF/` (configurabile via `gguf_models.json`)
|
||||
- **Requisito vision**: installa wheel `llama-cpp-python` con capacità vision che fornisce `Qwen3VLChatHandler` / `Qwen25VLChatHandler`
|
||||
Vedi [llama-cpp-python install guide](https://github.com/abetlen/llama-cpp-python)
|
||||
Vedi [docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md](docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md)
|
||||
|
||||
## **🗂️ File Configurazione**
|
||||
|
||||
@@ -378,7 +296,7 @@ Questo repository include nodi **GGUF** powered by `llama-cpp-python` (separati
|
||||
- `hf_vl_models`: modelli vision-linguaggio (usati da nodi QwenVL).
|
||||
- `hf_text_models`: modelli solo testo (usati da Prompt Enhancer).
|
||||
- **Modelli GGUF**: `gguf_models.json`
|
||||
- **Prompt di sistema**: `presets/*.json` — un file per famiglia (preset VL + stili prompt-enhancer); un `AILab_System_Prompts.json` creato dall'utente resta supportato come overlay.
|
||||
- **Prompt di sistema**: `AILab_System_Prompts.json` (include sia prompt VL che stili prompt-enhancer).
|
||||
|
||||
## **📥 Download Modelli**
|
||||
|
||||
@@ -403,6 +321,8 @@ I modelli saranno scaricati automaticamente al primo utilizzo. Se preferisci sca
|
||||
| Qwen3-VL-32B-Thinking | [Download](https://huggingface.co/Qwen/Qwen3-VL-32B-Thinking) |
|
||||
| Qwen3-VL-32B-Instruct-FP8 | [Download](https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct-FP8) |
|
||||
| Qwen3-VL-32B-Thinking-FP8 | [Download](https://huggingface.co/Qwen/Qwen3-VL-32B-Thinking-FP8) |
|
||||
| Qwen2.5-VL-3B-Instruct | [Download](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct) |
|
||||
| Qwen2.5-VL-7B-Instruct | [Download](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) |
|
||||
|
||||
### **Modelli Testo HF (Qwen3)**
|
||||
| Modello | Link |
|
||||
@@ -499,6 +419,7 @@ Questa integrazione include supporto per **Flash Attention 2**, un'implementazio
|
||||
|
||||
| Modello | Flash Attention 2 | Velocità |
|
||||
|-------|----------------|---------|
|
||||
| Qwen2.5-VL-3B | 100% | 200-300% | 2-3x |
|
||||
| Qwen3-VL-4B | 100% | 150-250% | 1.5-2.5x |
|
||||
|
||||
### **🎯 Come Usare**
|
||||
@@ -587,51 +508,11 @@ Questa versione migliorata include prompt specializzati per **WAN 2.2** generazi
|
||||
- Sfrutta specifiche cinematografia professionale per qualità video ottimale
|
||||
- Il sistema gestisce automaticamente ottimizzazione timeline per preset WAN 2.2
|
||||
|
||||
## **🎬 Integrazione LTX 2.3**
|
||||
|
||||
Questa versione include nuovi preset specializzati per **LTX 2.3** generazione video, supportando sia workflow I2V (immagine-a-video) che T2V (testo-a-video) con prompting guide ufficiali.
|
||||
|
||||
### **🎯 Prompt LTX 2.3 Disponibili**
|
||||
|
||||
| Tipo Prompt | Caso Uso | Input | Output | Posizione |
|
||||
|:---|:---|:---|:---|:---|
|
||||
| **🎥 LTX 2.3 NSFW I2V** | Immagine-a-Video | Immagine + Testo | Descrizione cambiamenti | Nodi QwenVL |
|
||||
| **🎥 LTX 2.3 NSFW T2V** | Testo-a-Video | Testo solo | Descrizione scena completa | Nodi Prompt Enhancer |
|
||||
|
||||
### **⚡ Funzionalità**
|
||||
|
||||
- **Prompting Guide Ufficiale**: Core Actions, Visual Details, Audio per generazione ottimizzata
|
||||
- **Supporto Multilingua**: Input in qualsiasi lingua → Output ottimizzato Inglese
|
||||
- **Formato Semplificato**: Nessuna timeline, singolo paragrafo fluido max 200 parole
|
||||
- **Descrizione Scena Professionale**: Direzione stile cinematografico inclusi illuminazione, camera, composizione
|
||||
- **Gestione NSFW**: Supporto completo NSFW per entrambi i preset
|
||||
- **Ottimizzazione LTX 2.3**: Formattato specificamente per migliori risultati generazione video
|
||||
|
||||
### **📝 Esempio Formato Output**
|
||||
|
||||
```
|
||||
A woman in a red dress walks slowly through a sunlit garden, her hands gently brushing against blooming roses as golden afternoon light filters through the leaves, creating dappled patterns on her flowing fabric while the camera follows her graceful movement with a smooth tracking shot.
|
||||
```
|
||||
|
||||
### **🔧 Utilizzo**
|
||||
|
||||
1. **Per I2V**: Usa "🎥 LTX 2.3 NSFW I2V" preset in nodi QwenVL con input immagine
|
||||
2. **Per T2V**: Usa "🎥 LTX 2.3 NSFW T2V" style in nodi Prompt Enhancer con testo solo
|
||||
3. **Prompting Guide**: Segui Core Actions, Visual Details, Audio per risultati ottimali
|
||||
|
||||
### **🎨 Best Practices**
|
||||
|
||||
- Descrivi eventi e azioni come si verificano nel tempo
|
||||
- Includi tutti i dettagli visivi che vuoi apparire nel video
|
||||
- Descrivi suoni e dialoghi necessari per la scena
|
||||
- Mantieni descrizioni letterali e precise
|
||||
- Pensa come un regista cinematografico che descrive una shot list
|
||||
|
||||
## **🗺️ Roadmap**
|
||||
|
||||
### **✅ Completato (v2.0.7)**
|
||||
|
||||
* ✅ Supporto per modelli Qwen3-VL.
|
||||
* ✅ Supporto per modelli Qwen3-VL e Qwen2.5-VL.
|
||||
* ✅ Supporto backend GGUF per inferenza più rapida.
|
||||
* ✅ Nodi Prompt Enhancer per workflow solo testo.
|
||||
* ✅ Integrazione Flash Attention 2 per 2-3x boost performance.
|
||||
@@ -656,8 +537,7 @@ A woman in a red dress walks slowly through a sunlit garden, her hands gently br
|
||||
* **Qwen Team**: [Alibaba Cloud](https://github.com/QwenLM) - Per sviluppo e open-source potenti modelli Qwen-VL.
|
||||
* **ComfyUI**: [comfyanonymous](https://github.com/comfyanonymous/ComfyUI) - Per incredibile e estensibile piattaforma ComfyUI.
|
||||
* **llama-cpp-python**: [JamePeng/llama-cpp-python](https://github.com/JamePeng/llama-cpp-python) - Backend GGUF con supporto vision usato da nodi GGUF.
|
||||
* **GenorTG**: [GenorTG/ComfyUI-Genor-QwenVL-Mod](https://github.com/GenorTG/ComfyUI-Genor-QwenVL-Mod) - Per innovativi miglioramenti di gestione memoria incluso il parametro `unload_after_run` e ottimizzazioni cache prompt che prevengono errori OOM in workflow multi-nodo.
|
||||
* **Deaquay**: [Deaquay/ComfyUI-Qwen3.5-Uncensored](https://github.com/Deaquay/ComfyUI-Qwen3.5-Uncensored) - Per la scoperta automatica dei modelli locali (GGUF e HF con supporto extra_model_paths.yaml), rilevamento architettura Qwen3.5, e le correzioni di headroom VRAM e gestione immagini portate in v2.4.
|
||||
* **GenorTG**: [GenorTG/ComfyUI-Genor-QwenVL-Mod](https://github.com/GenorTG/ComfyUI-Genor-QwenVL-Mod) - Per innovativi miglioramenti di gestione memoria incluso il parametro `unload_after_run` e ottimizzazioni cache prompt che prevengono errori OOM in workflow multi-nodo.
|
||||
* **Integrazione ComfyUI**: [1038lab](https://github.com/1038lab) - Sviluppatore di questo custom node.
|
||||
|
||||
## **👥 Autore**
|
||||
|
||||
+11
-112
@@ -1,70 +1,26 @@
|
||||
import asyncio
|
||||
import html
|
||||
"""
|
||||
{
|
||||
"name": "QwenVL-Mod: Enhanced Vision-Language",
|
||||
"description": "Enhanced QwenVL node with Flash Attention 2, WAN 2.2 video generation, free abliterated models, and comprehensive NSFW support. Advanced fork with major improvements over original for stable multimodal AI workflows.",
|
||||
"author": "huchukato",
|
||||
"version": "2.0.4",
|
||||
"url": "https://github.com/huchukato/ComfyUI-QwenVL-Mod",
|
||||
"category": "image"
|
||||
}
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
|
||||
from aiohttp import web
|
||||
import folder_paths
|
||||
from server import PromptServer
|
||||
|
||||
# Get the directory of the current script
|
||||
current_dir = os.path.dirname(__file__)
|
||||
sys.path.insert(0, current_dir)
|
||||
|
||||
# Vanilla installs (Vast.ai, bare ComfyUI) have no extra_model_paths.yaml LLM
|
||||
# section — register our models/LLM dir so the GGUF nodes, the model list UI
|
||||
# and the Director's Analyze discovery all see the same folder key.
|
||||
if "LLM" not in folder_paths.folder_names_and_paths:
|
||||
folder_paths.add_model_folder_path("LLM", os.path.join(folder_paths.models_dir, "LLM"))
|
||||
|
||||
NODE_CLASS_MAPPINGS = {}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {}
|
||||
WEB_DIRECTORY = "./web"
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Download log endpoint: serves last 30 lines of model download log at /dlstatus
|
||||
# Works on the same port as ComfyUI (8188), no extra port exposure needed.
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
_DL_LOGS = [
|
||||
"/var/log/minimax-h3-models.log",
|
||||
"/var/log/ltx-models.log",
|
||||
"/var/log/wan-models.log",
|
||||
]
|
||||
|
||||
@PromptServer.instance.routes.get("/dlstatus")
|
||||
async def _dlstatus(request):
|
||||
for log_path in _DL_LOGS:
|
||||
if os.path.exists(log_path):
|
||||
try:
|
||||
with open(log_path, "r") as f:
|
||||
lines = f.readlines()[-30:]
|
||||
return web.Response(text="".join(lines), content_type="text/plain")
|
||||
except Exception:
|
||||
pass
|
||||
return web.Response(text="Download log not yet available...", content_type="text/plain")
|
||||
|
||||
_DLSTATUS_PAGE = """<!DOCTYPE html>
|
||||
<html>
|
||||
<head><meta http-equiv="refresh" content="3"></head>
|
||||
<body><pre style="font-family:monospace;font-size:12px;white-space:pre-wrap;">%s</pre></body>
|
||||
</html>"""
|
||||
|
||||
|
||||
@PromptServer.instance.routes.get("/dlstatus.html")
|
||||
async def _dlstatus_html(request):
|
||||
for log_path in _DL_LOGS:
|
||||
if not os.path.exists(log_path):
|
||||
continue
|
||||
try:
|
||||
with open(log_path, "r") as f:
|
||||
tail = f.readlines()[-30:]
|
||||
except Exception:
|
||||
continue
|
||||
escaped = html.escape("".join(tail))
|
||||
return web.Response(text=_DLSTATUS_PAGE % escaped, content_type="text/html")
|
||||
return web.Response(text="Download log not yet available...", content_type="text/plain")
|
||||
|
||||
def load_modules_from_directory(directory):
|
||||
for file in os.listdir(directory):
|
||||
if file.endswith(".py"):
|
||||
@@ -85,10 +41,6 @@ def load_modules_from_directory(directory):
|
||||
NODE_DISPLAY_NAME_MAPPINGS.update(module.NODE_DISPLAY_NAME_MAPPINGS)
|
||||
except Exception as e:
|
||||
print(f"Error loading module {module_name}: {e}")
|
||||
# Drop the half-initialized module: leaving it in sys.modules makes
|
||||
# later `import` statements "succeed" and blow up with AttributeError
|
||||
# on the missing class instead of failing cleanly.
|
||||
sys.modules.pop(module_name, None)
|
||||
|
||||
load_modules_from_directory(current_dir)
|
||||
|
||||
@@ -96,59 +48,6 @@ load_modules_from_directory(current_dir)
|
||||
nodes_dir = os.path.join(current_dir, "nodes")
|
||||
if os.path.exists(nodes_dir):
|
||||
load_modules_from_directory(nodes_dir)
|
||||
|
||||
from chat_service import CHAT_RUNTIME, list_output_images
|
||||
|
||||
|
||||
@PromptServer.instance.routes.get("/qwenvl/chat/models")
|
||||
async def _chat_models(request):
|
||||
return web.json_response(CHAT_RUNTIME.models())
|
||||
|
||||
|
||||
@PromptServer.instance.routes.get("/qwenvl/chat/assets")
|
||||
async def _chat_assets(request):
|
||||
return web.json_response(await asyncio.to_thread(list_output_images, folder_paths.get_output_directory()))
|
||||
|
||||
|
||||
@PromptServer.instance.routes.post("/qwenvl/chat")
|
||||
async def _chat(request):
|
||||
try:
|
||||
data = await request.json()
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("request body must be an object")
|
||||
options = data.get("options") or {}
|
||||
if not isinstance(options, dict):
|
||||
raise ValueError("options must be an object")
|
||||
result = await asyncio.to_thread(
|
||||
CHAT_RUNTIME.chat,
|
||||
data.get("backend"),
|
||||
data.get("model"),
|
||||
data.get("messages"),
|
||||
data.get("graph"),
|
||||
options,
|
||||
data.get("images"),
|
||||
data.get("video"),
|
||||
data.get("directives"),
|
||||
)
|
||||
return web.json_response(result)
|
||||
except ValueError as error:
|
||||
return web.json_response({"error": str(error)}, status=400)
|
||||
except Exception as error:
|
||||
print(f"[QwenVL Chat] {error}")
|
||||
return web.json_response({"error": "Chat generation failed. Check the ComfyUI console."}, status=500)
|
||||
|
||||
|
||||
@PromptServer.instance.routes.post("/qwenvl/chat/unload")
|
||||
async def _chat_unload(request):
|
||||
try:
|
||||
data = await request.json()
|
||||
backend = data.get("backend", "all") if isinstance(data, dict) else "all"
|
||||
unloaded = await asyncio.to_thread(CHAT_RUNTIME.unload, backend)
|
||||
return web.json_response({"status": "success", "unloaded": unloaded})
|
||||
except ValueError as error:
|
||||
return web.json_response({"error": str(error)}, status=400)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = dict(sorted(NODE_CLASS_MAPPINGS.items(), key=lambda x: NODE_DISPLAY_NAME_MAPPINGS.get(x[0], x[0])))
|
||||
NODE_DISPLAY_NAME_MAPPINGS = dict(sorted(NODE_DISPLAY_NAME_MAPPINGS.items(), key=lambda x: x[1]))
|
||||
|
||||
|
||||
Binary file not shown.
BIN
Binary file not shown.
|
Before Width: | Height: | Size: 924 KiB |
-1549
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"hf_models": {
|
||||
"My QwenVL Finetune": {
|
||||
"repo_id": "myusername/MyQwenVL-Finetune",
|
||||
"default": false,
|
||||
"quantized": false,
|
||||
"vram_requirement": {
|
||||
"4bit": 4,
|
||||
"8bit": 6,
|
||||
"full": 10
|
||||
}
|
||||
},
|
||||
"QwenVL Mini 4bit": {
|
||||
"repo_id": "huggingface-user/qwenvl-mini-4bit",
|
||||
"default": false,
|
||||
"quantized": false,
|
||||
"vram_requirement": {
|
||||
"4bit": 2,
|
||||
"8bit": 4,
|
||||
"full": 8
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"author": "huchukato",
|
||||
"title": "QwenVL-Mod: Enhanced Vision-Language",
|
||||
"reference": "https://github.com/huchukato/ComfyUI-QwenVL-Mod",
|
||||
"files": [
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod"
|
||||
],
|
||||
"install_type": "git-clone",
|
||||
"description": "Enhanced QwenVL node with smart prompt caching, Fixed Seed Mode, professional cinematography specifications, WAN 2.2 video generation, Extended Storyboard preset, free abliterated models, and comprehensive NSFW support. Advanced fork with major improvements over original for stable multimodal AI workflows.",
|
||||
"category": "image",
|
||||
"tags": ["vision", "language", "multimodal", "qwen", "flash_attention", "performance", "wan2.2", "video", "i2v", "t2v", "storyboard", "cinematography", "professional", "caching", "fixed_seed", "abliterated", "nsfw", "enhanced", "fork"],
|
||||
"version": "2.0.7"
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
# Install `llama-cpp-python` (Vision / Qwen-VL GGUF)
|
||||
|
||||
This plugin’s **QwenVL (GGUF)** vision nodes require a `llama-cpp-python` build that includes multimodal chat handlers such as:
|
||||
|
||||
- `Qwen3VLChatHandler`
|
||||
- `Qwen25VLChatHandler`
|
||||
|
||||
The upstream `llama-cpp-python` from PyPI often does **not** include these vision handlers. Use a fork/build that provides them (e.g. JamePeng’s fork) and install a **Release wheel**.
|
||||
|
||||
Release wheels (download `.whl` here):
|
||||
|
||||
- [https://github.com/JamePeng/llama-cpp-python/releases](https://github.com/JamePeng/llama-cpp-python/releases/)
|
||||
|
||||
## 0) Close ComfyUI first
|
||||
|
||||
Stop ComfyUI before installing/replacing packages, especially on Windows portable.
|
||||
|
||||
## 1) Identify the exact Python ComfyUI uses
|
||||
|
||||
### Windows portable (common)
|
||||
|
||||
Your Python is usually:
|
||||
|
||||
`ComfyUI\\python_embeded\\python.exe`
|
||||
|
||||
Check:
|
||||
|
||||
```bat
|
||||
C:\AI\ComfyUI\python_embeded\python.exe -V
|
||||
C:\AI\ComfyUI\python_embeded\python.exe -c "import sys; print(sys.executable)"
|
||||
```
|
||||
|
||||
### venv / conda
|
||||
|
||||
Activate your env, then:
|
||||
|
||||
```bash
|
||||
python -V
|
||||
python -c "import sys; print(sys.executable)"
|
||||
```
|
||||
|
||||
## 2) Backup your environment (recommended)
|
||||
|
||||
```bat
|
||||
C:\AI\ComfyUI\python_embeded\python.exe -m pip freeze > C:\AI\ComfyUI\requirements-backup.txt
|
||||
```
|
||||
|
||||
## 3) Install the Release wheel (recommended)
|
||||
|
||||
Download a **Release `.whl`** from:
|
||||
[https://github.com/JamePeng/llama-cpp-python/releases/](https://github.com/JamePeng/llama-cpp-python/releases/)
|
||||
|
||||
The wheel **must match ALL of the following**:
|
||||
|
||||
* **Python version** used by ComfyUI
|
||||
(`cp310` / `cp311` / `cp312` / `cp313`)
|
||||
* **Platform**
|
||||
`win_amd64` (Windows 64-bit)
|
||||
* **Build type**
|
||||
|
||||
* **CPU wheel** → safest option (no CUDA toolkit required)
|
||||
* **CUDA wheel (`cuXXX`)** → requires a compatible CUDA runtime / toolkit
|
||||
|
||||
> [!WARNING]
|
||||
> **Windows note (important)**
|
||||
> If you install a CUDA wheel, the CUDA build tag (e.g. `cu121`, `cu122`) must be compatible with your installed CUDA runtime/toolkit.
|
||||
|
||||
A mismatch can cause errors like **“cannot load ggml.dll” even though the file exists**.
|
||||
|
||||
If you are unsure, **use a CPU wheel**.
|
||||
|
||||
Install with force-reinstall (safer than manual uninstall):
|
||||
|
||||
```
|
||||
C:\AI\ComfyUI\python_embeded\python.exe -m pip install --upgrade --force-reinstall C:\path\to\llama_cpp_python-*.whl
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
* Warnings about leftover folders like `~umpy` are usually safe to ignore while ComfyUI is closed.
|
||||
* Make sure ComfyUI is **fully stopped** before installing.
|
||||
|
||||
## 4) Verify vision handlers exist
|
||||
|
||||
```bat
|
||||
C:\AI\ComfyUI\python_embeded\python.exe -c "from llama_cpp.llama_chat_format import Qwen3VLChatHandler, Qwen25VLChatHandler; print('handlers OK')"
|
||||
```
|
||||
|
||||
If this fails, you installed a wheel that does not include vision support (or installed into the wrong Python environment).
|
||||
|
||||
## 5) Fix common dependency conflicts (Windows)
|
||||
|
||||
Some wheels may upgrade dependencies (notably `numpy` / `pillow`) and cause conflicts with other packages (like OpenCV).
|
||||
|
||||
### OpenCV conflict (recommended fix)
|
||||
|
||||
If you see errors like:
|
||||
|
||||
- `opencv-python ... requires numpy<2.3.0,>=2; ... but you have numpy 2.3.x`
|
||||
|
||||
Pin numpy back:
|
||||
|
||||
```bat
|
||||
C:\AI\ComfyUI\python_embeded\python.exe -m pip install --upgrade "numpy<2.3"
|
||||
```
|
||||
|
||||
### Pillow conflicts (optional)
|
||||
|
||||
If you don’t use packages that depend on an older Pillow, you can ignore Pillow warnings. Otherwise:
|
||||
|
||||
```bat
|
||||
C:\AI\ComfyUI\python_embeded\python.exe -m pip install --upgrade "pillow<12"
|
||||
```
|
||||
@@ -0,0 +1,62 @@
|
||||
# 🧩 ComfyUI-QwenVL — Custom Models Guide
|
||||
|
||||
You can add your own HuggingFace or local fine-tuned QwenVL models
|
||||
without modifying the main code.
|
||||
|
||||
---
|
||||
|
||||
## 📁 File location
|
||||
Place a file named `custom_models.json` in this folder:
|
||||
|
||||
```
|
||||
|
||||
ComfyUI/custom_nodes/ComfyUI-QwenVL/
|
||||
|
||||
````
|
||||
|
||||
When ComfyUI starts, it will automatically detect and merge this file
|
||||
with the default `config.json`.
|
||||
|
||||
---
|
||||
|
||||
## ⚙️ File format
|
||||
Use the following JSON structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"hf_models": {
|
||||
"My QwenVL Finetune": {
|
||||
"repo_id": "myusername/MyQwenVL-Finetune",
|
||||
"default": false,
|
||||
"quantized": false,
|
||||
"vram_requirement": {
|
||||
"4bit": 4,
|
||||
"8bit": 6,
|
||||
"full": 10
|
||||
}
|
||||
},
|
||||
"QwenVL Mini 4bit": {
|
||||
"repo_id": "huggingface-user/qwenvl-mini-4bit",
|
||||
"default": false,
|
||||
"quantized": false,
|
||||
"vram_requirement": {
|
||||
"4bit": 2,
|
||||
"8bit": 4,
|
||||
"full": 8
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
````
|
||||
|
||||
* The **key** (e.g. `"My QwenVL Finetune"`) is what appears in the model dropdown list.
|
||||
* `"repo_id"` can be a HuggingFace model name or a local directory path.
|
||||
* `"vram_requirement"` is optional but helps ComfyUI display VRAM recommendations.
|
||||
|
||||
---
|
||||
|
||||
## 🧠 Notes
|
||||
|
||||
* If `custom_models.json` doesn’t exist, the system just skips it (no errors).
|
||||
* You can edit and reload it anytime — simply restart ComfyUI to apply changes.
|
||||
* Compatible with both Qwen2-VL and Qwen3-VL model architectures.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 494 KiB |
@@ -0,0 +1,466 @@
|
||||
{
|
||||
"id": "b1b52a7e-e128-4de3-83e0-2238301079ca",
|
||||
"revision": 0,
|
||||
"last_node_id": 18,
|
||||
"last_link_id": 18,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 12,
|
||||
"type": "PreviewAny",
|
||||
"pos": [
|
||||
-470,
|
||||
70
|
||||
],
|
||||
"size": [
|
||||
890,
|
||||
350
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 16
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.65",
|
||||
"Node name for S&R": "PreviewAny",
|
||||
"enableTabs": false,
|
||||
"tabWidth": 65,
|
||||
"tabXOffset": 10,
|
||||
"hasSecondTab": false,
|
||||
"secondTabText": "Send Back",
|
||||
"secondTabOffset": 80,
|
||||
"secondTabWidth": 65,
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {},
|
||||
"version": "7.3"
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
null,
|
||||
null,
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
-1780,
|
||||
80
|
||||
],
|
||||
"size": [
|
||||
550,
|
||||
530
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"version": "7.5.2",
|
||||
"input_ue_unconnectable": {}
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"## **⚙️ Parameters**\n\n| Parameter | Description | Default | Range | Node(s) |\n| :---- | :---- | :---- | :---- | :---- |\n| **model\\_name** | The Qwen-VL model to use. | Qwen3-VL-4B-Instruct | \\- | Standard & Advanced |\n| **quantization** | On-the-fly quantization. Ignored for pre-quantized models (e.g., FP8). | 8-bit (Balanced) | 4-bit, 8-bit, None | Standard & Advanced |\n| **preset\\_prompt** | A selection of pre-defined prompts for common tasks. | \"Describe this...\" | Any text | Standard & Advanced |\n| **custom\\_prompt** | Overrides the preset prompt if provided. | | Any text | Standard & Advanced |\n| **max\\_tokens** | Maximum number of new tokens to generate. | 1024 | 64-2048 | Standard & Advanced |\n| **keep\\_model\\_loaded** | Keep the model in VRAM for faster subsequent runs. | True | True/False | Standard & Advanced |\n| **seed** | A seed for reproducible results. | 1 | 1 \\- 2^64-1 | Standard & Advanced |\n| **temperature** | Controls randomness. Higher values \\= more creative. (Used when num\\_beams is 1). | 0.6 | 0.1-1.0 | Advanced Only |\n| **top\\_p** | Nucleus sampling threshold. (Used when num\\_beams is 1). | 0.9 | 0.0-1.0 | Advanced Only |\n| **num\\_beams** | Number of beams for beam search. \\> 1 disables temperature/top\\_p sampling. | 1 | 1-10 | Advanced Only |\n| **repetition\\_penalty** | Discourages repeating tokens. | 1.2 | 0.0-2.0 | Advanced Only |\n| **frame\\_count** | Number of frames to sample from the video input. | 16 | 1-64 | Advanced Only |\n| **device** | Override automatic device selection. | auto | auto, cuda, cpu | Advanced Only |"
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#653"
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"type": "AILab_QwenVL",
|
||||
"pos": [
|
||||
-885,
|
||||
70
|
||||
],
|
||||
"size": [
|
||||
394.1806388251093,
|
||||
303.45832648511134
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": 15
|
||||
},
|
||||
{
|
||||
"name": "video",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "RESPONSE",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
16
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {}
|
||||
},
|
||||
"Node name for S&R": "AILab_QwenVL"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Qwen3-VL-2B-Instruct",
|
||||
"None (FP16)",
|
||||
"auto",
|
||||
"🖼️ Detailed Description",
|
||||
"",
|
||||
512,
|
||||
true,
|
||||
1,
|
||||
"randomize"
|
||||
],
|
||||
"color": "#28403f",
|
||||
"bgcolor": "#374539"
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "AILab_LoadImage",
|
||||
"pos": [
|
||||
-1200,
|
||||
70
|
||||
],
|
||||
"size": [
|
||||
300,
|
||||
720
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
15
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "WIDTH",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "HEIGHT",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "AILab_LoadImage",
|
||||
"enableTabs": false,
|
||||
"tabWidth": 65,
|
||||
"tabXOffset": 10,
|
||||
"hasSecondTab": false,
|
||||
"secondTabText": "Send Back",
|
||||
"secondTabOffset": 80,
|
||||
"secondTabWidth": 65,
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {},
|
||||
"version": "7.3"
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"",
|
||||
"2430364.png",
|
||||
"lanczos",
|
||||
1,
|
||||
1,
|
||||
"longest_side",
|
||||
0,
|
||||
"image"
|
||||
],
|
||||
"color": "#2e3e57",
|
||||
"bgcolor": "#4b5b73"
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "PreviewAny",
|
||||
"pos": [
|
||||
-200,
|
||||
480
|
||||
],
|
||||
"size": [
|
||||
620,
|
||||
370
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 18
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"cnr_id": "comfy-core",
|
||||
"ver": "0.3.65",
|
||||
"Node name for S&R": "PreviewAny",
|
||||
"enableTabs": false,
|
||||
"tabWidth": 65,
|
||||
"tabXOffset": 10,
|
||||
"hasSecondTab": false,
|
||||
"secondTabText": "Send Back",
|
||||
"secondTabOffset": 80,
|
||||
"secondTabWidth": 65,
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {},
|
||||
"version": "7.3"
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
null,
|
||||
null,
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"type": "AILab_QwenVL_Advanced",
|
||||
"pos": [
|
||||
-610.5085688039786,
|
||||
481.1527608367014
|
||||
],
|
||||
"size": [
|
||||
396.48624844852634,
|
||||
468
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "video",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": 17
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "RESPONSE",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
18
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {}
|
||||
},
|
||||
"Node name for S&R": "AILab_QwenVL_Advanced"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Qwen3-VL-2B-Instruct",
|
||||
"None (FP16)",
|
||||
"auto",
|
||||
false,
|
||||
"auto",
|
||||
"🖼️ Detailed Description",
|
||||
"",
|
||||
512,
|
||||
0.6,
|
||||
0.9,
|
||||
1,
|
||||
1.2,
|
||||
16,
|
||||
true,
|
||||
1,
|
||||
"randomize"
|
||||
],
|
||||
"color": "#28403f",
|
||||
"bgcolor": "#374539"
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "VHS_LoadVideo",
|
||||
"pos": [
|
||||
-880,
|
||||
480
|
||||
],
|
||||
"size": [
|
||||
255.8291015625,
|
||||
560.3298002919321
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "meta_batch",
|
||||
"shape": 7,
|
||||
"type": "VHS_BatchManager",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"shape": 7,
|
||||
"type": "VAE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
17
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "frame_count",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"type": "AUDIO",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "video_info",
|
||||
"type": "VHS_VIDEOINFO",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"cnr_id": "comfyui-videohelpersuite",
|
||||
"ver": "08e8df15db24da292d4b7f943c460dc2ab442b24",
|
||||
"Node name for S&R": "VHS_LoadVideo",
|
||||
"enableTabs": false,
|
||||
"tabWidth": 65,
|
||||
"tabXOffset": 10,
|
||||
"hasSecondTab": false,
|
||||
"secondTabText": "Send Back",
|
||||
"secondTabOffset": 80,
|
||||
"secondTabWidth": 65,
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {},
|
||||
"version": "7.3"
|
||||
}
|
||||
},
|
||||
"widgets_values": {
|
||||
"video": "sing_1.mp4",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1,
|
||||
"format": "AnimateDiff",
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "sing_1.mp4",
|
||||
"type": "input",
|
||||
"format": "video/mp4",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
15,
|
||||
7,
|
||||
0,
|
||||
17,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
16,
|
||||
17,
|
||||
0,
|
||||
12,
|
||||
0,
|
||||
"STRING"
|
||||
],
|
||||
[
|
||||
17,
|
||||
11,
|
||||
0,
|
||||
18,
|
||||
1,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
18,
|
||||
18,
|
||||
0,
|
||||
13,
|
||||
0,
|
||||
"STRING"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"frontendVersion": "1.34.9",
|
||||
"VHS_latentpreview": false,
|
||||
"VHS_latentpreviewrate": 0,
|
||||
"VHS_MetadataImage": true,
|
||||
"VHS_KeepIntermediate": true,
|
||||
"ue_links": [],
|
||||
"links_added_by_ue": [],
|
||||
"ds": {
|
||||
"scale": 0.693975512958857,
|
||||
"offset": [
|
||||
1764.4004214905397,
|
||||
359.36315462869436
|
||||
]
|
||||
},
|
||||
"workflowRendererVersion": "LG"
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 224 KiB |
@@ -0,0 +1,364 @@
|
||||
{
|
||||
"id": "afd623e3-56b5-44d1-915f-4b020a0c435d",
|
||||
"revision": 0,
|
||||
"last_node_id": 25,
|
||||
"last_link_id": 0,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 17,
|
||||
"type": "AILab_QwenVL_GGUF_Advanced",
|
||||
"pos": [
|
||||
1090.7435866951005,
|
||||
541.3632089660127
|
||||
],
|
||||
"size": [
|
||||
340,
|
||||
516
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "video",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "RESPONSE",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "AILab_QwenVL_GGUF_Advanced",
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {},
|
||||
"version": "7.5.2"
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"Qwen2.5-VL-3B-Instruct-BF16.gguf",
|
||||
"auto",
|
||||
"🖼️ Detailed Description",
|
||||
"",
|
||||
512,
|
||||
0.6,
|
||||
0.9,
|
||||
1.2,
|
||||
16,
|
||||
8192,
|
||||
512,
|
||||
-1,
|
||||
4096,
|
||||
0,
|
||||
4194304,
|
||||
true,
|
||||
1,
|
||||
"randomize"
|
||||
],
|
||||
"color": "#474539",
|
||||
"bgcolor": "#374445"
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "AILab_QwenVL_Advanced",
|
||||
"pos": [
|
||||
1090.7435866951005,
|
||||
1103.0165621245635
|
||||
],
|
||||
"size": [
|
||||
340,
|
||||
468
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "video",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "RESPONSE",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "AILab_QwenVL_Advanced",
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {},
|
||||
"version": "7.5.2"
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"Qwen3-VL-2B-Instruct",
|
||||
"None (FP16)",
|
||||
"auto",
|
||||
false,
|
||||
"auto",
|
||||
"🖼️ Detailed Description",
|
||||
"",
|
||||
512,
|
||||
0.6,
|
||||
0.9,
|
||||
1,
|
||||
1.2,
|
||||
16,
|
||||
true,
|
||||
1,
|
||||
"randomize"
|
||||
],
|
||||
"color": "#28403f",
|
||||
"bgcolor": "#374539"
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "AILab_QwenVL",
|
||||
"pos": [
|
||||
728.3779414208888,
|
||||
1103.0165621245635
|
||||
],
|
||||
"size": [
|
||||
340,
|
||||
300
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "video",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "RESPONSE",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "AILab_QwenVL",
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {},
|
||||
"version": "7.5.2"
|
||||
}
|
||||
},
|
||||
"widgets_values": [
|
||||
"Qwen3-VL-2B-Instruct",
|
||||
"None (FP16)",
|
||||
"auto",
|
||||
"🖼️ Detailed Description",
|
||||
"",
|
||||
512,
|
||||
true,
|
||||
1,
|
||||
"randomize"
|
||||
],
|
||||
"color": "#28403f",
|
||||
"bgcolor": "#374539"
|
||||
},
|
||||
{
|
||||
"id": 23,
|
||||
"type": "AILab_QwenVL_GGUF_PromptEnhancer",
|
||||
"pos": [
|
||||
1453.109231969312,
|
||||
541.3632089660127
|
||||
],
|
||||
"size": [
|
||||
340,
|
||||
382
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "ENHANCED_OUTPUT",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {}
|
||||
},
|
||||
"Node name for S&R": "AILab_QwenVL_GGUF_PromptEnhancer"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Qwen3-4B-Q4_K_M.gguf",
|
||||
"",
|
||||
"📝 Enhance",
|
||||
"",
|
||||
256,
|
||||
0.7,
|
||||
0.9,
|
||||
1.1,
|
||||
false,
|
||||
"auto",
|
||||
1,
|
||||
"randomize"
|
||||
],
|
||||
"color": "#374445",
|
||||
"bgcolor": "#474539"
|
||||
},
|
||||
{
|
||||
"id": 24,
|
||||
"type": "AILab_QwenVL_PromptEnhancer",
|
||||
"pos": [
|
||||
1453.109231969312,
|
||||
1103.0165621245635
|
||||
],
|
||||
"size": [
|
||||
340,
|
||||
454
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "ENHANCED_OUTPUT",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {}
|
||||
},
|
||||
"Node name for S&R": "AILab_QwenVL_PromptEnhancer"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Qwen3-0.6B",
|
||||
"None (FP16)",
|
||||
"auto",
|
||||
false,
|
||||
"auto",
|
||||
"",
|
||||
"📝 Enhance",
|
||||
"",
|
||||
256,
|
||||
0.7,
|
||||
0.9,
|
||||
1.1,
|
||||
true,
|
||||
1,
|
||||
"randomize"
|
||||
],
|
||||
"color": "#374445",
|
||||
"bgcolor": "#474539"
|
||||
},
|
||||
{
|
||||
"id": 25,
|
||||
"type": "AILab_QwenVL_GGUF",
|
||||
"pos": [
|
||||
728.3779414208888,
|
||||
541.3632089660127
|
||||
],
|
||||
"size": [
|
||||
340,
|
||||
252
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "video",
|
||||
"shape": 7,
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "RESPONSE",
|
||||
"type": "STRING",
|
||||
"links": null
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"ue_properties": {
|
||||
"widget_ue_connectable": {},
|
||||
"input_ue_unconnectable": {}
|
||||
},
|
||||
"Node name for S&R": "AILab_QwenVL_GGUF"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Qwen3-VL-4B-Thinking-Q4_K_M (bartowski)",
|
||||
"🖼️ Detailed Description",
|
||||
"",
|
||||
512,
|
||||
true,
|
||||
1,
|
||||
"randomize"
|
||||
],
|
||||
"color": "#474539",
|
||||
"bgcolor": "#2c4045"
|
||||
}
|
||||
],
|
||||
"links": [],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"workflowRendererVersion": "LG",
|
||||
"ue_links": [],
|
||||
"ds": {
|
||||
"scale": 0.8334122337245743,
|
||||
"offset": [
|
||||
-262.4276486336808,
|
||||
-189.67302442340656
|
||||
]
|
||||
},
|
||||
"frontendVersion": "1.34.9",
|
||||
"VHS_latentpreview": false,
|
||||
"VHS_latentpreviewrate": 0,
|
||||
"VHS_MetadataImage": true,
|
||||
"VHS_KeepIntermediate": true
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
+262
-39
@@ -1,36 +1,175 @@
|
||||
{
|
||||
"base_dir": "LLM/GGUF",
|
||||
"qwenVL_model": {
|
||||
"Qwen3.5-9B-Defiant-Fable-Heretic-GGUF": {
|
||||
"author": "DavidAU",
|
||||
"repo_name": "Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF",
|
||||
"repo_id": "DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF",
|
||||
"mmproj_file": "mmproj-BF16.gguf",
|
||||
"model_files": [
|
||||
"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q4_K_M.gguf",
|
||||
"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q5_K_M.gguf",
|
||||
"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q6_K.gguf",
|
||||
"Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q8_0.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 65536,
|
||||
"image_max_tokens": 4096,
|
||||
"n_batch": 512,
|
||||
"gpu_layers": -1,
|
||||
"top_k": 0,
|
||||
"pool_size": 4194304
|
||||
}
|
||||
},
|
||||
"Qwen3.5-4B-Claude-OS-HERETIC-GGUF": {
|
||||
"Qwen_model": {
|
||||
"Qwen3-4B-abliterated-GGUF": {
|
||||
"author": "mradermacher",
|
||||
"repo_name": "Qwen3.5-4B-Claude-4.6-OS-Auto-Variable-HERETIC-UNCENSORED-THINKING-GGUF",
|
||||
"repo_id": "mradermacher/Qwen3.5-4B-Claude-4.6-OS-Auto-Variable-HERETIC-UNCENSORED-THINKING-GGUF",
|
||||
"mmproj_file": "Qwen3.5-4B-Claude-4.6-OS-Auto-Variable-HERETIC-UNCENSORED-THINKING.mmproj-f16.gguf",
|
||||
"repo_name": "Qwen3-4B-abliterated-GGUF",
|
||||
"repo_id": "mradermacher/Qwen3-4B-abliterated-GGUF",
|
||||
"model_files": [
|
||||
"Qwen3.5-4B-Claude-4.6-OS-Auto-Variable-HERETIC-UNCENSORED-THINKING.Q4_K_M.gguf",
|
||||
"Qwen3.5-4B-Claude-4.6-OS-Auto-Variable-HERETIC-UNCENSORED-THINKING.Q5_K_M.gguf",
|
||||
"Qwen3.5-4B-Claude-4.6-OS-Auto-Variable-HERETIC-UNCENSORED-THINKING.Q6_K.gguf",
|
||||
"Qwen3.5-4B-Claude-4.6-OS-Auto-Variable-HERETIC-UNCENSORED-THINKING.Q8_0.gguf"
|
||||
"Qwen3-4B-abliterated-Q4_K_M.gguf",
|
||||
"Qwen3-4B-abliterated-Q5_K_M.gguf",
|
||||
"Qwen3-4B-abliterated-Q6_K.gguf",
|
||||
"Qwen3-4B-abliterated-Q8_0.gguf",
|
||||
"Qwen3-4B-abliterated-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768
|
||||
}
|
||||
},
|
||||
"Josiefied-Qwen3-4B-abliterated-v1-GGUF": {
|
||||
"author": "mradermacher",
|
||||
"repo_name": "Josiefied-Qwen3-4B-abliterated-v1-GGUF",
|
||||
"repo_id": "mradermacher/Josiefied-Qwen3-4B-abliterated-v1-GGUF",
|
||||
"model_files": [
|
||||
"Josiefied-Qwen3-4B-abliterated-v1.Q4_K_M.gguf",
|
||||
"Josiefied-Qwen3-4B-abliterated-v1.Q5_K_M.gguf",
|
||||
"Josiefied-Qwen3-4B-abliterated-v1.Q6_K.gguf",
|
||||
"Josiefied-Qwen3-4B-abliterated-v1.Q8_0.gguf",
|
||||
"Josiefied-Qwen3-4B-abliterated-v1-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768
|
||||
}
|
||||
},
|
||||
"Josiefied-Qwen3-8B-abliterated-v1-GGUF": {
|
||||
"author": "mradermacher",
|
||||
"repo_name": "Josiefied-Qwen3-8B-abliterated-v1-GGUF",
|
||||
"repo_id": "mradermacher/Josiefied-Qwen3-8B-abliterated-v1-GGUF",
|
||||
"model_files": [
|
||||
"Josiefied-Qwen3-8B-abliterated-v1.Q4_K_M.gguf",
|
||||
"Josiefied-Qwen3-8B-abliterated-v1.Q5_K_M.gguf",
|
||||
"Josiefied-Qwen3-8B-abliterated-v1.Q6_K.gguf",
|
||||
"Josiefied-Qwen3-8B-abliterated-v1.Q8_0.gguf",
|
||||
"Josiefied-Qwen3-8B-abliterated-v1-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2-GGUF": {
|
||||
"author": "mradermacher",
|
||||
"repo_name": "Qwen3-VL-4B-Instruct-c_abliterated-v2-GGUF",
|
||||
"repo_id": "mradermacher/Qwen3-VL-4B-Instruct-c_abliterated-v2-GGUF",
|
||||
"model_files": [
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2.Q4_K_M.gguf",
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2.Q5_K_M.gguf",
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2.Q6_K.gguf",
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2.Q8_0.gguf",
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3-GGUF": {
|
||||
"author": "mradermacher",
|
||||
"repo_name": "Qwen3-VL-8B-Instruct-c_abliterated-v3-GGUF",
|
||||
"repo_id": "mradermacher/Qwen3-VL-8B-Instruct-c_abliterated-v3-GGUF",
|
||||
"model_files": [
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3.Q4_K_M.gguf",
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3.Q5_K_M.gguf",
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3.Q6_K.gguf",
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3.Q8_0.gguf",
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 65536
|
||||
}
|
||||
},
|
||||
"Josiefied-Qwen3-4B-Instruct-2507-gabliterated-v1-GGUF": {
|
||||
"author": "DevQuasar",
|
||||
"repo_name": "Josiefied-Qwen3-4B-Instruct-2507-gabliterated-v1-GGUF",
|
||||
"repo_id": "DevQuasar/Goekdeniz-Guelmez.Josiefied-Qwen3-4B-Instruct-2507-gabliterated-v1-GGUF",
|
||||
"model_files": [
|
||||
"Goekdeniz-Guelmez.Josiefied-Qwen3-4B-Instruct-2507-gabliterated-v1.Q4_K_M.gguf",
|
||||
"Goekdeniz-Guelmez.Josiefied-Qwen3-4B-Instruct-2507-gabliterated-v1.Q5_K_M.gguf",
|
||||
"Goekdeniz-Guelmez.Josiefied-Qwen3-4B-Instruct-2507-gabliterated-v1.Q6_K.gguf",
|
||||
"Goekdeniz-Guelmez.Josiefied-Qwen3-4B-Instruct-2507-gabliterated-v1.Q8_0.gguf",
|
||||
"Goekdeniz-Guelmez.Josiefied-Qwen3-4B-Instruct-2507-gabliterated-v1.f16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768
|
||||
}
|
||||
},
|
||||
"Qwen3-4B-GGUF": {
|
||||
"author": "Qwen",
|
||||
"repo_name": "Qwen3-4B-GGUF",
|
||||
"repo_id": "Qwen/Qwen3-4B-GGUF",
|
||||
"model_files": [
|
||||
"Qwen3-4B-Q4_K_M.gguf",
|
||||
"Qwen3-4B-Q5_K_M.gguf",
|
||||
"Qwen3-4B-Q6_K.gguf",
|
||||
"Qwen3-4B-Q8_0.gguf",
|
||||
"Qwen3-4B-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768
|
||||
}
|
||||
},
|
||||
"Qwen3-4B-Instruct-GGUF": {
|
||||
"author": "Qwen",
|
||||
"repo_name": "Qwen3-4B-Instruct-GGUF",
|
||||
"repo_id": "Qwen/Qwen3-4B-Instruct-GGUF",
|
||||
"model_files": [
|
||||
"Qwen3-4B-Instruct-Q4_K_M.gguf",
|
||||
"Qwen3-4B-Instruct-Q5_K_M.gguf",
|
||||
"Qwen3-4B-Instruct-Q6_K.gguf",
|
||||
"Qwen3-4B-Instruct-Q8_0.gguf",
|
||||
"Qwen3-4B-Instruct-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768
|
||||
}
|
||||
},
|
||||
"Qwen3-8B-Instruct-GGUF": {
|
||||
"author": "Qwen",
|
||||
"repo_name": "Qwen3-8B-Instruct-GGUF",
|
||||
"repo_id": "Qwen/Qwen3-8B-Instruct-GGUF",
|
||||
"model_files": [
|
||||
"Qwen3-8B-Instruct-Q4_K_M.gguf",
|
||||
"Qwen3-8B-Instruct-Q5_K_M.gguf",
|
||||
"Qwen3-8B-Instruct-Q6_K.gguf",
|
||||
"Qwen3-8B-Instruct-Q8_0.gguf",
|
||||
"Qwen3-8B-Instruct-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768
|
||||
}
|
||||
}
|
||||
},
|
||||
"qwenVL_model": {
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2-GGUF": {
|
||||
"author": "mradermacher",
|
||||
"repo_name": "Qwen3-VL-4B-Instruct-c_abliterated-v2-GGUF",
|
||||
"repo_id": "mradermacher/Qwen3-VL-4B-Instruct-c_abliterated-v2-GGUF",
|
||||
"mmproj_file": "Qwen3-VL-4B-Instruct-c_abliterated-v2.mmproj-f16.gguf",
|
||||
"model_files": [
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2.Q4_K_M.gguf",
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2.Q5_K_M.gguf",
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2.Q6_K.gguf",
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2.Q8_0.gguf",
|
||||
"Qwen3-VL-4B-Instruct-c_abliterated-v2-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768,
|
||||
"image_max_tokens": 4096,
|
||||
"n_batch": 512,
|
||||
"gpu_layers": -1,
|
||||
"top_k": 0,
|
||||
"pool_size": 4194304
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3-GGUF": {
|
||||
"author": "mradermacher",
|
||||
"repo_name": "Qwen3-VL-8B-Instruct-c_abliterated-v3-GGUF",
|
||||
"repo_id": "mradermacher/Qwen3-VL-8B-Instruct-c_abliterated-v3-GGUF",
|
||||
"mmproj_file": "Qwen3-VL-8B-Instruct-c_abliterated-v3.mmproj-f16.gguf",
|
||||
"model_files": [
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3.Q4_K_M.gguf",
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3.Q5_K_M.gguf",
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3.Q6_K.gguf",
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3.Q8_0.gguf",
|
||||
"Qwen3-VL-8B-Instruct-c_abliterated-v3-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 65536,
|
||||
@@ -41,19 +180,103 @@
|
||||
"pool_size": 4194304
|
||||
}
|
||||
},
|
||||
"Qwen3.8-27B-TTURBO-Fable-CF-709-ULTRA-HERETIC-Uncen-GGUF": {
|
||||
"author": "DavidAU",
|
||||
"repo_name": "Qwen3.8-27B-TWIN-TURBO-Fable-Cold-Fusion-709-ULTRA-HERETIC-Uncensored-NM-DAU-NEO-MTP-GGUF",
|
||||
"repo_id": "DavidAU/Qwen3.8-27B-TWIN-TURBO-Fable-Cold-Fusion-709-ULTRA-HERETIC-Uncensored-NM-DAU-NEO-MTP-GGUF",
|
||||
"mmproj_file": "mmproj-BF16.gguf",
|
||||
"Qwen3-VL-32B-Heretic-v2-GGUF": {
|
||||
"author": "mradermacher",
|
||||
"repo_name": "Qwen3-VL-32B-Heretic-v2-GGUF",
|
||||
"repo_id": "mradermacher/Qwen3-VL-32B-Heretic-v2-GGUF",
|
||||
"mmproj_file": "Qwen3-VL-32B-Heretic-v2.mmproj-f16.gguf",
|
||||
"model_files": [
|
||||
"Qwen3.8-27B-TTURBO-Fable-C-Fusion-709-U-Httic-Uncen-NM-DAU-NEO-MAX-Q4_K_M.gguf",
|
||||
"Qwen3.8-27B-TTURBO-Fable-C-Fusion-709-U-Httic-Uncen-NM-DAU-NEO-MAX-Q5_K_M.gguf",
|
||||
"Qwen3.8-27B-TTURBO-Fable-C-Fusion-709-U-Httic-Uncen-NM-DAU-NEO-MAX-Q6_K.gguf",
|
||||
"Qwen3.8-27B-TTURBO-Fable-C-Fusion-709-U-Httic-Uncen-NM-DAU-NEO-MAX-Q8_0.gguf"
|
||||
"Qwen3-VL-32B-Heretic-v2.Q4_K_M.gguf",
|
||||
"Qwen3-VL-32B-Heretic-v2.Q5_K_M.gguf",
|
||||
"Qwen3-VL-32B-Heretic-v2.Q6_K.gguf",
|
||||
"Qwen3-VL-32B-Heretic-v2.Q8_0.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 65536,
|
||||
"context_length": 32768,
|
||||
"image_max_tokens": 4096,
|
||||
"n_batch": 512,
|
||||
"gpu_layers": -1,
|
||||
"top_k": 0,
|
||||
"pool_size": 4194304
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-4B-Instruct-GGUF": {
|
||||
"author": "Qwen",
|
||||
"repo_name": "Qwen3-VL-4B-Instruct-GGUF",
|
||||
"repo_id": "Qwen/Qwen3-VL-4B-Instruct-GGUF",
|
||||
"mmproj_file": "mmproj-Qwen3VL-4B-Instruct-F16.gguf",
|
||||
"model_files": [
|
||||
"Qwen3VL-4B-Instruct-Q4_K_M.gguf",
|
||||
"Qwen3-VL-4B-Instruct-Q5_K_M.gguf",
|
||||
"Qwen3-VL-4B-Instruct-Q6_K.gguf",
|
||||
"Qwen3VL-4B-Instruct-Q8_0.gguf",
|
||||
"Qwen3VL-4B-Instruct-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768,
|
||||
"image_max_tokens": 4096,
|
||||
"n_batch": 512,
|
||||
"gpu_layers": -1,
|
||||
"top_k": 0,
|
||||
"pool_size": 4194304
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-8B-Instruct-GGUF": {
|
||||
"author": "Qwen",
|
||||
"repo_name": "Qwen3-VL-8B-Instruct-GGUF",
|
||||
"repo_id": "Qwen/Qwen3-VL-8B-Instruct-GGUF",
|
||||
"mmproj_file": "mmproj-Qwen3VL-8B-Instruct-F16.gguf",
|
||||
"model_files": [
|
||||
"Qwen3VL-8B-Instruct-Q4_K_M.gguf",
|
||||
"Qwen3-VL-8B-Instruct-Q5_K_M.gguf",
|
||||
"Qwen3-VL-8B-Instruct-Q6_K.gguf",
|
||||
"Qwen3VL-8B-Instruct-Q8_0.gguf",
|
||||
"Qwen3VL-8B-Instruct-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768,
|
||||
"image_max_tokens": 4096,
|
||||
"n_batch": 512,
|
||||
"gpu_layers": -1,
|
||||
"top_k": 0,
|
||||
"pool_size": 4194304
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-4B-Thinking-GGUF": {
|
||||
"author": "Qwen",
|
||||
"repo_name": "Qwen3-VL-4B-Thinking-GGUF",
|
||||
"repo_id": "Qwen/Qwen3-VL-4B-Thinking-GGUF",
|
||||
"mmproj_file": "mmproj-Qwen3VL-4B-Thinking-F16.gguf",
|
||||
"model_files": [
|
||||
"Qwen3VL-4B-Thinking-Q4_K_M.gguf",
|
||||
"Qwen3-VL-4B-Thinking-Q5_K_M.gguf",
|
||||
"Qwen3-VL-4B-Thinking-Q6_K.gguf",
|
||||
"Qwen3VL-4B-Thinking-Q8_0.gguf",
|
||||
"Qwen3VL-4B-Thinking-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768,
|
||||
"image_max_tokens": 4096,
|
||||
"n_batch": 512,
|
||||
"gpu_layers": -1,
|
||||
"top_k": 0,
|
||||
"pool_size": 4194304
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-8B-Thinking-GGUF": {
|
||||
"author": "Qwen",
|
||||
"repo_name": "Qwen3-VL-8B-Thinking-GGUF",
|
||||
"repo_id": "Qwen/Qwen3-VL-8B-Thinking-GGUF",
|
||||
"mmproj_file": "mmproj-Qwen3VL-8B-Thinking-F16.gguf",
|
||||
"model_files": [
|
||||
"Qwen3VL-8B-Thinking-Q4_K_M.gguf",
|
||||
"Qwen3-VL-8B-Thinking-Q5_K_M.gguf",
|
||||
"Qwen3-VL-8B-Thinking-Q6_K.gguf",
|
||||
"Qwen3VL-8B-Thinking-Q8_0.gguf",
|
||||
"Qwen3VL-8B-Thinking-F16.gguf"
|
||||
],
|
||||
"defaults": {
|
||||
"context_length": 32768,
|
||||
"image_max_tokens": 4096,
|
||||
"n_batch": 512,
|
||||
"gpu_layers": -1,
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# GGUF backend requirements (vision-capable llama-cpp-python).
|
||||
# Follow the platform-specific install guide:
|
||||
# docs/LLAMA_CPP_PYTHON_VISION_INSTALL.md
|
||||
#
|
||||
# Linux (CUDA) example:
|
||||
# pip install --upgrade --force-reinstall --no-cache-dir \
|
||||
# llama-cpp-python==<version> --extra-index-url <vision-wheel-index>
|
||||
#
|
||||
# Windows example:
|
||||
# 1) Install a vision-capable wheel from the guide.
|
||||
# 2) Then verify with:
|
||||
# python -c "import llama_cpp; print(llama_cpp.__version__)"
|
||||
#
|
||||
# Placeholder only; do not rely on this line alone:
|
||||
# llama-cpp-python
|
||||
+83
-16
@@ -1,34 +1,101 @@
|
||||
{
|
||||
"hf_vl_models": {
|
||||
"Qwen3.5-9B-Defiant-Fable-Heretic": {
|
||||
"repo_id": "DavidAU/Qwen3.5-9B-The-Defiant-Fable-DARK-ROAST-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP",
|
||||
"Qwen3-VL-4B-Instruct-Abliterated": {
|
||||
"repo_id": "huihui-ai/Huihui-Qwen3-VL-4B-Instruct-abliterated",
|
||||
"default": true,
|
||||
"quantized": false,
|
||||
"quantized": true,
|
||||
"vram_requirement": {
|
||||
"full": 19.0,
|
||||
"8bit": 10.0,
|
||||
"4bit": 6.0
|
||||
"full": 6.0,
|
||||
"8bit": 3.5,
|
||||
"4bit": 2.0
|
||||
}
|
||||
},
|
||||
"Qwen3.5-4B-Deckard-Heretic-Thinking": {
|
||||
"repo_id": "DavidAU/Qwen3.5-4B-Deckard-HERETIC-UNCENSORED-Thinking",
|
||||
"Qwen3-VL-8B-Instruct-Abliterated": {
|
||||
"repo_id": "huihui-ai/Huihui-Qwen3-VL-8B-Instruct-abliterated",
|
||||
"default": false,
|
||||
"quantized": true,
|
||||
"vram_requirement": {
|
||||
"full": 12.0,
|
||||
"8bit": 7.0,
|
||||
"4bit": 4.5
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-4B-Instruct-Unredacted-MAX": {
|
||||
"repo_id": "prithivMLmods/Qwen3-VL-4B-Instruct-Unredacted-MAX",
|
||||
"default": false,
|
||||
"quantized": false,
|
||||
"vram_requirement": {
|
||||
"full": 9.0,
|
||||
"8bit": 5.0,
|
||||
"4bit": 3.0
|
||||
"full": 6.0,
|
||||
"8bit": 3.5,
|
||||
"4bit": 2.0
|
||||
}
|
||||
},
|
||||
"Qwen3.8-27B-TTURBO-Fable-CF-709-ULTRA-Heretic": {
|
||||
"repo_id": "DavidAU/Qwen3.8-27B-TWIN-TURBO-Fable-Cold-Fusion-709-ULTRA-HERETIC-Uncensored",
|
||||
"Qwen3-VL-8B-Instruct-Unredacted-MAX": {
|
||||
"repo_id": "prithivMLmods/Qwen3-VL-8B-Instruct-Unredacted-MAX",
|
||||
"default": false,
|
||||
"quantized": false,
|
||||
"vram_requirement": {
|
||||
"full": 55.0,
|
||||
"8bit": 29.0,
|
||||
"4bit": 17.0
|
||||
"full": 12.0,
|
||||
"8bit": 7.0,
|
||||
"4bit": 4.5
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-8B-Instruct-abliterated-v1": {
|
||||
"repo_id": "prithivMLmods/Qwen3-VL-8B-Instruct-abliterated-v1",
|
||||
"default": false,
|
||||
"quantized": false,
|
||||
"vram_requirement": {
|
||||
"full": 12.0,
|
||||
"8bit": 7.0,
|
||||
"4bit": 4.5
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-4B-Instruct-abliterated-v1": {
|
||||
"repo_id": "prithivMLmods/Qwen3-VL-4B-Instruct-abliterated-v1",
|
||||
"default": false,
|
||||
"quantized": false,
|
||||
"vram_requirement": {
|
||||
"full": 6.0,
|
||||
"8bit": 3.5,
|
||||
"4bit": 2.0
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-32B-Heretic-v2": {
|
||||
"repo_id": "rpDungeon/Qwen3-VL-32B-Heretic-v2",
|
||||
"default": false,
|
||||
"quantized": false,
|
||||
"vram_requirement": {
|
||||
"full": 48.0,
|
||||
"8bit": 28.0,
|
||||
"4bit": 16.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"hf_text_models": {
|
||||
"Qwen3-VL-4B-Instruct-Abliterated": {
|
||||
"repo_id": "huihui-ai/Huihui-Qwen3-VL-4B-Instruct-abliterated",
|
||||
"default": true,
|
||||
"quantized": true,
|
||||
"vram_requirement": {
|
||||
"full": 6.0,
|
||||
"8bit": 3.5,
|
||||
"4bit": 2.0
|
||||
}
|
||||
},
|
||||
"Qwen3-VL-8B-Instruct-Abliterated": {
|
||||
"repo_id": "huihui-ai/Huihui-Qwen3-VL-8B-Instruct-abliterated",
|
||||
"default": false,
|
||||
"quantized": true,
|
||||
"vram_requirement": {
|
||||
"full": 12.0,
|
||||
"8bit": 7.0,
|
||||
"4bit": 4.5
|
||||
}
|
||||
},
|
||||
"qwen3-4b-Z-Image-Engineer": {
|
||||
"repo_id": "BennyDaBall/qwen3-4b-Z-Image-Engineer",
|
||||
"default": false,
|
||||
"quantized": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
# ComfyUI-QwenVL-Mod — hybrid media loader
|
||||
# Lists images AND videos from input/ and output/ in one picker.
|
||||
# Image files -> IMAGE output; video files -> VIDEO output plus one
|
||||
# extracted frame on the IMAGE output (see frame_index).
|
||||
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
import folder_paths
|
||||
|
||||
try:
|
||||
from comfy_api.latest import InputImpl
|
||||
_VIDEO_FROM_FILE = getattr(InputImpl, "VideoFromFile", None)
|
||||
except Exception:
|
||||
try:
|
||||
from comfy_api.input_impl import VideoFromFile as _VIDEO_FROM_FILE
|
||||
except Exception:
|
||||
_VIDEO_FROM_FILE = None
|
||||
|
||||
_IMAGE_EXT = (".png", ".jpg", ".jpeg", ".webp")
|
||||
_VIDEO_EXT = (".mp4", ".webm", ".mov")
|
||||
_MAX_FILES = 2000
|
||||
|
||||
_SOURCES = (("input", folder_paths.get_input_directory),
|
||||
("output", folder_paths.get_output_directory))
|
||||
|
||||
|
||||
def _media_files():
|
||||
"""Relative paths of image/video files in input/ and output/, tagged."""
|
||||
files = []
|
||||
for tag, get_dir in _SOURCES:
|
||||
root_dir = get_dir()
|
||||
if not os.path.isdir(root_dir):
|
||||
continue
|
||||
for base, _, names in os.walk(root_dir):
|
||||
for name in sorted(names):
|
||||
if name.lower().endswith(_IMAGE_EXT + _VIDEO_EXT):
|
||||
rel = os.path.relpath(os.path.join(base, name), root_dir)
|
||||
files.append(f"{rel} [{tag}]")
|
||||
if len(files) >= _MAX_FILES:
|
||||
return files
|
||||
return files
|
||||
|
||||
|
||||
def _resolve(name):
|
||||
"""'clip.mp4 [output]' or uploaded 'clip.mp4' -> (abs path, tag)."""
|
||||
for tag, get_dir in _SOURCES:
|
||||
suffix = f" [{tag}]"
|
||||
if name.endswith(suffix):
|
||||
path = os.path.join(get_dir(), name[: -len(suffix)])
|
||||
if os.path.isfile(path):
|
||||
return path, tag
|
||||
return None, None
|
||||
# Untagged name (e.g. set by the upload widget): input/ wins, then output/
|
||||
for tag, get_dir in _SOURCES:
|
||||
path = os.path.join(get_dir(), name)
|
||||
if os.path.isfile(path):
|
||||
return path, tag
|
||||
return None, None
|
||||
|
||||
|
||||
def _pil_to_tensor(pil):
|
||||
arr = np.asarray(pil.convert("RGB")).astype("float32") / 255.0
|
||||
return torch.from_numpy(arr).unsqueeze(0)
|
||||
|
||||
|
||||
class QwenVL_LoadMedia:
|
||||
"""Load an image or a video from input//output/ in a single node."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"media": (_media_files() or ["<no media found>"], {"image_upload": True, "video_upload": True}),
|
||||
"frame_index": ("INT", {"default": 0, "min": -1, "max": 10000, "tooltip": "For videos: which frame to emit on the IMAGE output. 0 = first, -1 = last."}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "VIDEO", "STRING")
|
||||
RETURN_NAMES = ("image", "video", "path")
|
||||
FUNCTION = "load"
|
||||
CATEGORY = "QwenVL-Mod/Utils"
|
||||
OUTPUT_NODE = True
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, media, frame_index=0):
|
||||
path, _ = _resolve(media)
|
||||
return (path, os.path.getmtime(path)) if path else media
|
||||
|
||||
def load(self, media, frame_index=0):
|
||||
path, tag = _resolve(media)
|
||||
if not path:
|
||||
raise ValueError(f"media file not found: {media}")
|
||||
lower = path.lower()
|
||||
|
||||
if lower.endswith(_IMAGE_EXT):
|
||||
root = dict(input=folder_paths.get_input_directory(), output=folder_paths.get_output_directory())[tag]
|
||||
return {
|
||||
"ui": {"images": [{
|
||||
"filename": os.path.basename(path),
|
||||
"subfolder": os.path.dirname(os.path.relpath(path, root)),
|
||||
"type": tag,
|
||||
}]},
|
||||
"result": (_pil_to_tensor(Image.open(path)), None, path),
|
||||
}
|
||||
|
||||
if lower.endswith(_VIDEO_EXT):
|
||||
if _VIDEO_FROM_FILE is None:
|
||||
raise RuntimeError("VIDEO input type not available in this ComfyUI version")
|
||||
video = _VIDEO_FROM_FILE(path)
|
||||
frame_tensor = None
|
||||
try:
|
||||
frames = video.get_components().images
|
||||
if frames is not None and frames.shape[0]:
|
||||
index = frame_index if frame_index >= 0 else frames.shape[0] - 1
|
||||
frame_tensor = frames[min(index, frames.shape[0] - 1)].unsqueeze(0)
|
||||
except Exception:
|
||||
frame_tensor = None
|
||||
root = dict(input=folder_paths.get_input_directory(), output=folder_paths.get_output_directory())[tag]
|
||||
return {
|
||||
"ui": {"images": [{
|
||||
"filename": os.path.basename(path),
|
||||
"subfolder": os.path.dirname(os.path.relpath(path, root)),
|
||||
"type": tag,
|
||||
}], "animated": (True,)},
|
||||
"result": (frame_tensor, video, path),
|
||||
}
|
||||
|
||||
raise ValueError(f"unsupported media type: {media}")
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"QwenVL_LoadMedia": QwenVL_LoadMedia,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"QwenVL_LoadMedia": "🎞️ QwenVL Load Media (image+video)",
|
||||
}
|
||||
@@ -24,7 +24,7 @@ class StorySplitNode:
|
||||
RETURN_TYPES = ("STRING", "STRING", "STRING", "STRING", "STRING")
|
||||
RETURN_NAMES = ("prompt_1", "prompt_2", "prompt_3", "prompt_4", "debug_info")
|
||||
FUNCTION = "split_story"
|
||||
CATEGORY = "QwenVL-Mod/Utils"
|
||||
CATEGORY = "🔷 QwenVL-Mod/Utils"
|
||||
|
||||
def split_story(self, text, auto_split=True, custom_delimiter=""):
|
||||
"""
|
||||
@@ -81,5 +81,5 @@ NODE_CLASS_MAPPINGS = {
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"StorySplitNode": "✂️ QwenVL-Mod Story Split"
|
||||
"StorySplitNode": "Story Split Node"
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ class VRAMCleanup:
|
||||
RETURN_TYPES = ("*",) # Pass through the input
|
||||
RETURN_NAMES = ("output",)
|
||||
FUNCTION = "cleanup_vram_memory"
|
||||
CATEGORY = "QwenVL-Mod/Utils"
|
||||
CATEGORY = "🔷 QwenVL-Mod/Utils"
|
||||
OUTPUT_NODE = True
|
||||
|
||||
def cleanup_vram_memory(self, input, cleanup_mode):
|
||||
@@ -162,7 +162,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VRAMCleanup": "🧹 QwenVL-Mod VRAM Cleanup"
|
||||
"VRAMCleanup": "VRAM Cleanup"
|
||||
}
|
||||
|
||||
__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS']
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"vl": {
|
||||
"IMG › Simple": {
|
||||
"text": "Analyze the image and write a single concise sentence that describes the main subject and setting. Keep it grounded in visible details only."
|
||||
},
|
||||
"IMG › Detailed": {
|
||||
"text": "Write ONE detailed paragraph (8-14 sentences, ~120-220 words). Describe only what is visible: subject(s) and actions; people details if present (approx age group, gender expression if clear, hair, facial expression, pose, clothing, accessories); environment (location type, background elements, time cues); lighting (source, direction, softness/hardness, color temperature, shadows); camera viewpoint (eye-level/low/high, distance) and composition (framing, focal emphasis). No preface, no reasoning, no emojis."
|
||||
},
|
||||
"IMG › Ultra": {
|
||||
"text": "Write ONE ultra-detailed paragraph (10-16 sentences, ~180-320 words). Stay grounded in visible details. Include: subject micro-details (materials, textures, patterns, wear, reflections); people details if present (hair, skin tones, makeup, jewelry, fabric types, fit); environment depth (foreground/midground/background, signage/props, surface materials); lighting analysis (key/fill/back light, direction, softness, highlights, shadow shape); camera perspective (angle, lens feel, depth of field) and composition (leading lines, negative space, symmetry/asymmetry, visual hierarchy). No preface, no reasoning, no emojis."
|
||||
},
|
||||
"IMG › Cinematic": {
|
||||
"text": "Write ONE cinematic paragraph (8-12 sentences). Describe the scene like a film still: subject(s) and action; environment and atmosphere; lighting design (practical lights vs ambient, direction, contrast); camera language (shot type, angle, lens feel, depth of field, motion implied); composition and mood. Keep it vivid but factual (no made-up story). No preface, no reasoning, no emojis."
|
||||
},
|
||||
"IMG › Analysis": {
|
||||
"text": "Output ONLY these sections with short labels (no bullets): Subject; People (if any); Environment; Lighting; Camera/Composition; Color/Texture. In each section, write 2-4 sentences of concrete visible details. If something is not visible, write 'not visible'. No preface, no reasoning, no emojis."
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"vl": {
|
||||
"PONY › Danbooru Tags": {
|
||||
"text": "Your task is to generate a comprehensive list of Danbooru-style tags for a Pony/SDXL text-to-image model, based *only* on visual information in the image.\n\nOUTPUT FORMAT: Output ONLY comma-separated tags. No sentences, no explanations, no numbering. Start directly with the first tag.\n\nTAG STRUCTURE — follow this order:\n1. Quality tags (always start with): score_9, score_8_up, score_7_up\n2. Subject count: 1girl, 1boy, 2girls, solo, etc.\n3. Character attributes: hair color/style (blonde hair, long hair, ponytail), eye color (blue eyes), body type (slim, curvy, large breasts), skin tone, expression (smiling, blushing, looking at viewer)\n4. Clothing: detailed garment tags (school uniform, lingerie, dress, nude, partially undressed, cleavage)\n5. Pose/action: standing, sitting, lying, bent over, spread legs, reaching, kissing\n6. Environment: bedroom, classroom, outdoors, pool, bathroom, kitchen\n7. Lighting/atmosphere: warm lighting, soft light, sunlight, candlelight, night, dramatic lighting\n8. Camera/composition: close-up, full body, from above, from behind, wide shot, depth of field\n9. Style (if evident): photorealistic, anime, 3d, illustration, watercolor\n\nTAG RULES:\n- Use standard Danbooru tag naming: lowercase, underscores optional, space-separated phrases (e.g. \"long hair\" not \"long_hair\")\n- Be specific: \"blonde hair\" not \"light hair\", \"blue eyes\" not \"colored eyes\"\n- Include NSFW tags when applicable: nude, nipples, pussy, penis, sex, oral, spread pussy, etc. — do NOT censor\n- Include 30-60 tags for a rich prompt\n- Do NOT include meta tags like \"high quality\", \"masterpiece\", \"best quality\" — use the score_ tags instead\n- Do NOT include abstract concepts, emotions as single words, or non-visual tags\n- Do NOT repeat tags\n- If the image is NSFW, describe the act explicitly using standard danbooru NSFW tags\n\nEXAMPLE OUTPUT:\nscore_9, score_8_up, score_7_up, 1girl, solo, long blonde hair, blue eyes, large breasts, slim, pale skin, smiling, blushing, looking at viewer, school uniform, white shirt, plaid skirt, cleavage, sitting, legs crossed, classroom, desk, window, sunlight, warm lighting, soft shadows, upper body, depth of field, anime style"
|
||||
},
|
||||
"PONY › Natural Language": {
|
||||
"text": "You are a prompt converter that transforms Pony/SDXL Danbooru-style tag prompts into clean, model-agnostic natural-language prompts for image generation (Flux, Krea, Qwen-Image, Stable Diffusion, etc.).\n\nINPUT FORMAT — the user input may be:\n1. Danbooru-style tags (comma-separated, e.g. \"score_9, score_8_up, 1girl, solo, long blue hair, blue eyes, large breasts, school uniform, classroom, looking at viewer, spread legs\") → convert into flowing natural language.\n2. Natural language (Italian, English, or any language) → rewrite and enrich for image generation.\n3. Mix of tags and natural language → merge into one coherent prompt.\n\nCONVERSION RULES:\n- REMOVE all score_ tags (score_9, score_8_up, score_7_up, etc.).\n- REMOVE all Pony-specific meta tags (source_pony, rating_explicit_questionable, etc.).\n- Convert Danbooru tags into descriptive natural language SENTENCES, not tag lists. Examples:\n * \"1girl, solo\" → \"a young adult woman\"\n * \"long blue hair, blue eyes\" → \"with long blue hair and blue eyes\"\n * \"large breasts\" → \"with a curvaceous figure, full bust\"\n * \"school uniform\" → \"wearing a school uniform\"\n * \"spread legs\" → \"sitting with legs spread apart\"\n * \"looking at viewer\" → \"looking directly at the viewer\"\n * \"classroom\" → \"in a classroom\"\n * \"nude\" → \"fully nude\"\n * \"paizuri\" → \"performing paizuri, using her breasts to pleasure him\"\n * \"sex\" → \"having sex\"\n- Preserve ALL visual elements from the tags (hair, eyes, clothing, pose, expression, body type, environment). Do not drop tags.\n- DENSITY — expand every tag into descriptive detail, do not compress tags into bare clauses:\n * describe the subject's appearance with concrete physical detail (face, body, skin, hair texture and how it falls)\n * describe clothing with fabric, fit, and state of dress/undress\n * describe the action with explicit physical detail (positions, contact, motion)\n * describe the environment with props, surfaces, and atmosphere\n * describe lighting and mood (source, direction, warmth, shadows) and camera feel when photorealistic\n- Style tag priority: if the input starts with [anime], [photorealistic], [3dcg], [cartoon], [watercolor], [cyberpunk], [noir], [fantasy], [hentai], etc., that style DOMINATES the output. The bracketed tag itself must NEVER appear in the output — express the style in prose only.\n- STYLE DECLARATION — when a [style] tag is present, the FIRST sentence must declare it explicitly with concrete visual vocabulary (e.g. [ghibli] → \"A Studio Ghibli style hand-painted anime illustration of...\", [manga] → \"A black-and-white manga illustration of...\"). Do not just say \"whimsical\" or \"cinematic\" — name the rendering style: hand-painted, cel-shaded, 2D anime, painterly, ink lines, etc. Weave the style vocabulary through the whole paragraph so the target model cannot miss it.\n- If no style is specified, infer from the tags: anime-typical tags (1girl, kawaii, seifuku) default to anime-style; realistic tags default to photorealistic.\n- For photorealistic output, add concrete camera/lighting vocabulary when useful: shot type, lens feel, depth of field, natural or cinematic lighting.\n- For anime/stylized output, keep it as an anime/hentai/cartoon description rather than a photo caption.\n- NSFW content: describe explicitly and literally. Do NOT censor, soften, or add clothing.\n- Do NOT include quality prefixes, score_ tags, negative prompts, step counts, sampler settings, or meta commentary.\n\nOUTPUT:\n- Output exactly ONE flowing English paragraph — long and dense, not a summary.\n- 10-18 sentences, roughly 200-350 words depending on input richness.\n- Start with the style declaration (if a [style] tag is present) or the subject, then pose/action, then setting, then lighting/camera details.\n- No labels, bullets, or explanations.\n- The result must feel like a complete scene description, not a caption."
|
||||
}
|
||||
},
|
||||
"text": {
|
||||
"PONY › Natural Language": {
|
||||
"text": "You are a prompt converter that transforms Pony/SDXL Danbooru-style tag prompts into clean, model-agnostic natural-language prompts for image generation (Flux, Krea, Qwen-Image, Stable Diffusion, etc.).\n\nINPUT FORMAT — the user input may be:\n1. Danbooru-style tags (comma-separated, e.g. \"score_9, score_8_up, 1girl, solo, long blue hair, blue eyes, large breasts, school uniform, classroom, looking at viewer, spread legs\") → convert into flowing natural language.\n2. Natural language (Italian, English, or any language) → rewrite and enrich for image generation.\n3. Mix of tags and natural language → merge into one coherent prompt.\n\nCONVERSION RULES:\n- REMOVE all score_ tags (score_9, score_8_up, score_7_up, etc.).\n- REMOVE all Pony-specific meta tags (source_pony, rating_explicit_questionable, etc.).\n- Convert Danbooru tags into descriptive natural language SENTENCES, not tag lists. Examples:\n * \"1girl, solo\" → \"a young adult woman\"\n * \"long blue hair, blue eyes\" → \"with long blue hair and blue eyes\"\n * \"large breasts\" → \"with a curvaceous figure, full bust\"\n * \"school uniform\" → \"wearing a school uniform\"\n * \"spread legs\" → \"sitting with legs spread apart\"\n * \"looking at viewer\" → \"looking directly at the viewer\"\n * \"classroom\" → \"in a classroom\"\n * \"nude\" → \"fully nude\"\n * \"paizuri\" → \"performing paizuri, using her breasts to pleasure him\"\n * \"sex\" → \"having sex\"\n- Preserve ALL visual elements from the tags (hair, eyes, clothing, pose, expression, body type, environment). Do not drop tags.\n- Style tag priority: if the input starts with [anime], [photorealistic], [3dcg], [cartoon], [watercolor], [cyberpunk], [noir], [fantasy], [hentai], etc., that style DOMINATES the output. The bracketed tag itself must NEVER appear in the output — express the style in prose only.\n- STYLE DECLARATION — when a [style] tag is present, the FIRST sentence must declare it explicitly with concrete visual vocabulary (e.g. [ghibli] → \"A Studio Ghibli style hand-painted anime illustration of...\", [manga] → \"A black-and-white manga illustration of...\"). Do not just say \"whimsical\" or \"cinematic\" — name the rendering style: hand-painted, cel-shaded, 2D anime, painterly, ink lines, etc. Weave the style vocabulary through the whole paragraph so the target model cannot miss it.\n- If no style is specified, infer from the tags: anime-typical tags (1girl, kawaii, seifuku) default to anime-style; realistic tags default to photorealistic.\n- For photorealistic output, add concrete camera/lighting vocabulary when useful: shot type, lens feel, depth of field, natural or cinematic lighting.\n- For anime/stylized output, keep it as an anime/hentai/cartoon description rather than a photo caption.\n- NSFW content: describe explicitly and literally. Do NOT censor, soften, or add clothing.\n- Do NOT include quality prefixes, score_ tags, negative prompts, step counts, sampler settings, or meta commentary.\n\nOUTPUT:\n- Output exactly ONE flowing English paragraph.\n- 3–8 sentences depending on input richness.\n- Start with the style declaration (if a [style] tag is present) or the subject, then pose/action, then setting, then lighting/camera details.\n- No labels, bullets, or explanations."
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"text": {
|
||||
"Enhance": {
|
||||
"text": "You are a professional photography prompt writer. Respond in the same language as the user input.\n\nWrite ONE final cinematic photography prompt paragraph (150-300 words) based on the user text.\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- Do NOT output any reasoning, planning, or meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- Do NOT use <think>, quotes, markdown, code fences, JSON, headings, or bullet points.\n\nInclude naturally: subject + action/pose, environment, lighting, camera/lens/DoF, composition, color/texture, mood/style.\n\nIf input is short/ambiguous, infer minimal sensible details and keep it coherent."
|
||||
},
|
||||
"Refine": {
|
||||
"text": "You are a photography prompt refiner. Respond in the same language as the user input.\n\nWrite ONE clear, concise photography prompt paragraph (120-200 words) that preserves the user’s intent.\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nInclude: subject cues, environment context, lighting, camera parameters, composition focus, color/texture hints, and style tone. Remove redundancy."
|
||||
},
|
||||
"Creative Rewrite": {
|
||||
"text": "You are a creative photography prompt writer. Respond in the same language as the user input.\n\nRewrite the user’s scene into ONE fresh, imaginative photography prompt paragraph (150-250 words).\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nPreserve the core intent while adding vivid imagery and cohesive narrative flair. Integrate subject, environment, lighting, camera hints, composition, color/texture, and style."
|
||||
},
|
||||
"Detailed Visual": {
|
||||
"text": "You specialize in detailed visual photography prompts. Respond in the same language as the user input.\n\nWrite ONE flowing, highly visual photography prompt paragraph (180-260 words).\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nInclude concrete cues: subject traits and pose, foreground/midground/background, materials and textures, lighting direction/intensity/color temperature, colors and contrast, scale, atmosphere, and composition focus."
|
||||
},
|
||||
"Artistic Style": {
|
||||
"text": "You craft artistic photography prompts. Respond in the same language as the user input.\n\nWrite ONE artistic photography prompt paragraph (180-260 words).\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nWeave in subject, scene, and lighting with explicit style references (e.g., cinematic, fashion, fine art), mood, composition cues, and aesthetic adjectives. Keep it cohesive and visually rich."
|
||||
},
|
||||
"Technical Specs": {
|
||||
"text": "You convert scenes into technical photography directives. Respond in the same language as the user input.\n\nWrite ONE clear, actionable photography prompt paragraph (130-210 words).\n\nStrict output rules:\n- Output ONLY the prompt paragraph. Start immediately with the scene.\n- No reasoning, no planning, no meta text (no \"Okay\", no \"First/Next/Then\", no \"I/we\").\n- No <think>, no quotes, no markdown, no code fences, no JSON, no headings, no bullet points.\n\nCover: subject and scene plus focal length, aperture, depth of field, shooting angle, lighting type/direction, color temperature, focus target, and composition priorities as sentences."
|
||||
}
|
||||
},
|
||||
"translation_prompt": "You are a professional prompt translator. Return a single English paragraph (150-300 words). No prefixes, bullets, JSON, or . Preserve all visual and stylistic details."
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
+15
-16
@@ -1,16 +1,15 @@
|
||||
[project]
|
||||
name = "ComfyUI-QwenVL-Mod"
|
||||
description = "Enhanced QwenVL node with MiniMax H3 loop mode and presets, multilingual prompt generation, camera & style tag dropdowns, Pony prompt converter, Qwen 3.8 uncensored models, GGUF backend, LTX 2.3/2.5 FL2VA and audio-input music video presets, and local model discovery for professional multimodal video workflows."
|
||||
version = "2.12.12"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = ["transformers", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "llama-cpp-python"]
|
||||
|
||||
[project.urls]
|
||||
Repository = "https://github.com/huchukato/ComfyUI-QwenVL-Mod"
|
||||
|
||||
[tool.comfy]
|
||||
PublisherId = "huchukato"
|
||||
DisplayName = "QwenVL-Mod: Enhanced Vision-Language"
|
||||
Icon = "https://raw.githubusercontent.com/huchukato/ComfyUI-QwenVL-Mod/main/icon.png"
|
||||
Banner = "https://raw.githubusercontent.com/huchukato/ComfyUI-QwenVL-Mod/main/banner.png"
|
||||
Category = "video"
|
||||
[project]
|
||||
name = "ComfyUI-QwenVL-Mod"
|
||||
description = "Enhanced QwenVL node with WAN 2.2 workflows, multilingual support, visual style detection, smart prompt caching, and GGUF backend for professional video generation and multimodal AI."
|
||||
version = "2.2.3"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = ["transformers", "torch", "huggingface-hub", "hf_xet", "psutil", "numpy", "Pillow", "opencv-python", "bitsandbytes", "accelerate", "llama-cpp-python"]
|
||||
|
||||
[project.urls]
|
||||
Repository = "https://github.com/huchukato/ComfyUI-QwenVL-Mod"
|
||||
|
||||
[tool.comfy]
|
||||
PublisherId = "huchukato"
|
||||
DisplayName = "ComfyUI-QwenVL-Mod"
|
||||
Icon = ""
|
||||
Category = "video"
|
||||
|
||||
@@ -1,318 +0,0 @@
|
||||
"""Shared preset catalog for QwenVL nodes.
|
||||
|
||||
Presets live in presets/*.json, one file per family (minimax, wan, ltx,
|
||||
image, text). Each file may expose two sections:
|
||||
|
||||
- "vl": presets consumed by the VL nodes (preset_prompt widget)
|
||||
- "text": styles consumed by the Prompt Enhancer nodes (enhancement_style widget)
|
||||
|
||||
Entries are either {"text": "..."} or {"durations": {"5s": "...", ...},
|
||||
"default": "5s"} — duration variants stay grouped under a single dropdown
|
||||
entry and are selected by the node's "duration" widget.
|
||||
|
||||
Internally every entry is flattened to lookup keys of the form
|
||||
"Name" or "Name (Ns)" so legacy name-based matching keeps working.
|
||||
|
||||
A legacy AILab_System_Prompts.json placed next to the node (user
|
||||
customizations) is overlaid last so its "qwenvl"/"qwen_text" content still
|
||||
wins over the bundled presets.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
NODE_DIR = Path(__file__).resolve().parent
|
||||
PRESETS_DIR = NODE_DIR / "presets"
|
||||
LEGACY_PATH = NODE_DIR / "AILab_System_Prompts.json"
|
||||
|
||||
# Garage (huchukato/ComfyUI-Garage) is the source of truth for preset text —
|
||||
# synced into presets_remote/ at import so edits ship on ComfyUI restart
|
||||
# without waiting for a node release or image rebuild. Bundled presets stay
|
||||
# as offline fallback; remote files win per-key.
|
||||
_GARAGE_RAW_URL = "https://raw.githubusercontent.com/huchukato/ComfyUI-Garage/master/presets"
|
||||
_GARAGE_TREE_URL = "https://api.github.com/repos/huchukato/ComfyUI-Garage/git/trees/master?recursive=1"
|
||||
_REMOTE_DIR = NODE_DIR / "presets_remote"
|
||||
_MANIFEST = _REMOTE_DIR / ".garage-manifest"
|
||||
|
||||
DURATION_OPTIONS = ["5s", "10s", "15s", "20s"]
|
||||
DEFAULT_DURATION = "5s"
|
||||
|
||||
# Legacy dropdown names -> new base preset. Duration suffixes (Ns) are parsed
|
||||
# separately, so keys omit them.
|
||||
VL_ALIASES = {
|
||||
"MiniMax H3 NSFW": "MiniMax › NSFW I2VA",
|
||||
"MiniMax › NSFW": "MiniMax › NSFW I2VA",
|
||||
"MiniMax H3 NSFW R2VA": "MiniMax › NSFW R2VA",
|
||||
"MiniMax H3 NSFW FL2VA": "MiniMax › NSFW FL2VA",
|
||||
"MiniMax › SFW": "MiniMax › SFW I2VA",
|
||||
"MiniMax H3 SFW": "MiniMax › SFW I2VA",
|
||||
"MiniMax H3 SFW R2VA": "MiniMax › SFW R2VA",
|
||||
"MiniMax H3 SFW FL2VA": "MiniMax › SFW FL2VA",
|
||||
"Wan 2.2 NSFW I2V": "Wan22 › NSFW I2V",
|
||||
"Wan 2.2 NSFW I2V (20s)": "Wan22 › NSFW I2V Long",
|
||||
"Wan 2.2 NSFW FL2V": "Wan22 › NSFW FL2V",
|
||||
"LTX 2.3 NSFW I2V": "LTX › NSFW I2V",
|
||||
"LTX 2.3 NSFW FL2VA": "LTX › NSFW FL2VA",
|
||||
"LTX 2.3 NSFW Loop": "LTX › NSFW Loop",
|
||||
"LTX Audio-Input NSFW I2V": "LTX › Audio I2V",
|
||||
"Tags": "PONY › Danbooru Tags",
|
||||
"Pony→Natural Language": "PONY › Natural Language",
|
||||
"IMG › Tags": "PONY › Danbooru Tags",
|
||||
"IMG › Pony→Natural": "PONY › Natural Language",
|
||||
"IMG › Qwen Edit": "QWEN › Edit",
|
||||
"IMG › Storyboard": "QWEN › Storyboard",
|
||||
"Simple Description": "IMG › Simple",
|
||||
"Detailed Description": "IMG › Detailed",
|
||||
"Ultra Detailed Description": "IMG › Ultra",
|
||||
"Cinematic Description": "IMG › Cinematic",
|
||||
"Detailed Analysis": "IMG › Analysis",
|
||||
}
|
||||
|
||||
TEXT_ALIASES = {
|
||||
"MiniMax H3 NSFW": "MiniMax › NSFW T2V",
|
||||
"MiniMax › NSFW": "MiniMax › NSFW T2V",
|
||||
"MiniMax H3 NSFW R2VA": "MiniMax › NSFW R2VA T2V",
|
||||
"MiniMax H3 NSFW FL2VA": "MiniMax › NSFW FL2VA T2V",
|
||||
"MiniMax › SFW": "MiniMax › SFW T2V",
|
||||
"MiniMax H3 SFW": "MiniMax › SFW T2V",
|
||||
"MiniMax H3 SFW R2VA": "MiniMax › SFW R2VA T2V",
|
||||
"MiniMax H3 SFW FL2VA": "MiniMax › SFW FL2VA T2V",
|
||||
"Wan 2.2 NSFW T2V": "Wan22 › NSFW T2V",
|
||||
"Wan 2.2 NSFW T2V (20s)": "Wan22 › NSFW T2V Long",
|
||||
"LTX 2.3 NSFW T2V": "LTX › NSFW T2V",
|
||||
"Enhance": "Enhance",
|
||||
"Refine": "Refine",
|
||||
"Creative Rewrite": "Creative Rewrite",
|
||||
"Detailed Visual": "Detailed Visual",
|
||||
"Artistic Style": "Artistic Style",
|
||||
"Technical Specs": "Technical Specs",
|
||||
"Pony→Natural Language": "PONY › Natural Language",
|
||||
"Pony→Natural": "PONY › Natural Language",
|
||||
"Qwen Image Edit": "QWEN › Edit",
|
||||
"Qwen Image T2I": "QWEN › Txt2img",
|
||||
}
|
||||
|
||||
_EMOJI_PREFIX = re.compile(r"^[\U0001F000-\U0001FAFF☀-➿⬀-⯿️\u200d\ufe0f]+\s*")
|
||||
_DURATION_SUFFIX = re.compile(r"\s*\((\d+)s\)\s*$")
|
||||
|
||||
|
||||
def _strip_emoji(name):
|
||||
return _EMOJI_PREFIX.sub("", str(name or "")).strip()
|
||||
|
||||
|
||||
def _garage_preset_list():
|
||||
"""Map of presets/*.json blobs (path -> sha) in the Garage repo.
|
||||
None on failure — caller keeps bundled presets."""
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
_GARAGE_TREE_URL, headers={"User-Agent": "ComfyUI-QwenVL-Mod"})
|
||||
tree = json.loads(urllib.request.urlopen(req, timeout=10).read())
|
||||
return {
|
||||
blob["path"][len("presets/"):]: blob.get("sha", "")
|
||||
for blob in tree.get("tree", [])
|
||||
if blob.get("type") == "blob"
|
||||
and blob["path"].startswith("presets/")
|
||||
and blob["path"].endswith(".json")
|
||||
}
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL] Garage preset listing failed, using bundled presets: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def _sync_garage_presets():
|
||||
"""Blocking mirror of Garage presets/ into presets_remote/ (sha-skipped)."""
|
||||
remote = _garage_preset_list()
|
||||
if remote is None:
|
||||
return
|
||||
_REMOTE_DIR.mkdir(exist_ok=True)
|
||||
try:
|
||||
prev = dict(line.split(" ", 1) for line in
|
||||
_MANIFEST.read_text().splitlines() if " " in line)
|
||||
except Exception:
|
||||
prev = {}
|
||||
for rel, sha in remote.items():
|
||||
dest = _REMOTE_DIR / rel
|
||||
if prev.get(rel) == sha and dest.is_file():
|
||||
continue
|
||||
try:
|
||||
data = None
|
||||
for _ in range(2):
|
||||
try:
|
||||
# cache-bust: GitHub's raw CDN serves ~5-min-stale copies
|
||||
data = urllib.request.urlopen(
|
||||
f"{_GARAGE_RAW_URL}/{rel}?v={sha[:12]}", timeout=30).read()
|
||||
break
|
||||
except Exception:
|
||||
data = None
|
||||
if data is None:
|
||||
raise TimeoutError(f"download failed: {rel}")
|
||||
if dest.exists() and dest.read_bytes() == data:
|
||||
continue
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(data)
|
||||
print(f"[QwenVL] preset updated from Garage: {rel}")
|
||||
except Exception as exc:
|
||||
print(f"[QwenVL] preset sync skipped {rel}: {exc}")
|
||||
for p in _REMOTE_DIR.rglob("*.json"):
|
||||
if p.relative_to(_REMOTE_DIR).as_posix() not in remote:
|
||||
p.unlink()
|
||||
print(f"[QwenVL] preset removed (gone from Garage): {p.name}")
|
||||
try:
|
||||
_MANIFEST.write_text("\n".join(f"{rel} {sha}" for rel, sha in sorted(remote.items())))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _load_sections():
|
||||
"""Merge presets/*.json sections in deterministic file order.
|
||||
Bundled files load first; Garage-synced presets_remote/ wins per-key."""
|
||||
vl_entries, text_entries = {}, {}
|
||||
translation_prompt = ""
|
||||
order = ["minimax.json", "wan.json", "ltx.json", "qwen.json", "pony.json", "image.json", "text.json"]
|
||||
files = []
|
||||
for root in (PRESETS_DIR, _REMOTE_DIR):
|
||||
files += [root / name for name in order]
|
||||
files += sorted(p for p in root.glob("*.json") if p.name not in order)
|
||||
for path in files:
|
||||
if not path.exists():
|
||||
continue
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
for key, target in (("vl", vl_entries), ("text", text_entries)):
|
||||
section = data.get(key)
|
||||
if isinstance(section, dict):
|
||||
target.update(section)
|
||||
if isinstance(data.get("translation_prompt"), str):
|
||||
translation_prompt = data["translation_prompt"]
|
||||
return vl_entries, text_entries, translation_prompt
|
||||
|
||||
|
||||
_GROUP_ORDER = ("LTX ›", "MiniMax ›", "QWEN ›", "PONY ›", "Wan22 ›", "IMG ›", "VID ›")
|
||||
|
||||
def _group_key(name):
|
||||
for i, prefix in enumerate(_GROUP_ORDER):
|
||||
if name.startswith(prefix):
|
||||
return (i, name)
|
||||
return (len(_GROUP_ORDER), name) # generici in fondo
|
||||
|
||||
def _flatten(entries):
|
||||
"""Expand {"durations": {...}} entries into flat "Name (Ns)" keys.
|
||||
Returns (ordered_base_names, flat_dict, base_durations) — grouped by model
|
||||
prefix in _GROUP_ORDER, generics last."""
|
||||
base_names, flat, durations = [], {}, {}
|
||||
for name in sorted(entries, key=_group_key):
|
||||
entry = entries[name]
|
||||
base_names.append(name)
|
||||
if isinstance(entry, dict) and isinstance(entry.get("durations"), dict):
|
||||
durs = entry["durations"]
|
||||
durations[name] = (durs.get("default") or next(iter(durs), DEFAULT_DURATION))
|
||||
for dur, text in durs.items():
|
||||
if isinstance(text, str):
|
||||
flat[f"{name} ({dur})"] = text
|
||||
elif isinstance(entry, dict) and isinstance(entry.get("text"), str):
|
||||
flat[name] = entry["text"]
|
||||
elif isinstance(entry, str):
|
||||
flat[name] = entry
|
||||
return base_names, flat, durations
|
||||
|
||||
|
||||
def _apply_legacy_overlay(vl_entries, text_entries, translation_prompt):
|
||||
"""Overlay a user-provided legacy AILab_System_Prompts.json, if present."""
|
||||
if not LEGACY_PATH.exists():
|
||||
return vl_entries, text_entries, translation_prompt
|
||||
try:
|
||||
data = json.loads(LEGACY_PATH.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return vl_entries, text_entries, translation_prompt
|
||||
qwenvl = data.get("qwenvl")
|
||||
if isinstance(qwenvl, dict):
|
||||
for name, text in qwenvl.items():
|
||||
if isinstance(text, str):
|
||||
vl_entries[name] = {"text": text}
|
||||
qwen_text = data.get("qwen_text")
|
||||
if isinstance(qwen_text, dict):
|
||||
styles = qwen_text.get("styles")
|
||||
if isinstance(styles, dict):
|
||||
for name, text in styles.items():
|
||||
if isinstance(text, str):
|
||||
text_entries[name] = {"text": text}
|
||||
if isinstance(qwen_text.get("translation_prompt"), str):
|
||||
translation_prompt = qwen_text["translation_prompt"]
|
||||
return vl_entries, text_entries, translation_prompt
|
||||
|
||||
|
||||
def _load_all():
|
||||
_sync_garage_presets()
|
||||
vl_entries, text_entries, translation_prompt = _load_sections()
|
||||
vl_entries, text_entries, translation_prompt = _apply_legacy_overlay(
|
||||
vl_entries, text_entries, translation_prompt)
|
||||
vl_names, vl_flat, vl_durations = _flatten(vl_entries)
|
||||
text_names, text_flat, text_durations = _flatten(text_entries)
|
||||
return vl_names, vl_flat, vl_durations, text_names, text_flat, text_durations, translation_prompt
|
||||
|
||||
|
||||
(VL_PRESET_NAMES, VL_PROMPTS, VL_DURATIONS,
|
||||
TEXT_STYLE_NAMES, TEXT_PROMPTS, TEXT_DURATIONS,
|
||||
TRANSLATION_PROMPT) = _load_all()
|
||||
|
||||
|
||||
def resolve_preset(name, aliases):
|
||||
"""Map any preset name (new, legacy emoji, or duration-suffixed) to
|
||||
(canonical_base_name, duration_or_None)."""
|
||||
raw = str(name or "").strip()
|
||||
if not raw:
|
||||
return raw, None
|
||||
# Full-name alias first: "(20s)" legacy presets map to distinct Long
|
||||
# presets, not to a duration variant of the short one.
|
||||
stripped_full = _strip_emoji(raw)
|
||||
for old in sorted(aliases, key=len, reverse=True):
|
||||
if stripped_full == _strip_emoji(old):
|
||||
return aliases[old], None
|
||||
duration = None
|
||||
match = _DURATION_SUFFIX.search(raw)
|
||||
if match:
|
||||
duration = f"{match.group(1)}s"
|
||||
raw = raw[:match.start()].strip()
|
||||
stripped = _strip_emoji(raw)
|
||||
for old in sorted(aliases, key=len, reverse=True):
|
||||
if stripped == _strip_emoji(old):
|
||||
return aliases[old], duration
|
||||
return stripped, duration
|
||||
|
||||
|
||||
def resolve_prompt_key(name, duration, flat, durations, aliases):
|
||||
"""Resolve a widget preset + duration selection to a flat prompt key.
|
||||
Alias-derived durations win over the widget default."""
|
||||
base, alias_dur = resolve_preset(name, aliases)
|
||||
dur = alias_dur or duration or durations.get(base) or DEFAULT_DURATION
|
||||
key = f"{base} ({dur})"
|
||||
if key in flat:
|
||||
return base, key
|
||||
if base in flat:
|
||||
return base, base
|
||||
# Requested duration not defined for this preset (e.g. MiniMax has no
|
||||
# 20s): clamp to the closest available duration instead of the default.
|
||||
prefix = f"{base} ("
|
||||
avail = sorted(
|
||||
(int(k[len(prefix):-2].rstrip("s")), k)
|
||||
for k in flat
|
||||
if k.startswith(prefix) and k.endswith(")")
|
||||
and k[len(prefix):-2].rstrip("s").isdigit()
|
||||
)
|
||||
if avail:
|
||||
want = dur[:-1] if isinstance(dur, str) and dur.endswith("s") else ""
|
||||
pick = next((k for s, k in reversed(avail) if not want.isdigit() or s <= int(want)), avail[0][1])
|
||||
return base, pick
|
||||
return base, name # unknown preset: return raw (used as inline template)
|
||||
|
||||
|
||||
def resolve_vl_preset(name, duration=None):
|
||||
return resolve_prompt_key(name, duration, VL_PROMPTS, VL_DURATIONS, VL_ALIASES)
|
||||
|
||||
|
||||
def resolve_text_style(name, duration=None):
|
||||
return resolve_prompt_key(name, duration, TEXT_PROMPTS, TEXT_DURATIONS, TEXT_ALIASES)
|
||||
+7
-9
@@ -1,8 +1,7 @@
|
||||
# ComfyUI-QwenVL requirements
|
||||
# Note: Qwen3-VL models require transformers >= 4.57.0
|
||||
# Qwen3.5 (hybrid linear+full attention) requires transformers >= 5.2.0
|
||||
|
||||
transformers>=5.2.0
|
||||
transformers
|
||||
torch
|
||||
huggingface-hub
|
||||
hf_xet
|
||||
@@ -13,13 +12,12 @@ opencv-python
|
||||
bitsandbytes
|
||||
accelerate
|
||||
|
||||
# For running GGUF models with vision support:
|
||||
# pip install llama-cpp-python
|
||||
# Kept commented on purpose: llama-cpp-python often builds from source
|
||||
# (needs cmake/compiler) and a failed build would break the whole node
|
||||
# install via ComfyUI Manager. Pods ship a CUDA wheel in the base image;
|
||||
# local users should install it manually or follow docs/GGUF_MANUAL_INSTALL.md
|
||||
# llama-cpp-python
|
||||
# For running GGUF models with vision support
|
||||
# The [server] extra installs necessary components for multimodal models
|
||||
# llama-cpp-python[server]
|
||||
|
||||
# Optional: Flash Attention 2 for 2-3x performance boost (RTX 20+)
|
||||
# flash-attn>=2.0.0 # Uncomment for Flash Attention 2 support
|
||||
|
||||
# Optional: SageAttention for 2-5x performance boost (RTX 30+)
|
||||
# sageattention>=1.0.0 # Uncomment for SageAttention support
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
"""
|
||||
SageAttention patching functions for QwenVL models.
|
||||
This module contains the safe SageAttention implementation with proper error handling.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def set_sage_attention(model):
|
||||
"""
|
||||
Apply SageAttention patching to the model.
|
||||
Patches Qwen2Attention and Qwen3VLTextAttention modules to use SageAttention kernels.
|
||||
"""
|
||||
from AILab_QwenVL import sage_attn_available, get_sage_attention_config
|
||||
|
||||
if not sage_attn_available():
|
||||
raise ImportError("SageAttention library is not installed or GPU doesn't support it.")
|
||||
|
||||
SAGE_ATTN_FUNC, QK_QUANT_GRAN, PV_ACCUM_DTYPE = get_sage_attention_config()
|
||||
if SAGE_ATTN_FUNC is None:
|
||||
raise RuntimeError("No compatible SageAttention kernel found for this GPU.")
|
||||
|
||||
# Try to import different attention classes for different Qwen models
|
||||
attention_classes = []
|
||||
|
||||
# Qwen2 models
|
||||
try:
|
||||
from transformers.models.qwen2.modeling_qwen2 import Qwen2Attention, apply_rotary_pos_emb as qwen2_apply_rotary
|
||||
attention_classes.append((Qwen2Attention, qwen2_apply_rotary))
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Qwen3 models (Qwen3-VL, etc.)
|
||||
try:
|
||||
from transformers.models.qwen3.modeling_qwen3 import Qwen3Attention, apply_rotary_pos_emb as qwen3_apply_rotary
|
||||
attention_classes.append((Qwen3Attention, qwen3_apply_rotary))
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Qwen3-VL specific
|
||||
try:
|
||||
from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLTextAttention, apply_rotary_pos_emb as qwen3vl_apply_rotary
|
||||
attention_classes.append((Qwen3VLTextAttention, qwen3vl_apply_rotary))
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
if not attention_classes:
|
||||
print("[QwenVL] Could not import any attention classes for SageAttention patching")
|
||||
return
|
||||
|
||||
def make_sage_forward(AttentionClass, apply_rotary_pos_emb_func):
|
||||
def sage_attention_forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: tuple = None,
|
||||
attention_mask: torch.Tensor = None,
|
||||
past_key_values=None,
|
||||
cache_position: torch.LongTensor = None,
|
||||
position_ids: torch.LongTensor = None,
|
||||
**kwargs,
|
||||
):
|
||||
original_dtype = hidden_states.dtype
|
||||
|
||||
# Determine target dtype
|
||||
is_4bit = hasattr(self.q_proj, 'quant_state')
|
||||
if is_4bit:
|
||||
target_dtype = torch.bfloat16
|
||||
else:
|
||||
target_dtype = self.q_proj.weight.dtype
|
||||
|
||||
if hidden_states.dtype != target_dtype:
|
||||
hidden_states = hidden_states.to(target_dtype)
|
||||
|
||||
input_shape = hidden_states.shape[:-1]
|
||||
hidden_shape = (*input_shape, -1, self.head_dim)
|
||||
bsz, q_len = input_shape[0], input_shape[1] if len(input_shape) > 1 else hidden_states.size(1)
|
||||
|
||||
# Handle q_norm and k_norm for Qwen3-VL
|
||||
query_states = self.q_proj(hidden_states)
|
||||
key_states = self.k_proj(hidden_states)
|
||||
value_states = self.v_proj(hidden_states)
|
||||
|
||||
# Apply normalization if available (Qwen3-VL specific)
|
||||
if hasattr(self, 'q_norm'):
|
||||
query_states = self.q_norm(query_states.view(hidden_shape)).transpose(1, 2)
|
||||
else:
|
||||
query_states = query_states.view(hidden_shape).transpose(1, 2)
|
||||
|
||||
if hasattr(self, 'k_norm'):
|
||||
key_states = self.k_norm(key_states.view(hidden_shape)).transpose(1, 2)
|
||||
else:
|
||||
key_states = key_states.view(hidden_shape).transpose(1, 2)
|
||||
|
||||
value_states = value_states.view(hidden_shape).transpose(1, 2)
|
||||
|
||||
# Apply rotary embeddings
|
||||
if position_embeddings is not None:
|
||||
cos, sin = position_embeddings
|
||||
query_states, key_states = apply_rotary_pos_emb_func(query_states, key_states, cos, sin)
|
||||
|
||||
if past_key_values is not None:
|
||||
cache_kwargs = {"sin": sin if position_embeddings else None, "cos": cos if position_embeddings else None, "cache_position": cache_position}
|
||||
key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
||||
|
||||
is_causal = attention_mask is None and q_len > 1
|
||||
|
||||
attn_output = SAGE_ATTN_FUNC(
|
||||
query_states.to(target_dtype),
|
||||
key_states.to(target_dtype),
|
||||
value_states.to(target_dtype),
|
||||
tensor_layout="HND",
|
||||
is_causal=is_causal,
|
||||
qk_quant_gran=QK_QUANT_GRAN,
|
||||
pv_accum_dtype=PV_ACCUM_DTYPE,
|
||||
)
|
||||
|
||||
if isinstance(attn_output, tuple):
|
||||
attn_output = attn_output[0]
|
||||
|
||||
attn_output = attn_output.transpose(1, 2).contiguous()
|
||||
attn_output = attn_output.reshape(*input_shape, -1)
|
||||
|
||||
attn_output = self.o_proj(attn_output)
|
||||
|
||||
if attn_output.dtype != original_dtype:
|
||||
attn_output = attn_output.to(original_dtype)
|
||||
|
||||
return attn_output, None
|
||||
|
||||
return sage_attention_forward
|
||||
|
||||
# Apply patching to all supported attention modules
|
||||
patched_count = 0
|
||||
for AttentionClass, apply_rotary_func in attention_classes:
|
||||
sage_forward = make_sage_forward(AttentionClass, apply_rotary_func)
|
||||
for module in model.modules():
|
||||
if isinstance(module, AttentionClass):
|
||||
module.forward = sage_forward.__get__(module, AttentionClass)
|
||||
patched_count += 1
|
||||
|
||||
if patched_count > 0:
|
||||
print(f"[QwenVL] SageAttention: Patched {patched_count} attention layers")
|
||||
else:
|
||||
print("[QwenVL] SageAttention: No compatible attention layers found to patch")
|
||||
@@ -1,581 +0,0 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parents[1]))
|
||||
|
||||
from chat_service import (
|
||||
ChatRuntime,
|
||||
MINIMAX_I2VA_BINDING,
|
||||
THINK_CLOSE,
|
||||
THINK_OPEN,
|
||||
build_prompt,
|
||||
enforce_image_enhancer_routing,
|
||||
enforce_image_reference_bindings,
|
||||
list_output_images,
|
||||
parse_model_response,
|
||||
select_workflow_intent,
|
||||
validate_actions,
|
||||
validate_graph,
|
||||
validate_images,
|
||||
validate_messages,
|
||||
_fix_minimax_preset_actions,
|
||||
_match_minimax_preset,
|
||||
_minimax_preset_mode,
|
||||
_minimax_result,
|
||||
_is_image_enhancer_node,
|
||||
_has_image_enhancer_target,
|
||||
ensure_i2va_binding,
|
||||
normalize_minimax_output,
|
||||
)
|
||||
|
||||
|
||||
class ChatProtocolTests(unittest.TestCase):
|
||||
def test_validates_and_limits_messages(self):
|
||||
messages = [{"role": "user", "content": f"message {index}"} for index in range(25)]
|
||||
result = validate_messages(messages)
|
||||
self.assertEqual(len(result), 20)
|
||||
self.assertEqual(result[0]["content"], "message 5")
|
||||
|
||||
def test_rejects_invalid_message_role_and_content(self):
|
||||
for messages in ([], [{"role": "system", "content": "x"}], [{"role": "user", "content": ""}]):
|
||||
with self.subTest(messages=messages), self.assertRaises(ValueError):
|
||||
validate_messages(messages)
|
||||
|
||||
def test_validates_graph_snapshot(self):
|
||||
graph = {"nodes": [{"id": 1, "type": "KSampler", "title": "Sampler", "mode": 0, "widgets": [{"name": "steps", "value": 20}]}]}
|
||||
result = validate_graph(graph)
|
||||
self.assertEqual(result["nodes"][0]["id"], 1)
|
||||
self.assertEqual(result["nodes"][0]["widgets"][0]["value"], 20)
|
||||
|
||||
def test_rejects_invalid_graph(self):
|
||||
for graph in (None, {}, {"nodes": "bad"}, {"nodes": [{}]}):
|
||||
with self.subTest(graph=graph), self.assertRaises(ValueError):
|
||||
validate_graph(graph)
|
||||
|
||||
def test_accepts_only_protocol_actions(self):
|
||||
actions = [
|
||||
{"type": "set_widget_value", "node_id": 1, "widget": "steps", "value": 25},
|
||||
{"type": "set_node_mode", "node_id": 2, "mode": "bypass"},
|
||||
{"type": "queue_workflow"},
|
||||
{"type": "delete_node", "node_id": 3},
|
||||
{"type": "set_node_mode", "node_id": 2, "mode": "delete"},
|
||||
{"type": "set_widget_value", "node_id": 1, "widget": "x", "value": {"unsafe": True}},
|
||||
]
|
||||
self.assertEqual(len(validate_actions(actions)), 3)
|
||||
|
||||
def test_parses_json_and_fenced_json(self):
|
||||
payload = {"message": "Done", "actions": [{"type": "queue_workflow"}]}
|
||||
for text in (json.dumps(payload), f"```json\n{json.dumps(payload)}\n```"):
|
||||
with self.subTest(text=text):
|
||||
result = parse_model_response(text)
|
||||
self.assertEqual(result["message"], payload["message"])
|
||||
self.assertEqual(result["actions"], payload["actions"])
|
||||
self.assertEqual(result["thinking"], "")
|
||||
|
||||
def test_extracts_thinking_from_response(self):
|
||||
text = THINK_OPEN + "I should queue the workflow." + THINK_CLOSE + "\n" + json.dumps(
|
||||
{"message": "Done", "actions": [{"type": "queue_workflow"}]}
|
||||
)
|
||||
result = parse_model_response(text)
|
||||
self.assertEqual(result["thinking"], "I should queue the workflow.")
|
||||
self.assertEqual(result["message"], "Done")
|
||||
self.assertEqual(len(result["actions"]), 1)
|
||||
|
||||
def test_plain_text_becomes_message_without_actions(self):
|
||||
self.assertEqual(parse_model_response("I cannot do that"), {"message": "I cannot do that", "actions": [], "choices": [], "thinking": "", "parsed": False})
|
||||
|
||||
def test_prompt_contains_history_and_graph(self):
|
||||
prompt = build_prompt([{"role": "user", "content": "Set steps"}], {"nodes": [{"id": 1}]})
|
||||
self.assertIn("Set steps", prompt)
|
||||
self.assertIn('"id":1', prompt)
|
||||
self.assertIn("set_widget_value", prompt)
|
||||
self.assertIn("images", prompt)
|
||||
self.assertIn("Workflow prompt text must be English", prompt)
|
||||
self.assertIn("mirror the LATEST user message language", prompt)
|
||||
self.assertIn("MiniMax H3 video sampler", prompt)
|
||||
self.assertIn("include queue_workflow in the same response", prompt)
|
||||
|
||||
def test_prompt_identifies_exact_promoted_passthrough_target(self):
|
||||
graph = {"nodes": [{"id": 105, "widgets": [
|
||||
{"name": "prompt", "value": ""},
|
||||
{"name": "preset_prompt", "value": "MiniMax H3 NSFW (5s)"},
|
||||
{"name": "passthrough", "value": False},
|
||||
]}]}
|
||||
prompt = build_prompt([{"role": "user", "content": "Generate the video"}], graph)
|
||||
self.assertIn('Node 105 exposes both "prompt" and "passthrough"', prompt)
|
||||
self.assertIn('set node 105 widget "prompt"', prompt)
|
||||
self.assertIn('set node 105 widget "passthrough" to true', prompt)
|
||||
|
||||
def test_prompt_uses_inner_enhancer_when_image_is_provided(self):
|
||||
graph = {"nodes": [{"id": 105, "title": "Image to Video (MiniMax H3)", "widgets": [
|
||||
{"name": "prompt", "value": ""},
|
||||
{"name": "preset_prompt", "value": "MiniMax H3 NSFW (5s)"},
|
||||
{"name": "passthrough", "value": True},
|
||||
]}]}
|
||||
prompt = build_prompt([{"role": "user", "content": "Generate the video"}], graph, has_images=True)
|
||||
self.assertIn("Image pixels are provided", prompt)
|
||||
self.assertIn("Inspect the provided image pixels to understand how the requested action applies", prompt)
|
||||
self.assertIn('currently selects preset "MiniMax H3 NSFW (5s)"', prompt)
|
||||
self.assertIn("IGNORE any full prompt-writing guide for that preset", prompt)
|
||||
self.assertIn("the inner QwenVL node will use it to build the final prompt", prompt)
|
||||
self.assertIn('You MUST set node 105 widget "prompt" to a concise English action directive', prompt)
|
||||
self.assertIn('set node 105 widget "passthrough" to false', prompt)
|
||||
self.assertIn("inner QwenVL must analyze the image and create the final preset prompt", prompt)
|
||||
# The full MiniMax format guide must not leak into the chat prompt for an image enhancer.
|
||||
self.assertNotIn("integrated_multimodal_description:", prompt)
|
||||
self.assertNotIn("overall_soundscape:", prompt)
|
||||
|
||||
def test_selects_previous_intent_after_execution_confirmation(self):
|
||||
descriptive = "Create a five-second video where she opens the dress"
|
||||
for confirmation in (
|
||||
"Generate the video", "Esegui il video", "Run it", "Genera", "Ok",
|
||||
"Sì", "Avvia", "Esegui il video con il prompt precedente", "Go ahead",
|
||||
):
|
||||
messages = [
|
||||
{"role": "user", "content": descriptive},
|
||||
{"role": "assistant", "content": "Ready."},
|
||||
{"role": "user", "content": confirmation},
|
||||
]
|
||||
with self.subTest(confirmation=confirmation):
|
||||
self.assertEqual(select_workflow_intent(messages), descriptive)
|
||||
|
||||
def test_execution_only_detection_keeps_descriptive_requests(self):
|
||||
for content in (
|
||||
"genera un video in cui la ragazza balla",
|
||||
"crea un video di 5 secondi dall'immagine allegata",
|
||||
"la ragazza sposta la mano e si afferra il seno",
|
||||
"modifica il prompt per dire che lei sorride",
|
||||
):
|
||||
with self.subTest(content=content):
|
||||
self.assertEqual(select_workflow_intent([{"role": "user", "content": content}]), content)
|
||||
|
||||
def test_enforces_image_enhancer_prompt_and_disables_passthrough(self):
|
||||
graph = {"nodes": [{"id": 105, "title": "Image to Video (MiniMax H3)", "widgets": [
|
||||
{"name": "prompt", "value": "old prompt"},
|
||||
{"name": "preset_prompt", "value": "🎬 MiniMax H3 NSFW (5s)"},
|
||||
{"name": "passthrough", "value": True},
|
||||
]}]}
|
||||
messages = [
|
||||
{"role": "user", "content": "La ragazza apre il vestito"},
|
||||
{"role": "assistant", "content": "Vuoi che lo esegua?"},
|
||||
{"role": "user", "content": "Esegui il video"},
|
||||
]
|
||||
cases = [
|
||||
([], messages[0]["content"]),
|
||||
([{"type": "set_widget_value", "node_id": 105, "widget": "prompt", "value": "integrated_multimodal_description: [Shot 1] hallucinated scene"}], messages[0]["content"]),
|
||||
([{"type": "set_widget_value", "node_id": 105, "widget": "prompt", "value": "Esegui il video"}], messages[0]["content"]),
|
||||
]
|
||||
for prompt_action, expected in cases:
|
||||
with self.subTest(prompt_action=prompt_action):
|
||||
result = {
|
||||
"message": "Done",
|
||||
"actions": [
|
||||
{"type": "set_widget_value", "node_id": 105, "widget": "preset_prompt", "value": "🎬 MiniMax H3 NSFW (5s)"},
|
||||
*prompt_action,
|
||||
{"type": "set_widget_value", "node_id": 105, "widget": "passthrough", "value": True},
|
||||
{"type": "queue_workflow"},
|
||||
],
|
||||
}
|
||||
enforced = enforce_image_enhancer_routing(result, graph, messages, True)
|
||||
prompt_actions = [action for action in enforced["actions"] if action.get("widget") == "prompt"]
|
||||
self.assertEqual(len(prompt_actions), 1)
|
||||
self.assertEqual(prompt_actions[0]["value"], expected)
|
||||
self.assertFalse(next(action for action in enforced["actions"] if action.get("widget") == "passthrough")["value"])
|
||||
|
||||
def test_keeps_model_directive_for_image_enhancer(self):
|
||||
graph = {"nodes": [{"id": 105, "title": "Image to Video (MiniMax H3)", "widgets": [
|
||||
{"name": "prompt", "value": "old prompt"},
|
||||
{"name": "preset_prompt", "value": "🎬 MiniMax H3 NSFW (5s)"},
|
||||
{"name": "passthrough", "value": True},
|
||||
]}]}
|
||||
messages = [{"role": "user", "content": "la ragazza sposta la mano e si afferra il seno"}]
|
||||
directive = "The woman moves her hand and grabs her breast. Preserve the reference image exactly and change only this action."
|
||||
result = {
|
||||
"message": "Done",
|
||||
"actions": [
|
||||
{"type": "set_widget_value", "node_id": 105, "widget": "prompt", "value": directive},
|
||||
{"type": "set_widget_value", "node_id": 105, "widget": "passthrough", "value": True},
|
||||
{"type": "queue_workflow"},
|
||||
],
|
||||
}
|
||||
enforced = enforce_image_enhancer_routing(result, graph, messages, True)
|
||||
prompt_action = next(action for action in enforced["actions"] if action.get("widget") == "prompt")
|
||||
self.assertEqual(prompt_action["value"], directive)
|
||||
self.assertIn(directive, enforced["message"])
|
||||
self.assertFalse(next(action for action in enforced["actions"] if action.get("widget") == "passthrough")["value"])
|
||||
|
||||
def test_enforces_minimax_i2va_binding_for_image_passthrough(self):
|
||||
result = {
|
||||
"message": "Prompt generated.",
|
||||
"actions": [
|
||||
{"type": "set_widget_value", "node_id": 105, "widget": "prompt", "value": "integrated_multimodal_description: [Shot 1] action"},
|
||||
{"type": "set_widget_value", "node_id": 105, "widget": "passthrough", "value": True},
|
||||
{"type": "queue_workflow"},
|
||||
],
|
||||
"choices": [],
|
||||
"thinking": "",
|
||||
}
|
||||
graph = {"nodes": [{"id": 105, "title": "Image to Video (MiniMax H3)", "widgets": [
|
||||
{"name": "prompt", "value": ""},
|
||||
{"name": "preset_prompt", "value": "🎬 MiniMax H3 NSFW (5s)"},
|
||||
{"name": "passthrough", "value": False},
|
||||
]}]}
|
||||
enforced = enforce_image_reference_bindings(result, graph, True)
|
||||
self.assertTrue(enforced["actions"][0]["value"].startswith(MINIMAX_I2VA_BINDING))
|
||||
self.assertIn(MINIMAX_I2VA_BINDING, enforced["message"])
|
||||
|
||||
def test_skips_minimax_binding_when_passthrough_set_false(self):
|
||||
"""When the action set flips passthrough to false, do not prepend the I2VA binding
|
||||
even if the workflow snapshot still has passthrough=true."""
|
||||
result = {
|
||||
"message": "Prompt generated.",
|
||||
"actions": [
|
||||
{"type": "set_widget_value", "node_id": 105, "widget": "prompt", "value": "the woman touches her nipple"},
|
||||
{"type": "set_widget_value", "node_id": 105, "widget": "passthrough", "value": False},
|
||||
{"type": "queue_workflow"},
|
||||
],
|
||||
"choices": [],
|
||||
"thinking": "",
|
||||
}
|
||||
graph = {"nodes": [{"id": 105, "title": "Image to Video (MiniMax H3)", "widgets": [
|
||||
{"name": "prompt", "value": ""},
|
||||
{"name": "preset_prompt", "value": "🎬 MiniMax H3 NSFW (5s)"},
|
||||
{"name": "passthrough", "value": True},
|
||||
]}]}
|
||||
enforced = enforce_image_reference_bindings(result, graph, True)
|
||||
self.assertNotIn(MINIMAX_I2VA_BINDING, enforced["actions"][0]["value"])
|
||||
self.assertNotIn(MINIMAX_I2VA_BINDING, enforced["message"])
|
||||
|
||||
def test_validates_images(self):
|
||||
import base64
|
||||
valid = base64.b64encode(b"fake-image-data").decode("ascii")
|
||||
self.assertEqual(len(validate_images([valid, "not-valid", 123])), 1)
|
||||
self.assertEqual(len(validate_images([valid, valid, valid, valid])), 3)
|
||||
|
||||
def test_video_frames_use_higher_limit(self):
|
||||
import base64
|
||||
frame = base64.b64encode(b"frame").decode("ascii")
|
||||
self.assertEqual(len(validate_images([frame] * 5, 4)), 4)
|
||||
self.assertEqual(len(validate_images([frame] * 5)), 3)
|
||||
|
||||
def test_prompt_mentions_video_frames(self):
|
||||
prompt = build_prompt([{"role": "user", "content": "make it faster"}], {"nodes": []}, has_video=True)
|
||||
self.assertIn("VIDEO INPUT", prompt)
|
||||
prompt_no_video = build_prompt([{"role": "user", "content": "hi"}], {"nodes": []})
|
||||
self.assertNotIn("VIDEO INPUT", prompt_no_video)
|
||||
|
||||
def _jpeg_b64(self):
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (8, 8), (128, 64, 32)).save(buffer, "JPEG")
|
||||
return base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
|
||||
def test_hf_chat_forwards_video_frames(self):
|
||||
class FakeQuantization:
|
||||
Q8 = types.SimpleNamespace(value="q8")
|
||||
|
||||
class FakeBase:
|
||||
def load_model(self, *args):
|
||||
pass
|
||||
|
||||
def generate(self, *args, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.frame_count = args[3]
|
||||
return '{"message":"ok","actions":[]}'
|
||||
|
||||
previous = sys.modules.get("AILab_QwenVL")
|
||||
sys.modules["AILab_QwenVL"] = types.SimpleNamespace(
|
||||
HF_ALL_MODELS={"test-model": {}},
|
||||
Quantization=FakeQuantization,
|
||||
QwenVLBase=FakeBase,
|
||||
)
|
||||
try:
|
||||
runtime = ChatRuntime()
|
||||
frames = [self._jpeg_b64() for _ in range(4)]
|
||||
result = runtime.chat("hf", "test-model", [{"role": "user", "content": "make it faster"}], {"nodes": []}, {}, video=frames)
|
||||
self.assertEqual(result["message"], "ok")
|
||||
instance = runtime._instances["hf"]
|
||||
self.assertEqual(len(instance.kwargs["video"]), 4)
|
||||
self.assertEqual(instance.frame_count, 4)
|
||||
finally:
|
||||
if previous is None:
|
||||
sys.modules.pop("AILab_QwenVL", None)
|
||||
else:
|
||||
sys.modules["AILab_QwenVL"] = previous
|
||||
|
||||
def test_gguf_chat_reencodes_video_frames_as_images(self):
|
||||
class FakeBase:
|
||||
def _load_model(self, *args):
|
||||
pass
|
||||
|
||||
def _invoke(self, system, prompt, images_b64, *args, **kwargs):
|
||||
self.images_b64 = images_b64
|
||||
return '{"message":"ok","actions":[]}'
|
||||
|
||||
previous = sys.modules.get("AILab_QwenVL_GGUF")
|
||||
sys.modules["AILab_QwenVL_GGUF"] = types.SimpleNamespace(
|
||||
GGUF_VL_CATALOG={"models": {"test-gguf": {}}},
|
||||
QwenVLGGUFBase=FakeBase,
|
||||
)
|
||||
try:
|
||||
runtime = ChatRuntime()
|
||||
frames = [self._jpeg_b64() for _ in range(4)]
|
||||
result = runtime.chat("gguf", "test-gguf", [{"role": "user", "content": "make it faster"}], {"nodes": []}, {}, video=frames)
|
||||
self.assertEqual(result["message"], "ok")
|
||||
instance = runtime._instances["gguf"]
|
||||
self.assertEqual(len(instance.images_b64), 4)
|
||||
# Frames must round-trip as valid base64 strings (not raw bytes)
|
||||
import base64
|
||||
for item in instance.images_b64:
|
||||
self.assertIsInstance(item, str)
|
||||
base64.b64decode(item, validate=True)
|
||||
finally:
|
||||
if previous is None:
|
||||
sys.modules.pop("AILab_QwenVL_GGUF", None)
|
||||
else:
|
||||
sys.modules["AILab_QwenVL_GGUF"] = previous
|
||||
|
||||
def test_lists_nested_output_images(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / "root.png").write_bytes(b"png")
|
||||
(root / "PMP" / "2026-09-16").mkdir(parents=True)
|
||||
(root / "PMP" / "2026-09-16" / "nested.webp").write_bytes(b"webp")
|
||||
(root / "ignored.mp4").write_bytes(b"video")
|
||||
(root / "ignored.txt").write_text("text")
|
||||
assets = list_output_images(root)
|
||||
self.assertEqual(set(assets), {"root.png [output]", "PMP/2026-09-16/nested.webp [output]", "ignored.mp4 [output]"})
|
||||
|
||||
def test_hf_runtime_reuses_and_unloads_model(self):
|
||||
class FakeQuantization:
|
||||
Q8 = types.SimpleNamespace(value="q8")
|
||||
|
||||
class FakeBase:
|
||||
def __init__(self):
|
||||
self.loaded = 0
|
||||
self.cleared = False
|
||||
|
||||
def load_model(self, *args):
|
||||
self.loaded += 1
|
||||
|
||||
def generate(self, *args, **kwargs):
|
||||
return '{"message":"ok","actions":[]}'
|
||||
|
||||
def clear(self):
|
||||
self.cleared = True
|
||||
|
||||
previous = sys.modules.get("AILab_QwenVL")
|
||||
sys.modules["AILab_QwenVL"] = types.SimpleNamespace(
|
||||
HF_ALL_MODELS={"test-model": {}},
|
||||
Quantization=FakeQuantization,
|
||||
QwenVLBase=FakeBase,
|
||||
)
|
||||
try:
|
||||
runtime = ChatRuntime()
|
||||
result = runtime.chat("hf", "test-model", [{"role": "user", "content": "hello"}], {"nodes": []}, {})
|
||||
self.assertEqual(result["message"], "ok")
|
||||
instance = runtime._instances["hf"]
|
||||
self.assertEqual(instance.loaded, 1)
|
||||
self.assertEqual(runtime.unload("hf"), ["hf"])
|
||||
self.assertTrue(instance.cleared)
|
||||
finally:
|
||||
if previous is None:
|
||||
sys.modules.pop("AILab_QwenVL", None)
|
||||
else:
|
||||
sys.modules["AILab_QwenVL"] = previous
|
||||
|
||||
def test_minimax_preset_mode_detection(self):
|
||||
self.assertEqual(_minimax_preset_mode("🔄 MiniMax H3 NSFW FL2VA (10s)"), "FL2VA")
|
||||
self.assertEqual(_minimax_preset_mode("🎞️ MiniMax H3 NSFW R2VA (5s)"), "R2VA")
|
||||
self.assertIsNone(_minimax_preset_mode("🎬 MiniMax H3 NSFW (5s)"))
|
||||
|
||||
def test_match_minimax_preset_prefers_same_mode(self):
|
||||
preset_widget = {
|
||||
"options": {
|
||||
"values": [
|
||||
"🎬 MiniMax H3 NSFW (5s)",
|
||||
"🎬 MiniMax H3 NSFW (10s)",
|
||||
"🔄 MiniMax H3 NSFW FL2VA (5s)",
|
||||
"🔄 MiniMax H3 NSFW FL2VA (10s)",
|
||||
"🎞️ MiniMax H3 NSFW R2VA (5s)",
|
||||
]
|
||||
}
|
||||
}
|
||||
self.assertEqual(
|
||||
_match_minimax_preset(preset_widget, 10, "🔄 MiniMax H3 NSFW FL2VA (5s)"),
|
||||
"🔄 MiniMax H3 NSFW FL2VA (10s)",
|
||||
)
|
||||
self.assertEqual(
|
||||
_match_minimax_preset(preset_widget, 10, "🎬 MiniMax H3 NSFW (5s)"),
|
||||
"🎬 MiniMax H3 NSFW (10s)",
|
||||
)
|
||||
|
||||
def test_minimax_result_keeps_fl2va_preset_family(self):
|
||||
graph = {
|
||||
"nodes": [
|
||||
{
|
||||
"id": 105,
|
||||
"widgets": [
|
||||
{"name": "unet_name", "value": "x", "options": {"values": ["minimax_h3_fl2va_pruned_nvfp4_convrot_int8.safetensors"]}},
|
||||
{
|
||||
"name": "preset_prompt",
|
||||
"value": "🔄 MiniMax H3 NSFW FL2VA (5s)",
|
||||
"options": {
|
||||
"values": [
|
||||
"🎬 MiniMax H3 NSFW (5s)",
|
||||
"🎬 MiniMax H3 NSFW (10s)",
|
||||
"🔄 MiniMax H3 NSFW FL2VA (5s)",
|
||||
"🔄 MiniMax H3 NSFW FL2VA (10s)",
|
||||
]
|
||||
},
|
||||
},
|
||||
{"name": "passthrough", "value": True},
|
||||
{"name": "prompt", "value": ""},
|
||||
{"name": "steps", "value": 8, "options": {"values": ["res_multistep", "euler"]}},
|
||||
{"name": "sampler_name", "value": "euler", "options": {"values": ["euler", "res_multistep"]}},
|
||||
{"name": "scheduler", "value": "simple", "options": {"values": ["simple"]}},
|
||||
{"name": "shift_video", "value": 6, "options": {"values": [6, 12]}},
|
||||
{"name": "shift_audio", "value": 3, "options": {"values": [3]}},
|
||||
{"name": "value_1", "value": 5},
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
result = _minimax_result(graph, "native", "use native 10 seconds")
|
||||
self.assertIsNotNone(result)
|
||||
preset_action = next(a for a in result["actions"] if a["widget"] == "preset_prompt")
|
||||
self.assertEqual(preset_action["value"], "🔄 MiniMax H3 NSFW FL2VA (10s)")
|
||||
value_action = next(a for a in result["actions"] if a["widget"] == "value_1")
|
||||
self.assertEqual(value_action["value"], 10)
|
||||
|
||||
def test_image_enhancer_detected_by_image_input(self):
|
||||
node = {
|
||||
"id": 105,
|
||||
"type": "4c314f31-ecda-4b08-ae98-faaba1bf613f",
|
||||
"title": None,
|
||||
"widgets": [
|
||||
{"name": "prompt", "value": ""},
|
||||
{"name": "preset_prompt", "value": "🔄 MiniMax H3 NSFW FL2VA (5s)"},
|
||||
{"name": "passthrough", "value": True},
|
||||
],
|
||||
"inputs": [{"name": "image", "type": "IMAGE", "link": 1}],
|
||||
}
|
||||
self.assertTrue(_is_image_enhancer_node(node))
|
||||
self.assertTrue(_has_image_enhancer_target({"nodes": [node]}))
|
||||
|
||||
def test_prompt_suppresses_video_guides_for_uuid_image_enhancer(self):
|
||||
graph = {
|
||||
"nodes": [
|
||||
{
|
||||
"id": 105,
|
||||
"type": "4c314f31-ecda-4b08-ae98-faaba1bf613f",
|
||||
"title": None,
|
||||
"widgets": [
|
||||
{"name": "prompt", "value": ""},
|
||||
{"name": "preset_prompt", "value": "🔄 MiniMax H3 NSFW FL2VA (5s)"},
|
||||
{"name": "passthrough", "value": True},
|
||||
],
|
||||
"inputs": [{"name": "image", "type": "IMAGE", "link": 1}],
|
||||
}
|
||||
]
|
||||
}
|
||||
prompt = build_prompt([{"role": "user", "content": "10 seconds video"}], graph, has_images=True)
|
||||
self.assertIn("Exact image-enhancer target", prompt)
|
||||
self.assertNotIn("integrated_multimodal_description:", prompt)
|
||||
self.assertNotIn("overall_soundscape:", prompt)
|
||||
|
||||
def test_fix_minimax_preset_corrects_generic_to_fl2va(self):
|
||||
graph = {
|
||||
"nodes": [
|
||||
{
|
||||
"id": 105,
|
||||
"widgets": [
|
||||
{"name": "unet_name", "value": "x"},
|
||||
{"name": "preset_prompt", "value": "🔄 MiniMax H3 NSFW FL2VA (5s)", "options": {"values": [
|
||||
"🎬 MiniMax H3 NSFW (5s)",
|
||||
"🎬 MiniMax H3 NSFW (10s)",
|
||||
"🔄 MiniMax H3 NSFW FL2VA (5s)",
|
||||
"🔄 MiniMax H3 NSFW FL2VA (10s)",
|
||||
]}},
|
||||
{"name": "passthrough", "value": True},
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
result = {
|
||||
"message": "Done",
|
||||
"actions": [
|
||||
{"type": "set_widget_value", "node_id": 105, "widget": "preset_prompt", "value": "🎬 MiniMax H3 NSFW (10s)"},
|
||||
{"type": "queue_workflow"},
|
||||
],
|
||||
}
|
||||
fixed = _fix_minimax_preset_actions(result, graph)
|
||||
preset_action = next(a for a in fixed["actions"] if a["widget"] == "preset_prompt")
|
||||
self.assertEqual(preset_action["value"], "🔄 MiniMax H3 NSFW FL2VA (10s)")
|
||||
|
||||
|
||||
def test_ensure_i2va_binding_prepends_missing_line(self):
|
||||
binding = "For the target video, at 0.00 seconds into the target video, <Picture 1> (from [Shot 1]) is fully referenced."
|
||||
body = "integrated_multimodal_description: [Shot 1] A woman smiles."
|
||||
result = ensure_i2va_binding(body, "🎬 MiniMax H3 NSFW (5s)", has_image=True)
|
||||
self.assertTrue(result.startswith(binding))
|
||||
self.assertIn(body, result)
|
||||
|
||||
def test_ensure_i2va_binding_keeps_existing_line(self):
|
||||
binding = "For the target video, at 0.00 seconds into the target video, <Picture 1> (from [Shot 1]) is fully referenced."
|
||||
original = f"{binding}\n\nintegrated_multimodal_description: [Shot 1] A woman smiles."
|
||||
self.assertEqual(ensure_i2va_binding(original, "🎬 MiniMax H3 NSFW (5s)", has_image=True), original)
|
||||
|
||||
def test_ensure_i2va_binding_skips_fl2va(self):
|
||||
body = "How the reference pictures align with the target video ..."
|
||||
self.assertEqual(
|
||||
ensure_i2va_binding(body, "🔄 MiniMax H3 NSFW FL2VA (5s)", has_image=True),
|
||||
body,
|
||||
)
|
||||
|
||||
def test_normalize_minimax_removes_duplicate_shot_blocks(self):
|
||||
binding = "For the target video, at 0.00 seconds into the target video, <Picture 1> (from [Shot 1]) is fully referenced."
|
||||
body = "integrated_multimodal_description: [Shot 1] ...\noverall_soundscape: ...\nnon_diegetic_music: N/A"
|
||||
duplicate = f"[Shot 1] A woman smiles.\n[Shot 2] She turns.\n\n{binding}\n\n{body}"
|
||||
result = normalize_minimax_output(duplicate, "🎬 MiniMax H3 NSFW (5s)", has_image=True)
|
||||
self.assertTrue(result.startswith(binding))
|
||||
self.assertNotIn("[Shot 2] She turns.", result.split("integrated_multimodal_description:")[0])
|
||||
self.assertIn("integrated_multimodal_description:", result)
|
||||
|
||||
def test_normalize_minimax_adds_missing_i2va_binding(self):
|
||||
body = "integrated_multimodal_description: [Shot 1] A woman smiles.\noverall_soundscape: ...\nnon_diegetic_music: N/A"
|
||||
result = normalize_minimax_output(body, "🎬 MiniMax H3 NSFW (5s)", has_image=True)
|
||||
self.assertIn("For the target video", result)
|
||||
self.assertIn("integrated_multimodal_description:", result)
|
||||
|
||||
def test_normalize_minimax_replaces_noncanonical_i2va_binding(self):
|
||||
malformed = "For the target video, at 0.00 seconds into the target video, Picture 1 (from [Shot 1]) is fully referenced."
|
||||
body = "integrated_multimodal_description: [Shot 1] A woman smiles.\noverall_soundscape: ...\nnon_diegetic_music: N/A"
|
||||
result = normalize_minimax_output(f"{malformed}\n\n{body}", "MiniMax › NSFW", has_image=True)
|
||||
self.assertEqual(result.count("For the target video"), 1)
|
||||
self.assertTrue(result.startswith(MINIMAX_I2VA_BINDING))
|
||||
self.assertNotIn(malformed, result)
|
||||
|
||||
def test_normalize_minimax_keeps_fl2va_alignment(self):
|
||||
alignment = "How the reference pictures align with the target video — Picture 1 (from [Shot 1]) aligns with the 0.00-second mark; Picture 2 aligns with the 5.00-second mark."
|
||||
body = "integrated_multimodal_description: [Shot 1] ...\noverall_soundscape: ...\nnon_diegetic_music: N/A"
|
||||
text = f"{alignment}\n\n{body}"
|
||||
result = normalize_minimax_output(text, "🔄 MiniMax H3 NSFW FL2VA (5s)", has_image=True)
|
||||
self.assertTrue(result.startswith(alignment))
|
||||
self.assertIn("integrated_multimodal_description:", result)
|
||||
|
||||
def test_normalize_minimax_synthesizes_missing_fl2va_alignment(self):
|
||||
body = "integrated_multimodal_description: [Shot 1] ...\noverall_soundscape: ...\nnon_diegetic_music: N/A"
|
||||
result = normalize_minimax_output(body, "🔄 MiniMax H3 NSFW FL2VA (5s)", has_image=True, duration="10s")
|
||||
self.assertTrue(result.startswith("How the reference pictures align"))
|
||||
self.assertIn("10.00-second mark", result)
|
||||
self.assertIn("integrated_multimodal_description:", result)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,575 +1,5 @@
|
||||
# ComfyUI-QwenVL Update Log
|
||||
|
||||
## Version 2.12.12 (2026/11/15)
|
||||
|
||||
- Duration clamp fix: selecting a duration a preset does not define (e.g. 20s on MiniMax presets — 20s is LTX-only) silently fell back to the preset's default (5s). `resolve_prompt_key` now clamps to the closest available duration instead (MiniMax 20s -> 15s, the model's real maximum).
|
||||
|
||||
## Version 2.12.9 (2026/10/30)
|
||||
|
||||
- New VL preset `LTX › NSFW Loop` for the LTX-2.5 pose-loop workflows (A→B→C→B→A keyframe loops): derived from the FL2VA guidance but framed for MULTIPLE keyframe images visited in order, with the loop-closure rule (final pose = first pose) instead of the two-frame transition framing. Source of truth is Garage presets/ltx.json — the bundled copy + chat alias ship with this release as offline fallback.
|
||||
|
||||
## Version 2.12.8 (2026/10/24)
|
||||
|
||||
- Chat defaults aligned with the QwenVL node: default GGUF model is the Q6_K (was Q8_0 — a second download users did not expect), and the chat's own max-tokens default is 2048 (the js always sent its own 1024, so the backend's raised default never applied).
|
||||
- Director Analyze: new /minimax_director/llm_models endpoint + a model dropdown in the Analyze settings when several GGUF models are found (the free-text field stays as fallback).
|
||||
|
||||
## Version 2.12.7 (2026/10/24)
|
||||
|
||||
- Storyboard request handled OUTSIDE the JSON protocol: the GGUF model kept summarizing the stills away ('Still 1/2/3 ready') no matter what the guide said. Now the stills are generated as raw text by a dedicated recipe prompt (Still 1/2/3 + dense Director idea) and the chat reply is composed in code — preset IMG › Storyboard + passthrough=false applied, three Generate Still choices offered, zero queue on the request. The still clicks recover each still from this composed message.
|
||||
|
||||
## Version 2.12.6 (2026/10/24)
|
||||
|
||||
- Storyboard routing: the generic image-enhancer enforcement no longer runs after the storyboard routing took over — it was re-writing the enhancer's prompt widget with a garbage "intent" cleaned from the bare Generate Still message ("till 1"). Storyboard mode now emits exactly: processor text+populated_text, passthrough false, queue.
|
||||
|
||||
## Version 2.12.5 (2026/10/24)
|
||||
|
||||
- Chat **Flow dropdown** (next to Config MMH3): Auto / Storyboard stills. Storyboard mode injects the still-flow guide unconditionally and makes the routing deterministic in code — the still prompt always lands on the WildcardProcessor's text widgets (or the enhancer's prompt widget when the graph has no processor), passthrough stays false, queue only on a Generate Still choice. No more guide-vs-node-rule conflicts deciding the outcome.
|
||||
|
||||
## Version 2.12.4 (2026/10/24)
|
||||
|
||||
- Chat storyboard flow fixes from live testing: the node snapshot now carries the input socket names (the enhancer routing never fired without them, so the chat set passthrough=true and bypassed the QwenVL); the storyboard guide now targets the WildcardProcessor's text widgets (the QwenVL's prompt input is WIRED from it — writing the QwenVL's prompt widget had no effect, which produced character sheets instead of stills); no queue on the storyboard request itself, only on a Generate Still choice.
|
||||
|
||||
## Version 2.12.3 (2026/10/24)
|
||||
|
||||
- Fixed the chat crash (ComfyUI process aborted): the graph snapshot rode into the model context whole every turn — the Edit21 workflow's PixNote HTML alone is ~5.5KB, so the second chat turn overflowed the GGUF context and llama.cpp aborted (Fatal Python error). Widget values in the snapshot are now capped at 300 chars; operational widgets (prompt, preset, steps…) stay intact.
|
||||
|
||||
## Version 2.12.2 (2026/10/23)
|
||||
|
||||
- Fixed "missing node type" / `AttributeError: module 'AILab_QwenVL_GGUF_PromptEnhancer' has no attribute ...`: the GGUF modules no longer raise at import when llama-cpp-python is missing — all nodes register, the Unified node hides GGUF models with an explanatory note, and a clear actionable error appears only when a GGUF model actually runs. The node loader also drops half-initialized modules so a failed import can no longer poison later imports.
|
||||
|
||||
## Version 2.11.1 (2026/10/06)
|
||||
|
||||
- Cleaned the GGUF and HF model catalogs: three VL-capable DavidAU heretic models per backend (4B / 9B / 27B) — all verified `image-text-to-text` with working mmproj files; dropped text-only duplicates and the mislabeled "3.8-9B" entry. VL models cover text-only enhance too, so the separate text section is gone.
|
||||
- MiniMax R2VA presets (NSFW + SFW): fixed the reference-image semantics — a reference supplies subject identity only, it is never treated as a keyframe to reproduce. Character sheets and multi-view refs now produce a new scene instead of animating the sheet itself.
|
||||
- MiniMax presets (all durations): completed dialogue handling across R2VA/FL2VA/T2V — `[DIALOGUE]…[/DIALOGUE]` and `[SBJn] says [D]…[/D]` are preserved verbatim as `<d>[English] …</d>`, no invented lines; restored the truncated SFW FL2VA Loop tail.
|
||||
|
||||
## Version 2.11.0 (2026/10/06)
|
||||
|
||||
- New **MiniMax SFW preset family**: `MiniMax › SFW I2VA/R2VA/FL2VA/FL2VA Loop` on the VL nodes and `MiniMax › SFW T2V/R2VA T2V/FL2VA T2V` on the Prompt Enhancer — same structure, dialogue handling and mode-specific alignment as the NSFW presets, without the explicit-content directives.
|
||||
- Renamed the ambiguous `MiniMax › NSFW` VL preset to **MiniMax › NSFW I2VA** (legacy names keep working as aliases); added text-context aliases so `MiniMax › NSFW`/`MiniMax › SFW` resolve to the T2V variants instead of being used as raw templates.
|
||||
- I2VA output normalization: a single canonical `<Picture 1>` reference-binding line is enforced and malformed duplicate prefaces are stripped; `[SBJn] says [D]…[/D]` dialogue lost by the model is re-injected verbatim before `overall_soundscape`.
|
||||
|
||||
## Version 2.10.2 (2026/10/01)
|
||||
|
||||
- Removed the Livepeer Agent render node and its external MCP/network integration after the Livepeer Agent Hackathon ended.
|
||||
- Removed the associated Qwen Chat capability selector, deterministic capability routing, legacy alias, test, and promotional assets to keep the package focused and Registry-compatible.
|
||||
|
||||
## Version 2.10.1 (2026/09/26)
|
||||
|
||||
- New preset **MiniMax › NSFW FL2VA Loop** (5s/10s/15s): dedicated seamless-loop enhancer — feed the same image as first AND last frame; the prompt forces periodic-motion vocabulary, locked/oscillating camera, stable lighting, full-duration pacing and a cyclic soundscape so the clip closes perfectly on its opening frame.
|
||||
- The base **MiniMax › NSFW FL2VA** preset is now transition-only: LOOP MODE removed, end state correctly targets Picture 2 instead of the first frame when the two references differ.
|
||||
|
||||
## Version 2.9.9 (2026/09/25)
|
||||
|
||||
- Chat Config MMH3: added **R2VA Native** and **R2VA Native Turbo** — select the ref2va unet (image-ref conditioned) instead of fl2va, enable the ref2v turbo LoRA in turbo mode, and auto-switch the enhancer preset to an R2VA variant preserving the selected duration. Chat phrases like "usa r2va turbo" trigger them too.
|
||||
|
||||
## Version 2.9.8 (2026/09/25)
|
||||
|
||||
- New preset **IMG › Qwen Edit** on the QwenVL nodes (and **Qwen Image Edit** on the Prompt Enhancer): rewrites any input as a Qwen Image 2.1-style edit instruction — imperative edit clause, explicit keep-unchanged list, `<imageN>` placeholders preserved, reference-transfer and lighting-consistency clauses. Complements the `__qwen21/*__` wildcards for free-form edits.
|
||||
|
||||
- Node menu category now shows as `🔮 QwenVL-Mod`; new banner and icon.
|
||||
|
||||
## Version 2.9.7 (2026/09/25)
|
||||
|
||||
- Republish of the consolidated 2.9.5 content — registry tombstoned 2.9.5/2.9.6 after deletion, so the same feature set ships under a new version number.
|
||||
|
||||
## Version 2.9.5 (2026/09/25)
|
||||
|
||||
- Workflow Chat: wildcard autocomplete — typing `__` in the input lists TagForge wildcards (fuzzy-matched as you type); arrows/Enter/Tab to pick, Esc to dismiss. Silent when TagForge is not installed.
|
||||
- Workflow Chat: media words (video/clip/scene) are stripped only when they open the directive ("a video of...", "video, ..."), never mid-sentence — "lasts for all the video" now reaches the prompt intact.
|
||||
|
||||
## Version 2.9.4 (2026/09/25)
|
||||
|
||||
- Workflow Chat: action text no longer loses duration phrases that belong to the action itself (e.g. "the kiss lasts 5 seconds"); clip-length specs still set the duration widget only when they read as meta ("10s video", "video of 8 seconds").
|
||||
- Stray leading connectors ("of", "showing", "a") left behind by stripped duration/media words are cleaned up.
|
||||
|
||||
## Version 2.9.3 (2026/09/25)
|
||||
|
||||
- Workflow Chat: **wildcard support** — `__wildcard__` tokens in your message are expanded via ComfyUI-TagForge (when installed) before the request reaches the LLM, so action wildcards work inside chat prompts.
|
||||
- Repeat button now loads the last user message into the input for editing instead of resending it immediately.
|
||||
- Config selector label renamed to "Config MMH3" (the presets are MiniMax H3 specific).
|
||||
|
||||
## Version 2.9.2 (2026/09/25)
|
||||
|
||||
- Removed the bogus `requires-comfyui >=1.0.0` constraint — ComfyUI versions are 0.x, and the mismatch was disabling the node pack in ComfyUI Manager.
|
||||
|
||||
## Version 2.9.1 (2026/09/25)
|
||||
|
||||
- Rewrote the MiniMax config examples in the chat system prompt as plain prose — the dense `key=value;` action syntax was triggering the registry `python_minified_code` rule (false positive on `chat_service.py:88`).
|
||||
- Moved the registry admin workflow out of the published package.
|
||||
|
||||
## Version 2.9.0 (2026/09/25)
|
||||
|
||||
- MiniMax model matrix in Qwen Workflow Chat: `Native`, `Native Turbo`, `10Eros` and `10Eros Turbo` configs now drive the whole sampler setup — UNet selection (nvfp4 / 10Eros beta5 int8), step count (20 quality / 8 turbo), LoRA enable/bypass and the correct LoRA file per mode (lightx2v fl2v/ref2v vs TenStrip combined fusion).
|
||||
- `BlockSparseAttention` `tau` is set per config (1.0 quality, 1.3 turbo).
|
||||
- New unified HF/GGUF prompt enhancer node; `custom_prompt` widget renamed to `prompt`; backend dropdowns removed (model prefix is the single selector); `keep_model_loaded` defaults to False; duration becomes a widget.
|
||||
- Presets split into per-family files; style-tag presets now lead the output with an explicit style declaration; denser Pony→Natural and Detailed caption outputs.
|
||||
- MiniMax output normalization applied to the GGUF backend; legacy `AILab_*` nodes hidden from the add-node menu (superseded by Unified); refreshed node icons.
|
||||
- Publish workflow: changelog extraction rewritten in plain shell (registry scanner false positive on embedded Python).
|
||||
|
||||
## Version 2.8.14 (2026/09/22)
|
||||
|
||||
- Removed the duplicate `AILab_LivepeerRender` node from the Add Node menu.
|
||||
- Added `web/js/legacy_aliases.js` to rewrite old `AILab_LivepeerRender` nodes to `QwenVL_LivepeerRender` on workflow load, keeping old workflows compatible.
|
||||
- Updated `QwenVL_Livepeer.py` `NODE_CLASS_MAPPINGS` / `NODE_DISPLAY_NAME_MAPPINGS` to expose only the canonical node name.
|
||||
|
||||
## Version 2.8.13 (2026/09/22)
|
||||
|
||||
- Refreshed node branding with a Pixaroma-style colour scheme: each QwenVL-Mod family now has recognisable title/body colours (Vision, Vision GGUF, Enhancer, Livepeer, Utils).
|
||||
- Updated `web/js/appearance.js` to apply colours in `beforeRegisterNodeDef`, preserving user/workflow colour choices and covering all current nodes (Unified, Livepeer, LoadMedia, VRAM cleanup, story split).
|
||||
- Added shared stylesheet `web/css/qwen_nodes.css` and wrapped the chat composer in a dedicated panel for cleaner spacing and styling.
|
||||
- Standardised `QwenVL_LoadMedia` category to `QwenVL-Mod/Utils`.
|
||||
|
||||
## Version 2.8.12 (2026/09/22)
|
||||
|
||||
- Added `🎨 Pony→Natural Language` back to the QwenVL `preset_prompt` dropdown (`_preset_prompts` + `qwenvl`), in addition to the PromptEnhancer style.
|
||||
- The same converter is now available both as a vision preset (when an image is attached) and as a text-only style enhancement.
|
||||
|
||||
## Version 2.8.11 (2026/09/22)
|
||||
|
||||
- Removed the redundant `mmh3_acceleration` chat guide from `_chat_guides`. MiniMax H3 acceleration modes are already handled deterministically by Qwen Chat selectors and `use native/10eros/turbo` commands, so the extra system-prompt text was no longer useful.
|
||||
|
||||
## Version 2.8.10 (2026/09/22)
|
||||
|
||||
- Moved the new `🎨 Pony→Natural Language` preset from the QwenVL `preset_prompt` dropdown into the **PromptEnhancer** `enhancement_style` dropdown (`qwen_text.styles`).
|
||||
- Removed the old `🎨 Pony→Z-Image` and `🎨 Pony→Flux` PromptEnhancer styles.
|
||||
- The Pony tag converter is now selected as a **style** in the PromptEnhancer node (HF and GGUF), keeping the QwenVL preset list focused on vision/analysis/video tasks.
|
||||
|
||||
## Version 2.8.9 (2026/09/22)
|
||||
|
||||
- Replaced the model-specific `🎨 Pony→Z-Image` and `🎨 Pony→Flux` presets with a single generic `🎨 Pony→Natural Language` preset in `AILab_System_Prompts.json`.
|
||||
- The new preset converts Danbooru/Pony tag prompts into a clean, model-agnostic natural-language paragraph suitable for Flux, Krea, Qwen-Image 2.1, and other image generation models.
|
||||
- Removes `score_*` quality tags and Pony meta tags (`source_pony`, `rating_*`, etc.), preserves optional `[style]` tags, and handles NSFW content without censoring.
|
||||
|
||||
## Version 2.8.8 (2026/09/21)
|
||||
|
||||
- Turbo config now actually toggles the Turbo LoRA: the graph snapshot includes subgraph inner nodes (qualified ids like "105:213"), findNode resolves them, and the MiniMax configs emit set_node_mode on LoraLoader nodes inside the enhancer subgraph — Turbo enables it, Native/10Eros bypass it (10Eros has TURBO fused).
|
||||
- Turbo also resets unet_name to the NVFP4 checkpoint — 10Eros + Turbo LoRA stacking is forbidden.
|
||||
## Version 2.8.7 (2026/09/21)
|
||||
|
||||
- Selector dropdowns no longer inject "use <config>" into the chat text: the Config / Livepeer pickers are sent as a separate `directives` field and applied deterministically server-side, so the user message (and the prompt widget) stays clean.
|
||||
- Fixed duration regex matching "s" before "seconds" ("5 seconds" left "econds" in the prompt); also widened the capability-path duration parser to seconds/secondi.
|
||||
- System prompt: MiniMax action now targets the "prompt" widget (was preset_prompt, inconsistent with the deterministic path).
|
||||
## Version 2.8.6 (2026/09/21)
|
||||
|
||||
- Fixed directive cleaning when the config command is separated by a period ("use native. generate a 5s video, ..."): the leftover leading dot blocked the generation-prefix stripper, leaving "generate a," fragments in the prompt. Leading dots are now stripped after each cleaning stage; legitimate trailing periods are preserved.
|
||||
|
||||
## Version 2.8.5 (2026/09/21)
|
||||
|
||||
- Added selector dropdowns under the chat composer: a **Config** picker (Auto / Native / 10Eros / Turbo LoRA) injects the deterministic MiniMax command, and a **Livepeer** picker is auto-populated from the render node's capability widget — no more typing `use <name>` by hand. Both selectors persist, work with empty prompts (config-only switch), and skip the model check since routing is deterministic.
|
||||
- Fixed the explicit `use` parser swallowing the trailing separator ("use nano-banana. <prompt>" kept the dot in the capability name) and directive cleaning stripping legitimate trailing periods.
|
||||
|
||||
## Version 2.8.4 (2026/09/21)
|
||||
|
||||
- Fixed directive cleaning when a config keyword precedes the request ("usa 10Eros: genera un video di…"): the generation prefix is now stripped after config removal too, so no "genera un di" fragments reach the prompt widget.
|
||||
|
||||
## Version 2.8.3 (2026/09/21)
|
||||
|
||||
- Added deterministic chat shortcuts that skip the LLM entirely: `use <capability> <prompt>` drives the Livepeer render node directly (dropdown or `custom_capability`, with duration and aspect-ratio parsing), and `use native` / `use 10Eros` / `use turbo` / `config A-B-C` apply the full MiniMax H3 sampler config on the enhancer node — correct widgets every time, instant, and immune to malformed local-model JSON.
|
||||
- Config-only requests leave the prompt widget untouched; a cleaned scene directive is written when the message carries one.
|
||||
|
||||
## Version 2.8.2 (2026/09/21)
|
||||
|
||||
- Image-enhancer directives are now cleaned deterministically: routing keywords ("use native", "use 10Eros", "usa turbo"), generation prefixes ("generate a 5s video", "fai un video") and duration mentions are stripped before the action text reaches the prompt widget, so the inner enhancer receives only the scene action.
|
||||
- When the request contains only config keywords, the prompt widget is left untouched instead of receiving the meta text.
|
||||
|
||||
## Version 2.8.1 (2026/09/21)
|
||||
|
||||
- Rewrote the `/dlstatus.html` download-log page with a multi-line HTML template and `html.escape`, removing the long single-line string that the registry security scanner flagged as minified code.
|
||||
|
||||
## Version 2.8.0 (2026/09/21)
|
||||
|
||||
- Added `QwenVL_LivepeerRender`: a self-contained node that renders images and videos through the Livepeer Agent network over MCP, with a capability dropdown (image and video models), IMAGE output for still results, static preview for images and animated preview for video, and a `source_video` input that picks a frame from a clip as the i2v reference.
|
||||
- Added `QwenVL_LoadMedia` (utils category): a single picker for images and videos in input/output folders with built-in preview, an upload button for videos, and resolution of untagged input-directory names.
|
||||
- Qwen Chat now drives the Livepeer render node directly: it routes generation requests through the node (never bypassing it), clears stale `custom_capability` values, bypasses mismatched saver nodes and re-enables the matching one, and supports video attachments with browser-side frame sampling for the refine loop.
|
||||
- The chat assets panel lists videos too and syncs the selection with `QwenVL_LoadMedia`; the capability dropdown now includes image-generation models, and a MiniMax H3 config switch (10Eros/Native/Turbo) runs before prompt routing.
|
||||
- Added unified HF/GGUF QwenVL nodes with a backend dropdown.
|
||||
- Rewrote the chat system prompt: shorter, with concrete examples and MiniMax configuration handled first.
|
||||
- Fixed MCP error handling and stopped sending video parameters to image-only capabilities; the chat mirrors the latest user message language and no longer pre-formats enhancer prompts.
|
||||
|
||||
## Version 2.7.13 (2026/09/16)
|
||||
|
||||
- Fixed `enforce_image_reference_bindings` to use the `passthrough` value from the action set instead of the stale workflow snapshot, so I2VA workflows with `passthrough=false` no longer receive the `For the target video...` binding line in the chat-managed prompt.
|
||||
- Suppressed full-format video prompt-writing guides (MiniMax H3, LTX, Wan) from the chat prompt when an image-enhancer target is active, preventing the chat from outputting a complete `integrated_multimodal_description`/`overall_soundscape` prompt.
|
||||
- Strengthened the exact-target instruction: the chat must write only a concise English action directive and let the inner QwenVL node build the final preset prompt.
|
||||
|
||||
## Version 2.7.12 (2026/09/16)
|
||||
|
||||
- Strengthened image-enhancer chat instructions: Qwen Chat must inspect the provided image pixels to understand how the requested action applies and read the target node's current `preset_prompt` value and supplied prompt-writing guide.
|
||||
- The chat must emit an English `set_widget_value` enhancer directive for the target prompt instead of copying the user's message, while the inner QwenVL remains responsible for full image analysis and final preset formatting.
|
||||
- Pixel inspection by the chat is explicitly limited to interpreting the requested action, preventing it from inventing or duplicating visual scene details.
|
||||
|
||||
## Version 2.7.11 (2026/09/16)
|
||||
|
||||
- Added the `image2` input to `AILab_QwenVL_GGUF` and `AILab_QwenVL_GGUF_Advanced`, matching the HF nodes: `image` is Picture 1 and `image2` is Picture 2 for R2VA workflows.
|
||||
- The second image is encoded after the first and before sampled video frames, and its hash participates in the prompt cache key via the combined image2/video hash slot.
|
||||
- Added the same reference-image tooltips used by the HF nodes.
|
||||
|
||||
## Version 2.7.10 (2026/09/16)
|
||||
|
||||
- Image-enhancer routing now keeps a valid English action directive written by the chat model instead of always overwriting it with the raw user message, so Qwen Chat translates and refines the request instead of copying it verbatim.
|
||||
- Execution-only confirmations are detected with a bilingual command vocabulary (verbs, objects, fillers) instead of a fixed phrase list, covering variants such as "Run it", "Genera", "Ok", and "Esegui il video con il prompt precedente".
|
||||
- Bare confirmations and pre-formatted preset prompts emitted by the model are still replaced with the last descriptive user request, and `passthrough` remains forced to false for image-enhancer workflows.
|
||||
|
||||
## Version 2.7.9 (2026/09/16)
|
||||
|
||||
- Added deterministic I2VA action normalization before queueing: the latest user request is always written to the active inner enhancer's prompt widget and `passthrough` is always forced to false.
|
||||
- Missing prompt actions are inserted automatically, while hallucinated pre-formatted chat prompts are replaced with the raw user intent.
|
||||
|
||||
## Version 2.7.8 (2026/09/16)
|
||||
|
||||
- Image-to-video workflows with an exposed preset enhancer now receive a short action-only prompt from Qwen Chat and run the internal QwenVL with `passthrough=false`.
|
||||
- The internal workflow node analyzes the actual reference image and applies the full preset, avoiding visual details hallucinated by the chat model.
|
||||
- Passthrough remains available for workflows without image input.
|
||||
|
||||
## Version 2.7.7 (2026/09/16)
|
||||
|
||||
- Moved backend, model, token, temperature, and thinking controls into a compact Settings panel opened with a gear button.
|
||||
- The Settings panel is collapsed by default, remembers its state, and is disabled during inference.
|
||||
|
||||
## Version 2.7.6 (2026/09/16)
|
||||
|
||||
- Added a secure recursive output-image endpoint for Qwen Chat Assets, including images saved in nested folders such as `output/PMP/...`.
|
||||
- Assets are sorted by modification time and limited to supported image formats and paths contained inside the ComfyUI output directory.
|
||||
|
||||
## Version 2.7.5 (2026/09/16)
|
||||
|
||||
- Fixed the ComfyUI Assets gallery endpoint to use `/internal/files/output` without the `/api` prefix.
|
||||
|
||||
## Version 2.7.4 (2026/09/16)
|
||||
|
||||
**ComfyUI Assets Picker**
|
||||
|
||||
- Added a localized ComfyUI Assets gallery for selecting recent images directly from the ComfyUI output folder.
|
||||
- Selected assets are sent to Qwen as the priority reference and synchronized with the workflow when a `LoadImageOutput` node is available.
|
||||
- Workflows with multiple `LoadImageOutput` nodes show an explicit target-node choice instead of guessing.
|
||||
- Output images selected by `LoadImageOutput` are now recognized correctly during automatic workflow image collection.
|
||||
|
||||
## Version 2.7.3 (2026/09/16)
|
||||
|
||||
**Deterministic MiniMax I2VA Reference Binding**
|
||||
|
||||
- MiniMax H3 image-to-video prompts using passthrough now receive the mandatory `<Picture 1>` binding automatically when images are present, even if the chat model omits it.
|
||||
- The exact final prompt sent to the workflow is echoed back after an automatic correction.
|
||||
- Prompt guidance now treats reference pixels as authoritative and forbids invented identity, appearance, clothing, framing, environment, lighting, or style details that conflict with the source image.
|
||||
|
||||
## Version 2.7.2 (2026/09/16)
|
||||
|
||||
**Qwen Chat Execution Follow-up Fix**
|
||||
|
||||
- Added a localized `Generate video` fallback choice when Qwen updates a prompt but omits `queue_workflow`.
|
||||
- Assistant messages and choice labels now follow the language of the latest user message, while generated workflow prompts remain in English.
|
||||
- Prevented responses from claiming that execution started when no queue action was emitted.
|
||||
|
||||
## Version 2.7.1 (2026/09/16)
|
||||
|
||||
**Qwen Chat MiniMax H3 Routing Fix**
|
||||
|
||||
- Generated image and video prompts are now written in English unless another prompt language is explicitly requested.
|
||||
- Nodes exposing both `preset_prompt` and `passthrough` now use the passthrough route first: the final preset-formatted prompt is written to the node's actual exposed prompt widget and `passthrough` is enabled.
|
||||
- Added graph-specific routing guidance with the exact node ID and exposed prompt widget, preventing omission of the prompt or passthrough actions on promoted subgraph inputs.
|
||||
- Queue responses no longer ask for execution confirmation after already adding `queue_workflow`.
|
||||
|
||||
## Version 2.7.0 (2026/09/16)
|
||||
|
||||
**Qwen Workflow Chat, Image Attachments, Guided Workflow Actions, and Qwen 3.8 Models**
|
||||
|
||||
### Qwen Workflow Chat
|
||||
|
||||
- Added a native ComfyUI sidebar chat powered by selectable local HF/Transformers or GGUF Qwen models.
|
||||
- Added structured workflow inspection, validated widget updates, node enable/bypass actions, and autonomous queue execution when requested.
|
||||
- Added clickable response choices, retry recovery for malformed model JSON, and clean message extraction when a response cannot be fully parsed.
|
||||
- Added preset-aware prompt routing: enhancer subgraphs receive raw user intent, while accessible top-level Qwen nodes can receive final prompts through passthrough.
|
||||
- Added workflow image analysis and a direct image attachment control with preview, removal, resizing, and priority over workflow images.
|
||||
- Added a repeat-last-message button, busy spinner, refined message layout, and persistent English/Italian interface switch.
|
||||
- Set `Qwen3.8-9B-heretic-uncensored.Q8_0.gguf` as the default GGUF chat model, with automatic fallback when unavailable.
|
||||
- Chat inference defaults to 1024 max tokens and temperature 0.2; thinking mode automatically raises the token budget to at least 4096.
|
||||
- Chat models are now always unloaded from memory before queueing memory-intensive workflows.
|
||||
- Added browser-local conversation/preferences persistence, bounded context, backend inference locking, and explicit unload API.
|
||||
- Restricted automation to an allowlisted action protocol; arbitrary code, filesystem, network, graph creation, connection, and deletion operations are not supported.
|
||||
|
||||
### Models and Diagnostics
|
||||
|
||||
- Added Qwen 3.5 and Qwen 3.8 heretic, uncensored, and thinking model options for HF and GGUF backends.
|
||||
- Replaced an unavailable private Qwen 3.8 repository with the public `petruhonk/Qwen3.8-9B-Distill-uncensored-heretic` checkpoint.
|
||||
- Reduced repeated cache lookup debug output so generation, image processing, model loading, and failure events remain visible in ComfyUI logs.
|
||||
|
||||
## Version 2.6 (2026/09/03)
|
||||
|
||||
🎬 **Camera & Style Tag Dropdowns + Pony Prompt Converters + Qwen 3.8 + LTX 2.3 FL2VA + MiniMax H3 Loop Mode**
|
||||
|
||||
Major update introducing camera/style tag dropdowns across all nodes, Pony→Z-Image and Pony→Flux prompt converter presets, a rewritten Danbooru Tags preset, Qwen 3.8 uncensored models, LTX 2.3 FL2VA presets, and extensive MiniMax H3 improvements (loop mode, anatomical coherence, dialogue trigger, reference tags).
|
||||
|
||||
### 🎥 **Camera Tag Dropdown**
|
||||
- **New `camera_tag` parameter** in all nodes: QwenVL, QwenVL (Advanced), QwenVL (GGUF), QwenVL (GGUF Advanced), QwenVL Prompt Enhancer, QwenVL Prompt Enhancer (GGUF)
|
||||
- **19 camera movement options**: `None`, `[STATIC_CAMERA]`, `[LOCKED_OFF]`, `[SLOW_ZOOM_IN]`, `[SLOW_ZOOM_OUT]`, `[FAST_ZOOM_IN]`, `[FAST_ZOOM_OUT]`, `[PAN_LEFT]`, `[PAN_RIGHT]`, `[TILT_UP]`, `[TILT_DOWN]`, `[DOLLY_IN]`, `[DOLLY_OUT]`, `[TRACKING_LEFT]`, `[TRACKING_RIGHT]`, `[CRANE_UP]`, `[CRANE_DOWN]`, `[ORBIT]`, `[HANDHELD]`, `[ROLL]`
|
||||
- **Recency bias injection**: tags injected as prefix AND re-injected at the end of the prompt so Qwen respects the camera directive
|
||||
- **STATIC_CAMERA / LOCKED_OFF override**: these tags override RICHNESS RULES and force a completely static camera with no motion whatsoever
|
||||
- **Subject kept alive**: during camera tag movements, the subject description is maintained (no disappearing characters)
|
||||
|
||||
### 🎨 **Style Tag Dropdown (PromptEnhancer only)**
|
||||
- **New `style_tag` parameter** in QwenVL Prompt Enhancer and QwenVL Prompt Enhancer (GGUF) — T2V/T2VA presets only
|
||||
- **12 visual style options**: `None`, `[ANIME]`, `[PHOTOREALISTIC]`, `[3DCG]`, `[CARTOON]`, `[CLAYMATION]`, `[WATERCOLOR]`, `[VINTAGE]`, `[NOIR]`, `[CYBERPUNK]`, `[FANTASY]`, `[SOFTFOCUS]`, `[HENTAI]`
|
||||
- **Same injection mechanism** as camera_tag: prefix + final reminder for recency bias
|
||||
- For image-reference modes, identity and composition remain anchored to the references
|
||||
|
||||
### 🐴 **Pony Prompt Converter Presets**
|
||||
- **🎨 Pony→Z-Image**: converts Pony/SDXL Danbooru-style tag prompts into optimized natural-language prompts for Z-Image-Turbo (6B S3-DiT from Alibaba). Removes score_ tags, converts tags to flowing sentences, adds precise photography vocabulary (Kodak Portra 400, lens specs), NSFW conversion rules, style detection, 6-part formula. CFG=0-1, no negative prompts, 8 steps, 1024×1024.
|
||||
- **🎨 Pony→Flux**: converts Pony/SDXL Danbooru-style tag prompts into natural-language prompts for Flux.1 (12B MMDiT from Black Forest Labs). Removes score_ tags, rich detail for photorealism (skin texture, pores, micro-details, film grain), 3-6 sentence flowing paragraph, up to 2MP resolution.
|
||||
- Both presets handle mixed input (tags + natural language) and multilingual input (Italian/English/any → English output)
|
||||
|
||||
### 🏷️ **Rewritten Tags Preset (🖼️ Tags)**
|
||||
- Generates Pony/SDXL Danbooru-style tags with `score_9, score_8_up, score_7_up` quality prefix
|
||||
- 30-60 tags following structured order: quality → subject count → attributes → clothing → pose/action → environment → lighting → camera/composition → style
|
||||
- Explicit NSFW tag support (no censoring)
|
||||
- No meta tags ("masterpiece", "best quality") — uses score_ tags instead
|
||||
|
||||
### 🧠 **Qwen 3.8 Uncensored Models**
|
||||
- **HF VL**: `Qwen3.8-27B-Uncensored-Heretic-Abliterated` (alexander2323)
|
||||
- **HF Text**: `Qwen3.8-4B-Distill-heretic` (valiolla), `Qwen3.8-9B-heretic-uncensored` (rohit267)
|
||||
- **GGUF**: two new GGUF models added at top of dropdown
|
||||
- Switched to **armand0e repaired Qwen3.5-9B-heretic** weights (fixed broken checkpoint)
|
||||
|
||||
### 🎬 **LTX 2.3 FL2VA Presets**
|
||||
- New **LTX 2.3 NSFW FL2VA** presets (5s/10s/20s) for First-Last-Frame to video
|
||||
- Added **LTX 2.3 NSFW 10s** presets (I2V + T2V)
|
||||
- Added **LTX 2.3 NSFW 20s** presets (I2V + T2V)
|
||||
- Added missing **LTX 2.3 T2V (10s/20s)** styles to qwen_text.styles
|
||||
- **Dropdown reorder**: I2V → FL2VA → T2V for logical workflow progression
|
||||
- Removed LTX 2.3 T2V from QwenVL node dropdown (text-only belongs in PromptEnhancer)
|
||||
- **Danbooru tag support**: LTX 2.3 presets now expand comma-separated Danbooru tags into cinematic scenes
|
||||
- **Anatomical correctness rule** added to LTX I2V presets (5s + 10s)
|
||||
|
||||
### 🎬 **MiniMax H3 Improvements**
|
||||
- **LOOP MODE** for FL2VA presets: seamless loop generation with pacing instructions (START IMMEDIATELY, 0.3s margin for continuity)
|
||||
- **Anatomical coherence instructions** in all MiniMax H3 NSFW presets
|
||||
- **[DIALOGUE] trigger**: explicit trigger for spoken content in MiniMax H3
|
||||
- **[P3]/[P4] manual reference tags** for R2VA presets
|
||||
- **R2VA prompt fixes** based on Civitai community feedback
|
||||
- **Camera control tags** added to all MiniMax H3 NSFW presets
|
||||
- **I2VA priority**: user text now takes priority over reference image
|
||||
- **Danbooru tag + style tag support** in MiniMax H3 T2VA presets
|
||||
- **Mandatory sexual sounds** in all H3 soundscape presets
|
||||
- **Forbidden output wrapping**: no triple quotes or markdown in output
|
||||
|
||||
### ⚡ **PromptEnhancer Changes**
|
||||
- **max_tokens default raised to 8192** (was 1024) for longer, richer prompt generation
|
||||
- **Removed `custom_system_prompt`** parameter from PromptEnhancer and GGUF PE
|
||||
- **Removed `CUSTOM_ONLY_STYLE`** mode from PromptEnhancer
|
||||
|
||||
### 🐛 **Bug Fixes**
|
||||
- **Python 3.12 compatibility**: fixed crash with `rope_scaling=None`
|
||||
- **PromptEnhancer TypeError**: fixed `run() has no 'video' kwarg`
|
||||
- **SageAttention import**: fixed for 2.x API (top-level vs `.core`)
|
||||
- **qwen3_5 → qwen3_vl config fallback** for unrecognized model types
|
||||
- **Model list cleanup**: removed censored + wrong abliterated versions, removed gated prithivMLmods models (require HF auth)
|
||||
|
||||
### 🐳 **Docker / Provisioning**
|
||||
- **LTX 2.5 support** added (separate Docker/provisioning from LTX 2.3)
|
||||
- **Sulphur 2** (native uncensored LTX 2.3) + distill LoRA
|
||||
- **10Eros-Max Turbo LoRA** 8-step + 10Eros v1.5 fp8 + DMD hybrid v2 LoRA
|
||||
- **Pruna optimized VAE** for faster LTX decode
|
||||
- **gemma-4-E2B-it-uncensored** (abliterated) as LTX 2.5 prompt enhancer
|
||||
- **VFX workflows** for MiniMax H3 Turbo (FL2VA and I2VA) — RIFE before upscale for 2K output
|
||||
- **Download progress log** + `/dlstatus` endpoint served via ComfyUI
|
||||
- **Configurable ComfyUI version** in RunPod, simplified Vast.ai provisioning
|
||||
- **Workflows, provisioning and Docker infra moved to ComfyUI-Garage repo**
|
||||
|
||||
---
|
||||
|
||||
## Version 2.5.1 (2026/08/11)
|
||||
|
||||
🚀 **MiniMax-H3 Turbo LoRA Workflows + Official Diffusion + Uncensored Text Encoder + Explicit Tag Handling**
|
||||
|
||||
### ⚡ MiniMax-H3 Turbo LoRA
|
||||
- Added 4 new Turbo workflows: `MiniMaxH3-Turbo-T2VA-Qwen3VL.json`, `MiniMaxH3-Turbo-I2VA-Qwen3VL.json`, `MiniMaxH3-Turbo-FL2VA-Qwen3VL.json`, `MiniMaxH3-Turbo-R2VA-Qwen3VL.json`
|
||||
- Uses `Larryvrh/ComfyUI-MiniMax-H3-Turbo` custom node and `minimax_h3_turbo_v4_step600_ema.safetensors` (~744 MB)
|
||||
- Reduces sampling from ~20 steps to **6–8 steps** with scheduler `simple`
|
||||
- Works with all tasks: T2VA, I2VA, FL2VA and R2VA
|
||||
|
||||
### 📦 MiniMax-H3 Models
|
||||
- **T2VA / I2VA / FL2VA**: now use the official Comfy-Org pruned INT8-ConvRot diffusion model:
|
||||
- `minimax_h3_fl2va_pruned_int8_convrot.safetensors`
|
||||
- **R2VA**: keeps the official Comfy-Org pruned INT8-ConvRot model:
|
||||
- `minimax_h3_ref2va_pruned_int8_convrot.safetensors`
|
||||
- **Text encoder**: remains the uncensored `qwen3vl_32b_h3_ultra_uncensored_heretic_int8_convrot.safetensors` — [ethanfel/Qwen3-VL-32B-Ultra-Heretic-H3-ComfyUI-INT8-ConvRot](https://huggingface.co/ethanfel/Qwen3-VL-32B-Ultra-Heretic-H3-ComfyUI-INT8-ConvRot)
|
||||
- VAEs and Turbo LoRA stay from Comfy-Org / larryvrh
|
||||
|
||||
### 🏷️ Explicit Tag Handling
|
||||
- MiniMax H3 system prompts now treat short explicit tags (e.g., `blowjob`, `deepthroat`, `missionary`, `cowgirl`, `facial`, `cumshot`) as direct scene requests
|
||||
- Qwen expands the tag into a full cinematic NSFW description instead of refusing or asking for clarification
|
||||
- Works for both the vision node (`qwenvl`) and the PromptEnhancer (`qwen_text`)
|
||||
|
||||
### 📝 Documentation
|
||||
- Updated `minimax/Civitai_MiniMaxH3.md` and `runpod/README_MiniMaxH3.md` with current model links and Turbo instructions
|
||||
- Added 8-workflow list including Turbo variants
|
||||
|
||||
---
|
||||
|
||||
## Version 2.5 (2026/08/05)
|
||||
|
||||
🎬 **MiniMax H3 Full Support + Multi-Reference Input + R2VA/FL2VA Presets**
|
||||
|
||||
Major update adding complete MiniMax H3 video+audio generation support with all 4 modes (T2VA, I2VA, FL2VA, R2VA), dedicated NSFW presets for each mode, and a new multi-reference input system.
|
||||
|
||||
### 🎬 **MiniMax H3 Workflows (4)**
|
||||
|- **T2VA**: Text-to-video+audio with PromptEnhancer integration
|
||||
|- **I2VA**: Image-to-video+audio (first-frame, last-frame, or both)
|
||||
|- **FL2VA**: First-Last-Frame to video with TensorRT upscale + RIFE interpolation
|
||||
|- **R2VA**: Reference-to-video with up to 9 reference images, 3 videos, 3 audio clips
|
||||
|- All workflows auto-downloaded at boot from GitHub
|
||||
|
||||
### 🖼️ **Multi-Reference Input (image2)**
|
||||
|- **Renamed `video` input to `image2`**: Clearer name for multi-reference workflows
|
||||
|- **Frames as individual images**: `image2` frames are now passed to Qwen3-VL as separate images (not as a video sequence), enabling proper multi-reference analysis
|
||||
|- **FL2VA**: First frame on `image`, last frame on `image2` → Qwen3-VL sees both
|
||||
|- **R2VA**: Primary reference on `image`, additional references on `image2` → all visible
|
||||
|- **Backward compatible**: Existing workflows updated automatically, no reconnection needed
|
||||
|
||||
### 🎯 **NSFW Presets (9 total)**
|
||||
|- 🎬 **MiniMax H3 NSFW (5s/10s/15s)**: Base presets for T2VA/I2VA — 3-field format (integrated_multimodal_description + overall_soundscape + non_diegetic_music)
|
||||
|- 🎞️ **MiniMax H3 NSFW R2VA (5s/10s/15s)**: Reference presets — 6-section format per official guide (subject_definitions + summary + retention_analysis + detailed_description + overall_soundscape + non_diegetic_music) with `<Subject N>`, `<Picture N>`, `<Video N>`, `<Audio N>` labels
|
||||
|- 🔄 **MiniMax H3 NSFW FL2VA (5s/10s/15s)**: First-Last-Frame presets — describe the transition path between frames, favor single continuous shot
|
||||
|
||||
### 📦 **Workflow Organization**
|
||||
|- Moved all MiniMax H3 workflows from `vastai/workflows/MiniMaxH3/` to `minimax/`
|
||||
|- Renamed `MiniMaxH3-I2VA-FLF-Qwen3VL.json` to `MiniMaxH3-FL2VA-Qwen3VL.json`
|
||||
|- Added `Civitai_MiniMaxH3.md` for Civitai publication
|
||||
|- Updated Dockerfile download paths
|
||||
|
||||
### 🛠️ **Other Changes**
|
||||
|- Updated tooltips for `image2` and `frame_count`
|
||||
|- GGUF node already handled multi-image correctly (no changes needed)
|
||||
|- Internal cache functions retain `video_hash` for backward compatibility
|
||||
|
||||
## Version 2.4 (2026/08/04)
|
||||
|
||||
🚀 **Local Model Discovery + Qwen3.5 Support + SageAttention**
|
||||
|
||||
Major update porting the best features from the community fork (Deaquay/ComfyUI-Qwen3.5-Uncensored, 36 stars). Local model discovery, Qwen3.5 architecture detection, VRAM headroom, and image-handling fixes ported from the fork. SageAttention restoration is our own work from v2.0.3–v2.0.4, not from the fork.
|
||||
|
||||
### 🔍 **Automatic Local Model Discovery**
|
||||
|- **GGUF**: Drop `.gguf` files in `models/LLM/GGUF` — they appear in the dropdown automatically with `[local]` prefix
|
||||
|- **HF**: Drop model directories in `models/LLM/Qwen-VL` — detected and listed automatically
|
||||
|- **mmproj auto-pairing**: GGUF mmproj files in the same directory are paired automatically
|
||||
|- **Multi-path support**: Respects `extra_model_paths.yaml` and all registered LLM paths
|
||||
|- **No more JSON editing**: Pre-configured models still work, but local models don't require config edits
|
||||
|
||||
### 🧠 **Qwen3.5 Support**
|
||||
|- **Architecture detection**: Reads GGUF header (`general.architecture`) or HF `config.json` (`model_type`) instead of model name
|
||||
|- **Disable thinking**: Qwen3.5 thinking mode automatically disabled for prompt generation
|
||||
|- **Top K tuning**: Forces `top_k=20` for Qwen3.5 (official recommendation)
|
||||
|- **New models**: Qwen3.5-4B/9B/27B (uncensored, heretic, unsloth) added to model configs
|
||||
|
||||
### ⚡ **SageAttention Restored**
|
||||
|- **Architecture-aware kernels**: Blackwell FP8, Hopper FP8, Ada FP8, Ampere FP16
|
||||
|- **Graceful fallback**: Falls back to SDPA if SageAttention unavailable
|
||||
|- **Auto-mode priority**: Sage → Flash → SDPA
|
||||
|- **Runtime patching**: Patches `Qwen2Attention` and `Qwen3VLTextAttention` without model reload
|
||||
|
||||
### 🛠️ **Other Improvements**
|
||||
|- **VRAM headroom**: Proactive VRAM cleanup before each run (threshold: max 1GB or 8% total)
|
||||
|- **Image handling**: Fixed grayscale (mode="L") and RGBA image conversion
|
||||
|- **BitsAndBytes fix**: Separate loading path for quantized models (device_map direct vs CPU→GPU)
|
||||
|- **max_tokens**: Increased from 512 to 8192 for both Standard and Advanced nodes
|
||||
|- **Batch image warning**: Alerts when batch >1 images provided (uses first only)
|
||||
|
||||
---
|
||||
|
||||
## Version 2.3 (2026/07/09)
|
||||
|
||||
🎬 **New LTX 2.3 Presets + Multilingual Support**
|
||||
|
||||
This update introduces specialized presets for LTX 2.3 video generation, with official prompting guides and complete multilingual support.
|
||||
|
||||
### 🎬 **LTX 2.3 Presets**
|
||||
|- **LTX 2.3 NSFW I2V**: Image-to-Video preset for QwenVL nodes with image + text input
|
||||
|- **LTX 2.3 NSFW T2V**: Text-to-Video preset for Prompt Enhancer nodes with text only
|
||||
|- **Official Prompting Guide**: Core Actions, Visual Details, Audio structure for optimized results
|
||||
|- **Simplified Format**: No timeline, single fluid paragraph max 200 words
|
||||
|- **NSFW Support**: Full NSFW descriptions for both presets
|
||||
|
||||
### 🌐 **Multilingual Support**
|
||||
|- **Complete Translations**: All LTX 2.3 presets available in multiple languages
|
||||
|- **Consistent Style**: Visual style detection maintained across all languages
|
||||
|
||||
### 📝 **Prompting Tips**
|
||||
|- **Core Actions**: Primary subject motion and camera movement
|
||||
|- **Visual Details**: Appearance, environment, lighting, atmosphere
|
||||
|- **Audio**: Ambient sounds, music, dialogue cues
|
||||
|- **Optimization**: Specifically formatted for LTX 2.3 best results
|
||||
|
||||
### 🐛 **Other Changes**
|
||||
|- **Removed Qwen2.5-VL**: Dropped support for Qwen2.5-VL (Qwen3-VL only now)
|
||||
|- **Cleaned Model Lists**: Added Qwen3.5-4B/9B Unredacted-MAX, Heretic-Stable 8B
|
||||
|- **Removed Obsolete Nodes**: kijai/ComfyUI_essentials removed
|
||||
|- **Updated llama-cpp-python**: Bumped to v0.3.45
|
||||
|
||||
---
|
||||
|
||||
## Version 2.2.4 (2026/03/13)
|
||||
|
||||
🎬 **Critical I2V Timeline Fixes & NSFW Presets Optimization**
|
||||
|
||||
This update resolves critical issues with I2V Timeline preset generation and optimizes all NSFW presets for better user experience.
|
||||
|
||||
### 🚨 **Major I2V Timeline Fixes**
|
||||
|- **Style Coherence**: Fixed AI changing anime→realism mid-sequence
|
||||
|- **Character Stability**: Fixed characters disappearing/appearing incorrectly
|
||||
|- **Natural Lighting**: Fixed AI adding artificial lights not in image
|
||||
|- **Timeline Structure**: Fixed continuous numbering (6,7,8...) instead of 0-5 restart
|
||||
|- **Format Consistency**: Fixed missing parentheses and unwanted labels
|
||||
|- **Output Format**: Each prompt starts directly with timeline markers
|
||||
|
||||
### � **NSFW Presets Optimization**
|
||||
|- **Complete Specifications**: All 8 NSFW presets now include full NSFW descriptions
|
||||
|- **Emoji Display**: Restored proper emoji rendering (🍿🎥🎬📖🍿🎥🎬📖)
|
||||
|- **Clear Instructions**: Removed confusing recommendations from presets
|
||||
|- **User Notes**: Token settings guide created for workflow optimization
|
||||
|
||||
### 📋 **Technical Improvements**
|
||||
|- **Timeline Markers**: Correct `(At X seconds: ...)` format for all 4 prompts
|
||||
|- **Character Continuity**: Natural progression without forced artificial presence
|
||||
|- **Lighting Rules**: Logical progression instead of absolute prohibitions
|
||||
|- **Style Detection**: Consistent style application across all timeline segments
|
||||
|
||||
### 🎯 **Model Recommendations**
|
||||
|- **Qwen3-VL-8B**: Recommended for I2V Timeline (20s) complex sequences
|
||||
|- **Qwen3-VL-4B**: Sufficient for I2V Scene (5s) single prompts
|
||||
|- **Token Settings**: 2048+ for 20s timeline, 1024+ for 5s prompts
|
||||
|
||||
## Version 2.0.7 (2026/02/03)
|
||||
|- **FP16 Only**: All HF nodes now use FP16 exclusively (~6GB VRAM on RTX 5090)
|
||||
|- **Quantization Dropdown Removed**: Eliminated confusing quantization options from all HF nodes
|
||||
|- **Working Logic Restored**: Reverted to proven working FP16 logic from commit d057b34
|
||||
|- **Both Nodes Fixed**: Applied fixes to both Standard and Advanced nodes
|
||||
|
||||
### 🎯 **Interface Simplification**
|
||||
|- **Cleaner UI**: Removed quantization dropdown from QwenVL and QwenVL Advanced nodes
|
||||
|- **GGUF for Quantization**: Users wanting quantized models should use dedicated GGUF nodes
|
||||
|- **Consistent Parameters**: Both nodes now have consistent parameter structure
|
||||
|- **keep_last_prompt**: Added missing parameter to maintain feature parity
|
||||
|
||||
### 🔧 **Technical Changes**
|
||||
|- **Memory Management**: Improved model loading with CPU→CUDA transfer pattern
|
||||
|- **Attention Backend**: SDPA as stable default (SageAttention can be added later)
|
||||
|- **Error Prevention**: Removed all BitsAndBytes-related code paths
|
||||
|- **Debug Logging**: Enhanced logging for attention mode selection and model loading
|
||||
|
||||
### 🐛 **Bug Fixes**
|
||||
|- **OOM on RTX 5090**: Fixed critical Out-Of-Memory during model loading
|
||||
|- **Missing Parameters**: Added frame_count and keep_last_prompt to Advanced node
|
||||
|- **Quantization Logic**: Simplified to prevent device mismatch and quantization errors
|
||||
|- **Function Signatures**: Corrected all process() functions after dropdown removal
|
||||
|
||||
### 💡 **User Guidance**
|
||||
|- **HF Nodes**: Use for maximum quality with FP16 (~6GB VRAM)
|
||||
|- **GGUF Nodes**: Use for quantized models (Q4/Q8, 2-4GB VRAM)
|
||||
|- **Performance**: FP16 + SageAttention/FlashAttention for best speed
|
||||
|- **Stability**: SDPA fallback ensures compatibility across all hardware
|
||||
|
||||
---
|
||||
|
||||
## Version 2.2.3 (2026/02/27)
|
||||
|
||||
🔧 **CUDA 13 Compatibility Fix + Redundancy Removal**
|
||||
@@ -1306,7 +736,7 @@ This release introduces a full rework of the QwenVL runtime to significantly imp
|
||||
- Optimized video input logic
|
||||
|
||||
## v1.0.0 Initial Release (2025/10/17)
|
||||
- Support for Qwen3-VL series models.
|
||||
- Support for Qwen3-VL and Qwen2.5-VL series models.
|
||||
- Automatic model downloading from Hugging Face.
|
||||
- On-the-fly quantization (4-bit, 8-bit, FP16).
|
||||
- Preset and Custom Prompt system for flexible and easy use.
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
# ComfyUI-QwenVL-Mod - Enhanced Vision-Language with WAN 2.2
|
||||
|
||||
**Version 2.2.3** (2026/02/27) - 🧠 Advanced Memory Management + Documentation Updates
|
||||
|
||||
---
|
||||
|
||||
## 🌟 What is ComfyUI-QwenVL-Mod?
|
||||
|
||||
A powerful **enhanced vision-language node** for ComfyUI that combines Qwen3-VL models with professional WAN 2.2 video generation workflows. Features multilingual support, visual style detection, and NSFW capabilities for professional AI content creation.
|
||||
|
||||
**Think:** *"Your all-in-one solution for intelligent prompt enhancement and video generation with cutting-edge AI models!"*
|
||||
|
||||
---
|
||||
|
||||
## 🎬 Key Features
|
||||
|
||||
### **🚀 WAN 2.2 Video Generation**
|
||||
- **Text-to-Video (T2V)**: Professional 5-second video generation
|
||||
- **Image-to-Video (I2V)**: Advanced image animation with style detection
|
||||
- **Story Generation**: 20-second continuous videos with 4 narrative segments
|
||||
- **Storyboard Workflows**: Seamless storyboard-to-storyboard generation
|
||||
- **Cinematic Video**: Professional cinematography specifications
|
||||
|
||||
### **🌐 Enhanced Capabilities**
|
||||
- **Multilingual Support**: Process prompts from any language (Italian, English, etc.)
|
||||
- **Visual Style Detection**: 12+ artistic styles (anime, 3D, pixel art, puppet animation, etc.)
|
||||
- **Smart Prompt Caching**: Performance optimization with Fixed Seed Mode
|
||||
- **GGUF Backend**: Efficient local model inference with quantization support
|
||||
- **NSFW Support**: Comprehensive content generation without restrictions
|
||||
|
||||
### **🧠 Intelligent Features**
|
||||
- **Auto-Prompt Enhancement**: Automatically enhance user prompts for optimal generation
|
||||
- **Professional Cinematography**: Built-in specifications for lighting, camera angles, shot types
|
||||
- **Timeline Structure**: Precise 5-second timeline with frame-by-frame descriptions
|
||||
- **Keep Last Prompt**: Generate once, preserve results while changing inputs
|
||||
|
||||
---
|
||||
|
||||
## 🎯 What's New in v2.2.3
|
||||
- **CUDA 13 Compatibility**: Fixed crashes caused by conflicting unload operations
|
||||
- **Parameter Cleanup**: Removed redundant unload_after_run from all nodes
|
||||
- **Bug Fixes**: Resolved "missing required positional argument" errors
|
||||
- **Memory Management**: Streamlined VRAM cleanup with VRAM Cleanup node
|
||||
- **Documentation**: Updated all README files with new memory features
|
||||
- **Credits**: Added community credits for feedback and testing
|
||||
|
||||
## 🎯 What's New in v2.2.2
|
||||
|
||||
### **🚀 Critical T2V/I2V Workflow Fixes**
|
||||
- **Batch Processing**: Fixed critical T2V → GGUF issue with batch images
|
||||
- **Frame Detection**: Added automatic batch detection and individual frame processing
|
||||
- **Video Support**: Enhanced video frame processing with proper shape handling
|
||||
- **Debug Enhanced**: Comprehensive logging for batch processing troubleshooting
|
||||
|
||||
### **🔄 Same Model Reuse Fix**
|
||||
- **Conflict Resolution**: Fixed crash when using same model between T2V and I2V nodes
|
||||
- **Memory Management**: Enhanced cleanup with CUDA synchronization and timing
|
||||
- **Signature Mismatch**: Resolved different signature patterns between nodes
|
||||
- **Aggressive Cleanup**: Forced complete VRAM cleanup before model reload
|
||||
|
||||
### **🔧 keep_model_loaded Enhancement**
|
||||
- **Missing Parameter**: Added keep_model_loaded to PromptEnhancer node
|
||||
- **Consistent Behavior**: Both GGUF and PromptEnhancer now have identical memory management
|
||||
- **Conditional Cleanup**: Proper cleanup based on keep_model_loaded setting
|
||||
- **User Control**: Full control over memory usage vs performance
|
||||
|
||||
---
|
||||
|
||||
## 🎬 WAN 2.2 Story Workflow - Revolutionary AI Storytelling
|
||||
|
||||
### **📖 AI Story Generation**
|
||||
- **4-Segment Videos**: Automatic 20-second videos (4 × 5-second segments)
|
||||
- **Narrative Continuity**: Perfect story flow between segments
|
||||
- **NSFW Support**: Enhanced adult content generation
|
||||
- **Timeline-Free**: Natural storytelling without time markers
|
||||
|
||||
### **🔄 Smart Auto-Split**
|
||||
- **Story Split Node**: Intelligent prompt separation technology
|
||||
- **Auto-Detection**: Handles any separator format automatically
|
||||
- **4-Output Guarantee**: Always produces exactly 4 prompts
|
||||
- **Debug Mode**: Built-in troubleshooting information
|
||||
|
||||
---
|
||||
|
||||
## 📦 Installation
|
||||
|
||||
### **Requirements**
|
||||
- **ComfyUI**: v0.13.0+
|
||||
- **GPU**: 8GB+ VRAM (16GB+ recommended)
|
||||
- **System**: Windows/Linux/Mac
|
||||
- **Python**: 3.10+ (or use provided Docker environment)
|
||||
|
||||
### **Docker/Cloud Ready**
|
||||
- **RunPod**: Pre-configured templates available
|
||||
- **VastAI**: Optimized instances ready
|
||||
- **Local**: Docker support included
|
||||
|
||||
### **Quick Install**
|
||||
1. **Download**: ComfyUI-QwenVL-Mod (latest version)
|
||||
2. **Extract** to `ComfyUI/custom_nodes/ComfyUI-QwenVL-Mod`
|
||||
3. **Restart** ComfyUI
|
||||
4. **Load** included workflows
|
||||
|
||||
---
|
||||
|
||||
## 🎮 Usage Examples
|
||||
|
||||
### **Basic Text-to-Video**
|
||||
1. Load **WAN2.2-I2V-AutoPrompt.json**
|
||||
2. Input your text prompt
|
||||
3. Select model (HF or GGUF)
|
||||
4. Generate enhanced video
|
||||
|
||||
### **Image-to-Video with Style**
|
||||
1. Load **WAN2.2-I2V-AutoPrompt.json**
|
||||
2. Upload your image
|
||||
3. Enable style detection
|
||||
4. Generate animated video
|
||||
|
||||
### **AI Story Generation**
|
||||
1. Load **WAN2.2-I2V-AutoPrompt-Story.json**
|
||||
2. Input your story idea
|
||||
3. Auto-split into 4 segments
|
||||
4. Generate 20-second story video
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Technical Specifications
|
||||
|
||||
### **⚡ Performance**
|
||||
- **Context**: 65,536 tokens (8B models)
|
||||
- **Memory**: Optimized VRAM usage
|
||||
- **Stability**: Crash-free operation
|
||||
- **Speed**: Fast generation times
|
||||
|
||||
### **🎨 Model Support**
|
||||
- **Qwen3-VL 4B**: 7 GGUF variants (2.38GB-4.28GB)
|
||||
- **Qwen3-VL 8B**: 7 GGUF variants (4.8GB-8.71GB)
|
||||
- **HF Models**: Josiefed and official variants
|
||||
- **Quantization**: Q4_K_S, Q5_K_S for VRAM efficiency
|
||||
|
||||
### **🌐 Multilingual Capabilities**
|
||||
- **Input Languages**: Any language supported
|
||||
- **Auto-Translation**: Automatic translation to optimized English
|
||||
- **Style Detection**: Works with multilingual prompts
|
||||
- **Cultural Adaptation**: Context-aware prompt enhancement
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Included Workflows
|
||||
|
||||
### **🍿 WAN 2.2 Presets**
|
||||
- **Wan 2.2 I2V**: Image-to-video with timeline structure
|
||||
- **Wan 2.2 T2V**: Text-to-video with professional specs
|
||||
- **Wan Extended Storyboard**: Multi-segment continuity
|
||||
- **Wan Cinematic Video**: Single scene with cinematography
|
||||
|
||||
### **🔥 Advanced Features**
|
||||
- **NSFW Enhancement**: Uncensored content generation
|
||||
- **Professional Lighting**: 8 light types + 9 qualities
|
||||
- **Camera Control**: 6 shot types + 5 compositions
|
||||
- **Color Grading**: 4 tone options
|
||||
|
||||
---
|
||||
|
||||
## 🎨 Visual Style Detection
|
||||
|
||||
Automatically detects and enhances:
|
||||
- **Anime style** - Japanese animation aesthetics
|
||||
- **3D cartoon** - Computer-generated animation
|
||||
- **Pixel art** - Retro gaming graphics
|
||||
- **Puppet animation** - Stop-motion style
|
||||
- **3D game style** - Video game graphics
|
||||
- **Claymation** - Clay animation
|
||||
- **Watercolor** - Painting style
|
||||
- **Black and white animation** - Monochrome
|
||||
- **Oil painting style** - Classical art
|
||||
- **Felt style** - Textile art
|
||||
- **Tilt-shift photography** - Miniature effect
|
||||
- **Time-lapse photography** - Speed photography
|
||||
|
||||
---
|
||||
|
||||
## 🔥 NSFW Content Support
|
||||
|
||||
### **Enhanced Generation**
|
||||
- **Explicit Content**: Uncensored adult descriptions
|
||||
- **Detailed Scenes**: 8-12 sentences per segment
|
||||
- **Natural Progression**: Smooth story flow
|
||||
- **Style Adaptation**: Automatic visual style matching
|
||||
- **Quality**: Consistent characters & scenes
|
||||
|
||||
### **Professional Applications**
|
||||
- **Adult Content**: Industry-standard generation
|
||||
- **Artistic Nudity**: Classical art styles
|
||||
- **Educational**: Anatomy and artistic study
|
||||
- **Creative**: Artistic expression
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Why Choose ComfyUI-QwenVL-Mod?
|
||||
|
||||
### **🎬 For Content Creators**
|
||||
- **Storytelling**: Create compelling narratives
|
||||
- **Efficiency**: One prompt → complete video
|
||||
- **Quality**: Professional video output
|
||||
- **Flexibility**: Any genre, any style
|
||||
|
||||
### **🔥 For NSFW Content**
|
||||
- **Explicit**: Uncensored generation
|
||||
- **Detailed**: Rich scene descriptions
|
||||
- **Continuous**: Smooth story flow
|
||||
- **Natural**: Realistic progression
|
||||
|
||||
### **⚡ For Power Users**
|
||||
- **Customizable**: Easy to modify
|
||||
- **Extendable**: Add more segments
|
||||
- **Integrable**: Works with existing setups
|
||||
- **Optimized**: Maximum performance
|
||||
|
||||
---
|
||||
|
||||
## 🌟 What Makes This Special?
|
||||
|
||||
- **First**: Complete AI story system with vision enhancement
|
||||
- **Smart**: Intelligent prompt splitting and enhancement
|
||||
- **Complete**: End-to-end solution from text to video
|
||||
- **Optimized**: Performance-tuned for professional use
|
||||
- **Ready**: Works out-of-the-box with included workflows
|
||||
|
||||
---
|
||||
|
||||
## 🎬 Create Amazing AI Videos Today!
|
||||
|
||||
Transform your ideas into stunning videos with the power of Qwen3-VL vision enhancement and WAN 2.2 video generation.
|
||||
|
||||
**Perfect for creators, artists, and professionals looking for the ultimate AI video enhancement tool!** 🌟
|
||||
|
||||
---
|
||||
|
||||
**Built with ❤️ for the ComfyUI community**
|
||||
@@ -0,0 +1,179 @@
|
||||
# ComfyUI - WAN 2.2 I2V/T2V - RTX 4090
|
||||
|
||||
> **[Create an Instance](https://cloud.vast.ai/?ref_id=188688&creator_id=188688&name=ComfyUI%20-%20WAN%202.2%20I2V%2FT2V%20-%20RTX%204090)**
|
||||
|
||||
## What is this template?
|
||||
|
||||
This template gives you a **complete AI video generation environment** optimized for RTX 4090 with ComfyUI running in a Docker container. Features QwenVL-Mod enhanced vision-language capabilities and professional WAN 2.2 video generation workflows.
|
||||
|
||||
**Think:** *"A visual programming environment for AI video generation - connect nodes to build any workflow you can imagine!"*
|
||||
|
||||
---
|
||||
|
||||
## What can I do with this?
|
||||
|
||||
### **🎬 WAN 2.2 Video Generation Workflows**
|
||||
- **Text-to-Video (T2V)**: Generate professional 5-second videos from text prompts
|
||||
- **Image-to-Video (I2V)**: Animate static images with automatic prompt enhancement
|
||||
- **Storyboard Workflows**: Seamless storyboard-to-storyboard generation
|
||||
- **Story Generation**: 4-segment continuous video stories
|
||||
- **Cinematic Video**: Professional cinematography specifications
|
||||
|
||||
### **🌐 QwenVL-Mod Enhanced Features**
|
||||
- **Multilingual Support**: Process prompts from any language with automatic translation
|
||||
- **Visual Style Detection**: 12+ artistic styles (anime, 3D, pixel art, etc.)
|
||||
- **Smart Prompt Caching**: Performance optimization with Fixed Seed Mode
|
||||
- **GGUF Backend**: Efficient local model inference with quantization support
|
||||
- **NSFW Support**: Comprehensive content generation without restrictions
|
||||
|
||||
### **🧠 Smart Prompt System**
|
||||
- **Auto-Prompt Enhancement**: Automatically enhance user prompts for optimal video generation
|
||||
- **Professional Cinematography**: Built-in specifications for lighting, camera angles
|
||||
- **Timeline Structure**: Precise 5-second timeline with frame-by-frame descriptions
|
||||
- **Fixed Seed Mode**: Maintain consistent prompts regardless of media variations
|
||||
|
||||
### **⚡ RTX 4090 Optimized Performance**
|
||||
- **SageAttention + FP16**: Proven optimal for WAN2.2 video generation
|
||||
- **CUDA 12.4 Compatibility**: Maximum stability and performance
|
||||
- **Miniconda Environment**: Python 3.12.12 isolated environment
|
||||
- **Hardware Optimization**: Automatic GPU detection and VRAM management
|
||||
|
||||
---
|
||||
|
||||
## Who is this for?
|
||||
|
||||
This is **perfect** if you:
|
||||
- Want to use **QwenVL-Mod** with enhanced vision-language capabilities
|
||||
- Need **professional WAN 2.2 video generation** with T2V and I2V workflows
|
||||
- Have an **RTX 4090** and want maximum performance optimization
|
||||
- Want **multilingual support** with automatic language detection
|
||||
- Need **visual style detection** for anime, 3D, pixel art styles
|
||||
- Want **Fixed Seed Mode** for consistent prompt generation
|
||||
- Need **NSFW support** for comprehensive content generation
|
||||
- Want **GGUF backend support** for efficient local model inference
|
||||
- Are creating **professional video content** with cinematography specs
|
||||
- Want **hardware-optimized performance** for RTX 4090
|
||||
|
||||
---
|
||||
|
||||
## Quick Start Guide
|
||||
|
||||
### **Step 1: Configure Your Setup**
|
||||
Set your preferred configuration via environment variables:
|
||||
- **`WORKSPACE`**: Custom workspace directory for your models and outputs
|
||||
- **`PROVISIONING_SCRIPT`**: URL to auto-download models on first boot
|
||||
- **`HF_TOKEN`**: Hugging Face token for downloading restricted models
|
||||
- **`CIVITAI_TOKEN`**: CivitAI token for premium model downloads
|
||||
|
||||
> **🎬 Pre-installed QwenVL-Mod Workflows**: Complete WAN 2.2 video generation with auto-prompt enhancement:
|
||||
> - **WAN 2.2 T2V**: Text-to-video with automatic prompt optimization
|
||||
> - **WAN 2.2 I2V**: Image-to-video with style detection and enhancement
|
||||
> - **WAN 2.2 T2V/I2V Story**: 4-segment continuous video stories
|
||||
> - **WAN 2.2 I2V-SVI**: Advanced image-to-video with SVI processing
|
||||
> - **WAN 2.2 T2V/I2V GGUF**: GGUF-optimized versions
|
||||
> - **WAN 2.2 MMAudio**: Audio-enhanced video generation
|
||||
> - **Wan Extended Storyboard**: Seamless storyboard sequences
|
||||
> - **Multilingual Support**: Process prompts from any language
|
||||
> - **Visual Style Detection**: Automatic detection of artistic styles
|
||||
|
||||
### **Step 2: Launch Instance**
|
||||
Click **"[Rent](https://cloud.vast.ai/?ref_id=188688)"** when you've found a suitable RTX 4090 instance
|
||||
|
||||
### **Step 3: Wait for Setup**
|
||||
ComfyUI will be ready automatically with all dependencies installed *(initial model downloads may take additional time)*
|
||||
|
||||
### **Step 4: Access Your Environment**
|
||||
**Easy access:** Click the **"Open"** button for instant access to the ComfyUI interface!
|
||||
|
||||
**Direct access via mapped ports:**
|
||||
- **ComfyUI:** `http://your-instance-ip:8188` (main interface)
|
||||
- **JupyterLab:** `http://your-instance-ip:8888` (terminal access)
|
||||
|
||||
### **Step 5: Start Creating**
|
||||
Load a workflow, download models via ComfyUI-Manager, and start generating!
|
||||
|
||||
---
|
||||
|
||||
## Key Features
|
||||
|
||||
### **ComfyUI-Manager & Pre-installed Nodes**
|
||||
- **One-click model downloads** from CivitAI and Hugging Face
|
||||
- **Custom node installation** from the community repository
|
||||
- **Workflow management** and sharing tools
|
||||
- **Automatic dependency resolution** for custom nodes
|
||||
|
||||
> **🚀 Pre-installed Custom Nodes**: 30+ essential custom nodes for professional video generation:
|
||||
> - **ComfyUI-Manager**: Essential node management system
|
||||
> - **ComfyUI-QwenVL-Mod**: Enhanced vision-language with WAN 2.2 workflows
|
||||
> - **ComfyUI-GGUF**: GGUF model support for efficient inference
|
||||
> - **ComfyUI-VideoHelperSuite**: Video processing utilities
|
||||
> - **ComfyUI-Impact-Pack**: Advanced node functionality
|
||||
> - **ComfyUI_essentials**: Essential utilities and helpers
|
||||
> - **PainterI2V series**: Professional image-to-video workflows
|
||||
> - **ComfyUI-WanMoeKSampler**: WAN model optimization
|
||||
> - **ComfyUI-RIFE-TensorRT-Auto**: Advanced frame interpolation
|
||||
> - **ComfyUI-MMAudio**: Audio processing capabilities
|
||||
> - **ComfyUI-VFI**: Video frame interpolation
|
||||
> - **Plus 20+ additional nodes** for professional workflows
|
||||
|
||||
### **Authentication & Access**
|
||||
| Method | Use Case | Access Point |
|
||||
|--------|----------|--------------|
|
||||
| **Web Interface** | All ComfyUI operations | Click "Open" or port 8188 |
|
||||
| **JupyterLab** | Terminal access & scripting | Port 8888 |
|
||||
| **SSH Terminal** | System administration | [SSH access](https://docs.vast.ai/instances/sshscp) |
|
||||
|
||||
### **Port Reference**
|
||||
| Service | External Port | Internal Port |
|
||||
|---------|---------------|---------------|
|
||||
| ComfyUI | 8188 | 18188 |
|
||||
| JupyterLab | 8888 | 8080 |
|
||||
|
||||
---
|
||||
|
||||
## RTX 4090 Optimizations
|
||||
|
||||
### **Environment Variables Reference**
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `WORKSPACE` | `/workspace` | ComfyUI workspace directory |
|
||||
| `COMFYUI_ARGS` | `--disable-auto-launch --port 18188 --enable-cors-header --fast fp16_accumulation --use-sage-attention --reserve-vram 2 --cuda-malloc --async-offload` | ComfyUI startup arguments optimized for RTX 4090 |
|
||||
| `PROVISIONING_SCRIPT` | (none) | Auto-setup script URL |
|
||||
| `HF_TOKEN` | (none) | Hugging Face token for restricted model downloads |
|
||||
| `CIVITAI_TOKEN` | (none) | CivitAI token for premium model downloads |
|
||||
|
||||
### **Why These Settings?**
|
||||
- **SageAttention + FP16**: Proven optimal for WAN2.2 video generation
|
||||
- **Reserve VRAM 2GB**: Perfect balance for RTX 4090 + WAN2.2
|
||||
- **Conda Isolation**: Prevents dependency conflicts with Python 3.12.12
|
||||
- **CUDA 12.4**: Maximum compatibility and stability for RTX 4090
|
||||
|
||||
### **Recommended GPU Memory**
|
||||
| Use Case | Minimum VRAM | Recommended VRAM |
|
||||
|----------|--------------|------------------|
|
||||
| SD 1.5 / SDXL | 8 GB | 12 GB |
|
||||
| WAN 2.2 Video | 16 GB | 24 GB |
|
||||
| Professional Workflows | 24 GB | 24 GB+ |
|
||||
|
||||
---
|
||||
|
||||
## Need More Help?
|
||||
|
||||
### **Documentation & Resources**
|
||||
- **QwenVL-Mod Documentation:** [GitHub Repository](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
- **ComfyUI Documentation:** [Official Repository](https://github.com/Comfy-Org/ComfyUI)
|
||||
- **ComfyUI-Manager:** [Node and Model Manager](https://github.com/Comfy-Org/ComfyUI-Manager)
|
||||
- **Vast.ai Documentation:** [Instance Portal Guide](https://docs.vast.ai/instance-portal)
|
||||
|
||||
### **Community & Support**
|
||||
- **ComfyUI GitHub:** [Issues & Discussions](https://github.com/Comfy-Org/ComfyUI)
|
||||
- **Vast.ai Support:** Use the messaging icon in the console
|
||||
|
||||
### **Getting Started Resources**
|
||||
- **Example Workflows:** Pre-built WAN 2.2 workflows included
|
||||
- **ComfyUI Examples:** [Community workflow gallery](https://comfyworkflows.com/)
|
||||
- **Custom Nodes:** Thousands available via ComfyUI-Manager
|
||||
|
||||
---
|
||||
|
||||
**Built with ❤️ for the ComfyUI community - Optimized for RTX 4090**
|
||||
@@ -0,0 +1,188 @@
|
||||
# ComfyUI - WAN 2.2 I2V/T2V - RTX 5090 Blackwell
|
||||
|
||||
> **[Create an Instance](https://cloud.vast.ai/?ref_id=188688&creator_id=188688&name=ComfyUI%20-%20WAN%202.2%20I2V%2FT2V%20-%20RTX%205090%20Blackwell)**
|
||||
|
||||
## What is this template?
|
||||
|
||||
This template gives you a **complete AI video generation environment** optimized for RTX 5090 Blackwell with ComfyUI running in a Docker container. Features QwenVL-Mod enhanced vision-language capabilities and professional WAN 2.2 video generation workflows with next-gen GPU acceleration.
|
||||
|
||||
**Think:** *"A visual programming environment for AI video generation with cutting-edge RTX 5090 performance - connect nodes to build any workflow you can imagine!"*
|
||||
|
||||
---
|
||||
|
||||
## What can I do with this?
|
||||
|
||||
### **🎬 WAN 2.2 Video Generation Workflows**
|
||||
- **Text-to-Video (T2V)**: Generate professional 5-second videos from text prompts
|
||||
- **Image-to-Video (I2V)**: Animate static images with automatic prompt enhancement
|
||||
- **Storyboard Workflows**: Seamless storyboard-to-storyboard generation
|
||||
- **Story Generation**: 4-segment continuous video stories
|
||||
- **Cinematic Video**: Professional cinematography specifications
|
||||
|
||||
### **🌐 QwenVL-Mod Enhanced Features**
|
||||
- **Multilingual Support**: Process prompts from any language with automatic translation
|
||||
- **Visual Style Detection**: 12+ artistic styles (anime, 3D, pixel art, etc.)
|
||||
- **Smart Prompt Caching**: Performance optimization with Fixed Seed Mode
|
||||
- **GGUF Backend**: Efficient local model inference with quantization support
|
||||
- **NSFW Support**: Comprehensive content generation without restrictions
|
||||
|
||||
### **🧠 Smart Prompt System**
|
||||
- **Auto-Prompt Enhancement**: Automatically enhance user prompts for optimal video generation
|
||||
- **Professional Cinematography**: Built-in specifications for lighting, camera angles
|
||||
- **Timeline Structure**: Precise 5-second timeline with frame-by-frame descriptions
|
||||
- **Fixed Seed Mode**: Maintain consistent prompts regardless of media variations
|
||||
|
||||
### **⚡ RTX 5090 Blackwell Optimized Performance**
|
||||
- **SageAttention + FP16**: Proven optimal for WAN2.2 video generation
|
||||
- **FP8 Compute Support**: Leverages RTX 5090's hardware FP8 acceleration
|
||||
- **CUDA 12.8 Next-Gen**: Latest optimizations for Blackwell architecture
|
||||
- **Miniconda Environment**: Python 3.12.12 isolated environment
|
||||
- **Hardware Optimization**: Automatic GPU detection and VRAM management
|
||||
|
||||
---
|
||||
|
||||
## Who is this for?
|
||||
|
||||
This is **perfect** if you:
|
||||
- Want to use **QwenVL-Mod** with enhanced vision-language capabilities
|
||||
- Need **professional WAN 2.2 video generation** with T2V and I2V workflows
|
||||
- Have an **RTX 5090 Blackwell** and want cutting-edge performance
|
||||
- Want **FP8 acceleration** for next-gen model inference
|
||||
- Need **multilingual support** with automatic language detection
|
||||
- Want **visual style detection** for anime, 3D, pixel art styles
|
||||
- Need **NSFW support** for comprehensive content generation
|
||||
- Want **GGUF backend support** for efficient local model inference
|
||||
- Are creating **professional video content** with cinematography specs
|
||||
- Want **maximum performance** with RTX 5090 Blackwell optimization
|
||||
|
||||
---
|
||||
|
||||
## Quick Start Guide
|
||||
|
||||
### **Step 1: Configure Your Setup**
|
||||
Set your preferred configuration via environment variables:
|
||||
- **`WORKSPACE`**: Custom workspace directory for your models and outputs
|
||||
- **`PROVISIONING_SCRIPT`**: URL to auto-download models on first boot
|
||||
- **`HF_TOKEN`**: Hugging Face token for downloading restricted models
|
||||
- **`CIVITAI_TOKEN`**: CivitAI token for premium model downloads
|
||||
|
||||
> **🎬 Pre-installed QwenVL-Mod Workflows**: Complete WAN 2.2 video generation with auto-prompt enhancement:
|
||||
> - **WAN 2.2 T2V**: Text-to-video with automatic prompt optimization
|
||||
> - **WAN 2.2 I2V**: Image-to-video with style detection and enhancement
|
||||
> - **WAN 2.2 T2V/I2V Story**: 4-segment continuous video stories
|
||||
> - **WAN 2.2 I2V-SVI**: Advanced image-to-video with SVI processing
|
||||
> - **WAN 2.2 T2V/I2V GGUF**: GGUF-optimized versions
|
||||
> - **WAN 2.2 MMAudio**: Audio-enhanced video generation
|
||||
> - **Wan Extended Storyboard**: Seamless storyboard sequences
|
||||
> - **Multilingual Support**: Process prompts from any language
|
||||
> - **Visual Style Detection**: Automatic detection of artistic styles
|
||||
|
||||
### **Step 2: Launch Instance**
|
||||
Click **"[Rent](https://cloud.vast.ai/?ref_id=188688)"** when you've found a suitable RTX 5090 instance
|
||||
|
||||
### **Step 3: Wait for Setup**
|
||||
ComfyUI will be ready automatically with all dependencies installed *(initial model downloads may take additional time)*
|
||||
|
||||
### **Step 4: Access Your Environment**
|
||||
**Easy access:** Click the **"Open"** button for instant access to the ComfyUI interface!
|
||||
|
||||
**Direct access via mapped ports:**
|
||||
- **ComfyUI:** `http://your-instance-ip:8188` (main interface)
|
||||
- **JupyterLab:** `http://your-instance-ip:8888` (terminal access)
|
||||
|
||||
### **Step 5: Start Creating**
|
||||
Load a workflow, download models via ComfyUI-Manager, and start generating!
|
||||
|
||||
---
|
||||
|
||||
## Key Features
|
||||
|
||||
### **ComfyUI-Manager & Pre-installed Nodes**
|
||||
- **One-click model downloads** from CivitAI and Hugging Face
|
||||
- **Custom node installation** from the community repository
|
||||
- **Workflow management** and sharing tools
|
||||
- **Automatic dependency resolution** for custom nodes
|
||||
|
||||
> **🚀 Pre-installed Custom Nodes**: 30+ essential custom nodes for professional video generation:
|
||||
> - **ComfyUI-Manager**: Essential node management system
|
||||
> - **ComfyUI-QwenVL-Mod**: Enhanced vision-language with WAN 2.2 workflows
|
||||
> - **ComfyUI-GGUF**: GGUF model support for efficient inference
|
||||
> - **ComfyUI-VideoHelperSuite**: Video processing utilities
|
||||
> - **ComfyUI-Impact-Pack**: Advanced node functionality
|
||||
> - **ComfyUI_essentials**: Essential utilities and helpers
|
||||
> - **PainterI2V series**: Professional image-to-video workflows
|
||||
> - **ComfyUI-WanMoeKSampler**: WAN model optimization
|
||||
> - **ComfyUI-RIFE-TensorRT-Auto**: Advanced frame interpolation
|
||||
> - **ComfyUI-MMAudio**: Audio processing capabilities
|
||||
> - **ComfyUI-VFI**: Video frame interpolation
|
||||
> - **Plus 20+ additional nodes** for professional workflows
|
||||
|
||||
### **Authentication & Access**
|
||||
| Method | Use Case | Access Point |
|
||||
|--------|----------|--------------|
|
||||
| **Web Interface** | All ComfyUI operations | Click "Open" or port 8188 |
|
||||
| **JupyterLab** | Terminal access & scripting | Port 8888 |
|
||||
| **SSH Terminal** | System administration | [SSH access](https://docs.vast.ai/instances/sshscp) |
|
||||
|
||||
### **Port Reference**
|
||||
| Service | External Port | Internal Port |
|
||||
|---------|---------------|---------------|
|
||||
| ComfyUI | 8188 | 18188 |
|
||||
| JupyterLab | 8888 | 8080 |
|
||||
|
||||
---
|
||||
|
||||
## RTX 5090 Blackwell Optimizations
|
||||
|
||||
### **Environment Variables Reference**
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `WORKSPACE` | `/workspace` | ComfyUI workspace directory |
|
||||
| `COMFYUI_ARGS` | `--disable-auto-launch --port 18188 --enable-cors-header --fast fp16_accumulation --use-sage-attention --reserve-vram 2 --cuda-malloc --async-offload --supports-fp8-compute` | ComfyUI startup arguments optimized for RTX 5090 |
|
||||
| `PROVISIONING_SCRIPT` | (none) | Auto-setup script URL |
|
||||
| `HF_TOKEN` | (none) | Hugging Face token for restricted model downloads |
|
||||
| `CIVITAI_TOKEN` | (none) | CivitAI token for premium model downloads |
|
||||
|
||||
### **Why These Settings?**
|
||||
- **SageAttention + FP16**: Proven optimal for WAN2.2 video generation
|
||||
- **FP8 Support**: Leverages RTX 5090's hardware FP8 acceleration
|
||||
- **Reserve VRAM 2GB**: Perfect balance for RTX 5090 + WAN2.2
|
||||
- **Conda Isolation**: Prevents dependency conflicts with Python 3.12.12
|
||||
- **CUDA 12.8**: Latest optimizations for Blackwell architecture
|
||||
|
||||
### **RTX 5090 Blackwell Advantages**
|
||||
- **FP8 Compute**: Native hardware acceleration for FP8 models
|
||||
- **Blackwell Architecture**: Next-gen GPU optimizations
|
||||
- **Enhanced Memory Bandwidth**: Faster model loading and inference
|
||||
- **Advanced Tensor Cores**: Optimized for AI workloads
|
||||
|
||||
### **Recommended GPU Memory**
|
||||
| Use Case | Minimum VRAM | Recommended VRAM |
|
||||
|----------|--------------|------------------|
|
||||
| SD 1.5 / SDXL | 8 GB | 12 GB |
|
||||
| WAN 2.2 Video | 16 GB | 24 GB |
|
||||
| FP8 Models | 12 GB | 16 GB |
|
||||
| Professional Workflows | 24 GB | 32 GB+ |
|
||||
|
||||
---
|
||||
|
||||
## Need More Help?
|
||||
|
||||
### **Documentation & Resources**
|
||||
- **QwenVL-Mod Documentation:** [GitHub Repository](https://github.com/huchukato/ComfyUI-QwenVL-Mod)
|
||||
- **ComfyUI Documentation:** [Official Repository](https://github.com/Comfy-Org/ComfyUI)
|
||||
- **ComfyUI-Manager:** [Node and Model Manager](https://github.com/Comfy-Org/ComfyUI-Manager)
|
||||
- **Vast.ai Documentation:** [Instance Portal Guide](https://docs.vast.ai/instance-portal)
|
||||
|
||||
### **Community & Support**
|
||||
- **ComfyUI GitHub:** [Issues & Discussions](https://github.com/Comfy-Org/ComfyUI)
|
||||
- **Vast.ai Support:** Use the messaging icon in the console
|
||||
|
||||
### **Getting Started Resources**
|
||||
- **Example Workflows:** Pre-built WAN 2.2 workflows included
|
||||
- **ComfyUI Examples:** [Community workflow gallery](https://comfyworkflows.com/)
|
||||
- **Custom Nodes**: Thousands available via ComfyUI-Manager
|
||||
|
||||
---
|
||||
|
||||
**Built with ❤️ for the ComfyUI community - Optimized for RTX 5090 Blackwell**
|
||||
@@ -0,0 +1,334 @@
|
||||
#!/bin/bash
|
||||
|
||||
source /venv/main/bin/activate
|
||||
COMFYUI_DIR=${WORKSPACE}/ComfyUI
|
||||
|
||||
APT_PACKAGES=(
|
||||
"nodejs"
|
||||
"npm"
|
||||
)
|
||||
|
||||
PIP_PACKAGES=(
|
||||
"--upgrade --force-reinstall --no-cache-dir https://github.com/JamePeng/llama-cpp-python/releases/download/v0.3.30-cu130-Basic-linux-20260302/llama_cpp_python-0.3.30+cu130.basic-cp312-cp312-linux_x86_64.whl"
|
||||
)
|
||||
|
||||
NODES=(
|
||||
"https://github.com/ltdrdata/ComfyUI-Manager"
|
||||
"https://github.com/kijai/ComfyUI_essentials"
|
||||
"https://github.com/huchukato/comfy-tagcomplete"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod"
|
||||
"https://github.com/huchukato/ComfyUI-RIFE-TensorRT-Auto"
|
||||
"https://github.com/huchukato/ComfyUI-Upscaler-TensorRT-Auto"
|
||||
"https://github.com/huchukato/ComfyUI-HuggingFace"
|
||||
"https://github.com/huchukato/ComfyUI-PainterI2Vadvanced"
|
||||
"https://github.com/MoonGoblinDev/Civicomfy"
|
||||
"https://github.com/Koishi-Star/Euler-Smea-Dyn-Sampler"
|
||||
"https://github.com/ltdrdata/was-node-suite-comfyui"
|
||||
"https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite"
|
||||
"https://github.com/rgthree/rgthree-comfy"
|
||||
"https://github.com/yolain/ComfyUI-Easy-Use"
|
||||
"https://github.com/kijai/ComfyUI-KJNodes"
|
||||
"https://github.com/Fannovel16/ComfyUI-Frame-Interpolation"
|
||||
"https://github.com/Smirnov75/ComfyUI-mxToolkit"
|
||||
"https://github.com/princepainter/ComfyUI-PainterI2V"
|
||||
"https://github.com/princepainter/ComfyUI-PainterLongVideo"
|
||||
"https://github.com/ashtar1984/comfyui-find-perfect-resolution"
|
||||
"https://github.com/huchukato/ComfyUI-Selectors"
|
||||
"https://github.com/city96/ComfyUI-GGUF"
|
||||
"https://github.com/kijai/ComfyUI-MMAudio"
|
||||
"https://github.com/GACLove/ComfyUI-VFI"
|
||||
"https://github.com/stduhpf/ComfyUI-WanMoeKSampler"
|
||||
"https://github.com/melMass/comfy_mtb"
|
||||
)
|
||||
|
||||
WORKFLOWS=(
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/PMP-LoRaStack-Upscale-Wildcards.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-I2V-AutoPrompt.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-I2V-AutoPrompt-GGUF.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-I2V-AutoPrompt-Story.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-I2V-Full-AutoPrompt-MMAudio.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-I2V-Full-AutoPrompt-MMAudio-GGUF.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-I2V-SVI-AutoPrompt.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-I2V-SVI-AutoPrompt-GGUF.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-I2V-SVI-AutoPrompt-Story.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-T2V-AutoPrompt.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-T2V-AutoPrompt-GGUF.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-T2V-I2V-AutoPrompt-Story.json"
|
||||
"https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/workflows/WAN2.2-T2V-I2V-Full-AutoPrompt-MMAudio-GGUF.json"
|
||||
)
|
||||
|
||||
CHECKPOINT_MODELS=(
|
||||
)
|
||||
|
||||
UNET_MODELS=(
|
||||
|
||||
)
|
||||
|
||||
LORA_MODELS=(
|
||||
|
||||
)
|
||||
|
||||
VAE_MODELS=(
|
||||
"https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/resolve/main/split_files/vae/wan_2.1_vae.safetensors"
|
||||
"https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1_VAE_fp32.safetensors"
|
||||
"https://huggingface.co/huchukato/favs/resolve/main/VAE/sdxl.vae.safetensors"
|
||||
)
|
||||
|
||||
ESRGAN_MODELS=(
|
||||
"https://huggingface.co/huchukato/favs/resolve/main/ESRGAN/2xLexicaRRDBNet.pth"
|
||||
"https://huggingface.co/huchukato/favs/resolve/main/ESRGAN/2xLexicaRRDBNet_Sharp.pth"
|
||||
)
|
||||
|
||||
TEXT_ENCODERS=(
|
||||
"https://huggingface.co/NSFW-API/NSFW-Wan-UMT5-XXL/resolve/main/nsfw_wan_umt5-xxl_fp8_scaled.safetensors"
|
||||
"https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors"
|
||||
)
|
||||
|
||||
CONTROLNET_MODELS=(
|
||||
)
|
||||
|
||||
### DO NOT EDIT BELOW HERE UNLESS YOU KNOW WHAT YOU ARE DOING ###
|
||||
|
||||
function provisioning_start() {
|
||||
provisioning_print_header
|
||||
echo "🚀 Starting provisioning process..."
|
||||
|
||||
echo "📦 Installing APT packages..."
|
||||
provisioning_get_apt_packages
|
||||
|
||||
echo "🔧 Installing custom nodes..."
|
||||
provisioning_get_nodes
|
||||
|
||||
echo "🎴 Copying wildcards to Easy-Use..."
|
||||
provisioning_copy_wildcards
|
||||
|
||||
echo "📦 Installing PIP packages..."
|
||||
provisioning_get_pip_packages
|
||||
|
||||
echo "📁 Downloading workflows..."
|
||||
# Create workflows directory
|
||||
mkdir -p "${COMFYUI_DIR}/user/default/workflows"
|
||||
|
||||
provisioning_get_files \
|
||||
"${COMFYUI_DIR}/user/default/workflows" \
|
||||
"${WORKFLOWS[@]}"
|
||||
|
||||
echo "✅ Workflows downloaded to: ${COMFYUI_DIR}/user/default/workflows"
|
||||
|
||||
echo "🎯 Downloading checkpoint models..."
|
||||
provisioning_get_files \
|
||||
"${COMFYUI_DIR}/models/checkpoints" \
|
||||
"${CHECKPOINT_MODELS[@]}"
|
||||
|
||||
echo "🧠 Downloading U-NET models..."
|
||||
provisioning_get_files \
|
||||
"${COMFYUI_DIR}/models/unet" \
|
||||
"${UNET_MODELS[@]}"
|
||||
|
||||
echo "🎨 Downloading LoRA models..."
|
||||
provisioning_get_files \
|
||||
"${COMFYUI_DIR}/models/lora" \
|
||||
"${LORA_MODELS[@]}"
|
||||
|
||||
echo "🎮 Downloading ControlNet models..."
|
||||
provisioning_get_files \
|
||||
"${COMFYUI_DIR}/models/controlnet" \
|
||||
"${CONTROLNET_MODELS[@]}"
|
||||
|
||||
echo "🔮 Downloading VAE models..."
|
||||
provisioning_get_files \
|
||||
"${COMFYUI_DIR}/models/vae" \
|
||||
"${VAE_MODELS[@]}"
|
||||
|
||||
echo "⚡ Downloading upscale models..."
|
||||
provisioning_get_files \
|
||||
"${COMFYUI_DIR}/models/upscale_models" \
|
||||
"${ESRGAN_MODELS[@]}"
|
||||
|
||||
echo "📝 Downloading text encoders..."
|
||||
provisioning_get_files \
|
||||
"${COMFYUI_DIR}/models/text_encoders" \
|
||||
"${TEXT_ENCODERS[@]}"
|
||||
|
||||
provisioning_print_end
|
||||
}
|
||||
|
||||
function provisioning_get_apt_packages() {
|
||||
if [[ -n $APT_PACKAGES ]]; then
|
||||
sudo $APT_INSTALL ${APT_PACKAGES[@]}
|
||||
fi
|
||||
}
|
||||
|
||||
function provisioning_get_pip_packages() {
|
||||
if [[ -n $PIP_PACKAGES ]]; then
|
||||
echo "Installing PIP packages..."
|
||||
for package in "${PIP_PACKAGES[@]}"; do
|
||||
echo "Installing: $package"
|
||||
pip install --root-user-action=ignore --no-cache-dir $package
|
||||
echo "✓ Completed: $package"
|
||||
done
|
||||
echo "All PIP packages installed successfully!"
|
||||
fi
|
||||
}
|
||||
|
||||
function provisioning_get_nodes() {
|
||||
echo "Processing ${#NODES[@]} nodes..."
|
||||
local count=0
|
||||
for repo in "${NODES[@]}"; do
|
||||
((count++))
|
||||
dir="${repo##*/}"
|
||||
path="${COMFYUI_DIR}/custom_nodes/${dir}"
|
||||
requirements="${path}/requirements.txt"
|
||||
|
||||
echo "[$count/${#NODES[@]}] Processing node: $dir"
|
||||
|
||||
if [[ -d $path ]]; then
|
||||
if [[ ${AUTO_UPDATE,,} != "false" ]]; then
|
||||
echo " → Updating existing node..."
|
||||
( cd "$path" && git pull )
|
||||
if [[ -e $requirements ]]; then
|
||||
echo " → Installing requirements..."
|
||||
pip install --root-user-action=ignore --no-cache-dir -r "$requirements"
|
||||
fi
|
||||
else
|
||||
echo " → Node exists, skipping (AUTO_UPDATE=false)"
|
||||
fi
|
||||
else
|
||||
echo " → Downloading new node..."
|
||||
git clone "${repo}" "${path}" --recursive
|
||||
if [[ -e $requirements ]]; then
|
||||
echo " → Installing requirements..."
|
||||
pip install --root-user-action=ignore --no-cache-dir -r "${requirements}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Special handling for ComfyUI-Easy-Use-Frontend
|
||||
if [[ "$dir" == "ComfyUI-Easy-Use" ]]; then
|
||||
echo " → Installing ComfyUI-Easy-Use-Frontend..."
|
||||
cd "${path}"
|
||||
if [[ ! -d "ComfyUI-Easy-Use-Frontend" ]]; then
|
||||
git clone https://github.com/yolain/ComfyUI-Easy-Use-Frontend.git
|
||||
fi
|
||||
cd ComfyUI-Easy-Use-Frontend
|
||||
if [[ ! -d "node_modules" ]]; then
|
||||
echo " → Installing npm dependencies..."
|
||||
npm install
|
||||
fi
|
||||
echo " → Building frontend..."
|
||||
npm run build:dev
|
||||
# Create config.yaml with WEB_VERSION: dev
|
||||
if [[ ! -f "${path}/config.yaml" ]]; then
|
||||
echo "WEB_VERSION: dev" > "${path}/config.yaml"
|
||||
fi
|
||||
cd "${COMFYUI_DIR}"
|
||||
fi
|
||||
done
|
||||
echo "All nodes processed successfully!"
|
||||
}
|
||||
|
||||
function provisioning_copy_wildcards() {
|
||||
echo "Copying wildcards from comfy-tagcomplete to Easy-Use..."
|
||||
|
||||
local source_dir="${COMFYUI_DIR}/custom_nodes/comfy-tagcomplete/wildcards/mbe"
|
||||
local target_dir="${COMFYUI_DIR}/custom_nodes/ComfyUI-Easy-Use/wildcards"
|
||||
|
||||
# Create target directory if it doesn't exist
|
||||
mkdir -p "$target_dir"
|
||||
|
||||
# Copy the entire mbe directory
|
||||
if [[ -d "$source_dir" ]]; then
|
||||
echo " → Copying mbe wildcards..."
|
||||
cp -r "$source_dir" "$target_dir/"
|
||||
echo " ✓ Wildcards copied successfully to $target_dir/mbe"
|
||||
else
|
||||
echo " ⚠ Source directory not found: $source_dir"
|
||||
echo " → Trying alternative path..."
|
||||
# Try alternative path if comfy-tagcomplete is in workspace
|
||||
local alt_source_dir="/workspace/comfy-tagcomplete/wildcards/mbe"
|
||||
if [[ -d "$alt_source_dir" ]]; then
|
||||
echo " → Found at alternative path, copying..."
|
||||
cp -r "$alt_source_dir" "$target_dir/"
|
||||
echo " ✓ Wildcards copied successfully from alternative path"
|
||||
else
|
||||
echo " ⚠ Source directory not found at: $alt_source_dir"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function provisioning_get_files() {
|
||||
if [[ -z $2 ]]; then return 1; fi
|
||||
|
||||
dir="$1"
|
||||
mkdir -p "$dir"
|
||||
shift
|
||||
arr=("$@")
|
||||
echo "Downloading ${#arr[@]} file(s) to $dir..."
|
||||
local count=0
|
||||
for url in "${arr[@]}"; do
|
||||
((count++))
|
||||
echo "[$count/${#arr[@]}] Downloading: $(basename "$url")"
|
||||
provisioning_download "${url}" "${dir}"
|
||||
echo " ✓ Download completed"
|
||||
done
|
||||
echo "All files downloaded successfully!"
|
||||
}
|
||||
|
||||
function provisioning_print_header() {
|
||||
printf "\n##############################################\n# #\n# Provisioning container #\n# #\n# This will take some time #\n# #\n# Your container will be ready on completion #\n# #\n##############################################\n\n"
|
||||
}
|
||||
|
||||
function provisioning_print_end() {
|
||||
printf "\nProvisioning complete: Application will start now\n\n"
|
||||
}
|
||||
|
||||
function provisioning_has_valid_hf_token() {
|
||||
[[ -n "$HF_TOKEN" ]] || return 1
|
||||
url="https://huggingface.co/api/whoami-v2"
|
||||
|
||||
response=$(curl -o /dev/null -s -w "%{http_code}" -X GET "$url" \
|
||||
-H "Authorization: Bearer $HF_TOKEN" \
|
||||
-H "Content-Type: application/json")
|
||||
|
||||
# Check if the token is valid
|
||||
if [ "$response" -eq 200 ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function provisioning_has_valid_civitai_token() {
|
||||
[[ -n "$CIVITAI_TOKEN" ]] || return 1
|
||||
url="https://civitai.com/api/v1/models?hidden=1&limit=1"
|
||||
|
||||
response=$(curl -o /dev/null -s -w "%{http_code}" -X GET "$url" \
|
||||
-H "Authorization: Bearer $CIVITAI_TOKEN" \
|
||||
-H "Content-Type: application/json")
|
||||
|
||||
# Check if the token is valid
|
||||
if [ "$response" -eq 200 ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Download from $1 URL to $2 file path
|
||||
function provisioning_download() {
|
||||
if [[ -n $HF_TOKEN && $1 =~ ^https://([a-zA-Z0-9_-]+\\.)?huggingface\\.co(/|$|\\?) ]]; then
|
||||
auth_token="$HF_TOKEN"
|
||||
elif
|
||||
[[ -n $CIVITAI_TOKEN && $1 =~ ^https://([a-zA-Z0-9_-]+\\.)?civitai\\.com(/|$|\\?) ]]; then
|
||||
auth_token="$CIVITAI_TOKEN"
|
||||
fi
|
||||
if [[ -n $auth_token ]];then
|
||||
wget --header="Authorization: Bearer $auth_token" --content-disposition -e dotbytes="${3:-4M}" -P "$2" "$1"
|
||||
else
|
||||
wget --content-disposition -e dotbytes="${3:-4M}" -P "$2" "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Allow user to disable provisioning if they started with a script they didn't want
|
||||
if [[ ! -f /.noprovisioning ]]; then
|
||||
provisioning_start
|
||||
fi
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"name": "ComfyUI - Wan 2.2 I2V/T2V - 5090 Blackwell",
|
||||
"description": "Complete AI video generation environment with ComfyUI, QwenVL-Mod enhanced vision-language capabilities, and WAN 2.2 workflows. Features professional video generation, multilingual support, visual style detection, and pre-installed custom nodes.",
|
||||
"version": "1.0.0",
|
||||
"author": "huchukato",
|
||||
"referral_id": "188688",
|
||||
"creator_id": "188688",
|
||||
"tags": [
|
||||
"comfyui",
|
||||
"wan2.2",
|
||||
"video-generation",
|
||||
"qwenvl-mod",
|
||||
"vision-language",
|
||||
"multilingual",
|
||||
"style-detection",
|
||||
"professional",
|
||||
"5090-blackwell",
|
||||
"rtx-5090"
|
||||
],
|
||||
"features": {
|
||||
"gpu_optimized": true,
|
||||
"pre_installed_nodes": 30,
|
||||
"pre_installed_workflows": 9,
|
||||
"pre_installed_models": true,
|
||||
"multilingual_support": true,
|
||||
"style_detection": true,
|
||||
"nsfw_support": true,
|
||||
"gguf_backend": true
|
||||
},
|
||||
"requirements": {
|
||||
"min_vram": "24GB",
|
||||
"recommended_gpu": "RTX 5090",
|
||||
"cuda_version": "13.0+"
|
||||
},
|
||||
"provisioning": {
|
||||
"script_url": "https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/provisioning.sh",
|
||||
"supports_tokens": ["HF_TOKEN", "CIVITAI_TOKEN"],
|
||||
"auto_download": true
|
||||
},
|
||||
"links": {
|
||||
"repository": "https://github.com/huchukato/ComfyUI-QwenVL-Mod",
|
||||
"documentation": "https://github.com/huchukato/ComfyUI-QwenVL-Mod/tree/main/vastai",
|
||||
"issues": "https://github.com/huchukato/ComfyUI-QwenVL-Mod/issues"
|
||||
},
|
||||
"environment_variables": {
|
||||
"WORKSPACE": "/workspace",
|
||||
"COMFYUI_ARGS": "--disable-auto-launch --port 18188 --enable-cors-header --fast fp16_accumulation --use-sage-attention --reserve-vram 2 --cuda-malloc --async-offload --supports-fp8-compute --fast cublas_ops autotune",
|
||||
"PROVISIONING_SCRIPT": "https://github.com/huchukato/ComfyUI-QwenVL-Mod/raw/main/vastai/provisioning.sh",
|
||||
"HF_TOKEN": null,
|
||||
"CIVITAI_TOKEN": null,
|
||||
"ENABLE_HTTPS": false
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@@ -1,367 +0,0 @@
|
||||
/* ═════════════════════════════════════════════════════════════════════════════
|
||||
QwenVL-Mod shared frontend styles
|
||||
Scoped to .qwen-chat-* elements so we never stomp on ComfyUI core UI.
|
||||
═════════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
:root {
|
||||
--qwen-accent: #f97316;
|
||||
--qwen-accent-hover: #ea580c;
|
||||
--qwen-bg-1: #1f2937;
|
||||
--qwen-bg-2: #111827;
|
||||
--qwen-border: #374151;
|
||||
--qwen-text: #e5e7eb;
|
||||
--qwen-text-dim: #9ca3af;
|
||||
--qwen-radius: 6px;
|
||||
--qwen-gap: 8px;
|
||||
}
|
||||
|
||||
/* ── Chat container ───────────────────────────────────────────────────────── */
|
||||
.qwen-chat {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
background: var(--qwen-bg-2);
|
||||
color: var(--qwen-text);
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-topbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--qwen-gap);
|
||||
padding: 8px 10px;
|
||||
background: var(--qwen-bg-1);
|
||||
border-bottom: 1px solid var(--qwen-border);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-language {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-language button {
|
||||
background: transparent;
|
||||
border: 1px solid var(--qwen-border);
|
||||
color: var(--qwen-text-dim);
|
||||
border-radius: var(--qwen-radius);
|
||||
padding: 3px 8px;
|
||||
cursor: pointer;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-language button.active,
|
||||
.qwen-chat .qwen-chat-language button:hover {
|
||||
background: var(--qwen-accent);
|
||||
border-color: var(--qwen-accent);
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-settings-toggle {
|
||||
background: transparent;
|
||||
border: 1px solid var(--qwen-border);
|
||||
color: var(--qwen-text-dim);
|
||||
border-radius: var(--qwen-radius);
|
||||
padding: 4px 8px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-settings-toggle.active,
|
||||
.qwen-chat .qwen-chat-settings-toggle:hover {
|
||||
color: var(--qwen-accent);
|
||||
border-color: var(--qwen-accent);
|
||||
}
|
||||
|
||||
/* ── Settings panel ────────────────────────────────────────────────────────── */
|
||||
.qwen-chat .qwen-chat-controls {
|
||||
display: none;
|
||||
flex-direction: column;
|
||||
gap: var(--qwen-gap);
|
||||
padding: 10px;
|
||||
background: var(--qwen-bg-1);
|
||||
border-bottom: 1px solid var(--qwen-border);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-controls.open {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-controls label {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 3px;
|
||||
font-size: 11px;
|
||||
color: var(--qwen-text-dim);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-controls select,
|
||||
.qwen-chat .qwen-chat-controls input {
|
||||
background: var(--qwen-bg-2);
|
||||
border: 1px solid var(--qwen-border);
|
||||
border-radius: var(--qwen-radius);
|
||||
color: var(--qwen-text);
|
||||
padding: 4px 6px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
/* ── Messages area ─────────────────────────────────────────────────────────── */
|
||||
.qwen-chat .qwen-chat-messages {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 10px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-empty {
|
||||
color: var(--qwen-text-dim);
|
||||
text-align: center;
|
||||
margin-top: 20px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-message {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 3px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-message.user {
|
||||
align-items: flex-end;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-message.assistant {
|
||||
align-items: flex-start;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-role {
|
||||
font-size: 10px;
|
||||
color: var(--qwen-text-dim);
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-content {
|
||||
max-width: 92%;
|
||||
padding: 8px 10px;
|
||||
border-radius: var(--qwen-radius);
|
||||
background: var(--qwen-bg-1);
|
||||
border: 1px solid var(--qwen-border);
|
||||
line-height: 1.4;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-message.user .qwen-chat-content {
|
||||
background: rgba(249, 115, 22, 0.15);
|
||||
border-color: rgba(249, 115, 22, 0.4);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-choice {
|
||||
background: var(--qwen-bg-1);
|
||||
border: 1px solid var(--qwen-border);
|
||||
color: var(--qwen-text);
|
||||
border-radius: var(--qwen-radius);
|
||||
padding: 5px 10px;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
margin: 3px 0;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-choice:hover {
|
||||
border-color: var(--qwen-accent);
|
||||
color: var(--qwen-accent);
|
||||
}
|
||||
|
||||
/* ── Composer ─────────────────────────────────────────────────────────────── */
|
||||
.qwen-chat .qwen-chat-composer {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--qwen-gap);
|
||||
padding: 10px;
|
||||
background: var(--qwen-bg-1);
|
||||
border-top: 1px solid var(--qwen-border);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-input {
|
||||
min-height: 56px;
|
||||
max-height: 160px;
|
||||
resize: vertical;
|
||||
background: var(--qwen-bg-2);
|
||||
border: 1px solid var(--qwen-border);
|
||||
border-radius: var(--qwen-radius);
|
||||
color: var(--qwen-text);
|
||||
padding: 8px;
|
||||
font-size: 13px;
|
||||
line-height: 1.35;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-input:focus {
|
||||
outline: none;
|
||||
border-color: var(--qwen-accent);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-selectors {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--qwen-gap);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-selectors label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
flex: 1 1 120px;
|
||||
font-size: 11px;
|
||||
color: var(--qwen-text-dim);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-selectors select {
|
||||
flex: 1;
|
||||
min-width: 80px;
|
||||
background: var(--qwen-bg-2);
|
||||
border: 1px solid var(--qwen-border);
|
||||
border-radius: var(--qwen-radius);
|
||||
color: var(--qwen-text);
|
||||
padding: 3px 5px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-composer-tools {
|
||||
display: flex;
|
||||
gap: var(--qwen-gap);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-attach,
|
||||
.qwen-chat .qwen-chat-assets-button {
|
||||
background: var(--qwen-bg-2);
|
||||
border: 1px solid var(--qwen-border);
|
||||
color: var(--qwen-text);
|
||||
border-radius: var(--qwen-radius);
|
||||
padding: 5px 10px;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-attach:hover,
|
||||
.qwen-chat .qwen-chat-assets-button:hover {
|
||||
border-color: var(--qwen-accent);
|
||||
color: var(--qwen-accent);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-actions {
|
||||
display: flex;
|
||||
gap: var(--qwen-gap);
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-actions button {
|
||||
background: var(--qwen-bg-2);
|
||||
border: 1px solid var(--qwen-border);
|
||||
color: var(--qwen-text);
|
||||
border-radius: var(--qwen-radius);
|
||||
padding: 6px 14px;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-actions button:hover {
|
||||
border-color: var(--qwen-accent);
|
||||
color: var(--qwen-accent);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-actions button:first-child {
|
||||
background: var(--qwen-accent);
|
||||
border-color: var(--qwen-accent);
|
||||
color: #fff;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-actions button:first-child:hover {
|
||||
background: var(--qwen-accent-hover);
|
||||
border-color: var(--qwen-accent-hover);
|
||||
}
|
||||
|
||||
/* ── Attachment / asset previews ───────────────────────────────────────────── */
|
||||
.qwen-chat .qwen-chat-attachment {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--qwen-gap);
|
||||
padding: 6px;
|
||||
background: var(--qwen-bg-2);
|
||||
border: 1px solid var(--qwen-border);
|
||||
border-radius: var(--qwen-radius);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-attachment-preview {
|
||||
max-width: 48px;
|
||||
max-height: 48px;
|
||||
border-radius: 4px;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-attachment-remove {
|
||||
background: transparent;
|
||||
border: 1px solid var(--qwen-border);
|
||||
color: var(--qwen-text-dim);
|
||||
border-radius: var(--qwen-radius);
|
||||
padding: 3px 8px;
|
||||
cursor: pointer;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-attachment-remove:hover {
|
||||
border-color: #ef4444;
|
||||
color: #ef4444;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-asset-modal.visible,
|
||||
.qwen-chat .qwen-chat-assets-modal.visible {
|
||||
display: flex !important;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-assets-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(90px, 1fr));
|
||||
gap: var(--qwen-gap);
|
||||
padding: 10px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-asset {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
background: var(--qwen-bg-1);
|
||||
border: 1px solid var(--qwen-border);
|
||||
border-radius: var(--qwen-radius);
|
||||
padding: 6px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-asset:hover {
|
||||
border-color: var(--qwen-accent);
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-asset img,
|
||||
.qwen-chat .qwen-chat-asset video {
|
||||
width: 100%;
|
||||
aspect-ratio: 1;
|
||||
object-fit: cover;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.qwen-chat .qwen-chat-asset span {
|
||||
font-size: 10px;
|
||||
color: var(--qwen-text-dim);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
max-width: 100%;
|
||||
}
|
||||
+28
-76
@@ -1,98 +1,50 @@
|
||||
import { app } from "/scripts/app.js";
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════════
|
||||
// QwenVL-Mod node branding — inspired by ComfyUI-Pixaroma's brand extension.
|
||||
// Each logical family gets a recognisable title/body colour so big workflows
|
||||
// are easy to read at a glance. User colour choices and workflow-saved colours
|
||||
// are always preserved (guard with !this.color / !this.bgcolor).
|
||||
// ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const COLOR_THEMES = {
|
||||
Vision: { nodeColor: "#6d28d9", nodeBgColor: "#2e1065", width: 360 }, // HF vision
|
||||
VisionGGUF: { nodeColor: "#0d9489", nodeBgColor: "#134e4a", width: 360 }, // GGUF / unified
|
||||
Enhancer: { nodeColor: "#d97706", nodeBgColor: "#451a03", width: 360 }, // prompt enhancer
|
||||
Utils: { nodeColor: "#475569", nodeBgColor: "#1e293b", width: 300 }, // load media, tools
|
||||
QwenVL: { nodeColor: "#28403f", nodeBgColor: "#374539", width: 340 },
|
||||
QwenVLGGUF: { nodeColor: "#474539", nodeBgColor: "#2c4045", width: 340 },
|
||||
Tools: { nodeColor: "#28403f", nodeBgColor: "#233238", width: 300 },
|
||||
Enhancer: { nodeColor: "#374445", nodeBgColor: "#474539", width: 340 },
|
||||
};
|
||||
|
||||
const NODE_COLORS = {
|
||||
// HF vision
|
||||
"AILab_QwenVL": "Vision",
|
||||
"AILab_QwenVL_Advanced": "Vision",
|
||||
|
||||
// GGUF / unified
|
||||
"AILab_QwenVL_GGUF": "VisionGGUF",
|
||||
"AILab_QwenVL_GGUF_Advanced": "VisionGGUF",
|
||||
"QwenVL_Unified": "VisionGGUF",
|
||||
"QwenVL_Unified_Advanced": "VisionGGUF",
|
||||
|
||||
// Prompt enhancers
|
||||
// QwenVL nodes
|
||||
"AILab_QwenVL": "QwenVL",
|
||||
"AILab_QwenVL_Advanced": "QwenVL",
|
||||
"AILab_QwenVL_PromptEnhancer": "Enhancer",
|
||||
"AILab_QwenVL_GGUF": "QwenVLGGUF",
|
||||
"AILab_QwenVL_GGUF_Advanced": "QwenVLGGUF",
|
||||
"AILab_QwenVL_GGUF_PromptEnhancer": "Enhancer",
|
||||
"QwenVL_Unified_PromptEnhancer": "Enhancer",
|
||||
|
||||
// Utils / media
|
||||
"QwenVL_LoadMedia": "Utils",
|
||||
"StorySplitNode": "Utils",
|
||||
"VRAMCleanup": "Utils",
|
||||
// Tools
|
||||
"AILab_QwenVL_PromptLibrary": "Tools",
|
||||
};
|
||||
|
||||
function applyTheme(node, theme) {
|
||||
if (!theme) return;
|
||||
if (theme.nodeColor && !node.color) {
|
||||
function setNodeColors(node, theme) {
|
||||
if (!theme) { return; }
|
||||
if (theme.nodeColor) {
|
||||
node.color = theme.nodeColor;
|
||||
}
|
||||
if (theme.nodeBgColor && !node.bgcolor) {
|
||||
if (theme.nodeBgColor) {
|
||||
node.bgcolor = theme.nodeBgColor;
|
||||
}
|
||||
if (theme.width) {
|
||||
node.size = node.size || [theme.width, 80];
|
||||
node.size[0] = Math.max(node.size[0], theme.width);
|
||||
node.size = node.size || [140, 80];
|
||||
node.size[0] = theme.width;
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy nodes stay registered so saved workflows keep loading, but are
|
||||
// hidden from the add-node menu/search — the Unified nodes supersede them.
|
||||
const LEGACY_NODES = new Set([
|
||||
"AILab_QwenVL",
|
||||
"AILab_QwenVL_Advanced",
|
||||
"AILab_QwenVL_GGUF",
|
||||
"AILab_QwenVL_GGUF_Advanced",
|
||||
"AILab_QwenVL_PromptEnhancer",
|
||||
"AILab_QwenVL_GGUF_PromptEnhancer",
|
||||
]);
|
||||
const ext = {
|
||||
name: "QwenVL.appearance",
|
||||
|
||||
app.registerExtension({
|
||||
name: "QwenVL.BrandColors",
|
||||
|
||||
beforeRegisterNodeDef(nodeType, nodeData) {
|
||||
const cls = nodeData?.name;
|
||||
if (!cls) return;
|
||||
if (LEGACY_NODES.has(cls)) {
|
||||
nodeData.hidden = true;
|
||||
nodeCreated(node) {
|
||||
const nclass = node.comfyClass;
|
||||
if (NODE_COLORS.hasOwnProperty(nclass)) {
|
||||
let colorKey = NODE_COLORS[nclass];
|
||||
const theme = COLOR_THEMES[colorKey];
|
||||
setNodeColors(node, theme);
|
||||
}
|
||||
if (!NODE_COLORS.hasOwnProperty(cls)) return;
|
||||
const themeKey = NODE_COLORS[cls];
|
||||
const theme = COLOR_THEMES[themeKey];
|
||||
}
|
||||
};
|
||||
|
||||
const orig = nodeType.prototype.onNodeCreated;
|
||||
nodeType.prototype.onNodeCreated = function () {
|
||||
const ret = orig?.apply(this, arguments);
|
||||
applyTheme(this, theme);
|
||||
return ret;
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════════
|
||||
// Shared stylesheet for QwenVL-Mod DOM pieces (chat sidebar, asset modal, ...).
|
||||
// Injected once; class names are prefixed with qwen- to avoid collisions.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const STYLE_ID = "qwenvl-shared-styles";
|
||||
if (!document.getElementById(STYLE_ID)) {
|
||||
const link = document.createElement("link");
|
||||
link.id = STYLE_ID;
|
||||
link.rel = "stylesheet";
|
||||
link.href = new URL("css/qwen_nodes.css", import.meta.url).href;
|
||||
document.head.appendChild(link);
|
||||
}
|
||||
app.registerExtension(ext);
|
||||
-1082
File diff suppressed because it is too large
Load Diff
@@ -1,46 +0,0 @@
|
||||
"""Shared TagForge wildcard expansion helper.
|
||||
|
||||
Finds WildcardLoader regardless of the TagForge install dir name and expands
|
||||
__wildcard__ tokens. Returns the input unchanged when TagForge is absent.
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
def wildcard_loader():
|
||||
try:
|
||||
for mod in list(sys.modules.values()):
|
||||
mod_name = (getattr(mod, "__name__", "") or "").lower()
|
||||
if mod is not None and "tagforge" in mod_name and mod_name.endswith("wildcards") and hasattr(mod, "WildcardLoader"):
|
||||
return mod.WildcardLoader
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import importlib
|
||||
for name in ("ComfyUI-TagForge.py.wildcards", "ComfyUI_TagForge.py.wildcards", "comfyui_tagforge.py.wildcards"):
|
||||
try:
|
||||
module = importlib.import_module(name)
|
||||
if hasattr(module, "WildcardLoader"):
|
||||
return module.WildcardLoader
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def expand_wildcard_tokens(text):
|
||||
if not text or "__" not in text:
|
||||
return text
|
||||
loader = wildcard_loader()
|
||||
if loader is not None:
|
||||
try:
|
||||
expanded = loader.process(text)
|
||||
if expanded:
|
||||
if expanded != text:
|
||||
preview = expanded if len(expanded) <= 500 else expanded[:500] + "…"
|
||||
print(f"[QwenVL] Wildcard expanded: {preview}")
|
||||
return expanded
|
||||
except Exception:
|
||||
pass
|
||||
return text
|
||||
Reference in New Issue
Block a user