nodes/pharmapsychotic-comfy-cliption
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update readme with example workflow

Browse Source
This commit is contained in:
pharmapsychotic
2024-12-23 10:53:41 -06:00
parent 49f91986cc
commit a1e3fc3cc3
4 changed files with 443 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+19 -2
View File
@@ -2,16 +2,33 @@
**CLIPtion** is a fast and small captioning extension to [OpenAI CLIP](https://github.com/openai/CLIP/) ViT-L/14. You already have ViT-L loaded when using using Stable Diffusion, SDXL, SD3, FLUX, etc and with just an extra *100MB memory* you can include caption/prompt generation in your workflows!
I made this for fun and am sure bigger dedicated caption models and VLM's will give you more accurate captioning, but this guy is tiny, fast, reuses what you already have loaded, and has options to give better CLIP alignment so give it a try if you like!
Big thanks to [Ben Egan](https://github.com/ProGamerGov), [SilentAntagonist](https://huggingface.co/SilentAntagonist), [Alex Redden](https://github.com/aredden), [XWAVE](https://twitter.com/XWAVEart), and [Jacky-hate](https://github.com/Jacky-hate) whose synthetic caption datasets I included in the training.
# Example workflow
...
![example workflow](media/CLIPtion-example.png)
[CLIPtion-example.json](media/CLIPtion-example.json)
# Installation
1. Clone this repo to your ComfyUI/custom_nodes directory.
```shell
cd custom_nodes
git clone https://github.com/pharmapsychotic/comfy-cliption.git
pip install -r comfy-cliption/requirements.txt
```
2. Download [CLIPtion_20241219_fp16.safetensors](https://huggingface.co/pharmapsychotic/CLIPtion/blob/main/CLIPtion_20241219_fp16.safetensors) and put in your `ComfyUI/custom_nodes/comfy-cliption` directory. You can skip this step to let it auto-download on first use to your `HF_HOME` cache.
3. Restart ComfyUI
# Nodes
## CLIPtion Loader
The loader will automatically download the CLIPtion model for you the first time it is run from the [HuggingFace CLIPtion repo](https://huggingface.co/pharmapsychotic/CLIPtion). It gets stored in the [HuggingFace cache dir](https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache) (controlled by `HF_HOME` environment variable). You could also manually download the safetensor file and put in comfy-cliption directory and that will be used instead.
If `CLIPtion_20241219_fp16.safetensors` is not already downloaded (as in step 2 of Installation) then the loader will automatically download the CLIPtion model for you the first time it is run from the [HuggingFace CLIPtion repo](https://huggingface.co/pharmapsychotic/CLIPtion). It gets stored in the [HuggingFace cache dir](https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache) (controlled by `HF_HOME` environment variable).
## CLIPtion Generate
__init__.py
+20 -4
View File
@@ -148,6 +148,11 @@ class CLIPtionModel(nn.Module):
image_embeds = image_outputs.image_embeds.to(device, dtype=torch.float16)
image_embeds /= image_embeds.norm(dim=-1, keepdim=True)
if image_features.size(2) != 1024:
raise ValueError(
f"Expected image features to have 1024 dimensions but got {image_features.size(2)}. Please ensure you are using CLIP L."
)
captions = []
for image_idx in range(image_features.size(0)):
features = image_features[image_idx : image_idx + 1]
@@ -205,6 +210,11 @@ class CLIPtionModel(nn.Module):
image_embeds = image_outputs.image_embeds.to(device, dtype=torch.float16)
image_embeds /= image_embeds.norm(dim=-1, keepdim=True)
if image_features.size(2) != 1024:
raise ValueError(
f"Expected image features to have 1024 dimensions but got {image_features.size(2)}. Please ensure you are using CLIP L."
)
captions = []
for image_idx in range(image_features.size(0)):
features = image_features[image_idx].unsqueeze(0)
@@ -374,6 +384,12 @@ class CLIPtionModel(nn.Module):
if (current_tokens[:, -1] == tokenizer.eos_token_id).all():
break
# add final EOS token
current_tokens = torch.cat(
[current_tokens, torch.full((beam_width, 1), tokenizer.eos_token_id, device=device)],
dim=1,
)
# rank final candidates by CLIP similarity
candidates = []
for idx in range(beam_width):
@@ -402,7 +418,7 @@ class CLIPtionModel(nn.Module):
class CLIPtionLoader:
CATEGORY = "pharmapsychotic"
FUNCTION = "load"
RETURN_TYPES = ("CLIPTION_MODEL",)
RETURN_TYPES = ("CLIPTION",)
@classmethod
def INPUT_TYPES(s):
@@ -454,7 +470,7 @@ class CLIPtionGenerate:
def INPUT_TYPES(s):
return {
"required": {
"model": ("CLIPTION_MODEL", {"tooltip": "The CLIPtion model."}),
"model": ("CLIPTION", {"tooltip": "The CLIPtion model."}),
"image": ("IMAGE",),
"seed": (
"INT",
@@ -508,7 +524,7 @@ class CLIPtionBeamSearch:
def INPUT_TYPES(s):
return {
"required": {
"model": ("CLIPTION_MODEL", {"tooltip": "The CLIPtion model."}),
"model": ("CLIPTION", {"tooltip": "The CLIPtion model."}),
"image": ("IMAGE",),
"beam_width": (
"INT",
@@ -547,4 +563,4 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"CLIPtionBeamSearch": "CLIPtion Beam Search",
"CLIPtionGenerate": "CLIPtion Generate",
"CLIPtionLoader": "CLIPtion Loader",
}
}
media/CLIPtion-example.json
+404
View File
@@ -0,0 +1,404 @@
{
"last_node_id": 32,
"last_link_id": 65,
"nodes": [
{
"id": 11,
"type": "CLIPtionLoader",
"pos": [
465,
850
],
"size": [
220,
46
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"name": "clip",
"type": "CLIP",
"link": 50
},
{
"name": "clip_vision",
"type": "CLIP_VISION",
"link": 15
}
],
"outputs": [
{
"name": "CLIPTION",
"type": "CLIPTION",
"links": [
29,
40
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPtionLoader"
},
"widgets_values": []
},
{
"id": 14,
"type": "CLIPLoader",
"pos": [
115.46097564697266,
787.0906372070312
],
"size": [
315,
82
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP",
"type": "CLIP",
"links": [
50
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPLoader"
},
"widgets_values": [
"clip_l.safetensors",
"stable_diffusion"
]
},
{
"id": 15,
"type": "CLIPVisionLoader",
"pos": [
119,
928
],
"size": [
315,
58
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "CLIP_VISION",
"type": "CLIP_VISION",
"links": [
15
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "CLIPVisionLoader"
},
"widgets_values": [
"clip_vit_l_14.safetensors"
]
},
{
"id": 22,
"type": "LoadImage",
"pos": [
116,
1045
],
"size": [
315,
314
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
64,
65
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"strawberry_bot.png",
"image"
]
},
{
"id": 23,
"type": "CLIPtionGenerate",
"pos": [
730,
850
],
"size": [
268.73236083984375,
174
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "CLIPTION",
"link": 29
},
{
"name": "image",
"type": "IMAGE",
"link": 64
}
],
"outputs": [
{
"name": "STRING",
"type": "STRING",
"links": [
46
],
"slot_index": 0,
"shape": 6
}
],
"properties": {
"Node name for S&R": "CLIPtionGenerate"
},
"widgets_values": [
42,
"fixed",
0.7000000000000001,
8,
false
]
},
{
"id": 30,
"type": "ShowText|pysssss",
"pos": [
1020,
840
],
"size": [
308.9680480957031,
175.7525634765625
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "text",
"type": "STRING",
"link": 46,
"widget": {
"name": "text"
}
}
],
"outputs": [
{
"name": "STRING",
"type": "STRING",
"links": null,
"shape": 6
}
],
"properties": {
"Node name for S&R": "ShowText|pysssss"
},
"widgets_values": [
"",
"a strawberry robot with glowing blue eyes and a metallic body, featuring red and green details, stands against a teal background."
]
},
{
"id": 28,
"type": "CLIPtionBeamSearch",
"pos": [
760,
1110
],
"size": [
212.6961669921875,
102
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "model",
"type": "CLIPTION",
"link": 40
},
{
"name": "image",
"type": "IMAGE",
"link": 65
}
],
"outputs": [
{
"name": "STRING",
"type": "STRING",
"links": [
42
],
"slot_index": 0,
"shape": 6
}
],
"properties": {
"Node name for S&R": "CLIPtionBeamSearch"
},
"widgets_values": [
8,
false
]
},
{
"id": 29,
"type": "ShowText|pysssss",
"pos": [
1020,
1080
],
"size": [
299.4249267578125,
167.9679412841797
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "text",
"type": "STRING",
"link": 42,
"widget": {
"name": "text"
}
}
],
"outputs": [
{
"name": "STRING",
"type": "STRING",
"links": null,
"shape": 6
}
],
"properties": {
"Node name for S&R": "ShowText|pysssss"
},
"widgets_values": [
"",
"an anthropomorphic, strawberry - shaped robot with glowing green eyes stands against a dark blue background, featuring a strawberry - shaped head, red body, green leaf - like appendages, and a small red plant on its head. its body is predominantly red with green and blue accents, and it possesses a small green leaf on its left arm. it"
]
}
],
"links": [
[
15,
15,
0,
11,
1,
"CLIP_VISION"
],
[
29,
11,
0,
23,
0,
"CLIPTION_MODEL"
],
[
40,
11,
0,
28,
0,
"CLIPTION_MODEL"
],
[
42,
28,
0,
29,
0,
"STRING"
],
[
46,
23,
0,
30,
0,
"STRING"
],
[
50,
14,
0,
11,
0,
"CLIP"
],
[
64,
22,
0,
23,
1,
"IMAGE"
],
[
65,
22,
0,
28,
1,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.7627768444385489,
"offset": [
396.05767259320686,
-622.3565097653578
]
},
"ue_links": []
},
"version": 0.4
}
media/CLIPtion-example.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 375 KiB