modified: README.md
modified: nodes.py modified: pyproject.toml
This commit is contained in:
@@ -24,7 +24,7 @@ Loads the CLIPtion decoder on demand at caption time (rather than at workflow-lo
|
||||
| --- | --- | --- |
|
||||
| `clip` | `CLIP` | CLIP text encoder (must include CLIP-L). |
|
||||
| `clip_vision` | `CLIP_VISION` | CLIP vision encoder (must be CLIP-L). |
|
||||
| `image` | `IMAGE` | Input Image to caption. |
|
||||
| `image` | `IMAGE` | Image to caption. |
|
||||
| `beam_width` | `INT` | Number of beams to maintain during search (default: 4). |
|
||||
| `unload_after_run` | `BOOLEAN` | Unload the CLIPtion decoder from VRAM after captioning (default: False). |
|
||||
| `device` (optional) | `["default", "cpu"]` | Inference device override. |
|
||||
@@ -33,11 +33,11 @@ Loads the CLIPtion decoder on demand at caption time (rather than at workflow-lo
|
||||
|
||||
| Name | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `STRING` | `STRING` (list) | Generated caption |
|
||||
| `STRING` | `STRING` (list) | Generated caption. |
|
||||
|
||||
The CLIPtion decoder weights (`CLIPtion_20241219_fp16.safetensors`) are downloaded automatically from [easygoing0114/ComfyUI-use-models](https://huggingface.co/easygoing0114/ComfyUI-use-models/tree/main) on Hugging Face the first time the node runs, and are cached on disk afterwards. The weights have also been mirrored to that same repository for convenience.
|
||||
The CLIPtion decoder weights (`CLIPtion_20241219_fp16.safetensors`) are downloaded automatically from [easygoing0114/ComfyUI-use-models](https://huggingface.co/easygoing0114/ComfyUI-use-models/tree/main) on Hugging Face the first time the node runs, and are saved to `ComfyUI/models/cliption` for reuse on subsequent runs. You can also place a copy of the file in that folder yourself ahead of time to skip the automatic download.
|
||||
|
||||
### Recommended high-accuracy CLIP-L model
|
||||
### Recommended CLIP-L model
|
||||
|
||||
For the `clip` and `clip_vision` inputs, we recommend [zer0int/CLIP-SAE-ViT-L-14](https://huggingface.co/zer0int/CLIP-SAE-ViT-L-14), a free, high-accuracy fine-tune of CLIP ViT-L/14 released by zer0int. This model has also been mirrored to the [easygoing0114/ComfyUI-use-models](https://huggingface.co/easygoing0114/ComfyUI-use-models/tree/main) repository above.
|
||||
|
||||
@@ -58,7 +58,7 @@ Restart ComfyUI. The node should now appear in the node search as **CLIPtion Bea
|
||||
## Credits
|
||||
|
||||
- [pharmapsychotic/comfy-cliption](https://github.com/pharmapsychotic/comfy-cliption) — original ComfyUI node pack this fork is based on.
|
||||
- [pharmapsychotic/CLIPtion](https://github.com/pharmapsychotic/CLIPtion) — the underlying CLIPtion model.
|
||||
- [pharmapsychotic/CLIPtion](https://huggingface.co/pharmapsychotic/CLIPtion) — the underlying CLIPtion model.
|
||||
- [zer0int/CLIP-SAE-ViT-L-14](https://huggingface.co/zer0int/CLIP-SAE-ViT-L-14) — recommended high-accuracy CLIP-L model, freely released by zer0int.
|
||||
|
||||
## License
|
||||
|
||||
@@ -11,9 +11,17 @@ from huggingface_hub import hf_hub_download
|
||||
from safetensors import safe_open
|
||||
|
||||
import comfy.model_management
|
||||
import folder_paths
|
||||
from comfy_api.latest import ComfyExtension, io
|
||||
|
||||
|
||||
# register comfyui/models/cliption as a known model folder so the
|
||||
# downloaded decoder weights are cached alongside ComfyUI's other models
|
||||
_CLIPTION_MODELS_DIR = os.path.join(folder_paths.models_dir, "cliption")
|
||||
os.makedirs(_CLIPTION_MODELS_DIR, exist_ok=True)
|
||||
folder_paths.add_model_folder_path("cliption", _CLIPTION_MODELS_DIR)
|
||||
|
||||
|
||||
def _fix_punctuation_spacing(text: str) -> str:
|
||||
"""Remove stray whitespace before commas and periods left over from
|
||||
CLIPTokenizer's BPE decoding (e.g. "cats , dogs ." -> "cats, dogs.")."""
|
||||
@@ -354,14 +362,25 @@ class CLIPtionBeamSearchIntegrated(io.ComfyNode):
|
||||
if _DecoderCache.is_loaded():
|
||||
return
|
||||
|
||||
base_path = os.path.dirname(os.path.abspath(__file__))
|
||||
local_path = os.path.join(base_path, cls._SAFETENSORS_FILE)
|
||||
if os.path.exists(local_path):
|
||||
model_path = local_path
|
||||
# 1) allow a manually placed copy anywhere under comfyui/models/cliption
|
||||
# (covers extra_model_paths / multiple registered search paths too)
|
||||
existing_paths = folder_paths.get_filename_list("cliption")
|
||||
if cls._SAFETENSORS_FILE in existing_paths:
|
||||
model_path = folder_paths.get_full_path("cliption", cls._SAFETENSORS_FILE)
|
||||
else:
|
||||
model_path = hf_hub_download(
|
||||
repo_id=cls._HF_REPO_ID, filename=cls._SAFETENSORS_FILE, revision=cls._HF_REVISION
|
||||
)
|
||||
# 2) fall back to a copy sitting next to this node (legacy location)
|
||||
base_path = os.path.dirname(os.path.abspath(__file__))
|
||||
local_path = os.path.join(base_path, cls._SAFETENSORS_FILE)
|
||||
if os.path.exists(local_path):
|
||||
model_path = local_path
|
||||
else:
|
||||
# 3) download into comfyui/models/cliption and cache it there
|
||||
model_path = hf_hub_download(
|
||||
repo_id=cls._HF_REPO_ID,
|
||||
filename=cls._SAFETENSORS_FILE,
|
||||
revision=cls._HF_REVISION,
|
||||
local_dir=_CLIPTION_MODELS_DIR,
|
||||
)
|
||||
|
||||
state_dict = {}
|
||||
with safe_open(model_path, framework="pt", device="cpu") as f:
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "comfyui-cliption-integrated"
|
||||
description = "Integrated beam-search-only fork of pharmapsychotic's ComfyUI-Cliption, combining the CLIPtion loader and beam search into a single node"
|
||||
version = "0.1.1"
|
||||
version = "0.1.2"
|
||||
license = { text = "MIT" }
|
||||
|
||||
[project.urls]
|
||||
|
||||
Reference in New Issue
Block a user