diff --git a/README.md b/README.md index fc7f99d..600ae4a 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Loads the CLIPtion decoder on demand at caption time (rather than at workflow-lo | --- | --- | --- | | `clip` | `CLIP` | CLIP text encoder (must include CLIP-L). | | `clip_vision` | `CLIP_VISION` | CLIP vision encoder (must be CLIP-L). | -| `image` | `IMAGE` | Input Image to caption. | +| `image` | `IMAGE` | Image to caption. | | `beam_width` | `INT` | Number of beams to maintain during search (default: 4). | | `unload_after_run` | `BOOLEAN` | Unload the CLIPtion decoder from VRAM after captioning (default: False). | | `device` (optional) | `["default", "cpu"]` | Inference device override. | @@ -33,11 +33,11 @@ Loads the CLIPtion decoder on demand at caption time (rather than at workflow-lo | Name | Type | Description | | --- | --- | --- | -| `STRING` | `STRING` (list) | Generated caption | +| `STRING` | `STRING` (list) | Generated caption. | -The CLIPtion decoder weights (`CLIPtion_20241219_fp16.safetensors`) are downloaded automatically from [easygoing0114/ComfyUI-use-models](https://huggingface.co/easygoing0114/ComfyUI-use-models/tree/main) on Hugging Face the first time the node runs, and are cached on disk afterwards. The weights have also been mirrored to that same repository for convenience. +The CLIPtion decoder weights (`CLIPtion_20241219_fp16.safetensors`) are downloaded automatically from [easygoing0114/ComfyUI-use-models](https://huggingface.co/easygoing0114/ComfyUI-use-models/tree/main) on Hugging Face the first time the node runs, and are saved to `ComfyUI/models/cliption` for reuse on subsequent runs. You can also place a copy of the file in that folder yourself ahead of time to skip the automatic download. -### Recommended high-accuracy CLIP-L model +### Recommended CLIP-L model For the `clip` and `clip_vision` inputs, we recommend [zer0int/CLIP-SAE-ViT-L-14](https://huggingface.co/zer0int/CLIP-SAE-ViT-L-14), a free, high-accuracy fine-tune of CLIP ViT-L/14 released by zer0int. This model has also been mirrored to the [easygoing0114/ComfyUI-use-models](https://huggingface.co/easygoing0114/ComfyUI-use-models/tree/main) repository above. @@ -58,7 +58,7 @@ Restart ComfyUI. The node should now appear in the node search as **CLIPtion Bea ## Credits - [pharmapsychotic/comfy-cliption](https://github.com/pharmapsychotic/comfy-cliption) — original ComfyUI node pack this fork is based on. -- [pharmapsychotic/CLIPtion](https://github.com/pharmapsychotic/CLIPtion) — the underlying CLIPtion model. +- [pharmapsychotic/CLIPtion](https://huggingface.co/pharmapsychotic/CLIPtion) — the underlying CLIPtion model. - [zer0int/CLIP-SAE-ViT-L-14](https://huggingface.co/zer0int/CLIP-SAE-ViT-L-14) — recommended high-accuracy CLIP-L model, freely released by zer0int. ## License diff --git a/nodes.py b/nodes.py index 7759ad2..6ea6522 100644 --- a/nodes.py +++ b/nodes.py @@ -11,9 +11,17 @@ from huggingface_hub import hf_hub_download from safetensors import safe_open import comfy.model_management +import folder_paths from comfy_api.latest import ComfyExtension, io +# register comfyui/models/cliption as a known model folder so the +# downloaded decoder weights are cached alongside ComfyUI's other models +_CLIPTION_MODELS_DIR = os.path.join(folder_paths.models_dir, "cliption") +os.makedirs(_CLIPTION_MODELS_DIR, exist_ok=True) +folder_paths.add_model_folder_path("cliption", _CLIPTION_MODELS_DIR) + + def _fix_punctuation_spacing(text: str) -> str: """Remove stray whitespace before commas and periods left over from CLIPTokenizer's BPE decoding (e.g. "cats , dogs ." -> "cats, dogs.").""" @@ -354,14 +362,25 @@ class CLIPtionBeamSearchIntegrated(io.ComfyNode): if _DecoderCache.is_loaded(): return - base_path = os.path.dirname(os.path.abspath(__file__)) - local_path = os.path.join(base_path, cls._SAFETENSORS_FILE) - if os.path.exists(local_path): - model_path = local_path + # 1) allow a manually placed copy anywhere under comfyui/models/cliption + # (covers extra_model_paths / multiple registered search paths too) + existing_paths = folder_paths.get_filename_list("cliption") + if cls._SAFETENSORS_FILE in existing_paths: + model_path = folder_paths.get_full_path("cliption", cls._SAFETENSORS_FILE) else: - model_path = hf_hub_download( - repo_id=cls._HF_REPO_ID, filename=cls._SAFETENSORS_FILE, revision=cls._HF_REVISION - ) + # 2) fall back to a copy sitting next to this node (legacy location) + base_path = os.path.dirname(os.path.abspath(__file__)) + local_path = os.path.join(base_path, cls._SAFETENSORS_FILE) + if os.path.exists(local_path): + model_path = local_path + else: + # 3) download into comfyui/models/cliption and cache it there + model_path = hf_hub_download( + repo_id=cls._HF_REPO_ID, + filename=cls._SAFETENSORS_FILE, + revision=cls._HF_REVISION, + local_dir=_CLIPTION_MODELS_DIR, + ) state_dict = {} with safe_open(model_path, framework="pt", device="cpu") as f: diff --git a/pyproject.toml b/pyproject.toml index ecbb99e..0f2ffe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-cliption-integrated" description = "Integrated beam-search-only fork of pharmapsychotic's ComfyUI-Cliption, combining the CLIPtion loader and beam search into a single node" -version = "0.1.1" +version = "0.1.2" license = { text = "MIT" } [project.urls]