diff --git a/examples/lora_embedding.json b/examples/lora_embedding.json new file mode 100644 index 0000000..6b0eaa7 --- /dev/null +++ b/examples/lora_embedding.json @@ -0,0 +1,1185 @@ +{ + "last_node_id": 34, + "last_link_id": 89, + "nodes": [ + { + "id": 22, + "type": "PromptUtilitiesConstString", + "pos": [ + 21, + 204 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 29 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "PromptUtilitiesConstString" + }, + "widgets_values": [ + "1girl" + ] + }, + { + "id": 23, + "type": "PromptUtilitiesConstStringMultiLine", + "pos": [ + 23, + 313 + ], + "size": { + "0": 210, + "1": 93.83313751220703 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 30 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PromptUtilitiesConstStringMultiLine" + }, + "widgets_values": [ + "hatsune miku, vocaloid" + ] + }, + { + "id": 25, + "type": "PromptUtilitiesLoadPreset", + "pos": [ + 31, + 614 + ], + "size": { + "0": 306.8652038574219, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 32 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PromptUtilitiesLoadPreset" + }, + "widgets_values": [ + "sample\\animagineXL : pos postfix" + ] + }, + { + "id": 26, + "type": "PromptUtilitiesJoinStringList", + "pos": [ + 289, + 385 + ], + "size": { + "0": 210, + "1": 138 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "arg1", + "type": "STRING", + "link": 29, + "widget": { + "name": "arg1" + }, + "slot_index": 0 + }, + { + "name": "arg2", + "type": "STRING", + "link": 30 + }, + { + "name": "arg3", + "type": "STRING", + "link": 31, + "slot_index": 2 + }, + { + "name": "arg4", + "type": "STRING", + "link": 32 + }, + { + "name": "arg5", + "type": "STRING", + "link": null + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 33 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PromptUtilitiesJoinStringList" + }, + "widgets_values": [ + ", ", + "" + ] + }, + { + "id": 20, + "type": "PromptUtilitiesLoadPreset", + "pos": [ + 35, + 733 + ], + "size": { + "0": 270.42572021484375, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 75 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PromptUtilitiesLoadPreset" + }, + "widgets_values": [ + "sample\\animagineXL : neg" + ] + }, + { + "id": 2, + "type": "CLIPTextEncode", + "pos": [ + 552, + 365 + ], + "size": { + "0": 210, + "1": 54 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": null, + "slot_index": 0 + }, + { + "name": "text", + "type": "STRING", + "link": 33, + "widget": { + "name": "text" + } + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 39 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "1girl, hatsune miku, vocaloid, upper body, smile, waving, looking at viewer, masterpiece, best quality" + ] + }, + { + "id": 28, + "type": "Prompts Everywhere", + "pos": [ + 809, + 485 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "CONDITIONING", + "type": "*", + "link": 39, + "slot_index": 0, + "color_on": "#FFA931" + }, + { + "name": "CONDITIONING", + "type": "*", + "link": 40, + "slot_index": 1, + "color_on": "#FFA931" + } + ], + "properties": { + "Node name for S&R": "Prompts Everywhere", + "group_restricted": false, + "color_restricted": false + } + }, + { + "id": 14, + "type": "SaveImage", + "pos": [ + 1776, + 122 + ], + "size": { + "0": 263.2173767089844, + "1": 273.5193786621094 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 54, + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [ + "ComfyUI-without-meta" + ] + }, + { + "id": 16, + "type": "VAEDecode", + "pos": [ + 1470, + 170 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 21, + "slot_index": 0 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 54, + 56 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 9, + "type": "SaveImageWithMetaData", + "pos": [ + 1783, + 473 + ], + "size": { + "0": 267.79010009765625, + "1": 318 + }, + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 56, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "SaveImageWithMetaData" + }, + "widgets_values": [ + "ComfyUI-with-meta-far", + "Farthest", + 0 + ] + }, + { + "id": 21, + "type": "EmptyLatentImageFromPresetsSDXL", + "pos": [ + 701, + 189 + ], + "size": { + "0": 319.20001220703125, + "1": 122 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "latent", + "type": "LATENT", + "links": [ + 28 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "w", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "h", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImageFromPresetsSDXL" + }, + "widgets_values": [ + "1024 x 1024", + 1 + ] + }, + { + "id": 27, + "type": "Anything Everywhere3", + "pos": [ + 743, + 38 + ], + "size": { + "0": 210, + "1": 66 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "MODEL", + "type": "*", + "link": 74, + "slot_index": 0, + "color_on": "#B39DDB" + }, + { + "name": "CLIP", + "type": "*", + "link": 62, + "color_on": "#FFD500", + "slot_index": 1 + }, + { + "name": "VAE", + "type": "*", + "link": 38, + "slot_index": 2, + "color_on": "#FF6E6E" + } + ], + "properties": { + "Node name for S&R": "Anything Everywhere3", + "group_restricted": false, + "color_restricted": false + } + }, + { + "id": 24, + "type": "PromptUtilitiesConstStringMultiLine", + "pos": [ + 26, + 458 + ], + "size": { + "0": 210, + "1": 93.83313751220703 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 31 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "PromptUtilitiesConstStringMultiLine" + }, + "widgets_values": [ + "can't show this \\(meme\\), holding sign, \\> \\<" + ] + }, + { + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 19, + 37 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 60 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 59 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 38 + ], + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "animagine-xl-3.0.safetensors" + ] + }, + { + "id": 30, + "type": "LoraLoader", + "pos": [ + 367, + 36 + ], + "size": { + "0": 325.6763000488281, + "1": 126.6906509399414 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 60, + "slot_index": 0 + }, + { + "name": "clip", + "type": "CLIP", + "link": 59 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 74 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 62 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoraLoader" + }, + "widgets_values": [ + "xl\\cant_show_this_xl.safetensors", + 1, + 1 + ] + }, + { + "id": 3, + "type": "CLIPTextEncode", + "pos": [ + 575, + 668 + ], + "size": { + "0": 210, + "1": 54 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": null, + "slot_index": 0 + }, + { + "name": "text", + "type": "STRING", + "link": 77, + "widget": { + "name": "text" + } + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 40 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, artist name" + ] + }, + { + "id": 33, + "type": "PromptUtilitiesConstStringMultiLine", + "pos": [ + 32, + 851 + ], + "size": { + "0": 261.890625, + "1": 126.9643325805664 + }, + "flags": {}, + "order": 7, + "mode": 0, + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 76 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "PromptUtilitiesConstStringMultiLine" + }, + "widgets_values": [ + "embedding:negativeXL_D" + ] + }, + { + "id": 34, + "type": "PromptUtilitiesJoinStringList", + "pos": [ + 334, + 756 + ], + "size": { + "0": 210, + "1": 98 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "arg1", + "type": "STRING", + "link": 75, + "widget": { + "name": "arg1" + }, + "slot_index": 0 + }, + { + "name": "arg2", + "type": "STRING", + "link": 76, + "slot_index": 1 + }, + { + "name": "arg3", + "type": "STRING", + "link": null + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 77 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PromptUtilitiesJoinStringList" + }, + "widgets_values": [ + ", ", + "" + ] + }, + { + "id": 4, + "type": "KSampler", + "pos": [ + 1080, + 170 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": null + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null, + "slot_index": 2 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 28 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 21 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 3, + "fixed", + 20, + 8, + "euler_ancestral", + "normal", + 1 + ] + } + ], + "links": [ + [ + 21, + 4, + 0, + 16, + 0, + "LATENT" + ], + [ + 28, + 21, + 0, + 4, + 3, + "LATENT" + ], + [ + 29, + 22, + 0, + 26, + 0, + "STRING" + ], + [ + 30, + 23, + 0, + 26, + 1, + "STRING" + ], + [ + 31, + 24, + 0, + 26, + 2, + "STRING" + ], + [ + 32, + 25, + 0, + 26, + 3, + "STRING" + ], + [ + 33, + 26, + 0, + 2, + 1, + "STRING" + ], + [ + 38, + 1, + 2, + 27, + 2, + "*" + ], + [ + 39, + 2, + 0, + 28, + 0, + "*" + ], + [ + 40, + 3, + 0, + 28, + 1, + "*" + ], + [ + 43, + 1, + 0, + 10, + 0, + "MODEL" + ], + [ + 44, + 2, + 0, + 10, + 1, + "CONDITIONING" + ], + [ + 45, + 3, + 0, + 10, + 2, + "CONDITIONING" + ], + [ + 46, + 1, + 2, + 8, + 1, + "VAE" + ], + [ + 47, + 1, + 2, + 16, + 1, + "VAE" + ], + [ + 48, + 1, + 0, + 4, + 0, + "MODEL" + ], + [ + 49, + 2, + 0, + 4, + 1, + "CONDITIONING" + ], + [ + 50, + 3, + 0, + 4, + 2, + "CONDITIONING" + ], + [ + 51, + 29, + 0, + 3, + 0, + "CLIP" + ], + [ + 52, + 29, + 0, + 2, + 0, + "CLIP" + ], + [ + 54, + 16, + 0, + 14, + 0, + "IMAGE" + ], + [ + 56, + 16, + 0, + 9, + 0, + "IMAGE" + ], + [ + 59, + 1, + 1, + 30, + 1, + "CLIP" + ], + [ + 60, + 1, + 0, + 30, + 0, + "MODEL" + ], + [ + 62, + 30, + 1, + 27, + 1, + "*" + ], + [ + 67, + 1, + 1, + 3, + 0, + "CLIP" + ], + [ + 68, + 1, + 1, + 2, + 0, + "CLIP" + ], + [ + 69, + 1, + 2, + 16, + 1, + "VAE" + ], + [ + 70, + 32, + 0, + 4, + 0, + "MODEL" + ], + [ + 71, + 2, + 0, + 4, + 1, + "CONDITIONING" + ], + [ + 72, + 3, + 0, + 4, + 2, + "CONDITIONING" + ], + [ + 74, + 30, + 0, + 27, + 0, + "*" + ], + [ + 75, + 20, + 0, + 34, + 0, + "STRING" + ], + [ + 76, + 33, + 0, + 34, + 1, + "STRING" + ], + [ + 77, + 34, + 0, + 3, + 1, + "STRING" + ], + [ + 78, + 30, + 1, + 2, + 0, + "CLIP" + ], + [ + 79, + 1, + 2, + 16, + 1, + "VAE" + ], + [ + 80, + 30, + 0, + 4, + 0, + "MODEL" + ], + [ + 81, + 2, + 0, + 4, + 1, + "CONDITIONING" + ], + [ + 82, + 3, + 0, + 4, + 2, + "CONDITIONING" + ], + [ + 83, + 30, + 1, + 3, + 0, + "CLIP" + ], + [ + 84, + 30, + 1, + 2, + 0, + "CLIP" + ], + [ + 85, + 1, + 2, + 16, + 1, + "VAE" + ], + [ + 86, + 30, + 1, + 3, + 0, + "CLIP" + ], + [ + 87, + 30, + 0, + 4, + 0, + "MODEL" + ], + [ + 88, + 2, + 0, + 4, + 1, + "CONDITIONING" + ], + [ + 89, + 3, + 0, + 4, + 2, + "CONDITIONING" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/examples/lora_embedding.png b/examples/lora_embedding.png new file mode 100644 index 0000000..61afe4b Binary files /dev/null and b/examples/lora_embedding.png differ diff --git a/py/capture.py b/py/capture.py index 7685450..858dffb 100644 --- a/py/capture.py +++ b/py/capture.py @@ -50,7 +50,11 @@ class Capture: v = value[0] if format is not None: v = format(v, input_data) - inputs[meta].append((node_id, v)) + if isinstance(v, list): + for x in v: + inputs[meta].append((node_id, x)) + else: + inputs[meta].append((node_id, v)) return inputs @classmethod @@ -91,6 +95,7 @@ class Capture: update_pnginfo_dict(MetaField.MODEL_HASH, "Model hash") pnginfo_dict.update(cls.gen_loras(inputs)) + pnginfo_dict.update(cls.gen_embeddings(inputs)) hashes_for_civitai = cls.get_hashes_for_civitai(inputs) if len(hashes_for_civitai) > 0: @@ -130,6 +135,12 @@ class Capture: lora_model_name = os.path.splitext(os.path.basename(lora_model_name[1]))[0] resource_hashes[f"lora:{lora_model_name}"] = lora_model_hash[1] + embedding_names = inputs.get(MetaField.EMBEDDING_NAME, []) + embedding_hashes = inputs.get(MetaField.EMBEDDING_HASH, []) + for embedding_name, embedding_hash in zip(embedding_names, embedding_hashes): + embedding_name = os.path.splitext(os.path.basename(embedding_name[1]))[0] + resource_hashes[f"embed:{embedding_name}"] = embedding_hash[1] + return resource_hashes @classmethod @@ -153,3 +164,19 @@ class Capture: index += 1 return pnginfo_dict + + @classmethod + def gen_embeddings(cls, inputs): + pnginfo_dict = {} + + embedding_names = inputs.get(MetaField.EMBEDDING_NAME, []) + embedding_hashes = inputs.get(MetaField.EMBEDDING_HASH, []) + + index = 0 + for embedding_name, embedding_hashe in zip(embedding_names, embedding_hashes): + field_prefix = f"Embedding_{index}" + pnginfo_dict[f"{field_prefix} name"] = os.path.basename(embedding_name[1]) + pnginfo_dict[f"{field_prefix} hash"] = embedding_hashe[1] + index += 1 + + return pnginfo_dict diff --git a/py/defs/captures.py b/py/defs/captures.py index a36197d..16baa01 100644 --- a/py/defs/captures.py +++ b/py/defs/captures.py @@ -6,6 +6,8 @@ from .formatters import ( convert_skip_clip, get_scaled_width, get_scaled_height, + extract_embedding_names, + extract_embedding_hashes, ) @@ -33,6 +35,14 @@ CAPTURE_FIELD_LIST = { "field_name": "text", "validate": is_negative_prompt, }, + MetaField.EMBEDDING_NAME: { + "field_name": "text", + "format": extract_embedding_names, + }, + MetaField.EMBEDDING_HASH: { + "field_name": "text", + "format": extract_embedding_hashes, + }, }, "KSampler": { MetaField.SEED: {"field_name": "seed"}, diff --git a/py/defs/formatters.py b/py/defs/formatters.py index 7f7997a..edc133d 100644 --- a/py/defs/formatters.py +++ b/py/defs/formatters.py @@ -1,6 +1,12 @@ +import os + import folder_paths from ..utils.hash import calc_hash +from ..utils.embedding import get_embedding_file_path + +from comfy.sd1_clip import escape_important, token_weights, unescape_important +from comfy.sdxl_clip import SDXLTokenizer cache_model_hash = {} @@ -27,3 +33,45 @@ def get_scaled_width(scaled_by, input_data): def get_scaled_height(scaled_by, input_data): samples = input_data["samples"][0]["samples"] return round(samples.shape[2] * scaled_by * 8) + + +def extract_embedding_names(text, input_data): + embedding_names, _ = _extract_embedding_names(text, input_data) + + return [os.path.basename(embedding_name) for embedding_name in embedding_names] + + +def extract_embedding_hashes(text, input_data): + embedding_names, tokenizer = _extract_embedding_names(text, input_data) + embedding_hashes = [] + for embedding_name in embedding_names: + embedding_file_path = get_embedding_file_path(embedding_name, tokenizer) + embedding_hashes.append(calc_hash(embedding_file_path)) + + return embedding_hashes + + +def _extract_embedding_names(text, input_data): + clip = input_data["clip"][0] + tokenizer = clip.tokenizer + if isinstance(tokenizer, SDXLTokenizer): + tokenizer = tokenizer.clip_l + text = escape_important(text) + parsed_weights = token_weights(text, 1.0) + + # tokenize words + embedding_names = [] + for weighted_segment, weight in parsed_weights: + to_tokenize = unescape_important(weighted_segment).replace("\n", " ").split(" ") + to_tokenize = [x for x in to_tokenize if x != ""] + for word in to_tokenize: + # if we find an embedding, deal with the embedding + if ( + word.startswith(tokenizer.embedding_identifier) + and tokenizer.embedding_directory is not None + ): + embedding_name = word[len(tokenizer.embedding_identifier) :].strip("\n") + + embedding_names.append(embedding_name) + + return embedding_names, tokenizer diff --git a/py/defs/meta.py b/py/defs/meta.py index b6dc108..773ee0c 100644 --- a/py/defs/meta.py +++ b/py/defs/meta.py @@ -15,6 +15,7 @@ class MetaField(IntEnum): IMAGE_WIDTH = 30 IMAGE_HEIGHT = 31 EMBEDDING_NAME = 40 + EMBEDDING_HASH = 41 LORA_MODEL_NAME = 50 LORA_MODEL_HASH = 51 LORA_STRENGTH_MODEL = 52 diff --git a/py/utils/embedding.py b/py/utils/embedding.py new file mode 100644 index 0000000..70c1f3a --- /dev/null +++ b/py/utils/embedding.py @@ -0,0 +1,35 @@ +import os + +from comfy.sd1_clip import expand_directory_list + + +# refer. https://github.com/comfyanonymous/ComfyUI/blob/dccca1daa5af1954d55918f365e83a3331019549/comfy/sd1_clip.py#L284-L309 +def get_embedding_file_path(embedding_name, tokenizer): + embedding_directory = tokenizer.embedding_directory + if isinstance(embedding_directory, str): + embedding_directory = [embedding_directory] + + embedding_directory = expand_directory_list(embedding_directory) + + valid_file = None + for embed_dir in embedding_directory: + embed_path = os.path.abspath(os.path.join(embed_dir, embedding_name)) + embed_dir = os.path.abspath(embed_dir) + try: + if os.path.commonpath((embed_dir, embed_path)) != embed_dir: + continue + except Exception: + continue + if not os.path.isfile(embed_path): + extensions = [".safetensors", ".pt", ".bin"] + for x in extensions: + t = embed_path + x + if os.path.isfile(t): + valid_file = t + break + else: + valid_file = embed_path + if valid_file is not None: + break + + return valid_file