Node to obtain json of the tokenizer vocabulary.

This commit is contained in:
Ivor Wanders
2024-06-21 20:10:22 -04:00
parent c66027ac7e
commit f25752b59d
3 changed files with 45 additions and 2 deletions
+5 -2
View File
@@ -1,11 +1,14 @@
from .save_string import SaveString
from .tokenizer_vocab import TokenizerVocab
NODE_CLASS_MAPPINGS = {
"SaveString": SaveString
"SaveString": SaveString,
"TokenizerVocab": TokenizerVocab,
}
# A dictionary that contains the friendly/humanly readable titles for the nodes
NODE_DISPLAY_NAME_MAPPINGS = {
"SaveString": "SaveString"
"SaveString": "IW SaveString",
"TokenizerVocab": "IW TokenizerVocab"
}
+5
View File
@@ -1,5 +1,6 @@
import folder_paths
import os
import json
class SaveString:
def __init__(self):
@@ -30,6 +31,10 @@ class SaveString:
full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir)
file = f"{filename}_{counter:05}_{filename_suffix}.{extension}"
path = os.path.join(full_output_folder, file)
if extension == "json":
string_field = json.dumps(json.loads(string_field), indent=4)
with open(path, "w") as f:
f.write(string_field)
return tuple()
+35
View File
@@ -0,0 +1,35 @@
import folder_paths
from comfy import sdxl_clip
import os
import json
class TokenizerVocab:
def __init__(self):
pass
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"clip": ("CLIP",),
},
"optional": {
},
}
OUTPUT_NODE = False
RETURN_TYPES = ("STRING", )
FUNCTION = "execute"
CATEGORY = "iw"
def execute(self, clip):
tokenizer = clip.tokenizer
vocab = {}
# Is clip l and g different?
if isinstance(tokenizer, sdxl_clip.SDXLTokenizer):
clip_l_tokenizer = tokenizer.clip_l.tokenizer
vocab = clip_l_tokenizer.get_vocab()
return (json.dumps(vocab),)