fix(tokenizer): Fix issue when prompt is just an action, and has no tree

This commit is contained in:
Michael Poutre
2023-08-25 18:59:28 -07:00
parent c1cbdbe9df
commit bdb4d00910
+7 -1
View File
@@ -1,3 +1,5 @@
from lark import Tree
from comfy.sd1_clip import SD1Tokenizer
from custom_nodes.ClipStuff.lib.actions.types import SegOrAction
@@ -28,7 +30,11 @@ class PromptLangTokenizer(SD1Tokenizer):
batch = [PromptSegment(text="[SOT]", tokens=[self.start_token])]
# batched_segments.append(batch)
batch_size = 1
for segment in parsed_actions.children:
if isinstance(parsed_actions, Tree):
segments_to_process = parsed_actions.children
else:
segments_to_process = [parsed_actions]
for segment in segments_to_process:
num_tokens = segment.token_length()
# determine if we're going to try and keep the tokens in a single batch
is_large = num_tokens >= self.max_word_length