nodes/alpertunga-bile-prompt-generator-comfyui
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

preprocess is refactored

Browse Source
This commit is contained in:
alpertunga-bile
2024-08-28 09:49:46 +03:00
parent 96601bb905
commit e1a6fbc201
1 changed files with 48 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
generator/preprocess.py
+48 -53
View File
@@ -1,28 +1,5 @@
from re import compile
from collections import OrderedDict
# compiled regex strings
remove_nonprompts_regex = compile(r"[^a-zA-Z()\[\]{}]*")
remove_nonweighters_regex = compile(r"[()\[\]{}]*")
remove_inside_regex = compile(r"[^()\[\]{}]*")
find_empty_parantheses_regex = compile(r"\(\s*\)")
remove_multiwhitespaces_regex = compile(r"\s+")
remove_nonpromptcommas_regex = compile(r"(,\s){2,}")
remove_scalarweights_regex = compile(r",\s*:[0-9]*\.?[0-9]+")
remove_emptyprompts_regex = compile(r",\s+[()\[\]{}]+\s*,")
remove_danglingparantheses_regex = compile(r"\B\s+|\s+\B")
replace_dict = {
compile(r"\(\s*,"): "(",
compile(r"\[\s*,"): "[",
compile(r"{\s*,"): "{",
compile(r",\s*\)"): ")",
compile(r",\s*\]"): "]",
compile(r",\s*}"): "}",
}
from collections import OrderedDict, namedtuple
def get_unique_list(sequence: list[str]) -> list[str]:
@@ -33,11 +10,16 @@ def get_unique_list(sequence: list[str]) -> list[str]:
def remove_exact_keywords(line: str) -> list[str]:
find_empty_parantheses_regex = compile(r"\(\s*\)")
remove_nonprompts_regex = compile(r"[^a-zA-Z_\-()\[\]{}]*")
remove_nonweighters_regex = compile(r"[()\[\]{}]*")
remove_inside_regex = compile(r"[^()\[\]{}]*")
# remove exact prompts
prompts = get_unique_list(line.split(","))
pure_prompts = OrderedDict() # order matters, it contains prompts' original forms
extracted_pure_prompts = {} # order isn't important, it contains prompt keywords
extracted_pure_prompts = set() # order doesn't matters, it contains prompt keywords
# remove exact keyword
for prompt in prompts:
@@ -45,7 +27,7 @@ def remove_exact_keywords(line: str) -> list[str]:
"", prompt
).lstrip() # from -> ((masterpiece:1.2)) | to -> ((masterpiece))
if tempPrompt == "":
if len(tempPrompt) == 0:
continue
tempPrompt = remove_nonweighters_regex.sub(
@@ -59,8 +41,7 @@ def remove_exact_keywords(line: str) -> list[str]:
if (
len(find_empty_parantheses_regex.findall(tempPrompt))
> 0 # find () count
or tempPrompt == ""
> 0 # find balanced parantheses count
):
# check balanced parantheses
inner_parant_count = tempPrompt.count("(")
@@ -72,7 +53,7 @@ def remove_exact_keywords(line: str) -> list[str]:
lowest_count = min(inner_parant_count, outer_parant_count)
# remove balanced parantheses
# because it is going to be appended to previous string
# because it is going to be appended to a string
for _ in range(lowest_count):
tempPrompt = tempPrompt.replace("()", "")
@@ -80,30 +61,53 @@ def remove_exact_keywords(line: str) -> list[str]:
continue
if tempPrompt == "":
tempPrompt = remove_nonprompts_regex.sub("", prompt).lstrip()
pure_prompts[tempPrompt] = True
else:
extracted_pure_prompts[tempPrompt] = True
pure_prompts[prompt] = True
extracted_pure_prompts.add(tempPrompt)
pure_prompts[prompt] = True
return pure_prompts.keys()
def fix_commas(string: str) -> str:
remove_multiwhitespaces_regex = compile(r"\s+")
remove_nonpromptcommas_regex = compile(r"(,\s){2,}")
temp_str = remove_multiwhitespaces_regex.sub(" ", string)
return remove_nonpromptcommas_regex.sub(", ", temp_str)
def fix_artifacts(string: str) -> str:
temp_string = string
ArtifactFix = namedtuple("ArtifactFix", ["regex", "new_str"])
replace_list = [
ArtifactFix(regex=compile(r"\(\s*,"), new_str="("),
ArtifactFix(regex=compile(r"\[\s*,"), new_str="["),
ArtifactFix(regex=compile(r"{\s*,"), new_str="{"),
ArtifactFix(regex=compile(r",\s*\)"), new_str=")"),
ArtifactFix(regex=compile(r",\s*\]"), new_str="]"),
ArtifactFix(regex=compile(r",\s*}"), new_str="}"),
]
# fixing the artifacts
for artifact_fix in replace_list:
temp_string = artifact_fix.regex.sub(artifact_fix.new_str, temp_string)
return temp_string
def preprocess(line: str, preprocess_mode: str) -> str:
remove_scalarweights_regex = compile(r",\s*:[0-9]*\.?[0-9]+")
remove_emptyprompts_regex = compile(r",\s+[()\[\]{}]+\s*,")
remove_danglingparantheses_regex = compile(r"\B\s+|\s+\B")
temp_line = line.encode("ascii", "xmlcharrefreplace").decode()
temp_line = temp_line.encode("utf-8", "xmlcharrefreplace").decode()
temp_line = temp_line.replace("\xa0", " ")
temp_line = temp_line.replace("\n", ", ")
temp_line = remove_multiwhitespaces_regex.sub(
" ", temp_line
) # from -> , , , ( prompt) | to -> , , , (prompt)
temp_line = remove_nonpromptcommas_regex.sub(
", ", temp_line
) # from -> , , , , | to -> ,
temp_line = fix_commas(temp_line)
temp_line = remove_scalarweights_regex.sub(
"", temp_line
@@ -112,17 +116,8 @@ def preprocess(line: str, preprocess_mode: str) -> str:
if preprocess_mode == "exact_keyword":
temp_line = ", ".join(remove_exact_keywords(temp_line))
temp_line = remove_multiwhitespaces_regex.sub(
" ", temp_line
) # from -> , , , ( prompt) | to -> , , , (prompt)
temp_line = remove_nonpromptcommas_regex.sub(
", ", temp_line
) # from -> , , , , | to -> ,
# fixing the artifacts
for regex_pattern, new_str in replace_dict.items():
temp_line = regex_pattern.sub(new_str, temp_line)
temp_line = fix_commas(temp_line)
temp_line = fix_artifacts(temp_line)
temp_line = remove_emptyprompts_regex.sub(
",", temp_line
@@ -133,6 +128,6 @@ def preprocess(line: str, preprocess_mode: str) -> str:
) # from -> (( ((prompt)) | to -> ((prompt))
elif preprocess_mode == "exact_prompt":
temp_line = ",".join(get_unique_list(temp_line.split(",")))
temp_line = ", ".join(get_unique_list(temp_line.split(",")))
return temp_line