preprocess is refactored

generator/preprocess.py

@@ -1,28 +1,5 @@
 from re import compile
-from collections import OrderedDict
+from collections import OrderedDict, namedtuple
-
-# compiled regex strings
-
-remove_nonprompts_regex = compile(r"[^a-zA-Z()\[\]{}]*")
-remove_nonweighters_regex = compile(r"[()\[\]{}]*")
-remove_inside_regex = compile(r"[^()\[\]{}]*")
-
-find_empty_parantheses_regex = compile(r"\(\s*\)")
-
-remove_multiwhitespaces_regex = compile(r"\s+")
-remove_nonpromptcommas_regex = compile(r"(,\s){2,}")
-remove_scalarweights_regex = compile(r",\s*:[0-9]*\.?[0-9]+")
-remove_emptyprompts_regex = compile(r",\s+[()\[\]{}]+\s*,")
-remove_danglingparantheses_regex = compile(r"\B\s+|\s+\B")
-
-replace_dict = {
-    compile(r"\(\s*,"): "(",
-    compile(r"\[\s*,"): "[",
-    compile(r"{\s*,"): "{",
-    compile(r",\s*\)"): ")",
-    compile(r",\s*\]"): "]",
-    compile(r",\s*}"): "}",
-}
 
 
 def get_unique_list(sequence: list[str]) -> list[str]:
@@ -33,11 +10,16 @@ def get_unique_list(sequence: list[str]) -> list[str]:
 
 
 def remove_exact_keywords(line: str) -> list[str]:
+    find_empty_parantheses_regex = compile(r"\(\s*\)")
+    remove_nonprompts_regex = compile(r"[^a-zA-Z_\-()\[\]{}]*")
+    remove_nonweighters_regex = compile(r"[()\[\]{}]*")
+    remove_inside_regex = compile(r"[^()\[\]{}]*")
+
     # remove exact prompts
     prompts = get_unique_list(line.split(","))
 
     pure_prompts = OrderedDict()  # order matters, it contains prompts' original forms
-    extracted_pure_prompts = {}  # order isn't important, it contains prompt keywords
+    extracted_pure_prompts = set()  # order doesn't matters, it contains prompt keywords
 
     # remove exact keyword
     for prompt in prompts:
@@ -45,7 +27,7 @@ def remove_exact_keywords(line: str) -> list[str]:
             "", prompt
         ).lstrip()  # from -> ((masterpiece:1.2)) | to -> ((masterpiece))
 
-        if tempPrompt == "":
+        if len(tempPrompt) == 0:
             continue
 
         tempPrompt = remove_nonweighters_regex.sub(
@@ -59,8 +41,7 @@ def remove_exact_keywords(line: str) -> list[str]:
 
             if (
                 len(find_empty_parantheses_regex.findall(tempPrompt))
-                > 0  # find () count
+                > 0  # find balanced parantheses count
-                or tempPrompt == ""
             ):
                 # check balanced parantheses
                 inner_parant_count = tempPrompt.count("(")
@@ -72,7 +53,7 @@ def remove_exact_keywords(line: str) -> list[str]:
                 lowest_count = min(inner_parant_count, outer_parant_count)
 
                 # remove balanced parantheses
-                # because it is going to be appended to previous string
+                # because it is going to be appended to a string
                 for _ in range(lowest_count):
                     tempPrompt = tempPrompt.replace("()", "")
 
@@ -80,30 +61,53 @@ def remove_exact_keywords(line: str) -> list[str]:
 
             continue
 
-        if tempPrompt == "":
+        extracted_pure_prompts.add(tempPrompt)
-            tempPrompt = remove_nonprompts_regex.sub("", prompt).lstrip()
+        pure_prompts[prompt] = True
-            pure_prompts[tempPrompt] = True
-        else:
-            extracted_pure_prompts[tempPrompt] = True
-            pure_prompts[prompt] = True
 
     return pure_prompts.keys()
 
 
+def fix_commas(string: str) -> str:
+    remove_multiwhitespaces_regex = compile(r"\s+")
+    remove_nonpromptcommas_regex = compile(r"(,\s){2,}")
+
+    temp_str = remove_multiwhitespaces_regex.sub(" ", string)
+    return remove_nonpromptcommas_regex.sub(", ", temp_str)
+
+
+def fix_artifacts(string: str) -> str:
+    temp_string = string
+
+    ArtifactFix = namedtuple("ArtifactFix", ["regex", "new_str"])
+
+    replace_list = [
+        ArtifactFix(regex=compile(r"\(\s*,"), new_str="("),
+        ArtifactFix(regex=compile(r"\[\s*,"), new_str="["),
+        ArtifactFix(regex=compile(r"{\s*,"), new_str="{"),
+        ArtifactFix(regex=compile(r",\s*\)"), new_str=")"),
+        ArtifactFix(regex=compile(r",\s*\]"), new_str="]"),
+        ArtifactFix(regex=compile(r",\s*}"), new_str="}"),
+    ]
+
+    # fixing the artifacts
+    for artifact_fix in replace_list:
+        temp_string = artifact_fix.regex.sub(artifact_fix.new_str, temp_string)
+
+    return temp_string
+
+
 def preprocess(line: str, preprocess_mode: str) -> str:
+    remove_scalarweights_regex = compile(r",\s*:[0-9]*\.?[0-9]+")
+    remove_emptyprompts_regex = compile(r",\s+[()\[\]{}]+\s*,")
+    remove_danglingparantheses_regex = compile(r"\B\s+|\s+\B")
 
     temp_line = line.encode("ascii", "xmlcharrefreplace").decode()
     temp_line = temp_line.encode("utf-8", "xmlcharrefreplace").decode()
 
     temp_line = temp_line.replace("\xa0", " ")
     temp_line = temp_line.replace("\n", ", ")
-    temp_line = remove_multiwhitespaces_regex.sub(
-        " ", temp_line
-    )  # from -> ,       ,  , (     prompt) | to -> , , , (prompt)
 
-    temp_line = remove_nonpromptcommas_regex.sub(
+    temp_line = fix_commas(temp_line)
-        ", ", temp_line
-    )  # from -> , , , , | to -> ,
 
     temp_line = remove_scalarweights_regex.sub(
         "", temp_line
@@ -112,17 +116,8 @@ def preprocess(line: str, preprocess_mode: str) -> str:
     if preprocess_mode == "exact_keyword":
         temp_line = ", ".join(remove_exact_keywords(temp_line))
 
-        temp_line = remove_multiwhitespaces_regex.sub(
+        temp_line = fix_commas(temp_line)
-            " ", temp_line
+        temp_line = fix_artifacts(temp_line)
-        )  # from -> ,       ,  , (       prompt) | to -> , , , (prompt)
-
-        temp_line = remove_nonpromptcommas_regex.sub(
-            ", ", temp_line
-        )  # from -> , , , , | to -> ,
-
-        # fixing the artifacts
-        for regex_pattern, new_str in replace_dict.items():
-            temp_line = regex_pattern.sub(new_str, temp_line)
 
         temp_line = remove_emptyprompts_regex.sub(
             ",", temp_line
@@ -133,6 +128,6 @@ def preprocess(line: str, preprocess_mode: str) -> str:
         )  # from -> (( ((prompt)) | to -> ((prompt))
 
     elif preprocess_mode == "exact_prompt":
-        temp_line = ",".join(get_unique_list(temp_line.split(",")))
+        temp_line = ", ".join(get_unique_list(temp_line.split(",")))
 
     return temp_line