feat(syntax): tokenize dp_brace + dp_multi with nesting support
This commit is contained in:
DumiFlex
@@ -17,6 +17,85 @@ from engine.syntax.types import Token, TokenKind
|
||||
_VAR_RE = re.compile(r"\$([A-Za-z_][A-Za-z0-9_]*)")
|
||||
# `@{8hex}` — exactly 8 lowercase hex chars in braces
|
||||
_REF_RE = re.compile(r"@\{([0-9a-f]{8})\}")
|
||||
# Multi-pick prefix: {N$$sep$$ where N is one or more digits and sep can be empty.
|
||||
_MULTI_PREFIX_RE = re.compile(r"^\{(\d+)\$\$(.*?)\$\$", flags=re.DOTALL)
|
||||
|
||||
|
||||
def _split_top_level_pipes(s: str) -> list[str]:
|
||||
"""Split `s` on `|` at brace-depth zero only. Nested `{a|b}` stays intact."""
|
||||
parts: list[str] = []
|
||||
depth = 0
|
||||
last = 0
|
||||
for idx, c in enumerate(s):
|
||||
if c == "{":
|
||||
depth += 1
|
||||
elif c == "}":
|
||||
depth -= 1
|
||||
elif c == "|" and depth == 0:
|
||||
parts.append(s[last:idx])
|
||||
last = idx + 1
|
||||
parts.append(s[last:])
|
||||
return parts
|
||||
|
||||
|
||||
def _scan_brace_block(
|
||||
text: str, start: int
|
||||
) -> tuple[int, list[str], int | None, str | None] | None:
|
||||
"""Try to scan a `{...}` block starting at `text[start] == '{'`.
|
||||
|
||||
Returns:
|
||||
(end_index, branches, count_or_None, sep_or_None)
|
||||
|
||||
where end_index is the position AFTER the closing brace, branches is
|
||||
the list of branch source strings (split at top-level `|`), count is
|
||||
the parsed integer count for multi-pick or None for single-pick, and
|
||||
sep is the separator string for multi-pick or None for single-pick.
|
||||
|
||||
Returns None if the block is malformed (unclosed, no pipes for single-pick).
|
||||
The caller falls back to literal text.
|
||||
"""
|
||||
n = len(text)
|
||||
if start >= n or text[start] != "{":
|
||||
return None
|
||||
|
||||
# Walk to the matching `}` while tracking brace depth so nested
|
||||
# {a|{b|c}|d} works.
|
||||
i = start + 1
|
||||
depth = 1
|
||||
while i < n and depth > 0:
|
||||
c = text[i]
|
||||
if c == "{":
|
||||
depth += 1
|
||||
elif c == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
break
|
||||
i += 1
|
||||
if depth != 0:
|
||||
return None # unclosed
|
||||
|
||||
body_start = start + 1
|
||||
body_end = i # index of closing brace
|
||||
end_index = i + 1
|
||||
body = text[body_start:body_end]
|
||||
|
||||
# Detect multi-pick prefix: N$$sep$$
|
||||
body_with_braces = "{" + body # match against `{N$$sep$$...`
|
||||
multi_match = _MULTI_PREFIX_RE.match(body_with_braces)
|
||||
if multi_match:
|
||||
count = int(multi_match.group(1))
|
||||
sep = multi_match.group(2)
|
||||
# Body remaining after the prefix is the branch list
|
||||
prefix_len = multi_match.end() - 1 # subtract the leading `{` we added
|
||||
rest = body[prefix_len:]
|
||||
branches = _split_top_level_pipes(rest)
|
||||
return (end_index, branches, count, sep)
|
||||
|
||||
# Single-pick: must contain at least one top-level `|`
|
||||
branches = _split_top_level_pipes(body)
|
||||
if len(branches) < 2:
|
||||
return None # not a pick — fall back to literal
|
||||
return (end_index, branches, None, None)
|
||||
|
||||
|
||||
def tokenize_text(text: str) -> list[Token]:
|
||||
@@ -102,6 +181,32 @@ def tokenize_text(text: str) -> list[Token]:
|
||||
i = m.end()
|
||||
continue
|
||||
|
||||
# Brace block: {...} for inline pick or {N$$sep$$...} for multi-pick
|
||||
if ch == "{":
|
||||
scanned = _scan_brace_block(text, i)
|
||||
if scanned is not None:
|
||||
end_index, branches, count, sep = scanned
|
||||
_flush_text(i)
|
||||
if count is None:
|
||||
out.append(Token(
|
||||
kind=TokenKind.DP_BRACE,
|
||||
raw=text[i:end_index],
|
||||
start=i,
|
||||
end=end_index,
|
||||
meta={"branches": branches},
|
||||
))
|
||||
else:
|
||||
out.append(Token(
|
||||
kind=TokenKind.DP_MULTI,
|
||||
raw=text[i:end_index],
|
||||
start=i,
|
||||
end=end_index,
|
||||
meta={"count": count, "sep": sep, "branches": branches},
|
||||
))
|
||||
i = end_index
|
||||
continue
|
||||
# Malformed brace — fall through to literal text
|
||||
|
||||
# Default: accumulate literal text
|
||||
if text_start is None:
|
||||
text_start = i
|
||||
|
||||
Vendored
+64
@@ -105,6 +105,70 @@
|
||||
"expected_tokens": [
|
||||
{"kind": "text", "raw": "@{abc} short", "start": 0, "end": 12, "meta": {}}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "inline_pick_simple",
|
||||
"input": "{red|blue|green}",
|
||||
"expected_tokens": [
|
||||
{"kind": "dp_brace", "raw": "{red|blue|green}", "start": 0, "end": 16,
|
||||
"meta": {"branches": ["red", "blue", "green"]}}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "inline_pick_in_text",
|
||||
"input": "a {red|blue} cat",
|
||||
"expected_tokens": [
|
||||
{"kind": "text", "raw": "a ", "start": 0, "end": 2, "meta": {}},
|
||||
{"kind": "dp_brace", "raw": "{red|blue}", "start": 2, "end": 12,
|
||||
"meta": {"branches": ["red", "blue"]}},
|
||||
{"kind": "text", "raw": " cat", "start": 12, "end": 16, "meta": {}}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "inline_pick_with_empty_branch",
|
||||
"input": "{a||c}",
|
||||
"expected_tokens": [
|
||||
{"kind": "dp_brace", "raw": "{a||c}", "start": 0, "end": 6,
|
||||
"meta": {"branches": ["a", "", "c"]}}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "inline_pick_nested",
|
||||
"input": "{a|{b|c}|d}",
|
||||
"expected_tokens": [
|
||||
{"kind": "dp_brace", "raw": "{a|{b|c}|d}", "start": 0, "end": 11,
|
||||
"meta": {"branches": ["a", "{b|c}", "d"]}}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "multi_pick_simple",
|
||||
"input": "{2$$, $$red|blue|green}",
|
||||
"expected_tokens": [
|
||||
{"kind": "dp_multi", "raw": "{2$$, $$red|blue|green}", "start": 0, "end": 23,
|
||||
"meta": {"count": 2, "sep": ", ", "branches": ["red", "blue", "green"]}}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "multi_pick_empty_sep",
|
||||
"input": "{3$$$$a|b|c}",
|
||||
"expected_tokens": [
|
||||
{"kind": "dp_multi", "raw": "{3$$$$a|b|c}", "start": 0, "end": 12,
|
||||
"meta": {"count": 3, "sep": "", "branches": ["a", "b", "c"]}}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "unclosed_brace_is_literal_text",
|
||||
"input": "{a|b incomplete",
|
||||
"expected_tokens": [
|
||||
{"kind": "text", "raw": "{a|b incomplete", "start": 0, "end": 15, "meta": {}}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_branch_no_pipe_is_literal",
|
||||
"input": "{just_one}",
|
||||
"expected_tokens": [
|
||||
{"kind": "text", "raw": "{just_one}", "start": 0, "end": 10, "meta": {}}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -65,3 +65,15 @@ def test_tokenize_lossless_for_var_ref():
|
||||
toks = tokenize_text(s)
|
||||
joined = "".join(t.raw for t in toks)
|
||||
assert joined == s, f"lossless invariant broken for {s!r}: got {joined!r}"
|
||||
|
||||
|
||||
def test_tokenize_nested_braces_lossless():
|
||||
s = "{a|{b|c}|d} and {1$$, $$x|y}"
|
||||
toks = tokenize_text(s)
|
||||
assert "".join(t.raw for t in toks) == s
|
||||
|
||||
|
||||
def test_tokenize_unicode_preserved():
|
||||
s = "🎨 $color {red|blue|绿}"
|
||||
toks = tokenize_text(s)
|
||||
assert "".join(t.raw for t in toks) == s
|
||||
|
||||
Reference in New Issue
Block a user