feat(syntax): tokenize_text — text + escape kinds with lossless invariant

engine/syntax/__init__.py

@@ -3,6 +3,7 @@
 See `docs/superpowers/specs/2026-04-28-engine-syntax-design.md` for the
 language contract and parity rules.
 """
+from engine.syntax.tokenize import tokenize_text
 from engine.syntax.types import (
     CycleDetectedError,
     MalformedSyntaxError,
@@ -27,4 +28,5 @@ __all__ = [
     "Token",
     "TokenKind",
     "UnknownRefError",
+    "tokenize_text",
 ]

engine/syntax/tokenize.py

@@ -0,0 +1,73 @@
+"""Single-grammar tokenizer for wildcard syntax. Mirrors `richTokenize.ts`.
+
+Returns a flat list of `Token` covering the input contiguously (joining all
+`.raw` fields reproduces the input). The "lossless" invariant is asserted
+in tests + fuzz.
+
+This file ships in increments — Task 3 lands text + escape, Task 4 adds
+var + ref, Task 5 adds dp_brace + dp_pipe + dp_multi.
+"""
+from __future__ import annotations
+
+from engine.syntax.types import Token, TokenKind
+
+
+def tokenize_text(text: str) -> list[Token]:
+    """Tokenize `text` into a flat list of Token. Single-pass, O(n)."""
+    if not text:
+        return []
+
+    out: list[Token] = []
+    i = 0
+    n = len(text)
+    text_start: int | None = None
+
+    def _flush_text(end_at: int) -> None:
+        nonlocal text_start
+        if text_start is None:
+            return
+        if end_at > text_start:
+            out.append(Token(
+                kind=TokenKind.TEXT,
+                raw=text[text_start:end_at],
+                start=text_start,
+                end=end_at,
+                meta={},
+            ))
+        text_start = None
+
+    while i < n:
+        ch = text[i]
+
+        # Escape sequences: $$ → literal $, @@ → literal @
+        if ch == "$" and i + 1 < n and text[i + 1] == "$":
+            _flush_text(i)
+            out.append(Token(
+                kind=TokenKind.ESCAPE,
+                raw="$$",
+                start=i,
+                end=i + 2,
+                meta={"literal": "$"},
+            ))
+            i += 2
+            continue
+
+        if ch == "@" and i + 1 < n and text[i + 1] == "@":
+            _flush_text(i)
+            out.append(Token(
+                kind=TokenKind.ESCAPE,
+                raw="@@",
+                start=i,
+                end=i + 2,
+                meta={"literal": "@"},
+            ))
+            i += 2
+            continue
+
+        # Default: accumulate literal text
+        if text_start is None:
+            text_start = i
+        i += 1
+
+    _flush_text(n)
+    return out

tests/syntax/test_tokenize.py

@@ -0,0 +1,53 @@
+"""Python tokenizer reads parity corpus and asserts identical token output."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from engine.syntax import Token, tokenize_text
+
+CORPUS_PATH = Path(__file__).resolve().parent.parent / "fixtures" / "syntax-corpus.json"
+
+
+def _token_to_dict(t: Token) -> dict:
+    return {
+        "kind": t.kind.value,
+        "raw": t.raw,
+        "start": t.start,
+        "end": t.end,
+        "meta": t.meta,
+    }
+
+
+def _load_cases() -> list[dict]:
+    return json.loads(CORPUS_PATH.read_text(encoding="utf-8"))["cases"]
+
+
+@pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["name"])
+def test_tokenize_matches_corpus(case):
+    actual = [_token_to_dict(t) for t in tokenize_text(case["input"])]
+    assert actual == case["expected_tokens"], (
+        f"tokenize_text({case['input']!r}) diverged from corpus expectation"
+    )
+
+
+def test_tokenize_empty_returns_empty_list():
+    assert tokenize_text("") == []
+
+
+def test_tokenize_lossless_concatenation_invariant():
+    """The tokenizer must be lossless — joining all token .raw fields
+    reproduces the input exactly, for any input."""
+    samples = [
+        "hello",
+        "$$",
+        "@@",
+        "a$$b@@c",
+        "spaces  and\ttabs\n",
+    ]
+    for s in samples:
+        toks = tokenize_text(s)
+        joined = "".join(t.raw for t in toks)
+        assert joined == s, f"lossless invariant broken for {s!r}: got {joined!r}"