feat(syntax): resolveTokens TS twin of Python resolve_text
This commit is contained in:
DumiFlex
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Deterministic seeded RNG for tests. Mulberry32 is fast, small, and
|
||||
* suitable for test reproducibility (NOT cryptographically secure).
|
||||
*
|
||||
* Returns an object with .random() and .randrange(n) matching the API
|
||||
* the resolver uses, so the same code shape works in both Python (using
|
||||
* stdlib random.Random) and TS (using this helper) tests.
|
||||
*/
|
||||
export interface SeededRng {
|
||||
random(): number;
|
||||
randrange(n: number): number;
|
||||
}
|
||||
|
||||
export function mulberry32(seed: number): SeededRng {
|
||||
let s = seed | 0;
|
||||
return {
|
||||
random(): number {
|
||||
s = (s + 0x6d2b79f5) | 0;
|
||||
let t = s;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
},
|
||||
randrange(n: number): number {
|
||||
return Math.floor(this.random() * n);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
resolveTokens,
|
||||
type ResolveContext,
|
||||
type ResolveModule,
|
||||
UnknownRefError,
|
||||
RefOutOfSurfaceError,
|
||||
} from "../../utils/resolveTokens";
|
||||
import { mulberry32 } from "./_rng-helper";
|
||||
|
||||
function makeCtx(opts: Partial<ResolveContext> = {}): ResolveContext {
|
||||
return {
|
||||
rng: mulberry32(opts.rngSeed ?? 42),
|
||||
maxRefDepth: 8,
|
||||
strict: false,
|
||||
surface: "wildcard",
|
||||
developerMode: false,
|
||||
warnings: [],
|
||||
vars: {},
|
||||
modules: {},
|
||||
...opts,
|
||||
} as ResolveContext;
|
||||
}
|
||||
|
||||
describe("resolveTokens — text/escape/var", () => {
|
||||
it("empty input returns empty string", () => {
|
||||
expect(resolveTokens("", makeCtx())).toBe("");
|
||||
});
|
||||
|
||||
it("plain text passthrough", () => {
|
||||
expect(resolveTokens("hello world", makeCtx())).toBe("hello world");
|
||||
});
|
||||
|
||||
it("escape sequences resolve to literals", () => {
|
||||
expect(resolveTokens("price: $$5", makeCtx())).toBe("price: $5");
|
||||
expect(resolveTokens("user@@host", makeCtx())).toBe("user@host");
|
||||
});
|
||||
|
||||
it("var resolves from ctx.vars", () => {
|
||||
const ctx = makeCtx({ vars: { color: "red" } });
|
||||
expect(resolveTokens("$color thing", ctx)).toBe("red thing");
|
||||
});
|
||||
|
||||
it("missing var emits empty string", () => {
|
||||
expect(resolveTokens("$missing thing", makeCtx())).toBe(" thing");
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveTokens — ref", () => {
|
||||
const wcModule = (_uuid: string, varBinding: string, options: Array<{ value: string; weight: number }> = [{ value: "default", weight: 1 }]): ResolveModule => ({
|
||||
type: "wildcard",
|
||||
var_binding: varBinding,
|
||||
options,
|
||||
});
|
||||
|
||||
it("ref in wildcard surface picks an option", () => {
|
||||
const ctx = makeCtx({
|
||||
modules: { a4f7b2e1: wcModule("a4f7b2e1", "color", [{ value: "red", weight: 1 }]) },
|
||||
});
|
||||
expect(resolveTokens("@{a4f7b2e1}", ctx)).toBe("red");
|
||||
});
|
||||
|
||||
it("unknown ref strict raises", () => {
|
||||
expect(() => resolveTokens("@{00000000}", makeCtx({ strict: true }))).toThrow(UnknownRefError);
|
||||
});
|
||||
|
||||
it("unknown ref lenient emits empty + warning", () => {
|
||||
const ctx = makeCtx({ strict: false });
|
||||
const out = resolveTokens("a @{00000000} b", ctx);
|
||||
expect(out).toBe("a b");
|
||||
expect(ctx.warnings.find((w) => w.type === "unknown_ref")).toBeDefined();
|
||||
});
|
||||
|
||||
it("ref out of surface lenient emits empty", () => {
|
||||
const ctx = makeCtx({
|
||||
surface: "combine",
|
||||
modules: { a4f7b2e1: wcModule("a4f7b2e1", "color") },
|
||||
});
|
||||
expect(resolveTokens("a @{a4f7b2e1} b", ctx)).toBe("a b");
|
||||
expect(ctx.warnings.find((w) => w.type === "ref_out_of_surface")).toBeDefined();
|
||||
});
|
||||
|
||||
it("ref out of surface strict raises", () => {
|
||||
const ctx = makeCtx({
|
||||
surface: "combine",
|
||||
strict: true,
|
||||
modules: { a4f7b2e1: wcModule("a4f7b2e1", "color") },
|
||||
});
|
||||
expect(() => resolveTokens("@{a4f7b2e1}", ctx)).toThrow(RefOutOfSurfaceError);
|
||||
});
|
||||
|
||||
it("cycle detected lenient emits empty + warning", () => {
|
||||
const ctx = makeCtx({
|
||||
modules: {
|
||||
a0000001: wcModule("a0000001", "a", [{ value: "@{b0000002}", weight: 1 }]),
|
||||
b0000002: wcModule("b0000002", "b", [{ value: "@{a0000001}", weight: 1 }]),
|
||||
},
|
||||
});
|
||||
expect(resolveTokens("@{a0000001}", ctx)).toBe("");
|
||||
expect(ctx.warnings.find((w) => w.type === "cycle_detected")).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveTokens — inline + multi-pick", () => {
|
||||
it("inline pick returns one of branches", () => {
|
||||
const out = resolveTokens("{a|b|c}", makeCtx());
|
||||
expect(["a", "b", "c"]).toContain(out);
|
||||
});
|
||||
|
||||
it("multi-pick zero count empty", () => {
|
||||
expect(resolveTokens("{0$$, $$a|b|c}", makeCtx())).toBe("");
|
||||
});
|
||||
|
||||
it("multi-pick returns joined picks", () => {
|
||||
const out = resolveTokens("{2$$, $$a|b|c|d}", makeCtx());
|
||||
const parts = out.split(", ");
|
||||
expect(parts.length).toBe(2);
|
||||
parts.forEach((p) => expect(["a", "b", "c", "d"]).toContain(p));
|
||||
});
|
||||
|
||||
it("multi-pick clamps when N > branches", () => {
|
||||
const out = resolveTokens("{5$$, $$a|b|c}", makeCtx());
|
||||
expect(out.split(", ").length).toBe(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,269 @@
|
||||
/**
|
||||
* TS twin of engine/syntax/resolve.py. Walks the token list emitted by
|
||||
* tokenizeRich, dispatching by kind. Used by Test Runner; same semantics
|
||||
* as the Python engine.
|
||||
*/
|
||||
import { tokenizeRich, type RichToken } from "./richTokenize";
|
||||
|
||||
export type SurfaceKind = "wildcard" | "combine" | "derivation" | "assembler";
|
||||
|
||||
export interface ResolveModule {
|
||||
type: string;
|
||||
var_binding: string;
|
||||
options: Array<{ value: string; weight: number }>;
|
||||
}
|
||||
|
||||
export interface ResolveWarning {
|
||||
type: string;
|
||||
severity: "info" | "warn" | "error";
|
||||
module_id: string;
|
||||
source_field: string;
|
||||
position: number;
|
||||
token_index: number | null;
|
||||
detail: Record<string, unknown>;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface ResolveContext {
|
||||
rng: { random(): number; randrange(n: number): number };
|
||||
rngSeed?: number;
|
||||
maxRefDepth: number;
|
||||
strict: boolean;
|
||||
surface: SurfaceKind;
|
||||
developerMode: boolean;
|
||||
warnings: ResolveWarning[];
|
||||
vars: Record<string, string>;
|
||||
modules: Record<string, ResolveModule>;
|
||||
}
|
||||
|
||||
export class UnknownRefError extends Error {
|
||||
constructor(public uuid: string) {
|
||||
super(`unknown wildcard ref: @{${uuid}}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class RefOutOfSurfaceError extends Error {
|
||||
constructor(public uuid: string, public surface: string) {
|
||||
super(`ref @{${uuid}} not allowed in ${surface!} surface`);
|
||||
}
|
||||
}
|
||||
|
||||
export class RecursionLimitExceeded extends Error {}
|
||||
|
||||
export class CycleDetectedError extends Error {
|
||||
constructor(public chain: string[]) {
|
||||
super("cycle: " + chain.map((u) => `@{${u}}`).join(" → "));
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveTokens(text: string, ctx: ResolveContext): string {
|
||||
const tokens = tokenizeRich(text);
|
||||
return resolveTokenList(tokens, ctx, 0, []);
|
||||
}
|
||||
|
||||
function resolveTokenList(
|
||||
tokens: RichToken[],
|
||||
ctx: ResolveContext,
|
||||
depth: number,
|
||||
visited: string[],
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
for (const tok of tokens) {
|
||||
switch (tok.kind) {
|
||||
case "text":
|
||||
parts.push(tok.raw);
|
||||
break;
|
||||
case "escape":
|
||||
parts.push((tok.meta?.literal as string) ?? "");
|
||||
break;
|
||||
case "var":
|
||||
parts.push(resolveVar(tok, ctx));
|
||||
break;
|
||||
case "ref":
|
||||
parts.push(resolveRef(tok, ctx, depth, visited));
|
||||
break;
|
||||
case "dp-brace":
|
||||
parts.push(resolveInlinePick(tok, ctx, depth, visited));
|
||||
break;
|
||||
case "dp-multi":
|
||||
parts.push(resolveMultiPick(tok, ctx, depth, visited));
|
||||
break;
|
||||
// dp-pipe, dp-weight not emitted at top level — assertion failure if we hit them
|
||||
default:
|
||||
throw new Error(`unknown token kind: ${tok.kind}`);
|
||||
}
|
||||
}
|
||||
return parts.join("");
|
||||
}
|
||||
|
||||
function resolveVar(tok: RichToken, ctx: ResolveContext): string {
|
||||
const name = (tok.meta?.name as string) ?? "";
|
||||
if (name.startsWith("__")) return "";
|
||||
return ctx.vars[name] ?? "";
|
||||
}
|
||||
|
||||
function resolveRef(
|
||||
tok: RichToken,
|
||||
ctx: ResolveContext,
|
||||
depth: number,
|
||||
visited: string[],
|
||||
): string {
|
||||
const uuid = (tok.meta?.uuid as string) ?? "";
|
||||
|
||||
if (depth >= ctx.maxRefDepth) {
|
||||
const chain = [...visited, uuid];
|
||||
if (ctx.strict) throw new RecursionLimitExceeded("max ref depth exceeded");
|
||||
pushWarning(ctx, {
|
||||
type: "recursion_limit", severity: "error", position: tok.start,
|
||||
detail: { chain, limit: ctx.maxRefDepth },
|
||||
message: `Recursion limit reached at depth ${ctx.maxRefDepth}`,
|
||||
});
|
||||
return "";
|
||||
}
|
||||
|
||||
if (visited.includes(uuid)) {
|
||||
const chain = [...visited, uuid];
|
||||
if (ctx.strict) throw new CycleDetectedError(chain);
|
||||
pushWarning(ctx, {
|
||||
type: "cycle_detected", severity: "error", position: tok.start,
|
||||
detail: { chain },
|
||||
message: "Cycle: " + chain.map((u) => `@{${u}}`).join(" → "),
|
||||
});
|
||||
return "";
|
||||
}
|
||||
|
||||
if (ctx.surface !== "wildcard") {
|
||||
if (ctx.strict) throw new RefOutOfSurfaceError(uuid, ctx.surface);
|
||||
const mod = ctx.modules[uuid];
|
||||
pushWarning(ctx, {
|
||||
type: "ref_out_of_surface", severity: "info", position: tok.start,
|
||||
detail: { uuid, name: mod?.var_binding ?? null, surface: ctx.surface },
|
||||
message: `@{${uuid}} ignored in ${ctx.surface!} surface`,
|
||||
});
|
||||
return "";
|
||||
}
|
||||
|
||||
const module = ctx.modules[uuid];
|
||||
if (!module) {
|
||||
if (ctx.strict) throw new UnknownRefError(uuid);
|
||||
pushWarning(ctx, {
|
||||
type: "unknown_ref", severity: "warn", position: tok.start,
|
||||
detail: { uuid, name: null },
|
||||
message: `Unknown wildcard ref @{${uuid}}`,
|
||||
});
|
||||
return "";
|
||||
}
|
||||
|
||||
const chosen = pickWeighted(module.options ?? [], ctx.rng);
|
||||
if (!chosen) return "";
|
||||
const chosenValue = String(chosen.value ?? "");
|
||||
if (!chosenValue) return "";
|
||||
|
||||
const nested = tokenizeRich(chosenValue);
|
||||
return resolveTokenList(nested, ctx, depth + 1, [...visited, uuid]);
|
||||
}
|
||||
|
||||
function resolveInlinePick(
|
||||
tok: RichToken,
|
||||
ctx: ResolveContext,
|
||||
depth: number,
|
||||
visited: string[],
|
||||
): string {
|
||||
const branches = (tok.meta?.branches as string[]) ?? [];
|
||||
if (!branches.length) return "";
|
||||
const idx = ctx.rng.randrange(branches.length);
|
||||
const chosen = branches[idx];
|
||||
if (!chosen) return "";
|
||||
return resolveTokenList(tokenizeRich(chosen), ctx, depth, visited);
|
||||
}
|
||||
|
||||
function resolveMultiPick(
|
||||
tok: RichToken,
|
||||
ctx: ResolveContext,
|
||||
depth: number,
|
||||
visited: string[],
|
||||
): string {
|
||||
const count = (tok.meta?.count as number) ?? 0;
|
||||
const sep = (tok.meta?.sep as string) ?? "";
|
||||
const branches = [...((tok.meta?.branches as string[]) ?? [])];
|
||||
|
||||
if (count <= 0) return "";
|
||||
const nPicks = Math.min(count, branches.length);
|
||||
if (nPicks === 0) return "";
|
||||
|
||||
const weights = branches.map(parseBranchWeight);
|
||||
const values = branches.map(stripBranchWeight);
|
||||
|
||||
const chosenIndices: number[] = [];
|
||||
const availableIdx = branches.map((_, i) => i);
|
||||
const availableW = [...weights];
|
||||
|
||||
for (let i = 0; i < nPicks; i++) {
|
||||
if (!availableIdx.length) break;
|
||||
const pickIdx = weightedPickIndex(availableW, ctx.rng);
|
||||
chosenIndices.push(availableIdx.splice(pickIdx, 1)[0]);
|
||||
availableW.splice(pickIdx, 1);
|
||||
}
|
||||
|
||||
return chosenIndices
|
||||
.map((idx) => resolveTokenList(tokenizeRich(values[idx]), ctx, depth, visited))
|
||||
.join(sep);
|
||||
}
|
||||
|
||||
function pickWeighted(
|
||||
options: Array<{ value: string; weight: number }>,
|
||||
rng: ResolveContext["rng"],
|
||||
): { value: string; weight: number } | null {
|
||||
if (!options.length) return null;
|
||||
const total = options.reduce((s, o) => s + Math.max(0, Number(o.weight) || 0), 0);
|
||||
if (total <= 0) return options[0];
|
||||
const r = rng.random() * total;
|
||||
let acc = 0;
|
||||
for (const o of options) {
|
||||
acc += Math.max(0, Number(o.weight) || 0);
|
||||
if (r <= acc) return o;
|
||||
}
|
||||
return options[options.length - 1];
|
||||
}
|
||||
|
||||
function parseBranchWeight(branch: string): number {
|
||||
if (!branch.includes("::")) return 1;
|
||||
const [prefix] = branch.split("::");
|
||||
const n = Number(prefix);
|
||||
if (!Number.isFinite(n) || n <= 0) return 1;
|
||||
return n;
|
||||
}
|
||||
|
||||
function stripBranchWeight(branch: string): string {
|
||||
if (!branch.includes("::")) return branch;
|
||||
const idx = branch.indexOf("::");
|
||||
const prefix = branch.slice(0, idx);
|
||||
if (!Number.isFinite(Number(prefix))) return branch;
|
||||
return branch.slice(idx + 2);
|
||||
}
|
||||
|
||||
function weightedPickIndex(weights: number[], rng: ResolveContext["rng"]): number {
|
||||
const total = weights.reduce((s, w) => s + Math.max(0, w), 0);
|
||||
if (total <= 0) return 0;
|
||||
const r = rng.random() * total;
|
||||
let acc = 0;
|
||||
for (let i = 0; i < weights.length; i++) {
|
||||
acc += Math.max(0, weights[i]);
|
||||
if (r <= acc) return i;
|
||||
}
|
||||
return weights.length - 1;
|
||||
}
|
||||
|
||||
function pushWarning(ctx: ResolveContext, partial: Partial<ResolveWarning>): void {
|
||||
ctx.warnings.push({
|
||||
type: "",
|
||||
severity: "warn",
|
||||
module_id: "",
|
||||
source_field: "",
|
||||
position: 0,
|
||||
token_index: null,
|
||||
detail: {},
|
||||
message: "",
|
||||
...partial,
|
||||
} as ResolveWarning);
|
||||
}
|
||||
Reference in New Issue
Block a user