100 lines
3.1 KiB
JavaScript
100 lines
3.1 KiB
JavaScript
export const TokenType = {
|
|
TEXT: "text",
|
|
LORA_OPEN: "lora_open",
|
|
LORA_CLOSE: "lora_close",
|
|
PAREN_OPEN: "paren_open",
|
|
PAREN_CLOSE: "paren_close",
|
|
EMBEDDING: "embedding",
|
|
ESCAPE: "escape",
|
|
INVALID_ESCAPE: "invalid_escape",
|
|
WILDCARD_OPEN: "wildcard_open",
|
|
WILDCARD_CLOSE: "wildcard_close",
|
|
COMMENT_LINE: "comment_line",
|
|
};
|
|
|
|
export class SyntaxTokenizer {
|
|
constructor() {
|
|
this.patterns = [
|
|
{ type: TokenType.COMMENT_LINE, regex: /^\s*#[^\n]*/gm },
|
|
{ type: TokenType.ESCAPE, regex: /\\[()]/g },
|
|
{ type: TokenType.INVALID_ESCAPE, regex: /\/[()]/g },
|
|
{ type: TokenType.EMBEDDING, regex: /embedding:[^,\s]+/gi },
|
|
{ type: TokenType.LORA_OPEN, regex: /<(?=lora:)/gi },
|
|
{ type: TokenType.LORA_CLOSE, regex: />/g },
|
|
{ type: TokenType.PAREN_OPEN, regex: /\(/g },
|
|
{ type: TokenType.PAREN_CLOSE, regex: /\)/g },
|
|
{ type: TokenType.WILDCARD_OPEN, regex: /\{/g },
|
|
{ type: TokenType.WILDCARD_CLOSE, regex: /\}/g },
|
|
];
|
|
}
|
|
|
|
tokenize(text) {
|
|
const tokens = [];
|
|
const matches = [];
|
|
|
|
for (const pattern of this.patterns) {
|
|
pattern.regex.lastIndex = 0; // Reset regex
|
|
let match;
|
|
while ((match = pattern.regex.exec(text)) !== null) {
|
|
matches.push({
|
|
type: pattern.type,
|
|
value: match[0],
|
|
start: match.index,
|
|
end: match.index + match[0].length,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Sort by start position, then by type priority (escapes first)
|
|
matches.sort((a, b) => {
|
|
if (a.start !== b.start) return a.start - b.start;
|
|
const priority = {
|
|
comment_line: -1,
|
|
escape: 0,
|
|
invalid_escape: 1,
|
|
paren_open: 2,
|
|
paren_close: 2,
|
|
};
|
|
return (priority[a.type] || 3) - (priority[b.type] || 3);
|
|
});
|
|
|
|
// Filter out overlapping matches, keeping higher priority ones
|
|
const filteredMatches = [];
|
|
for (const match of matches) {
|
|
const overlaps = filteredMatches.some(
|
|
(existing) =>
|
|
match.start < existing.end && match.end > existing.start
|
|
);
|
|
if (!overlaps) {
|
|
filteredMatches.push(match);
|
|
}
|
|
}
|
|
|
|
let lastIndex = 0;
|
|
for (const match of filteredMatches) {
|
|
if (match.start > lastIndex) {
|
|
tokens.push({
|
|
type: TokenType.TEXT,
|
|
value: text.slice(lastIndex, match.start),
|
|
start: lastIndex,
|
|
end: match.start,
|
|
});
|
|
}
|
|
|
|
tokens.push(match);
|
|
lastIndex = match.end;
|
|
}
|
|
|
|
if (lastIndex < text.length) {
|
|
tokens.push({
|
|
type: TokenType.TEXT,
|
|
value: text.slice(lastIndex),
|
|
start: lastIndex,
|
|
end: text.length,
|
|
});
|
|
}
|
|
|
|
return tokens;
|
|
}
|
|
}
|