// Tokenizer for .tsus source, built on TypeScript's own scanner.//// ts.createScanner knows strings, comments, numeric literals and every// operator, but two things depend on parser context and need a rescan://// - `/` starts either a division or a regex literal. We decide from the// previous significant token (an expression just ended -> division) and call// reScanSlashToken for the regex case.// - `}` either closes a block or resumes a template literal after `${ ... }`.// We track the brace depth at each TemplateHead and call reScanTemplateToken// when the matching `}` arrives.//// The result is a flat array of tokens with source offsets. Trivia (spaces,// comments) is skipped; callers copy it from the source text between tokens.import ts from "typescript";const K = ts.SyntaxKind;// Keywords after which a `/` starts a regex rather than dividing.const REGEX_AFTER_KEYWORD = new Set([ K.ReturnKeyword, K.TypeOfKeyword, K.InstanceOfKeyword, K.InKeyword, K.OfKeyword, K.NewKeyword, K.DeleteKeyword, K.VoidKeyword, K.ThrowKeyword, K.CaseKeyword, K.DoKeyword, K.ElseKeyword, K.YieldKeyword, K.AwaitKeyword, K.ExtendsKeyword,]);// Tokens that end an expression: a following `/` is division.const ENDS_EXPRESSION = new Set([ K.Identifier, K.PrivateIdentifier, K.NumericLiteral, K.BigIntLiteral, K.StringLiteral, K.RegularExpressionLiteral, K.NoSubstitutionTemplateLiteral, K.TemplateTail, K.CloseParenToken, K.CloseBracketToken, K.CloseBraceToken, K.PlusPlusToken, K.MinusMinusToken, K.ThisKeyword, K.SuperKeyword, K.TrueKeyword, K.FalseKeyword, K.NullKeyword,]);const isKeyword = (kind) => kind >= K.FirstKeyword && kind <= K.LastKeyword;const slashStartsRegex = (prev) => { if (prev === undefined) { return true; } if (ENDS_EXPRESSION.has(prev.kind)) { return false; } if (isKeyword(prev.kind)) { // Reserved operators-as-words (return, typeof, ...) precede regexes; any // other keyword here is being used as a name (`type / 2`). return REGEX_AFTER_KEYWORD.has(prev.kind); } return true;};export const tokenize = (text) => { const scanner = ts.createScanner(ts.ScriptTarget.Latest, true, ts.LanguageVariant.Standard, text); const tokens = []; // Brace depth at which each open template substitution started. const templateDepths = []; let braceDepth = 0; while (true) { let kind = scanner.scan(); if (kind === K.EndOfFileToken) { break; } const prev = tokens.length > 0 ? tokens[tokens.length - 1] : undefined; if ((kind === K.SlashToken || kind === K.SlashEqualsToken) && slashStartsRegex(prev)) { kind = scanner.reScanSlashToken(); } else if (kind === K.OpenBraceToken) { braceDepth += 1; } else if (kind === K.CloseBraceToken) { const top = templateDepths.length > 0 ? templateDepths[templateDepths.length - 1] : -1; if (top === braceDepth) { kind = scanner.reScanTemplateToken(false); if (kind === K.TemplateTail) { templateDepths.pop(); } } else { braceDepth -= 1; } } else if (kind === K.TemplateHead) { templateDepths.push(braceDepth); } tokens.push({ kind, start: scanner.getTokenStart(), end: scanner.getTokenEnd(), text: scanner.getTokenText(), newlineBefore: scanner.hasPrecedingLineBreak(), }); } return tokens;};const OPENERS = new Set([K.OpenParenToken, K.OpenBracketToken, K.OpenBraceToken, K.TemplateHead]);const CLOSERS = new Set([K.CloseParenToken, K.CloseBracketToken, K.CloseBraceToken, K.TemplateTail]);export const isOpener = (kind) => OPENERS.has(kind);export const isCloser = (kind) => CLOSERS.has(kind);// Index of the token that closes the bracket opened at tokens[i], or -1.// Template literals count as a bracket pair (head ... tail), so braces inside// `${}` never confuse the count.export const findClose = (tokens, i, limit = tokens.length) => { let depth = 0; for (let j = i; j < limit; j += 1) { const kind = tokens[j].kind; if (OPENERS.has(kind)) { depth += 1; } else if (CLOSERS.has(kind)) { depth -= 1; if (depth === 0) { return j; } } } return -1;};export { K as SyntaxKind };