Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

JavaScript · 142 lines · 4.2 KiB · at 74947de

// Tokenizer for .tsus source, built on TypeScript's own scanner.//// ts.createScanner knows strings, comments, numeric literals and every// operator, but two things depend on parser context and need a rescan://// - `/` starts either a division or a regex literal. We decide from the//   previous significant token (an expression just ended -> division) and call//   reScanSlashToken for the regex case.// - `}` either closes a block or resumes a template literal after `${ ... }`.//   We track the brace depth at each TemplateHead and call reScanTemplateToken//   when the matching `}` arrives.//// The result is a flat array of tokens with source offsets. Trivia (spaces,// comments) is skipped; callers copy it from the source text between tokens.import ts from "typescript";const K = ts.SyntaxKind;// Keywords after which a `/` starts a regex rather than dividing.const REGEX_AFTER_KEYWORD = new Set([  K.ReturnKeyword,  K.TypeOfKeyword,  K.InstanceOfKeyword,  K.InKeyword,  K.OfKeyword,  K.NewKeyword,  K.DeleteKeyword,  K.VoidKeyword,  K.ThrowKeyword,  K.CaseKeyword,  K.DoKeyword,  K.ElseKeyword,  K.YieldKeyword,  K.AwaitKeyword,  K.ExtendsKeyword,]);// Tokens that end an expression: a following `/` is division.const ENDS_EXPRESSION = new Set([  K.Identifier,  K.PrivateIdentifier,  K.NumericLiteral,  K.BigIntLiteral,  K.StringLiteral,  K.RegularExpressionLiteral,  K.NoSubstitutionTemplateLiteral,  K.TemplateTail,  K.CloseParenToken,  K.CloseBracketToken,  K.CloseBraceToken,  K.PlusPlusToken,  K.MinusMinusToken,  K.ThisKeyword,  K.SuperKeyword,  K.TrueKeyword,  K.FalseKeyword,  K.NullKeyword,]);const isKeyword = (kind) => kind >= K.FirstKeyword && kind <= K.LastKeyword;const slashStartsRegex = (prev) => {  if (prev === undefined) {    return true;  }  if (ENDS_EXPRESSION.has(prev.kind)) {    return false;  }  if (isKeyword(prev.kind)) {    // Reserved operators-as-words (return, typeof, ...) precede regexes; any    // other keyword here is being used as a name (`type / 2`).    return REGEX_AFTER_KEYWORD.has(prev.kind);  }  return true;};export const tokenize = (text) => {  const scanner = ts.createScanner(ts.ScriptTarget.Latest, true, ts.LanguageVariant.Standard, text);  const tokens = [];  // Brace depth at which each open template substitution started.  const templateDepths = [];  let braceDepth = 0;  while (true) {    let kind = scanner.scan();    if (kind === K.EndOfFileToken) {      break;    }    const prev = tokens.length > 0 ? tokens[tokens.length - 1] : undefined;    if ((kind === K.SlashToken || kind === K.SlashEqualsToken) && slashStartsRegex(prev)) {      kind = scanner.reScanSlashToken();    } else if (kind === K.OpenBraceToken) {      braceDepth += 1;    } else if (kind === K.CloseBraceToken) {      const top = templateDepths.length > 0 ? templateDepths[templateDepths.length - 1] : -1;      if (top === braceDepth) {        kind = scanner.reScanTemplateToken(false);        if (kind === K.TemplateTail) {          templateDepths.pop();        }      } else {        braceDepth -= 1;      }    } else if (kind === K.TemplateHead) {      templateDepths.push(braceDepth);    }    tokens.push({      kind,      start: scanner.getTokenStart(),      end: scanner.getTokenEnd(),      text: scanner.getTokenText(),      newlineBefore: scanner.hasPrecedingLineBreak(),    });  }  return tokens;};const OPENERS = new Set([K.OpenParenToken, K.OpenBracketToken, K.OpenBraceToken, K.TemplateHead]);const CLOSERS = new Set([K.CloseParenToken, K.CloseBracketToken, K.CloseBraceToken, K.TemplateTail]);export const isOpener = (kind) => OPENERS.has(kind);export const isCloser = (kind) => CLOSERS.has(kind);// Index of the token that closes the bracket opened at tokens[i], or -1.// Template literals count as a bracket pair (head ... tail), so braces inside// `${}` never confuse the count.export const findClose = (tokens, i, limit = tokens.length) => {  let depth = 0;  for (let j = i; j < limit; j += 1) {    const kind = tokens[j].kind;    if (OPENERS.has(kind)) {      depth += 1;    } else if (CLOSERS.has(kind)) {      depth -= 1;      if (depth === 0) {        return j;      }    }  }  return -1;};export { K as SyntaxKind };