Add capacitorjs runtime

node_modules/chevrotain/src/parse/constants.ts

@@ -0,0 +1,2 @@
+// TODO: can this be removed? where is it used?
+export let IN = "_~IN~_"

node_modules/chevrotain/src/parse/cst/cst.ts

@@ -0,0 +1,87 @@
+import { CstNode, CstNodeLocation, IToken } from "../../../api"
+
+/**
+ * This nodeLocation tracking is not efficient and should only be used
+ * when error recovery is enabled or the Token Vector contains virtual Tokens
+ * (e.g, Python Indent/Outdent)
+ * As it executes the calculation for every single terminal/nonTerminal
+ * and does not rely on the fact the token vector is **sorted**
+ */
+export function setNodeLocationOnlyOffset(
+  currNodeLocation: CstNodeLocation,
+  newLocationInfo: IToken
+): void {
+  // First (valid) update for this cst node
+  if (isNaN(currNodeLocation.startOffset) === true) {
+    // assumption1: Token location information is either NaN or a valid number
+    // assumption2: Token location information is fully valid if it exist
+    // (both start/end offsets exist and are numbers).
+    currNodeLocation.startOffset = newLocationInfo.startOffset
+    currNodeLocation.endOffset = newLocationInfo.endOffset
+  }
+  // Once the startOffset has been updated with a valid number it should never receive
+  // any farther updates as the Token vector is sorted.
+  // We still have to check this this condition for every new possible location info
+  // because with error recovery enabled we may encounter invalid tokens (NaN location props)
+  else if (currNodeLocation.endOffset < newLocationInfo.endOffset === true) {
+    currNodeLocation.endOffset = newLocationInfo.endOffset
+  }
+}
+
+/**
+ * This nodeLocation tracking is not efficient and should only be used
+ * when error recovery is enabled or the Token Vector contains virtual Tokens
+ * (e.g, Python Indent/Outdent)
+ * As it executes the calculation for every single terminal/nonTerminal
+ * and does not rely on the fact the token vector is **sorted**
+ */
+export function setNodeLocationFull(
+  currNodeLocation: CstNodeLocation,
+  newLocationInfo: CstNodeLocation
+): void {
+  // First (valid) update for this cst node
+  if (isNaN(currNodeLocation.startOffset) === true) {
+    // assumption1: Token location information is either NaN or a valid number
+    // assumption2: Token location information is fully valid if it exist
+    // (all start/end props exist and are numbers).
+    currNodeLocation.startOffset = newLocationInfo.startOffset
+    currNodeLocation.startColumn = newLocationInfo.startColumn
+    currNodeLocation.startLine = newLocationInfo.startLine
+    currNodeLocation.endOffset = newLocationInfo.endOffset
+    currNodeLocation.endColumn = newLocationInfo.endColumn
+    currNodeLocation.endLine = newLocationInfo.endLine
+  }
+  // Once the start props has been updated with a valid number it should never receive
+  // any farther updates as the Token vector is sorted.
+  // We still have to check this this condition for every new possible location info
+  // because with error recovery enabled we may encounter invalid tokens (NaN location props)
+  else if (currNodeLocation.endOffset < newLocationInfo.endOffset === true) {
+    currNodeLocation.endOffset = newLocationInfo.endOffset
+    currNodeLocation.endColumn = newLocationInfo.endColumn
+    currNodeLocation.endLine = newLocationInfo.endLine
+  }
+}
+
+export function addTerminalToCst(
+  node: CstNode,
+  token: IToken,
+  tokenTypeName: string
+): void {
+  if (node.children[tokenTypeName] === undefined) {
+    node.children[tokenTypeName] = [token]
+  } else {
+    node.children[tokenTypeName].push(token)
+  }
+}
+
+export function addNoneTerminalToCst(
+  node: CstNode,
+  ruleName: string,
+  ruleResult: any
+): void {
+  if (node.children[ruleName] === undefined) {
+    node.children[ruleName] = [ruleResult]
+  } else {
+    node.children[ruleName].push(ruleResult)
+  }
+}

node_modules/chevrotain/src/parse/cst/cst_visitor.ts

@@ -0,0 +1,178 @@
+import {
+  compact,
+  contains,
+  forEach,
+  isArray,
+  isEmpty,
+  isFunction,
+  isUndefined,
+  keys,
+  map
+} from "../../utils/utils"
+import { defineNameProp, functionName } from "../../lang/lang_extensions"
+import { ICstVisitor } from "../../../api"
+
+export function defaultVisit<IN, OUT>(ctx: any, param: IN): OUT {
+  let childrenNames = keys(ctx)
+  let childrenNamesLength = childrenNames.length
+  for (let i = 0; i < childrenNamesLength; i++) {
+    let currChildName = childrenNames[i]
+    let currChildArray = ctx[currChildName]
+    let currChildArrayLength = currChildArray.length
+    for (let j = 0; j < currChildArrayLength; j++) {
+      let currChild: any = currChildArray[j]
+      // distinction between Tokens Children and CstNode children
+      if (currChild.tokenTypeIdx === undefined) {
+        this[currChild.name](currChild.children, param)
+      }
+    }
+  }
+  // defaultVisit does not support generic out param
+  return undefined
+}
+
+export function createBaseSemanticVisitorConstructor(
+  grammarName: string,
+  ruleNames: string[]
+): {
+  new (...args: any[]): ICstVisitor<any, any>
+} {
+  let derivedConstructor: any = function () {}
+
+  // can be overwritten according to:
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/
+  // name?redirectlocale=en-US&redirectslug=JavaScript%2FReference%2FGlobal_Objects%2FFunction%2Fname
+  defineNameProp(derivedConstructor, grammarName + "BaseSemantics")
+
+  let semanticProto = {
+    visit: function (cstNode, param) {
+      // enables writing more concise visitor methods when CstNode has only a single child
+      if (isArray(cstNode)) {
+        // A CST Node's children dictionary can never have empty arrays as values
+        // If a key is defined there will be at least one element in the corresponding value array.
+        cstNode = cstNode[0]
+      }
+
+      // enables passing optional CstNodes concisely.
+      if (isUndefined(cstNode)) {
+        return undefined
+      }
+
+      return this[cstNode.name](cstNode.children, param)
+    },
+
+    validateVisitor: function () {
+      let semanticDefinitionErrors = validateVisitor(this, ruleNames)
+      if (!isEmpty(semanticDefinitionErrors)) {
+        let errorMessages = map(
+          semanticDefinitionErrors,
+          (currDefError) => currDefError.msg
+        )
+        throw Error(
+          `Errors Detected in CST Visitor <${functionName(
+            this.constructor
+          )}>:\n\t` + `${errorMessages.join("\n\n").replace(/\n/g, "\n\t")}`
+        )
+      }
+    }
+  }
+
+  derivedConstructor.prototype = semanticProto
+  derivedConstructor.prototype.constructor = derivedConstructor
+
+  derivedConstructor._RULE_NAMES = ruleNames
+
+  return derivedConstructor
+}
+
+export function createBaseVisitorConstructorWithDefaults(
+  grammarName: string,
+  ruleNames: string[],
+  baseConstructor: Function
+): {
+  new (...args: any[]): ICstVisitor<any, any>
+} {
+  let derivedConstructor: any = function () {}
+
+  // can be overwritten according to:
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/
+  // name?redirectlocale=en-US&redirectslug=JavaScript%2FReference%2FGlobal_Objects%2FFunction%2Fname
+  defineNameProp(derivedConstructor, grammarName + "BaseSemanticsWithDefaults")
+
+  let withDefaultsProto = Object.create(baseConstructor.prototype)
+  forEach(ruleNames, (ruleName) => {
+    withDefaultsProto[ruleName] = defaultVisit
+  })
+
+  derivedConstructor.prototype = withDefaultsProto
+  derivedConstructor.prototype.constructor = derivedConstructor
+
+  return derivedConstructor
+}
+
+export enum CstVisitorDefinitionError {
+  REDUNDANT_METHOD,
+  MISSING_METHOD
+}
+
+export interface IVisitorDefinitionError {
+  msg: string
+  type: CstVisitorDefinitionError
+  methodName: string
+}
+
+export function validateVisitor(
+  visitorInstance: Function,
+  ruleNames: string[]
+): IVisitorDefinitionError[] {
+  let missingErrors = validateMissingCstMethods(visitorInstance, ruleNames)
+  let redundantErrors = validateRedundantMethods(visitorInstance, ruleNames)
+
+  return missingErrors.concat(redundantErrors)
+}
+
+export function validateMissingCstMethods(
+  visitorInstance: Function,
+  ruleNames: string[]
+): IVisitorDefinitionError[] {
+  let errors: IVisitorDefinitionError[] = map(ruleNames, (currRuleName) => {
+    if (!isFunction(visitorInstance[currRuleName])) {
+      return {
+        msg: `Missing visitor method: <${currRuleName}> on ${functionName(
+          <any>visitorInstance.constructor
+        )} CST Visitor.`,
+        type: CstVisitorDefinitionError.MISSING_METHOD,
+        methodName: currRuleName
+      }
+    }
+  })
+
+  return compact<IVisitorDefinitionError>(errors)
+}
+
+const VALID_PROP_NAMES = ["constructor", "visit", "validateVisitor"]
+export function validateRedundantMethods(
+  visitorInstance: Function,
+  ruleNames: string[]
+): IVisitorDefinitionError[] {
+  let errors = []
+
+  for (let prop in visitorInstance) {
+    if (
+      isFunction(visitorInstance[prop]) &&
+      !contains(VALID_PROP_NAMES, prop) &&
+      !contains(ruleNames, prop)
+    ) {
+      errors.push({
+        msg:
+          `Redundant visitor method: <${prop}> on ${functionName(
+            <any>visitorInstance.constructor
+          )} CST Visitor\n` +
+          `There is no Grammar Rule corresponding to this method's name.\n`,
+        type: CstVisitorDefinitionError.REDUNDANT_METHOD,
+        methodName: prop
+      })
+    }
+  }
+  return errors
+}

node_modules/chevrotain/src/parse/errors_public.ts

@@ -0,0 +1,320 @@
+import { hasTokenLabel, tokenLabel } from "../scan/tokens_public"
+import * as utils from "../utils/utils"
+import { first, map, reduce } from "../utils/utils"
+import {
+  Alternation,
+  NonTerminal,
+  Rule,
+  Terminal
+} from "./grammar/gast/gast_public"
+import { getProductionDslName } from "./grammar/gast/gast"
+import {
+  IGrammarResolverErrorMessageProvider,
+  IGrammarValidatorErrorMessageProvider,
+  IParserErrorMessageProvider,
+  IProductionWithOccurrence,
+  TokenType
+} from "../../api"
+
+export const defaultParserErrorProvider: IParserErrorMessageProvider = {
+  buildMismatchTokenMessage({ expected, actual, previous, ruleName }): string {
+    let hasLabel = hasTokenLabel(expected)
+    let expectedMsg = hasLabel
+      ? `--> ${tokenLabel(expected)} <--`
+      : `token of type --> ${expected.name} <--`
+
+    let msg = `Expecting ${expectedMsg} but found --> '${actual.image}' <--`
+
+    return msg
+  },
+
+  buildNotAllInputParsedMessage({ firstRedundant, ruleName }): string {
+    return "Redundant input, expecting EOF but found: " + firstRedundant.image
+  },
+
+  buildNoViableAltMessage({
+    expectedPathsPerAlt,
+    actual,
+    previous,
+    customUserDescription,
+    ruleName
+  }): string {
+    let errPrefix = "Expecting: "
+    // TODO: issue: No Viable Alternative Error may have incomplete details. #502
+    let actualText = first(actual).image
+    let errSuffix = "\nbut found: '" + actualText + "'"
+
+    if (customUserDescription) {
+      return errPrefix + customUserDescription + errSuffix
+    } else {
+      let allLookAheadPaths = reduce(
+        expectedPathsPerAlt,
+        (result, currAltPaths) => result.concat(currAltPaths),
+        []
+      )
+      let nextValidTokenSequences = map(
+        allLookAheadPaths,
+        (currPath) =>
+          `[${map(currPath, (currTokenType) => tokenLabel(currTokenType)).join(
+            ", "
+          )}]`
+      )
+      let nextValidSequenceItems = map(
+        nextValidTokenSequences,
+        (itemMsg, idx) => `  ${idx + 1}. ${itemMsg}`
+      )
+      let calculatedDescription = `one of these possible Token sequences:\n${nextValidSequenceItems.join(
+        "\n"
+      )}`
+
+      return errPrefix + calculatedDescription + errSuffix
+    }
+  },
+
+  buildEarlyExitMessage({
+    expectedIterationPaths,
+    actual,
+    customUserDescription,
+    ruleName
+  }): string {
+    let errPrefix = "Expecting: "
+    // TODO: issue: No Viable Alternative Error may have incomplete details. #502
+    let actualText = first(actual).image
+    let errSuffix = "\nbut found: '" + actualText + "'"
+
+    if (customUserDescription) {
+      return errPrefix + customUserDescription + errSuffix
+    } else {
+      let nextValidTokenSequences = map(
+        expectedIterationPaths,
+        (currPath) =>
+          `[${map(currPath, (currTokenType) => tokenLabel(currTokenType)).join(
+            ","
+          )}]`
+      )
+      let calculatedDescription =
+        `expecting at least one iteration which starts with one of these possible Token sequences::\n  ` +
+        `<${nextValidTokenSequences.join(" ,")}>`
+
+      return errPrefix + calculatedDescription + errSuffix
+    }
+  }
+}
+
+Object.freeze(defaultParserErrorProvider)
+
+export const defaultGrammarResolverErrorProvider: IGrammarResolverErrorMessageProvider = {
+  buildRuleNotFoundError(
+    topLevelRule: Rule,
+    undefinedRule: NonTerminal
+  ): string {
+    const msg =
+      "Invalid grammar, reference to a rule which is not defined: ->" +
+      undefinedRule.nonTerminalName +
+      "<-\n" +
+      "inside top level rule: ->" +
+      topLevelRule.name +
+      "<-"
+    return msg
+  }
+}
+
+export const defaultGrammarValidatorErrorProvider: IGrammarValidatorErrorMessageProvider = {
+  buildDuplicateFoundError(
+    topLevelRule: Rule,
+    duplicateProds: IProductionWithOccurrence[]
+  ): string {
+    function getExtraProductionArgument(
+      prod: IProductionWithOccurrence
+    ): string {
+      if (prod instanceof Terminal) {
+        return prod.terminalType.name
+      } else if (prod instanceof NonTerminal) {
+        return prod.nonTerminalName
+      } else {
+        return ""
+      }
+    }
+
+    const topLevelName = topLevelRule.name
+    const duplicateProd = first(duplicateProds)
+    const index = duplicateProd.idx
+    const dslName = getProductionDslName(duplicateProd)
+    let extraArgument = getExtraProductionArgument(duplicateProd)
+
+    const hasExplicitIndex = index > 0
+    let msg = `->${dslName}${hasExplicitIndex ? index : ""}<- ${
+      extraArgument ? `with argument: ->${extraArgument}<-` : ""
+    }
+                  appears more than once (${
+                    duplicateProds.length
+                  } times) in the top level rule: ->${topLevelName}<-.                  
+                  For further details see: https://sap.github.io/chevrotain/docs/FAQ.html#NUMERICAL_SUFFIXES 
+                  `
+
+    // white space trimming time! better to trim afterwards as it allows to use WELL formatted multi line template strings...
+    msg = msg.replace(/[ \t]+/g, " ")
+    msg = msg.replace(/\s\s+/g, "\n")
+
+    return msg
+  },
+
+  buildNamespaceConflictError(rule: Rule): string {
+    const errMsg =
+      `Namespace conflict found in grammar.\n` +
+      `The grammar has both a Terminal(Token) and a Non-Terminal(Rule) named: <${rule.name}>.\n` +
+      `To resolve this make sure each Terminal and Non-Terminal names are unique\n` +
+      `This is easy to accomplish by using the convention that Terminal names start with an uppercase letter\n` +
+      `and Non-Terminal names start with a lower case letter.`
+
+    return errMsg
+  },
+
+  buildAlternationPrefixAmbiguityError(options: {
+    topLevelRule: Rule
+    prefixPath: TokenType[]
+    ambiguityIndices: number[]
+    alternation: Alternation
+  }): string {
+    const pathMsg = map(options.prefixPath, (currTok) =>
+      tokenLabel(currTok)
+    ).join(", ")
+    const occurrence =
+      options.alternation.idx === 0 ? "" : options.alternation.idx
+    const errMsg =
+      `Ambiguous alternatives: <${options.ambiguityIndices.join(
+        " ,"
+      )}> due to common lookahead prefix\n` +
+      `in <OR${occurrence}> inside <${options.topLevelRule.name}> Rule,\n` +
+      `<${pathMsg}> may appears as a prefix path in all these alternatives.\n` +
+      `See: https://sap.github.io/chevrotain/docs/guide/resolving_grammar_errors.html#COMMON_PREFIX\n` +
+      `For Further details.`
+
+    return errMsg
+  },
+
+  buildAlternationAmbiguityError(options: {
+    topLevelRule: Rule
+    prefixPath: TokenType[]
+    ambiguityIndices: number[]
+    alternation: Alternation
+  }): string {
+    let pathMsg = map(options.prefixPath, (currtok) =>
+      tokenLabel(currtok)
+    ).join(", ")
+    let occurrence =
+      options.alternation.idx === 0 ? "" : options.alternation.idx
+    let currMessage =
+      `Ambiguous Alternatives Detected: <${options.ambiguityIndices.join(
+        " ,"
+      )}> in <OR${occurrence}>` +
+      ` inside <${options.topLevelRule.name}> Rule,\n` +
+      `<${pathMsg}> may appears as a prefix path in all these alternatives.\n`
+
+    currMessage =
+      currMessage +
+      `See: https://sap.github.io/chevrotain/docs/guide/resolving_grammar_errors.html#AMBIGUOUS_ALTERNATIVES\n` +
+      `For Further details.`
+    return currMessage
+  },
+
+  buildEmptyRepetitionError(options: {
+    topLevelRule: Rule
+    repetition: IProductionWithOccurrence
+  }): string {
+    let dslName = getProductionDslName(options.repetition)
+    if (options.repetition.idx !== 0) {
+      dslName += options.repetition.idx
+    }
+
+    const errMsg =
+      `The repetition <${dslName}> within Rule <${options.topLevelRule.name}> can never consume any tokens.\n` +
+      `This could lead to an infinite loop.`
+
+    return errMsg
+  },
+
+  // TODO: remove - `errors_public` from nyc.config.js exclude
+  //       once this method is fully removed from this file
+  buildTokenNameError(options: {
+    tokenType: TokenType
+    expectedPattern: RegExp
+  }): string {
+    /* istanbul ignore next */
+    return "deprecated"
+  },
+
+  buildEmptyAlternationError(options: {
+    topLevelRule: Rule
+    alternation: Alternation
+    emptyChoiceIdx: number
+  }): string {
+    const errMsg =
+      `Ambiguous empty alternative: <${options.emptyChoiceIdx + 1}>` +
+      ` in <OR${options.alternation.idx}> inside <${options.topLevelRule.name}> Rule.\n` +
+      `Only the last alternative may be an empty alternative.`
+
+    return errMsg
+  },
+
+  buildTooManyAlternativesError(options: {
+    topLevelRule: Rule
+    alternation: Alternation
+  }): string {
+    const errMsg =
+      `An Alternation cannot have more than 256 alternatives:\n` +
+      `<OR${options.alternation.idx}> inside <${
+        options.topLevelRule.name
+      }> Rule.\n has ${options.alternation.definition.length + 1} alternatives.`
+
+    return errMsg
+  },
+
+  buildLeftRecursionError(options: {
+    topLevelRule: Rule
+    leftRecursionPath: Rule[]
+  }): string {
+    const ruleName = options.topLevelRule.name
+    let pathNames = utils.map(
+      options.leftRecursionPath,
+      (currRule) => currRule.name
+    )
+    let leftRecursivePath = `${ruleName} --> ${pathNames
+      .concat([ruleName])
+      .join(" --> ")}`
+    let errMsg =
+      `Left Recursion found in grammar.\n` +
+      `rule: <${ruleName}> can be invoked from itself (directly or indirectly)\n` +
+      `without consuming any Tokens. The grammar path that causes this is: \n ${leftRecursivePath}\n` +
+      ` To fix this refactor your grammar to remove the left recursion.\n` +
+      `see: https://en.wikipedia.org/wiki/LL_parser#Left_Factoring.`
+
+    return errMsg
+  },
+
+  // TODO: remove - `errors_public` from nyc.config.js exclude
+  //       once this method is fully removed from this file
+  buildInvalidRuleNameError(options: {
+    topLevelRule: Rule
+    expectedPattern: RegExp
+  }): string {
+    /* istanbul ignore next */
+    return "deprecated"
+  },
+
+  buildDuplicateRuleNameError(options: {
+    topLevelRule: Rule | string
+    grammarName: string
+  }): string {
+    let ruleName
+    if (options.topLevelRule instanceof Rule) {
+      ruleName = options.topLevelRule.name
+    } else {
+      ruleName = options.topLevelRule
+    }
+
+    const errMsg = `Duplicate definition, rule: ->${ruleName}<- is already defined in the grammar: ->${options.grammarName}<-`
+
+    return errMsg
+  }
+}

node_modules/chevrotain/src/parse/exceptions_public.ts

@@ -0,0 +1,69 @@
+import { contains } from "../utils/utils"
+import { IToken, IRecognitionException, IRecognizerContext } from "../../api"
+
+const MISMATCHED_TOKEN_EXCEPTION = "MismatchedTokenException"
+const NO_VIABLE_ALT_EXCEPTION = "NoViableAltException"
+const EARLY_EXIT_EXCEPTION = "EarlyExitException"
+const NOT_ALL_INPUT_PARSED_EXCEPTION = "NotAllInputParsedException"
+
+const RECOGNITION_EXCEPTION_NAMES = [
+  MISMATCHED_TOKEN_EXCEPTION,
+  NO_VIABLE_ALT_EXCEPTION,
+  EARLY_EXIT_EXCEPTION,
+  NOT_ALL_INPUT_PARSED_EXCEPTION
+]
+
+Object.freeze(RECOGNITION_EXCEPTION_NAMES)
+
+// hacks to bypass no support for custom Errors in javascript/typescript
+export function isRecognitionException(error: Error) {
+  // can't do instanceof on hacked custom js exceptions
+  return contains(RECOGNITION_EXCEPTION_NAMES, error.name)
+}
+
+abstract class RecognitionException
+  extends Error
+  implements IRecognitionException {
+  context: IRecognizerContext
+  resyncedTokens = []
+
+  protected constructor(message: string, public token: IToken) {
+    super(message)
+
+    // fix prototype chain when typescript target is ES5
+    Object.setPrototypeOf(this, new.target.prototype)
+
+    /* istanbul ignore next - V8 workaround to remove constructor from stacktrace when typescript target is ES5 */
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, this.constructor)
+    }
+  }
+}
+
+export class MismatchedTokenException extends RecognitionException {
+  constructor(message: string, token: IToken, public previousToken: IToken) {
+    super(message, token)
+    this.name = MISMATCHED_TOKEN_EXCEPTION
+  }
+}
+
+export class NoViableAltException extends RecognitionException {
+  constructor(message: string, token: IToken, public previousToken: IToken) {
+    super(message, token)
+    this.name = NO_VIABLE_ALT_EXCEPTION
+  }
+}
+
+export class NotAllInputParsedException extends RecognitionException {
+  constructor(message: string, token: IToken) {
+    super(message, token)
+    this.name = NOT_ALL_INPUT_PARSED_EXCEPTION
+  }
+}
+
+export class EarlyExitException extends RecognitionException {
+  constructor(message: string, token: IToken, public previousToken: IToken) {
+    super(message, token)
+    this.name = EARLY_EXIT_EXCEPTION
+  }
+}

node_modules/chevrotain/src/parse/grammar/checks.ts

@@ -0,0 +1,725 @@
+import * as utils from "../../utils/utils"
+import {
+  contains,
+  every,
+  findAll,
+  flatten,
+  forEach,
+  isEmpty,
+  map,
+  reduce,
+  reject
+} from "../../utils/utils"
+import {
+  IParserAmbiguousAlternativesDefinitionError,
+  IParserDuplicatesDefinitionError,
+  IParserEmptyAlternativeDefinitionError,
+  ParserDefinitionErrorType
+} from "../parser/parser"
+import { getProductionDslName, isOptionalProd } from "./gast/gast"
+import {
+  Alternative,
+  containsPath,
+  getLookaheadPathsForOptionalProd,
+  getLookaheadPathsForOr,
+  getProdType,
+  isStrictPrefixOfPath
+} from "./lookahead"
+import { nextPossibleTokensAfter } from "./interpreter"
+import {
+  Alternation,
+  Alternative as AlternativeGAST,
+  NonTerminal,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Rule,
+  Terminal
+} from "./gast/gast_public"
+import { GAstVisitor } from "./gast/gast_visitor_public"
+import {
+  IGrammarValidatorErrorMessageProvider,
+  IParserDefinitionError,
+  IProduction,
+  IProductionWithOccurrence,
+  TokenType
+} from "../../../api"
+
+export function validateGrammar(
+  topLevels: Rule[],
+  globalMaxLookahead: number,
+  tokenTypes: TokenType[],
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+  grammarName: string
+): IParserDefinitionError[] {
+  let duplicateErrors: any = utils.map(topLevels, (currTopLevel) =>
+    validateDuplicateProductions(currTopLevel, errMsgProvider)
+  )
+  let leftRecursionErrors: any = utils.map(topLevels, (currTopRule) =>
+    validateNoLeftRecursion(currTopRule, currTopRule, errMsgProvider)
+  )
+
+  let emptyAltErrors = []
+  let ambiguousAltsErrors = []
+  let emptyRepetitionErrors = []
+
+  // left recursion could cause infinite loops in the following validations.
+  // It is safest to first have the user fix the left recursion errors first and only then examine Further issues.
+  if (every(leftRecursionErrors, isEmpty)) {
+    emptyAltErrors = map(topLevels, (currTopRule) =>
+      validateEmptyOrAlternative(currTopRule, errMsgProvider)
+    )
+    ambiguousAltsErrors = map(topLevels, (currTopRule) =>
+      validateAmbiguousAlternationAlternatives(
+        currTopRule,
+        globalMaxLookahead,
+        errMsgProvider
+      )
+    )
+
+    emptyRepetitionErrors = validateSomeNonEmptyLookaheadPath(
+      topLevels,
+      globalMaxLookahead,
+      errMsgProvider
+    )
+  }
+
+  let termsNamespaceConflictErrors = checkTerminalAndNoneTerminalsNameSpace(
+    topLevels,
+    tokenTypes,
+    errMsgProvider
+  )
+
+  const tooManyAltsErrors = map(topLevels, (curRule) =>
+    validateTooManyAlts(curRule, errMsgProvider)
+  )
+
+  const duplicateRulesError = map(topLevels, (curRule) =>
+    validateRuleDoesNotAlreadyExist(
+      curRule,
+      topLevels,
+      grammarName,
+      errMsgProvider
+    )
+  )
+
+  return <any>(
+    utils.flatten(
+      duplicateErrors.concat(
+        emptyRepetitionErrors,
+        leftRecursionErrors,
+        emptyAltErrors,
+        ambiguousAltsErrors,
+        termsNamespaceConflictErrors,
+        tooManyAltsErrors,
+        duplicateRulesError
+      )
+    )
+  )
+}
+
+function validateDuplicateProductions(
+  topLevelRule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+): IParserDuplicatesDefinitionError[] {
+  let collectorVisitor = new OccurrenceValidationCollector()
+  topLevelRule.accept(collectorVisitor)
+  let allRuleProductions = collectorVisitor.allProductions
+
+  let productionGroups = utils.groupBy(
+    allRuleProductions,
+    identifyProductionForDuplicates
+  )
+
+  let duplicates: any = utils.pick(productionGroups, (currGroup) => {
+    return currGroup.length > 1
+  })
+
+  let errors = utils.map(utils.values(duplicates), (currDuplicates: any) => {
+    let firstProd: any = utils.first(currDuplicates)
+    let msg = errMsgProvider.buildDuplicateFoundError(
+      topLevelRule,
+      currDuplicates
+    )
+    let dslName = getProductionDslName(firstProd)
+    let defError: IParserDuplicatesDefinitionError = {
+      message: msg,
+      type: ParserDefinitionErrorType.DUPLICATE_PRODUCTIONS,
+      ruleName: topLevelRule.name,
+      dslName: dslName,
+      occurrence: firstProd.idx
+    }
+
+    let param = getExtraProductionArgument(firstProd)
+    if (param) {
+      defError.parameter = param
+    }
+
+    return defError
+  })
+  return errors
+}
+
+export function identifyProductionForDuplicates(
+  prod: IProductionWithOccurrence
+): string {
+  return `${getProductionDslName(prod)}_#_${
+    prod.idx
+  }_#_${getExtraProductionArgument(prod)}`
+}
+
+function getExtraProductionArgument(prod: IProductionWithOccurrence): string {
+  if (prod instanceof Terminal) {
+    return prod.terminalType.name
+  } else if (prod instanceof NonTerminal) {
+    return prod.nonTerminalName
+  } else {
+    return ""
+  }
+}
+
+export class OccurrenceValidationCollector extends GAstVisitor {
+  public allProductions: IProduction[] = []
+
+  public visitNonTerminal(subrule: NonTerminal): void {
+    this.allProductions.push(subrule)
+  }
+
+  public visitOption(option: Option): void {
+    this.allProductions.push(option)
+  }
+
+  public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void {
+    this.allProductions.push(manySep)
+  }
+
+  public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void {
+    this.allProductions.push(atLeastOne)
+  }
+
+  public visitRepetitionMandatoryWithSeparator(
+    atLeastOneSep: RepetitionMandatoryWithSeparator
+  ): void {
+    this.allProductions.push(atLeastOneSep)
+  }
+
+  public visitRepetition(many: Repetition): void {
+    this.allProductions.push(many)
+  }
+
+  public visitAlternation(or: Alternation): void {
+    this.allProductions.push(or)
+  }
+
+  public visitTerminal(terminal: Terminal): void {
+    this.allProductions.push(terminal)
+  }
+}
+
+export function validateRuleDoesNotAlreadyExist(
+  rule: Rule,
+  allRules: Rule[],
+  className,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+): IParserDefinitionError[] {
+  let errors = []
+  const occurrences = reduce(
+    allRules,
+    (result, curRule) => {
+      if (curRule.name === rule.name) {
+        return result + 1
+      }
+      return result
+    },
+    0
+  )
+  if (occurrences > 1) {
+    const errMsg = errMsgProvider.buildDuplicateRuleNameError({
+      topLevelRule: rule,
+      grammarName: className
+    })
+    errors.push({
+      message: errMsg,
+      type: ParserDefinitionErrorType.DUPLICATE_RULE_NAME,
+      ruleName: rule.name
+    })
+  }
+
+  return errors
+}
+
+// TODO: is there anyway to get only the rule names of rules inherited from the super grammars?
+// This is not part of the IGrammarErrorProvider because the validation cannot be performed on
+// The grammar structure, only at runtime.
+export function validateRuleIsOverridden(
+  ruleName: string,
+  definedRulesNames: string[],
+  className
+): IParserDefinitionError[] {
+  let errors = []
+  let errMsg
+
+  if (!utils.contains(definedRulesNames, ruleName)) {
+    errMsg =
+      `Invalid rule override, rule: ->${ruleName}<- cannot be overridden in the grammar: ->${className}<-` +
+      `as it is not defined in any of the super grammars `
+    errors.push({
+      message: errMsg,
+      type: ParserDefinitionErrorType.INVALID_RULE_OVERRIDE,
+      ruleName: ruleName
+    })
+  }
+
+  return errors
+}
+
+export function validateNoLeftRecursion(
+  topRule: Rule,
+  currRule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+  path: Rule[] = []
+): IParserDefinitionError[] {
+  let errors = []
+  let nextNonTerminals = getFirstNoneTerminal(currRule.definition)
+  if (utils.isEmpty(nextNonTerminals)) {
+    return []
+  } else {
+    let ruleName = topRule.name
+    let foundLeftRecursion = utils.contains(<any>nextNonTerminals, topRule)
+    if (foundLeftRecursion) {
+      errors.push({
+        message: errMsgProvider.buildLeftRecursionError({
+          topLevelRule: topRule,
+          leftRecursionPath: path
+        }),
+        type: ParserDefinitionErrorType.LEFT_RECURSION,
+        ruleName: ruleName
+      })
+    }
+
+    // we are only looking for cyclic paths leading back to the specific topRule
+    // other cyclic paths are ignored, we still need this difference to avoid infinite loops...
+    let validNextSteps = utils.difference(
+      nextNonTerminals,
+      path.concat([topRule])
+    )
+    let errorsFromNextSteps = utils.map(validNextSteps, (currRefRule) => {
+      let newPath = utils.cloneArr(path)
+      newPath.push(currRefRule)
+      return validateNoLeftRecursion(
+        topRule,
+        currRefRule,
+        errMsgProvider,
+        newPath
+      )
+    })
+
+    return errors.concat(utils.flatten(errorsFromNextSteps))
+  }
+}
+
+export function getFirstNoneTerminal(definition: IProduction[]): Rule[] {
+  let result = []
+  if (utils.isEmpty(definition)) {
+    return result
+  }
+  let firstProd = utils.first(definition)
+
+  /* istanbul ignore else */
+  if (firstProd instanceof NonTerminal) {
+    result.push(firstProd.referencedRule)
+  } else if (
+    firstProd instanceof AlternativeGAST ||
+    firstProd instanceof Option ||
+    firstProd instanceof RepetitionMandatory ||
+    firstProd instanceof RepetitionMandatoryWithSeparator ||
+    firstProd instanceof RepetitionWithSeparator ||
+    firstProd instanceof Repetition
+  ) {
+    result = result.concat(
+      getFirstNoneTerminal(<IProduction[]>firstProd.definition)
+    )
+  } else if (firstProd instanceof Alternation) {
+    // each sub definition in alternation is a FLAT
+    result = utils.flatten(
+      utils.map(firstProd.definition, (currSubDef) =>
+        getFirstNoneTerminal((<AlternativeGAST>currSubDef).definition)
+      )
+    )
+  } else if (firstProd instanceof Terminal) {
+    // nothing to see, move along
+  } else {
+    throw Error("non exhaustive match")
+  }
+
+  let isFirstOptional = isOptionalProd(firstProd)
+  let hasMore = definition.length > 1
+  if (isFirstOptional && hasMore) {
+    let rest = utils.drop(definition)
+    return result.concat(getFirstNoneTerminal(rest))
+  } else {
+    return result
+  }
+}
+
+class OrCollector extends GAstVisitor {
+  public alternations = []
+
+  public visitAlternation(node: Alternation): void {
+    this.alternations.push(node)
+  }
+}
+
+export function validateEmptyOrAlternative(
+  topLevelRule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+): IParserEmptyAlternativeDefinitionError[] {
+  let orCollector = new OrCollector()
+  topLevelRule.accept(orCollector)
+  let ors = orCollector.alternations
+
+  let errors = utils.reduce(
+    ors,
+    (errors, currOr) => {
+      let exceptLast = utils.dropRight(currOr.definition)
+      let currErrors = utils.map(
+        exceptLast,
+        (currAlternative: IProduction, currAltIdx) => {
+          const possibleFirstInAlt = nextPossibleTokensAfter(
+            [currAlternative],
+            [],
+            null,
+            1
+          )
+          if (utils.isEmpty(possibleFirstInAlt)) {
+            return {
+              message: errMsgProvider.buildEmptyAlternationError({
+                topLevelRule: topLevelRule,
+                alternation: currOr,
+                emptyChoiceIdx: currAltIdx
+              }),
+              type: ParserDefinitionErrorType.NONE_LAST_EMPTY_ALT,
+              ruleName: topLevelRule.name,
+              occurrence: currOr.idx,
+              alternative: currAltIdx + 1
+            }
+          } else {
+            return null
+          }
+        }
+      )
+      return errors.concat(utils.compact(currErrors))
+    },
+    []
+  )
+
+  return errors
+}
+
+export function validateAmbiguousAlternationAlternatives(
+  topLevelRule: Rule,
+  globalMaxLookahead: number,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+): IParserAmbiguousAlternativesDefinitionError[] {
+  let orCollector = new OrCollector()
+  topLevelRule.accept(orCollector)
+  let ors = orCollector.alternations
+
+  // New Handling of ignoring ambiguities
+  // - https://github.com/SAP/chevrotain/issues/869
+  ors = reject(ors, (currOr) => currOr.ignoreAmbiguities === true)
+
+  let errors = utils.reduce(
+    ors,
+    (result, currOr: Alternation) => {
+      let currOccurrence = currOr.idx
+      const actualMaxLookahead = currOr.maxLookahead || globalMaxLookahead
+      let alternatives = getLookaheadPathsForOr(
+        currOccurrence,
+        topLevelRule,
+        actualMaxLookahead,
+        currOr
+      )
+      let altsAmbiguityErrors = checkAlternativesAmbiguities(
+        alternatives,
+        currOr,
+        topLevelRule,
+        errMsgProvider
+      )
+      let altsPrefixAmbiguityErrors = checkPrefixAlternativesAmbiguities(
+        alternatives,
+        currOr,
+        topLevelRule,
+        errMsgProvider
+      )
+
+      return result.concat(altsAmbiguityErrors, altsPrefixAmbiguityErrors)
+    },
+    []
+  )
+
+  return errors
+}
+
+export class RepetionCollector extends GAstVisitor {
+  public allProductions: IProduction[] = []
+
+  public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void {
+    this.allProductions.push(manySep)
+  }
+
+  public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void {
+    this.allProductions.push(atLeastOne)
+  }
+
+  public visitRepetitionMandatoryWithSeparator(
+    atLeastOneSep: RepetitionMandatoryWithSeparator
+  ): void {
+    this.allProductions.push(atLeastOneSep)
+  }
+
+  public visitRepetition(many: Repetition): void {
+    this.allProductions.push(many)
+  }
+}
+
+export function validateTooManyAlts(
+  topLevelRule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+): IParserDefinitionError[] {
+  let orCollector = new OrCollector()
+  topLevelRule.accept(orCollector)
+  let ors = orCollector.alternations
+
+  let errors = utils.reduce(
+    ors,
+    (errors, currOr) => {
+      if (currOr.definition.length > 255) {
+        errors.push({
+          message: errMsgProvider.buildTooManyAlternativesError({
+            topLevelRule: topLevelRule,
+            alternation: currOr
+          }),
+          type: ParserDefinitionErrorType.TOO_MANY_ALTS,
+          ruleName: topLevelRule.name,
+          occurrence: currOr.idx
+        })
+      }
+      return errors
+    },
+    []
+  )
+
+  return errors
+}
+
+export function validateSomeNonEmptyLookaheadPath(
+  topLevelRules: Rule[],
+  maxLookahead: number,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+): IParserDefinitionError[] {
+  let errors = []
+  forEach(topLevelRules, (currTopRule) => {
+    let collectorVisitor = new RepetionCollector()
+    currTopRule.accept(collectorVisitor)
+    let allRuleProductions = collectorVisitor.allProductions
+    forEach(allRuleProductions, (currProd) => {
+      let prodType = getProdType(currProd)
+      const actualMaxLookahead = currProd.maxLookahead || maxLookahead
+      let currOccurrence = currProd.idx
+      let paths = getLookaheadPathsForOptionalProd(
+        currOccurrence,
+        currTopRule,
+        prodType,
+        actualMaxLookahead
+      )
+      let pathsInsideProduction = paths[0]
+      if (isEmpty(flatten(pathsInsideProduction))) {
+        const errMsg = errMsgProvider.buildEmptyRepetitionError({
+          topLevelRule: currTopRule,
+          repetition: currProd
+        })
+        errors.push({
+          message: errMsg,
+          type: ParserDefinitionErrorType.NO_NON_EMPTY_LOOKAHEAD,
+          ruleName: currTopRule.name
+        })
+      }
+    })
+  })
+
+  return errors
+}
+
+export interface IAmbiguityDescriptor {
+  alts: number[]
+  path: TokenType[]
+}
+
+function checkAlternativesAmbiguities(
+  alternatives: Alternative[],
+  alternation: Alternation,
+  rule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+): IParserAmbiguousAlternativesDefinitionError[] {
+  let foundAmbiguousPaths = []
+  let identicalAmbiguities = reduce(
+    alternatives,
+    (result, currAlt, currAltIdx) => {
+      // ignore (skip) ambiguities with this alternative
+      if (alternation.definition[currAltIdx].ignoreAmbiguities === true) {
+        return result
+      }
+
+      forEach(currAlt, (currPath) => {
+        let altsCurrPathAppearsIn = [currAltIdx]
+        forEach(alternatives, (currOtherAlt, currOtherAltIdx) => {
+          if (
+            currAltIdx !== currOtherAltIdx &&
+            containsPath(currOtherAlt, currPath) &&
+            // ignore (skip) ambiguities with this "other" alternative
+            alternation.definition[currOtherAltIdx].ignoreAmbiguities !== true
+          ) {
+            altsCurrPathAppearsIn.push(currOtherAltIdx)
+          }
+        })
+
+        if (
+          altsCurrPathAppearsIn.length > 1 &&
+          !containsPath(foundAmbiguousPaths, currPath)
+        ) {
+          foundAmbiguousPaths.push(currPath)
+          result.push({
+            alts: altsCurrPathAppearsIn,
+            path: currPath
+          })
+        }
+      })
+      return result
+    },
+    []
+  )
+
+  let currErrors = utils.map(identicalAmbiguities, (currAmbDescriptor) => {
+    let ambgIndices = map(
+      currAmbDescriptor.alts,
+      (currAltIdx) => currAltIdx + 1
+    )
+
+    const currMessage = errMsgProvider.buildAlternationAmbiguityError({
+      topLevelRule: rule,
+      alternation: alternation,
+      ambiguityIndices: ambgIndices,
+      prefixPath: currAmbDescriptor.path
+    })
+
+    return {
+      message: currMessage,
+      type: ParserDefinitionErrorType.AMBIGUOUS_ALTS,
+      ruleName: rule.name,
+      occurrence: alternation.idx,
+      alternatives: [currAmbDescriptor.alts]
+    }
+  })
+
+  return currErrors
+}
+
+export function checkPrefixAlternativesAmbiguities(
+  alternatives: Alternative[],
+  alternation: Alternation,
+  rule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+): IAmbiguityDescriptor[] {
+  let errors = []
+
+  // flatten
+  let pathsAndIndices = reduce(
+    alternatives,
+    (result, currAlt, idx) => {
+      let currPathsAndIdx = map(currAlt, (currPath) => {
+        return { idx: idx, path: currPath }
+      })
+      return result.concat(currPathsAndIdx)
+    },
+    []
+  )
+
+  forEach(pathsAndIndices, (currPathAndIdx) => {
+    const alternativeGast = alternation.definition[currPathAndIdx.idx]
+    // ignore (skip) ambiguities with this alternative
+    if (alternativeGast.ignoreAmbiguities === true) {
+      return
+    }
+    let targetIdx = currPathAndIdx.idx
+    let targetPath = currPathAndIdx.path
+
+    let prefixAmbiguitiesPathsAndIndices = findAll(
+      pathsAndIndices,
+      (searchPathAndIdx) => {
+        // prefix ambiguity can only be created from lower idx (higher priority) path
+        return (
+          // ignore (skip) ambiguities with this "other" alternative
+          alternation.definition[searchPathAndIdx.idx].ignoreAmbiguities !==
+            true &&
+          searchPathAndIdx.idx < targetIdx &&
+          // checking for strict prefix because identical lookaheads
+          // will be be detected using a different validation.
+          isStrictPrefixOfPath(searchPathAndIdx.path, targetPath)
+        )
+      }
+    )
+
+    let currPathPrefixErrors = map(
+      prefixAmbiguitiesPathsAndIndices,
+      (currAmbPathAndIdx) => {
+        let ambgIndices = [currAmbPathAndIdx.idx + 1, targetIdx + 1]
+        const occurrence = alternation.idx === 0 ? "" : alternation.idx
+
+        const message = errMsgProvider.buildAlternationPrefixAmbiguityError({
+          topLevelRule: rule,
+          alternation: alternation,
+          ambiguityIndices: ambgIndices,
+          prefixPath: currAmbPathAndIdx.path
+        })
+        return {
+          message: message,
+          type: ParserDefinitionErrorType.AMBIGUOUS_PREFIX_ALTS,
+          ruleName: rule.name,
+          occurrence: occurrence,
+          alternatives: ambgIndices
+        }
+      }
+    )
+    errors = errors.concat(currPathPrefixErrors)
+  })
+
+  return errors
+}
+
+function checkTerminalAndNoneTerminalsNameSpace(
+  topLevels: Rule[],
+  tokenTypes: TokenType[],
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+): IParserDefinitionError[] {
+  let errors = []
+
+  let tokenNames = map(tokenTypes, (currToken) => currToken.name)
+
+  forEach(topLevels, (currRule) => {
+    const currRuleName = currRule.name
+    if (contains(tokenNames, currRuleName)) {
+      let errMsg = errMsgProvider.buildNamespaceConflictError(currRule)
+
+      errors.push({
+        message: errMsg,
+        type: ParserDefinitionErrorType.CONFLICT_TOKENS_RULES_NAMESPACE,
+        ruleName: currRuleName
+      })
+    }
+  })
+
+  return errors
+}

node_modules/chevrotain/src/parse/grammar/first.ts

@@ -0,0 +1,61 @@
+import { uniq, map, flatten } from "../../utils/utils"
+import { AbstractProduction, NonTerminal, Terminal } from "./gast/gast_public"
+import { isBranchingProd, isOptionalProd, isSequenceProd } from "./gast/gast"
+import { IProduction, TokenType } from "../../../api"
+
+export function first(prod: IProduction): TokenType[] {
+  /* istanbul ignore else */
+  if (prod instanceof NonTerminal) {
+    // this could in theory cause infinite loops if
+    // (1) prod A refs prod B.
+    // (2) prod B refs prod A
+    // (3) AB can match the empty set
+    // in other words a cycle where everything is optional so the first will keep
+    // looking ahead for the next optional part and will never exit
+    // currently there is no safeguard for this unique edge case because
+    // (1) not sure a grammar in which this can happen is useful for anything (productive)
+    return first((<NonTerminal>prod).referencedRule)
+  } else if (prod instanceof Terminal) {
+    return firstForTerminal(<Terminal>prod)
+  } else if (isSequenceProd(prod)) {
+    return firstForSequence(<AbstractProduction>prod)
+  } else if (isBranchingProd(prod)) {
+    return firstForBranching(<AbstractProduction>prod)
+  } else {
+    throw Error("non exhaustive match")
+  }
+}
+
+export function firstForSequence(prod: AbstractProduction): TokenType[] {
+  let firstSet: TokenType[] = []
+  let seq = prod.definition
+  let nextSubProdIdx = 0
+  let hasInnerProdsRemaining = seq.length > nextSubProdIdx
+  let currSubProd
+  // so we enter the loop at least once (if the definition is not empty
+  let isLastInnerProdOptional = true
+  // scan a sequence until it's end or until we have found a NONE optional production in it
+  while (hasInnerProdsRemaining && isLastInnerProdOptional) {
+    currSubProd = seq[nextSubProdIdx]
+    isLastInnerProdOptional = isOptionalProd(currSubProd)
+    firstSet = firstSet.concat(first(currSubProd))
+    nextSubProdIdx = nextSubProdIdx + 1
+    hasInnerProdsRemaining = seq.length > nextSubProdIdx
+  }
+
+  return uniq(firstSet)
+}
+
+export function firstForBranching(prod: AbstractProduction): TokenType[] {
+  let allAlternativesFirsts: TokenType[][] = map(
+    prod.definition,
+    (innerProd) => {
+      return first(innerProd)
+    }
+  )
+  return uniq(flatten<TokenType>(allAlternativesFirsts))
+}
+
+export function firstForTerminal(terminal: Terminal): TokenType[] {
+  return [terminal.terminalType]
+}

node_modules/chevrotain/src/parse/grammar/follow.ts

@@ -0,0 +1,67 @@
+import { RestWalker } from "./rest"
+import { first } from "./first"
+import { assign, forEach } from "../../utils/utils"
+import { IN } from "../constants"
+import { Alternative, NonTerminal, Rule, Terminal } from "./gast/gast_public"
+import { IProduction, TokenType } from "../../../api"
+
+// This ResyncFollowsWalker computes all of the follows required for RESYNC
+// (skipping reference production).
+export class ResyncFollowsWalker extends RestWalker {
+  public follows = {}
+
+  constructor(private topProd: Rule) {
+    super()
+  }
+
+  startWalking(): Record<string, TokenType[]> {
+    this.walk(this.topProd)
+    return this.follows
+  }
+
+  walkTerminal(
+    terminal: Terminal,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    // do nothing! just like in the public sector after 13:00
+  }
+
+  walkProdRef(
+    refProd: NonTerminal,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    let followName =
+      buildBetweenProdsFollowPrefix(refProd.referencedRule, refProd.idx) +
+      this.topProd.name
+    let fullRest: IProduction[] = currRest.concat(prevRest)
+    let restProd = new Alternative({ definition: fullRest })
+    let t_in_topProd_follows = first(restProd)
+    this.follows[followName] = t_in_topProd_follows
+  }
+}
+
+export function computeAllProdsFollows(
+  topProductions: Rule[]
+): Record<string, TokenType[]> {
+  let reSyncFollows = {}
+
+  forEach(topProductions, (topProd) => {
+    let currRefsFollow = new ResyncFollowsWalker(topProd).startWalking()
+    assign(reSyncFollows, currRefsFollow)
+  })
+  return reSyncFollows
+}
+
+export function buildBetweenProdsFollowPrefix(
+  inner: Rule,
+  occurenceInParent: number
+): string {
+  return inner.name + occurenceInParent + IN
+}
+
+export function buildInProdFollowPrefix(terminal: Terminal): string {
+  let terminalName = terminal.terminalType.name
+  return terminalName + terminal.idx + IN
+}

node_modules/chevrotain/src/parse/grammar/gast/gast.ts

@@ -0,0 +1,179 @@
+import { contains, every, has, some } from "../../../utils/utils"
+import {
+  AbstractProduction,
+  Alternation,
+  Alternative,
+  NonTerminal,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Rule,
+  Terminal
+} from "./gast_public"
+import { GAstVisitor } from "./gast_visitor_public"
+import { IProduction, IProductionWithOccurrence } from "../../../../api"
+
+export function isSequenceProd(prod: IProduction): boolean {
+  return (
+    prod instanceof Alternative ||
+    prod instanceof Option ||
+    prod instanceof Repetition ||
+    prod instanceof RepetitionMandatory ||
+    prod instanceof RepetitionMandatoryWithSeparator ||
+    prod instanceof RepetitionWithSeparator ||
+    prod instanceof Terminal ||
+    prod instanceof Rule
+  )
+}
+
+export function isOptionalProd(
+  prod: IProduction,
+  alreadyVisited: NonTerminal[] = []
+): boolean {
+  let isDirectlyOptional =
+    prod instanceof Option ||
+    prod instanceof Repetition ||
+    prod instanceof RepetitionWithSeparator
+  if (isDirectlyOptional) {
+    return true
+  }
+
+  // note that this can cause infinite loop if one optional empty TOP production has a cyclic dependency with another
+  // empty optional top rule
+  // may be indirectly optional ((A?B?C?) | (D?E?F?))
+  if (prod instanceof Alternation) {
+    // for OR its enough for just one of the alternatives to be optional
+    return some((<Alternation>prod).definition, (subProd: IProduction) => {
+      return isOptionalProd(subProd, alreadyVisited)
+    })
+  } else if (prod instanceof NonTerminal && contains(alreadyVisited, prod)) {
+    // avoiding stack overflow due to infinite recursion
+    return false
+  } else if (prod instanceof AbstractProduction) {
+    if (prod instanceof NonTerminal) {
+      alreadyVisited.push(prod)
+    }
+    return every(
+      (<AbstractProduction>prod).definition,
+      (subProd: IProduction) => {
+        return isOptionalProd(subProd, alreadyVisited)
+      }
+    )
+  } else {
+    return false
+  }
+}
+
+export function isBranchingProd(prod: IProduction): boolean {
+  return prod instanceof Alternation
+}
+
+export function getProductionDslName(prod: IProductionWithOccurrence): string {
+  /* istanbul ignore else */
+  if (prod instanceof NonTerminal) {
+    return "SUBRULE"
+  } else if (prod instanceof Option) {
+    return "OPTION"
+  } else if (prod instanceof Alternation) {
+    return "OR"
+  } else if (prod instanceof RepetitionMandatory) {
+    return "AT_LEAST_ONE"
+  } else if (prod instanceof RepetitionMandatoryWithSeparator) {
+    return "AT_LEAST_ONE_SEP"
+  } else if (prod instanceof RepetitionWithSeparator) {
+    return "MANY_SEP"
+  } else if (prod instanceof Repetition) {
+    return "MANY"
+  } else if (prod instanceof Terminal) {
+    return "CONSUME"
+  } else {
+    throw Error("non exhaustive match")
+  }
+}
+
+export class DslMethodsCollectorVisitor extends GAstVisitor {
+  // A minus is never valid in an identifier name
+  public separator = "-"
+  public dslMethods = {
+    option: [],
+    alternation: [],
+    repetition: [],
+    repetitionWithSeparator: [],
+    repetitionMandatory: [],
+    repetitionMandatoryWithSeparator: []
+  }
+
+  reset() {
+    this.dslMethods = {
+      option: [],
+      alternation: [],
+      repetition: [],
+      repetitionWithSeparator: [],
+      repetitionMandatory: [],
+      repetitionMandatoryWithSeparator: []
+    }
+  }
+
+  public visitTerminal(terminal: Terminal): void {
+    const key = terminal.terminalType.name + this.separator + "Terminal"
+    if (!has(this.dslMethods, key)) {
+      this.dslMethods[key] = []
+    }
+    this.dslMethods[key].push(terminal)
+  }
+
+  public visitNonTerminal(subrule: NonTerminal): void {
+    const key = subrule.nonTerminalName + this.separator + "Terminal"
+    if (!has(this.dslMethods, key)) {
+      this.dslMethods[key] = []
+    }
+    this.dslMethods[key].push(subrule)
+  }
+
+  public visitOption(option: Option): void {
+    this.dslMethods.option.push(option)
+  }
+
+  public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void {
+    this.dslMethods.repetitionWithSeparator.push(manySep)
+  }
+
+  public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void {
+    this.dslMethods.repetitionMandatory.push(atLeastOne)
+  }
+
+  public visitRepetitionMandatoryWithSeparator(
+    atLeastOneSep: RepetitionMandatoryWithSeparator
+  ): void {
+    this.dslMethods.repetitionMandatoryWithSeparator.push(atLeastOneSep)
+  }
+
+  public visitRepetition(many: Repetition): void {
+    this.dslMethods.repetition.push(many)
+  }
+
+  public visitAlternation(or: Alternation): void {
+    this.dslMethods.alternation.push(or)
+  }
+}
+
+const collectorVisitor = new DslMethodsCollectorVisitor()
+export function collectMethods(
+  rule: Rule
+): {
+  option: Option[]
+  alternation: Alternation[]
+  repetition: Repetition[]
+  repetitionWithSeparator: RepetitionWithSeparator[]
+  repetitionMandatory: RepetitionMandatory[]
+  repetitionMandatoryWithSeparator: RepetitionMandatoryWithSeparator
+} {
+  collectorVisitor.reset()
+  rule.accept(collectorVisitor)
+  const dslMethods = collectorVisitor.dslMethods
+  // avoid uncleaned references
+  collectorVisitor.reset()
+  return <any>dslMethods
+}

node_modules/chevrotain/src/parse/grammar/gast/gast_public.ts

@@ -0,0 +1,371 @@
+import { assign, forEach, isRegExp, map, pick } from "../../../utils/utils"
+import { tokenLabel } from "../../../scan/tokens_public"
+import {
+  IGASTVisitor,
+  IProduction,
+  IProductionWithOccurrence,
+  ISerializedGast,
+  TokenType
+} from "../../../../api"
+
+export abstract class AbstractProduction<T extends IProduction = IProduction>
+  implements IProduction {
+  public get definition(): T[] {
+    return this._definition
+  }
+  public set definition(value: T[]) {
+    this._definition = value
+  }
+
+  constructor(protected _definition: T[]) {}
+
+  accept(visitor: IGASTVisitor): void {
+    visitor.visit(this)
+    forEach(this.definition, (prod) => {
+      prod.accept(visitor)
+    })
+  }
+}
+
+export class NonTerminal
+  extends AbstractProduction
+  implements IProductionWithOccurrence {
+  public nonTerminalName: string
+  public referencedRule: Rule
+  public idx: number = 1
+
+  constructor(options: {
+    nonTerminalName: string
+    referencedRule?: Rule
+    idx?: number
+  }) {
+    super([])
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+
+  set definition(definition: IProduction[]) {
+    // immutable
+  }
+
+  get definition(): IProduction[] {
+    if (this.referencedRule !== undefined) {
+      return this.referencedRule.definition
+    }
+    return []
+  }
+
+  accept(visitor: IGASTVisitor): void {
+    visitor.visit(this)
+    // don't visit children of a reference, we will get cyclic infinite loops if we do so
+  }
+}
+
+export class Rule extends AbstractProduction {
+  public name: string
+  public orgText: string = ""
+
+  constructor(options: {
+    name: string
+    definition: IProduction[]
+    orgText?: string
+  }) {
+    super(options.definition)
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+}
+
+export class Alternative extends AbstractProduction {
+  public ignoreAmbiguities: boolean = false
+
+  constructor(options: {
+    definition: IProduction[]
+    ignoreAmbiguities?: boolean
+  }) {
+    super(options.definition)
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+}
+
+export class Option
+  extends AbstractProduction
+  implements IProductionWithOccurrence {
+  public idx: number = 1
+  public maxLookahead?: number
+
+  constructor(options: {
+    definition: IProduction[]
+    idx?: number
+    maxLookahead?: number
+  }) {
+    super(options.definition)
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+}
+
+export class RepetitionMandatory
+  extends AbstractProduction
+  implements IProductionWithOccurrence {
+  public idx: number = 1
+  public maxLookahead?: number
+
+  constructor(options: {
+    definition: IProduction[]
+    idx?: number
+    maxLookahead?: number
+  }) {
+    super(options.definition)
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+}
+
+export class RepetitionMandatoryWithSeparator
+  extends AbstractProduction
+  implements IProductionWithOccurrence {
+  public separator: TokenType
+  public idx: number = 1
+
+  constructor(options: {
+    definition: IProduction[]
+    separator: TokenType
+    idx?: number
+  }) {
+    super(options.definition)
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+}
+
+export class Repetition
+  extends AbstractProduction
+  implements IProductionWithOccurrence {
+  public separator: TokenType
+  public idx: number = 1
+  public maxLookahead?: number
+
+  constructor(options: {
+    definition: IProduction[]
+    idx?: number
+    maxLookahead?: number
+  }) {
+    super(options.definition)
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+}
+
+export class RepetitionWithSeparator
+  extends AbstractProduction
+  implements IProductionWithOccurrence {
+  public separator: TokenType
+  public idx: number = 1
+
+  constructor(options: {
+    definition: IProduction[]
+    separator: TokenType
+    idx?: number
+  }) {
+    super(options.definition)
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+}
+
+export class Alternation
+  extends AbstractProduction<Alternative>
+  implements IProductionWithOccurrence {
+  public idx: number = 1
+  public ignoreAmbiguities: boolean = false
+  public hasPredicates: boolean = false
+  public maxLookahead?: number
+
+  public get definition(): Alternative[] {
+    return this._definition
+  }
+  public set definition(value: Alternative[]) {
+    this._definition = value
+  }
+
+  constructor(options: {
+    definition: Alternative[]
+    idx?: number
+    ignoreAmbiguities?: boolean
+    hasPredicates?: boolean
+    maxLookahead?: number
+  }) {
+    super(options.definition)
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+}
+
+export class Terminal implements IProductionWithOccurrence {
+  public terminalType: TokenType
+  public idx: number = 1
+
+  constructor(options: { terminalType: TokenType; idx?: number }) {
+    assign(
+      this,
+      pick(options, (v) => v !== undefined)
+    )
+  }
+
+  accept(visitor: IGASTVisitor): void {
+    visitor.visit(this)
+  }
+}
+
+export interface ISerializedBasic extends ISerializedGast {
+  type:
+    | "Alternative"
+    | "Option"
+    | "RepetitionMandatory"
+    | "Repetition"
+    | "Alternation"
+  idx?: number
+}
+
+export interface ISerializedGastRule extends ISerializedGast {
+  type: "Rule"
+  name: string
+  orgText: string
+}
+
+export interface ISerializedNonTerminal extends ISerializedGast {
+  type: "NonTerminal"
+  name: string
+  idx: number
+}
+
+export interface ISerializedTerminal extends ISerializedGast {
+  type: "Terminal"
+  name: string
+  label?: string
+  pattern?: string
+  idx: number
+}
+
+export interface ISerializedTerminalWithSeparator extends ISerializedGast {
+  type: "RepetitionMandatoryWithSeparator" | "RepetitionWithSeparator"
+  idx: number
+  separator: ISerializedTerminal
+}
+
+export type ISerializedGastAny =
+  | ISerializedBasic
+  | ISerializedGastRule
+  | ISerializedNonTerminal
+  | ISerializedTerminal
+  | ISerializedTerminalWithSeparator
+
+export function serializeGrammar(topRules: Rule[]): ISerializedGast[] {
+  return map(topRules, serializeProduction)
+}
+
+export function serializeProduction(node: IProduction): ISerializedGast {
+  function convertDefinition(definition: IProduction[]): ISerializedGast[] {
+    return map(definition, serializeProduction)
+  }
+  /* istanbul ignore else */
+  if (node instanceof NonTerminal) {
+    return <ISerializedNonTerminal>{
+      type: "NonTerminal",
+      name: node.nonTerminalName,
+      idx: node.idx
+    }
+  } else if (node instanceof Alternative) {
+    return <ISerializedBasic>{
+      type: "Alternative",
+      definition: convertDefinition(node.definition)
+    }
+  } else if (node instanceof Option) {
+    return <ISerializedBasic>{
+      type: "Option",
+      idx: node.idx,
+      definition: convertDefinition(node.definition)
+    }
+  } else if (node instanceof RepetitionMandatory) {
+    return <ISerializedBasic>{
+      type: "RepetitionMandatory",
+      idx: node.idx,
+      definition: convertDefinition(node.definition)
+    }
+  } else if (node instanceof RepetitionMandatoryWithSeparator) {
+    return <ISerializedTerminalWithSeparator>{
+      type: "RepetitionMandatoryWithSeparator",
+      idx: node.idx,
+      separator: <ISerializedTerminal>(
+        serializeProduction(new Terminal({ terminalType: node.separator }))
+      ),
+      definition: convertDefinition(node.definition)
+    }
+  } else if (node instanceof RepetitionWithSeparator) {
+    return <ISerializedTerminalWithSeparator>{
+      type: "RepetitionWithSeparator",
+      idx: node.idx,
+      separator: <ISerializedTerminal>(
+        serializeProduction(new Terminal({ terminalType: node.separator }))
+      ),
+      definition: convertDefinition(node.definition)
+    }
+  } else if (node instanceof Repetition) {
+    return <ISerializedBasic>{
+      type: "Repetition",
+      idx: node.idx,
+      definition: convertDefinition(node.definition)
+    }
+  } else if (node instanceof Alternation) {
+    return <ISerializedBasic>{
+      type: "Alternation",
+      idx: node.idx,
+      definition: convertDefinition(node.definition)
+    }
+  } else if (node instanceof Terminal) {
+    let serializedTerminal = <ISerializedTerminal>{
+      type: "Terminal",
+      name: node.terminalType.name,
+      label: tokenLabel(node.terminalType),
+      idx: node.idx
+    }
+
+    let pattern = node.terminalType.PATTERN
+    if (node.terminalType.PATTERN) {
+      serializedTerminal.pattern = isRegExp(pattern)
+        ? (<any>pattern).source
+        : pattern
+    }
+
+    return serializedTerminal
+  } else if (node instanceof Rule) {
+    return <ISerializedGastRule>{
+      type: "Rule",
+      name: node.name,
+      orgText: node.orgText,
+      definition: convertDefinition(node.definition)
+    }
+  } else {
+    throw Error("non exhaustive match")
+  }
+}

node_modules/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts

@@ -0,0 +1,64 @@
+import { Rule } from "./gast_public"
+
+import { defaults, forEach } from "../../../utils/utils"
+import { resolveGrammar as orgResolveGrammar } from "../resolver"
+import { validateGrammar as orgValidateGrammar } from "../checks"
+import {
+  defaultGrammarResolverErrorProvider,
+  defaultGrammarValidatorErrorProvider
+} from "../../errors_public"
+import { DslMethodsCollectorVisitor } from "./gast"
+import {
+  IGrammarResolverErrorMessageProvider,
+  IGrammarValidatorErrorMessageProvider,
+  IParserDefinitionError,
+  IProductionWithOccurrence,
+  TokenType
+} from "../../../../api"
+
+export function resolveGrammar(options: {
+  rules: Rule[]
+  errMsgProvider?: IGrammarResolverErrorMessageProvider
+}): IParserDefinitionError[] {
+  options = defaults(options, {
+    errMsgProvider: defaultGrammarResolverErrorProvider
+  })
+
+  const topRulesTable = {}
+  forEach(options.rules, (rule) => {
+    topRulesTable[rule.name] = rule
+  })
+  return orgResolveGrammar(topRulesTable, options.errMsgProvider)
+}
+
+export function validateGrammar(options: {
+  rules: Rule[]
+  maxLookahead: number
+  tokenTypes: TokenType[]
+  grammarName: string
+  errMsgProvider: IGrammarValidatorErrorMessageProvider
+}): IParserDefinitionError[] {
+  options = defaults(options, {
+    errMsgProvider: defaultGrammarValidatorErrorProvider
+  })
+
+  return orgValidateGrammar(
+    options.rules,
+    options.maxLookahead,
+    options.tokenTypes,
+    options.errMsgProvider,
+    options.grammarName
+  )
+}
+
+export function assignOccurrenceIndices(options: { rules: Rule[] }): void {
+  forEach(options.rules, (currRule) => {
+    const methodsCollector = new DslMethodsCollectorVisitor()
+    currRule.accept(methodsCollector)
+    forEach(methodsCollector.dslMethods, (methods) => {
+      forEach(methods, (currMethod: IProductionWithOccurrence, arrIdx) => {
+        currMethod.idx = arrIdx + 1
+      })
+    })
+  })
+}

node_modules/chevrotain/src/parse/grammar/gast/gast_visitor_public.ts

@@ -0,0 +1,66 @@
+import {
+  Alternation,
+  Alternative,
+  NonTerminal,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Rule,
+  Terminal
+} from "./gast_public"
+import { IProduction } from "../../../../api"
+
+export abstract class GAstVisitor {
+  public visit(node: IProduction): any {
+    const nodeAny: any = node
+    switch (nodeAny.constructor) {
+      case NonTerminal:
+        return this.visitNonTerminal(nodeAny)
+      case Alternative:
+        return this.visitAlternative(nodeAny)
+      case Option:
+        return this.visitOption(nodeAny)
+      case RepetitionMandatory:
+        return this.visitRepetitionMandatory(nodeAny)
+      case RepetitionMandatoryWithSeparator:
+        return this.visitRepetitionMandatoryWithSeparator(nodeAny)
+      case RepetitionWithSeparator:
+        return this.visitRepetitionWithSeparator(nodeAny)
+      case Repetition:
+        return this.visitRepetition(nodeAny)
+      case Alternation:
+        return this.visitAlternation(nodeAny)
+      case Terminal:
+        return this.visitTerminal(nodeAny)
+      case Rule:
+        return this.visitRule(nodeAny)
+      /* istanbul ignore next */
+      default:
+        throw Error("non exhaustive match")
+    }
+  }
+
+  public visitNonTerminal(node: NonTerminal): any {}
+
+  public visitAlternative(node: Alternative): any {}
+
+  public visitOption(node: Option): any {}
+
+  public visitRepetition(node: Repetition): any {}
+
+  public visitRepetitionMandatory(node: RepetitionMandatory): any {}
+
+  public visitRepetitionMandatoryWithSeparator(
+    node: RepetitionMandatoryWithSeparator
+  ): any {}
+
+  public visitRepetitionWithSeparator(node: RepetitionWithSeparator): any {}
+
+  public visitAlternation(node: Alternation): any {}
+
+  public visitTerminal(node: Terminal): any {}
+
+  public visitRule(node: Rule): any {}
+}

node_modules/chevrotain/src/parse/grammar/interpreter.ts

@@ -0,0 +1,610 @@
+import { RestWalker } from "./rest"
+import {
+  cloneArr,
+  drop,
+  dropRight,
+  first as _first,
+  forEach,
+  isEmpty,
+  last
+} from "../../utils/utils"
+import { first } from "./first"
+import { TokenMatcher } from "../parser/parser"
+import {
+  AbstractProduction,
+  Alternation,
+  Alternative,
+  NonTerminal,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Rule,
+  Terminal
+} from "./gast/gast_public"
+import {
+  IGrammarPath,
+  IProduction,
+  ISyntacticContentAssistPath,
+  IToken,
+  ITokenGrammarPath,
+  TokenType
+} from "../../../api"
+
+export abstract class AbstractNextPossibleTokensWalker extends RestWalker {
+  protected possibleTokTypes: TokenType[] = []
+  protected ruleStack: string[]
+  protected occurrenceStack: number[]
+
+  protected nextProductionName = ""
+  protected nextProductionOccurrence = 0
+  protected found = false
+  protected isAtEndOfPath = false
+
+  constructor(protected topProd: Rule, protected path: IGrammarPath) {
+    super()
+  }
+
+  startWalking(): TokenType[] {
+    this.found = false
+
+    if (this.path.ruleStack[0] !== this.topProd.name) {
+      throw Error("The path does not start with the walker's top Rule!")
+    }
+
+    // immutable for the win
+    this.ruleStack = cloneArr(this.path.ruleStack).reverse() // intelij bug requires assertion
+    this.occurrenceStack = cloneArr(this.path.occurrenceStack).reverse() // intelij bug requires assertion
+
+    // already verified that the first production is valid, we now seek the 2nd production
+    this.ruleStack.pop()
+    this.occurrenceStack.pop()
+
+    this.updateExpectedNext()
+    this.walk(this.topProd)
+
+    return this.possibleTokTypes
+  }
+
+  walk(prod: AbstractProduction, prevRest: IProduction[] = []): void {
+    // stop scanning once we found the path
+    if (!this.found) {
+      super.walk(prod, prevRest)
+    }
+  }
+
+  walkProdRef(
+    refProd: NonTerminal,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    // found the next production, need to keep walking in it
+    if (
+      refProd.referencedRule.name === this.nextProductionName &&
+      refProd.idx === this.nextProductionOccurrence
+    ) {
+      let fullRest = currRest.concat(prevRest)
+      this.updateExpectedNext()
+      this.walk(refProd.referencedRule, <any>fullRest)
+    }
+  }
+
+  updateExpectedNext(): void {
+    // need to consume the Terminal
+    if (isEmpty(this.ruleStack)) {
+      // must reset nextProductionXXX to avoid walking down another Top Level production while what we are
+      // really seeking is the last Terminal...
+      this.nextProductionName = ""
+      this.nextProductionOccurrence = 0
+      this.isAtEndOfPath = true
+    } else {
+      this.nextProductionName = this.ruleStack.pop()
+      this.nextProductionOccurrence = this.occurrenceStack.pop()
+    }
+  }
+}
+
+export class NextAfterTokenWalker extends AbstractNextPossibleTokensWalker {
+  private nextTerminalName = ""
+  private nextTerminalOccurrence = 0
+
+  constructor(topProd: Rule, protected path: ITokenGrammarPath) {
+    super(topProd, path)
+    this.nextTerminalName = this.path.lastTok.name
+    this.nextTerminalOccurrence = this.path.lastTokOccurrence
+  }
+
+  walkTerminal(
+    terminal: Terminal,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (
+      this.isAtEndOfPath &&
+      terminal.terminalType.name === this.nextTerminalName &&
+      terminal.idx === this.nextTerminalOccurrence &&
+      !this.found
+    ) {
+      let fullRest = currRest.concat(prevRest)
+      let restProd = new Alternative({ definition: fullRest })
+      this.possibleTokTypes = first(restProd)
+      this.found = true
+    }
+  }
+}
+
+export type AlternativesFirstTokens = TokenType[][]
+
+export interface IFirstAfterRepetition {
+  token: TokenType
+  occurrence: number
+  isEndOfRule: boolean
+}
+
+/**
+ * This walker only "walks" a single "TOP" level in the Grammar Ast, this means
+ * it never "follows" production refs
+ */
+export class AbstractNextTerminalAfterProductionWalker extends RestWalker {
+  protected result = {
+    token: undefined,
+    occurrence: undefined,
+    isEndOfRule: undefined
+  }
+
+  constructor(protected topRule: Rule, protected occurrence: number) {
+    super()
+  }
+
+  startWalking(): IFirstAfterRepetition {
+    this.walk(this.topRule)
+    return this.result
+  }
+}
+
+export class NextTerminalAfterManyWalker extends AbstractNextTerminalAfterProductionWalker {
+  walkMany(
+    manyProd: Repetition,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (manyProd.idx === this.occurrence) {
+      let firstAfterMany = _first(currRest.concat(prevRest))
+      this.result.isEndOfRule = firstAfterMany === undefined
+      if (firstAfterMany instanceof Terminal) {
+        this.result.token = firstAfterMany.terminalType
+        this.result.occurrence = firstAfterMany.idx
+      }
+    } else {
+      super.walkMany(manyProd, currRest, prevRest)
+    }
+  }
+}
+
+export class NextTerminalAfterManySepWalker extends AbstractNextTerminalAfterProductionWalker {
+  walkManySep(
+    manySepProd: RepetitionWithSeparator,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (manySepProd.idx === this.occurrence) {
+      let firstAfterManySep = _first(currRest.concat(prevRest))
+      this.result.isEndOfRule = firstAfterManySep === undefined
+      if (firstAfterManySep instanceof Terminal) {
+        this.result.token = firstAfterManySep.terminalType
+        this.result.occurrence = firstAfterManySep.idx
+      }
+    } else {
+      super.walkManySep(manySepProd, currRest, prevRest)
+    }
+  }
+}
+
+export class NextTerminalAfterAtLeastOneWalker extends AbstractNextTerminalAfterProductionWalker {
+  walkAtLeastOne(
+    atLeastOneProd: RepetitionMandatory,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (atLeastOneProd.idx === this.occurrence) {
+      let firstAfterAtLeastOne = _first(currRest.concat(prevRest))
+      this.result.isEndOfRule = firstAfterAtLeastOne === undefined
+      if (firstAfterAtLeastOne instanceof Terminal) {
+        this.result.token = firstAfterAtLeastOne.terminalType
+        this.result.occurrence = firstAfterAtLeastOne.idx
+      }
+    } else {
+      super.walkAtLeastOne(atLeastOneProd, currRest, prevRest)
+    }
+  }
+}
+
+// TODO: reduce code duplication in the AfterWalkers
+export class NextTerminalAfterAtLeastOneSepWalker extends AbstractNextTerminalAfterProductionWalker {
+  walkAtLeastOneSep(
+    atleastOneSepProd: RepetitionMandatoryWithSeparator,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (atleastOneSepProd.idx === this.occurrence) {
+      let firstAfterfirstAfterAtLeastOneSep = _first(currRest.concat(prevRest))
+      this.result.isEndOfRule = firstAfterfirstAfterAtLeastOneSep === undefined
+      if (firstAfterfirstAfterAtLeastOneSep instanceof Terminal) {
+        this.result.token = firstAfterfirstAfterAtLeastOneSep.terminalType
+        this.result.occurrence = firstAfterfirstAfterAtLeastOneSep.idx
+      }
+    } else {
+      super.walkAtLeastOneSep(atleastOneSepProd, currRest, prevRest)
+    }
+  }
+}
+
+export interface PartialPathAndSuffixes {
+  partialPath: TokenType[]
+  suffixDef: IProduction[]
+}
+
+export function possiblePathsFrom(
+  targetDef: IProduction[],
+  maxLength: number,
+  currPath = []
+): PartialPathAndSuffixes[] {
+  // avoid side effects
+  currPath = cloneArr(currPath)
+  let result = []
+  let i = 0
+
+  // TODO: avoid inner funcs
+  function remainingPathWith(nextDef: IProduction[]) {
+    return nextDef.concat(drop(targetDef, i + 1))
+  }
+
+  // TODO: avoid inner funcs
+  function getAlternativesForProd(definition: IProduction[]) {
+    let alternatives = possiblePathsFrom(
+      remainingPathWith(definition),
+      maxLength,
+      currPath
+    )
+    return result.concat(alternatives)
+  }
+
+  /**
+   * Mandatory productions will halt the loop as the paths computed from their recursive calls will already contain the
+   * following (rest) of the targetDef.
+   *
+   * For optional productions (Option/Repetition/...) the loop will continue to represent the paths that do not include the
+   * the optional production.
+   */
+  while (currPath.length < maxLength && i < targetDef.length) {
+    let prod = targetDef[i]
+
+    /* istanbul ignore else */
+    if (prod instanceof Alternative) {
+      return getAlternativesForProd(prod.definition)
+    } else if (prod instanceof NonTerminal) {
+      return getAlternativesForProd(prod.definition)
+    } else if (prod instanceof Option) {
+      result = getAlternativesForProd(prod.definition)
+    } else if (prod instanceof RepetitionMandatory) {
+      const newDef = prod.definition.concat([
+        new Repetition({
+          definition: prod.definition
+        })
+      ])
+      return getAlternativesForProd(newDef)
+    } else if (prod instanceof RepetitionMandatoryWithSeparator) {
+      const newDef = [
+        new Alternative({ definition: prod.definition }),
+        new Repetition({
+          definition: [new Terminal({ terminalType: prod.separator })].concat(
+            <any>prod.definition
+          )
+        })
+      ]
+      return getAlternativesForProd(newDef)
+    } else if (prod instanceof RepetitionWithSeparator) {
+      const newDef = prod.definition.concat([
+        new Repetition({
+          definition: [new Terminal({ terminalType: prod.separator })].concat(
+            <any>prod.definition
+          )
+        })
+      ])
+      result = getAlternativesForProd(newDef)
+    } else if (prod instanceof Repetition) {
+      const newDef = prod.definition.concat([
+        new Repetition({
+          definition: prod.definition
+        })
+      ])
+      result = getAlternativesForProd(newDef)
+    } else if (prod instanceof Alternation) {
+      forEach(prod.definition, (currAlt) => {
+        // TODO: this is a limited check for empty alternatives
+        //   It would prevent a common case of infinite loops during parser initialization.
+        //   However **in-directly** empty alternatives may still cause issues.
+        if (isEmpty(currAlt.definition) === false) {
+          result = getAlternativesForProd(currAlt.definition)
+        }
+      })
+      return result
+    } else if (prod instanceof Terminal) {
+      currPath.push(prod.terminalType)
+    } else {
+      throw Error("non exhaustive match")
+    }
+
+    i++
+  }
+  result.push({
+    partialPath: currPath,
+    suffixDef: drop(targetDef, i)
+  })
+
+  return result
+}
+
+interface IPathToExamine {
+  idx: number
+  def: IProduction[]
+  ruleStack: string[]
+  occurrenceStack: number[]
+}
+
+export function nextPossibleTokensAfter(
+  initialDef: IProduction[],
+  tokenVector: IToken[],
+  tokMatcher: TokenMatcher,
+  maxLookAhead: number
+): ISyntacticContentAssistPath[] {
+  const EXIT_NON_TERMINAL: any = "EXIT_NONE_TERMINAL"
+  // to avoid creating a new Array each time.
+  const EXIT_NON_TERMINAL_ARR = [EXIT_NON_TERMINAL]
+  const EXIT_ALTERNATIVE: any = "EXIT_ALTERNATIVE"
+  let foundCompletePath = false
+
+  const tokenVectorLength = tokenVector.length
+  let minimalAlternativesIndex = tokenVectorLength - maxLookAhead - 1
+
+  let result: ISyntacticContentAssistPath[] = []
+
+  let possiblePaths: IPathToExamine[] = []
+  possiblePaths.push({
+    idx: -1,
+    def: initialDef,
+    ruleStack: [],
+    occurrenceStack: []
+  })
+
+  while (!isEmpty(possiblePaths)) {
+    let currPath = possiblePaths.pop()
+
+    // skip alternatives if no more results can be found (assuming deterministic grammar with fixed lookahead)
+    if (currPath === EXIT_ALTERNATIVE) {
+      if (
+        foundCompletePath &&
+        last(possiblePaths).idx <= minimalAlternativesIndex
+      ) {
+        // remove irrelevant alternative
+        possiblePaths.pop()
+      }
+      continue
+    }
+
+    let currDef = currPath.def
+    let currIdx = currPath.idx
+    let currRuleStack = currPath.ruleStack
+    let currOccurrenceStack = currPath.occurrenceStack
+
+    // For Example: an empty path could exist in a valid grammar in the case of an EMPTY_ALT
+    if (isEmpty(currDef)) {
+      continue
+    }
+
+    let prod = currDef[0]
+    /* istanbul ignore else */
+    if (prod === EXIT_NON_TERMINAL) {
+      let nextPath = {
+        idx: currIdx,
+        def: drop(currDef),
+        ruleStack: dropRight(currRuleStack),
+        occurrenceStack: dropRight(currOccurrenceStack)
+      }
+      possiblePaths.push(nextPath)
+    } else if (prod instanceof Terminal) {
+      /* istanbul ignore else */
+      if (currIdx < tokenVectorLength - 1) {
+        let nextIdx = currIdx + 1
+        let actualToken = tokenVector[nextIdx]
+        if (tokMatcher(actualToken, prod.terminalType)) {
+          let nextPath = {
+            idx: nextIdx,
+            def: drop(currDef),
+            ruleStack: currRuleStack,
+            occurrenceStack: currOccurrenceStack
+          }
+          possiblePaths.push(nextPath)
+        }
+        // end of the line
+      } else if (currIdx === tokenVectorLength - 1) {
+        // IGNORE ABOVE ELSE
+        result.push({
+          nextTokenType: prod.terminalType,
+          nextTokenOccurrence: prod.idx,
+          ruleStack: currRuleStack,
+          occurrenceStack: currOccurrenceStack
+        })
+        foundCompletePath = true
+      } else {
+        throw Error("non exhaustive match")
+      }
+    } else if (prod instanceof NonTerminal) {
+      let newRuleStack = cloneArr(currRuleStack)
+      newRuleStack.push(prod.nonTerminalName)
+
+      let newOccurrenceStack = cloneArr(currOccurrenceStack)
+      newOccurrenceStack.push(prod.idx)
+
+      let nextPath = {
+        idx: currIdx,
+        def: prod.definition.concat(EXIT_NON_TERMINAL_ARR, drop(currDef)),
+        ruleStack: newRuleStack,
+        occurrenceStack: newOccurrenceStack
+      }
+      possiblePaths.push(nextPath)
+    } else if (prod instanceof Option) {
+      // the order of alternatives is meaningful, FILO (Last path will be traversed first).
+      let nextPathWithout = {
+        idx: currIdx,
+        def: drop(currDef),
+        ruleStack: currRuleStack,
+        occurrenceStack: currOccurrenceStack
+      }
+      possiblePaths.push(nextPathWithout)
+      // required marker to avoid backtracking paths whose higher priority alternatives already matched
+      possiblePaths.push(EXIT_ALTERNATIVE)
+
+      let nextPathWith = {
+        idx: currIdx,
+        def: prod.definition.concat(drop(currDef)),
+        ruleStack: currRuleStack,
+        occurrenceStack: currOccurrenceStack
+      }
+      possiblePaths.push(nextPathWith)
+    } else if (prod instanceof RepetitionMandatory) {
+      // TODO:(THE NEW operators here take a while...) (convert once?)
+      let secondIteration = new Repetition({
+        definition: prod.definition,
+        idx: prod.idx
+      })
+      let nextDef = prod.definition.concat([secondIteration], drop(currDef))
+      let nextPath = {
+        idx: currIdx,
+        def: nextDef,
+        ruleStack: currRuleStack,
+        occurrenceStack: currOccurrenceStack
+      }
+      possiblePaths.push(nextPath)
+    } else if (prod instanceof RepetitionMandatoryWithSeparator) {
+      // TODO:(THE NEW operators here take a while...) (convert once?)
+      let separatorGast = new Terminal({
+        terminalType: prod.separator
+      })
+      let secondIteration = new Repetition({
+        definition: [<any>separatorGast].concat(prod.definition),
+        idx: prod.idx
+      })
+      let nextDef = prod.definition.concat([secondIteration], drop(currDef))
+      let nextPath = {
+        idx: currIdx,
+        def: nextDef,
+        ruleStack: currRuleStack,
+        occurrenceStack: currOccurrenceStack
+      }
+      possiblePaths.push(nextPath)
+    } else if (prod instanceof RepetitionWithSeparator) {
+      // the order of alternatives is meaningful, FILO (Last path will be traversed first).
+      let nextPathWithout = {
+        idx: currIdx,
+        def: drop(currDef),
+        ruleStack: currRuleStack,
+        occurrenceStack: currOccurrenceStack
+      }
+      possiblePaths.push(nextPathWithout)
+      // required marker to avoid backtracking paths whose higher priority alternatives already matched
+      possiblePaths.push(EXIT_ALTERNATIVE)
+
+      let separatorGast = new Terminal({
+        terminalType: prod.separator
+      })
+      let nthRepetition = new Repetition({
+        definition: [<any>separatorGast].concat(prod.definition),
+        idx: prod.idx
+      })
+      let nextDef = prod.definition.concat([nthRepetition], drop(currDef))
+      let nextPathWith = {
+        idx: currIdx,
+        def: nextDef,
+        ruleStack: currRuleStack,
+        occurrenceStack: currOccurrenceStack
+      }
+      possiblePaths.push(nextPathWith)
+    } else if (prod instanceof Repetition) {
+      // the order of alternatives is meaningful, FILO (Last path will be traversed first).
+      let nextPathWithout = {
+        idx: currIdx,
+        def: drop(currDef),
+        ruleStack: currRuleStack,
+        occurrenceStack: currOccurrenceStack
+      }
+      possiblePaths.push(nextPathWithout)
+      // required marker to avoid backtracking paths whose higher priority alternatives already matched
+      possiblePaths.push(EXIT_ALTERNATIVE)
+
+      // TODO: an empty repetition will cause infinite loops here, will the parser detect this in selfAnalysis?
+      let nthRepetition = new Repetition({
+        definition: prod.definition,
+        idx: prod.idx
+      })
+      let nextDef = prod.definition.concat([nthRepetition], drop(currDef))
+      let nextPathWith = {
+        idx: currIdx,
+        def: nextDef,
+        ruleStack: currRuleStack,
+        occurrenceStack: currOccurrenceStack
+      }
+      possiblePaths.push(nextPathWith)
+    } else if (prod instanceof Alternation) {
+      // the order of alternatives is meaningful, FILO (Last path will be traversed first).
+      for (let i = prod.definition.length - 1; i >= 0; i--) {
+        let currAlt: any = prod.definition[i]
+        let currAltPath = {
+          idx: currIdx,
+          def: currAlt.definition.concat(drop(currDef)),
+          ruleStack: currRuleStack,
+          occurrenceStack: currOccurrenceStack
+        }
+        possiblePaths.push(currAltPath)
+        possiblePaths.push(EXIT_ALTERNATIVE)
+      }
+    } else if (prod instanceof Alternative) {
+      possiblePaths.push({
+        idx: currIdx,
+        def: prod.definition.concat(drop(currDef)),
+        ruleStack: currRuleStack,
+        occurrenceStack: currOccurrenceStack
+      })
+    } else if (prod instanceof Rule) {
+      // last because we should only encounter at most a single one of these per invocation.
+      possiblePaths.push(
+        expandTopLevelRule(prod, currIdx, currRuleStack, currOccurrenceStack)
+      )
+    } else {
+      throw Error("non exhaustive match")
+    }
+  }
+  return result
+}
+
+function expandTopLevelRule(
+  topRule: Rule,
+  currIdx: number,
+  currRuleStack: string[],
+  currOccurrenceStack: number[]
+): IPathToExamine {
+  let newRuleStack = cloneArr(currRuleStack)
+  newRuleStack.push(topRule.name)
+
+  let newCurrOccurrenceStack = cloneArr(currOccurrenceStack)
+  // top rule is always assumed to have been called with occurrence index 1
+  newCurrOccurrenceStack.push(1)
+
+  return {
+    idx: currIdx,
+    def: topRule.definition,
+    ruleStack: newRuleStack,
+    occurrenceStack: newCurrOccurrenceStack
+  }
+}

node_modules/chevrotain/src/parse/grammar/keys.ts

@@ -0,0 +1,37 @@
+// Lookahead keys are 32Bit integers in the form
+// TTTTTTTT-ZZZZZZZZZZZZ-YYYY-XXXXXXXX
+// XXXX -> Occurrence Index bitmap.
+// YYYY -> DSL Method Type bitmap.
+// ZZZZZZZZZZZZZZZ -> Rule short Index bitmap.
+// TTTTTTTTT -> alternation alternative index bitmap
+
+export const BITS_FOR_METHOD_TYPE = 4
+export const BITS_FOR_OCCURRENCE_IDX = 8
+export const BITS_FOR_RULE_IDX = 12
+// TODO: validation, this means that there may at most 2^8 --> 256 alternatives for an alternation.
+export const BITS_FOR_ALT_IDX = 8
+
+// short string used as part of mapping keys.
+// being short improves the performance when composing KEYS for maps out of these
+// The 5 - 8 bits (16 possible values, are reserved for the DSL method indices)
+/* tslint:disable */
+export const OR_IDX = 1 << BITS_FOR_OCCURRENCE_IDX
+export const OPTION_IDX = 2 << BITS_FOR_OCCURRENCE_IDX
+export const MANY_IDX = 3 << BITS_FOR_OCCURRENCE_IDX
+export const AT_LEAST_ONE_IDX = 4 << BITS_FOR_OCCURRENCE_IDX
+export const MANY_SEP_IDX = 5 << BITS_FOR_OCCURRENCE_IDX
+export const AT_LEAST_ONE_SEP_IDX = 6 << BITS_FOR_OCCURRENCE_IDX
+/* tslint:enable */
+
+// this actually returns a number, but it is always used as a string (object prop key)
+export function getKeyForAutomaticLookahead(
+  ruleIdx: number,
+  dslMethodIdx: number,
+  occurrence: number
+): number {
+  /* tslint:disable */
+  return occurrence | dslMethodIdx | ruleIdx
+  /* tslint:enable */
+}
+
+const BITS_START_FOR_ALT_IDX = 32 - BITS_FOR_ALT_IDX

node_modules/chevrotain/src/parse/grammar/lookahead.ts

@@ -0,0 +1,695 @@
+import {
+  map,
+  reduce,
+  every,
+  isEmpty,
+  flatten,
+  forEach,
+  has
+} from "../../utils/utils"
+import { possiblePathsFrom } from "./interpreter"
+import { RestWalker } from "./rest"
+import { Predicate, TokenMatcher, lookAheadSequence } from "../parser/parser"
+import {
+  tokenStructuredMatcher,
+  tokenStructuredMatcherNoCategories
+} from "../../scan/tokens"
+import {
+  AbstractProduction,
+  Alternation,
+  Alternative as AlternativeGAST,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Rule
+} from "./gast/gast_public"
+import { GAstVisitor } from "./gast/gast_visitor_public"
+import {
+  IOrAlt,
+  IProduction,
+  IProductionWithOccurrence,
+  TokenType
+} from "../../../api"
+
+export enum PROD_TYPE {
+  OPTION,
+  REPETITION,
+  REPETITION_MANDATORY,
+  REPETITION_MANDATORY_WITH_SEPARATOR,
+  REPETITION_WITH_SEPARATOR,
+  ALTERNATION
+}
+
+export function getProdType(prod: IProduction): PROD_TYPE {
+  /* istanbul ignore else */
+  if (prod instanceof Option) {
+    return PROD_TYPE.OPTION
+  } else if (prod instanceof Repetition) {
+    return PROD_TYPE.REPETITION
+  } else if (prod instanceof RepetitionMandatory) {
+    return PROD_TYPE.REPETITION_MANDATORY
+  } else if (prod instanceof RepetitionMandatoryWithSeparator) {
+    return PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR
+  } else if (prod instanceof RepetitionWithSeparator) {
+    return PROD_TYPE.REPETITION_WITH_SEPARATOR
+  } else if (prod instanceof Alternation) {
+    return PROD_TYPE.ALTERNATION
+  } else {
+    throw Error("non exhaustive match")
+  }
+}
+
+export function buildLookaheadFuncForOr(
+  occurrence: number,
+  ruleGrammar: Rule,
+  maxLookahead: number,
+  hasPredicates: boolean,
+  dynamicTokensEnabled: boolean,
+  laFuncBuilder: Function
+): (orAlts?: IOrAlt<any>[]) => number {
+  let lookAheadPaths = getLookaheadPathsForOr(
+    occurrence,
+    ruleGrammar,
+    maxLookahead
+  )
+
+  const tokenMatcher = areTokenCategoriesNotUsed(lookAheadPaths)
+    ? tokenStructuredMatcherNoCategories
+    : tokenStructuredMatcher
+
+  return laFuncBuilder(
+    lookAheadPaths,
+    hasPredicates,
+    tokenMatcher,
+    dynamicTokensEnabled
+  )
+}
+
+/**
+ *  When dealing with an Optional production (OPTION/MANY/2nd iteration of AT_LEAST_ONE/...) we need to compare
+ *  the lookahead "inside" the production and the lookahead immediately "after" it in the same top level rule (context free).
+ *
+ *  Example: given a production:
+ *  ABC(DE)?DF
+ *
+ *  The optional '(DE)?' should only be entered if we see 'DE'. a single Token 'D' is not sufficient to distinguish between the two
+ *  alternatives.
+ *
+ *  @returns A Lookahead function which will return true IFF the parser should parse the Optional production.
+ */
+export function buildLookaheadFuncForOptionalProd(
+  occurrence: number,
+  ruleGrammar: Rule,
+  k: number,
+  dynamicTokensEnabled: boolean,
+  prodType: PROD_TYPE,
+  lookaheadBuilder: (lookAheadSequence, TokenMatcher, boolean) => () => boolean
+): () => boolean {
+  let lookAheadPaths = getLookaheadPathsForOptionalProd(
+    occurrence,
+    ruleGrammar,
+    prodType,
+    k
+  )
+
+  const tokenMatcher = areTokenCategoriesNotUsed(lookAheadPaths)
+    ? tokenStructuredMatcherNoCategories
+    : tokenStructuredMatcher
+
+  return lookaheadBuilder(lookAheadPaths[0], tokenMatcher, dynamicTokensEnabled)
+}
+
+export type Alternative = TokenType[][]
+
+export function buildAlternativesLookAheadFunc(
+  alts: lookAheadSequence[],
+  hasPredicates: boolean,
+  tokenMatcher: TokenMatcher,
+  dynamicTokensEnabled: boolean
+): (orAlts?: IOrAlt<any>[]) => number {
+  let numOfAlts = alts.length
+  let areAllOneTokenLookahead = every(alts, (currAlt) => {
+    return every(currAlt, (currPath) => {
+      return currPath.length === 1
+    })
+  })
+
+  // This version takes into account the predicates as well.
+  if (hasPredicates) {
+    /**
+     * @returns {number} - The chosen alternative index
+     */
+    return function (orAlts: IOrAlt<any>[]): number {
+      // unfortunately the predicates must be extracted every single time
+      // as they cannot be cached due to references to parameters(vars) which are no longer valid.
+      // note that in the common case of no predicates, no cpu time will be wasted on this (see else block)
+      let predicates: Predicate[] = map(orAlts, (currAlt) => currAlt.GATE)
+
+      for (let t = 0; t < numOfAlts; t++) {
+        let currAlt = alts[t]
+        let currNumOfPaths = currAlt.length
+
+        let currPredicate = predicates[t]
+        if (currPredicate !== undefined && currPredicate.call(this) === false) {
+          // if the predicate does not match there is no point in checking the paths
+          continue
+        }
+        nextPath: for (let j = 0; j < currNumOfPaths; j++) {
+          let currPath = currAlt[j]
+          let currPathLength = currPath.length
+          for (let i = 0; i < currPathLength; i++) {
+            let nextToken = this.LA(i + 1)
+            if (tokenMatcher(nextToken, currPath[i]) === false) {
+              // mismatch in current path
+              // try the next pth
+              continue nextPath
+            }
+          }
+          // found a full path that matches.
+          // this will also work for an empty ALT as the loop will be skipped
+          return t
+        }
+        // none of the paths for the current alternative matched
+        // try the next alternative
+      }
+      // none of the alternatives could be matched
+      return undefined
+    }
+  } else if (areAllOneTokenLookahead && !dynamicTokensEnabled) {
+    // optimized (common) case of all the lookaheads paths requiring only
+    // a single token lookahead. These Optimizations cannot work if dynamically defined Tokens are used.
+    let singleTokenAlts = map(alts, (currAlt) => {
+      return flatten(currAlt)
+    })
+
+    let choiceToAlt = reduce(
+      singleTokenAlts,
+      (result, currAlt, idx) => {
+        forEach(currAlt, (currTokType) => {
+          if (!has(result, currTokType.tokenTypeIdx)) {
+            result[currTokType.tokenTypeIdx] = idx
+          }
+          forEach(currTokType.categoryMatches, (currExtendingType) => {
+            if (!has(result, currExtendingType)) {
+              result[currExtendingType] = idx
+            }
+          })
+        })
+        return result
+      },
+      []
+    )
+
+    /**
+     * @returns {number} - The chosen alternative index
+     */
+    return function (): number {
+      let nextToken = this.LA(1)
+      return choiceToAlt[nextToken.tokenTypeIdx]
+    }
+  } else {
+    // optimized lookahead without needing to check the predicates at all.
+    // this causes code duplication which is intentional to improve performance.
+    /**
+     * @returns {number} - The chosen alternative index
+     */
+    return function (): number {
+      for (let t = 0; t < numOfAlts; t++) {
+        let currAlt = alts[t]
+        let currNumOfPaths = currAlt.length
+        nextPath: for (let j = 0; j < currNumOfPaths; j++) {
+          let currPath = currAlt[j]
+          let currPathLength = currPath.length
+          for (let i = 0; i < currPathLength; i++) {
+            let nextToken = this.LA(i + 1)
+            if (tokenMatcher(nextToken, currPath[i]) === false) {
+              // mismatch in current path
+              // try the next pth
+              continue nextPath
+            }
+          }
+          // found a full path that matches.
+          // this will also work for an empty ALT as the loop will be skipped
+          return t
+        }
+        // none of the paths for the current alternative matched
+        // try the next alternative
+      }
+      // none of the alternatives could be matched
+      return undefined
+    }
+  }
+}
+
+export function buildSingleAlternativeLookaheadFunction(
+  alt: lookAheadSequence,
+  tokenMatcher: TokenMatcher,
+  dynamicTokensEnabled: boolean
+): () => boolean {
+  let areAllOneTokenLookahead = every(alt, (currPath) => {
+    return currPath.length === 1
+  })
+
+  let numOfPaths = alt.length
+
+  // optimized (common) case of all the lookaheads paths requiring only
+  // a single token lookahead.
+  if (areAllOneTokenLookahead && !dynamicTokensEnabled) {
+    let singleTokensTypes = flatten(alt)
+
+    if (
+      singleTokensTypes.length === 1 &&
+      isEmpty((<any>singleTokensTypes[0]).categoryMatches)
+    ) {
+      let expectedTokenType = singleTokensTypes[0]
+      let expectedTokenUniqueKey = (<any>expectedTokenType).tokenTypeIdx
+
+      return function (): boolean {
+        return this.LA(1).tokenTypeIdx === expectedTokenUniqueKey
+      }
+    } else {
+      let choiceToAlt = reduce(
+        singleTokensTypes,
+        (result, currTokType, idx) => {
+          result[currTokType.tokenTypeIdx] = true
+          forEach(currTokType.categoryMatches, (currExtendingType) => {
+            result[currExtendingType] = true
+          })
+          return result
+        },
+        []
+      )
+
+      return function (): boolean {
+        let nextToken = this.LA(1)
+        return choiceToAlt[nextToken.tokenTypeIdx] === true
+      }
+    }
+  } else {
+    return function (): boolean {
+      nextPath: for (let j = 0; j < numOfPaths; j++) {
+        let currPath = alt[j]
+        let currPathLength = currPath.length
+        for (let i = 0; i < currPathLength; i++) {
+          let nextToken = this.LA(i + 1)
+          if (tokenMatcher(nextToken, currPath[i]) === false) {
+            // mismatch in current path
+            // try the next pth
+            continue nextPath
+          }
+        }
+        // found a full path that matches.
+        return true
+      }
+
+      // none of the paths matched
+      return false
+    }
+  }
+}
+
+class RestDefinitionFinderWalker extends RestWalker {
+  private restDef: IProduction[]
+
+  constructor(
+    private topProd: Rule,
+    private targetOccurrence: number,
+    private targetProdType: PROD_TYPE
+  ) {
+    super()
+  }
+
+  startWalking(): IProduction[] {
+    this.walk(this.topProd)
+    return this.restDef
+  }
+
+  private checkIsTarget(
+    node: AbstractProduction & IProductionWithOccurrence,
+    expectedProdType: PROD_TYPE,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): boolean {
+    if (
+      node.idx === this.targetOccurrence &&
+      this.targetProdType === expectedProdType
+    ) {
+      this.restDef = currRest.concat(prevRest)
+      return true
+    }
+    // performance optimization, do not iterate over the entire Grammar ast after we have found the target
+    return false
+  }
+
+  walkOption(
+    optionProd: Option,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (!this.checkIsTarget(optionProd, PROD_TYPE.OPTION, currRest, prevRest)) {
+      super.walkOption(optionProd, currRest, prevRest)
+    }
+  }
+
+  walkAtLeastOne(
+    atLeastOneProd: RepetitionMandatory,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (
+      !this.checkIsTarget(
+        atLeastOneProd,
+        PROD_TYPE.REPETITION_MANDATORY,
+        currRest,
+        prevRest
+      )
+    ) {
+      super.walkOption(atLeastOneProd, currRest, prevRest)
+    }
+  }
+
+  walkAtLeastOneSep(
+    atLeastOneSepProd: RepetitionMandatoryWithSeparator,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (
+      !this.checkIsTarget(
+        atLeastOneSepProd,
+        PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR,
+        currRest,
+        prevRest
+      )
+    ) {
+      super.walkOption(atLeastOneSepProd, currRest, prevRest)
+    }
+  }
+
+  walkMany(
+    manyProd: Repetition,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (
+      !this.checkIsTarget(manyProd, PROD_TYPE.REPETITION, currRest, prevRest)
+    ) {
+      super.walkOption(manyProd, currRest, prevRest)
+    }
+  }
+
+  walkManySep(
+    manySepProd: RepetitionWithSeparator,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    if (
+      !this.checkIsTarget(
+        manySepProd,
+        PROD_TYPE.REPETITION_WITH_SEPARATOR,
+        currRest,
+        prevRest
+      )
+    ) {
+      super.walkOption(manySepProd, currRest, prevRest)
+    }
+  }
+}
+
+/**
+ * Returns the definition of a target production in a top level level rule.
+ */
+class InsideDefinitionFinderVisitor extends GAstVisitor {
+  public result: IProduction[] = []
+
+  constructor(
+    private targetOccurrence: number,
+    private targetProdType: PROD_TYPE,
+    private targetRef?: any
+  ) {
+    super()
+  }
+
+  private checkIsTarget(
+    node: AbstractProduction & IProductionWithOccurrence,
+    expectedProdName: PROD_TYPE
+  ): void {
+    if (
+      node.idx === this.targetOccurrence &&
+      this.targetProdType === expectedProdName &&
+      (this.targetRef === undefined || node === this.targetRef)
+    ) {
+      this.result = node.definition
+    }
+  }
+
+  public visitOption(node: Option): void {
+    this.checkIsTarget(node, PROD_TYPE.OPTION)
+  }
+
+  public visitRepetition(node: Repetition): void {
+    this.checkIsTarget(node, PROD_TYPE.REPETITION)
+  }
+
+  public visitRepetitionMandatory(node: RepetitionMandatory): void {
+    this.checkIsTarget(node, PROD_TYPE.REPETITION_MANDATORY)
+  }
+
+  public visitRepetitionMandatoryWithSeparator(
+    node: RepetitionMandatoryWithSeparator
+  ): void {
+    this.checkIsTarget(node, PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR)
+  }
+
+  public visitRepetitionWithSeparator(node: RepetitionWithSeparator): void {
+    this.checkIsTarget(node, PROD_TYPE.REPETITION_WITH_SEPARATOR)
+  }
+
+  public visitAlternation(node: Alternation): void {
+    this.checkIsTarget(node, PROD_TYPE.ALTERNATION)
+  }
+}
+
+function initializeArrayOfArrays(size): any[][] {
+  let result = new Array(size)
+  for (let i = 0; i < size; i++) {
+    result[i] = []
+  }
+  return result
+}
+
+/**
+ * A sort of hash function between a Path in the grammar and a string.
+ * Note that this returns multiple "hashes" to support the scenario of token categories.
+ * -  A single path with categories may match multiple **actual** paths.
+ */
+function pathToHashKeys(path: TokenType[]): string[] {
+  let keys = [""]
+  for (let i = 0; i < path.length; i++) {
+    const tokType = path[i]
+    let longerKeys = []
+    for (let j = 0; j < keys.length; j++) {
+      const currShorterKey = keys[j]
+      longerKeys.push(currShorterKey + "_" + tokType.tokenTypeIdx)
+      for (let t = 0; t < tokType.categoryMatches.length; t++) {
+        const categoriesKeySuffix = "_" + tokType.categoryMatches[t]
+        longerKeys.push(currShorterKey + categoriesKeySuffix)
+      }
+    }
+    keys = longerKeys
+  }
+  return keys
+}
+
+/**
+ * Imperative style due to being called from a hot spot
+ */
+function isUniquePrefixHash(
+  altKnownPathsKeys: Record<string, boolean>[],
+  searchPathKeys: string[],
+  idx: number
+): boolean {
+  for (
+    let currAltIdx = 0;
+    currAltIdx < altKnownPathsKeys.length;
+    currAltIdx++
+  ) {
+    // We only want to test vs the other alternatives
+    if (currAltIdx === idx) {
+      continue
+    }
+    const otherAltKnownPathsKeys = altKnownPathsKeys[currAltIdx]
+    for (let searchIdx = 0; searchIdx < searchPathKeys.length; searchIdx++) {
+      const searchKey = searchPathKeys[searchIdx]
+      if (otherAltKnownPathsKeys[searchKey] === true) {
+        return false
+      }
+    }
+  }
+  // None of the SearchPathKeys were found in any of the other alternatives
+  return true
+}
+
+export function lookAheadSequenceFromAlternatives(
+  altsDefs: IProduction[],
+  k: number
+): lookAheadSequence[] {
+  let partialAlts = map(altsDefs, (currAlt) => possiblePathsFrom([currAlt], 1))
+  let finalResult = initializeArrayOfArrays(partialAlts.length)
+  const altsHashes = map(partialAlts, (currAltPaths) => {
+    const dict = {}
+    forEach(currAltPaths, (item) => {
+      const keys = pathToHashKeys(item.partialPath)
+      forEach(keys, (currKey) => {
+        dict[currKey] = true
+      })
+    })
+    return dict
+  })
+  let newData = partialAlts
+
+  // maxLookahead loop
+  for (let pathLength = 1; pathLength <= k; pathLength++) {
+    let currDataset = newData
+    newData = initializeArrayOfArrays(currDataset.length)
+
+    // alternatives loop
+    for (let altIdx = 0; altIdx < currDataset.length; altIdx++) {
+      let currAltPathsAndSuffixes = currDataset[altIdx]
+      // paths in current alternative loop
+      for (
+        let currPathIdx = 0;
+        currPathIdx < currAltPathsAndSuffixes.length;
+        currPathIdx++
+      ) {
+        let currPathPrefix = currAltPathsAndSuffixes[currPathIdx].partialPath
+        let suffixDef = currAltPathsAndSuffixes[currPathIdx].suffixDef
+        const prefixKeys = pathToHashKeys(currPathPrefix)
+        let isUnique = isUniquePrefixHash(altsHashes, prefixKeys, altIdx)
+        // End of the line for this path.
+        if (isUnique || isEmpty(suffixDef) || currPathPrefix.length === k) {
+          let currAltResult = finalResult[altIdx]
+          // TODO: Can we implement a containsPath using Maps/Dictionaries?
+          if (containsPath(currAltResult, currPathPrefix) === false) {
+            currAltResult.push(currPathPrefix)
+            // Update all new  keys for the current path.
+            for (let j = 0; j < prefixKeys.length; j++) {
+              const currKey = prefixKeys[j]
+              altsHashes[altIdx][currKey] = true
+            }
+          }
+        }
+        // Expand longer paths
+        else {
+          let newPartialPathsAndSuffixes = possiblePathsFrom(
+            suffixDef,
+            pathLength + 1,
+            currPathPrefix
+          )
+          newData[altIdx] = newData[altIdx].concat(newPartialPathsAndSuffixes)
+
+          // Update keys for new known paths
+          forEach(newPartialPathsAndSuffixes, (item) => {
+            const prefixKeys = pathToHashKeys(item.partialPath)
+            forEach(prefixKeys, (key) => {
+              altsHashes[altIdx][key] = true
+            })
+          })
+        }
+      }
+    }
+  }
+
+  return finalResult
+}
+
+export function getLookaheadPathsForOr(
+  occurrence: number,
+  ruleGrammar: Rule,
+  k: number,
+  orProd?: Alternation
+): lookAheadSequence[] {
+  const visitor = new InsideDefinitionFinderVisitor(
+    occurrence,
+    PROD_TYPE.ALTERNATION,
+    orProd
+  )
+  ruleGrammar.accept(visitor)
+  return lookAheadSequenceFromAlternatives(visitor.result, k)
+}
+
+export function getLookaheadPathsForOptionalProd(
+  occurrence: number,
+  ruleGrammar: Rule,
+  prodType: PROD_TYPE,
+  k: number
+): lookAheadSequence[] {
+  let insideDefVisitor = new InsideDefinitionFinderVisitor(occurrence, prodType)
+  ruleGrammar.accept(insideDefVisitor)
+  let insideDef = insideDefVisitor.result
+
+  let afterDefWalker = new RestDefinitionFinderWalker(
+    ruleGrammar,
+    occurrence,
+    prodType
+  )
+  let afterDef = afterDefWalker.startWalking()
+
+  let insideFlat = new AlternativeGAST({ definition: insideDef })
+  let afterFlat = new AlternativeGAST({ definition: afterDef })
+
+  return lookAheadSequenceFromAlternatives([insideFlat, afterFlat], k)
+}
+
+export function containsPath(
+  alternative: Alternative,
+  searchPath: TokenType[]
+): boolean {
+  compareOtherPath: for (let i = 0; i < alternative.length; i++) {
+    const otherPath = alternative[i]
+    if (otherPath.length !== searchPath.length) {
+      continue
+    }
+    for (let j = 0; j < otherPath.length; j++) {
+      const searchTok = searchPath[j]
+      const otherTok = otherPath[j]
+
+      const matchingTokens =
+        searchTok === otherTok ||
+        otherTok.categoryMatchesMap[searchTok.tokenTypeIdx] !== undefined
+      if (matchingTokens === false) {
+        continue compareOtherPath
+      }
+    }
+    return true
+  }
+
+  return false
+}
+
+export function isStrictPrefixOfPath(
+  prefix: TokenType[],
+  other: TokenType[]
+): boolean {
+  return (
+    prefix.length < other.length &&
+    every(prefix, (tokType, idx) => {
+      const otherTokType = other[idx]
+      return (
+        tokType === otherTokType ||
+        otherTokType.categoryMatchesMap[tokType.tokenTypeIdx]
+      )
+    })
+  )
+}
+
+export function areTokenCategoriesNotUsed(
+  lookAheadPaths: lookAheadSequence[]
+): boolean {
+  return every(lookAheadPaths, (singleAltPaths) =>
+    every(singleAltPaths, (singlePath) =>
+      every(singlePath, (token) => isEmpty(token.categoryMatches))
+    )
+  )
+}

node_modules/chevrotain/src/parse/grammar/resolver.ts

@@ -0,0 +1,58 @@
+import {
+  IParserUnresolvedRefDefinitionError,
+  ParserDefinitionErrorType
+} from "../parser/parser"
+import { forEach, values } from "../../utils/utils"
+import { NonTerminal, Rule } from "./gast/gast_public"
+import { GAstVisitor } from "./gast/gast_visitor_public"
+import {
+  IGrammarResolverErrorMessageProvider,
+  IParserDefinitionError
+} from "../../../api"
+
+export function resolveGrammar(
+  topLevels: Record<string, Rule>,
+  errMsgProvider: IGrammarResolverErrorMessageProvider
+): IParserDefinitionError[] {
+  let refResolver = new GastRefResolverVisitor(topLevels, errMsgProvider)
+  refResolver.resolveRefs()
+  return refResolver.errors
+}
+
+export class GastRefResolverVisitor extends GAstVisitor {
+  public errors: IParserUnresolvedRefDefinitionError[] = []
+  private currTopLevel: Rule
+
+  constructor(
+    private nameToTopRule: Record<string, Rule>,
+    private errMsgProvider: IGrammarResolverErrorMessageProvider
+  ) {
+    super()
+  }
+
+  public resolveRefs(): void {
+    forEach(values(this.nameToTopRule), (prod) => {
+      this.currTopLevel = prod
+      prod.accept(this)
+    })
+  }
+
+  public visitNonTerminal(node: NonTerminal): void {
+    let ref = this.nameToTopRule[node.nonTerminalName]
+
+    if (!ref) {
+      let msg = this.errMsgProvider.buildRuleNotFoundError(
+        this.currTopLevel,
+        node
+      )
+      this.errors.push({
+        message: msg,
+        type: ParserDefinitionErrorType.UNRESOLVED_SUBRULE_REF,
+        ruleName: this.currTopLevel.name,
+        unresolvedRefName: node.nonTerminalName
+      })
+    } else {
+      node.referencedRule = ref
+    }
+  }
+}

node_modules/chevrotain/src/parse/grammar/rest.ts

@@ -0,0 +1,163 @@
+import { drop, forEach } from "../../utils/utils"
+import {
+  AbstractProduction,
+  Alternation,
+  Alternative,
+  NonTerminal,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Terminal
+} from "./gast/gast_public"
+import { IProduction } from "../../../api"
+
+/**
+ *  A Grammar Walker that computes the "remaining" grammar "after" a productions in the grammar.
+ */
+export abstract class RestWalker {
+  walk(prod: AbstractProduction, prevRest: any[] = []): void {
+    forEach(prod.definition, (subProd: IProduction, index) => {
+      let currRest = drop(prod.definition, index + 1)
+      /* istanbul ignore else */
+      if (subProd instanceof NonTerminal) {
+        this.walkProdRef(subProd, currRest, prevRest)
+      } else if (subProd instanceof Terminal) {
+        this.walkTerminal(subProd, currRest, prevRest)
+      } else if (subProd instanceof Alternative) {
+        this.walkFlat(subProd, currRest, prevRest)
+      } else if (subProd instanceof Option) {
+        this.walkOption(subProd, currRest, prevRest)
+      } else if (subProd instanceof RepetitionMandatory) {
+        this.walkAtLeastOne(subProd, currRest, prevRest)
+      } else if (subProd instanceof RepetitionMandatoryWithSeparator) {
+        this.walkAtLeastOneSep(subProd, currRest, prevRest)
+      } else if (subProd instanceof RepetitionWithSeparator) {
+        this.walkManySep(subProd, currRest, prevRest)
+      } else if (subProd instanceof Repetition) {
+        this.walkMany(subProd, currRest, prevRest)
+      } else if (subProd instanceof Alternation) {
+        this.walkOr(subProd, currRest, prevRest)
+      } else {
+        throw Error("non exhaustive match")
+      }
+    })
+  }
+
+  walkTerminal(
+    terminal: Terminal,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {}
+
+  walkProdRef(
+    refProd: NonTerminal,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {}
+
+  walkFlat(
+    flatProd: Alternative,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    // ABCDEF => after the D the rest is EF
+    let fullOrRest = currRest.concat(prevRest)
+    this.walk(flatProd, <any>fullOrRest)
+  }
+
+  walkOption(
+    optionProd: Option,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    // ABC(DE)?F => after the (DE)? the rest is F
+    let fullOrRest = currRest.concat(prevRest)
+    this.walk(optionProd, <any>fullOrRest)
+  }
+
+  walkAtLeastOne(
+    atLeastOneProd: RepetitionMandatory,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    // ABC(DE)+F => after the (DE)+ the rest is (DE)?F
+    let fullAtLeastOneRest: IProduction[] = [
+      new Option({ definition: atLeastOneProd.definition })
+    ].concat(<any>currRest, <any>prevRest)
+    this.walk(atLeastOneProd, fullAtLeastOneRest)
+  }
+
+  walkAtLeastOneSep(
+    atLeastOneSepProd: RepetitionMandatoryWithSeparator,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    // ABC DE(,DE)* F => after the (,DE)+ the rest is (,DE)?F
+    let fullAtLeastOneSepRest = restForRepetitionWithSeparator(
+      atLeastOneSepProd,
+      currRest,
+      prevRest
+    )
+    this.walk(atLeastOneSepProd, fullAtLeastOneSepRest)
+  }
+
+  walkMany(
+    manyProd: Repetition,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    // ABC(DE)*F => after the (DE)* the rest is (DE)?F
+    let fullManyRest: IProduction[] = [
+      new Option({ definition: manyProd.definition })
+    ].concat(<any>currRest, <any>prevRest)
+    this.walk(manyProd, fullManyRest)
+  }
+
+  walkManySep(
+    manySepProd: RepetitionWithSeparator,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    // ABC (DE(,DE)*)? F => after the (,DE)* the rest is (,DE)?F
+    let fullManySepRest = restForRepetitionWithSeparator(
+      manySepProd,
+      currRest,
+      prevRest
+    )
+    this.walk(manySepProd, fullManySepRest)
+  }
+
+  walkOr(
+    orProd: Alternation,
+    currRest: IProduction[],
+    prevRest: IProduction[]
+  ): void {
+    // ABC(D|E|F)G => when finding the (D|E|F) the rest is G
+    let fullOrRest = currRest.concat(prevRest)
+    // walk all different alternatives
+    forEach(orProd.definition, (alt) => {
+      // wrapping each alternative in a single definition wrapper
+      // to avoid errors in computing the rest of that alternative in the invocation to computeInProdFollows
+      // (otherwise for OR([alt1,alt2]) alt2 will be considered in 'rest' of alt1
+      let prodWrapper = new Alternative({ definition: [alt] })
+      this.walk(prodWrapper, <any>fullOrRest)
+    })
+  }
+}
+
+function restForRepetitionWithSeparator(repSepProd, currRest, prevRest) {
+  let repSepRest = [
+    new Option({
+      definition: [new Terminal({ terminalType: repSepProd.separator })].concat(
+        repSepProd.definition
+      )
+    })
+  ]
+  let fullRepSepRest: IProduction[] = repSepRest.concat(
+    <any>currRest,
+    <any>prevRest
+  )
+  return fullRepSepRest
+}

node_modules/chevrotain/src/parse/parser/parser.ts

@@ -0,0 +1,310 @@
+import {
+  applyMixins,
+  cloneObj,
+  forEach,
+  has,
+  isEmpty,
+  map,
+  toFastProperties,
+  values
+} from "../../utils/utils"
+import { computeAllProdsFollows } from "../grammar/follow"
+import { createTokenInstance, EOF } from "../../scan/tokens_public"
+import {
+  defaultGrammarValidatorErrorProvider,
+  defaultParserErrorProvider
+} from "../errors_public"
+import {
+  resolveGrammar,
+  validateGrammar
+} from "../grammar/gast/gast_resolver_public"
+import {
+  CstNode,
+  IParserConfig,
+  IParserDefinitionError,
+  IRecognitionException,
+  IRuleConfig,
+  IToken,
+  TokenType,
+  TokenVocabulary
+} from "../../../api"
+import { Recoverable } from "./traits/recoverable"
+import { LooksAhead } from "./traits/looksahead"
+import { TreeBuilder } from "./traits/tree_builder"
+import { LexerAdapter } from "./traits/lexer_adapter"
+import { RecognizerApi } from "./traits/recognizer_api"
+import { RecognizerEngine } from "./traits/recognizer_engine"
+
+import { ErrorHandler } from "./traits/error_handler"
+import { MixedInParser } from "./traits/parser_traits"
+import { ContentAssist } from "./traits/context_assist"
+import { GastRecorder } from "./traits/gast_recorder"
+import { PerformanceTracer } from "./traits/perf_tracer"
+
+export const END_OF_FILE = createTokenInstance(
+  EOF,
+  "",
+  NaN,
+  NaN,
+  NaN,
+  NaN,
+  NaN,
+  NaN
+)
+Object.freeze(END_OF_FILE)
+
+export type TokenMatcher = (token: IToken, tokType: TokenType) => boolean
+
+export type lookAheadSequence = TokenType[][]
+
+export const DEFAULT_PARSER_CONFIG: IParserConfig = Object.freeze({
+  recoveryEnabled: false,
+  maxLookahead: 3,
+  dynamicTokensEnabled: false,
+  outputCst: true,
+  errorMessageProvider: defaultParserErrorProvider,
+  nodeLocationTracking: "none",
+  traceInitPerf: false,
+  skipValidations: false
+})
+
+export const DEFAULT_RULE_CONFIG: IRuleConfig<any> = Object.freeze({
+  recoveryValueFunc: () => undefined,
+  resyncEnabled: true
+})
+
+export enum ParserDefinitionErrorType {
+  INVALID_RULE_NAME = 0,
+  DUPLICATE_RULE_NAME = 1,
+  INVALID_RULE_OVERRIDE = 2,
+  DUPLICATE_PRODUCTIONS = 3,
+  UNRESOLVED_SUBRULE_REF = 4,
+  LEFT_RECURSION = 5,
+  NONE_LAST_EMPTY_ALT = 6,
+  AMBIGUOUS_ALTS = 7,
+  CONFLICT_TOKENS_RULES_NAMESPACE = 8,
+  INVALID_TOKEN_NAME = 9,
+  NO_NON_EMPTY_LOOKAHEAD = 10,
+  AMBIGUOUS_PREFIX_ALTS = 11,
+  TOO_MANY_ALTS = 12
+}
+
+export interface IParserDuplicatesDefinitionError
+  extends IParserDefinitionError {
+  dslName: string
+  occurrence: number
+  parameter?: string
+}
+
+export interface IParserEmptyAlternativeDefinitionError
+  extends IParserDefinitionError {
+  occurrence: number
+  alternative: number
+}
+
+export interface IParserAmbiguousAlternativesDefinitionError
+  extends IParserDefinitionError {
+  occurrence: number
+  alternatives: number[]
+}
+
+export interface IParserUnresolvedRefDefinitionError
+  extends IParserDefinitionError {
+  unresolvedRefName: string
+}
+
+export interface IParserState {
+  errors: IRecognitionException[]
+  lexerState: any
+  RULE_STACK: string[]
+  CST_STACK: CstNode[]
+}
+
+export type Predicate = () => boolean
+
+export function EMPTY_ALT<T>(value: T = undefined): () => T {
+  return function () {
+    return value
+  }
+}
+
+export class Parser {
+  // Set this flag to true if you don't want the Parser to throw error when problems in it's definition are detected.
+  // (normally during the parser's constructor).
+  // This is a design time flag, it will not affect the runtime error handling of the parser, just design time errors,
+  // for example: duplicate rule names, referencing an unresolved subrule, ect...
+  // This flag should not be enabled during normal usage, it is used in special situations, for example when
+  // needing to display the parser definition errors in some GUI(online playground).
+  static DEFER_DEFINITION_ERRORS_HANDLING: boolean = false
+
+  /**
+   *  @deprecated use the **instance** method with the same name instead
+   */
+  static performSelfAnalysis(parserInstance: Parser): void {
+    throw Error(
+      "The **static** `performSelfAnalysis` method has been deprecated." +
+        "\t\nUse the **instance** method with the same name instead."
+    )
+  }
+
+  public performSelfAnalysis(this: MixedInParser): void {
+    this.TRACE_INIT("performSelfAnalysis", () => {
+      let defErrorsMsgs
+
+      this.selfAnalysisDone = true
+      let className = this.className
+
+      this.TRACE_INIT("toFastProps", () => {
+        // Without this voodoo magic the parser would be x3-x4 slower
+        // It seems it is better to invoke `toFastProperties` **before**
+        // Any manipulations of the `this` object done during the recording phase.
+        toFastProperties(this)
+      })
+
+      this.TRACE_INIT("Grammar Recording", () => {
+        try {
+          this.enableRecording()
+          // Building the GAST
+          forEach(this.definedRulesNames, (currRuleName) => {
+            const wrappedRule = this[currRuleName]
+            const originalGrammarAction = wrappedRule["originalGrammarAction"]
+            let recordedRuleGast = undefined
+            this.TRACE_INIT(`${currRuleName} Rule`, () => {
+              recordedRuleGast = this.topLevelRuleRecord(
+                currRuleName,
+                originalGrammarAction
+              )
+            })
+            this.gastProductionsCache[currRuleName] = recordedRuleGast
+          })
+        } finally {
+          this.disableRecording()
+        }
+      })
+
+      let resolverErrors = []
+      this.TRACE_INIT("Grammar Resolving", () => {
+        resolverErrors = resolveGrammar({
+          rules: values(this.gastProductionsCache)
+        })
+        this.definitionErrors.push.apply(this.definitionErrors, resolverErrors) // mutability for the win?
+      })
+
+      this.TRACE_INIT("Grammar Validations", () => {
+        // only perform additional grammar validations IFF no resolving errors have occurred.
+        // as unresolved grammar may lead to unhandled runtime exceptions in the follow up validations.
+        if (isEmpty(resolverErrors) && this.skipValidations === false) {
+          let validationErrors = validateGrammar({
+            rules: values(this.gastProductionsCache),
+            maxLookahead: this.maxLookahead,
+            tokenTypes: values(this.tokensMap),
+            errMsgProvider: defaultGrammarValidatorErrorProvider,
+            grammarName: className
+          })
+
+          this.definitionErrors.push.apply(
+            this.definitionErrors,
+            validationErrors
+          ) // mutability for the win?
+        }
+      })
+
+      // this analysis may fail if the grammar is not perfectly valid
+      if (isEmpty(this.definitionErrors)) {
+        // The results of these computations are not needed unless error recovery is enabled.
+        if (this.recoveryEnabled) {
+          this.TRACE_INIT("computeAllProdsFollows", () => {
+            let allFollows = computeAllProdsFollows(
+              values(this.gastProductionsCache)
+            )
+            this.resyncFollows = allFollows
+          })
+        }
+
+        this.TRACE_INIT("ComputeLookaheadFunctions", () => {
+          this.preComputeLookaheadFunctions(values(this.gastProductionsCache))
+        })
+      }
+
+      if (
+        !Parser.DEFER_DEFINITION_ERRORS_HANDLING &&
+        !isEmpty(this.definitionErrors)
+      ) {
+        defErrorsMsgs = map(
+          this.definitionErrors,
+          (defError) => defError.message
+        )
+        throw new Error(
+          `Parser Definition Errors detected:\n ${defErrorsMsgs.join(
+            "\n-------------------------------\n"
+          )}`
+        )
+      }
+    })
+  }
+
+  definitionErrors: IParserDefinitionError[] = []
+  selfAnalysisDone = false
+  protected skipValidations: boolean
+
+  constructor(tokenVocabulary: TokenVocabulary, config: IParserConfig) {
+    const that: MixedInParser = this as any
+    that.initErrorHandler(config)
+    that.initLexerAdapter()
+    that.initLooksAhead(config)
+    that.initRecognizerEngine(tokenVocabulary, config)
+    that.initRecoverable(config)
+    that.initTreeBuilder(config)
+    that.initContentAssist()
+    that.initGastRecorder(config)
+    that.initPerformanceTracer(config)
+
+    if (has(config, "ignoredIssues")) {
+      throw new Error(
+        "The <ignoredIssues> IParserConfig property has been deprecated.\n\t" +
+          "Please use the <IGNORE_AMBIGUITIES> flag on the relevant DSL method instead.\n\t" +
+          "See: https://sap.github.io/chevrotain/docs/guide/resolving_grammar_errors.html#IGNORING_AMBIGUITIES\n\t" +
+          "For further details."
+      )
+    }
+
+    this.skipValidations = has(config, "skipValidations")
+      ? config.skipValidations
+      : DEFAULT_PARSER_CONFIG.skipValidations
+  }
+}
+
+applyMixins(Parser, [
+  Recoverable,
+  LooksAhead,
+  TreeBuilder,
+  LexerAdapter,
+  RecognizerEngine,
+  RecognizerApi,
+  ErrorHandler,
+  ContentAssist,
+  GastRecorder,
+  PerformanceTracer
+])
+
+export class CstParser extends Parser {
+  constructor(
+    tokenVocabulary: TokenVocabulary,
+    config: IParserConfig = DEFAULT_PARSER_CONFIG
+  ) {
+    const configClone = cloneObj(config)
+    configClone.outputCst = true
+    super(tokenVocabulary, configClone)
+  }
+}
+
+export class EmbeddedActionsParser extends Parser {
+  constructor(
+    tokenVocabulary: TokenVocabulary,
+    config: IParserConfig = DEFAULT_PARSER_CONFIG
+  ) {
+    const configClone = cloneObj(config)
+    configClone.outputCst = false
+    super(tokenVocabulary, configClone)
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/README.md

@@ -0,0 +1,71 @@
+### Parser Traits (mixins)
+
+The Chevrotain Parser class is implemented as multiple classes mixed-in (combined) together
+to provide the required functionality.
+
+A mix-in approach has been chosen to:
+
+1. Split up the large (~3,000 LOC) Parser Class into smaller files.
+   - Similar to C# [Partial Classes](https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/classes-and-structs/partial-classes-and-methods)
+2. Avoid performance regressions of the common "classic" Object oriented pattern of composition & delegation.
+   - For example, consider: `this.LA()` vs `this.traitX.LA()`
+   - Past attempts at extracting an API for Lexer Adaptors using composition have [shown](https://github.com/SAP/chevrotain/issues/528#issuecomment-313863665)
+     significant performance regressions.
+
+### Background
+
+The mixin pattern used here is derived from the mixins pattern found in the [TypeScript Handbook](https://www.typescriptlang.org/docs/handbook/mixins.html).
+There are a few Issues With The TypeScript Handbook Pattern:
+
+- Duplicate declarations of the instance methods.
+- Missing scenario of interaction between different mixins included by the same class.
+  - This scenario is more similar to partial classes.
+- Instance fields initialization would not be executed.
+  - As instance fields are copied to the constructor by the TypeScript compiler
+    and only one constructor would get invoked.
+
+Therefor a slightly modified pattern has been used.
+
+### The "Upgraded" Pattern
+
+The building blocks are as follows:
+
+- **Define the full combined type**
+  - by using Intersection Types to define the complete Type (after all the mixins).
+  - [Source Snippet](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/traits/parser_traits.ts#L20-L28)
+- **Make every method aware of its full execution context**
+  - By specifying the type of "this" context in methods as the "full combined type" Type
+    to allow "interaction" between different mixed-ins of the same class.
+  - [e.g calling a method from another trait](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/traits/recognizer_api.ts#L35-L41)
+- **Define state initialization for each trait/mixin**?:
+  - By defining init methods for each trait/mixin which would be called during the combined type initialization.
+  - [A single trait's initializer definition](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/traits/lexer_adapter.ts#L17-L21)
+- **Invoke the state initialization in the combined class**
+  - By using a type assertion in the main class constructor to enable calling these init methods.
+  - [Invoking the init methods](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/parser.ts#L225-L232)
+- **Ensuring alignment with the public API**
+  - By using a dummy assignment statement to leverge the TypeScript compiler to ensure the internal Parser implementation matches
+    the exposed public API of Chevrotain.
+  - [Dummy assignment](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/api.ts#L193-L197)
+- **Enriching the combined class's prototype**
+  - [Invoking ApplyMixings](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/parser.ts#L240-L249)
+  - [Upgraded ApplyMixings with setter/getter handling](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/utils/utils.ts#L433-L460)
+
+* Pros
+
+  - Avoid duplication.
+  - Allows splitting up large classes to multiple files if/when class composition is not appropriate.
+    - a.k.a ["partial classes"](https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/classes-and-structs/partial-classes-and-methods).
+
+* Cons
+  - Breaks the semantics of TypeScript a bit, What the compiler knows about "SmartObject"
+    is no longer the "full story".
+
+### References
+
+- [Wikipedia Article on Mixins](https://en.wikipedia.org/wiki/Mixin)
+- [Trait Linearization in Scala](https://www.trivento.io/trait-linearization/)
+
+### To Investigate
+
+Would TypeScript 2.2 enabled a simpler & clearer pattern by using its support for ["mixin classes"](https://github.com/Microsoft/TypeScript/wiki/What%27s-new-in-TypeScript#support-for-mix-in-classes)?

node_modules/chevrotain/src/parse/parser/traits/context_assist.ts

@@ -0,0 +1,51 @@
+import {
+  ISyntacticContentAssistPath,
+  IToken,
+  ITokenGrammarPath,
+  TokenType
+} from "../../../../api"
+import {
+  NextAfterTokenWalker,
+  nextPossibleTokensAfter
+} from "../../grammar/interpreter"
+import { first, isUndefined } from "../../../utils/utils"
+import { MixedInParser } from "./parser_traits"
+
+export class ContentAssist {
+  initContentAssist() {}
+
+  public computeContentAssist(
+    this: MixedInParser,
+    startRuleName: string,
+    precedingInput: IToken[]
+  ): ISyntacticContentAssistPath[] {
+    let startRuleGast = this.gastProductionsCache[startRuleName]
+
+    if (isUndefined(startRuleGast)) {
+      throw Error(`Rule ->${startRuleName}<- does not exist in this grammar.`)
+    }
+
+    return nextPossibleTokensAfter(
+      [startRuleGast],
+      precedingInput,
+      this.tokenMatcher,
+      this.maxLookahead
+    )
+  }
+
+  // TODO: should this be a member method or a utility? it does not have any state or usage of 'this'...
+  // TODO: should this be more explicitly part of the public API?
+  public getNextPossibleTokenTypes(
+    this: MixedInParser,
+    grammarPath: ITokenGrammarPath
+  ): TokenType[] {
+    let topRuleName = first(grammarPath.ruleStack)
+    let gastProductions = this.getGAstProductions()
+    let topProduction = gastProductions[topRuleName]
+    let nextPossibleTokenTypes = new NextAfterTokenWalker(
+      topProduction,
+      grammarPath
+    ).startWalking()
+    return nextPossibleTokenTypes
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/error_handler.ts

@@ -0,0 +1,122 @@
+import {
+  IParserConfig,
+  IParserErrorMessageProvider,
+  IRecognitionException
+} from "../../../../api"
+import {
+  EarlyExitException,
+  isRecognitionException,
+  NoViableAltException
+} from "../../exceptions_public"
+import { cloneArr, has } from "../../../utils/utils"
+import {
+  getLookaheadPathsForOptionalProd,
+  getLookaheadPathsForOr,
+  PROD_TYPE
+} from "../../grammar/lookahead"
+import { MixedInParser } from "./parser_traits"
+import { DEFAULT_PARSER_CONFIG } from "../parser"
+
+/**
+ * Trait responsible for runtime parsing errors.
+ */
+export class ErrorHandler {
+  _errors: IRecognitionException[]
+  errorMessageProvider: IParserErrorMessageProvider
+
+  initErrorHandler(config: IParserConfig) {
+    this._errors = []
+    this.errorMessageProvider = has(config, "errorMessageProvider")
+      ? config.errorMessageProvider
+      : DEFAULT_PARSER_CONFIG.errorMessageProvider
+  }
+
+  SAVE_ERROR(
+    this: MixedInParser,
+    error: IRecognitionException
+  ): IRecognitionException {
+    if (isRecognitionException(error)) {
+      error.context = {
+        ruleStack: this.getHumanReadableRuleStack(),
+        ruleOccurrenceStack: cloneArr(this.RULE_OCCURRENCE_STACK)
+      }
+      this._errors.push(error)
+      return error
+    } else {
+      throw Error("Trying to save an Error which is not a RecognitionException")
+    }
+  }
+
+  get errors(): IRecognitionException[] {
+    return cloneArr(this._errors)
+  }
+
+  set errors(newErrors: IRecognitionException[]) {
+    this._errors = newErrors
+  }
+
+  // TODO: consider caching the error message computed information
+  raiseEarlyExitException(
+    this: MixedInParser,
+    occurrence: number,
+    prodType: PROD_TYPE,
+    userDefinedErrMsg: string
+  ): void {
+    let ruleName = this.getCurrRuleFullName()
+    let ruleGrammar = this.getGAstProductions()[ruleName]
+    let lookAheadPathsPerAlternative = getLookaheadPathsForOptionalProd(
+      occurrence,
+      ruleGrammar,
+      prodType,
+      this.maxLookahead
+    )
+    let insideProdPaths = lookAheadPathsPerAlternative[0]
+    let actualTokens = []
+    for (let i = 1; i <= this.maxLookahead; i++) {
+      actualTokens.push(this.LA(i))
+    }
+    let msg = this.errorMessageProvider.buildEarlyExitMessage({
+      expectedIterationPaths: insideProdPaths,
+      actual: actualTokens,
+      previous: this.LA(0),
+      customUserDescription: userDefinedErrMsg,
+      ruleName: ruleName
+    })
+
+    throw this.SAVE_ERROR(new EarlyExitException(msg, this.LA(1), this.LA(0)))
+  }
+
+  // TODO: consider caching the error message computed information
+  raiseNoAltException(
+    this: MixedInParser,
+    occurrence: number,
+    errMsgTypes: string
+  ): void {
+    let ruleName = this.getCurrRuleFullName()
+    let ruleGrammar = this.getGAstProductions()[ruleName]
+    // TODO: getLookaheadPathsForOr can be slow for large enough maxLookahead and certain grammars, consider caching ?
+    let lookAheadPathsPerAlternative = getLookaheadPathsForOr(
+      occurrence,
+      ruleGrammar,
+      this.maxLookahead
+    )
+
+    let actualTokens = []
+    for (let i = 1; i <= this.maxLookahead; i++) {
+      actualTokens.push(this.LA(i))
+    }
+    let previousToken = this.LA(0)
+
+    let errMsg = this.errorMessageProvider.buildNoViableAltMessage({
+      expectedPathsPerAlt: lookAheadPathsPerAlternative,
+      actual: actualTokens,
+      previous: previousToken,
+      customUserDescription: errMsgTypes,
+      ruleName: this.getCurrRuleFullName()
+    })
+
+    throw this.SAVE_ERROR(
+      new NoViableAltException(errMsg, this.LA(1), previousToken)
+    )
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/gast_recorder.ts

@@ -0,0 +1,443 @@
+import {
+  AtLeastOneSepMethodOpts,
+  ConsumeMethodOpts,
+  CstNode,
+  DSLMethodOpts,
+  DSLMethodOptsWithErr,
+  GrammarAction,
+  IOrAlt,
+  IParserConfig,
+  IProduction,
+  IToken,
+  ManySepMethodOpts,
+  OrMethodOpts,
+  SubruleMethodOpts,
+  TokenType
+} from "../../../../api"
+import {
+  forEach,
+  has,
+  isArray,
+  isFunction,
+  peek,
+  some
+} from "../../../utils/utils"
+import { MixedInParser } from "./parser_traits"
+import {
+  Alternation,
+  Alternative,
+  NonTerminal,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Rule,
+  Terminal
+} from "../../grammar/gast/gast_public"
+import { Lexer } from "../../../scan/lexer_public"
+import { augmentTokenTypes, hasShortKeyProperty } from "../../../scan/tokens"
+import { createToken, createTokenInstance } from "../../../scan/tokens_public"
+import { END_OF_FILE } from "../parser"
+import { BITS_FOR_OCCURRENCE_IDX } from "../../grammar/keys"
+
+type ProdWithDef = IProduction & { definition?: IProduction[] }
+const RECORDING_NULL_OBJECT = {
+  description: "This Object indicates the Parser is during Recording Phase"
+}
+Object.freeze(RECORDING_NULL_OBJECT)
+
+const HANDLE_SEPARATOR = true
+const MAX_METHOD_IDX = Math.pow(2, BITS_FOR_OCCURRENCE_IDX) - 1
+
+const RFT = createToken({ name: "RECORDING_PHASE_TOKEN", pattern: Lexer.NA })
+augmentTokenTypes([RFT])
+const RECORDING_PHASE_TOKEN = createTokenInstance(
+  RFT,
+  "This IToken indicates the Parser is in Recording Phase\n\t" +
+    "" +
+    "See: https://sap.github.io/chevrotain/docs/guide/internals.html#grammar-recording for details",
+  // Using "-1" instead of NaN (as in EOF) because an actual number is less likely to
+  // cause errors if the output of LA or CONSUME would be (incorrectly) used during the recording phase.
+  -1,
+  -1,
+  -1,
+  -1,
+  -1,
+  -1
+)
+Object.freeze(RECORDING_PHASE_TOKEN)
+
+const RECORDING_PHASE_CSTNODE: CstNode = {
+  name:
+    "This CSTNode indicates the Parser is in Recording Phase\n\t" +
+    "See: https://sap.github.io/chevrotain/docs/guide/internals.html#grammar-recording for details",
+  children: {}
+}
+
+/**
+ * This trait handles the creation of the GAST structure for Chevrotain Grammars
+ */
+export class GastRecorder {
+  recordingProdStack: ProdWithDef[]
+  RECORDING_PHASE: boolean
+
+  initGastRecorder(this: MixedInParser, config: IParserConfig): void {
+    this.recordingProdStack = []
+    this.RECORDING_PHASE = false
+  }
+
+  enableRecording(this: MixedInParser): void {
+    this.RECORDING_PHASE = true
+
+    this.TRACE_INIT("Enable Recording", () => {
+      /**
+       * Warning Dark Voodoo Magic upcoming!
+       * We are "replacing" the public parsing DSL methods API
+       * With **new** alternative implementations on the Parser **instance**
+       *
+       * So far this is the only way I've found to avoid performance regressions during parsing time.
+       * - Approx 30% performance regression was measured on Chrome 75 Canary when attempting to replace the "internal"
+       *   implementations directly instead.
+       */
+      for (let i = 0; i < 10; i++) {
+        const idx = i > 0 ? i : ""
+        this[`CONSUME${idx}`] = function (arg1, arg2) {
+          return this.consumeInternalRecord(arg1, i, arg2)
+        }
+        this[`SUBRULE${idx}`] = function (arg1, arg2) {
+          return this.subruleInternalRecord(arg1, i, arg2)
+        }
+        this[`OPTION${idx}`] = function (arg1) {
+          return this.optionInternalRecord(arg1, i)
+        }
+        this[`OR${idx}`] = function (arg1) {
+          return this.orInternalRecord(arg1, i)
+        }
+        this[`MANY${idx}`] = function (arg1) {
+          this.manyInternalRecord(i, arg1)
+        }
+        this[`MANY_SEP${idx}`] = function (arg1) {
+          this.manySepFirstInternalRecord(i, arg1)
+        }
+        this[`AT_LEAST_ONE${idx}`] = function (arg1) {
+          this.atLeastOneInternalRecord(i, arg1)
+        }
+        this[`AT_LEAST_ONE_SEP${idx}`] = function (arg1) {
+          this.atLeastOneSepFirstInternalRecord(i, arg1)
+        }
+      }
+
+      // DSL methods with the idx(suffix) as an argument
+      this[`consume`] = function (idx, arg1, arg2) {
+        return this.consumeInternalRecord(arg1, idx, arg2)
+      }
+      this[`subrule`] = <any>function (idx, arg1, arg2) {
+        return this.subruleInternalRecord(arg1, idx, arg2)
+      }
+      this[`option`] = function (idx, arg1) {
+        return this.optionInternalRecord(arg1, idx)
+      }
+      this[`or`] = function (idx, arg1) {
+        return this.orInternalRecord(arg1, idx)
+      }
+      this[`many`] = function (idx, arg1) {
+        this.manyInternalRecord(idx, arg1)
+      }
+      this[`atLeastOne`] = function (idx, arg1) {
+        this.atLeastOneInternalRecord(idx, arg1)
+      }
+
+      this.ACTION = this.ACTION_RECORD
+      this.BACKTRACK = this.BACKTRACK_RECORD
+      this.LA = this.LA_RECORD
+    })
+  }
+
+  disableRecording(this: MixedInParser) {
+    this.RECORDING_PHASE = false
+    // By deleting these **instance** properties, any future invocation
+    // will be deferred to the original methods on the **prototype** object
+    // This seems to get rid of any incorrect optimizations that V8 may
+    // do during the recording phase.
+    this.TRACE_INIT("Deleting Recording methods", () => {
+      for (let i = 0; i < 10; i++) {
+        const idx = i > 0 ? i : ""
+        delete this[`CONSUME${idx}`]
+        delete this[`SUBRULE${idx}`]
+        delete this[`OPTION${idx}`]
+        delete this[`OR${idx}`]
+        delete this[`MANY${idx}`]
+        delete this[`MANY_SEP${idx}`]
+        delete this[`AT_LEAST_ONE${idx}`]
+        delete this[`AT_LEAST_ONE_SEP${idx}`]
+      }
+
+      delete this[`consume`]
+      delete this[`subrule`]
+      delete this[`option`]
+      delete this[`or`]
+      delete this[`many`]
+      delete this[`atLeastOne`]
+
+      delete this.ACTION
+      delete this.BACKTRACK
+      delete this.LA
+    })
+  }
+
+  // TODO: is there any way to use this method to check no
+  //   Parser methods are called inside an ACTION?
+  //   Maybe try/catch/finally on ACTIONS while disabling the recorders state changes?
+  ACTION_RECORD<T>(this: MixedInParser, impl: () => T): T {
+    // NO-OP during recording
+    return
+  }
+
+  // Executing backtracking logic will break our recording logic assumptions
+  BACKTRACK_RECORD<T>(
+    grammarRule: (...args: any[]) => T,
+    args?: any[]
+  ): () => boolean {
+    return () => true
+  }
+
+  // LA is part of the official API and may be used for custom lookahead logic
+  // by end users who may forget to wrap it in ACTION or inside a GATE
+  LA_RECORD(howMuch: number): IToken {
+    // We cannot use the RECORD_PHASE_TOKEN here because someone may depend
+    // On LA return EOF at the end of the input so an infinite loop may occur.
+    return END_OF_FILE
+  }
+
+  topLevelRuleRecord(name: string, def: Function): Rule {
+    try {
+      const newTopLevelRule = new Rule({ definition: [], name: name })
+      newTopLevelRule.name = name
+      this.recordingProdStack.push(newTopLevelRule)
+      def.call(this)
+      this.recordingProdStack.pop()
+      return newTopLevelRule
+    } catch (originalError) {
+      if (originalError.KNOWN_RECORDER_ERROR !== true) {
+        try {
+          originalError.message =
+            originalError.message +
+            '\n\t This error was thrown during the "grammar recording phase" For more info see:\n\t' +
+            "https://sap.github.io/chevrotain/docs/guide/internals.html#grammar-recording"
+        } catch (mutabilityError) {
+          // We may not be able to modify the original error object
+          throw originalError
+        }
+      }
+      throw originalError
+    }
+  }
+
+  // Implementation of parsing DSL
+  optionInternalRecord<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>,
+    occurrence: number
+  ): OUT {
+    return recordProd.call(this, Option, actionORMethodDef, occurrence)
+  }
+
+  atLeastOneInternalRecord<OUT>(
+    this: MixedInParser,
+    occurrence: number,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    recordProd.call(this, RepetitionMandatory, actionORMethodDef, occurrence)
+  }
+
+  atLeastOneSepFirstInternalRecord<OUT>(
+    this: MixedInParser,
+    occurrence: number,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    recordProd.call(
+      this,
+      RepetitionMandatoryWithSeparator,
+      options,
+      occurrence,
+      HANDLE_SEPARATOR
+    )
+  }
+
+  manyInternalRecord<OUT>(
+    this: MixedInParser,
+    occurrence: number,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    recordProd.call(this, Repetition, actionORMethodDef, occurrence)
+  }
+
+  manySepFirstInternalRecord<OUT>(
+    this: MixedInParser,
+    occurrence: number,
+    options: ManySepMethodOpts<OUT>
+  ): void {
+    recordProd.call(
+      this,
+      RepetitionWithSeparator,
+      options,
+      occurrence,
+      HANDLE_SEPARATOR
+    )
+  }
+
+  orInternalRecord<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>,
+    occurrence: number
+  ): T {
+    return recordOrProd.call(this, altsOrOpts, occurrence)
+  }
+
+  subruleInternalRecord<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    occurrence: number,
+    options?: SubruleMethodOpts
+  ): T | CstNode {
+    assertMethodIdxIsValid(occurrence)
+    if (!ruleToCall || has(ruleToCall, "ruleName") === false) {
+      const error: any = new Error(
+        `<SUBRULE${getIdxSuffix(occurrence)}> argument is invalid` +
+          ` expecting a Parser method reference but got: <${JSON.stringify(
+            ruleToCall
+          )}>` +
+          `\n inside top level rule: <${
+            (<Rule>this.recordingProdStack[0]).name
+          }>`
+      )
+      error.KNOWN_RECORDER_ERROR = true
+      throw error
+    }
+
+    const prevProd: any = peek(this.recordingProdStack)
+    const ruleName = ruleToCall["ruleName"]
+    const newNoneTerminal = new NonTerminal({
+      idx: occurrence,
+      nonTerminalName: ruleName,
+      // The resolving of the `referencedRule` property will be done once all the Rule's GASTs have been created
+      referencedRule: undefined
+    })
+    prevProd.definition.push(newNoneTerminal)
+
+    return this.outputCst ? RECORDING_PHASE_CSTNODE : <any>RECORDING_NULL_OBJECT
+  }
+
+  consumeInternalRecord(
+    this: MixedInParser,
+    tokType: TokenType,
+    occurrence: number,
+    options: ConsumeMethodOpts
+  ): IToken {
+    assertMethodIdxIsValid(occurrence)
+    if (!hasShortKeyProperty(tokType)) {
+      const error: any = new Error(
+        `<CONSUME${getIdxSuffix(occurrence)}> argument is invalid` +
+          ` expecting a TokenType reference but got: <${JSON.stringify(
+            tokType
+          )}>` +
+          `\n inside top level rule: <${
+            (<Rule>this.recordingProdStack[0]).name
+          }>`
+      )
+      error.KNOWN_RECORDER_ERROR = true
+      throw error
+    }
+    const prevProd: any = peek(this.recordingProdStack)
+    const newNoneTerminal = new Terminal({
+      idx: occurrence,
+      terminalType: tokType
+    })
+    prevProd.definition.push(newNoneTerminal)
+
+    return RECORDING_PHASE_TOKEN
+  }
+}
+
+function recordProd(
+  prodConstructor: any,
+  mainProdArg: any,
+  occurrence: number,
+  handleSep: boolean = false
+): any {
+  assertMethodIdxIsValid(occurrence)
+  const prevProd: any = peek(this.recordingProdStack)
+  const grammarAction = isFunction(mainProdArg) ? mainProdArg : mainProdArg.DEF
+
+  const newProd = new prodConstructor({ definition: [], idx: occurrence })
+  if (handleSep) {
+    newProd.separator = mainProdArg.SEP
+  }
+  if (has(mainProdArg, "MAX_LOOKAHEAD")) {
+    newProd.maxLookahead = mainProdArg.MAX_LOOKAHEAD
+  }
+
+  this.recordingProdStack.push(newProd)
+  grammarAction.call(this)
+  prevProd.definition.push(newProd)
+  this.recordingProdStack.pop()
+
+  return RECORDING_NULL_OBJECT
+}
+
+function recordOrProd(mainProdArg: any, occurrence: number): any {
+  assertMethodIdxIsValid(occurrence)
+  const prevProd: any = peek(this.recordingProdStack)
+  // Only an array of alternatives
+  const hasOptions = isArray(mainProdArg) === false
+  const alts = hasOptions === false ? mainProdArg : mainProdArg.DEF
+
+  const newOrProd = new Alternation({
+    definition: [],
+    idx: occurrence,
+    ignoreAmbiguities: hasOptions && mainProdArg.IGNORE_AMBIGUITIES === true
+  })
+  if (has(mainProdArg, "MAX_LOOKAHEAD")) {
+    newOrProd.maxLookahead = mainProdArg.MAX_LOOKAHEAD
+  }
+
+  const hasPredicates = some(alts, (currAlt: any) => isFunction(currAlt.GATE))
+  newOrProd.hasPredicates = hasPredicates
+
+  prevProd.definition.push(newOrProd)
+
+  forEach(alts, (currAlt) => {
+    const currAltFlat = new Alternative({ definition: [] })
+    newOrProd.definition.push(currAltFlat)
+    if (has(currAlt, "IGNORE_AMBIGUITIES")) {
+      currAltFlat.ignoreAmbiguities = currAlt.IGNORE_AMBIGUITIES
+    }
+    // **implicit** ignoreAmbiguities due to usage of gate
+    else if (has(currAlt, "GATE")) {
+      currAltFlat.ignoreAmbiguities = true
+    }
+    this.recordingProdStack.push(currAltFlat)
+    currAlt.ALT.call(this)
+    this.recordingProdStack.pop()
+  })
+  return RECORDING_NULL_OBJECT
+}
+
+function getIdxSuffix(idx: number): string {
+  return idx === 0 ? "" : `${idx}`
+}
+
+function assertMethodIdxIsValid(idx): void {
+  if (idx < 0 || idx > MAX_METHOD_IDX) {
+    const error: any = new Error(
+      // The stack trace will contain all the needed details
+      `Invalid DSL Method idx value: <${idx}>\n\t` +
+        `Idx value must be a none negative value smaller than ${
+          MAX_METHOD_IDX + 1
+        }`
+    )
+    error.KNOWN_RECORDER_ERROR = true
+    throw error
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/lexer_adapter.ts

@@ -0,0 +1,86 @@
+import { END_OF_FILE } from "../parser"
+import { IToken } from "../../../../api"
+import { MixedInParser } from "./parser_traits"
+
+/**
+ * Trait responsible abstracting over the interaction with Lexer output (Token vector).
+ *
+ * This could be generalized to support other kinds of lexers, e.g.
+ * - Just in Time Lexing / Lexer-Less parsing.
+ * - Streaming Lexer.
+ */
+export class LexerAdapter {
+  tokVector: IToken[]
+  tokVectorLength
+  currIdx: number
+
+  initLexerAdapter() {
+    this.tokVector = []
+    this.tokVectorLength = 0
+    this.currIdx = -1
+  }
+
+  set input(newInput: IToken[]) {
+    // @ts-ignore - `this parameter` not supported in setters/getters
+    //   - https://www.typescriptlang.org/docs/handbook/functions.html#this-parameters
+    if (this.selfAnalysisDone !== true) {
+      throw Error(
+        `Missing <performSelfAnalysis> invocation at the end of the Parser's constructor.`
+      )
+    }
+    // @ts-ignore - `this parameter` not supported in setters/getters
+    //   - https://www.typescriptlang.org/docs/handbook/functions.html#this-parameters
+    this.reset()
+    this.tokVector = newInput
+    this.tokVectorLength = newInput.length
+  }
+
+  get input(): IToken[] {
+    return this.tokVector
+  }
+
+  // skips a token and returns the next token
+  SKIP_TOKEN(this: MixedInParser): IToken {
+    if (this.currIdx <= this.tokVector.length - 2) {
+      this.consumeToken()
+      return this.LA(1)
+    } else {
+      return END_OF_FILE
+    }
+  }
+
+  // Lexer (accessing Token vector) related methods which can be overridden to implement lazy lexers
+  // or lexers dependent on parser context.
+  LA(this: MixedInParser, howMuch: number): IToken {
+    const soughtIdx = this.currIdx + howMuch
+    if (soughtIdx < 0 || this.tokVectorLength <= soughtIdx) {
+      return END_OF_FILE
+    } else {
+      return this.tokVector[soughtIdx]
+    }
+  }
+
+  consumeToken(this: MixedInParser) {
+    this.currIdx++
+  }
+
+  exportLexerState(this: MixedInParser): number {
+    return this.currIdx
+  }
+
+  importLexerState(this: MixedInParser, newState: number) {
+    this.currIdx = newState
+  }
+
+  resetLexerState(this: MixedInParser): void {
+    this.currIdx = -1
+  }
+
+  moveToTerminatedState(this: MixedInParser): void {
+    this.currIdx = this.tokVector.length - 1
+  }
+
+  getLexerPosition(this: MixedInParser): number {
+    return this.exportLexerState()
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/looksahead.ts

@@ -0,0 +1,252 @@
+import {
+  buildAlternativesLookAheadFunc,
+  buildLookaheadFuncForOptionalProd,
+  buildLookaheadFuncForOr,
+  buildSingleAlternativeLookaheadFunction,
+  PROD_TYPE
+} from "../../grammar/lookahead"
+import { forEach, has, isES2015MapSupported } from "../../../utils/utils"
+import {
+  DEFAULT_PARSER_CONFIG,
+  lookAheadSequence,
+  TokenMatcher
+} from "../parser"
+import { IOrAlt, IParserConfig } from "../../../../api"
+import {
+  AT_LEAST_ONE_IDX,
+  AT_LEAST_ONE_SEP_IDX,
+  getKeyForAutomaticLookahead,
+  MANY_IDX,
+  MANY_SEP_IDX,
+  OPTION_IDX,
+  OR_IDX
+} from "../../grammar/keys"
+import { MixedInParser } from "./parser_traits"
+import { Rule } from "../../grammar/gast/gast_public"
+import { collectMethods, getProductionDslName } from "../../grammar/gast/gast"
+
+/**
+ * Trait responsible for the lookahead related utilities and optimizations.
+ */
+export class LooksAhead {
+  maxLookahead: number
+  lookAheadFuncsCache: any
+  dynamicTokensEnabled: boolean
+
+  initLooksAhead(config: IParserConfig) {
+    this.dynamicTokensEnabled = has(config, "dynamicTokensEnabled")
+      ? config.dynamicTokensEnabled
+      : DEFAULT_PARSER_CONFIG.dynamicTokensEnabled
+
+    this.maxLookahead = has(config, "maxLookahead")
+      ? config.maxLookahead
+      : DEFAULT_PARSER_CONFIG.maxLookahead
+
+    /* istanbul ignore next - Using plain array as dictionary will be tested on older node.js versions and IE11 */
+    this.lookAheadFuncsCache = isES2015MapSupported() ? new Map() : []
+
+    // Performance optimization on newer engines that support ES6 Map
+    // For larger Maps this is slightly faster than using a plain object (array in our case).
+    /* istanbul ignore else - The else branch will be tested on older node.js versions and IE11 */
+    if (isES2015MapSupported()) {
+      this.getLaFuncFromCache = this.getLaFuncFromMap
+      this.setLaFuncCache = this.setLaFuncCacheUsingMap
+    } else {
+      this.getLaFuncFromCache = this.getLaFuncFromObj
+      this.setLaFuncCache = this.setLaFuncUsingObj
+    }
+  }
+
+  preComputeLookaheadFunctions(this: MixedInParser, rules: Rule[]): void {
+    forEach(rules, (currRule) => {
+      this.TRACE_INIT(`${currRule.name} Rule Lookahead`, () => {
+        const {
+          alternation,
+          repetition,
+          option,
+          repetitionMandatory,
+          repetitionMandatoryWithSeparator,
+          repetitionWithSeparator
+        } = collectMethods(currRule)
+
+        forEach(alternation, (currProd) => {
+          const prodIdx = currProd.idx === 0 ? "" : currProd.idx
+          this.TRACE_INIT(`${getProductionDslName(currProd)}${prodIdx}`, () => {
+            const laFunc = buildLookaheadFuncForOr(
+              currProd.idx,
+              currRule,
+              currProd.maxLookahead || this.maxLookahead,
+              currProd.hasPredicates,
+              this.dynamicTokensEnabled,
+              this.lookAheadBuilderForAlternatives
+            )
+
+            const key = getKeyForAutomaticLookahead(
+              this.fullRuleNameToShort[currRule.name],
+              OR_IDX,
+              currProd.idx
+            )
+            this.setLaFuncCache(key, laFunc)
+          })
+        })
+
+        forEach(repetition, (currProd) => {
+          this.computeLookaheadFunc(
+            currRule,
+            currProd.idx,
+            MANY_IDX,
+            PROD_TYPE.REPETITION,
+            currProd.maxLookahead,
+            getProductionDslName(currProd)
+          )
+        })
+
+        forEach(option, (currProd) => {
+          this.computeLookaheadFunc(
+            currRule,
+            currProd.idx,
+            OPTION_IDX,
+            PROD_TYPE.OPTION,
+            currProd.maxLookahead,
+            getProductionDslName(currProd)
+          )
+        })
+
+        forEach(repetitionMandatory, (currProd) => {
+          this.computeLookaheadFunc(
+            currRule,
+            currProd.idx,
+            AT_LEAST_ONE_IDX,
+            PROD_TYPE.REPETITION_MANDATORY,
+            currProd.maxLookahead,
+            getProductionDslName(currProd)
+          )
+        })
+
+        forEach(repetitionMandatoryWithSeparator, (currProd) => {
+          this.computeLookaheadFunc(
+            currRule,
+            currProd.idx,
+            AT_LEAST_ONE_SEP_IDX,
+            PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR,
+            currProd.maxLookahead,
+            getProductionDslName(currProd)
+          )
+        })
+
+        forEach(repetitionWithSeparator, (currProd) => {
+          this.computeLookaheadFunc(
+            currRule,
+            currProd.idx,
+            MANY_SEP_IDX,
+            PROD_TYPE.REPETITION_WITH_SEPARATOR,
+            currProd.maxLookahead,
+            getProductionDslName(currProd)
+          )
+        })
+      })
+    })
+  }
+
+  computeLookaheadFunc(
+    this: MixedInParser,
+    rule: Rule,
+    prodOccurrence: number,
+    prodKey: number,
+    prodType: PROD_TYPE,
+    prodMaxLookahead: number,
+    dslMethodName: string
+  ): void {
+    this.TRACE_INIT(
+      `${dslMethodName}${prodOccurrence === 0 ? "" : prodOccurrence}`,
+      () => {
+        const laFunc = buildLookaheadFuncForOptionalProd(
+          prodOccurrence,
+          rule,
+          prodMaxLookahead || this.maxLookahead,
+          this.dynamicTokensEnabled,
+          prodType,
+          this.lookAheadBuilderForOptional
+        )
+        const key = getKeyForAutomaticLookahead(
+          this.fullRuleNameToShort[rule.name],
+          prodKey,
+          prodOccurrence
+        )
+        this.setLaFuncCache(key, laFunc)
+      }
+    )
+  }
+
+  lookAheadBuilderForOptional(
+    this: MixedInParser,
+    alt: lookAheadSequence,
+    tokenMatcher: TokenMatcher,
+    dynamicTokensEnabled: boolean
+  ): () => boolean {
+    return buildSingleAlternativeLookaheadFunction(
+      alt,
+      tokenMatcher,
+      dynamicTokensEnabled
+    )
+  }
+
+  lookAheadBuilderForAlternatives(
+    this: MixedInParser,
+    alts: lookAheadSequence[],
+    hasPredicates: boolean,
+    tokenMatcher: TokenMatcher,
+    dynamicTokensEnabled: boolean
+  ): (orAlts?: IOrAlt<any>[]) => number | undefined {
+    return buildAlternativesLookAheadFunc(
+      alts,
+      hasPredicates,
+      tokenMatcher,
+      dynamicTokensEnabled
+    )
+  }
+
+  // this actually returns a number, but it is always used as a string (object prop key)
+  getKeyForAutomaticLookahead(
+    this: MixedInParser,
+    dslMethodIdx: number,
+    occurrence: number
+  ): number {
+    let currRuleShortName: any = this.getLastExplicitRuleShortName()
+    return getKeyForAutomaticLookahead(
+      currRuleShortName,
+      dslMethodIdx,
+      occurrence
+    )
+  }
+
+  /* istanbul ignore next */
+  getLaFuncFromCache(this: MixedInParser, key: number): Function {
+    return undefined
+  }
+
+  getLaFuncFromMap(this: MixedInParser, key: number): Function {
+    return this.lookAheadFuncsCache.get(key)
+  }
+
+  /* istanbul ignore next - Using plain array as dictionary will be tested on older node.js versions and IE11 */
+  getLaFuncFromObj(this: MixedInParser, key: number): Function {
+    return this.lookAheadFuncsCache[key]
+  }
+
+  /* istanbul ignore next */
+  setLaFuncCache(this: MixedInParser, key: number, value: Function): void {}
+
+  setLaFuncCacheUsingMap(
+    this: MixedInParser,
+    key: number,
+    value: Function
+  ): void {
+    this.lookAheadFuncsCache.set(key, value)
+  }
+
+  /* istanbul ignore next - Using plain array as dictionary will be tested on older node.js versions and IE11 */
+  setLaFuncUsingObj(this: MixedInParser, key: number, value: Function): void {
+    this.lookAheadFuncsCache[key] = value
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/parser_traits.ts

@@ -0,0 +1,58 @@
+import { ErrorHandler } from "./error_handler"
+import { LexerAdapter } from "./lexer_adapter"
+import { LooksAhead } from "./looksahead"
+import { RecognizerApi } from "./recognizer_api"
+import { RecognizerEngine } from "./recognizer_engine"
+import { Recoverable } from "./recoverable"
+import { TreeBuilder } from "./tree_builder"
+import {
+  Parser as ParserConstructorImpel,
+  CstParser as CstParserConstructorImpel,
+  EmbeddedActionsParser as EmbeddedActionsParserConstructorImpel
+} from "../parser"
+import * as defs from "../../../../api"
+import { ContentAssist } from "./context_assist"
+import { GastRecorder } from "./gast_recorder"
+import { PerformanceTracer } from "./perf_tracer"
+
+/**
+ * This Type combines all the Parser traits.
+ * It is used in all traits in the "this type assertion"
+ * - https://github.com/Microsoft/TypeScript/wiki/What%27s-new-in-TypeScript#specifying-the-type-of-this-for-functions
+ * This enables strong Type Checks inside trait methods that invoke methods from other traits.
+ * This pattern is very similar to "self types" in Scala.
+ * - https://docs.scala-lang.org/tour/self-types.html
+ */
+export type MixedInParser = ParserConstructorImpel &
+  ErrorHandler &
+  LexerAdapter &
+  LooksAhead &
+  RecognizerApi &
+  RecognizerEngine &
+  Recoverable &
+  TreeBuilder &
+  ContentAssist &
+  GastRecorder &
+  PerformanceTracer
+
+interface MixedInCstParserConstructor {
+  new (
+    tokenVocabulary: defs.TokenVocabulary,
+    config?: defs.IParserConfig
+  ): defs.CstParser
+}
+
+export const CstParser: MixedInCstParserConstructor = <any>(
+  CstParserConstructorImpel
+)
+
+interface MixedInEmbeddedActionsParserConstructor {
+  new (
+    tokenVocabulary: defs.TokenVocabulary,
+    config?: defs.IParserConfig
+  ): defs.EmbeddedActionsParser
+}
+
+export const EmbeddedActionsParser: MixedInEmbeddedActionsParserConstructor = <
+  any
+>EmbeddedActionsParserConstructorImpel

node_modules/chevrotain/src/parse/parser/traits/perf_tracer.ts

@@ -0,0 +1,53 @@
+import { IParserConfig } from "../../../../api"
+import { has, timer } from "../../../utils/utils"
+import { MixedInParser } from "./parser_traits"
+import { DEFAULT_PARSER_CONFIG } from "../parser"
+
+/**
+ * Trait responsible for runtime parsing errors.
+ */
+export class PerformanceTracer {
+  traceInitPerf: boolean | number
+  traceInitMaxIdent: number
+  traceInitIndent: number
+
+  initPerformanceTracer(config: IParserConfig) {
+    if (has(config, "traceInitPerf")) {
+      const userTraceInitPerf = config.traceInitPerf
+      const traceIsNumber = typeof userTraceInitPerf === "number"
+      this.traceInitMaxIdent = traceIsNumber
+        ? <number>userTraceInitPerf
+        : Infinity
+      this.traceInitPerf = traceIsNumber
+        ? userTraceInitPerf > 0
+        : userTraceInitPerf
+    } else {
+      this.traceInitMaxIdent = 0
+      this.traceInitPerf = DEFAULT_PARSER_CONFIG.traceInitPerf
+    }
+
+    this.traceInitIndent = -1
+  }
+
+  TRACE_INIT<T>(this: MixedInParser, phaseDesc: string, phaseImpl: () => T): T {
+    // No need to optimize this using NOOP pattern because
+    // It is not called in a hot spot...
+    if (this.traceInitPerf === true) {
+      this.traceInitIndent++
+      const indent = new Array(this.traceInitIndent + 1).join("\t")
+      if (this.traceInitIndent < this.traceInitMaxIdent) {
+        console.log(`${indent}--> <${phaseDesc}>`)
+      }
+      const { time, value } = timer(phaseImpl)
+      /* istanbul ignore next - Difficult to reproduce specific performance behavior (>10ms) in tests */
+      const traceMethod = time > 10 ? console.warn : console.log
+      if (this.traceInitIndent < this.traceInitMaxIdent) {
+        traceMethod(`${indent}<-- <${phaseDesc}> time: ${time}ms`)
+      }
+      this.traceInitIndent--
+      return value
+    } else {
+      return phaseImpl()
+    }
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/recognizer_api.ts

@@ -0,0 +1,717 @@
+import {
+  AtLeastOneSepMethodOpts,
+  ConsumeMethodOpts,
+  DSLMethodOpts,
+  DSLMethodOptsWithErr,
+  GrammarAction,
+  IOrAlt,
+  IRuleConfig,
+  ISerializedGast,
+  IToken,
+  ManySepMethodOpts,
+  OrMethodOpts,
+  SubruleMethodOpts,
+  TokenType
+} from "../../../../api"
+import { contains, values } from "../../../utils/utils"
+import { isRecognitionException } from "../../exceptions_public"
+import { DEFAULT_RULE_CONFIG, ParserDefinitionErrorType } from "../parser"
+import { defaultGrammarValidatorErrorProvider } from "../../errors_public"
+import { validateRuleIsOverridden } from "../../grammar/checks"
+import { MixedInParser } from "./parser_traits"
+import { Rule, serializeGrammar } from "../../grammar/gast/gast_public"
+
+/**
+ * This trait is responsible for implementing the public API
+ * for defining Chevrotain parsers, i.e:
+ * - CONSUME
+ * - RULE
+ * - OPTION
+ * - ...
+ */
+export class RecognizerApi {
+  ACTION<T>(this: MixedInParser, impl: () => T): T {
+    return impl.call(this)
+  }
+
+  consume(
+    this: MixedInParser,
+    idx: number,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, idx, options)
+  }
+
+  subrule<T>(
+    this: MixedInParser,
+    idx: number,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, idx, options)
+  }
+
+  option<OUT>(
+    this: MixedInParser,
+    idx: number,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, idx)
+  }
+
+  or(
+    this: MixedInParser,
+    idx: number,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<any>
+  ): any {
+    return this.orInternal(altsOrOpts, idx)
+  }
+
+  many(
+    this: MixedInParser,
+    idx: number,
+    actionORMethodDef: GrammarAction<any> | DSLMethodOpts<any>
+  ): void {
+    return this.manyInternal(idx, actionORMethodDef)
+  }
+
+  atLeastOne(
+    this: MixedInParser,
+    idx: number,
+    actionORMethodDef: GrammarAction<any> | DSLMethodOptsWithErr<any>
+  ): void {
+    return this.atLeastOneInternal(idx, actionORMethodDef)
+  }
+
+  CONSUME(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 0, options)
+  }
+
+  CONSUME1(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 1, options)
+  }
+
+  CONSUME2(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 2, options)
+  }
+
+  CONSUME3(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 3, options)
+  }
+
+  CONSUME4(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 4, options)
+  }
+
+  CONSUME5(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 5, options)
+  }
+
+  CONSUME6(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 6, options)
+  }
+
+  CONSUME7(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 7, options)
+  }
+
+  CONSUME8(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 8, options)
+  }
+
+  CONSUME9(
+    this: MixedInParser,
+    tokType: TokenType,
+    options?: ConsumeMethodOpts
+  ): IToken {
+    return this.consumeInternal(tokType, 9, options)
+  }
+
+  SUBRULE<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 0, options)
+  }
+
+  SUBRULE1<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 1, options)
+  }
+
+  SUBRULE2<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 2, options)
+  }
+
+  SUBRULE3<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 3, options)
+  }
+
+  SUBRULE4<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 4, options)
+  }
+
+  SUBRULE5<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 5, options)
+  }
+
+  SUBRULE6<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 6, options)
+  }
+
+  SUBRULE7<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 7, options)
+  }
+
+  SUBRULE8<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 8, options)
+  }
+
+  SUBRULE9<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    options?: SubruleMethodOpts
+  ): T {
+    return this.subruleInternal(ruleToCall, 9, options)
+  }
+
+  OPTION<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 0)
+  }
+
+  OPTION1<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 1)
+  }
+
+  OPTION2<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 2)
+  }
+
+  OPTION3<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 3)
+  }
+
+  OPTION4<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 4)
+  }
+
+  OPTION5<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 5)
+  }
+
+  OPTION6<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 6)
+  }
+
+  OPTION7<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 7)
+  }
+
+  OPTION8<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 8)
+  }
+
+  OPTION9<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): OUT {
+    return this.optionInternal(actionORMethodDef, 9)
+  }
+
+  OR<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 0)
+  }
+
+  OR1<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 1)
+  }
+
+  OR2<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 2)
+  }
+
+  OR3<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 3)
+  }
+
+  OR4<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 4)
+  }
+
+  OR5<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 5)
+  }
+
+  OR6<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 6)
+  }
+
+  OR7<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 7)
+  }
+
+  OR8<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 8)
+  }
+
+  OR9<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
+  ): T {
+    return this.orInternal(altsOrOpts, 9)
+  }
+
+  MANY<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(0, actionORMethodDef)
+  }
+
+  MANY1<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(1, actionORMethodDef)
+  }
+
+  MANY2<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(2, actionORMethodDef)
+  }
+
+  MANY3<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(3, actionORMethodDef)
+  }
+
+  MANY4<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(4, actionORMethodDef)
+  }
+
+  MANY5<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(5, actionORMethodDef)
+  }
+
+  MANY6<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(6, actionORMethodDef)
+  }
+
+  MANY7<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(7, actionORMethodDef)
+  }
+
+  MANY8<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(8, actionORMethodDef)
+  }
+
+  MANY9<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    this.manyInternal(9, actionORMethodDef)
+  }
+
+  MANY_SEP<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(0, options)
+  }
+
+  MANY_SEP1<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(1, options)
+  }
+
+  MANY_SEP2<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(2, options)
+  }
+
+  MANY_SEP3<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(3, options)
+  }
+
+  MANY_SEP4<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(4, options)
+  }
+
+  MANY_SEP5<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(5, options)
+  }
+
+  MANY_SEP6<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(6, options)
+  }
+
+  MANY_SEP7<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(7, options)
+  }
+
+  MANY_SEP8<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(8, options)
+  }
+
+  MANY_SEP9<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
+    this.manySepFirstInternal(9, options)
+  }
+
+  AT_LEAST_ONE<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    this.atLeastOneInternal(0, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE1<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    return this.atLeastOneInternal(1, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE2<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    this.atLeastOneInternal(2, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE3<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    this.atLeastOneInternal(3, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE4<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    this.atLeastOneInternal(4, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE5<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    this.atLeastOneInternal(5, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE6<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    this.atLeastOneInternal(6, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE7<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    this.atLeastOneInternal(7, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE8<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    this.atLeastOneInternal(8, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE9<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    this.atLeastOneInternal(9, actionORMethodDef)
+  }
+
+  AT_LEAST_ONE_SEP<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(0, options)
+  }
+
+  AT_LEAST_ONE_SEP1<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(1, options)
+  }
+
+  AT_LEAST_ONE_SEP2<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(2, options)
+  }
+
+  AT_LEAST_ONE_SEP3<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(3, options)
+  }
+
+  AT_LEAST_ONE_SEP4<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(4, options)
+  }
+
+  AT_LEAST_ONE_SEP5<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(5, options)
+  }
+
+  AT_LEAST_ONE_SEP6<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(6, options)
+  }
+
+  AT_LEAST_ONE_SEP7<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(7, options)
+  }
+
+  AT_LEAST_ONE_SEP8<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(8, options)
+  }
+
+  AT_LEAST_ONE_SEP9<OUT>(
+    this: MixedInParser,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    this.atLeastOneSepFirstInternal(9, options)
+  }
+
+  RULE<T>(
+    this: MixedInParser,
+    name: string,
+    implementation: (...implArgs: any[]) => T,
+    config: IRuleConfig<T> = DEFAULT_RULE_CONFIG
+  ): (idxInCallingRule?: number, ...args: any[]) => T | any {
+    if (contains(this.definedRulesNames, name)) {
+      const errMsg = defaultGrammarValidatorErrorProvider.buildDuplicateRuleNameError(
+        {
+          topLevelRule: name,
+          grammarName: this.className
+        }
+      )
+
+      const error = {
+        message: errMsg,
+        type: ParserDefinitionErrorType.DUPLICATE_RULE_NAME,
+        ruleName: name
+      }
+      this.definitionErrors.push(error)
+    }
+
+    this.definedRulesNames.push(name)
+
+    let ruleImplementation = this.defineRule(name, implementation, config)
+    this[name] = ruleImplementation
+    return ruleImplementation
+  }
+
+  OVERRIDE_RULE<T>(
+    this: MixedInParser,
+    name: string,
+    impl: (...implArgs: any[]) => T,
+    config: IRuleConfig<T> = DEFAULT_RULE_CONFIG
+  ): (idxInCallingRule?: number, ...args: any[]) => T {
+    let ruleErrors = []
+    ruleErrors = ruleErrors.concat(
+      validateRuleIsOverridden(name, this.definedRulesNames, this.className)
+    )
+    this.definitionErrors.push.apply(this.definitionErrors, ruleErrors) // mutability for the win
+
+    let ruleImplementation = this.defineRule(name, impl, config)
+    this[name] = ruleImplementation
+    return ruleImplementation
+  }
+
+  BACKTRACK<T>(
+    this: MixedInParser,
+    grammarRule: (...args: any[]) => T,
+    args?: any[]
+  ): () => boolean {
+    return function () {
+      // save org state
+      this.isBackTrackingStack.push(1)
+      const orgState = this.saveRecogState()
+      try {
+        grammarRule.apply(this, args)
+        // if no exception was thrown we have succeed parsing the rule.
+        return true
+      } catch (e) {
+        if (isRecognitionException(e)) {
+          return false
+        } else {
+          throw e
+        }
+      } finally {
+        this.reloadRecogState(orgState)
+        this.isBackTrackingStack.pop()
+      }
+    }
+  }
+
+  // GAST export APIs
+  public getGAstProductions(this: MixedInParser): Record<string, Rule> {
+    return this.gastProductionsCache
+  }
+
+  public getSerializedGastProductions(this: MixedInParser): ISerializedGast[] {
+    return serializeGrammar(values(this.gastProductionsCache))
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/recognizer_engine.ts

@@ -0,0 +1,854 @@
+import {
+  AtLeastOneSepMethodOpts,
+  ConsumeMethodOpts,
+  DSLMethodOpts,
+  DSLMethodOptsWithErr,
+  GrammarAction,
+  IOrAlt,
+  IParserConfig,
+  IRuleConfig,
+  IToken,
+  ManySepMethodOpts,
+  OrMethodOpts,
+  SubruleMethodOpts,
+  TokenType,
+  TokenVocabulary
+} from "../../../../api"
+import {
+  cloneArr,
+  cloneObj,
+  every,
+  flatten,
+  has,
+  isArray,
+  isEmpty,
+  isObject,
+  reduce,
+  uniq,
+  values
+} from "../../../utils/utils"
+import {
+  AT_LEAST_ONE_IDX,
+  AT_LEAST_ONE_SEP_IDX,
+  BITS_FOR_METHOD_TYPE,
+  BITS_FOR_OCCURRENCE_IDX,
+  MANY_IDX,
+  MANY_SEP_IDX,
+  OPTION_IDX,
+  OR_IDX
+} from "../../grammar/keys"
+import {
+  isRecognitionException,
+  MismatchedTokenException,
+  NotAllInputParsedException
+} from "../../exceptions_public"
+import { PROD_TYPE } from "../../grammar/lookahead"
+import {
+  AbstractNextTerminalAfterProductionWalker,
+  NextTerminalAfterAtLeastOneSepWalker,
+  NextTerminalAfterAtLeastOneWalker,
+  NextTerminalAfterManySepWalker,
+  NextTerminalAfterManyWalker
+} from "../../grammar/interpreter"
+import { DEFAULT_RULE_CONFIG, IParserState, TokenMatcher } from "../parser"
+import { IN_RULE_RECOVERY_EXCEPTION } from "./recoverable"
+import { EOF } from "../../../scan/tokens_public"
+import { MixedInParser } from "./parser_traits"
+import {
+  augmentTokenTypes,
+  isTokenType,
+  tokenStructuredMatcher,
+  tokenStructuredMatcherNoCategories
+} from "../../../scan/tokens"
+import { classNameFromInstance } from "../../../lang/lang_extensions"
+import { Rule } from "../../grammar/gast/gast_public"
+
+/**
+ * This trait is responsible for the runtime parsing engine
+ * Used by the official API (recognizer_api.ts)
+ */
+export class RecognizerEngine {
+  isBackTrackingStack
+  className: string
+  RULE_STACK: string[]
+  RULE_OCCURRENCE_STACK: number[]
+  definedRulesNames: string[]
+  tokensMap: { [fqn: string]: TokenType }
+  gastProductionsCache: Record<string, Rule>
+  shortRuleNameToFull: Record<string, string>
+  fullRuleNameToShort: Record<string, number>
+  // The shortName Index must be coded "after" the first 8bits to enable building unique lookahead keys
+  ruleShortNameIdx: number
+  tokenMatcher: TokenMatcher
+
+  initRecognizerEngine(
+    tokenVocabulary: TokenVocabulary,
+    config: IParserConfig
+  ) {
+    this.className = classNameFromInstance(this)
+    // TODO: would using an ES6 Map or plain object be faster (CST building scenario)
+    this.shortRuleNameToFull = {}
+    this.fullRuleNameToShort = {}
+    this.ruleShortNameIdx = 256
+    this.tokenMatcher = tokenStructuredMatcherNoCategories
+
+    this.definedRulesNames = []
+    this.tokensMap = {}
+    this.isBackTrackingStack = []
+    this.RULE_STACK = []
+    this.RULE_OCCURRENCE_STACK = []
+    this.gastProductionsCache = {}
+
+    if (has(config, "serializedGrammar")) {
+      throw Error(
+        "The Parser's configuration can no longer contain a <serializedGrammar> property.\n" +
+          "\tSee: https://sap.github.io/chevrotain/docs/changes/BREAKING_CHANGES.html#_6-0-0\n" +
+          "\tFor Further details."
+      )
+    }
+
+    if (isArray(tokenVocabulary)) {
+      // This only checks for Token vocabularies provided as arrays.
+      // That is good enough because the main objective is to detect users of pre-V4.0 APIs
+      // rather than all edge cases of empty Token vocabularies.
+      if (isEmpty(tokenVocabulary as any[])) {
+        throw Error(
+          "A Token Vocabulary cannot be empty.\n" +
+            "\tNote that the first argument for the parser constructor\n" +
+            "\tis no longer a Token vector (since v4.0)."
+        )
+      }
+
+      if (typeof (tokenVocabulary as any[])[0].startOffset === "number") {
+        throw Error(
+          "The Parser constructor no longer accepts a token vector as the first argument.\n" +
+            "\tSee: https://sap.github.io/chevrotain/docs/changes/BREAKING_CHANGES.html#_4-0-0\n" +
+            "\tFor Further details."
+        )
+      }
+    }
+
+    if (isArray(tokenVocabulary)) {
+      this.tokensMap = <any>reduce(
+        <any>tokenVocabulary,
+        (acc, tokType: TokenType) => {
+          acc[tokType.name] = tokType
+          return acc
+        },
+        {}
+      )
+    } else if (
+      has(tokenVocabulary, "modes") &&
+      every(flatten(values((<any>tokenVocabulary).modes)), isTokenType)
+    ) {
+      let allTokenTypes = flatten(values((<any>tokenVocabulary).modes))
+      let uniqueTokens = uniq(allTokenTypes)
+      this.tokensMap = <any>reduce(
+        uniqueTokens,
+        (acc, tokType: TokenType) => {
+          acc[tokType.name] = tokType
+          return acc
+        },
+        {}
+      )
+    } else if (isObject(tokenVocabulary)) {
+      this.tokensMap = cloneObj(tokenVocabulary)
+    } else {
+      throw new Error(
+        "<tokensDictionary> argument must be An Array of Token constructors," +
+          " A dictionary of Token constructors or an IMultiModeLexerDefinition"
+      )
+    }
+
+    // always add EOF to the tokenNames -> constructors map. it is useful to assure all the input has been
+    // parsed with a clear error message ("expecting EOF but found ...")
+    /* tslint:disable */
+    this.tokensMap["EOF"] = EOF
+
+    // TODO: This check may not be accurate for multi mode lexers
+    const noTokenCategoriesUsed = every(
+      values(tokenVocabulary),
+      (tokenConstructor) => isEmpty(tokenConstructor.categoryMatches)
+    )
+
+    this.tokenMatcher = noTokenCategoriesUsed
+      ? tokenStructuredMatcherNoCategories
+      : tokenStructuredMatcher
+
+    // Because ES2015+ syntax should be supported for creating Token classes
+    // We cannot assume that the Token classes were created using the "extendToken" utilities
+    // Therefore we must augment the Token classes both on Lexer initialization and on Parser initialization
+    augmentTokenTypes(values(this.tokensMap))
+  }
+
+  defineRule<T>(
+    this: MixedInParser,
+    ruleName: string,
+    impl: (...implArgs: any[]) => T,
+    config: IRuleConfig<T>
+  ): (idxInCallingRule?: number, ...args: any[]) => T {
+    if (this.selfAnalysisDone) {
+      throw Error(
+        `Grammar rule <${ruleName}> may not be defined after the 'performSelfAnalysis' method has been called'\n` +
+          `Make sure that all grammar rule definitions are done before 'performSelfAnalysis' is called.`
+      )
+    }
+    let resyncEnabled = has(config, "resyncEnabled")
+      ? config.resyncEnabled
+      : DEFAULT_RULE_CONFIG.resyncEnabled
+    let recoveryValueFunc = has(config, "recoveryValueFunc")
+      ? config.recoveryValueFunc
+      : DEFAULT_RULE_CONFIG.recoveryValueFunc
+
+    // performance optimization: Use small integers as keys for the longer human readable "full" rule names.
+    // this greatly improves Map access time (as much as 8% for some performance benchmarks).
+    /* tslint:disable */
+    let shortName =
+      this.ruleShortNameIdx << (BITS_FOR_METHOD_TYPE + BITS_FOR_OCCURRENCE_IDX)
+    /* tslint:enable */
+
+    this.ruleShortNameIdx++
+    this.shortRuleNameToFull[shortName] = ruleName
+    this.fullRuleNameToShort[ruleName] = shortName
+
+    function invokeRuleWithTry(args: any[]) {
+      try {
+        if (this.outputCst === true) {
+          impl.apply(this, args)
+          const cst = this.CST_STACK[this.CST_STACK.length - 1]
+          this.cstPostRule(cst)
+          return cst
+        } else {
+          return impl.apply(this, args)
+        }
+      } catch (e) {
+        return this.invokeRuleCatch(e, resyncEnabled, recoveryValueFunc)
+      } finally {
+        this.ruleFinallyStateUpdate()
+      }
+    }
+
+    let wrappedGrammarRule
+
+    wrappedGrammarRule = function (idxInCallingRule: number = 0, args: any[]) {
+      this.ruleInvocationStateUpdate(shortName, ruleName, idxInCallingRule)
+      return invokeRuleWithTry.call(this, args)
+    }
+
+    let ruleNamePropName = "ruleName"
+    wrappedGrammarRule[ruleNamePropName] = ruleName
+    wrappedGrammarRule["originalGrammarAction"] = impl
+    return wrappedGrammarRule
+  }
+
+  invokeRuleCatch(
+    this: MixedInParser,
+    e: Error,
+    resyncEnabledConfig: boolean,
+    recoveryValueFunc: Function
+  ): void {
+    let isFirstInvokedRule = this.RULE_STACK.length === 1
+    // note the reSync is always enabled for the first rule invocation, because we must always be able to
+    // reSync with EOF and just output some INVALID ParseTree
+    // during backtracking reSync recovery is disabled, otherwise we can't be certain the backtracking
+    // path is really the most valid one
+    let reSyncEnabled =
+      resyncEnabledConfig && !this.isBackTracking() && this.recoveryEnabled
+
+    if (isRecognitionException(e)) {
+      const recogError: any = e
+      if (reSyncEnabled) {
+        let reSyncTokType = this.findReSyncTokenType()
+        if (this.isInCurrentRuleReSyncSet(reSyncTokType)) {
+          recogError.resyncedTokens = this.reSyncTo(reSyncTokType)
+          if (this.outputCst) {
+            let partialCstResult: any = this.CST_STACK[
+              this.CST_STACK.length - 1
+            ]
+            partialCstResult.recoveredNode = true
+            return partialCstResult
+          } else {
+            return recoveryValueFunc()
+          }
+        } else {
+          if (this.outputCst) {
+            const partialCstResult: any = this.CST_STACK[
+              this.CST_STACK.length - 1
+            ]
+            partialCstResult.recoveredNode = true
+            recogError.partialCstResult = partialCstResult
+          }
+          // to be handled Further up the call stack
+          throw recogError
+        }
+      } else if (isFirstInvokedRule) {
+        // otherwise a Redundant input error will be created as well and we cannot guarantee that this is indeed the case
+        this.moveToTerminatedState()
+        // the parser should never throw one of its own errors outside its flow.
+        // even if error recovery is disabled
+        return recoveryValueFunc()
+      } else {
+        // to be recovered Further up the call stack
+        throw recogError
+      }
+    } else {
+      // some other Error type which we don't know how to handle (for example a built in JavaScript Error)
+      throw e
+    }
+  }
+
+  // Implementation of parsing DSL
+  optionInternal<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>,
+    occurrence: number
+  ): OUT {
+    let key = this.getKeyForAutomaticLookahead(OPTION_IDX, occurrence)
+    return this.optionInternalLogic(actionORMethodDef, occurrence, key)
+  }
+
+  optionInternalLogic<OUT>(
+    this: MixedInParser,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>,
+    occurrence: number,
+    key: number
+  ): OUT {
+    let lookAheadFunc = this.getLaFuncFromCache(key)
+    let action
+    let predicate
+    if ((<DSLMethodOpts<OUT>>actionORMethodDef).DEF !== undefined) {
+      action = (<DSLMethodOpts<OUT>>actionORMethodDef).DEF
+      predicate = (<DSLMethodOpts<OUT>>actionORMethodDef).GATE
+      // predicate present
+      if (predicate !== undefined) {
+        let orgLookaheadFunction = lookAheadFunc
+        lookAheadFunc = () => {
+          return predicate.call(this) && orgLookaheadFunction.call(this)
+        }
+      }
+    } else {
+      action = actionORMethodDef
+    }
+
+    if (lookAheadFunc.call(this) === true) {
+      return action.call(this)
+    }
+    return undefined
+  }
+
+  atLeastOneInternal<OUT>(
+    this: MixedInParser,
+    prodOccurrence: number,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
+  ): void {
+    let laKey = this.getKeyForAutomaticLookahead(
+      AT_LEAST_ONE_IDX,
+      prodOccurrence
+    )
+    return this.atLeastOneInternalLogic(
+      prodOccurrence,
+      actionORMethodDef,
+      laKey
+    )
+  }
+
+  atLeastOneInternalLogic<OUT>(
+    this: MixedInParser,
+    prodOccurrence: number,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>,
+    key: number
+  ): void {
+    let lookAheadFunc = this.getLaFuncFromCache(key)
+
+    let action
+    let predicate
+    if ((<DSLMethodOptsWithErr<OUT>>actionORMethodDef).DEF !== undefined) {
+      action = (<DSLMethodOptsWithErr<OUT>>actionORMethodDef).DEF
+      predicate = (<DSLMethodOptsWithErr<OUT>>actionORMethodDef).GATE
+      // predicate present
+      if (predicate !== undefined) {
+        let orgLookaheadFunction = lookAheadFunc
+        lookAheadFunc = () => {
+          return predicate.call(this) && orgLookaheadFunction.call(this)
+        }
+      }
+    } else {
+      action = actionORMethodDef
+    }
+
+    if ((<Function>lookAheadFunc).call(this) === true) {
+      let notStuck = this.doSingleRepetition(action)
+      while (
+        (<Function>lookAheadFunc).call(this) === true &&
+        notStuck === true
+      ) {
+        notStuck = this.doSingleRepetition(action)
+      }
+    } else {
+      throw this.raiseEarlyExitException(
+        prodOccurrence,
+        PROD_TYPE.REPETITION_MANDATORY,
+        (<DSLMethodOptsWithErr<OUT>>actionORMethodDef).ERR_MSG
+      )
+    }
+
+    // note that while it may seem that this can cause an error because by using a recursive call to
+    // AT_LEAST_ONE we change the grammar to AT_LEAST_TWO, AT_LEAST_THREE ... , the possible recursive call
+    // from the tryInRepetitionRecovery(...) will only happen IFF there really are TWO/THREE/.... items.
+
+    // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
+    this.attemptInRepetitionRecovery(
+      this.atLeastOneInternal,
+      [prodOccurrence, actionORMethodDef],
+      <any>lookAheadFunc,
+      AT_LEAST_ONE_IDX,
+      prodOccurrence,
+      NextTerminalAfterAtLeastOneWalker
+    )
+  }
+
+  atLeastOneSepFirstInternal<OUT>(
+    this: MixedInParser,
+    prodOccurrence: number,
+    options: AtLeastOneSepMethodOpts<OUT>
+  ): void {
+    let laKey = this.getKeyForAutomaticLookahead(
+      AT_LEAST_ONE_SEP_IDX,
+      prodOccurrence
+    )
+    this.atLeastOneSepFirstInternalLogic(prodOccurrence, options, laKey)
+  }
+
+  atLeastOneSepFirstInternalLogic<OUT>(
+    this: MixedInParser,
+    prodOccurrence: number,
+    options: AtLeastOneSepMethodOpts<OUT>,
+    key: number
+  ): void {
+    let action = options.DEF
+    let separator = options.SEP
+
+    let firstIterationLookaheadFunc = this.getLaFuncFromCache(key)
+
+    // 1st iteration
+    if (firstIterationLookaheadFunc.call(this) === true) {
+      ;(<GrammarAction<OUT>>action).call(this)
+
+      //  TODO: Optimization can move this function construction into "attemptInRepetitionRecovery"
+      //  because it is only needed in error recovery scenarios.
+      let separatorLookAheadFunc = () => {
+        return this.tokenMatcher(this.LA(1), separator)
+      }
+
+      // 2nd..nth iterations
+      while (this.tokenMatcher(this.LA(1), separator) === true) {
+        // note that this CONSUME will never enter recovery because
+        // the separatorLookAheadFunc checks that the separator really does exist.
+        this.CONSUME(separator)
+        // No need for checking infinite loop here due to consuming the separator.
+        ;(<GrammarAction<OUT>>action).call(this)
+      }
+
+      // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
+      this.attemptInRepetitionRecovery(
+        this.repetitionSepSecondInternal,
+        [
+          prodOccurrence,
+          separator,
+          separatorLookAheadFunc,
+          action,
+          NextTerminalAfterAtLeastOneSepWalker
+        ],
+        separatorLookAheadFunc,
+        AT_LEAST_ONE_SEP_IDX,
+        prodOccurrence,
+        NextTerminalAfterAtLeastOneSepWalker
+      )
+    } else {
+      throw this.raiseEarlyExitException(
+        prodOccurrence,
+        PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR,
+        options.ERR_MSG
+      )
+    }
+  }
+
+  manyInternal<OUT>(
+    this: MixedInParser,
+    prodOccurrence: number,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
+  ): void {
+    let laKey = this.getKeyForAutomaticLookahead(MANY_IDX, prodOccurrence)
+    return this.manyInternalLogic(prodOccurrence, actionORMethodDef, laKey)
+  }
+
+  manyInternalLogic<OUT>(
+    this: MixedInParser,
+    prodOccurrence: number,
+    actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>,
+    key: number
+  ) {
+    let lookaheadFunction = this.getLaFuncFromCache(key)
+
+    let action
+    let predicate
+    if ((<DSLMethodOpts<OUT>>actionORMethodDef).DEF !== undefined) {
+      action = (<DSLMethodOpts<OUT>>actionORMethodDef).DEF
+      predicate = (<DSLMethodOpts<OUT>>actionORMethodDef).GATE
+      // predicate present
+      if (predicate !== undefined) {
+        let orgLookaheadFunction = lookaheadFunction
+        lookaheadFunction = () => {
+          return predicate.call(this) && orgLookaheadFunction.call(this)
+        }
+      }
+    } else {
+      action = actionORMethodDef
+    }
+
+    let notStuck = true
+    while (lookaheadFunction.call(this) === true && notStuck === true) {
+      notStuck = this.doSingleRepetition(action)
+    }
+
+    // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
+    this.attemptInRepetitionRecovery(
+      this.manyInternal,
+      [prodOccurrence, actionORMethodDef],
+      <any>lookaheadFunction,
+      MANY_IDX,
+      prodOccurrence,
+      NextTerminalAfterManyWalker,
+      // The notStuck parameter is only relevant when "attemptInRepetitionRecovery"
+      // is invoked from manyInternal, in the MANY_SEP case and AT_LEAST_ONE[_SEP]
+      // An infinite loop cannot occur as:
+      // - Either the lookahead is guaranteed to consume something (Single Token Separator)
+      // - AT_LEAST_ONE by definition is guaranteed to consume something (or error out).
+      notStuck
+    )
+  }
+
+  manySepFirstInternal<OUT>(
+    this: MixedInParser,
+    prodOccurrence: number,
+    options: ManySepMethodOpts<OUT>
+  ): void {
+    let laKey = this.getKeyForAutomaticLookahead(MANY_SEP_IDX, prodOccurrence)
+    this.manySepFirstInternalLogic(prodOccurrence, options, laKey)
+  }
+
+  manySepFirstInternalLogic<OUT>(
+    this: MixedInParser,
+    prodOccurrence: number,
+    options: ManySepMethodOpts<OUT>,
+    key: number
+  ): void {
+    let action = options.DEF
+    let separator = options.SEP
+    let firstIterationLaFunc = this.getLaFuncFromCache(key)
+
+    // 1st iteration
+    if (firstIterationLaFunc.call(this) === true) {
+      action.call(this)
+
+      let separatorLookAheadFunc = () => {
+        return this.tokenMatcher(this.LA(1), separator)
+      }
+      // 2nd..nth iterations
+      while (this.tokenMatcher(this.LA(1), separator) === true) {
+        // note that this CONSUME will never enter recovery because
+        // the separatorLookAheadFunc checks that the separator really does exist.
+        this.CONSUME(separator)
+        // No need for checking infinite loop here due to consuming the separator.
+        action.call(this)
+      }
+
+      // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
+      this.attemptInRepetitionRecovery(
+        this.repetitionSepSecondInternal,
+        [
+          prodOccurrence,
+          separator,
+          separatorLookAheadFunc,
+          action,
+          NextTerminalAfterManySepWalker
+        ],
+        separatorLookAheadFunc,
+        MANY_SEP_IDX,
+        prodOccurrence,
+        NextTerminalAfterManySepWalker
+      )
+    }
+  }
+
+  repetitionSepSecondInternal<OUT>(
+    this: MixedInParser,
+    prodOccurrence: number,
+    separator: TokenType,
+    separatorLookAheadFunc: () => boolean,
+    action: GrammarAction<OUT>,
+    nextTerminalAfterWalker: typeof AbstractNextTerminalAfterProductionWalker
+  ): void {
+    while (separatorLookAheadFunc()) {
+      // note that this CONSUME will never enter recovery because
+      // the separatorLookAheadFunc checks that the separator really does exist.
+      this.CONSUME(separator)
+      action.call(this)
+    }
+
+    // we can only arrive to this function after an error
+    // has occurred (hence the name 'second') so the following
+    // IF will always be entered, its possible to remove it...
+    // however it is kept to avoid confusion and be consistent.
+    // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
+    /* istanbul ignore else */
+    this.attemptInRepetitionRecovery(
+      this.repetitionSepSecondInternal,
+      [
+        prodOccurrence,
+        separator,
+        separatorLookAheadFunc,
+        action,
+        nextTerminalAfterWalker
+      ],
+      separatorLookAheadFunc,
+      AT_LEAST_ONE_SEP_IDX,
+      prodOccurrence,
+      nextTerminalAfterWalker
+    )
+  }
+
+  doSingleRepetition(this: MixedInParser, action: Function): any {
+    const beforeIteration = this.getLexerPosition()
+    action.call(this)
+    const afterIteration = this.getLexerPosition()
+
+    // This boolean will indicate if this repetition progressed
+    // or if we are "stuck" (potential infinite loop in the repetition).
+    return afterIteration > beforeIteration
+  }
+
+  orInternal<T>(
+    this: MixedInParser,
+    altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>,
+    occurrence: number
+  ): T {
+    let laKey = this.getKeyForAutomaticLookahead(OR_IDX, occurrence)
+    let alts = isArray(altsOrOpts)
+      ? (altsOrOpts as IOrAlt<any>[])
+      : (altsOrOpts as OrMethodOpts<unknown>).DEF
+
+    const laFunc = this.getLaFuncFromCache(laKey)
+    let altIdxToTake = laFunc.call(this, alts)
+    if (altIdxToTake !== undefined) {
+      let chosenAlternative: any = alts[altIdxToTake]
+      return chosenAlternative.ALT.call(this)
+    }
+    this.raiseNoAltException(
+      occurrence,
+      (altsOrOpts as OrMethodOpts<unknown>).ERR_MSG
+    )
+  }
+
+  ruleFinallyStateUpdate(this: MixedInParser): void {
+    this.RULE_STACK.pop()
+    this.RULE_OCCURRENCE_STACK.pop()
+
+    // NOOP when cst is disabled
+    this.cstFinallyStateUpdate()
+
+    if (this.RULE_STACK.length === 0 && this.isAtEndOfInput() === false) {
+      let firstRedundantTok = this.LA(1)
+      let errMsg = this.errorMessageProvider.buildNotAllInputParsedMessage({
+        firstRedundant: firstRedundantTok,
+        ruleName: this.getCurrRuleFullName()
+      })
+      this.SAVE_ERROR(new NotAllInputParsedException(errMsg, firstRedundantTok))
+    }
+  }
+
+  subruleInternal<T>(
+    this: MixedInParser,
+    ruleToCall: (idx: number) => T,
+    idx: number,
+    options?: SubruleMethodOpts
+  ) {
+    let ruleResult
+    try {
+      const args = options !== undefined ? options.ARGS : undefined
+      ruleResult = ruleToCall.call(this, idx, args)
+      this.cstPostNonTerminal(
+        ruleResult,
+        options !== undefined && options.LABEL !== undefined
+          ? options.LABEL
+          : (<any>ruleToCall).ruleName
+      )
+      return ruleResult
+    } catch (e) {
+      this.subruleInternalError(e, options, (<any>ruleToCall).ruleName)
+    }
+  }
+
+  subruleInternalError(
+    this: MixedInParser,
+    e: any,
+    options: SubruleMethodOpts,
+    ruleName: string
+  ): void {
+    if (isRecognitionException(e) && e.partialCstResult !== undefined) {
+      this.cstPostNonTerminal(
+        e.partialCstResult,
+        options !== undefined && options.LABEL !== undefined
+          ? options.LABEL
+          : ruleName
+      )
+
+      delete e.partialCstResult
+    }
+    throw e
+  }
+
+  consumeInternal(
+    this: MixedInParser,
+    tokType: TokenType,
+    idx: number,
+    options: ConsumeMethodOpts
+  ): IToken {
+    let consumedToken
+    try {
+      let nextToken = this.LA(1)
+      if (this.tokenMatcher(nextToken, tokType) === true) {
+        this.consumeToken()
+        consumedToken = nextToken
+      } else {
+        this.consumeInternalError(tokType, nextToken, options)
+      }
+    } catch (eFromConsumption) {
+      consumedToken = this.consumeInternalRecovery(
+        tokType,
+        idx,
+        eFromConsumption
+      )
+    }
+
+    this.cstPostTerminal(
+      options !== undefined && options.LABEL !== undefined
+        ? options.LABEL
+        : tokType.name,
+      consumedToken
+    )
+    return consumedToken
+  }
+
+  consumeInternalError(
+    this: MixedInParser,
+    tokType: TokenType,
+    nextToken: IToken,
+    options: ConsumeMethodOpts
+  ): void {
+    let msg
+    let previousToken = this.LA(0)
+    if (options !== undefined && options.ERR_MSG) {
+      msg = options.ERR_MSG
+    } else {
+      msg = this.errorMessageProvider.buildMismatchTokenMessage({
+        expected: tokType,
+        actual: nextToken,
+        previous: previousToken,
+        ruleName: this.getCurrRuleFullName()
+      })
+    }
+    throw this.SAVE_ERROR(
+      new MismatchedTokenException(msg, nextToken, previousToken)
+    )
+  }
+
+  consumeInternalRecovery(
+    this: MixedInParser,
+    tokType: TokenType,
+    idx: number,
+    eFromConsumption: Error
+  ): IToken {
+    // no recovery allowed during backtracking, otherwise backtracking may recover invalid syntax and accept it
+    // but the original syntax could have been parsed successfully without any backtracking + recovery
+    if (
+      this.recoveryEnabled &&
+      // TODO: more robust checking of the exception type. Perhaps Typescript extending expressions?
+      eFromConsumption.name === "MismatchedTokenException" &&
+      !this.isBackTracking()
+    ) {
+      let follows = this.getFollowsForInRuleRecovery(<any>tokType, idx)
+      try {
+        return this.tryInRuleRecovery(<any>tokType, follows)
+      } catch (eFromInRuleRecovery) {
+        if (eFromInRuleRecovery.name === IN_RULE_RECOVERY_EXCEPTION) {
+          // failed in RuleRecovery.
+          // throw the original error in order to trigger reSync error recovery
+          throw eFromConsumption
+        } else {
+          throw eFromInRuleRecovery
+        }
+      }
+    } else {
+      throw eFromConsumption
+    }
+  }
+
+  saveRecogState(this: MixedInParser): IParserState {
+    // errors is a getter which will clone the errors array
+    let savedErrors = this.errors
+    let savedRuleStack = cloneArr(this.RULE_STACK)
+    return {
+      errors: savedErrors,
+      lexerState: this.exportLexerState(),
+      RULE_STACK: savedRuleStack,
+      CST_STACK: this.CST_STACK
+    }
+  }
+
+  reloadRecogState(this: MixedInParser, newState: IParserState) {
+    this.errors = newState.errors
+    this.importLexerState(newState.lexerState)
+    this.RULE_STACK = newState.RULE_STACK
+  }
+
+  ruleInvocationStateUpdate(
+    this: MixedInParser,
+    shortName: string,
+    fullName: string,
+    idxInCallingRule: number
+  ): void {
+    this.RULE_OCCURRENCE_STACK.push(idxInCallingRule)
+    this.RULE_STACK.push(shortName)
+    // NOOP when cst is disabled
+    this.cstInvocationStateUpdate(fullName, shortName)
+  }
+
+  isBackTracking(this: MixedInParser): boolean {
+    return this.isBackTrackingStack.length !== 0
+  }
+
+  getCurrRuleFullName(this: MixedInParser): string {
+    let shortName = this.getLastExplicitRuleShortName()
+    return this.shortRuleNameToFull[shortName]
+  }
+
+  shortRuleNameToFullName(this: MixedInParser, shortName: string) {
+    return this.shortRuleNameToFull[shortName]
+  }
+
+  public isAtEndOfInput(this: MixedInParser): boolean {
+    return this.tokenMatcher(this.LA(1), EOF)
+  }
+
+  public reset(this: MixedInParser): void {
+    this.resetLexerState()
+
+    this.isBackTrackingStack = []
+    this.errors = []
+    this.RULE_STACK = []
+    // TODO: extract a specific reset for TreeBuilder trait
+    this.CST_STACK = []
+    this.RULE_OCCURRENCE_STACK = []
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/recoverable.ts

@@ -0,0 +1,459 @@
+import { createTokenInstance, EOF } from "../../../scan/tokens_public"
+import {
+  AbstractNextTerminalAfterProductionWalker,
+  IFirstAfterRepetition
+} from "../../grammar/interpreter"
+import {
+  cloneArr,
+  contains,
+  dropRight,
+  find,
+  flatten,
+  has,
+  isEmpty,
+  map
+} from "../../../utils/utils"
+import {
+  IParserConfig,
+  IToken,
+  ITokenGrammarPath,
+  TokenType
+} from "../../../../api"
+import { MismatchedTokenException } from "../../exceptions_public"
+import { IN } from "../../constants"
+import { MixedInParser } from "./parser_traits"
+import { DEFAULT_PARSER_CONFIG } from "../parser"
+
+export const EOF_FOLLOW_KEY: any = {}
+
+export interface IFollowKey {
+  ruleName: string
+  idxInCallingRule: number
+  inRule: string
+}
+
+export const IN_RULE_RECOVERY_EXCEPTION = "InRuleRecoveryException"
+
+export function InRuleRecoveryException(message: string) {
+  this.name = IN_RULE_RECOVERY_EXCEPTION
+  this.message = message
+}
+
+InRuleRecoveryException.prototype = Error.prototype
+
+/**
+ * This trait is responsible for the error recovery and fault tolerant logic
+ */
+export class Recoverable {
+  recoveryEnabled: boolean
+  firstAfterRepMap: Record<string, IFirstAfterRepetition>
+  resyncFollows: Record<string, TokenType[]>
+
+  initRecoverable(config: IParserConfig) {
+    this.firstAfterRepMap = {}
+    this.resyncFollows = {}
+
+    this.recoveryEnabled = has(config, "recoveryEnabled")
+      ? config.recoveryEnabled
+      : DEFAULT_PARSER_CONFIG.recoveryEnabled
+
+    // performance optimization, NOOP will be inlined which
+    // effectively means that this optional feature does not exist
+    // when not used.
+    if (this.recoveryEnabled) {
+      this.attemptInRepetitionRecovery = attemptInRepetitionRecovery
+    }
+  }
+
+  public getTokenToInsert(tokType: TokenType): IToken {
+    let tokToInsert = createTokenInstance(
+      tokType,
+      "",
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN,
+      NaN
+    )
+    tokToInsert.isInsertedInRecovery = true
+    return tokToInsert
+  }
+
+  public canTokenTypeBeInsertedInRecovery(tokType: TokenType) {
+    return true
+  }
+
+  tryInRepetitionRecovery(
+    this: MixedInParser,
+    grammarRule: Function,
+    grammarRuleArgs: any[],
+    lookAheadFunc: () => boolean,
+    expectedTokType: TokenType
+  ): void {
+    // TODO: can the resyncTokenType be cached?
+    let reSyncTokType = this.findReSyncTokenType()
+    let savedLexerState = this.exportLexerState()
+    let resyncedTokens = []
+    let passedResyncPoint = false
+
+    let nextTokenWithoutResync = this.LA(1)
+    let currToken = this.LA(1)
+
+    let generateErrorMessage = () => {
+      let previousToken = this.LA(0)
+      // we are preemptively re-syncing before an error has been detected, therefor we must reproduce
+      // the error that would have been thrown
+      let msg = this.errorMessageProvider.buildMismatchTokenMessage({
+        expected: expectedTokType,
+        actual: nextTokenWithoutResync,
+        previous: previousToken,
+        ruleName: this.getCurrRuleFullName()
+      })
+      let error = new MismatchedTokenException(
+        msg,
+        nextTokenWithoutResync,
+        this.LA(0)
+      )
+      // the first token here will be the original cause of the error, this is not part of the resyncedTokens property.
+      error.resyncedTokens = dropRight(resyncedTokens)
+      this.SAVE_ERROR(error)
+    }
+
+    while (!passedResyncPoint) {
+      // re-synced to a point where we can safely exit the repetition/
+      if (this.tokenMatcher(currToken, expectedTokType)) {
+        generateErrorMessage()
+        return // must return here to avoid reverting the inputIdx
+      } else if (lookAheadFunc.call(this)) {
+        // we skipped enough tokens so we can resync right back into another iteration of the repetition grammar rule
+        generateErrorMessage()
+        // recursive invocation in other to support multiple re-syncs in the same top level repetition grammar rule
+        grammarRule.apply(this, grammarRuleArgs)
+        return // must return here to avoid reverting the inputIdx
+      } else if (this.tokenMatcher(currToken, reSyncTokType)) {
+        passedResyncPoint = true
+      } else {
+        currToken = this.SKIP_TOKEN()
+        this.addToResyncTokens(currToken, resyncedTokens)
+      }
+    }
+
+    // we were unable to find a CLOSER point to resync inside the Repetition, reset the state.
+    // The parsing exception we were trying to prevent will happen in the NEXT parsing step. it may be handled by
+    // "between rules" resync recovery later in the flow.
+    this.importLexerState(savedLexerState)
+  }
+
+  shouldInRepetitionRecoveryBeTried(
+    this: MixedInParser,
+    expectTokAfterLastMatch: TokenType,
+    nextTokIdx: number,
+    notStuck: boolean | undefined
+  ): boolean {
+    // Edge case of arriving from a MANY repetition which is stuck
+    // Attempting recovery in this case could cause an infinite loop
+    if (notStuck === false) {
+      return false
+    }
+
+    // arguments to try and perform resync into the next iteration of the many are missing
+    if (expectTokAfterLastMatch === undefined || nextTokIdx === undefined) {
+      return false
+    }
+
+    // no need to recover, next token is what we expect...
+    if (this.tokenMatcher(this.LA(1), expectTokAfterLastMatch)) {
+      return false
+    }
+
+    // error recovery is disabled during backtracking as it can make the parser ignore a valid grammar path
+    // and prefer some backtracking path that includes recovered errors.
+    if (this.isBackTracking()) {
+      return false
+    }
+
+    // if we can perform inRule recovery (single token insertion or deletion) we always prefer that recovery algorithm
+    // because if it works, it makes the least amount of changes to the input stream (greedy algorithm)
+    //noinspection RedundantIfStatementJS
+    if (
+      this.canPerformInRuleRecovery(
+        expectTokAfterLastMatch,
+        this.getFollowsForInRuleRecovery(expectTokAfterLastMatch, nextTokIdx)
+      )
+    ) {
+      return false
+    }
+
+    return true
+  }
+
+  // Error Recovery functionality
+  getFollowsForInRuleRecovery(
+    this: MixedInParser,
+    tokType: TokenType,
+    tokIdxInRule: number
+  ): TokenType[] {
+    let grammarPath = this.getCurrentGrammarPath(tokType, tokIdxInRule)
+    let follows = this.getNextPossibleTokenTypes(grammarPath)
+    return follows
+  }
+
+  tryInRuleRecovery(
+    this: MixedInParser,
+    expectedTokType: TokenType,
+    follows: TokenType[]
+  ): IToken {
+    if (this.canRecoverWithSingleTokenInsertion(expectedTokType, follows)) {
+      let tokToInsert = this.getTokenToInsert(expectedTokType)
+      return tokToInsert
+    }
+
+    if (this.canRecoverWithSingleTokenDeletion(expectedTokType)) {
+      let nextTok = this.SKIP_TOKEN()
+      this.consumeToken()
+      return nextTok
+    }
+
+    throw new InRuleRecoveryException("sad sad panda")
+  }
+
+  canPerformInRuleRecovery(
+    this: MixedInParser,
+    expectedToken: TokenType,
+    follows: TokenType[]
+  ): boolean {
+    return (
+      this.canRecoverWithSingleTokenInsertion(expectedToken, follows) ||
+      this.canRecoverWithSingleTokenDeletion(expectedToken)
+    )
+  }
+
+  canRecoverWithSingleTokenInsertion(
+    this: MixedInParser,
+    expectedTokType: TokenType,
+    follows: TokenType[]
+  ): boolean {
+    if (!this.canTokenTypeBeInsertedInRecovery(expectedTokType)) {
+      return false
+    }
+
+    // must know the possible following tokens to perform single token insertion
+    if (isEmpty(follows)) {
+      return false
+    }
+
+    let mismatchedTok = this.LA(1)
+    let isMisMatchedTokInFollows =
+      find(follows, (possibleFollowsTokType: TokenType) => {
+        return this.tokenMatcher(mismatchedTok, possibleFollowsTokType)
+      }) !== undefined
+
+    return isMisMatchedTokInFollows
+  }
+
+  canRecoverWithSingleTokenDeletion(
+    this: MixedInParser,
+    expectedTokType: TokenType
+  ): boolean {
+    let isNextTokenWhatIsExpected = this.tokenMatcher(
+      this.LA(2),
+      expectedTokType
+    )
+    return isNextTokenWhatIsExpected
+  }
+
+  isInCurrentRuleReSyncSet(
+    this: MixedInParser,
+    tokenTypeIdx: TokenType
+  ): boolean {
+    let followKey = this.getCurrFollowKey()
+    let currentRuleReSyncSet = this.getFollowSetFromFollowKey(followKey)
+    return contains(currentRuleReSyncSet, tokenTypeIdx)
+  }
+
+  findReSyncTokenType(this: MixedInParser): TokenType {
+    let allPossibleReSyncTokTypes = this.flattenFollowSet()
+    // this loop will always terminate as EOF is always in the follow stack and also always (virtually) in the input
+    let nextToken = this.LA(1)
+    let k = 2
+    while (true) {
+      let nextTokenType: any = nextToken.tokenType
+      if (contains(allPossibleReSyncTokTypes, nextTokenType)) {
+        return nextTokenType
+      }
+      nextToken = this.LA(k)
+      k++
+    }
+  }
+
+  getCurrFollowKey(this: MixedInParser): IFollowKey {
+    // the length is at least one as we always add the ruleName to the stack before invoking the rule.
+    if (this.RULE_STACK.length === 1) {
+      return EOF_FOLLOW_KEY
+    }
+    let currRuleShortName = this.getLastExplicitRuleShortName()
+    let currRuleIdx = this.getLastExplicitRuleOccurrenceIndex()
+    let prevRuleShortName = this.getPreviousExplicitRuleShortName()
+
+    return {
+      ruleName: this.shortRuleNameToFullName(currRuleShortName),
+      idxInCallingRule: currRuleIdx,
+      inRule: this.shortRuleNameToFullName(prevRuleShortName)
+    }
+  }
+
+  buildFullFollowKeyStack(this: MixedInParser): IFollowKey[] {
+    let explicitRuleStack = this.RULE_STACK
+    let explicitOccurrenceStack = this.RULE_OCCURRENCE_STACK
+
+    return map(explicitRuleStack, (ruleName, idx) => {
+      if (idx === 0) {
+        return EOF_FOLLOW_KEY
+      }
+      return {
+        ruleName: this.shortRuleNameToFullName(ruleName),
+        idxInCallingRule: explicitOccurrenceStack[idx],
+        inRule: this.shortRuleNameToFullName(explicitRuleStack[idx - 1])
+      }
+    })
+  }
+
+  flattenFollowSet(this: MixedInParser): TokenType[] {
+    let followStack = map(this.buildFullFollowKeyStack(), (currKey) => {
+      return this.getFollowSetFromFollowKey(currKey)
+    })
+    return <any>flatten(followStack)
+  }
+
+  getFollowSetFromFollowKey(
+    this: MixedInParser,
+    followKey: IFollowKey
+  ): TokenType[] {
+    if (followKey === EOF_FOLLOW_KEY) {
+      return [EOF]
+    }
+
+    let followName =
+      followKey.ruleName + followKey.idxInCallingRule + IN + followKey.inRule
+
+    return this.resyncFollows[followName]
+  }
+
+  // It does not make any sense to include a virtual EOF token in the list of resynced tokens
+  // as EOF does not really exist and thus does not contain any useful information (line/column numbers)
+  addToResyncTokens(
+    this: MixedInParser,
+    token: IToken,
+    resyncTokens: IToken[]
+  ): IToken[] {
+    if (!this.tokenMatcher(token, EOF)) {
+      resyncTokens.push(token)
+    }
+    return resyncTokens
+  }
+
+  reSyncTo(this: MixedInParser, tokType: TokenType): IToken[] {
+    let resyncedTokens = []
+    let nextTok = this.LA(1)
+    while (this.tokenMatcher(nextTok, tokType) === false) {
+      nextTok = this.SKIP_TOKEN()
+      this.addToResyncTokens(nextTok, resyncedTokens)
+    }
+    // the last token is not part of the error.
+    return dropRight(resyncedTokens)
+  }
+
+  attemptInRepetitionRecovery(
+    this: MixedInParser,
+    prodFunc: Function,
+    args: any[],
+    lookaheadFunc: () => boolean,
+    dslMethodIdx: number,
+    prodOccurrence: number,
+    nextToksWalker: typeof AbstractNextTerminalAfterProductionWalker,
+    notStuck?: boolean
+  ): void {
+    // by default this is a NO-OP
+    // The actual implementation is with the function(not method) below
+  }
+
+  getCurrentGrammarPath(
+    this: MixedInParser,
+    tokType: TokenType,
+    tokIdxInRule: number
+  ): ITokenGrammarPath {
+    let pathRuleStack: string[] = this.getHumanReadableRuleStack()
+    let pathOccurrenceStack: number[] = cloneArr(this.RULE_OCCURRENCE_STACK)
+    let grammarPath: any = {
+      ruleStack: pathRuleStack,
+      occurrenceStack: pathOccurrenceStack,
+      lastTok: tokType,
+      lastTokOccurrence: tokIdxInRule
+    }
+
+    return grammarPath
+  }
+  getHumanReadableRuleStack(this: MixedInParser): string[] {
+    return map(this.RULE_STACK, (currShortName) =>
+      this.shortRuleNameToFullName(currShortName)
+    )
+  }
+}
+
+export function attemptInRepetitionRecovery(
+  this: MixedInParser,
+  prodFunc: Function,
+  args: any[],
+  lookaheadFunc: () => boolean,
+  dslMethodIdx: number,
+  prodOccurrence: number,
+  nextToksWalker: typeof AbstractNextTerminalAfterProductionWalker,
+  notStuck?: boolean
+) {
+  let key = this.getKeyForAutomaticLookahead(dslMethodIdx, prodOccurrence)
+  let firstAfterRepInfo = this.firstAfterRepMap[key]
+  if (firstAfterRepInfo === undefined) {
+    let currRuleName = this.getCurrRuleFullName()
+    let ruleGrammar = this.getGAstProductions()[currRuleName]
+    let walker: AbstractNextTerminalAfterProductionWalker = new nextToksWalker(
+      ruleGrammar,
+      prodOccurrence
+    )
+    firstAfterRepInfo = walker.startWalking()
+    this.firstAfterRepMap[key] = firstAfterRepInfo
+  }
+
+  let expectTokAfterLastMatch = firstAfterRepInfo.token
+  let nextTokIdx = firstAfterRepInfo.occurrence
+  let isEndOfRule = firstAfterRepInfo.isEndOfRule
+
+  // special edge case of a TOP most repetition after which the input should END.
+  // this will force an attempt for inRule recovery in that scenario.
+  if (
+    this.RULE_STACK.length === 1 &&
+    isEndOfRule &&
+    expectTokAfterLastMatch === undefined
+  ) {
+    expectTokAfterLastMatch = EOF
+    nextTokIdx = 1
+  }
+
+  if (
+    this.shouldInRepetitionRecoveryBeTried(
+      expectTokAfterLastMatch,
+      nextTokIdx,
+      notStuck
+    )
+  ) {
+    // TODO: performance optimization: instead of passing the original args here, we modify
+    // the args param (or create a new one) and make sure the lookahead func is explicitly provided
+    // to avoid searching the cache for it once more.
+    this.tryInRepetitionRecovery(
+      prodFunc,
+      args,
+      lookaheadFunc,
+      expectTokAfterLastMatch
+    )
+  }
+}

node_modules/chevrotain/src/parse/parser/traits/tree_builder.ts

@@ -0,0 +1,275 @@
+import {
+  addNoneTerminalToCst,
+  addTerminalToCst,
+  setNodeLocationFull,
+  setNodeLocationOnlyOffset
+} from "../../cst/cst"
+import { has, isUndefined, keys, NOOP } from "../../../utils/utils"
+import {
+  createBaseSemanticVisitorConstructor,
+  createBaseVisitorConstructorWithDefaults
+} from "../../cst/cst_visitor"
+import {
+  CstNode,
+  CstNodeLocation,
+  ICstVisitor,
+  IParserConfig,
+  IToken,
+  nodeLocationTrackingOptions
+} from "../../../../api"
+import { MixedInParser } from "./parser_traits"
+import { DEFAULT_PARSER_CONFIG } from "../parser"
+
+/**
+ * This trait is responsible for the CST building logic.
+ */
+export class TreeBuilder {
+  outputCst: boolean
+  CST_STACK: CstNode[]
+  baseCstVisitorConstructor: Function
+  baseCstVisitorWithDefaultsConstructor: Function
+
+  // dynamically assigned Methods
+  setNodeLocationFromNode: (
+    nodeLocation: CstNodeLocation,
+    locationInformation: CstNodeLocation
+  ) => void
+  setNodeLocationFromToken: (
+    nodeLocation: CstNodeLocation,
+    locationInformation: CstNodeLocation
+  ) => void
+  cstPostRule: (this: MixedInParser, ruleCstNode: CstNode) => void
+
+  setInitialNodeLocation: (cstNode: CstNode) => void
+  nodeLocationTracking: nodeLocationTrackingOptions
+
+  initTreeBuilder(this: MixedInParser, config: IParserConfig) {
+    this.CST_STACK = []
+
+    // outputCst is no longer exposed/defined in the pubic API
+    this.outputCst = (config as any).outputCst
+
+    this.nodeLocationTracking = has(config, "nodeLocationTracking")
+      ? config.nodeLocationTracking
+      : DEFAULT_PARSER_CONFIG.nodeLocationTracking
+
+    if (!this.outputCst) {
+      this.cstInvocationStateUpdate = NOOP
+      this.cstFinallyStateUpdate = NOOP
+      this.cstPostTerminal = NOOP
+      this.cstPostNonTerminal = NOOP
+      this.cstPostRule = NOOP
+    } else {
+      if (/full/i.test(this.nodeLocationTracking)) {
+        if (this.recoveryEnabled) {
+          this.setNodeLocationFromToken = setNodeLocationFull
+          this.setNodeLocationFromNode = setNodeLocationFull
+          this.cstPostRule = NOOP
+          this.setInitialNodeLocation = this.setInitialNodeLocationFullRecovery
+        } else {
+          this.setNodeLocationFromToken = NOOP
+          this.setNodeLocationFromNode = NOOP
+          this.cstPostRule = this.cstPostRuleFull
+          this.setInitialNodeLocation = this.setInitialNodeLocationFullRegular
+        }
+      } else if (/onlyOffset/i.test(this.nodeLocationTracking)) {
+        if (this.recoveryEnabled) {
+          this.setNodeLocationFromToken = <any>setNodeLocationOnlyOffset
+          this.setNodeLocationFromNode = <any>setNodeLocationOnlyOffset
+          this.cstPostRule = NOOP
+          this.setInitialNodeLocation = this.setInitialNodeLocationOnlyOffsetRecovery
+        } else {
+          this.setNodeLocationFromToken = NOOP
+          this.setNodeLocationFromNode = NOOP
+          this.cstPostRule = this.cstPostRuleOnlyOffset
+          this.setInitialNodeLocation = this.setInitialNodeLocationOnlyOffsetRegular
+        }
+      } else if (/none/i.test(this.nodeLocationTracking)) {
+        this.setNodeLocationFromToken = NOOP
+        this.setNodeLocationFromNode = NOOP
+        this.cstPostRule = NOOP
+        this.setInitialNodeLocation = NOOP
+      } else {
+        throw Error(
+          `Invalid <nodeLocationTracking> config option: "${config.nodeLocationTracking}"`
+        )
+      }
+    }
+  }
+
+  setInitialNodeLocationOnlyOffsetRecovery(
+    this: MixedInParser,
+    cstNode: any
+  ): void {
+    cstNode.location = {
+      startOffset: NaN,
+      endOffset: NaN
+    }
+  }
+
+  setInitialNodeLocationOnlyOffsetRegular(
+    this: MixedInParser,
+    cstNode: any
+  ): void {
+    cstNode.location = {
+      // without error recovery the starting Location of a new CstNode is guaranteed
+      // To be the next Token's startOffset (for valid inputs).
+      // For invalid inputs there won't be any CSTOutput so this potential
+      // inaccuracy does not matter
+      startOffset: this.LA(1).startOffset,
+      endOffset: NaN
+    }
+  }
+
+  setInitialNodeLocationFullRecovery(this: MixedInParser, cstNode: any): void {
+    cstNode.location = {
+      startOffset: NaN,
+      startLine: NaN,
+      startColumn: NaN,
+      endOffset: NaN,
+      endLine: NaN,
+      endColumn: NaN
+    }
+  }
+
+  /**
+     *  @see setInitialNodeLocationOnlyOffsetRegular for explanation why this work
+
+     * @param cstNode
+     */
+  setInitialNodeLocationFullRegular(this: MixedInParser, cstNode: any): void {
+    const nextToken = this.LA(1)
+    cstNode.location = {
+      startOffset: nextToken.startOffset,
+      startLine: nextToken.startLine,
+      startColumn: nextToken.startColumn,
+      endOffset: NaN,
+      endLine: NaN,
+      endColumn: NaN
+    }
+  }
+
+  cstInvocationStateUpdate(
+    this: MixedInParser,
+    fullRuleName: string,
+    shortName: string | number
+  ): void {
+    const cstNode: CstNode = {
+      name: fullRuleName,
+      children: {}
+    }
+
+    this.setInitialNodeLocation(cstNode)
+    this.CST_STACK.push(cstNode)
+  }
+
+  cstFinallyStateUpdate(this: MixedInParser): void {
+    this.CST_STACK.pop()
+  }
+
+  cstPostRuleFull(this: MixedInParser, ruleCstNode: CstNode): void {
+    const prevToken = this.LA(0)
+    const loc = ruleCstNode.location
+
+    // If this condition is true it means we consumed at least one Token
+    // In this CstNode.
+    if (loc.startOffset <= prevToken.startOffset === true) {
+      loc.endOffset = prevToken.endOffset
+      loc.endLine = prevToken.endLine
+      loc.endColumn = prevToken.endColumn
+    }
+    // "empty" CstNode edge case
+    else {
+      loc.startOffset = NaN
+      loc.startLine = NaN
+      loc.startColumn = NaN
+    }
+  }
+
+  cstPostRuleOnlyOffset(this: MixedInParser, ruleCstNode: CstNode): void {
+    const prevToken = this.LA(0)
+    const loc = ruleCstNode.location
+
+    // If this condition is true it means we consumed at least one Token
+    // In this CstNode.
+    if (loc.startOffset <= prevToken.startOffset === true) {
+      loc.endOffset = prevToken.endOffset
+    }
+    // "empty" CstNode edge case
+    else {
+      loc.startOffset = NaN
+    }
+  }
+
+  cstPostTerminal(
+    this: MixedInParser,
+    key: string,
+    consumedToken: IToken
+  ): void {
+    const rootCst = this.CST_STACK[this.CST_STACK.length - 1]
+    addTerminalToCst(rootCst, consumedToken, key)
+    // This is only used when **both** error recovery and CST Output are enabled.
+    this.setNodeLocationFromToken(rootCst.location, <any>consumedToken)
+  }
+
+  cstPostNonTerminal(
+    this: MixedInParser,
+    ruleCstResult: CstNode,
+    ruleName: string
+  ): void {
+    const preCstNode = this.CST_STACK[this.CST_STACK.length - 1]
+    addNoneTerminalToCst(preCstNode, ruleName, ruleCstResult)
+    // This is only used when **both** error recovery and CST Output are enabled.
+    this.setNodeLocationFromNode(preCstNode.location, ruleCstResult.location)
+  }
+
+  getBaseCstVisitorConstructor<IN = any, OUT = any>(
+    this: MixedInParser
+  ): {
+    new (...args: any[]): ICstVisitor<IN, OUT>
+  } {
+    if (isUndefined(this.baseCstVisitorConstructor)) {
+      const newBaseCstVisitorConstructor = createBaseSemanticVisitorConstructor(
+        this.className,
+        keys(this.gastProductionsCache)
+      )
+      this.baseCstVisitorConstructor = newBaseCstVisitorConstructor
+      return newBaseCstVisitorConstructor
+    }
+
+    return <any>this.baseCstVisitorConstructor
+  }
+
+  getBaseCstVisitorConstructorWithDefaults<IN = any, OUT = any>(
+    this: MixedInParser
+  ): {
+    new (...args: any[]): ICstVisitor<IN, OUT>
+  } {
+    if (isUndefined(this.baseCstVisitorWithDefaultsConstructor)) {
+      const newConstructor = createBaseVisitorConstructorWithDefaults(
+        this.className,
+        keys(this.gastProductionsCache),
+        this.getBaseCstVisitorConstructor()
+      )
+      this.baseCstVisitorWithDefaultsConstructor = newConstructor
+      return newConstructor
+    }
+
+    return <any>this.baseCstVisitorWithDefaultsConstructor
+  }
+
+  getLastExplicitRuleShortName(this: MixedInParser): string {
+    let ruleStack = this.RULE_STACK
+    return ruleStack[ruleStack.length - 1]
+  }
+
+  getPreviousExplicitRuleShortName(this: MixedInParser): string {
+    let ruleStack = this.RULE_STACK
+    return ruleStack[ruleStack.length - 2]
+  }
+
+  getLastExplicitRuleOccurrenceIndex(this: MixedInParser): number {
+    let occurrenceStack = this.RULE_OCCURRENCE_STACK
+    return occurrenceStack[occurrenceStack.length - 1]
+  }
+}