Add capacitorjs runtime
This commit is contained in:
parent
d0ece489ee
commit
f90c0e6c40
8362 changed files with 1502407 additions and 1 deletions
97
node_modules/chevrotain/src/api.ts
generated
vendored
Normal file
97
node_modules/chevrotain/src/api.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
/* istanbul ignore file - tricky to import some things from this module during testing */
|
||||
|
||||
// semantic version
|
||||
export { VERSION } from "./version"
|
||||
|
||||
export {
|
||||
CstParser,
|
||||
EmbeddedActionsParser,
|
||||
ParserDefinitionErrorType,
|
||||
EMPTY_ALT
|
||||
} from "./parse/parser/parser"
|
||||
|
||||
export { Lexer, LexerDefinitionErrorType } from "./scan/lexer_public"
|
||||
|
||||
// Tokens utilities
|
||||
|
||||
export {
|
||||
createToken,
|
||||
createTokenInstance,
|
||||
EOF,
|
||||
tokenLabel,
|
||||
tokenMatcher,
|
||||
tokenName
|
||||
} from "./scan/tokens_public"
|
||||
|
||||
// Other Utilities
|
||||
|
||||
export {
|
||||
defaultGrammarResolverErrorProvider,
|
||||
defaultGrammarValidatorErrorProvider,
|
||||
defaultParserErrorProvider
|
||||
} from "./parse/errors_public"
|
||||
|
||||
export {
|
||||
EarlyExitException,
|
||||
isRecognitionException,
|
||||
MismatchedTokenException,
|
||||
NotAllInputParsedException,
|
||||
NoViableAltException
|
||||
} from "./parse/exceptions_public"
|
||||
|
||||
export { defaultLexerErrorProvider } from "./scan/lexer_errors_public"
|
||||
|
||||
// grammar reflection API
|
||||
|
||||
export {
|
||||
Alternation,
|
||||
Alternative,
|
||||
NonTerminal,
|
||||
Option,
|
||||
Repetition,
|
||||
RepetitionMandatory,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
RepetitionWithSeparator,
|
||||
Rule,
|
||||
Terminal
|
||||
} from "./parse/grammar/gast/gast_public"
|
||||
|
||||
// GAST Utilities
|
||||
|
||||
export {
|
||||
serializeGrammar,
|
||||
serializeProduction
|
||||
} from "./parse/grammar/gast/gast_public"
|
||||
|
||||
export { GAstVisitor } from "./parse/grammar/gast/gast_visitor_public"
|
||||
|
||||
export {
|
||||
assignOccurrenceIndices,
|
||||
resolveGrammar,
|
||||
validateGrammar
|
||||
} from "./parse/grammar/gast/gast_resolver_public"
|
||||
|
||||
/* istanbul ignore next */
|
||||
export function clearCache() {
|
||||
console.warn(
|
||||
"The clearCache function was 'soft' removed from the Chevrotain API." +
|
||||
"\n\t It performs no action other than printing this message." +
|
||||
"\n\t Please avoid using it as it will be completely removed in the future"
|
||||
)
|
||||
}
|
||||
|
||||
export { createSyntaxDiagramsCode } from "./diagrams/render_public"
|
||||
|
||||
export {
|
||||
generateParserFactory,
|
||||
generateParserModule
|
||||
} from "./generate/generate_public"
|
||||
|
||||
export class Parser {
|
||||
constructor() {
|
||||
throw new Error(
|
||||
"The Parser class has been deprecated, use CstParser or EmbeddedActionsParser instead.\t\n" +
|
||||
"See: https://sap.github.io/chevrotain/docs/changes/BREAKING_CHANGES.html#_7-0-0"
|
||||
)
|
||||
}
|
||||
}
|
||||
53
node_modules/chevrotain/src/diagrams/render_public.ts
generated
vendored
Normal file
53
node_modules/chevrotain/src/diagrams/render_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { VERSION } from "../version"
|
||||
import { ISerializedGast } from "../../api"
|
||||
|
||||
export function createSyntaxDiagramsCode(
|
||||
grammar: ISerializedGast[],
|
||||
{
|
||||
resourceBase = `https://unpkg.com/chevrotain@${VERSION}/diagrams/`,
|
||||
css = `https://unpkg.com/chevrotain@${VERSION}/diagrams/diagrams.css`
|
||||
}: {
|
||||
resourceBase?: string
|
||||
css?: string
|
||||
} = {}
|
||||
) {
|
||||
const header = `
|
||||
<!-- This is a generated file -->
|
||||
<!DOCTYPE html>
|
||||
<meta charset="utf-8">
|
||||
<style>
|
||||
body {
|
||||
background-color: hsl(30, 20%, 95%)
|
||||
}
|
||||
</style>
|
||||
|
||||
`
|
||||
const cssHtml = `
|
||||
<link rel='stylesheet' href='${css}'>
|
||||
`
|
||||
|
||||
const scripts = `
|
||||
<script src='${resourceBase}vendor/railroad-diagrams.js'></script>
|
||||
<script src='${resourceBase}src/diagrams_builder.js'></script>
|
||||
<script src='${resourceBase}src/diagrams_behavior.js'></script>
|
||||
<script src='${resourceBase}src/main.js'></script>
|
||||
`
|
||||
const diagramsDiv = `
|
||||
<div id="diagrams" align="center"></div>
|
||||
`
|
||||
const serializedGrammar = `
|
||||
<script>
|
||||
window.serializedGrammar = ${JSON.stringify(grammar, null, " ")};
|
||||
</script>
|
||||
`
|
||||
|
||||
const initLogic = `
|
||||
<script>
|
||||
var diagramsDiv = document.getElementById("diagrams");
|
||||
main.drawDiagramsFromSerializedGrammar(serializedGrammar, diagramsDiv);
|
||||
</script>
|
||||
`
|
||||
return (
|
||||
header + cssHtml + scripts + diagramsDiv + serializedGrammar + initLogic
|
||||
)
|
||||
}
|
||||
199
node_modules/chevrotain/src/generate/generate.ts
generated
vendored
Normal file
199
node_modules/chevrotain/src/generate/generate.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
import { forEach, map } from "../utils/utils"
|
||||
import {
|
||||
RepetitionMandatory,
|
||||
Option,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
RepetitionWithSeparator,
|
||||
Terminal,
|
||||
NonTerminal,
|
||||
Alternation,
|
||||
Alternative,
|
||||
Repetition
|
||||
} from "../parse/grammar/gast/gast_public"
|
||||
import { IProduction, TokenType, Rule } from "../../api"
|
||||
|
||||
/**
|
||||
* Missing features
|
||||
* 1. Rule arguments
|
||||
* 2. Gates
|
||||
* 3. embedded actions
|
||||
*/
|
||||
|
||||
const NL = "\n"
|
||||
|
||||
export function genUmdModule(options: { name: string; rules: Rule[] }): string {
|
||||
return `
|
||||
(function (root, factory) {
|
||||
if (typeof define === 'function' && define.amd) {
|
||||
// AMD. Register as an anonymous module.
|
||||
define(['chevrotain'], factory);
|
||||
} else if (typeof module === 'object' && module.exports) {
|
||||
// Node. Does not work with strict CommonJS, but
|
||||
// only CommonJS-like environments that support module.exports,
|
||||
// like Node.
|
||||
module.exports = factory(require('chevrotain'));
|
||||
} else {
|
||||
// Browser globals (root is window)
|
||||
root.returnExports = factory(root.b);
|
||||
}
|
||||
}(typeof self !== 'undefined' ? self : this, function (chevrotain) {
|
||||
|
||||
${genClass(options)}
|
||||
|
||||
return {
|
||||
${options.name}: ${options.name}
|
||||
}
|
||||
}));
|
||||
`
|
||||
}
|
||||
|
||||
export function genWrapperFunction(options: {
|
||||
name: string
|
||||
rules: Rule[]
|
||||
}): string {
|
||||
return `
|
||||
${genClass(options)}
|
||||
return new ${options.name}(tokenVocabulary, config)
|
||||
`
|
||||
}
|
||||
|
||||
export function genClass(options: { name: string; rules: Rule[] }): string {
|
||||
// TODO: how to pass the token vocabulary? Constructor? other?
|
||||
let result = `
|
||||
function ${options.name}(tokenVocabulary, config) {
|
||||
// invoke super constructor
|
||||
// No support for embedded actions currently, so we can 'hardcode'
|
||||
// The use of CstParser.
|
||||
chevrotain.CstParser.call(this, tokenVocabulary, config)
|
||||
|
||||
const $ = this
|
||||
|
||||
${genAllRules(options.rules)}
|
||||
|
||||
// very important to call this after all the rules have been defined.
|
||||
// otherwise the parser may not work correctly as it will lack information
|
||||
// derived during the self analysis phase.
|
||||
this.performSelfAnalysis(this)
|
||||
}
|
||||
|
||||
// inheritance as implemented in javascript in the previous decade... :(
|
||||
${options.name}.prototype = Object.create(chevrotain.CstParser.prototype)
|
||||
${options.name}.prototype.constructor = ${options.name}
|
||||
`
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export function genAllRules(rules: Rule[]): string {
|
||||
let rulesText = map(rules, (currRule) => {
|
||||
return genRule(currRule, 1)
|
||||
})
|
||||
|
||||
return rulesText.join("\n")
|
||||
}
|
||||
|
||||
export function genRule(prod: Rule, n: number): string {
|
||||
let result = indent(n, `$.RULE("${prod.name}", function() {`) + NL
|
||||
result += genDefinition(prod.definition, n + 1)
|
||||
result += indent(n + 1, `})`) + NL
|
||||
return result
|
||||
}
|
||||
|
||||
export function genTerminal(prod: Terminal, n: number): string {
|
||||
const name = prod.terminalType.name
|
||||
// TODO: potential performance optimization, avoid tokenMap Dictionary access
|
||||
return indent(n, `$.CONSUME${prod.idx}(this.tokensMap.${name})` + NL)
|
||||
}
|
||||
|
||||
export function genNonTerminal(prod: NonTerminal, n: number): string {
|
||||
return indent(n, `$.SUBRULE${prod.idx}($.${prod.nonTerminalName})` + NL)
|
||||
}
|
||||
|
||||
export function genAlternation(prod: Alternation, n: number): string {
|
||||
let result = indent(n, `$.OR${prod.idx}([`) + NL
|
||||
const alts = map(prod.definition, (altDef) => genSingleAlt(altDef, n + 1))
|
||||
result += alts.join("," + NL)
|
||||
result += NL + indent(n, `])` + NL)
|
||||
return result
|
||||
}
|
||||
|
||||
export function genSingleAlt(prod: Alternative, n: number): string {
|
||||
let result = indent(n, `{`) + NL
|
||||
|
||||
result += indent(n + 1, "ALT: function() {") + NL
|
||||
result += genDefinition(prod.definition, n + 1)
|
||||
result += indent(n + 1, `}`) + NL
|
||||
result += indent(n, `}`)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
function genProd(prod: IProduction, n: number): string {
|
||||
/* istanbul ignore else */
|
||||
if (prod instanceof NonTerminal) {
|
||||
return genNonTerminal(prod, n)
|
||||
} else if (prod instanceof Option) {
|
||||
return genDSLRule("OPTION", prod, n)
|
||||
} else if (prod instanceof RepetitionMandatory) {
|
||||
return genDSLRule("AT_LEAST_ONE", prod, n)
|
||||
} else if (prod instanceof RepetitionMandatoryWithSeparator) {
|
||||
return genDSLRule("AT_LEAST_ONE_SEP", prod, n)
|
||||
} else if (prod instanceof RepetitionWithSeparator) {
|
||||
return genDSLRule("MANY_SEP", prod, n)
|
||||
} else if (prod instanceof Repetition) {
|
||||
return genDSLRule("MANY", prod, n)
|
||||
} else if (prod instanceof Alternation) {
|
||||
return genAlternation(prod, n)
|
||||
} else if (prod instanceof Terminal) {
|
||||
return genTerminal(prod, n)
|
||||
} else if (prod instanceof Alternative) {
|
||||
return genDefinition(prod.definition, n)
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
}
|
||||
|
||||
function genDSLRule(
|
||||
dslName,
|
||||
prod: {
|
||||
definition: IProduction[]
|
||||
idx: number
|
||||
separator?: TokenType
|
||||
},
|
||||
n: number
|
||||
): string {
|
||||
let result = indent(n, `$.${dslName + prod.idx}(`)
|
||||
|
||||
if (prod.separator) {
|
||||
result += "{" + NL
|
||||
result +=
|
||||
indent(n + 1, `SEP: this.tokensMap.${prod.separator.name}`) + "," + NL
|
||||
result += `DEF: ${genDefFunction(prod.definition, n + 2)}` + NL
|
||||
result += indent(n, "}") + NL
|
||||
} else {
|
||||
result += genDefFunction(prod.definition, n + 1)
|
||||
}
|
||||
|
||||
result += indent(n, `)`) + NL
|
||||
return result
|
||||
}
|
||||
|
||||
function genDefFunction(definition: IProduction[], n: number): string {
|
||||
let def = "function() {" + NL
|
||||
def += genDefinition(definition, n)
|
||||
def += indent(n, `}`) + NL
|
||||
return def
|
||||
}
|
||||
|
||||
function genDefinition(def: IProduction[], n: number): string {
|
||||
let result = ""
|
||||
forEach(def, (prod) => {
|
||||
result += genProd(prod, n + 1)
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
function indent(howMuch: number, text: string): string {
|
||||
const spaces = Array(howMuch * 4 + 1).join(" ")
|
||||
return spaces + text
|
||||
}
|
||||
36
node_modules/chevrotain/src/generate/generate_public.ts
generated
vendored
Normal file
36
node_modules/chevrotain/src/generate/generate_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import { Rule, IParserConfig, TokenVocabulary, BaseParser } from "../../api"
|
||||
import { genUmdModule, genWrapperFunction } from "./generate"
|
||||
|
||||
export function generateParserFactory<T extends BaseParser>(options: {
|
||||
name: string
|
||||
rules: Rule[]
|
||||
tokenVocabulary: TokenVocabulary
|
||||
}): (config?: IParserConfig) => T {
|
||||
const wrapperText = genWrapperFunction({
|
||||
name: options.name,
|
||||
rules: options.rules
|
||||
})
|
||||
|
||||
const constructorWrapper = new Function(
|
||||
"tokenVocabulary",
|
||||
"config",
|
||||
"chevrotain",
|
||||
wrapperText
|
||||
)
|
||||
|
||||
return function (config) {
|
||||
return constructorWrapper(
|
||||
options.tokenVocabulary,
|
||||
config,
|
||||
// TODO: check how the require is transpiled/webpacked
|
||||
require("../api")
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function generateParserModule(options: {
|
||||
name: string
|
||||
rules: Rule[]
|
||||
}): string {
|
||||
return genUmdModule({ name: options.name, rules: options.rules })
|
||||
}
|
||||
45
node_modules/chevrotain/src/lang/lang_extensions.ts
generated
vendored
Normal file
45
node_modules/chevrotain/src/lang/lang_extensions.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import { TokenType } from "../../api"
|
||||
import { isUndefined } from "../utils/utils"
|
||||
|
||||
export function classNameFromInstance(instance: any): string {
|
||||
return functionName(instance.constructor)
|
||||
}
|
||||
|
||||
const NAME = "name"
|
||||
|
||||
/**
|
||||
* Utility to obtain Function names.
|
||||
* Note that there should not be an assumptions on the result of this function.
|
||||
* E.g: When running from minified source code the result may be auto generated.
|
||||
*/
|
||||
export function functionName(func: TokenType): string {
|
||||
// Engines that support Function.prototype.name OR the nth (n>1) time after
|
||||
// the name has been computed in the following else block.
|
||||
let existingNameProp = (<any>func).name
|
||||
/* istanbul ignore else - too many hacks for IE/old versions of node.js here*/
|
||||
if (existingNameProp) {
|
||||
return existingNameProp
|
||||
} else {
|
||||
return "anonymous"
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {boolean} - has the property been successfully defined
|
||||
*/
|
||||
export function defineNameProp(obj, nameValue): boolean {
|
||||
let namePropDescriptor = Object.getOwnPropertyDescriptor(obj, NAME)
|
||||
/* istanbul ignore else -> will only run in old versions of node.js */
|
||||
if (isUndefined(namePropDescriptor) || namePropDescriptor.configurable) {
|
||||
Object.defineProperty(obj, NAME, {
|
||||
enumerable: false,
|
||||
configurable: true,
|
||||
writable: false,
|
||||
value: nameValue
|
||||
})
|
||||
|
||||
return true
|
||||
}
|
||||
/* istanbul ignore next -> will only run in old versions of node.js */
|
||||
return false
|
||||
}
|
||||
2
node_modules/chevrotain/src/parse/constants.ts
generated
vendored
Normal file
2
node_modules/chevrotain/src/parse/constants.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
// TODO: can this be removed? where is it used?
|
||||
export let IN = "_~IN~_"
|
||||
87
node_modules/chevrotain/src/parse/cst/cst.ts
generated
vendored
Normal file
87
node_modules/chevrotain/src/parse/cst/cst.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import { CstNode, CstNodeLocation, IToken } from "../../../api"
|
||||
|
||||
/**
|
||||
* This nodeLocation tracking is not efficient and should only be used
|
||||
* when error recovery is enabled or the Token Vector contains virtual Tokens
|
||||
* (e.g, Python Indent/Outdent)
|
||||
* As it executes the calculation for every single terminal/nonTerminal
|
||||
* and does not rely on the fact the token vector is **sorted**
|
||||
*/
|
||||
export function setNodeLocationOnlyOffset(
|
||||
currNodeLocation: CstNodeLocation,
|
||||
newLocationInfo: IToken
|
||||
): void {
|
||||
// First (valid) update for this cst node
|
||||
if (isNaN(currNodeLocation.startOffset) === true) {
|
||||
// assumption1: Token location information is either NaN or a valid number
|
||||
// assumption2: Token location information is fully valid if it exist
|
||||
// (both start/end offsets exist and are numbers).
|
||||
currNodeLocation.startOffset = newLocationInfo.startOffset
|
||||
currNodeLocation.endOffset = newLocationInfo.endOffset
|
||||
}
|
||||
// Once the startOffset has been updated with a valid number it should never receive
|
||||
// any farther updates as the Token vector is sorted.
|
||||
// We still have to check this this condition for every new possible location info
|
||||
// because with error recovery enabled we may encounter invalid tokens (NaN location props)
|
||||
else if (currNodeLocation.endOffset < newLocationInfo.endOffset === true) {
|
||||
currNodeLocation.endOffset = newLocationInfo.endOffset
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This nodeLocation tracking is not efficient and should only be used
|
||||
* when error recovery is enabled or the Token Vector contains virtual Tokens
|
||||
* (e.g, Python Indent/Outdent)
|
||||
* As it executes the calculation for every single terminal/nonTerminal
|
||||
* and does not rely on the fact the token vector is **sorted**
|
||||
*/
|
||||
export function setNodeLocationFull(
|
||||
currNodeLocation: CstNodeLocation,
|
||||
newLocationInfo: CstNodeLocation
|
||||
): void {
|
||||
// First (valid) update for this cst node
|
||||
if (isNaN(currNodeLocation.startOffset) === true) {
|
||||
// assumption1: Token location information is either NaN or a valid number
|
||||
// assumption2: Token location information is fully valid if it exist
|
||||
// (all start/end props exist and are numbers).
|
||||
currNodeLocation.startOffset = newLocationInfo.startOffset
|
||||
currNodeLocation.startColumn = newLocationInfo.startColumn
|
||||
currNodeLocation.startLine = newLocationInfo.startLine
|
||||
currNodeLocation.endOffset = newLocationInfo.endOffset
|
||||
currNodeLocation.endColumn = newLocationInfo.endColumn
|
||||
currNodeLocation.endLine = newLocationInfo.endLine
|
||||
}
|
||||
// Once the start props has been updated with a valid number it should never receive
|
||||
// any farther updates as the Token vector is sorted.
|
||||
// We still have to check this this condition for every new possible location info
|
||||
// because with error recovery enabled we may encounter invalid tokens (NaN location props)
|
||||
else if (currNodeLocation.endOffset < newLocationInfo.endOffset === true) {
|
||||
currNodeLocation.endOffset = newLocationInfo.endOffset
|
||||
currNodeLocation.endColumn = newLocationInfo.endColumn
|
||||
currNodeLocation.endLine = newLocationInfo.endLine
|
||||
}
|
||||
}
|
||||
|
||||
export function addTerminalToCst(
|
||||
node: CstNode,
|
||||
token: IToken,
|
||||
tokenTypeName: string
|
||||
): void {
|
||||
if (node.children[tokenTypeName] === undefined) {
|
||||
node.children[tokenTypeName] = [token]
|
||||
} else {
|
||||
node.children[tokenTypeName].push(token)
|
||||
}
|
||||
}
|
||||
|
||||
export function addNoneTerminalToCst(
|
||||
node: CstNode,
|
||||
ruleName: string,
|
||||
ruleResult: any
|
||||
): void {
|
||||
if (node.children[ruleName] === undefined) {
|
||||
node.children[ruleName] = [ruleResult]
|
||||
} else {
|
||||
node.children[ruleName].push(ruleResult)
|
||||
}
|
||||
}
|
||||
178
node_modules/chevrotain/src/parse/cst/cst_visitor.ts
generated
vendored
Normal file
178
node_modules/chevrotain/src/parse/cst/cst_visitor.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
import {
|
||||
compact,
|
||||
contains,
|
||||
forEach,
|
||||
isArray,
|
||||
isEmpty,
|
||||
isFunction,
|
||||
isUndefined,
|
||||
keys,
|
||||
map
|
||||
} from "../../utils/utils"
|
||||
import { defineNameProp, functionName } from "../../lang/lang_extensions"
|
||||
import { ICstVisitor } from "../../../api"
|
||||
|
||||
export function defaultVisit<IN, OUT>(ctx: any, param: IN): OUT {
|
||||
let childrenNames = keys(ctx)
|
||||
let childrenNamesLength = childrenNames.length
|
||||
for (let i = 0; i < childrenNamesLength; i++) {
|
||||
let currChildName = childrenNames[i]
|
||||
let currChildArray = ctx[currChildName]
|
||||
let currChildArrayLength = currChildArray.length
|
||||
for (let j = 0; j < currChildArrayLength; j++) {
|
||||
let currChild: any = currChildArray[j]
|
||||
// distinction between Tokens Children and CstNode children
|
||||
if (currChild.tokenTypeIdx === undefined) {
|
||||
this[currChild.name](currChild.children, param)
|
||||
}
|
||||
}
|
||||
}
|
||||
// defaultVisit does not support generic out param
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function createBaseSemanticVisitorConstructor(
|
||||
grammarName: string,
|
||||
ruleNames: string[]
|
||||
): {
|
||||
new (...args: any[]): ICstVisitor<any, any>
|
||||
} {
|
||||
let derivedConstructor: any = function () {}
|
||||
|
||||
// can be overwritten according to:
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/
|
||||
// name?redirectlocale=en-US&redirectslug=JavaScript%2FReference%2FGlobal_Objects%2FFunction%2Fname
|
||||
defineNameProp(derivedConstructor, grammarName + "BaseSemantics")
|
||||
|
||||
let semanticProto = {
|
||||
visit: function (cstNode, param) {
|
||||
// enables writing more concise visitor methods when CstNode has only a single child
|
||||
if (isArray(cstNode)) {
|
||||
// A CST Node's children dictionary can never have empty arrays as values
|
||||
// If a key is defined there will be at least one element in the corresponding value array.
|
||||
cstNode = cstNode[0]
|
||||
}
|
||||
|
||||
// enables passing optional CstNodes concisely.
|
||||
if (isUndefined(cstNode)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return this[cstNode.name](cstNode.children, param)
|
||||
},
|
||||
|
||||
validateVisitor: function () {
|
||||
let semanticDefinitionErrors = validateVisitor(this, ruleNames)
|
||||
if (!isEmpty(semanticDefinitionErrors)) {
|
||||
let errorMessages = map(
|
||||
semanticDefinitionErrors,
|
||||
(currDefError) => currDefError.msg
|
||||
)
|
||||
throw Error(
|
||||
`Errors Detected in CST Visitor <${functionName(
|
||||
this.constructor
|
||||
)}>:\n\t` + `${errorMessages.join("\n\n").replace(/\n/g, "\n\t")}`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
derivedConstructor.prototype = semanticProto
|
||||
derivedConstructor.prototype.constructor = derivedConstructor
|
||||
|
||||
derivedConstructor._RULE_NAMES = ruleNames
|
||||
|
||||
return derivedConstructor
|
||||
}
|
||||
|
||||
export function createBaseVisitorConstructorWithDefaults(
|
||||
grammarName: string,
|
||||
ruleNames: string[],
|
||||
baseConstructor: Function
|
||||
): {
|
||||
new (...args: any[]): ICstVisitor<any, any>
|
||||
} {
|
||||
let derivedConstructor: any = function () {}
|
||||
|
||||
// can be overwritten according to:
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/
|
||||
// name?redirectlocale=en-US&redirectslug=JavaScript%2FReference%2FGlobal_Objects%2FFunction%2Fname
|
||||
defineNameProp(derivedConstructor, grammarName + "BaseSemanticsWithDefaults")
|
||||
|
||||
let withDefaultsProto = Object.create(baseConstructor.prototype)
|
||||
forEach(ruleNames, (ruleName) => {
|
||||
withDefaultsProto[ruleName] = defaultVisit
|
||||
})
|
||||
|
||||
derivedConstructor.prototype = withDefaultsProto
|
||||
derivedConstructor.prototype.constructor = derivedConstructor
|
||||
|
||||
return derivedConstructor
|
||||
}
|
||||
|
||||
export enum CstVisitorDefinitionError {
|
||||
REDUNDANT_METHOD,
|
||||
MISSING_METHOD
|
||||
}
|
||||
|
||||
export interface IVisitorDefinitionError {
|
||||
msg: string
|
||||
type: CstVisitorDefinitionError
|
||||
methodName: string
|
||||
}
|
||||
|
||||
export function validateVisitor(
|
||||
visitorInstance: Function,
|
||||
ruleNames: string[]
|
||||
): IVisitorDefinitionError[] {
|
||||
let missingErrors = validateMissingCstMethods(visitorInstance, ruleNames)
|
||||
let redundantErrors = validateRedundantMethods(visitorInstance, ruleNames)
|
||||
|
||||
return missingErrors.concat(redundantErrors)
|
||||
}
|
||||
|
||||
export function validateMissingCstMethods(
|
||||
visitorInstance: Function,
|
||||
ruleNames: string[]
|
||||
): IVisitorDefinitionError[] {
|
||||
let errors: IVisitorDefinitionError[] = map(ruleNames, (currRuleName) => {
|
||||
if (!isFunction(visitorInstance[currRuleName])) {
|
||||
return {
|
||||
msg: `Missing visitor method: <${currRuleName}> on ${functionName(
|
||||
<any>visitorInstance.constructor
|
||||
)} CST Visitor.`,
|
||||
type: CstVisitorDefinitionError.MISSING_METHOD,
|
||||
methodName: currRuleName
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return compact<IVisitorDefinitionError>(errors)
|
||||
}
|
||||
|
||||
const VALID_PROP_NAMES = ["constructor", "visit", "validateVisitor"]
|
||||
export function validateRedundantMethods(
|
||||
visitorInstance: Function,
|
||||
ruleNames: string[]
|
||||
): IVisitorDefinitionError[] {
|
||||
let errors = []
|
||||
|
||||
for (let prop in visitorInstance) {
|
||||
if (
|
||||
isFunction(visitorInstance[prop]) &&
|
||||
!contains(VALID_PROP_NAMES, prop) &&
|
||||
!contains(ruleNames, prop)
|
||||
) {
|
||||
errors.push({
|
||||
msg:
|
||||
`Redundant visitor method: <${prop}> on ${functionName(
|
||||
<any>visitorInstance.constructor
|
||||
)} CST Visitor\n` +
|
||||
`There is no Grammar Rule corresponding to this method's name.\n`,
|
||||
type: CstVisitorDefinitionError.REDUNDANT_METHOD,
|
||||
methodName: prop
|
||||
})
|
||||
}
|
||||
}
|
||||
return errors
|
||||
}
|
||||
320
node_modules/chevrotain/src/parse/errors_public.ts
generated
vendored
Normal file
320
node_modules/chevrotain/src/parse/errors_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
import { hasTokenLabel, tokenLabel } from "../scan/tokens_public"
|
||||
import * as utils from "../utils/utils"
|
||||
import { first, map, reduce } from "../utils/utils"
|
||||
import {
|
||||
Alternation,
|
||||
NonTerminal,
|
||||
Rule,
|
||||
Terminal
|
||||
} from "./grammar/gast/gast_public"
|
||||
import { getProductionDslName } from "./grammar/gast/gast"
|
||||
import {
|
||||
IGrammarResolverErrorMessageProvider,
|
||||
IGrammarValidatorErrorMessageProvider,
|
||||
IParserErrorMessageProvider,
|
||||
IProductionWithOccurrence,
|
||||
TokenType
|
||||
} from "../../api"
|
||||
|
||||
export const defaultParserErrorProvider: IParserErrorMessageProvider = {
|
||||
buildMismatchTokenMessage({ expected, actual, previous, ruleName }): string {
|
||||
let hasLabel = hasTokenLabel(expected)
|
||||
let expectedMsg = hasLabel
|
||||
? `--> ${tokenLabel(expected)} <--`
|
||||
: `token of type --> ${expected.name} <--`
|
||||
|
||||
let msg = `Expecting ${expectedMsg} but found --> '${actual.image}' <--`
|
||||
|
||||
return msg
|
||||
},
|
||||
|
||||
buildNotAllInputParsedMessage({ firstRedundant, ruleName }): string {
|
||||
return "Redundant input, expecting EOF but found: " + firstRedundant.image
|
||||
},
|
||||
|
||||
buildNoViableAltMessage({
|
||||
expectedPathsPerAlt,
|
||||
actual,
|
||||
previous,
|
||||
customUserDescription,
|
||||
ruleName
|
||||
}): string {
|
||||
let errPrefix = "Expecting: "
|
||||
// TODO: issue: No Viable Alternative Error may have incomplete details. #502
|
||||
let actualText = first(actual).image
|
||||
let errSuffix = "\nbut found: '" + actualText + "'"
|
||||
|
||||
if (customUserDescription) {
|
||||
return errPrefix + customUserDescription + errSuffix
|
||||
} else {
|
||||
let allLookAheadPaths = reduce(
|
||||
expectedPathsPerAlt,
|
||||
(result, currAltPaths) => result.concat(currAltPaths),
|
||||
[]
|
||||
)
|
||||
let nextValidTokenSequences = map(
|
||||
allLookAheadPaths,
|
||||
(currPath) =>
|
||||
`[${map(currPath, (currTokenType) => tokenLabel(currTokenType)).join(
|
||||
", "
|
||||
)}]`
|
||||
)
|
||||
let nextValidSequenceItems = map(
|
||||
nextValidTokenSequences,
|
||||
(itemMsg, idx) => ` ${idx + 1}. ${itemMsg}`
|
||||
)
|
||||
let calculatedDescription = `one of these possible Token sequences:\n${nextValidSequenceItems.join(
|
||||
"\n"
|
||||
)}`
|
||||
|
||||
return errPrefix + calculatedDescription + errSuffix
|
||||
}
|
||||
},
|
||||
|
||||
buildEarlyExitMessage({
|
||||
expectedIterationPaths,
|
||||
actual,
|
||||
customUserDescription,
|
||||
ruleName
|
||||
}): string {
|
||||
let errPrefix = "Expecting: "
|
||||
// TODO: issue: No Viable Alternative Error may have incomplete details. #502
|
||||
let actualText = first(actual).image
|
||||
let errSuffix = "\nbut found: '" + actualText + "'"
|
||||
|
||||
if (customUserDescription) {
|
||||
return errPrefix + customUserDescription + errSuffix
|
||||
} else {
|
||||
let nextValidTokenSequences = map(
|
||||
expectedIterationPaths,
|
||||
(currPath) =>
|
||||
`[${map(currPath, (currTokenType) => tokenLabel(currTokenType)).join(
|
||||
","
|
||||
)}]`
|
||||
)
|
||||
let calculatedDescription =
|
||||
`expecting at least one iteration which starts with one of these possible Token sequences::\n ` +
|
||||
`<${nextValidTokenSequences.join(" ,")}>`
|
||||
|
||||
return errPrefix + calculatedDescription + errSuffix
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Object.freeze(defaultParserErrorProvider)
|
||||
|
||||
export const defaultGrammarResolverErrorProvider: IGrammarResolverErrorMessageProvider = {
|
||||
buildRuleNotFoundError(
|
||||
topLevelRule: Rule,
|
||||
undefinedRule: NonTerminal
|
||||
): string {
|
||||
const msg =
|
||||
"Invalid grammar, reference to a rule which is not defined: ->" +
|
||||
undefinedRule.nonTerminalName +
|
||||
"<-\n" +
|
||||
"inside top level rule: ->" +
|
||||
topLevelRule.name +
|
||||
"<-"
|
||||
return msg
|
||||
}
|
||||
}
|
||||
|
||||
export const defaultGrammarValidatorErrorProvider: IGrammarValidatorErrorMessageProvider = {
|
||||
buildDuplicateFoundError(
|
||||
topLevelRule: Rule,
|
||||
duplicateProds: IProductionWithOccurrence[]
|
||||
): string {
|
||||
function getExtraProductionArgument(
|
||||
prod: IProductionWithOccurrence
|
||||
): string {
|
||||
if (prod instanceof Terminal) {
|
||||
return prod.terminalType.name
|
||||
} else if (prod instanceof NonTerminal) {
|
||||
return prod.nonTerminalName
|
||||
} else {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
const topLevelName = topLevelRule.name
|
||||
const duplicateProd = first(duplicateProds)
|
||||
const index = duplicateProd.idx
|
||||
const dslName = getProductionDslName(duplicateProd)
|
||||
let extraArgument = getExtraProductionArgument(duplicateProd)
|
||||
|
||||
const hasExplicitIndex = index > 0
|
||||
let msg = `->${dslName}${hasExplicitIndex ? index : ""}<- ${
|
||||
extraArgument ? `with argument: ->${extraArgument}<-` : ""
|
||||
}
|
||||
appears more than once (${
|
||||
duplicateProds.length
|
||||
} times) in the top level rule: ->${topLevelName}<-.
|
||||
For further details see: https://sap.github.io/chevrotain/docs/FAQ.html#NUMERICAL_SUFFIXES
|
||||
`
|
||||
|
||||
// white space trimming time! better to trim afterwards as it allows to use WELL formatted multi line template strings...
|
||||
msg = msg.replace(/[ \t]+/g, " ")
|
||||
msg = msg.replace(/\s\s+/g, "\n")
|
||||
|
||||
return msg
|
||||
},
|
||||
|
||||
buildNamespaceConflictError(rule: Rule): string {
|
||||
const errMsg =
|
||||
`Namespace conflict found in grammar.\n` +
|
||||
`The grammar has both a Terminal(Token) and a Non-Terminal(Rule) named: <${rule.name}>.\n` +
|
||||
`To resolve this make sure each Terminal and Non-Terminal names are unique\n` +
|
||||
`This is easy to accomplish by using the convention that Terminal names start with an uppercase letter\n` +
|
||||
`and Non-Terminal names start with a lower case letter.`
|
||||
|
||||
return errMsg
|
||||
},
|
||||
|
||||
buildAlternationPrefixAmbiguityError(options: {
|
||||
topLevelRule: Rule
|
||||
prefixPath: TokenType[]
|
||||
ambiguityIndices: number[]
|
||||
alternation: Alternation
|
||||
}): string {
|
||||
const pathMsg = map(options.prefixPath, (currTok) =>
|
||||
tokenLabel(currTok)
|
||||
).join(", ")
|
||||
const occurrence =
|
||||
options.alternation.idx === 0 ? "" : options.alternation.idx
|
||||
const errMsg =
|
||||
`Ambiguous alternatives: <${options.ambiguityIndices.join(
|
||||
" ,"
|
||||
)}> due to common lookahead prefix\n` +
|
||||
`in <OR${occurrence}> inside <${options.topLevelRule.name}> Rule,\n` +
|
||||
`<${pathMsg}> may appears as a prefix path in all these alternatives.\n` +
|
||||
`See: https://sap.github.io/chevrotain/docs/guide/resolving_grammar_errors.html#COMMON_PREFIX\n` +
|
||||
`For Further details.`
|
||||
|
||||
return errMsg
|
||||
},
|
||||
|
||||
buildAlternationAmbiguityError(options: {
|
||||
topLevelRule: Rule
|
||||
prefixPath: TokenType[]
|
||||
ambiguityIndices: number[]
|
||||
alternation: Alternation
|
||||
}): string {
|
||||
let pathMsg = map(options.prefixPath, (currtok) =>
|
||||
tokenLabel(currtok)
|
||||
).join(", ")
|
||||
let occurrence =
|
||||
options.alternation.idx === 0 ? "" : options.alternation.idx
|
||||
let currMessage =
|
||||
`Ambiguous Alternatives Detected: <${options.ambiguityIndices.join(
|
||||
" ,"
|
||||
)}> in <OR${occurrence}>` +
|
||||
` inside <${options.topLevelRule.name}> Rule,\n` +
|
||||
`<${pathMsg}> may appears as a prefix path in all these alternatives.\n`
|
||||
|
||||
currMessage =
|
||||
currMessage +
|
||||
`See: https://sap.github.io/chevrotain/docs/guide/resolving_grammar_errors.html#AMBIGUOUS_ALTERNATIVES\n` +
|
||||
`For Further details.`
|
||||
return currMessage
|
||||
},
|
||||
|
||||
buildEmptyRepetitionError(options: {
|
||||
topLevelRule: Rule
|
||||
repetition: IProductionWithOccurrence
|
||||
}): string {
|
||||
let dslName = getProductionDslName(options.repetition)
|
||||
if (options.repetition.idx !== 0) {
|
||||
dslName += options.repetition.idx
|
||||
}
|
||||
|
||||
const errMsg =
|
||||
`The repetition <${dslName}> within Rule <${options.topLevelRule.name}> can never consume any tokens.\n` +
|
||||
`This could lead to an infinite loop.`
|
||||
|
||||
return errMsg
|
||||
},
|
||||
|
||||
// TODO: remove - `errors_public` from nyc.config.js exclude
|
||||
// once this method is fully removed from this file
|
||||
buildTokenNameError(options: {
|
||||
tokenType: TokenType
|
||||
expectedPattern: RegExp
|
||||
}): string {
|
||||
/* istanbul ignore next */
|
||||
return "deprecated"
|
||||
},
|
||||
|
||||
buildEmptyAlternationError(options: {
|
||||
topLevelRule: Rule
|
||||
alternation: Alternation
|
||||
emptyChoiceIdx: number
|
||||
}): string {
|
||||
const errMsg =
|
||||
`Ambiguous empty alternative: <${options.emptyChoiceIdx + 1}>` +
|
||||
` in <OR${options.alternation.idx}> inside <${options.topLevelRule.name}> Rule.\n` +
|
||||
`Only the last alternative may be an empty alternative.`
|
||||
|
||||
return errMsg
|
||||
},
|
||||
|
||||
buildTooManyAlternativesError(options: {
|
||||
topLevelRule: Rule
|
||||
alternation: Alternation
|
||||
}): string {
|
||||
const errMsg =
|
||||
`An Alternation cannot have more than 256 alternatives:\n` +
|
||||
`<OR${options.alternation.idx}> inside <${
|
||||
options.topLevelRule.name
|
||||
}> Rule.\n has ${options.alternation.definition.length + 1} alternatives.`
|
||||
|
||||
return errMsg
|
||||
},
|
||||
|
||||
buildLeftRecursionError(options: {
|
||||
topLevelRule: Rule
|
||||
leftRecursionPath: Rule[]
|
||||
}): string {
|
||||
const ruleName = options.topLevelRule.name
|
||||
let pathNames = utils.map(
|
||||
options.leftRecursionPath,
|
||||
(currRule) => currRule.name
|
||||
)
|
||||
let leftRecursivePath = `${ruleName} --> ${pathNames
|
||||
.concat([ruleName])
|
||||
.join(" --> ")}`
|
||||
let errMsg =
|
||||
`Left Recursion found in grammar.\n` +
|
||||
`rule: <${ruleName}> can be invoked from itself (directly or indirectly)\n` +
|
||||
`without consuming any Tokens. The grammar path that causes this is: \n ${leftRecursivePath}\n` +
|
||||
` To fix this refactor your grammar to remove the left recursion.\n` +
|
||||
`see: https://en.wikipedia.org/wiki/LL_parser#Left_Factoring.`
|
||||
|
||||
return errMsg
|
||||
},
|
||||
|
||||
// TODO: remove - `errors_public` from nyc.config.js exclude
|
||||
// once this method is fully removed from this file
|
||||
buildInvalidRuleNameError(options: {
|
||||
topLevelRule: Rule
|
||||
expectedPattern: RegExp
|
||||
}): string {
|
||||
/* istanbul ignore next */
|
||||
return "deprecated"
|
||||
},
|
||||
|
||||
buildDuplicateRuleNameError(options: {
|
||||
topLevelRule: Rule | string
|
||||
grammarName: string
|
||||
}): string {
|
||||
let ruleName
|
||||
if (options.topLevelRule instanceof Rule) {
|
||||
ruleName = options.topLevelRule.name
|
||||
} else {
|
||||
ruleName = options.topLevelRule
|
||||
}
|
||||
|
||||
const errMsg = `Duplicate definition, rule: ->${ruleName}<- is already defined in the grammar: ->${options.grammarName}<-`
|
||||
|
||||
return errMsg
|
||||
}
|
||||
}
|
||||
69
node_modules/chevrotain/src/parse/exceptions_public.ts
generated
vendored
Normal file
69
node_modules/chevrotain/src/parse/exceptions_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
import { contains } from "../utils/utils"
|
||||
import { IToken, IRecognitionException, IRecognizerContext } from "../../api"
|
||||
|
||||
const MISMATCHED_TOKEN_EXCEPTION = "MismatchedTokenException"
|
||||
const NO_VIABLE_ALT_EXCEPTION = "NoViableAltException"
|
||||
const EARLY_EXIT_EXCEPTION = "EarlyExitException"
|
||||
const NOT_ALL_INPUT_PARSED_EXCEPTION = "NotAllInputParsedException"
|
||||
|
||||
const RECOGNITION_EXCEPTION_NAMES = [
|
||||
MISMATCHED_TOKEN_EXCEPTION,
|
||||
NO_VIABLE_ALT_EXCEPTION,
|
||||
EARLY_EXIT_EXCEPTION,
|
||||
NOT_ALL_INPUT_PARSED_EXCEPTION
|
||||
]
|
||||
|
||||
Object.freeze(RECOGNITION_EXCEPTION_NAMES)
|
||||
|
||||
// hacks to bypass no support for custom Errors in javascript/typescript
|
||||
export function isRecognitionException(error: Error) {
|
||||
// can't do instanceof on hacked custom js exceptions
|
||||
return contains(RECOGNITION_EXCEPTION_NAMES, error.name)
|
||||
}
|
||||
|
||||
abstract class RecognitionException
|
||||
extends Error
|
||||
implements IRecognitionException {
|
||||
context: IRecognizerContext
|
||||
resyncedTokens = []
|
||||
|
||||
protected constructor(message: string, public token: IToken) {
|
||||
super(message)
|
||||
|
||||
// fix prototype chain when typescript target is ES5
|
||||
Object.setPrototypeOf(this, new.target.prototype)
|
||||
|
||||
/* istanbul ignore next - V8 workaround to remove constructor from stacktrace when typescript target is ES5 */
|
||||
if (Error.captureStackTrace) {
|
||||
Error.captureStackTrace(this, this.constructor)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class MismatchedTokenException extends RecognitionException {
|
||||
constructor(message: string, token: IToken, public previousToken: IToken) {
|
||||
super(message, token)
|
||||
this.name = MISMATCHED_TOKEN_EXCEPTION
|
||||
}
|
||||
}
|
||||
|
||||
export class NoViableAltException extends RecognitionException {
|
||||
constructor(message: string, token: IToken, public previousToken: IToken) {
|
||||
super(message, token)
|
||||
this.name = NO_VIABLE_ALT_EXCEPTION
|
||||
}
|
||||
}
|
||||
|
||||
export class NotAllInputParsedException extends RecognitionException {
|
||||
constructor(message: string, token: IToken) {
|
||||
super(message, token)
|
||||
this.name = NOT_ALL_INPUT_PARSED_EXCEPTION
|
||||
}
|
||||
}
|
||||
|
||||
export class EarlyExitException extends RecognitionException {
|
||||
constructor(message: string, token: IToken, public previousToken: IToken) {
|
||||
super(message, token)
|
||||
this.name = EARLY_EXIT_EXCEPTION
|
||||
}
|
||||
}
|
||||
725
node_modules/chevrotain/src/parse/grammar/checks.ts
generated
vendored
Normal file
725
node_modules/chevrotain/src/parse/grammar/checks.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,725 @@
|
|||
import * as utils from "../../utils/utils"
|
||||
import {
|
||||
contains,
|
||||
every,
|
||||
findAll,
|
||||
flatten,
|
||||
forEach,
|
||||
isEmpty,
|
||||
map,
|
||||
reduce,
|
||||
reject
|
||||
} from "../../utils/utils"
|
||||
import {
|
||||
IParserAmbiguousAlternativesDefinitionError,
|
||||
IParserDuplicatesDefinitionError,
|
||||
IParserEmptyAlternativeDefinitionError,
|
||||
ParserDefinitionErrorType
|
||||
} from "../parser/parser"
|
||||
import { getProductionDslName, isOptionalProd } from "./gast/gast"
|
||||
import {
|
||||
Alternative,
|
||||
containsPath,
|
||||
getLookaheadPathsForOptionalProd,
|
||||
getLookaheadPathsForOr,
|
||||
getProdType,
|
||||
isStrictPrefixOfPath
|
||||
} from "./lookahead"
|
||||
import { nextPossibleTokensAfter } from "./interpreter"
|
||||
import {
|
||||
Alternation,
|
||||
Alternative as AlternativeGAST,
|
||||
NonTerminal,
|
||||
Option,
|
||||
Repetition,
|
||||
RepetitionMandatory,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
RepetitionWithSeparator,
|
||||
Rule,
|
||||
Terminal
|
||||
} from "./gast/gast_public"
|
||||
import { GAstVisitor } from "./gast/gast_visitor_public"
|
||||
import {
|
||||
IGrammarValidatorErrorMessageProvider,
|
||||
IParserDefinitionError,
|
||||
IProduction,
|
||||
IProductionWithOccurrence,
|
||||
TokenType
|
||||
} from "../../../api"
|
||||
|
||||
export function validateGrammar(
|
||||
topLevels: Rule[],
|
||||
globalMaxLookahead: number,
|
||||
tokenTypes: TokenType[],
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider,
|
||||
grammarName: string
|
||||
): IParserDefinitionError[] {
|
||||
let duplicateErrors: any = utils.map(topLevels, (currTopLevel) =>
|
||||
validateDuplicateProductions(currTopLevel, errMsgProvider)
|
||||
)
|
||||
let leftRecursionErrors: any = utils.map(topLevels, (currTopRule) =>
|
||||
validateNoLeftRecursion(currTopRule, currTopRule, errMsgProvider)
|
||||
)
|
||||
|
||||
let emptyAltErrors = []
|
||||
let ambiguousAltsErrors = []
|
||||
let emptyRepetitionErrors = []
|
||||
|
||||
// left recursion could cause infinite loops in the following validations.
|
||||
// It is safest to first have the user fix the left recursion errors first and only then examine Further issues.
|
||||
if (every(leftRecursionErrors, isEmpty)) {
|
||||
emptyAltErrors = map(topLevels, (currTopRule) =>
|
||||
validateEmptyOrAlternative(currTopRule, errMsgProvider)
|
||||
)
|
||||
ambiguousAltsErrors = map(topLevels, (currTopRule) =>
|
||||
validateAmbiguousAlternationAlternatives(
|
||||
currTopRule,
|
||||
globalMaxLookahead,
|
||||
errMsgProvider
|
||||
)
|
||||
)
|
||||
|
||||
emptyRepetitionErrors = validateSomeNonEmptyLookaheadPath(
|
||||
topLevels,
|
||||
globalMaxLookahead,
|
||||
errMsgProvider
|
||||
)
|
||||
}
|
||||
|
||||
let termsNamespaceConflictErrors = checkTerminalAndNoneTerminalsNameSpace(
|
||||
topLevels,
|
||||
tokenTypes,
|
||||
errMsgProvider
|
||||
)
|
||||
|
||||
const tooManyAltsErrors = map(topLevels, (curRule) =>
|
||||
validateTooManyAlts(curRule, errMsgProvider)
|
||||
)
|
||||
|
||||
const duplicateRulesError = map(topLevels, (curRule) =>
|
||||
validateRuleDoesNotAlreadyExist(
|
||||
curRule,
|
||||
topLevels,
|
||||
grammarName,
|
||||
errMsgProvider
|
||||
)
|
||||
)
|
||||
|
||||
return <any>(
|
||||
utils.flatten(
|
||||
duplicateErrors.concat(
|
||||
emptyRepetitionErrors,
|
||||
leftRecursionErrors,
|
||||
emptyAltErrors,
|
||||
ambiguousAltsErrors,
|
||||
termsNamespaceConflictErrors,
|
||||
tooManyAltsErrors,
|
||||
duplicateRulesError
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
function validateDuplicateProductions(
|
||||
topLevelRule: Rule,
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
): IParserDuplicatesDefinitionError[] {
|
||||
let collectorVisitor = new OccurrenceValidationCollector()
|
||||
topLevelRule.accept(collectorVisitor)
|
||||
let allRuleProductions = collectorVisitor.allProductions
|
||||
|
||||
let productionGroups = utils.groupBy(
|
||||
allRuleProductions,
|
||||
identifyProductionForDuplicates
|
||||
)
|
||||
|
||||
let duplicates: any = utils.pick(productionGroups, (currGroup) => {
|
||||
return currGroup.length > 1
|
||||
})
|
||||
|
||||
let errors = utils.map(utils.values(duplicates), (currDuplicates: any) => {
|
||||
let firstProd: any = utils.first(currDuplicates)
|
||||
let msg = errMsgProvider.buildDuplicateFoundError(
|
||||
topLevelRule,
|
||||
currDuplicates
|
||||
)
|
||||
let dslName = getProductionDslName(firstProd)
|
||||
let defError: IParserDuplicatesDefinitionError = {
|
||||
message: msg,
|
||||
type: ParserDefinitionErrorType.DUPLICATE_PRODUCTIONS,
|
||||
ruleName: topLevelRule.name,
|
||||
dslName: dslName,
|
||||
occurrence: firstProd.idx
|
||||
}
|
||||
|
||||
let param = getExtraProductionArgument(firstProd)
|
||||
if (param) {
|
||||
defError.parameter = param
|
||||
}
|
||||
|
||||
return defError
|
||||
})
|
||||
return errors
|
||||
}
|
||||
|
||||
export function identifyProductionForDuplicates(
|
||||
prod: IProductionWithOccurrence
|
||||
): string {
|
||||
return `${getProductionDslName(prod)}_#_${
|
||||
prod.idx
|
||||
}_#_${getExtraProductionArgument(prod)}`
|
||||
}
|
||||
|
||||
function getExtraProductionArgument(prod: IProductionWithOccurrence): string {
|
||||
if (prod instanceof Terminal) {
|
||||
return prod.terminalType.name
|
||||
} else if (prod instanceof NonTerminal) {
|
||||
return prod.nonTerminalName
|
||||
} else {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
export class OccurrenceValidationCollector extends GAstVisitor {
|
||||
public allProductions: IProduction[] = []
|
||||
|
||||
public visitNonTerminal(subrule: NonTerminal): void {
|
||||
this.allProductions.push(subrule)
|
||||
}
|
||||
|
||||
public visitOption(option: Option): void {
|
||||
this.allProductions.push(option)
|
||||
}
|
||||
|
||||
public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void {
|
||||
this.allProductions.push(manySep)
|
||||
}
|
||||
|
||||
public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void {
|
||||
this.allProductions.push(atLeastOne)
|
||||
}
|
||||
|
||||
public visitRepetitionMandatoryWithSeparator(
|
||||
atLeastOneSep: RepetitionMandatoryWithSeparator
|
||||
): void {
|
||||
this.allProductions.push(atLeastOneSep)
|
||||
}
|
||||
|
||||
public visitRepetition(many: Repetition): void {
|
||||
this.allProductions.push(many)
|
||||
}
|
||||
|
||||
public visitAlternation(or: Alternation): void {
|
||||
this.allProductions.push(or)
|
||||
}
|
||||
|
||||
public visitTerminal(terminal: Terminal): void {
|
||||
this.allProductions.push(terminal)
|
||||
}
|
||||
}
|
||||
|
||||
export function validateRuleDoesNotAlreadyExist(
|
||||
rule: Rule,
|
||||
allRules: Rule[],
|
||||
className,
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
): IParserDefinitionError[] {
|
||||
let errors = []
|
||||
const occurrences = reduce(
|
||||
allRules,
|
||||
(result, curRule) => {
|
||||
if (curRule.name === rule.name) {
|
||||
return result + 1
|
||||
}
|
||||
return result
|
||||
},
|
||||
0
|
||||
)
|
||||
if (occurrences > 1) {
|
||||
const errMsg = errMsgProvider.buildDuplicateRuleNameError({
|
||||
topLevelRule: rule,
|
||||
grammarName: className
|
||||
})
|
||||
errors.push({
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.DUPLICATE_RULE_NAME,
|
||||
ruleName: rule.name
|
||||
})
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
// TODO: is there anyway to get only the rule names of rules inherited from the super grammars?
|
||||
// This is not part of the IGrammarErrorProvider because the validation cannot be performed on
|
||||
// The grammar structure, only at runtime.
|
||||
export function validateRuleIsOverridden(
|
||||
ruleName: string,
|
||||
definedRulesNames: string[],
|
||||
className
|
||||
): IParserDefinitionError[] {
|
||||
let errors = []
|
||||
let errMsg
|
||||
|
||||
if (!utils.contains(definedRulesNames, ruleName)) {
|
||||
errMsg =
|
||||
`Invalid rule override, rule: ->${ruleName}<- cannot be overridden in the grammar: ->${className}<-` +
|
||||
`as it is not defined in any of the super grammars `
|
||||
errors.push({
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.INVALID_RULE_OVERRIDE,
|
||||
ruleName: ruleName
|
||||
})
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
export function validateNoLeftRecursion(
|
||||
topRule: Rule,
|
||||
currRule: Rule,
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider,
|
||||
path: Rule[] = []
|
||||
): IParserDefinitionError[] {
|
||||
let errors = []
|
||||
let nextNonTerminals = getFirstNoneTerminal(currRule.definition)
|
||||
if (utils.isEmpty(nextNonTerminals)) {
|
||||
return []
|
||||
} else {
|
||||
let ruleName = topRule.name
|
||||
let foundLeftRecursion = utils.contains(<any>nextNonTerminals, topRule)
|
||||
if (foundLeftRecursion) {
|
||||
errors.push({
|
||||
message: errMsgProvider.buildLeftRecursionError({
|
||||
topLevelRule: topRule,
|
||||
leftRecursionPath: path
|
||||
}),
|
||||
type: ParserDefinitionErrorType.LEFT_RECURSION,
|
||||
ruleName: ruleName
|
||||
})
|
||||
}
|
||||
|
||||
// we are only looking for cyclic paths leading back to the specific topRule
|
||||
// other cyclic paths are ignored, we still need this difference to avoid infinite loops...
|
||||
let validNextSteps = utils.difference(
|
||||
nextNonTerminals,
|
||||
path.concat([topRule])
|
||||
)
|
||||
let errorsFromNextSteps = utils.map(validNextSteps, (currRefRule) => {
|
||||
let newPath = utils.cloneArr(path)
|
||||
newPath.push(currRefRule)
|
||||
return validateNoLeftRecursion(
|
||||
topRule,
|
||||
currRefRule,
|
||||
errMsgProvider,
|
||||
newPath
|
||||
)
|
||||
})
|
||||
|
||||
return errors.concat(utils.flatten(errorsFromNextSteps))
|
||||
}
|
||||
}
|
||||
|
||||
export function getFirstNoneTerminal(definition: IProduction[]): Rule[] {
|
||||
let result = []
|
||||
if (utils.isEmpty(definition)) {
|
||||
return result
|
||||
}
|
||||
let firstProd = utils.first(definition)
|
||||
|
||||
/* istanbul ignore else */
|
||||
if (firstProd instanceof NonTerminal) {
|
||||
result.push(firstProd.referencedRule)
|
||||
} else if (
|
||||
firstProd instanceof AlternativeGAST ||
|
||||
firstProd instanceof Option ||
|
||||
firstProd instanceof RepetitionMandatory ||
|
||||
firstProd instanceof RepetitionMandatoryWithSeparator ||
|
||||
firstProd instanceof RepetitionWithSeparator ||
|
||||
firstProd instanceof Repetition
|
||||
) {
|
||||
result = result.concat(
|
||||
getFirstNoneTerminal(<IProduction[]>firstProd.definition)
|
||||
)
|
||||
} else if (firstProd instanceof Alternation) {
|
||||
// each sub definition in alternation is a FLAT
|
||||
result = utils.flatten(
|
||||
utils.map(firstProd.definition, (currSubDef) =>
|
||||
getFirstNoneTerminal((<AlternativeGAST>currSubDef).definition)
|
||||
)
|
||||
)
|
||||
} else if (firstProd instanceof Terminal) {
|
||||
// nothing to see, move along
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
|
||||
let isFirstOptional = isOptionalProd(firstProd)
|
||||
let hasMore = definition.length > 1
|
||||
if (isFirstOptional && hasMore) {
|
||||
let rest = utils.drop(definition)
|
||||
return result.concat(getFirstNoneTerminal(rest))
|
||||
} else {
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
class OrCollector extends GAstVisitor {
|
||||
public alternations = []
|
||||
|
||||
public visitAlternation(node: Alternation): void {
|
||||
this.alternations.push(node)
|
||||
}
|
||||
}
|
||||
|
||||
export function validateEmptyOrAlternative(
|
||||
topLevelRule: Rule,
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
): IParserEmptyAlternativeDefinitionError[] {
|
||||
let orCollector = new OrCollector()
|
||||
topLevelRule.accept(orCollector)
|
||||
let ors = orCollector.alternations
|
||||
|
||||
let errors = utils.reduce(
|
||||
ors,
|
||||
(errors, currOr) => {
|
||||
let exceptLast = utils.dropRight(currOr.definition)
|
||||
let currErrors = utils.map(
|
||||
exceptLast,
|
||||
(currAlternative: IProduction, currAltIdx) => {
|
||||
const possibleFirstInAlt = nextPossibleTokensAfter(
|
||||
[currAlternative],
|
||||
[],
|
||||
null,
|
||||
1
|
||||
)
|
||||
if (utils.isEmpty(possibleFirstInAlt)) {
|
||||
return {
|
||||
message: errMsgProvider.buildEmptyAlternationError({
|
||||
topLevelRule: topLevelRule,
|
||||
alternation: currOr,
|
||||
emptyChoiceIdx: currAltIdx
|
||||
}),
|
||||
type: ParserDefinitionErrorType.NONE_LAST_EMPTY_ALT,
|
||||
ruleName: topLevelRule.name,
|
||||
occurrence: currOr.idx,
|
||||
alternative: currAltIdx + 1
|
||||
}
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
)
|
||||
return errors.concat(utils.compact(currErrors))
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
export function validateAmbiguousAlternationAlternatives(
|
||||
topLevelRule: Rule,
|
||||
globalMaxLookahead: number,
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
): IParserAmbiguousAlternativesDefinitionError[] {
|
||||
let orCollector = new OrCollector()
|
||||
topLevelRule.accept(orCollector)
|
||||
let ors = orCollector.alternations
|
||||
|
||||
// New Handling of ignoring ambiguities
|
||||
// - https://github.com/SAP/chevrotain/issues/869
|
||||
ors = reject(ors, (currOr) => currOr.ignoreAmbiguities === true)
|
||||
|
||||
let errors = utils.reduce(
|
||||
ors,
|
||||
(result, currOr: Alternation) => {
|
||||
let currOccurrence = currOr.idx
|
||||
const actualMaxLookahead = currOr.maxLookahead || globalMaxLookahead
|
||||
let alternatives = getLookaheadPathsForOr(
|
||||
currOccurrence,
|
||||
topLevelRule,
|
||||
actualMaxLookahead,
|
||||
currOr
|
||||
)
|
||||
let altsAmbiguityErrors = checkAlternativesAmbiguities(
|
||||
alternatives,
|
||||
currOr,
|
||||
topLevelRule,
|
||||
errMsgProvider
|
||||
)
|
||||
let altsPrefixAmbiguityErrors = checkPrefixAlternativesAmbiguities(
|
||||
alternatives,
|
||||
currOr,
|
||||
topLevelRule,
|
||||
errMsgProvider
|
||||
)
|
||||
|
||||
return result.concat(altsAmbiguityErrors, altsPrefixAmbiguityErrors)
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
export class RepetionCollector extends GAstVisitor {
|
||||
public allProductions: IProduction[] = []
|
||||
|
||||
public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void {
|
||||
this.allProductions.push(manySep)
|
||||
}
|
||||
|
||||
public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void {
|
||||
this.allProductions.push(atLeastOne)
|
||||
}
|
||||
|
||||
public visitRepetitionMandatoryWithSeparator(
|
||||
atLeastOneSep: RepetitionMandatoryWithSeparator
|
||||
): void {
|
||||
this.allProductions.push(atLeastOneSep)
|
||||
}
|
||||
|
||||
public visitRepetition(many: Repetition): void {
|
||||
this.allProductions.push(many)
|
||||
}
|
||||
}
|
||||
|
||||
export function validateTooManyAlts(
|
||||
topLevelRule: Rule,
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
): IParserDefinitionError[] {
|
||||
let orCollector = new OrCollector()
|
||||
topLevelRule.accept(orCollector)
|
||||
let ors = orCollector.alternations
|
||||
|
||||
let errors = utils.reduce(
|
||||
ors,
|
||||
(errors, currOr) => {
|
||||
if (currOr.definition.length > 255) {
|
||||
errors.push({
|
||||
message: errMsgProvider.buildTooManyAlternativesError({
|
||||
topLevelRule: topLevelRule,
|
||||
alternation: currOr
|
||||
}),
|
||||
type: ParserDefinitionErrorType.TOO_MANY_ALTS,
|
||||
ruleName: topLevelRule.name,
|
||||
occurrence: currOr.idx
|
||||
})
|
||||
}
|
||||
return errors
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
export function validateSomeNonEmptyLookaheadPath(
|
||||
topLevelRules: Rule[],
|
||||
maxLookahead: number,
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
): IParserDefinitionError[] {
|
||||
let errors = []
|
||||
forEach(topLevelRules, (currTopRule) => {
|
||||
let collectorVisitor = new RepetionCollector()
|
||||
currTopRule.accept(collectorVisitor)
|
||||
let allRuleProductions = collectorVisitor.allProductions
|
||||
forEach(allRuleProductions, (currProd) => {
|
||||
let prodType = getProdType(currProd)
|
||||
const actualMaxLookahead = currProd.maxLookahead || maxLookahead
|
||||
let currOccurrence = currProd.idx
|
||||
let paths = getLookaheadPathsForOptionalProd(
|
||||
currOccurrence,
|
||||
currTopRule,
|
||||
prodType,
|
||||
actualMaxLookahead
|
||||
)
|
||||
let pathsInsideProduction = paths[0]
|
||||
if (isEmpty(flatten(pathsInsideProduction))) {
|
||||
const errMsg = errMsgProvider.buildEmptyRepetitionError({
|
||||
topLevelRule: currTopRule,
|
||||
repetition: currProd
|
||||
})
|
||||
errors.push({
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.NO_NON_EMPTY_LOOKAHEAD,
|
||||
ruleName: currTopRule.name
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
export interface IAmbiguityDescriptor {
|
||||
alts: number[]
|
||||
path: TokenType[]
|
||||
}
|
||||
|
||||
function checkAlternativesAmbiguities(
|
||||
alternatives: Alternative[],
|
||||
alternation: Alternation,
|
||||
rule: Rule,
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
): IParserAmbiguousAlternativesDefinitionError[] {
|
||||
let foundAmbiguousPaths = []
|
||||
let identicalAmbiguities = reduce(
|
||||
alternatives,
|
||||
(result, currAlt, currAltIdx) => {
|
||||
// ignore (skip) ambiguities with this alternative
|
||||
if (alternation.definition[currAltIdx].ignoreAmbiguities === true) {
|
||||
return result
|
||||
}
|
||||
|
||||
forEach(currAlt, (currPath) => {
|
||||
let altsCurrPathAppearsIn = [currAltIdx]
|
||||
forEach(alternatives, (currOtherAlt, currOtherAltIdx) => {
|
||||
if (
|
||||
currAltIdx !== currOtherAltIdx &&
|
||||
containsPath(currOtherAlt, currPath) &&
|
||||
// ignore (skip) ambiguities with this "other" alternative
|
||||
alternation.definition[currOtherAltIdx].ignoreAmbiguities !== true
|
||||
) {
|
||||
altsCurrPathAppearsIn.push(currOtherAltIdx)
|
||||
}
|
||||
})
|
||||
|
||||
if (
|
||||
altsCurrPathAppearsIn.length > 1 &&
|
||||
!containsPath(foundAmbiguousPaths, currPath)
|
||||
) {
|
||||
foundAmbiguousPaths.push(currPath)
|
||||
result.push({
|
||||
alts: altsCurrPathAppearsIn,
|
||||
path: currPath
|
||||
})
|
||||
}
|
||||
})
|
||||
return result
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
let currErrors = utils.map(identicalAmbiguities, (currAmbDescriptor) => {
|
||||
let ambgIndices = map(
|
||||
currAmbDescriptor.alts,
|
||||
(currAltIdx) => currAltIdx + 1
|
||||
)
|
||||
|
||||
const currMessage = errMsgProvider.buildAlternationAmbiguityError({
|
||||
topLevelRule: rule,
|
||||
alternation: alternation,
|
||||
ambiguityIndices: ambgIndices,
|
||||
prefixPath: currAmbDescriptor.path
|
||||
})
|
||||
|
||||
return {
|
||||
message: currMessage,
|
||||
type: ParserDefinitionErrorType.AMBIGUOUS_ALTS,
|
||||
ruleName: rule.name,
|
||||
occurrence: alternation.idx,
|
||||
alternatives: [currAmbDescriptor.alts]
|
||||
}
|
||||
})
|
||||
|
||||
return currErrors
|
||||
}
|
||||
|
||||
export function checkPrefixAlternativesAmbiguities(
|
||||
alternatives: Alternative[],
|
||||
alternation: Alternation,
|
||||
rule: Rule,
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
): IAmbiguityDescriptor[] {
|
||||
let errors = []
|
||||
|
||||
// flatten
|
||||
let pathsAndIndices = reduce(
|
||||
alternatives,
|
||||
(result, currAlt, idx) => {
|
||||
let currPathsAndIdx = map(currAlt, (currPath) => {
|
||||
return { idx: idx, path: currPath }
|
||||
})
|
||||
return result.concat(currPathsAndIdx)
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
forEach(pathsAndIndices, (currPathAndIdx) => {
|
||||
const alternativeGast = alternation.definition[currPathAndIdx.idx]
|
||||
// ignore (skip) ambiguities with this alternative
|
||||
if (alternativeGast.ignoreAmbiguities === true) {
|
||||
return
|
||||
}
|
||||
let targetIdx = currPathAndIdx.idx
|
||||
let targetPath = currPathAndIdx.path
|
||||
|
||||
let prefixAmbiguitiesPathsAndIndices = findAll(
|
||||
pathsAndIndices,
|
||||
(searchPathAndIdx) => {
|
||||
// prefix ambiguity can only be created from lower idx (higher priority) path
|
||||
return (
|
||||
// ignore (skip) ambiguities with this "other" alternative
|
||||
alternation.definition[searchPathAndIdx.idx].ignoreAmbiguities !==
|
||||
true &&
|
||||
searchPathAndIdx.idx < targetIdx &&
|
||||
// checking for strict prefix because identical lookaheads
|
||||
// will be be detected using a different validation.
|
||||
isStrictPrefixOfPath(searchPathAndIdx.path, targetPath)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
let currPathPrefixErrors = map(
|
||||
prefixAmbiguitiesPathsAndIndices,
|
||||
(currAmbPathAndIdx) => {
|
||||
let ambgIndices = [currAmbPathAndIdx.idx + 1, targetIdx + 1]
|
||||
const occurrence = alternation.idx === 0 ? "" : alternation.idx
|
||||
|
||||
const message = errMsgProvider.buildAlternationPrefixAmbiguityError({
|
||||
topLevelRule: rule,
|
||||
alternation: alternation,
|
||||
ambiguityIndices: ambgIndices,
|
||||
prefixPath: currAmbPathAndIdx.path
|
||||
})
|
||||
return {
|
||||
message: message,
|
||||
type: ParserDefinitionErrorType.AMBIGUOUS_PREFIX_ALTS,
|
||||
ruleName: rule.name,
|
||||
occurrence: occurrence,
|
||||
alternatives: ambgIndices
|
||||
}
|
||||
}
|
||||
)
|
||||
errors = errors.concat(currPathPrefixErrors)
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function checkTerminalAndNoneTerminalsNameSpace(
|
||||
topLevels: Rule[],
|
||||
tokenTypes: TokenType[],
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
): IParserDefinitionError[] {
|
||||
let errors = []
|
||||
|
||||
let tokenNames = map(tokenTypes, (currToken) => currToken.name)
|
||||
|
||||
forEach(topLevels, (currRule) => {
|
||||
const currRuleName = currRule.name
|
||||
if (contains(tokenNames, currRuleName)) {
|
||||
let errMsg = errMsgProvider.buildNamespaceConflictError(currRule)
|
||||
|
||||
errors.push({
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.CONFLICT_TOKENS_RULES_NAMESPACE,
|
||||
ruleName: currRuleName
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
61
node_modules/chevrotain/src/parse/grammar/first.ts
generated
vendored
Normal file
61
node_modules/chevrotain/src/parse/grammar/first.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import { uniq, map, flatten } from "../../utils/utils"
|
||||
import { AbstractProduction, NonTerminal, Terminal } from "./gast/gast_public"
|
||||
import { isBranchingProd, isOptionalProd, isSequenceProd } from "./gast/gast"
|
||||
import { IProduction, TokenType } from "../../../api"
|
||||
|
||||
export function first(prod: IProduction): TokenType[] {
|
||||
/* istanbul ignore else */
|
||||
if (prod instanceof NonTerminal) {
|
||||
// this could in theory cause infinite loops if
|
||||
// (1) prod A refs prod B.
|
||||
// (2) prod B refs prod A
|
||||
// (3) AB can match the empty set
|
||||
// in other words a cycle where everything is optional so the first will keep
|
||||
// looking ahead for the next optional part and will never exit
|
||||
// currently there is no safeguard for this unique edge case because
|
||||
// (1) not sure a grammar in which this can happen is useful for anything (productive)
|
||||
return first((<NonTerminal>prod).referencedRule)
|
||||
} else if (prod instanceof Terminal) {
|
||||
return firstForTerminal(<Terminal>prod)
|
||||
} else if (isSequenceProd(prod)) {
|
||||
return firstForSequence(<AbstractProduction>prod)
|
||||
} else if (isBranchingProd(prod)) {
|
||||
return firstForBranching(<AbstractProduction>prod)
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
}
|
||||
|
||||
export function firstForSequence(prod: AbstractProduction): TokenType[] {
|
||||
let firstSet: TokenType[] = []
|
||||
let seq = prod.definition
|
||||
let nextSubProdIdx = 0
|
||||
let hasInnerProdsRemaining = seq.length > nextSubProdIdx
|
||||
let currSubProd
|
||||
// so we enter the loop at least once (if the definition is not empty
|
||||
let isLastInnerProdOptional = true
|
||||
// scan a sequence until it's end or until we have found a NONE optional production in it
|
||||
while (hasInnerProdsRemaining && isLastInnerProdOptional) {
|
||||
currSubProd = seq[nextSubProdIdx]
|
||||
isLastInnerProdOptional = isOptionalProd(currSubProd)
|
||||
firstSet = firstSet.concat(first(currSubProd))
|
||||
nextSubProdIdx = nextSubProdIdx + 1
|
||||
hasInnerProdsRemaining = seq.length > nextSubProdIdx
|
||||
}
|
||||
|
||||
return uniq(firstSet)
|
||||
}
|
||||
|
||||
export function firstForBranching(prod: AbstractProduction): TokenType[] {
|
||||
let allAlternativesFirsts: TokenType[][] = map(
|
||||
prod.definition,
|
||||
(innerProd) => {
|
||||
return first(innerProd)
|
||||
}
|
||||
)
|
||||
return uniq(flatten<TokenType>(allAlternativesFirsts))
|
||||
}
|
||||
|
||||
export function firstForTerminal(terminal: Terminal): TokenType[] {
|
||||
return [terminal.terminalType]
|
||||
}
|
||||
67
node_modules/chevrotain/src/parse/grammar/follow.ts
generated
vendored
Normal file
67
node_modules/chevrotain/src/parse/grammar/follow.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import { RestWalker } from "./rest"
|
||||
import { first } from "./first"
|
||||
import { assign, forEach } from "../../utils/utils"
|
||||
import { IN } from "../constants"
|
||||
import { Alternative, NonTerminal, Rule, Terminal } from "./gast/gast_public"
|
||||
import { IProduction, TokenType } from "../../../api"
|
||||
|
||||
// This ResyncFollowsWalker computes all of the follows required for RESYNC
|
||||
// (skipping reference production).
|
||||
export class ResyncFollowsWalker extends RestWalker {
|
||||
public follows = {}
|
||||
|
||||
constructor(private topProd: Rule) {
|
||||
super()
|
||||
}
|
||||
|
||||
startWalking(): Record<string, TokenType[]> {
|
||||
this.walk(this.topProd)
|
||||
return this.follows
|
||||
}
|
||||
|
||||
walkTerminal(
|
||||
terminal: Terminal,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
// do nothing! just like in the public sector after 13:00
|
||||
}
|
||||
|
||||
walkProdRef(
|
||||
refProd: NonTerminal,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
let followName =
|
||||
buildBetweenProdsFollowPrefix(refProd.referencedRule, refProd.idx) +
|
||||
this.topProd.name
|
||||
let fullRest: IProduction[] = currRest.concat(prevRest)
|
||||
let restProd = new Alternative({ definition: fullRest })
|
||||
let t_in_topProd_follows = first(restProd)
|
||||
this.follows[followName] = t_in_topProd_follows
|
||||
}
|
||||
}
|
||||
|
||||
export function computeAllProdsFollows(
|
||||
topProductions: Rule[]
|
||||
): Record<string, TokenType[]> {
|
||||
let reSyncFollows = {}
|
||||
|
||||
forEach(topProductions, (topProd) => {
|
||||
let currRefsFollow = new ResyncFollowsWalker(topProd).startWalking()
|
||||
assign(reSyncFollows, currRefsFollow)
|
||||
})
|
||||
return reSyncFollows
|
||||
}
|
||||
|
||||
export function buildBetweenProdsFollowPrefix(
|
||||
inner: Rule,
|
||||
occurenceInParent: number
|
||||
): string {
|
||||
return inner.name + occurenceInParent + IN
|
||||
}
|
||||
|
||||
export function buildInProdFollowPrefix(terminal: Terminal): string {
|
||||
let terminalName = terminal.terminalType.name
|
||||
return terminalName + terminal.idx + IN
|
||||
}
|
||||
179
node_modules/chevrotain/src/parse/grammar/gast/gast.ts
generated
vendored
Normal file
179
node_modules/chevrotain/src/parse/grammar/gast/gast.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
import { contains, every, has, some } from "../../../utils/utils"
|
||||
import {
|
||||
AbstractProduction,
|
||||
Alternation,
|
||||
Alternative,
|
||||
NonTerminal,
|
||||
Option,
|
||||
Repetition,
|
||||
RepetitionMandatory,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
RepetitionWithSeparator,
|
||||
Rule,
|
||||
Terminal
|
||||
} from "./gast_public"
|
||||
import { GAstVisitor } from "./gast_visitor_public"
|
||||
import { IProduction, IProductionWithOccurrence } from "../../../../api"
|
||||
|
||||
export function isSequenceProd(prod: IProduction): boolean {
|
||||
return (
|
||||
prod instanceof Alternative ||
|
||||
prod instanceof Option ||
|
||||
prod instanceof Repetition ||
|
||||
prod instanceof RepetitionMandatory ||
|
||||
prod instanceof RepetitionMandatoryWithSeparator ||
|
||||
prod instanceof RepetitionWithSeparator ||
|
||||
prod instanceof Terminal ||
|
||||
prod instanceof Rule
|
||||
)
|
||||
}
|
||||
|
||||
export function isOptionalProd(
|
||||
prod: IProduction,
|
||||
alreadyVisited: NonTerminal[] = []
|
||||
): boolean {
|
||||
let isDirectlyOptional =
|
||||
prod instanceof Option ||
|
||||
prod instanceof Repetition ||
|
||||
prod instanceof RepetitionWithSeparator
|
||||
if (isDirectlyOptional) {
|
||||
return true
|
||||
}
|
||||
|
||||
// note that this can cause infinite loop if one optional empty TOP production has a cyclic dependency with another
|
||||
// empty optional top rule
|
||||
// may be indirectly optional ((A?B?C?) | (D?E?F?))
|
||||
if (prod instanceof Alternation) {
|
||||
// for OR its enough for just one of the alternatives to be optional
|
||||
return some((<Alternation>prod).definition, (subProd: IProduction) => {
|
||||
return isOptionalProd(subProd, alreadyVisited)
|
||||
})
|
||||
} else if (prod instanceof NonTerminal && contains(alreadyVisited, prod)) {
|
||||
// avoiding stack overflow due to infinite recursion
|
||||
return false
|
||||
} else if (prod instanceof AbstractProduction) {
|
||||
if (prod instanceof NonTerminal) {
|
||||
alreadyVisited.push(prod)
|
||||
}
|
||||
return every(
|
||||
(<AbstractProduction>prod).definition,
|
||||
(subProd: IProduction) => {
|
||||
return isOptionalProd(subProd, alreadyVisited)
|
||||
}
|
||||
)
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export function isBranchingProd(prod: IProduction): boolean {
|
||||
return prod instanceof Alternation
|
||||
}
|
||||
|
||||
export function getProductionDslName(prod: IProductionWithOccurrence): string {
|
||||
/* istanbul ignore else */
|
||||
if (prod instanceof NonTerminal) {
|
||||
return "SUBRULE"
|
||||
} else if (prod instanceof Option) {
|
||||
return "OPTION"
|
||||
} else if (prod instanceof Alternation) {
|
||||
return "OR"
|
||||
} else if (prod instanceof RepetitionMandatory) {
|
||||
return "AT_LEAST_ONE"
|
||||
} else if (prod instanceof RepetitionMandatoryWithSeparator) {
|
||||
return "AT_LEAST_ONE_SEP"
|
||||
} else if (prod instanceof RepetitionWithSeparator) {
|
||||
return "MANY_SEP"
|
||||
} else if (prod instanceof Repetition) {
|
||||
return "MANY"
|
||||
} else if (prod instanceof Terminal) {
|
||||
return "CONSUME"
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
}
|
||||
|
||||
export class DslMethodsCollectorVisitor extends GAstVisitor {
|
||||
// A minus is never valid in an identifier name
|
||||
public separator = "-"
|
||||
public dslMethods = {
|
||||
option: [],
|
||||
alternation: [],
|
||||
repetition: [],
|
||||
repetitionWithSeparator: [],
|
||||
repetitionMandatory: [],
|
||||
repetitionMandatoryWithSeparator: []
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.dslMethods = {
|
||||
option: [],
|
||||
alternation: [],
|
||||
repetition: [],
|
||||
repetitionWithSeparator: [],
|
||||
repetitionMandatory: [],
|
||||
repetitionMandatoryWithSeparator: []
|
||||
}
|
||||
}
|
||||
|
||||
public visitTerminal(terminal: Terminal): void {
|
||||
const key = terminal.terminalType.name + this.separator + "Terminal"
|
||||
if (!has(this.dslMethods, key)) {
|
||||
this.dslMethods[key] = []
|
||||
}
|
||||
this.dslMethods[key].push(terminal)
|
||||
}
|
||||
|
||||
public visitNonTerminal(subrule: NonTerminal): void {
|
||||
const key = subrule.nonTerminalName + this.separator + "Terminal"
|
||||
if (!has(this.dslMethods, key)) {
|
||||
this.dslMethods[key] = []
|
||||
}
|
||||
this.dslMethods[key].push(subrule)
|
||||
}
|
||||
|
||||
public visitOption(option: Option): void {
|
||||
this.dslMethods.option.push(option)
|
||||
}
|
||||
|
||||
public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void {
|
||||
this.dslMethods.repetitionWithSeparator.push(manySep)
|
||||
}
|
||||
|
||||
public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void {
|
||||
this.dslMethods.repetitionMandatory.push(atLeastOne)
|
||||
}
|
||||
|
||||
public visitRepetitionMandatoryWithSeparator(
|
||||
atLeastOneSep: RepetitionMandatoryWithSeparator
|
||||
): void {
|
||||
this.dslMethods.repetitionMandatoryWithSeparator.push(atLeastOneSep)
|
||||
}
|
||||
|
||||
public visitRepetition(many: Repetition): void {
|
||||
this.dslMethods.repetition.push(many)
|
||||
}
|
||||
|
||||
public visitAlternation(or: Alternation): void {
|
||||
this.dslMethods.alternation.push(or)
|
||||
}
|
||||
}
|
||||
|
||||
const collectorVisitor = new DslMethodsCollectorVisitor()
|
||||
export function collectMethods(
|
||||
rule: Rule
|
||||
): {
|
||||
option: Option[]
|
||||
alternation: Alternation[]
|
||||
repetition: Repetition[]
|
||||
repetitionWithSeparator: RepetitionWithSeparator[]
|
||||
repetitionMandatory: RepetitionMandatory[]
|
||||
repetitionMandatoryWithSeparator: RepetitionMandatoryWithSeparator
|
||||
} {
|
||||
collectorVisitor.reset()
|
||||
rule.accept(collectorVisitor)
|
||||
const dslMethods = collectorVisitor.dslMethods
|
||||
// avoid uncleaned references
|
||||
collectorVisitor.reset()
|
||||
return <any>dslMethods
|
||||
}
|
||||
371
node_modules/chevrotain/src/parse/grammar/gast/gast_public.ts
generated
vendored
Normal file
371
node_modules/chevrotain/src/parse/grammar/gast/gast_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,371 @@
|
|||
import { assign, forEach, isRegExp, map, pick } from "../../../utils/utils"
|
||||
import { tokenLabel } from "../../../scan/tokens_public"
|
||||
import {
|
||||
IGASTVisitor,
|
||||
IProduction,
|
||||
IProductionWithOccurrence,
|
||||
ISerializedGast,
|
||||
TokenType
|
||||
} from "../../../../api"
|
||||
|
||||
export abstract class AbstractProduction<T extends IProduction = IProduction>
|
||||
implements IProduction {
|
||||
public get definition(): T[] {
|
||||
return this._definition
|
||||
}
|
||||
public set definition(value: T[]) {
|
||||
this._definition = value
|
||||
}
|
||||
|
||||
constructor(protected _definition: T[]) {}
|
||||
|
||||
accept(visitor: IGASTVisitor): void {
|
||||
visitor.visit(this)
|
||||
forEach(this.definition, (prod) => {
|
||||
prod.accept(visitor)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export class NonTerminal
|
||||
extends AbstractProduction
|
||||
implements IProductionWithOccurrence {
|
||||
public nonTerminalName: string
|
||||
public referencedRule: Rule
|
||||
public idx: number = 1
|
||||
|
||||
constructor(options: {
|
||||
nonTerminalName: string
|
||||
referencedRule?: Rule
|
||||
idx?: number
|
||||
}) {
|
||||
super([])
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
|
||||
set definition(definition: IProduction[]) {
|
||||
// immutable
|
||||
}
|
||||
|
||||
get definition(): IProduction[] {
|
||||
if (this.referencedRule !== undefined) {
|
||||
return this.referencedRule.definition
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
||||
accept(visitor: IGASTVisitor): void {
|
||||
visitor.visit(this)
|
||||
// don't visit children of a reference, we will get cyclic infinite loops if we do so
|
||||
}
|
||||
}
|
||||
|
||||
export class Rule extends AbstractProduction {
|
||||
public name: string
|
||||
public orgText: string = ""
|
||||
|
||||
constructor(options: {
|
||||
name: string
|
||||
definition: IProduction[]
|
||||
orgText?: string
|
||||
}) {
|
||||
super(options.definition)
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export class Alternative extends AbstractProduction {
|
||||
public ignoreAmbiguities: boolean = false
|
||||
|
||||
constructor(options: {
|
||||
definition: IProduction[]
|
||||
ignoreAmbiguities?: boolean
|
||||
}) {
|
||||
super(options.definition)
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export class Option
|
||||
extends AbstractProduction
|
||||
implements IProductionWithOccurrence {
|
||||
public idx: number = 1
|
||||
public maxLookahead?: number
|
||||
|
||||
constructor(options: {
|
||||
definition: IProduction[]
|
||||
idx?: number
|
||||
maxLookahead?: number
|
||||
}) {
|
||||
super(options.definition)
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export class RepetitionMandatory
|
||||
extends AbstractProduction
|
||||
implements IProductionWithOccurrence {
|
||||
public idx: number = 1
|
||||
public maxLookahead?: number
|
||||
|
||||
constructor(options: {
|
||||
definition: IProduction[]
|
||||
idx?: number
|
||||
maxLookahead?: number
|
||||
}) {
|
||||
super(options.definition)
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export class RepetitionMandatoryWithSeparator
|
||||
extends AbstractProduction
|
||||
implements IProductionWithOccurrence {
|
||||
public separator: TokenType
|
||||
public idx: number = 1
|
||||
|
||||
constructor(options: {
|
||||
definition: IProduction[]
|
||||
separator: TokenType
|
||||
idx?: number
|
||||
}) {
|
||||
super(options.definition)
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export class Repetition
|
||||
extends AbstractProduction
|
||||
implements IProductionWithOccurrence {
|
||||
public separator: TokenType
|
||||
public idx: number = 1
|
||||
public maxLookahead?: number
|
||||
|
||||
constructor(options: {
|
||||
definition: IProduction[]
|
||||
idx?: number
|
||||
maxLookahead?: number
|
||||
}) {
|
||||
super(options.definition)
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export class RepetitionWithSeparator
|
||||
extends AbstractProduction
|
||||
implements IProductionWithOccurrence {
|
||||
public separator: TokenType
|
||||
public idx: number = 1
|
||||
|
||||
constructor(options: {
|
||||
definition: IProduction[]
|
||||
separator: TokenType
|
||||
idx?: number
|
||||
}) {
|
||||
super(options.definition)
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export class Alternation
|
||||
extends AbstractProduction<Alternative>
|
||||
implements IProductionWithOccurrence {
|
||||
public idx: number = 1
|
||||
public ignoreAmbiguities: boolean = false
|
||||
public hasPredicates: boolean = false
|
||||
public maxLookahead?: number
|
||||
|
||||
public get definition(): Alternative[] {
|
||||
return this._definition
|
||||
}
|
||||
public set definition(value: Alternative[]) {
|
||||
this._definition = value
|
||||
}
|
||||
|
||||
constructor(options: {
|
||||
definition: Alternative[]
|
||||
idx?: number
|
||||
ignoreAmbiguities?: boolean
|
||||
hasPredicates?: boolean
|
||||
maxLookahead?: number
|
||||
}) {
|
||||
super(options.definition)
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export class Terminal implements IProductionWithOccurrence {
|
||||
public terminalType: TokenType
|
||||
public idx: number = 1
|
||||
|
||||
constructor(options: { terminalType: TokenType; idx?: number }) {
|
||||
assign(
|
||||
this,
|
||||
pick(options, (v) => v !== undefined)
|
||||
)
|
||||
}
|
||||
|
||||
accept(visitor: IGASTVisitor): void {
|
||||
visitor.visit(this)
|
||||
}
|
||||
}
|
||||
|
||||
export interface ISerializedBasic extends ISerializedGast {
|
||||
type:
|
||||
| "Alternative"
|
||||
| "Option"
|
||||
| "RepetitionMandatory"
|
||||
| "Repetition"
|
||||
| "Alternation"
|
||||
idx?: number
|
||||
}
|
||||
|
||||
export interface ISerializedGastRule extends ISerializedGast {
|
||||
type: "Rule"
|
||||
name: string
|
||||
orgText: string
|
||||
}
|
||||
|
||||
export interface ISerializedNonTerminal extends ISerializedGast {
|
||||
type: "NonTerminal"
|
||||
name: string
|
||||
idx: number
|
||||
}
|
||||
|
||||
export interface ISerializedTerminal extends ISerializedGast {
|
||||
type: "Terminal"
|
||||
name: string
|
||||
label?: string
|
||||
pattern?: string
|
||||
idx: number
|
||||
}
|
||||
|
||||
export interface ISerializedTerminalWithSeparator extends ISerializedGast {
|
||||
type: "RepetitionMandatoryWithSeparator" | "RepetitionWithSeparator"
|
||||
idx: number
|
||||
separator: ISerializedTerminal
|
||||
}
|
||||
|
||||
export type ISerializedGastAny =
|
||||
| ISerializedBasic
|
||||
| ISerializedGastRule
|
||||
| ISerializedNonTerminal
|
||||
| ISerializedTerminal
|
||||
| ISerializedTerminalWithSeparator
|
||||
|
||||
export function serializeGrammar(topRules: Rule[]): ISerializedGast[] {
|
||||
return map(topRules, serializeProduction)
|
||||
}
|
||||
|
||||
export function serializeProduction(node: IProduction): ISerializedGast {
|
||||
function convertDefinition(definition: IProduction[]): ISerializedGast[] {
|
||||
return map(definition, serializeProduction)
|
||||
}
|
||||
/* istanbul ignore else */
|
||||
if (node instanceof NonTerminal) {
|
||||
return <ISerializedNonTerminal>{
|
||||
type: "NonTerminal",
|
||||
name: node.nonTerminalName,
|
||||
idx: node.idx
|
||||
}
|
||||
} else if (node instanceof Alternative) {
|
||||
return <ISerializedBasic>{
|
||||
type: "Alternative",
|
||||
definition: convertDefinition(node.definition)
|
||||
}
|
||||
} else if (node instanceof Option) {
|
||||
return <ISerializedBasic>{
|
||||
type: "Option",
|
||||
idx: node.idx,
|
||||
definition: convertDefinition(node.definition)
|
||||
}
|
||||
} else if (node instanceof RepetitionMandatory) {
|
||||
return <ISerializedBasic>{
|
||||
type: "RepetitionMandatory",
|
||||
idx: node.idx,
|
||||
definition: convertDefinition(node.definition)
|
||||
}
|
||||
} else if (node instanceof RepetitionMandatoryWithSeparator) {
|
||||
return <ISerializedTerminalWithSeparator>{
|
||||
type: "RepetitionMandatoryWithSeparator",
|
||||
idx: node.idx,
|
||||
separator: <ISerializedTerminal>(
|
||||
serializeProduction(new Terminal({ terminalType: node.separator }))
|
||||
),
|
||||
definition: convertDefinition(node.definition)
|
||||
}
|
||||
} else if (node instanceof RepetitionWithSeparator) {
|
||||
return <ISerializedTerminalWithSeparator>{
|
||||
type: "RepetitionWithSeparator",
|
||||
idx: node.idx,
|
||||
separator: <ISerializedTerminal>(
|
||||
serializeProduction(new Terminal({ terminalType: node.separator }))
|
||||
),
|
||||
definition: convertDefinition(node.definition)
|
||||
}
|
||||
} else if (node instanceof Repetition) {
|
||||
return <ISerializedBasic>{
|
||||
type: "Repetition",
|
||||
idx: node.idx,
|
||||
definition: convertDefinition(node.definition)
|
||||
}
|
||||
} else if (node instanceof Alternation) {
|
||||
return <ISerializedBasic>{
|
||||
type: "Alternation",
|
||||
idx: node.idx,
|
||||
definition: convertDefinition(node.definition)
|
||||
}
|
||||
} else if (node instanceof Terminal) {
|
||||
let serializedTerminal = <ISerializedTerminal>{
|
||||
type: "Terminal",
|
||||
name: node.terminalType.name,
|
||||
label: tokenLabel(node.terminalType),
|
||||
idx: node.idx
|
||||
}
|
||||
|
||||
let pattern = node.terminalType.PATTERN
|
||||
if (node.terminalType.PATTERN) {
|
||||
serializedTerminal.pattern = isRegExp(pattern)
|
||||
? (<any>pattern).source
|
||||
: pattern
|
||||
}
|
||||
|
||||
return serializedTerminal
|
||||
} else if (node instanceof Rule) {
|
||||
return <ISerializedGastRule>{
|
||||
type: "Rule",
|
||||
name: node.name,
|
||||
orgText: node.orgText,
|
||||
definition: convertDefinition(node.definition)
|
||||
}
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
}
|
||||
64
node_modules/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts
generated
vendored
Normal file
64
node_modules/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
import { Rule } from "./gast_public"
|
||||
|
||||
import { defaults, forEach } from "../../../utils/utils"
|
||||
import { resolveGrammar as orgResolveGrammar } from "../resolver"
|
||||
import { validateGrammar as orgValidateGrammar } from "../checks"
|
||||
import {
|
||||
defaultGrammarResolverErrorProvider,
|
||||
defaultGrammarValidatorErrorProvider
|
||||
} from "../../errors_public"
|
||||
import { DslMethodsCollectorVisitor } from "./gast"
|
||||
import {
|
||||
IGrammarResolverErrorMessageProvider,
|
||||
IGrammarValidatorErrorMessageProvider,
|
||||
IParserDefinitionError,
|
||||
IProductionWithOccurrence,
|
||||
TokenType
|
||||
} from "../../../../api"
|
||||
|
||||
export function resolveGrammar(options: {
|
||||
rules: Rule[]
|
||||
errMsgProvider?: IGrammarResolverErrorMessageProvider
|
||||
}): IParserDefinitionError[] {
|
||||
options = defaults(options, {
|
||||
errMsgProvider: defaultGrammarResolverErrorProvider
|
||||
})
|
||||
|
||||
const topRulesTable = {}
|
||||
forEach(options.rules, (rule) => {
|
||||
topRulesTable[rule.name] = rule
|
||||
})
|
||||
return orgResolveGrammar(topRulesTable, options.errMsgProvider)
|
||||
}
|
||||
|
||||
export function validateGrammar(options: {
|
||||
rules: Rule[]
|
||||
maxLookahead: number
|
||||
tokenTypes: TokenType[]
|
||||
grammarName: string
|
||||
errMsgProvider: IGrammarValidatorErrorMessageProvider
|
||||
}): IParserDefinitionError[] {
|
||||
options = defaults(options, {
|
||||
errMsgProvider: defaultGrammarValidatorErrorProvider
|
||||
})
|
||||
|
||||
return orgValidateGrammar(
|
||||
options.rules,
|
||||
options.maxLookahead,
|
||||
options.tokenTypes,
|
||||
options.errMsgProvider,
|
||||
options.grammarName
|
||||
)
|
||||
}
|
||||
|
||||
export function assignOccurrenceIndices(options: { rules: Rule[] }): void {
|
||||
forEach(options.rules, (currRule) => {
|
||||
const methodsCollector = new DslMethodsCollectorVisitor()
|
||||
currRule.accept(methodsCollector)
|
||||
forEach(methodsCollector.dslMethods, (methods) => {
|
||||
forEach(methods, (currMethod: IProductionWithOccurrence, arrIdx) => {
|
||||
currMethod.idx = arrIdx + 1
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
66
node_modules/chevrotain/src/parse/grammar/gast/gast_visitor_public.ts
generated
vendored
Normal file
66
node_modules/chevrotain/src/parse/grammar/gast/gast_visitor_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import {
|
||||
Alternation,
|
||||
Alternative,
|
||||
NonTerminal,
|
||||
Option,
|
||||
Repetition,
|
||||
RepetitionMandatory,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
RepetitionWithSeparator,
|
||||
Rule,
|
||||
Terminal
|
||||
} from "./gast_public"
|
||||
import { IProduction } from "../../../../api"
|
||||
|
||||
export abstract class GAstVisitor {
|
||||
public visit(node: IProduction): any {
|
||||
const nodeAny: any = node
|
||||
switch (nodeAny.constructor) {
|
||||
case NonTerminal:
|
||||
return this.visitNonTerminal(nodeAny)
|
||||
case Alternative:
|
||||
return this.visitAlternative(nodeAny)
|
||||
case Option:
|
||||
return this.visitOption(nodeAny)
|
||||
case RepetitionMandatory:
|
||||
return this.visitRepetitionMandatory(nodeAny)
|
||||
case RepetitionMandatoryWithSeparator:
|
||||
return this.visitRepetitionMandatoryWithSeparator(nodeAny)
|
||||
case RepetitionWithSeparator:
|
||||
return this.visitRepetitionWithSeparator(nodeAny)
|
||||
case Repetition:
|
||||
return this.visitRepetition(nodeAny)
|
||||
case Alternation:
|
||||
return this.visitAlternation(nodeAny)
|
||||
case Terminal:
|
||||
return this.visitTerminal(nodeAny)
|
||||
case Rule:
|
||||
return this.visitRule(nodeAny)
|
||||
/* istanbul ignore next */
|
||||
default:
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
}
|
||||
|
||||
public visitNonTerminal(node: NonTerminal): any {}
|
||||
|
||||
public visitAlternative(node: Alternative): any {}
|
||||
|
||||
public visitOption(node: Option): any {}
|
||||
|
||||
public visitRepetition(node: Repetition): any {}
|
||||
|
||||
public visitRepetitionMandatory(node: RepetitionMandatory): any {}
|
||||
|
||||
public visitRepetitionMandatoryWithSeparator(
|
||||
node: RepetitionMandatoryWithSeparator
|
||||
): any {}
|
||||
|
||||
public visitRepetitionWithSeparator(node: RepetitionWithSeparator): any {}
|
||||
|
||||
public visitAlternation(node: Alternation): any {}
|
||||
|
||||
public visitTerminal(node: Terminal): any {}
|
||||
|
||||
public visitRule(node: Rule): any {}
|
||||
}
|
||||
610
node_modules/chevrotain/src/parse/grammar/interpreter.ts
generated
vendored
Normal file
610
node_modules/chevrotain/src/parse/grammar/interpreter.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,610 @@
|
|||
import { RestWalker } from "./rest"
|
||||
import {
|
||||
cloneArr,
|
||||
drop,
|
||||
dropRight,
|
||||
first as _first,
|
||||
forEach,
|
||||
isEmpty,
|
||||
last
|
||||
} from "../../utils/utils"
|
||||
import { first } from "./first"
|
||||
import { TokenMatcher } from "../parser/parser"
|
||||
import {
|
||||
AbstractProduction,
|
||||
Alternation,
|
||||
Alternative,
|
||||
NonTerminal,
|
||||
Option,
|
||||
Repetition,
|
||||
RepetitionMandatory,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
RepetitionWithSeparator,
|
||||
Rule,
|
||||
Terminal
|
||||
} from "./gast/gast_public"
|
||||
import {
|
||||
IGrammarPath,
|
||||
IProduction,
|
||||
ISyntacticContentAssistPath,
|
||||
IToken,
|
||||
ITokenGrammarPath,
|
||||
TokenType
|
||||
} from "../../../api"
|
||||
|
||||
export abstract class AbstractNextPossibleTokensWalker extends RestWalker {
|
||||
protected possibleTokTypes: TokenType[] = []
|
||||
protected ruleStack: string[]
|
||||
protected occurrenceStack: number[]
|
||||
|
||||
protected nextProductionName = ""
|
||||
protected nextProductionOccurrence = 0
|
||||
protected found = false
|
||||
protected isAtEndOfPath = false
|
||||
|
||||
constructor(protected topProd: Rule, protected path: IGrammarPath) {
|
||||
super()
|
||||
}
|
||||
|
||||
startWalking(): TokenType[] {
|
||||
this.found = false
|
||||
|
||||
if (this.path.ruleStack[0] !== this.topProd.name) {
|
||||
throw Error("The path does not start with the walker's top Rule!")
|
||||
}
|
||||
|
||||
// immutable for the win
|
||||
this.ruleStack = cloneArr(this.path.ruleStack).reverse() // intelij bug requires assertion
|
||||
this.occurrenceStack = cloneArr(this.path.occurrenceStack).reverse() // intelij bug requires assertion
|
||||
|
||||
// already verified that the first production is valid, we now seek the 2nd production
|
||||
this.ruleStack.pop()
|
||||
this.occurrenceStack.pop()
|
||||
|
||||
this.updateExpectedNext()
|
||||
this.walk(this.topProd)
|
||||
|
||||
return this.possibleTokTypes
|
||||
}
|
||||
|
||||
walk(prod: AbstractProduction, prevRest: IProduction[] = []): void {
|
||||
// stop scanning once we found the path
|
||||
if (!this.found) {
|
||||
super.walk(prod, prevRest)
|
||||
}
|
||||
}
|
||||
|
||||
walkProdRef(
|
||||
refProd: NonTerminal,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
// found the next production, need to keep walking in it
|
||||
if (
|
||||
refProd.referencedRule.name === this.nextProductionName &&
|
||||
refProd.idx === this.nextProductionOccurrence
|
||||
) {
|
||||
let fullRest = currRest.concat(prevRest)
|
||||
this.updateExpectedNext()
|
||||
this.walk(refProd.referencedRule, <any>fullRest)
|
||||
}
|
||||
}
|
||||
|
||||
updateExpectedNext(): void {
|
||||
// need to consume the Terminal
|
||||
if (isEmpty(this.ruleStack)) {
|
||||
// must reset nextProductionXXX to avoid walking down another Top Level production while what we are
|
||||
// really seeking is the last Terminal...
|
||||
this.nextProductionName = ""
|
||||
this.nextProductionOccurrence = 0
|
||||
this.isAtEndOfPath = true
|
||||
} else {
|
||||
this.nextProductionName = this.ruleStack.pop()
|
||||
this.nextProductionOccurrence = this.occurrenceStack.pop()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class NextAfterTokenWalker extends AbstractNextPossibleTokensWalker {
|
||||
private nextTerminalName = ""
|
||||
private nextTerminalOccurrence = 0
|
||||
|
||||
constructor(topProd: Rule, protected path: ITokenGrammarPath) {
|
||||
super(topProd, path)
|
||||
this.nextTerminalName = this.path.lastTok.name
|
||||
this.nextTerminalOccurrence = this.path.lastTokOccurrence
|
||||
}
|
||||
|
||||
walkTerminal(
|
||||
terminal: Terminal,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (
|
||||
this.isAtEndOfPath &&
|
||||
terminal.terminalType.name === this.nextTerminalName &&
|
||||
terminal.idx === this.nextTerminalOccurrence &&
|
||||
!this.found
|
||||
) {
|
||||
let fullRest = currRest.concat(prevRest)
|
||||
let restProd = new Alternative({ definition: fullRest })
|
||||
this.possibleTokTypes = first(restProd)
|
||||
this.found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type AlternativesFirstTokens = TokenType[][]
|
||||
|
||||
export interface IFirstAfterRepetition {
|
||||
token: TokenType
|
||||
occurrence: number
|
||||
isEndOfRule: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* This walker only "walks" a single "TOP" level in the Grammar Ast, this means
|
||||
* it never "follows" production refs
|
||||
*/
|
||||
export class AbstractNextTerminalAfterProductionWalker extends RestWalker {
|
||||
protected result = {
|
||||
token: undefined,
|
||||
occurrence: undefined,
|
||||
isEndOfRule: undefined
|
||||
}
|
||||
|
||||
constructor(protected topRule: Rule, protected occurrence: number) {
|
||||
super()
|
||||
}
|
||||
|
||||
startWalking(): IFirstAfterRepetition {
|
||||
this.walk(this.topRule)
|
||||
return this.result
|
||||
}
|
||||
}
|
||||
|
||||
export class NextTerminalAfterManyWalker extends AbstractNextTerminalAfterProductionWalker {
|
||||
walkMany(
|
||||
manyProd: Repetition,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (manyProd.idx === this.occurrence) {
|
||||
let firstAfterMany = _first(currRest.concat(prevRest))
|
||||
this.result.isEndOfRule = firstAfterMany === undefined
|
||||
if (firstAfterMany instanceof Terminal) {
|
||||
this.result.token = firstAfterMany.terminalType
|
||||
this.result.occurrence = firstAfterMany.idx
|
||||
}
|
||||
} else {
|
||||
super.walkMany(manyProd, currRest, prevRest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class NextTerminalAfterManySepWalker extends AbstractNextTerminalAfterProductionWalker {
|
||||
walkManySep(
|
||||
manySepProd: RepetitionWithSeparator,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (manySepProd.idx === this.occurrence) {
|
||||
let firstAfterManySep = _first(currRest.concat(prevRest))
|
||||
this.result.isEndOfRule = firstAfterManySep === undefined
|
||||
if (firstAfterManySep instanceof Terminal) {
|
||||
this.result.token = firstAfterManySep.terminalType
|
||||
this.result.occurrence = firstAfterManySep.idx
|
||||
}
|
||||
} else {
|
||||
super.walkManySep(manySepProd, currRest, prevRest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class NextTerminalAfterAtLeastOneWalker extends AbstractNextTerminalAfterProductionWalker {
|
||||
walkAtLeastOne(
|
||||
atLeastOneProd: RepetitionMandatory,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (atLeastOneProd.idx === this.occurrence) {
|
||||
let firstAfterAtLeastOne = _first(currRest.concat(prevRest))
|
||||
this.result.isEndOfRule = firstAfterAtLeastOne === undefined
|
||||
if (firstAfterAtLeastOne instanceof Terminal) {
|
||||
this.result.token = firstAfterAtLeastOne.terminalType
|
||||
this.result.occurrence = firstAfterAtLeastOne.idx
|
||||
}
|
||||
} else {
|
||||
super.walkAtLeastOne(atLeastOneProd, currRest, prevRest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: reduce code duplication in the AfterWalkers
|
||||
export class NextTerminalAfterAtLeastOneSepWalker extends AbstractNextTerminalAfterProductionWalker {
|
||||
walkAtLeastOneSep(
|
||||
atleastOneSepProd: RepetitionMandatoryWithSeparator,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (atleastOneSepProd.idx === this.occurrence) {
|
||||
let firstAfterfirstAfterAtLeastOneSep = _first(currRest.concat(prevRest))
|
||||
this.result.isEndOfRule = firstAfterfirstAfterAtLeastOneSep === undefined
|
||||
if (firstAfterfirstAfterAtLeastOneSep instanceof Terminal) {
|
||||
this.result.token = firstAfterfirstAfterAtLeastOneSep.terminalType
|
||||
this.result.occurrence = firstAfterfirstAfterAtLeastOneSep.idx
|
||||
}
|
||||
} else {
|
||||
super.walkAtLeastOneSep(atleastOneSepProd, currRest, prevRest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface PartialPathAndSuffixes {
|
||||
partialPath: TokenType[]
|
||||
suffixDef: IProduction[]
|
||||
}
|
||||
|
||||
export function possiblePathsFrom(
|
||||
targetDef: IProduction[],
|
||||
maxLength: number,
|
||||
currPath = []
|
||||
): PartialPathAndSuffixes[] {
|
||||
// avoid side effects
|
||||
currPath = cloneArr(currPath)
|
||||
let result = []
|
||||
let i = 0
|
||||
|
||||
// TODO: avoid inner funcs
|
||||
function remainingPathWith(nextDef: IProduction[]) {
|
||||
return nextDef.concat(drop(targetDef, i + 1))
|
||||
}
|
||||
|
||||
// TODO: avoid inner funcs
|
||||
function getAlternativesForProd(definition: IProduction[]) {
|
||||
let alternatives = possiblePathsFrom(
|
||||
remainingPathWith(definition),
|
||||
maxLength,
|
||||
currPath
|
||||
)
|
||||
return result.concat(alternatives)
|
||||
}
|
||||
|
||||
/**
|
||||
* Mandatory productions will halt the loop as the paths computed from their recursive calls will already contain the
|
||||
* following (rest) of the targetDef.
|
||||
*
|
||||
* For optional productions (Option/Repetition/...) the loop will continue to represent the paths that do not include the
|
||||
* the optional production.
|
||||
*/
|
||||
while (currPath.length < maxLength && i < targetDef.length) {
|
||||
let prod = targetDef[i]
|
||||
|
||||
/* istanbul ignore else */
|
||||
if (prod instanceof Alternative) {
|
||||
return getAlternativesForProd(prod.definition)
|
||||
} else if (prod instanceof NonTerminal) {
|
||||
return getAlternativesForProd(prod.definition)
|
||||
} else if (prod instanceof Option) {
|
||||
result = getAlternativesForProd(prod.definition)
|
||||
} else if (prod instanceof RepetitionMandatory) {
|
||||
const newDef = prod.definition.concat([
|
||||
new Repetition({
|
||||
definition: prod.definition
|
||||
})
|
||||
])
|
||||
return getAlternativesForProd(newDef)
|
||||
} else if (prod instanceof RepetitionMandatoryWithSeparator) {
|
||||
const newDef = [
|
||||
new Alternative({ definition: prod.definition }),
|
||||
new Repetition({
|
||||
definition: [new Terminal({ terminalType: prod.separator })].concat(
|
||||
<any>prod.definition
|
||||
)
|
||||
})
|
||||
]
|
||||
return getAlternativesForProd(newDef)
|
||||
} else if (prod instanceof RepetitionWithSeparator) {
|
||||
const newDef = prod.definition.concat([
|
||||
new Repetition({
|
||||
definition: [new Terminal({ terminalType: prod.separator })].concat(
|
||||
<any>prod.definition
|
||||
)
|
||||
})
|
||||
])
|
||||
result = getAlternativesForProd(newDef)
|
||||
} else if (prod instanceof Repetition) {
|
||||
const newDef = prod.definition.concat([
|
||||
new Repetition({
|
||||
definition: prod.definition
|
||||
})
|
||||
])
|
||||
result = getAlternativesForProd(newDef)
|
||||
} else if (prod instanceof Alternation) {
|
||||
forEach(prod.definition, (currAlt) => {
|
||||
// TODO: this is a limited check for empty alternatives
|
||||
// It would prevent a common case of infinite loops during parser initialization.
|
||||
// However **in-directly** empty alternatives may still cause issues.
|
||||
if (isEmpty(currAlt.definition) === false) {
|
||||
result = getAlternativesForProd(currAlt.definition)
|
||||
}
|
||||
})
|
||||
return result
|
||||
} else if (prod instanceof Terminal) {
|
||||
currPath.push(prod.terminalType)
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
result.push({
|
||||
partialPath: currPath,
|
||||
suffixDef: drop(targetDef, i)
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
interface IPathToExamine {
|
||||
idx: number
|
||||
def: IProduction[]
|
||||
ruleStack: string[]
|
||||
occurrenceStack: number[]
|
||||
}
|
||||
|
||||
export function nextPossibleTokensAfter(
|
||||
initialDef: IProduction[],
|
||||
tokenVector: IToken[],
|
||||
tokMatcher: TokenMatcher,
|
||||
maxLookAhead: number
|
||||
): ISyntacticContentAssistPath[] {
|
||||
const EXIT_NON_TERMINAL: any = "EXIT_NONE_TERMINAL"
|
||||
// to avoid creating a new Array each time.
|
||||
const EXIT_NON_TERMINAL_ARR = [EXIT_NON_TERMINAL]
|
||||
const EXIT_ALTERNATIVE: any = "EXIT_ALTERNATIVE"
|
||||
let foundCompletePath = false
|
||||
|
||||
const tokenVectorLength = tokenVector.length
|
||||
let minimalAlternativesIndex = tokenVectorLength - maxLookAhead - 1
|
||||
|
||||
let result: ISyntacticContentAssistPath[] = []
|
||||
|
||||
let possiblePaths: IPathToExamine[] = []
|
||||
possiblePaths.push({
|
||||
idx: -1,
|
||||
def: initialDef,
|
||||
ruleStack: [],
|
||||
occurrenceStack: []
|
||||
})
|
||||
|
||||
while (!isEmpty(possiblePaths)) {
|
||||
let currPath = possiblePaths.pop()
|
||||
|
||||
// skip alternatives if no more results can be found (assuming deterministic grammar with fixed lookahead)
|
||||
if (currPath === EXIT_ALTERNATIVE) {
|
||||
if (
|
||||
foundCompletePath &&
|
||||
last(possiblePaths).idx <= minimalAlternativesIndex
|
||||
) {
|
||||
// remove irrelevant alternative
|
||||
possiblePaths.pop()
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
let currDef = currPath.def
|
||||
let currIdx = currPath.idx
|
||||
let currRuleStack = currPath.ruleStack
|
||||
let currOccurrenceStack = currPath.occurrenceStack
|
||||
|
||||
// For Example: an empty path could exist in a valid grammar in the case of an EMPTY_ALT
|
||||
if (isEmpty(currDef)) {
|
||||
continue
|
||||
}
|
||||
|
||||
let prod = currDef[0]
|
||||
/* istanbul ignore else */
|
||||
if (prod === EXIT_NON_TERMINAL) {
|
||||
let nextPath = {
|
||||
idx: currIdx,
|
||||
def: drop(currDef),
|
||||
ruleStack: dropRight(currRuleStack),
|
||||
occurrenceStack: dropRight(currOccurrenceStack)
|
||||
}
|
||||
possiblePaths.push(nextPath)
|
||||
} else if (prod instanceof Terminal) {
|
||||
/* istanbul ignore else */
|
||||
if (currIdx < tokenVectorLength - 1) {
|
||||
let nextIdx = currIdx + 1
|
||||
let actualToken = tokenVector[nextIdx]
|
||||
if (tokMatcher(actualToken, prod.terminalType)) {
|
||||
let nextPath = {
|
||||
idx: nextIdx,
|
||||
def: drop(currDef),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPath)
|
||||
}
|
||||
// end of the line
|
||||
} else if (currIdx === tokenVectorLength - 1) {
|
||||
// IGNORE ABOVE ELSE
|
||||
result.push({
|
||||
nextTokenType: prod.terminalType,
|
||||
nextTokenOccurrence: prod.idx,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
})
|
||||
foundCompletePath = true
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
} else if (prod instanceof NonTerminal) {
|
||||
let newRuleStack = cloneArr(currRuleStack)
|
||||
newRuleStack.push(prod.nonTerminalName)
|
||||
|
||||
let newOccurrenceStack = cloneArr(currOccurrenceStack)
|
||||
newOccurrenceStack.push(prod.idx)
|
||||
|
||||
let nextPath = {
|
||||
idx: currIdx,
|
||||
def: prod.definition.concat(EXIT_NON_TERMINAL_ARR, drop(currDef)),
|
||||
ruleStack: newRuleStack,
|
||||
occurrenceStack: newOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPath)
|
||||
} else if (prod instanceof Option) {
|
||||
// the order of alternatives is meaningful, FILO (Last path will be traversed first).
|
||||
let nextPathWithout = {
|
||||
idx: currIdx,
|
||||
def: drop(currDef),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPathWithout)
|
||||
// required marker to avoid backtracking paths whose higher priority alternatives already matched
|
||||
possiblePaths.push(EXIT_ALTERNATIVE)
|
||||
|
||||
let nextPathWith = {
|
||||
idx: currIdx,
|
||||
def: prod.definition.concat(drop(currDef)),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPathWith)
|
||||
} else if (prod instanceof RepetitionMandatory) {
|
||||
// TODO:(THE NEW operators here take a while...) (convert once?)
|
||||
let secondIteration = new Repetition({
|
||||
definition: prod.definition,
|
||||
idx: prod.idx
|
||||
})
|
||||
let nextDef = prod.definition.concat([secondIteration], drop(currDef))
|
||||
let nextPath = {
|
||||
idx: currIdx,
|
||||
def: nextDef,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPath)
|
||||
} else if (prod instanceof RepetitionMandatoryWithSeparator) {
|
||||
// TODO:(THE NEW operators here take a while...) (convert once?)
|
||||
let separatorGast = new Terminal({
|
||||
terminalType: prod.separator
|
||||
})
|
||||
let secondIteration = new Repetition({
|
||||
definition: [<any>separatorGast].concat(prod.definition),
|
||||
idx: prod.idx
|
||||
})
|
||||
let nextDef = prod.definition.concat([secondIteration], drop(currDef))
|
||||
let nextPath = {
|
||||
idx: currIdx,
|
||||
def: nextDef,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPath)
|
||||
} else if (prod instanceof RepetitionWithSeparator) {
|
||||
// the order of alternatives is meaningful, FILO (Last path will be traversed first).
|
||||
let nextPathWithout = {
|
||||
idx: currIdx,
|
||||
def: drop(currDef),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPathWithout)
|
||||
// required marker to avoid backtracking paths whose higher priority alternatives already matched
|
||||
possiblePaths.push(EXIT_ALTERNATIVE)
|
||||
|
||||
let separatorGast = new Terminal({
|
||||
terminalType: prod.separator
|
||||
})
|
||||
let nthRepetition = new Repetition({
|
||||
definition: [<any>separatorGast].concat(prod.definition),
|
||||
idx: prod.idx
|
||||
})
|
||||
let nextDef = prod.definition.concat([nthRepetition], drop(currDef))
|
||||
let nextPathWith = {
|
||||
idx: currIdx,
|
||||
def: nextDef,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPathWith)
|
||||
} else if (prod instanceof Repetition) {
|
||||
// the order of alternatives is meaningful, FILO (Last path will be traversed first).
|
||||
let nextPathWithout = {
|
||||
idx: currIdx,
|
||||
def: drop(currDef),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPathWithout)
|
||||
// required marker to avoid backtracking paths whose higher priority alternatives already matched
|
||||
possiblePaths.push(EXIT_ALTERNATIVE)
|
||||
|
||||
// TODO: an empty repetition will cause infinite loops here, will the parser detect this in selfAnalysis?
|
||||
let nthRepetition = new Repetition({
|
||||
definition: prod.definition,
|
||||
idx: prod.idx
|
||||
})
|
||||
let nextDef = prod.definition.concat([nthRepetition], drop(currDef))
|
||||
let nextPathWith = {
|
||||
idx: currIdx,
|
||||
def: nextDef,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(nextPathWith)
|
||||
} else if (prod instanceof Alternation) {
|
||||
// the order of alternatives is meaningful, FILO (Last path will be traversed first).
|
||||
for (let i = prod.definition.length - 1; i >= 0; i--) {
|
||||
let currAlt: any = prod.definition[i]
|
||||
let currAltPath = {
|
||||
idx: currIdx,
|
||||
def: currAlt.definition.concat(drop(currDef)),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
}
|
||||
possiblePaths.push(currAltPath)
|
||||
possiblePaths.push(EXIT_ALTERNATIVE)
|
||||
}
|
||||
} else if (prod instanceof Alternative) {
|
||||
possiblePaths.push({
|
||||
idx: currIdx,
|
||||
def: prod.definition.concat(drop(currDef)),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack
|
||||
})
|
||||
} else if (prod instanceof Rule) {
|
||||
// last because we should only encounter at most a single one of these per invocation.
|
||||
possiblePaths.push(
|
||||
expandTopLevelRule(prod, currIdx, currRuleStack, currOccurrenceStack)
|
||||
)
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
function expandTopLevelRule(
|
||||
topRule: Rule,
|
||||
currIdx: number,
|
||||
currRuleStack: string[],
|
||||
currOccurrenceStack: number[]
|
||||
): IPathToExamine {
|
||||
let newRuleStack = cloneArr(currRuleStack)
|
||||
newRuleStack.push(topRule.name)
|
||||
|
||||
let newCurrOccurrenceStack = cloneArr(currOccurrenceStack)
|
||||
// top rule is always assumed to have been called with occurrence index 1
|
||||
newCurrOccurrenceStack.push(1)
|
||||
|
||||
return {
|
||||
idx: currIdx,
|
||||
def: topRule.definition,
|
||||
ruleStack: newRuleStack,
|
||||
occurrenceStack: newCurrOccurrenceStack
|
||||
}
|
||||
}
|
||||
37
node_modules/chevrotain/src/parse/grammar/keys.ts
generated
vendored
Normal file
37
node_modules/chevrotain/src/parse/grammar/keys.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
// Lookahead keys are 32Bit integers in the form
|
||||
// TTTTTTTT-ZZZZZZZZZZZZ-YYYY-XXXXXXXX
|
||||
// XXXX -> Occurrence Index bitmap.
|
||||
// YYYY -> DSL Method Type bitmap.
|
||||
// ZZZZZZZZZZZZZZZ -> Rule short Index bitmap.
|
||||
// TTTTTTTTT -> alternation alternative index bitmap
|
||||
|
||||
export const BITS_FOR_METHOD_TYPE = 4
|
||||
export const BITS_FOR_OCCURRENCE_IDX = 8
|
||||
export const BITS_FOR_RULE_IDX = 12
|
||||
// TODO: validation, this means that there may at most 2^8 --> 256 alternatives for an alternation.
|
||||
export const BITS_FOR_ALT_IDX = 8
|
||||
|
||||
// short string used as part of mapping keys.
|
||||
// being short improves the performance when composing KEYS for maps out of these
|
||||
// The 5 - 8 bits (16 possible values, are reserved for the DSL method indices)
|
||||
/* tslint:disable */
|
||||
export const OR_IDX = 1 << BITS_FOR_OCCURRENCE_IDX
|
||||
export const OPTION_IDX = 2 << BITS_FOR_OCCURRENCE_IDX
|
||||
export const MANY_IDX = 3 << BITS_FOR_OCCURRENCE_IDX
|
||||
export const AT_LEAST_ONE_IDX = 4 << BITS_FOR_OCCURRENCE_IDX
|
||||
export const MANY_SEP_IDX = 5 << BITS_FOR_OCCURRENCE_IDX
|
||||
export const AT_LEAST_ONE_SEP_IDX = 6 << BITS_FOR_OCCURRENCE_IDX
|
||||
/* tslint:enable */
|
||||
|
||||
// this actually returns a number, but it is always used as a string (object prop key)
|
||||
export function getKeyForAutomaticLookahead(
|
||||
ruleIdx: number,
|
||||
dslMethodIdx: number,
|
||||
occurrence: number
|
||||
): number {
|
||||
/* tslint:disable */
|
||||
return occurrence | dslMethodIdx | ruleIdx
|
||||
/* tslint:enable */
|
||||
}
|
||||
|
||||
const BITS_START_FOR_ALT_IDX = 32 - BITS_FOR_ALT_IDX
|
||||
695
node_modules/chevrotain/src/parse/grammar/lookahead.ts
generated
vendored
Normal file
695
node_modules/chevrotain/src/parse/grammar/lookahead.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,695 @@
|
|||
import {
|
||||
map,
|
||||
reduce,
|
||||
every,
|
||||
isEmpty,
|
||||
flatten,
|
||||
forEach,
|
||||
has
|
||||
} from "../../utils/utils"
|
||||
import { possiblePathsFrom } from "./interpreter"
|
||||
import { RestWalker } from "./rest"
|
||||
import { Predicate, TokenMatcher, lookAheadSequence } from "../parser/parser"
|
||||
import {
|
||||
tokenStructuredMatcher,
|
||||
tokenStructuredMatcherNoCategories
|
||||
} from "../../scan/tokens"
|
||||
import {
|
||||
AbstractProduction,
|
||||
Alternation,
|
||||
Alternative as AlternativeGAST,
|
||||
Option,
|
||||
Repetition,
|
||||
RepetitionMandatory,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
RepetitionWithSeparator,
|
||||
Rule
|
||||
} from "./gast/gast_public"
|
||||
import { GAstVisitor } from "./gast/gast_visitor_public"
|
||||
import {
|
||||
IOrAlt,
|
||||
IProduction,
|
||||
IProductionWithOccurrence,
|
||||
TokenType
|
||||
} from "../../../api"
|
||||
|
||||
export enum PROD_TYPE {
|
||||
OPTION,
|
||||
REPETITION,
|
||||
REPETITION_MANDATORY,
|
||||
REPETITION_MANDATORY_WITH_SEPARATOR,
|
||||
REPETITION_WITH_SEPARATOR,
|
||||
ALTERNATION
|
||||
}
|
||||
|
||||
export function getProdType(prod: IProduction): PROD_TYPE {
|
||||
/* istanbul ignore else */
|
||||
if (prod instanceof Option) {
|
||||
return PROD_TYPE.OPTION
|
||||
} else if (prod instanceof Repetition) {
|
||||
return PROD_TYPE.REPETITION
|
||||
} else if (prod instanceof RepetitionMandatory) {
|
||||
return PROD_TYPE.REPETITION_MANDATORY
|
||||
} else if (prod instanceof RepetitionMandatoryWithSeparator) {
|
||||
return PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR
|
||||
} else if (prod instanceof RepetitionWithSeparator) {
|
||||
return PROD_TYPE.REPETITION_WITH_SEPARATOR
|
||||
} else if (prod instanceof Alternation) {
|
||||
return PROD_TYPE.ALTERNATION
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
}
|
||||
|
||||
export function buildLookaheadFuncForOr(
|
||||
occurrence: number,
|
||||
ruleGrammar: Rule,
|
||||
maxLookahead: number,
|
||||
hasPredicates: boolean,
|
||||
dynamicTokensEnabled: boolean,
|
||||
laFuncBuilder: Function
|
||||
): (orAlts?: IOrAlt<any>[]) => number {
|
||||
let lookAheadPaths = getLookaheadPathsForOr(
|
||||
occurrence,
|
||||
ruleGrammar,
|
||||
maxLookahead
|
||||
)
|
||||
|
||||
const tokenMatcher = areTokenCategoriesNotUsed(lookAheadPaths)
|
||||
? tokenStructuredMatcherNoCategories
|
||||
: tokenStructuredMatcher
|
||||
|
||||
return laFuncBuilder(
|
||||
lookAheadPaths,
|
||||
hasPredicates,
|
||||
tokenMatcher,
|
||||
dynamicTokensEnabled
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* When dealing with an Optional production (OPTION/MANY/2nd iteration of AT_LEAST_ONE/...) we need to compare
|
||||
* the lookahead "inside" the production and the lookahead immediately "after" it in the same top level rule (context free).
|
||||
*
|
||||
* Example: given a production:
|
||||
* ABC(DE)?DF
|
||||
*
|
||||
* The optional '(DE)?' should only be entered if we see 'DE'. a single Token 'D' is not sufficient to distinguish between the two
|
||||
* alternatives.
|
||||
*
|
||||
* @returns A Lookahead function which will return true IFF the parser should parse the Optional production.
|
||||
*/
|
||||
export function buildLookaheadFuncForOptionalProd(
|
||||
occurrence: number,
|
||||
ruleGrammar: Rule,
|
||||
k: number,
|
||||
dynamicTokensEnabled: boolean,
|
||||
prodType: PROD_TYPE,
|
||||
lookaheadBuilder: (lookAheadSequence, TokenMatcher, boolean) => () => boolean
|
||||
): () => boolean {
|
||||
let lookAheadPaths = getLookaheadPathsForOptionalProd(
|
||||
occurrence,
|
||||
ruleGrammar,
|
||||
prodType,
|
||||
k
|
||||
)
|
||||
|
||||
const tokenMatcher = areTokenCategoriesNotUsed(lookAheadPaths)
|
||||
? tokenStructuredMatcherNoCategories
|
||||
: tokenStructuredMatcher
|
||||
|
||||
return lookaheadBuilder(lookAheadPaths[0], tokenMatcher, dynamicTokensEnabled)
|
||||
}
|
||||
|
||||
export type Alternative = TokenType[][]
|
||||
|
||||
export function buildAlternativesLookAheadFunc(
|
||||
alts: lookAheadSequence[],
|
||||
hasPredicates: boolean,
|
||||
tokenMatcher: TokenMatcher,
|
||||
dynamicTokensEnabled: boolean
|
||||
): (orAlts?: IOrAlt<any>[]) => number {
|
||||
let numOfAlts = alts.length
|
||||
let areAllOneTokenLookahead = every(alts, (currAlt) => {
|
||||
return every(currAlt, (currPath) => {
|
||||
return currPath.length === 1
|
||||
})
|
||||
})
|
||||
|
||||
// This version takes into account the predicates as well.
|
||||
if (hasPredicates) {
|
||||
/**
|
||||
* @returns {number} - The chosen alternative index
|
||||
*/
|
||||
return function (orAlts: IOrAlt<any>[]): number {
|
||||
// unfortunately the predicates must be extracted every single time
|
||||
// as they cannot be cached due to references to parameters(vars) which are no longer valid.
|
||||
// note that in the common case of no predicates, no cpu time will be wasted on this (see else block)
|
||||
let predicates: Predicate[] = map(orAlts, (currAlt) => currAlt.GATE)
|
||||
|
||||
for (let t = 0; t < numOfAlts; t++) {
|
||||
let currAlt = alts[t]
|
||||
let currNumOfPaths = currAlt.length
|
||||
|
||||
let currPredicate = predicates[t]
|
||||
if (currPredicate !== undefined && currPredicate.call(this) === false) {
|
||||
// if the predicate does not match there is no point in checking the paths
|
||||
continue
|
||||
}
|
||||
nextPath: for (let j = 0; j < currNumOfPaths; j++) {
|
||||
let currPath = currAlt[j]
|
||||
let currPathLength = currPath.length
|
||||
for (let i = 0; i < currPathLength; i++) {
|
||||
let nextToken = this.LA(i + 1)
|
||||
if (tokenMatcher(nextToken, currPath[i]) === false) {
|
||||
// mismatch in current path
|
||||
// try the next pth
|
||||
continue nextPath
|
||||
}
|
||||
}
|
||||
// found a full path that matches.
|
||||
// this will also work for an empty ALT as the loop will be skipped
|
||||
return t
|
||||
}
|
||||
// none of the paths for the current alternative matched
|
||||
// try the next alternative
|
||||
}
|
||||
// none of the alternatives could be matched
|
||||
return undefined
|
||||
}
|
||||
} else if (areAllOneTokenLookahead && !dynamicTokensEnabled) {
|
||||
// optimized (common) case of all the lookaheads paths requiring only
|
||||
// a single token lookahead. These Optimizations cannot work if dynamically defined Tokens are used.
|
||||
let singleTokenAlts = map(alts, (currAlt) => {
|
||||
return flatten(currAlt)
|
||||
})
|
||||
|
||||
let choiceToAlt = reduce(
|
||||
singleTokenAlts,
|
||||
(result, currAlt, idx) => {
|
||||
forEach(currAlt, (currTokType) => {
|
||||
if (!has(result, currTokType.tokenTypeIdx)) {
|
||||
result[currTokType.tokenTypeIdx] = idx
|
||||
}
|
||||
forEach(currTokType.categoryMatches, (currExtendingType) => {
|
||||
if (!has(result, currExtendingType)) {
|
||||
result[currExtendingType] = idx
|
||||
}
|
||||
})
|
||||
})
|
||||
return result
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
/**
|
||||
* @returns {number} - The chosen alternative index
|
||||
*/
|
||||
return function (): number {
|
||||
let nextToken = this.LA(1)
|
||||
return choiceToAlt[nextToken.tokenTypeIdx]
|
||||
}
|
||||
} else {
|
||||
// optimized lookahead without needing to check the predicates at all.
|
||||
// this causes code duplication which is intentional to improve performance.
|
||||
/**
|
||||
* @returns {number} - The chosen alternative index
|
||||
*/
|
||||
return function (): number {
|
||||
for (let t = 0; t < numOfAlts; t++) {
|
||||
let currAlt = alts[t]
|
||||
let currNumOfPaths = currAlt.length
|
||||
nextPath: for (let j = 0; j < currNumOfPaths; j++) {
|
||||
let currPath = currAlt[j]
|
||||
let currPathLength = currPath.length
|
||||
for (let i = 0; i < currPathLength; i++) {
|
||||
let nextToken = this.LA(i + 1)
|
||||
if (tokenMatcher(nextToken, currPath[i]) === false) {
|
||||
// mismatch in current path
|
||||
// try the next pth
|
||||
continue nextPath
|
||||
}
|
||||
}
|
||||
// found a full path that matches.
|
||||
// this will also work for an empty ALT as the loop will be skipped
|
||||
return t
|
||||
}
|
||||
// none of the paths for the current alternative matched
|
||||
// try the next alternative
|
||||
}
|
||||
// none of the alternatives could be matched
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildSingleAlternativeLookaheadFunction(
|
||||
alt: lookAheadSequence,
|
||||
tokenMatcher: TokenMatcher,
|
||||
dynamicTokensEnabled: boolean
|
||||
): () => boolean {
|
||||
let areAllOneTokenLookahead = every(alt, (currPath) => {
|
||||
return currPath.length === 1
|
||||
})
|
||||
|
||||
let numOfPaths = alt.length
|
||||
|
||||
// optimized (common) case of all the lookaheads paths requiring only
|
||||
// a single token lookahead.
|
||||
if (areAllOneTokenLookahead && !dynamicTokensEnabled) {
|
||||
let singleTokensTypes = flatten(alt)
|
||||
|
||||
if (
|
||||
singleTokensTypes.length === 1 &&
|
||||
isEmpty((<any>singleTokensTypes[0]).categoryMatches)
|
||||
) {
|
||||
let expectedTokenType = singleTokensTypes[0]
|
||||
let expectedTokenUniqueKey = (<any>expectedTokenType).tokenTypeIdx
|
||||
|
||||
return function (): boolean {
|
||||
return this.LA(1).tokenTypeIdx === expectedTokenUniqueKey
|
||||
}
|
||||
} else {
|
||||
let choiceToAlt = reduce(
|
||||
singleTokensTypes,
|
||||
(result, currTokType, idx) => {
|
||||
result[currTokType.tokenTypeIdx] = true
|
||||
forEach(currTokType.categoryMatches, (currExtendingType) => {
|
||||
result[currExtendingType] = true
|
||||
})
|
||||
return result
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
return function (): boolean {
|
||||
let nextToken = this.LA(1)
|
||||
return choiceToAlt[nextToken.tokenTypeIdx] === true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return function (): boolean {
|
||||
nextPath: for (let j = 0; j < numOfPaths; j++) {
|
||||
let currPath = alt[j]
|
||||
let currPathLength = currPath.length
|
||||
for (let i = 0; i < currPathLength; i++) {
|
||||
let nextToken = this.LA(i + 1)
|
||||
if (tokenMatcher(nextToken, currPath[i]) === false) {
|
||||
// mismatch in current path
|
||||
// try the next pth
|
||||
continue nextPath
|
||||
}
|
||||
}
|
||||
// found a full path that matches.
|
||||
return true
|
||||
}
|
||||
|
||||
// none of the paths matched
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class RestDefinitionFinderWalker extends RestWalker {
|
||||
private restDef: IProduction[]
|
||||
|
||||
constructor(
|
||||
private topProd: Rule,
|
||||
private targetOccurrence: number,
|
||||
private targetProdType: PROD_TYPE
|
||||
) {
|
||||
super()
|
||||
}
|
||||
|
||||
startWalking(): IProduction[] {
|
||||
this.walk(this.topProd)
|
||||
return this.restDef
|
||||
}
|
||||
|
||||
private checkIsTarget(
|
||||
node: AbstractProduction & IProductionWithOccurrence,
|
||||
expectedProdType: PROD_TYPE,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): boolean {
|
||||
if (
|
||||
node.idx === this.targetOccurrence &&
|
||||
this.targetProdType === expectedProdType
|
||||
) {
|
||||
this.restDef = currRest.concat(prevRest)
|
||||
return true
|
||||
}
|
||||
// performance optimization, do not iterate over the entire Grammar ast after we have found the target
|
||||
return false
|
||||
}
|
||||
|
||||
walkOption(
|
||||
optionProd: Option,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (!this.checkIsTarget(optionProd, PROD_TYPE.OPTION, currRest, prevRest)) {
|
||||
super.walkOption(optionProd, currRest, prevRest)
|
||||
}
|
||||
}
|
||||
|
||||
walkAtLeastOne(
|
||||
atLeastOneProd: RepetitionMandatory,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (
|
||||
!this.checkIsTarget(
|
||||
atLeastOneProd,
|
||||
PROD_TYPE.REPETITION_MANDATORY,
|
||||
currRest,
|
||||
prevRest
|
||||
)
|
||||
) {
|
||||
super.walkOption(atLeastOneProd, currRest, prevRest)
|
||||
}
|
||||
}
|
||||
|
||||
walkAtLeastOneSep(
|
||||
atLeastOneSepProd: RepetitionMandatoryWithSeparator,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (
|
||||
!this.checkIsTarget(
|
||||
atLeastOneSepProd,
|
||||
PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR,
|
||||
currRest,
|
||||
prevRest
|
||||
)
|
||||
) {
|
||||
super.walkOption(atLeastOneSepProd, currRest, prevRest)
|
||||
}
|
||||
}
|
||||
|
||||
walkMany(
|
||||
manyProd: Repetition,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (
|
||||
!this.checkIsTarget(manyProd, PROD_TYPE.REPETITION, currRest, prevRest)
|
||||
) {
|
||||
super.walkOption(manyProd, currRest, prevRest)
|
||||
}
|
||||
}
|
||||
|
||||
walkManySep(
|
||||
manySepProd: RepetitionWithSeparator,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
if (
|
||||
!this.checkIsTarget(
|
||||
manySepProd,
|
||||
PROD_TYPE.REPETITION_WITH_SEPARATOR,
|
||||
currRest,
|
||||
prevRest
|
||||
)
|
||||
) {
|
||||
super.walkOption(manySepProd, currRest, prevRest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the definition of a target production in a top level level rule.
|
||||
*/
|
||||
class InsideDefinitionFinderVisitor extends GAstVisitor {
|
||||
public result: IProduction[] = []
|
||||
|
||||
constructor(
|
||||
private targetOccurrence: number,
|
||||
private targetProdType: PROD_TYPE,
|
||||
private targetRef?: any
|
||||
) {
|
||||
super()
|
||||
}
|
||||
|
||||
private checkIsTarget(
|
||||
node: AbstractProduction & IProductionWithOccurrence,
|
||||
expectedProdName: PROD_TYPE
|
||||
): void {
|
||||
if (
|
||||
node.idx === this.targetOccurrence &&
|
||||
this.targetProdType === expectedProdName &&
|
||||
(this.targetRef === undefined || node === this.targetRef)
|
||||
) {
|
||||
this.result = node.definition
|
||||
}
|
||||
}
|
||||
|
||||
public visitOption(node: Option): void {
|
||||
this.checkIsTarget(node, PROD_TYPE.OPTION)
|
||||
}
|
||||
|
||||
public visitRepetition(node: Repetition): void {
|
||||
this.checkIsTarget(node, PROD_TYPE.REPETITION)
|
||||
}
|
||||
|
||||
public visitRepetitionMandatory(node: RepetitionMandatory): void {
|
||||
this.checkIsTarget(node, PROD_TYPE.REPETITION_MANDATORY)
|
||||
}
|
||||
|
||||
public visitRepetitionMandatoryWithSeparator(
|
||||
node: RepetitionMandatoryWithSeparator
|
||||
): void {
|
||||
this.checkIsTarget(node, PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR)
|
||||
}
|
||||
|
||||
public visitRepetitionWithSeparator(node: RepetitionWithSeparator): void {
|
||||
this.checkIsTarget(node, PROD_TYPE.REPETITION_WITH_SEPARATOR)
|
||||
}
|
||||
|
||||
public visitAlternation(node: Alternation): void {
|
||||
this.checkIsTarget(node, PROD_TYPE.ALTERNATION)
|
||||
}
|
||||
}
|
||||
|
||||
function initializeArrayOfArrays(size): any[][] {
|
||||
let result = new Array(size)
|
||||
for (let i = 0; i < size; i++) {
|
||||
result[i] = []
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* A sort of hash function between a Path in the grammar and a string.
|
||||
* Note that this returns multiple "hashes" to support the scenario of token categories.
|
||||
* - A single path with categories may match multiple **actual** paths.
|
||||
*/
|
||||
function pathToHashKeys(path: TokenType[]): string[] {
|
||||
let keys = [""]
|
||||
for (let i = 0; i < path.length; i++) {
|
||||
const tokType = path[i]
|
||||
let longerKeys = []
|
||||
for (let j = 0; j < keys.length; j++) {
|
||||
const currShorterKey = keys[j]
|
||||
longerKeys.push(currShorterKey + "_" + tokType.tokenTypeIdx)
|
||||
for (let t = 0; t < tokType.categoryMatches.length; t++) {
|
||||
const categoriesKeySuffix = "_" + tokType.categoryMatches[t]
|
||||
longerKeys.push(currShorterKey + categoriesKeySuffix)
|
||||
}
|
||||
}
|
||||
keys = longerKeys
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
/**
|
||||
* Imperative style due to being called from a hot spot
|
||||
*/
|
||||
function isUniquePrefixHash(
|
||||
altKnownPathsKeys: Record<string, boolean>[],
|
||||
searchPathKeys: string[],
|
||||
idx: number
|
||||
): boolean {
|
||||
for (
|
||||
let currAltIdx = 0;
|
||||
currAltIdx < altKnownPathsKeys.length;
|
||||
currAltIdx++
|
||||
) {
|
||||
// We only want to test vs the other alternatives
|
||||
if (currAltIdx === idx) {
|
||||
continue
|
||||
}
|
||||
const otherAltKnownPathsKeys = altKnownPathsKeys[currAltIdx]
|
||||
for (let searchIdx = 0; searchIdx < searchPathKeys.length; searchIdx++) {
|
||||
const searchKey = searchPathKeys[searchIdx]
|
||||
if (otherAltKnownPathsKeys[searchKey] === true) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
// None of the SearchPathKeys were found in any of the other alternatives
|
||||
return true
|
||||
}
|
||||
|
||||
export function lookAheadSequenceFromAlternatives(
|
||||
altsDefs: IProduction[],
|
||||
k: number
|
||||
): lookAheadSequence[] {
|
||||
let partialAlts = map(altsDefs, (currAlt) => possiblePathsFrom([currAlt], 1))
|
||||
let finalResult = initializeArrayOfArrays(partialAlts.length)
|
||||
const altsHashes = map(partialAlts, (currAltPaths) => {
|
||||
const dict = {}
|
||||
forEach(currAltPaths, (item) => {
|
||||
const keys = pathToHashKeys(item.partialPath)
|
||||
forEach(keys, (currKey) => {
|
||||
dict[currKey] = true
|
||||
})
|
||||
})
|
||||
return dict
|
||||
})
|
||||
let newData = partialAlts
|
||||
|
||||
// maxLookahead loop
|
||||
for (let pathLength = 1; pathLength <= k; pathLength++) {
|
||||
let currDataset = newData
|
||||
newData = initializeArrayOfArrays(currDataset.length)
|
||||
|
||||
// alternatives loop
|
||||
for (let altIdx = 0; altIdx < currDataset.length; altIdx++) {
|
||||
let currAltPathsAndSuffixes = currDataset[altIdx]
|
||||
// paths in current alternative loop
|
||||
for (
|
||||
let currPathIdx = 0;
|
||||
currPathIdx < currAltPathsAndSuffixes.length;
|
||||
currPathIdx++
|
||||
) {
|
||||
let currPathPrefix = currAltPathsAndSuffixes[currPathIdx].partialPath
|
||||
let suffixDef = currAltPathsAndSuffixes[currPathIdx].suffixDef
|
||||
const prefixKeys = pathToHashKeys(currPathPrefix)
|
||||
let isUnique = isUniquePrefixHash(altsHashes, prefixKeys, altIdx)
|
||||
// End of the line for this path.
|
||||
if (isUnique || isEmpty(suffixDef) || currPathPrefix.length === k) {
|
||||
let currAltResult = finalResult[altIdx]
|
||||
// TODO: Can we implement a containsPath using Maps/Dictionaries?
|
||||
if (containsPath(currAltResult, currPathPrefix) === false) {
|
||||
currAltResult.push(currPathPrefix)
|
||||
// Update all new keys for the current path.
|
||||
for (let j = 0; j < prefixKeys.length; j++) {
|
||||
const currKey = prefixKeys[j]
|
||||
altsHashes[altIdx][currKey] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
// Expand longer paths
|
||||
else {
|
||||
let newPartialPathsAndSuffixes = possiblePathsFrom(
|
||||
suffixDef,
|
||||
pathLength + 1,
|
||||
currPathPrefix
|
||||
)
|
||||
newData[altIdx] = newData[altIdx].concat(newPartialPathsAndSuffixes)
|
||||
|
||||
// Update keys for new known paths
|
||||
forEach(newPartialPathsAndSuffixes, (item) => {
|
||||
const prefixKeys = pathToHashKeys(item.partialPath)
|
||||
forEach(prefixKeys, (key) => {
|
||||
altsHashes[altIdx][key] = true
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return finalResult
|
||||
}
|
||||
|
||||
export function getLookaheadPathsForOr(
|
||||
occurrence: number,
|
||||
ruleGrammar: Rule,
|
||||
k: number,
|
||||
orProd?: Alternation
|
||||
): lookAheadSequence[] {
|
||||
const visitor = new InsideDefinitionFinderVisitor(
|
||||
occurrence,
|
||||
PROD_TYPE.ALTERNATION,
|
||||
orProd
|
||||
)
|
||||
ruleGrammar.accept(visitor)
|
||||
return lookAheadSequenceFromAlternatives(visitor.result, k)
|
||||
}
|
||||
|
||||
export function getLookaheadPathsForOptionalProd(
|
||||
occurrence: number,
|
||||
ruleGrammar: Rule,
|
||||
prodType: PROD_TYPE,
|
||||
k: number
|
||||
): lookAheadSequence[] {
|
||||
let insideDefVisitor = new InsideDefinitionFinderVisitor(occurrence, prodType)
|
||||
ruleGrammar.accept(insideDefVisitor)
|
||||
let insideDef = insideDefVisitor.result
|
||||
|
||||
let afterDefWalker = new RestDefinitionFinderWalker(
|
||||
ruleGrammar,
|
||||
occurrence,
|
||||
prodType
|
||||
)
|
||||
let afterDef = afterDefWalker.startWalking()
|
||||
|
||||
let insideFlat = new AlternativeGAST({ definition: insideDef })
|
||||
let afterFlat = new AlternativeGAST({ definition: afterDef })
|
||||
|
||||
return lookAheadSequenceFromAlternatives([insideFlat, afterFlat], k)
|
||||
}
|
||||
|
||||
export function containsPath(
|
||||
alternative: Alternative,
|
||||
searchPath: TokenType[]
|
||||
): boolean {
|
||||
compareOtherPath: for (let i = 0; i < alternative.length; i++) {
|
||||
const otherPath = alternative[i]
|
||||
if (otherPath.length !== searchPath.length) {
|
||||
continue
|
||||
}
|
||||
for (let j = 0; j < otherPath.length; j++) {
|
||||
const searchTok = searchPath[j]
|
||||
const otherTok = otherPath[j]
|
||||
|
||||
const matchingTokens =
|
||||
searchTok === otherTok ||
|
||||
otherTok.categoryMatchesMap[searchTok.tokenTypeIdx] !== undefined
|
||||
if (matchingTokens === false) {
|
||||
continue compareOtherPath
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
export function isStrictPrefixOfPath(
|
||||
prefix: TokenType[],
|
||||
other: TokenType[]
|
||||
): boolean {
|
||||
return (
|
||||
prefix.length < other.length &&
|
||||
every(prefix, (tokType, idx) => {
|
||||
const otherTokType = other[idx]
|
||||
return (
|
||||
tokType === otherTokType ||
|
||||
otherTokType.categoryMatchesMap[tokType.tokenTypeIdx]
|
||||
)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
export function areTokenCategoriesNotUsed(
|
||||
lookAheadPaths: lookAheadSequence[]
|
||||
): boolean {
|
||||
return every(lookAheadPaths, (singleAltPaths) =>
|
||||
every(singleAltPaths, (singlePath) =>
|
||||
every(singlePath, (token) => isEmpty(token.categoryMatches))
|
||||
)
|
||||
)
|
||||
}
|
||||
58
node_modules/chevrotain/src/parse/grammar/resolver.ts
generated
vendored
Normal file
58
node_modules/chevrotain/src/parse/grammar/resolver.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import {
|
||||
IParserUnresolvedRefDefinitionError,
|
||||
ParserDefinitionErrorType
|
||||
} from "../parser/parser"
|
||||
import { forEach, values } from "../../utils/utils"
|
||||
import { NonTerminal, Rule } from "./gast/gast_public"
|
||||
import { GAstVisitor } from "./gast/gast_visitor_public"
|
||||
import {
|
||||
IGrammarResolverErrorMessageProvider,
|
||||
IParserDefinitionError
|
||||
} from "../../../api"
|
||||
|
||||
export function resolveGrammar(
|
||||
topLevels: Record<string, Rule>,
|
||||
errMsgProvider: IGrammarResolverErrorMessageProvider
|
||||
): IParserDefinitionError[] {
|
||||
let refResolver = new GastRefResolverVisitor(topLevels, errMsgProvider)
|
||||
refResolver.resolveRefs()
|
||||
return refResolver.errors
|
||||
}
|
||||
|
||||
export class GastRefResolverVisitor extends GAstVisitor {
|
||||
public errors: IParserUnresolvedRefDefinitionError[] = []
|
||||
private currTopLevel: Rule
|
||||
|
||||
constructor(
|
||||
private nameToTopRule: Record<string, Rule>,
|
||||
private errMsgProvider: IGrammarResolverErrorMessageProvider
|
||||
) {
|
||||
super()
|
||||
}
|
||||
|
||||
public resolveRefs(): void {
|
||||
forEach(values(this.nameToTopRule), (prod) => {
|
||||
this.currTopLevel = prod
|
||||
prod.accept(this)
|
||||
})
|
||||
}
|
||||
|
||||
public visitNonTerminal(node: NonTerminal): void {
|
||||
let ref = this.nameToTopRule[node.nonTerminalName]
|
||||
|
||||
if (!ref) {
|
||||
let msg = this.errMsgProvider.buildRuleNotFoundError(
|
||||
this.currTopLevel,
|
||||
node
|
||||
)
|
||||
this.errors.push({
|
||||
message: msg,
|
||||
type: ParserDefinitionErrorType.UNRESOLVED_SUBRULE_REF,
|
||||
ruleName: this.currTopLevel.name,
|
||||
unresolvedRefName: node.nonTerminalName
|
||||
})
|
||||
} else {
|
||||
node.referencedRule = ref
|
||||
}
|
||||
}
|
||||
}
|
||||
163
node_modules/chevrotain/src/parse/grammar/rest.ts
generated
vendored
Normal file
163
node_modules/chevrotain/src/parse/grammar/rest.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
import { drop, forEach } from "../../utils/utils"
|
||||
import {
|
||||
AbstractProduction,
|
||||
Alternation,
|
||||
Alternative,
|
||||
NonTerminal,
|
||||
Option,
|
||||
Repetition,
|
||||
RepetitionMandatory,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
RepetitionWithSeparator,
|
||||
Terminal
|
||||
} from "./gast/gast_public"
|
||||
import { IProduction } from "../../../api"
|
||||
|
||||
/**
|
||||
* A Grammar Walker that computes the "remaining" grammar "after" a productions in the grammar.
|
||||
*/
|
||||
export abstract class RestWalker {
|
||||
walk(prod: AbstractProduction, prevRest: any[] = []): void {
|
||||
forEach(prod.definition, (subProd: IProduction, index) => {
|
||||
let currRest = drop(prod.definition, index + 1)
|
||||
/* istanbul ignore else */
|
||||
if (subProd instanceof NonTerminal) {
|
||||
this.walkProdRef(subProd, currRest, prevRest)
|
||||
} else if (subProd instanceof Terminal) {
|
||||
this.walkTerminal(subProd, currRest, prevRest)
|
||||
} else if (subProd instanceof Alternative) {
|
||||
this.walkFlat(subProd, currRest, prevRest)
|
||||
} else if (subProd instanceof Option) {
|
||||
this.walkOption(subProd, currRest, prevRest)
|
||||
} else if (subProd instanceof RepetitionMandatory) {
|
||||
this.walkAtLeastOne(subProd, currRest, prevRest)
|
||||
} else if (subProd instanceof RepetitionMandatoryWithSeparator) {
|
||||
this.walkAtLeastOneSep(subProd, currRest, prevRest)
|
||||
} else if (subProd instanceof RepetitionWithSeparator) {
|
||||
this.walkManySep(subProd, currRest, prevRest)
|
||||
} else if (subProd instanceof Repetition) {
|
||||
this.walkMany(subProd, currRest, prevRest)
|
||||
} else if (subProd instanceof Alternation) {
|
||||
this.walkOr(subProd, currRest, prevRest)
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
walkTerminal(
|
||||
terminal: Terminal,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {}
|
||||
|
||||
walkProdRef(
|
||||
refProd: NonTerminal,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {}
|
||||
|
||||
walkFlat(
|
||||
flatProd: Alternative,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
// ABCDEF => after the D the rest is EF
|
||||
let fullOrRest = currRest.concat(prevRest)
|
||||
this.walk(flatProd, <any>fullOrRest)
|
||||
}
|
||||
|
||||
walkOption(
|
||||
optionProd: Option,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
// ABC(DE)?F => after the (DE)? the rest is F
|
||||
let fullOrRest = currRest.concat(prevRest)
|
||||
this.walk(optionProd, <any>fullOrRest)
|
||||
}
|
||||
|
||||
walkAtLeastOne(
|
||||
atLeastOneProd: RepetitionMandatory,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
// ABC(DE)+F => after the (DE)+ the rest is (DE)?F
|
||||
let fullAtLeastOneRest: IProduction[] = [
|
||||
new Option({ definition: atLeastOneProd.definition })
|
||||
].concat(<any>currRest, <any>prevRest)
|
||||
this.walk(atLeastOneProd, fullAtLeastOneRest)
|
||||
}
|
||||
|
||||
walkAtLeastOneSep(
|
||||
atLeastOneSepProd: RepetitionMandatoryWithSeparator,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
// ABC DE(,DE)* F => after the (,DE)+ the rest is (,DE)?F
|
||||
let fullAtLeastOneSepRest = restForRepetitionWithSeparator(
|
||||
atLeastOneSepProd,
|
||||
currRest,
|
||||
prevRest
|
||||
)
|
||||
this.walk(atLeastOneSepProd, fullAtLeastOneSepRest)
|
||||
}
|
||||
|
||||
walkMany(
|
||||
manyProd: Repetition,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
// ABC(DE)*F => after the (DE)* the rest is (DE)?F
|
||||
let fullManyRest: IProduction[] = [
|
||||
new Option({ definition: manyProd.definition })
|
||||
].concat(<any>currRest, <any>prevRest)
|
||||
this.walk(manyProd, fullManyRest)
|
||||
}
|
||||
|
||||
walkManySep(
|
||||
manySepProd: RepetitionWithSeparator,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
// ABC (DE(,DE)*)? F => after the (,DE)* the rest is (,DE)?F
|
||||
let fullManySepRest = restForRepetitionWithSeparator(
|
||||
manySepProd,
|
||||
currRest,
|
||||
prevRest
|
||||
)
|
||||
this.walk(manySepProd, fullManySepRest)
|
||||
}
|
||||
|
||||
walkOr(
|
||||
orProd: Alternation,
|
||||
currRest: IProduction[],
|
||||
prevRest: IProduction[]
|
||||
): void {
|
||||
// ABC(D|E|F)G => when finding the (D|E|F) the rest is G
|
||||
let fullOrRest = currRest.concat(prevRest)
|
||||
// walk all different alternatives
|
||||
forEach(orProd.definition, (alt) => {
|
||||
// wrapping each alternative in a single definition wrapper
|
||||
// to avoid errors in computing the rest of that alternative in the invocation to computeInProdFollows
|
||||
// (otherwise for OR([alt1,alt2]) alt2 will be considered in 'rest' of alt1
|
||||
let prodWrapper = new Alternative({ definition: [alt] })
|
||||
this.walk(prodWrapper, <any>fullOrRest)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function restForRepetitionWithSeparator(repSepProd, currRest, prevRest) {
|
||||
let repSepRest = [
|
||||
new Option({
|
||||
definition: [new Terminal({ terminalType: repSepProd.separator })].concat(
|
||||
repSepProd.definition
|
||||
)
|
||||
})
|
||||
]
|
||||
let fullRepSepRest: IProduction[] = repSepRest.concat(
|
||||
<any>currRest,
|
||||
<any>prevRest
|
||||
)
|
||||
return fullRepSepRest
|
||||
}
|
||||
310
node_modules/chevrotain/src/parse/parser/parser.ts
generated
vendored
Normal file
310
node_modules/chevrotain/src/parse/parser/parser.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,310 @@
|
|||
import {
|
||||
applyMixins,
|
||||
cloneObj,
|
||||
forEach,
|
||||
has,
|
||||
isEmpty,
|
||||
map,
|
||||
toFastProperties,
|
||||
values
|
||||
} from "../../utils/utils"
|
||||
import { computeAllProdsFollows } from "../grammar/follow"
|
||||
import { createTokenInstance, EOF } from "../../scan/tokens_public"
|
||||
import {
|
||||
defaultGrammarValidatorErrorProvider,
|
||||
defaultParserErrorProvider
|
||||
} from "../errors_public"
|
||||
import {
|
||||
resolveGrammar,
|
||||
validateGrammar
|
||||
} from "../grammar/gast/gast_resolver_public"
|
||||
import {
|
||||
CstNode,
|
||||
IParserConfig,
|
||||
IParserDefinitionError,
|
||||
IRecognitionException,
|
||||
IRuleConfig,
|
||||
IToken,
|
||||
TokenType,
|
||||
TokenVocabulary
|
||||
} from "../../../api"
|
||||
import { Recoverable } from "./traits/recoverable"
|
||||
import { LooksAhead } from "./traits/looksahead"
|
||||
import { TreeBuilder } from "./traits/tree_builder"
|
||||
import { LexerAdapter } from "./traits/lexer_adapter"
|
||||
import { RecognizerApi } from "./traits/recognizer_api"
|
||||
import { RecognizerEngine } from "./traits/recognizer_engine"
|
||||
|
||||
import { ErrorHandler } from "./traits/error_handler"
|
||||
import { MixedInParser } from "./traits/parser_traits"
|
||||
import { ContentAssist } from "./traits/context_assist"
|
||||
import { GastRecorder } from "./traits/gast_recorder"
|
||||
import { PerformanceTracer } from "./traits/perf_tracer"
|
||||
|
||||
export const END_OF_FILE = createTokenInstance(
|
||||
EOF,
|
||||
"",
|
||||
NaN,
|
||||
NaN,
|
||||
NaN,
|
||||
NaN,
|
||||
NaN,
|
||||
NaN
|
||||
)
|
||||
Object.freeze(END_OF_FILE)
|
||||
|
||||
export type TokenMatcher = (token: IToken, tokType: TokenType) => boolean
|
||||
|
||||
export type lookAheadSequence = TokenType[][]
|
||||
|
||||
export const DEFAULT_PARSER_CONFIG: IParserConfig = Object.freeze({
|
||||
recoveryEnabled: false,
|
||||
maxLookahead: 3,
|
||||
dynamicTokensEnabled: false,
|
||||
outputCst: true,
|
||||
errorMessageProvider: defaultParserErrorProvider,
|
||||
nodeLocationTracking: "none",
|
||||
traceInitPerf: false,
|
||||
skipValidations: false
|
||||
})
|
||||
|
||||
export const DEFAULT_RULE_CONFIG: IRuleConfig<any> = Object.freeze({
|
||||
recoveryValueFunc: () => undefined,
|
||||
resyncEnabled: true
|
||||
})
|
||||
|
||||
export enum ParserDefinitionErrorType {
|
||||
INVALID_RULE_NAME = 0,
|
||||
DUPLICATE_RULE_NAME = 1,
|
||||
INVALID_RULE_OVERRIDE = 2,
|
||||
DUPLICATE_PRODUCTIONS = 3,
|
||||
UNRESOLVED_SUBRULE_REF = 4,
|
||||
LEFT_RECURSION = 5,
|
||||
NONE_LAST_EMPTY_ALT = 6,
|
||||
AMBIGUOUS_ALTS = 7,
|
||||
CONFLICT_TOKENS_RULES_NAMESPACE = 8,
|
||||
INVALID_TOKEN_NAME = 9,
|
||||
NO_NON_EMPTY_LOOKAHEAD = 10,
|
||||
AMBIGUOUS_PREFIX_ALTS = 11,
|
||||
TOO_MANY_ALTS = 12
|
||||
}
|
||||
|
||||
export interface IParserDuplicatesDefinitionError
|
||||
extends IParserDefinitionError {
|
||||
dslName: string
|
||||
occurrence: number
|
||||
parameter?: string
|
||||
}
|
||||
|
||||
export interface IParserEmptyAlternativeDefinitionError
|
||||
extends IParserDefinitionError {
|
||||
occurrence: number
|
||||
alternative: number
|
||||
}
|
||||
|
||||
export interface IParserAmbiguousAlternativesDefinitionError
|
||||
extends IParserDefinitionError {
|
||||
occurrence: number
|
||||
alternatives: number[]
|
||||
}
|
||||
|
||||
export interface IParserUnresolvedRefDefinitionError
|
||||
extends IParserDefinitionError {
|
||||
unresolvedRefName: string
|
||||
}
|
||||
|
||||
export interface IParserState {
|
||||
errors: IRecognitionException[]
|
||||
lexerState: any
|
||||
RULE_STACK: string[]
|
||||
CST_STACK: CstNode[]
|
||||
}
|
||||
|
||||
export type Predicate = () => boolean
|
||||
|
||||
export function EMPTY_ALT<T>(value: T = undefined): () => T {
|
||||
return function () {
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
export class Parser {
|
||||
// Set this flag to true if you don't want the Parser to throw error when problems in it's definition are detected.
|
||||
// (normally during the parser's constructor).
|
||||
// This is a design time flag, it will not affect the runtime error handling of the parser, just design time errors,
|
||||
// for example: duplicate rule names, referencing an unresolved subrule, ect...
|
||||
// This flag should not be enabled during normal usage, it is used in special situations, for example when
|
||||
// needing to display the parser definition errors in some GUI(online playground).
|
||||
static DEFER_DEFINITION_ERRORS_HANDLING: boolean = false
|
||||
|
||||
/**
|
||||
* @deprecated use the **instance** method with the same name instead
|
||||
*/
|
||||
static performSelfAnalysis(parserInstance: Parser): void {
|
||||
throw Error(
|
||||
"The **static** `performSelfAnalysis` method has been deprecated." +
|
||||
"\t\nUse the **instance** method with the same name instead."
|
||||
)
|
||||
}
|
||||
|
||||
public performSelfAnalysis(this: MixedInParser): void {
|
||||
this.TRACE_INIT("performSelfAnalysis", () => {
|
||||
let defErrorsMsgs
|
||||
|
||||
this.selfAnalysisDone = true
|
||||
let className = this.className
|
||||
|
||||
this.TRACE_INIT("toFastProps", () => {
|
||||
// Without this voodoo magic the parser would be x3-x4 slower
|
||||
// It seems it is better to invoke `toFastProperties` **before**
|
||||
// Any manipulations of the `this` object done during the recording phase.
|
||||
toFastProperties(this)
|
||||
})
|
||||
|
||||
this.TRACE_INIT("Grammar Recording", () => {
|
||||
try {
|
||||
this.enableRecording()
|
||||
// Building the GAST
|
||||
forEach(this.definedRulesNames, (currRuleName) => {
|
||||
const wrappedRule = this[currRuleName]
|
||||
const originalGrammarAction = wrappedRule["originalGrammarAction"]
|
||||
let recordedRuleGast = undefined
|
||||
this.TRACE_INIT(`${currRuleName} Rule`, () => {
|
||||
recordedRuleGast = this.topLevelRuleRecord(
|
||||
currRuleName,
|
||||
originalGrammarAction
|
||||
)
|
||||
})
|
||||
this.gastProductionsCache[currRuleName] = recordedRuleGast
|
||||
})
|
||||
} finally {
|
||||
this.disableRecording()
|
||||
}
|
||||
})
|
||||
|
||||
let resolverErrors = []
|
||||
this.TRACE_INIT("Grammar Resolving", () => {
|
||||
resolverErrors = resolveGrammar({
|
||||
rules: values(this.gastProductionsCache)
|
||||
})
|
||||
this.definitionErrors.push.apply(this.definitionErrors, resolverErrors) // mutability for the win?
|
||||
})
|
||||
|
||||
this.TRACE_INIT("Grammar Validations", () => {
|
||||
// only perform additional grammar validations IFF no resolving errors have occurred.
|
||||
// as unresolved grammar may lead to unhandled runtime exceptions in the follow up validations.
|
||||
if (isEmpty(resolverErrors) && this.skipValidations === false) {
|
||||
let validationErrors = validateGrammar({
|
||||
rules: values(this.gastProductionsCache),
|
||||
maxLookahead: this.maxLookahead,
|
||||
tokenTypes: values(this.tokensMap),
|
||||
errMsgProvider: defaultGrammarValidatorErrorProvider,
|
||||
grammarName: className
|
||||
})
|
||||
|
||||
this.definitionErrors.push.apply(
|
||||
this.definitionErrors,
|
||||
validationErrors
|
||||
) // mutability for the win?
|
||||
}
|
||||
})
|
||||
|
||||
// this analysis may fail if the grammar is not perfectly valid
|
||||
if (isEmpty(this.definitionErrors)) {
|
||||
// The results of these computations are not needed unless error recovery is enabled.
|
||||
if (this.recoveryEnabled) {
|
||||
this.TRACE_INIT("computeAllProdsFollows", () => {
|
||||
let allFollows = computeAllProdsFollows(
|
||||
values(this.gastProductionsCache)
|
||||
)
|
||||
this.resyncFollows = allFollows
|
||||
})
|
||||
}
|
||||
|
||||
this.TRACE_INIT("ComputeLookaheadFunctions", () => {
|
||||
this.preComputeLookaheadFunctions(values(this.gastProductionsCache))
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
!Parser.DEFER_DEFINITION_ERRORS_HANDLING &&
|
||||
!isEmpty(this.definitionErrors)
|
||||
) {
|
||||
defErrorsMsgs = map(
|
||||
this.definitionErrors,
|
||||
(defError) => defError.message
|
||||
)
|
||||
throw new Error(
|
||||
`Parser Definition Errors detected:\n ${defErrorsMsgs.join(
|
||||
"\n-------------------------------\n"
|
||||
)}`
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
definitionErrors: IParserDefinitionError[] = []
|
||||
selfAnalysisDone = false
|
||||
protected skipValidations: boolean
|
||||
|
||||
constructor(tokenVocabulary: TokenVocabulary, config: IParserConfig) {
|
||||
const that: MixedInParser = this as any
|
||||
that.initErrorHandler(config)
|
||||
that.initLexerAdapter()
|
||||
that.initLooksAhead(config)
|
||||
that.initRecognizerEngine(tokenVocabulary, config)
|
||||
that.initRecoverable(config)
|
||||
that.initTreeBuilder(config)
|
||||
that.initContentAssist()
|
||||
that.initGastRecorder(config)
|
||||
that.initPerformanceTracer(config)
|
||||
|
||||
if (has(config, "ignoredIssues")) {
|
||||
throw new Error(
|
||||
"The <ignoredIssues> IParserConfig property has been deprecated.\n\t" +
|
||||
"Please use the <IGNORE_AMBIGUITIES> flag on the relevant DSL method instead.\n\t" +
|
||||
"See: https://sap.github.io/chevrotain/docs/guide/resolving_grammar_errors.html#IGNORING_AMBIGUITIES\n\t" +
|
||||
"For further details."
|
||||
)
|
||||
}
|
||||
|
||||
this.skipValidations = has(config, "skipValidations")
|
||||
? config.skipValidations
|
||||
: DEFAULT_PARSER_CONFIG.skipValidations
|
||||
}
|
||||
}
|
||||
|
||||
applyMixins(Parser, [
|
||||
Recoverable,
|
||||
LooksAhead,
|
||||
TreeBuilder,
|
||||
LexerAdapter,
|
||||
RecognizerEngine,
|
||||
RecognizerApi,
|
||||
ErrorHandler,
|
||||
ContentAssist,
|
||||
GastRecorder,
|
||||
PerformanceTracer
|
||||
])
|
||||
|
||||
export class CstParser extends Parser {
|
||||
constructor(
|
||||
tokenVocabulary: TokenVocabulary,
|
||||
config: IParserConfig = DEFAULT_PARSER_CONFIG
|
||||
) {
|
||||
const configClone = cloneObj(config)
|
||||
configClone.outputCst = true
|
||||
super(tokenVocabulary, configClone)
|
||||
}
|
||||
}
|
||||
|
||||
export class EmbeddedActionsParser extends Parser {
|
||||
constructor(
|
||||
tokenVocabulary: TokenVocabulary,
|
||||
config: IParserConfig = DEFAULT_PARSER_CONFIG
|
||||
) {
|
||||
const configClone = cloneObj(config)
|
||||
configClone.outputCst = false
|
||||
super(tokenVocabulary, configClone)
|
||||
}
|
||||
}
|
||||
71
node_modules/chevrotain/src/parse/parser/traits/README.md
generated
vendored
Normal file
71
node_modules/chevrotain/src/parse/parser/traits/README.md
generated
vendored
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
### Parser Traits (mixins)
|
||||
|
||||
The Chevrotain Parser class is implemented as multiple classes mixed-in (combined) together
|
||||
to provide the required functionality.
|
||||
|
||||
A mix-in approach has been chosen to:
|
||||
|
||||
1. Split up the large (~3,000 LOC) Parser Class into smaller files.
|
||||
- Similar to C# [Partial Classes](https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/classes-and-structs/partial-classes-and-methods)
|
||||
2. Avoid performance regressions of the common "classic" Object oriented pattern of composition & delegation.
|
||||
- For example, consider: `this.LA()` vs `this.traitX.LA()`
|
||||
- Past attempts at extracting an API for Lexer Adaptors using composition have [shown](https://github.com/SAP/chevrotain/issues/528#issuecomment-313863665)
|
||||
significant performance regressions.
|
||||
|
||||
### Background
|
||||
|
||||
The mixin pattern used here is derived from the mixins pattern found in the [TypeScript Handbook](https://www.typescriptlang.org/docs/handbook/mixins.html).
|
||||
There are a few Issues With The TypeScript Handbook Pattern:
|
||||
|
||||
- Duplicate declarations of the instance methods.
|
||||
- Missing scenario of interaction between different mixins included by the same class.
|
||||
- This scenario is more similar to partial classes.
|
||||
- Instance fields initialization would not be executed.
|
||||
- As instance fields are copied to the constructor by the TypeScript compiler
|
||||
and only one constructor would get invoked.
|
||||
|
||||
Therefor a slightly modified pattern has been used.
|
||||
|
||||
### The "Upgraded" Pattern
|
||||
|
||||
The building blocks are as follows:
|
||||
|
||||
- **Define the full combined type**
|
||||
- by using Intersection Types to define the complete Type (after all the mixins).
|
||||
- [Source Snippet](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/traits/parser_traits.ts#L20-L28)
|
||||
- **Make every method aware of its full execution context**
|
||||
- By specifying the type of "this" context in methods as the "full combined type" Type
|
||||
to allow "interaction" between different mixed-ins of the same class.
|
||||
- [e.g calling a method from another trait](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/traits/recognizer_api.ts#L35-L41)
|
||||
- **Define state initialization for each trait/mixin**?:
|
||||
- By defining init methods for each trait/mixin which would be called during the combined type initialization.
|
||||
- [A single trait's initializer definition](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/traits/lexer_adapter.ts#L17-L21)
|
||||
- **Invoke the state initialization in the combined class**
|
||||
- By using a type assertion in the main class constructor to enable calling these init methods.
|
||||
- [Invoking the init methods](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/parser.ts#L225-L232)
|
||||
- **Ensuring alignment with the public API**
|
||||
- By using a dummy assignment statement to leverge the TypeScript compiler to ensure the internal Parser implementation matches
|
||||
the exposed public API of Chevrotain.
|
||||
- [Dummy assignment](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/api.ts#L193-L197)
|
||||
- **Enriching the combined class's prototype**
|
||||
- [Invoking ApplyMixings](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/parse/parser/parser.ts#L240-L249)
|
||||
- [Upgraded ApplyMixings with setter/getter handling](https://github.com/SAP/chevrotain/blob/8a1c3594165849c179f6c9fd67078ba96af0ea34/src/utils/utils.ts#L433-L460)
|
||||
|
||||
* Pros
|
||||
|
||||
- Avoid duplication.
|
||||
- Allows splitting up large classes to multiple files if/when class composition is not appropriate.
|
||||
- a.k.a ["partial classes"](https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/classes-and-structs/partial-classes-and-methods).
|
||||
|
||||
* Cons
|
||||
- Breaks the semantics of TypeScript a bit, What the compiler knows about "SmartObject"
|
||||
is no longer the "full story".
|
||||
|
||||
### References
|
||||
|
||||
- [Wikipedia Article on Mixins](https://en.wikipedia.org/wiki/Mixin)
|
||||
- [Trait Linearization in Scala](https://www.trivento.io/trait-linearization/)
|
||||
|
||||
### To Investigate
|
||||
|
||||
Would TypeScript 2.2 enabled a simpler & clearer pattern by using its support for ["mixin classes"](https://github.com/Microsoft/TypeScript/wiki/What%27s-new-in-TypeScript#support-for-mix-in-classes)?
|
||||
51
node_modules/chevrotain/src/parse/parser/traits/context_assist.ts
generated
vendored
Normal file
51
node_modules/chevrotain/src/parse/parser/traits/context_assist.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import {
|
||||
ISyntacticContentAssistPath,
|
||||
IToken,
|
||||
ITokenGrammarPath,
|
||||
TokenType
|
||||
} from "../../../../api"
|
||||
import {
|
||||
NextAfterTokenWalker,
|
||||
nextPossibleTokensAfter
|
||||
} from "../../grammar/interpreter"
|
||||
import { first, isUndefined } from "../../../utils/utils"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
|
||||
export class ContentAssist {
|
||||
initContentAssist() {}
|
||||
|
||||
public computeContentAssist(
|
||||
this: MixedInParser,
|
||||
startRuleName: string,
|
||||
precedingInput: IToken[]
|
||||
): ISyntacticContentAssistPath[] {
|
||||
let startRuleGast = this.gastProductionsCache[startRuleName]
|
||||
|
||||
if (isUndefined(startRuleGast)) {
|
||||
throw Error(`Rule ->${startRuleName}<- does not exist in this grammar.`)
|
||||
}
|
||||
|
||||
return nextPossibleTokensAfter(
|
||||
[startRuleGast],
|
||||
precedingInput,
|
||||
this.tokenMatcher,
|
||||
this.maxLookahead
|
||||
)
|
||||
}
|
||||
|
||||
// TODO: should this be a member method or a utility? it does not have any state or usage of 'this'...
|
||||
// TODO: should this be more explicitly part of the public API?
|
||||
public getNextPossibleTokenTypes(
|
||||
this: MixedInParser,
|
||||
grammarPath: ITokenGrammarPath
|
||||
): TokenType[] {
|
||||
let topRuleName = first(grammarPath.ruleStack)
|
||||
let gastProductions = this.getGAstProductions()
|
||||
let topProduction = gastProductions[topRuleName]
|
||||
let nextPossibleTokenTypes = new NextAfterTokenWalker(
|
||||
topProduction,
|
||||
grammarPath
|
||||
).startWalking()
|
||||
return nextPossibleTokenTypes
|
||||
}
|
||||
}
|
||||
122
node_modules/chevrotain/src/parse/parser/traits/error_handler.ts
generated
vendored
Normal file
122
node_modules/chevrotain/src/parse/parser/traits/error_handler.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import {
|
||||
IParserConfig,
|
||||
IParserErrorMessageProvider,
|
||||
IRecognitionException
|
||||
} from "../../../../api"
|
||||
import {
|
||||
EarlyExitException,
|
||||
isRecognitionException,
|
||||
NoViableAltException
|
||||
} from "../../exceptions_public"
|
||||
import { cloneArr, has } from "../../../utils/utils"
|
||||
import {
|
||||
getLookaheadPathsForOptionalProd,
|
||||
getLookaheadPathsForOr,
|
||||
PROD_TYPE
|
||||
} from "../../grammar/lookahead"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser"
|
||||
|
||||
/**
|
||||
* Trait responsible for runtime parsing errors.
|
||||
*/
|
||||
export class ErrorHandler {
|
||||
_errors: IRecognitionException[]
|
||||
errorMessageProvider: IParserErrorMessageProvider
|
||||
|
||||
initErrorHandler(config: IParserConfig) {
|
||||
this._errors = []
|
||||
this.errorMessageProvider = has(config, "errorMessageProvider")
|
||||
? config.errorMessageProvider
|
||||
: DEFAULT_PARSER_CONFIG.errorMessageProvider
|
||||
}
|
||||
|
||||
SAVE_ERROR(
|
||||
this: MixedInParser,
|
||||
error: IRecognitionException
|
||||
): IRecognitionException {
|
||||
if (isRecognitionException(error)) {
|
||||
error.context = {
|
||||
ruleStack: this.getHumanReadableRuleStack(),
|
||||
ruleOccurrenceStack: cloneArr(this.RULE_OCCURRENCE_STACK)
|
||||
}
|
||||
this._errors.push(error)
|
||||
return error
|
||||
} else {
|
||||
throw Error("Trying to save an Error which is not a RecognitionException")
|
||||
}
|
||||
}
|
||||
|
||||
get errors(): IRecognitionException[] {
|
||||
return cloneArr(this._errors)
|
||||
}
|
||||
|
||||
set errors(newErrors: IRecognitionException[]) {
|
||||
this._errors = newErrors
|
||||
}
|
||||
|
||||
// TODO: consider caching the error message computed information
|
||||
raiseEarlyExitException(
|
||||
this: MixedInParser,
|
||||
occurrence: number,
|
||||
prodType: PROD_TYPE,
|
||||
userDefinedErrMsg: string
|
||||
): void {
|
||||
let ruleName = this.getCurrRuleFullName()
|
||||
let ruleGrammar = this.getGAstProductions()[ruleName]
|
||||
let lookAheadPathsPerAlternative = getLookaheadPathsForOptionalProd(
|
||||
occurrence,
|
||||
ruleGrammar,
|
||||
prodType,
|
||||
this.maxLookahead
|
||||
)
|
||||
let insideProdPaths = lookAheadPathsPerAlternative[0]
|
||||
let actualTokens = []
|
||||
for (let i = 1; i <= this.maxLookahead; i++) {
|
||||
actualTokens.push(this.LA(i))
|
||||
}
|
||||
let msg = this.errorMessageProvider.buildEarlyExitMessage({
|
||||
expectedIterationPaths: insideProdPaths,
|
||||
actual: actualTokens,
|
||||
previous: this.LA(0),
|
||||
customUserDescription: userDefinedErrMsg,
|
||||
ruleName: ruleName
|
||||
})
|
||||
|
||||
throw this.SAVE_ERROR(new EarlyExitException(msg, this.LA(1), this.LA(0)))
|
||||
}
|
||||
|
||||
// TODO: consider caching the error message computed information
|
||||
raiseNoAltException(
|
||||
this: MixedInParser,
|
||||
occurrence: number,
|
||||
errMsgTypes: string
|
||||
): void {
|
||||
let ruleName = this.getCurrRuleFullName()
|
||||
let ruleGrammar = this.getGAstProductions()[ruleName]
|
||||
// TODO: getLookaheadPathsForOr can be slow for large enough maxLookahead and certain grammars, consider caching ?
|
||||
let lookAheadPathsPerAlternative = getLookaheadPathsForOr(
|
||||
occurrence,
|
||||
ruleGrammar,
|
||||
this.maxLookahead
|
||||
)
|
||||
|
||||
let actualTokens = []
|
||||
for (let i = 1; i <= this.maxLookahead; i++) {
|
||||
actualTokens.push(this.LA(i))
|
||||
}
|
||||
let previousToken = this.LA(0)
|
||||
|
||||
let errMsg = this.errorMessageProvider.buildNoViableAltMessage({
|
||||
expectedPathsPerAlt: lookAheadPathsPerAlternative,
|
||||
actual: actualTokens,
|
||||
previous: previousToken,
|
||||
customUserDescription: errMsgTypes,
|
||||
ruleName: this.getCurrRuleFullName()
|
||||
})
|
||||
|
||||
throw this.SAVE_ERROR(
|
||||
new NoViableAltException(errMsg, this.LA(1), previousToken)
|
||||
)
|
||||
}
|
||||
}
|
||||
443
node_modules/chevrotain/src/parse/parser/traits/gast_recorder.ts
generated
vendored
Normal file
443
node_modules/chevrotain/src/parse/parser/traits/gast_recorder.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,443 @@
|
|||
import {
|
||||
AtLeastOneSepMethodOpts,
|
||||
ConsumeMethodOpts,
|
||||
CstNode,
|
||||
DSLMethodOpts,
|
||||
DSLMethodOptsWithErr,
|
||||
GrammarAction,
|
||||
IOrAlt,
|
||||
IParserConfig,
|
||||
IProduction,
|
||||
IToken,
|
||||
ManySepMethodOpts,
|
||||
OrMethodOpts,
|
||||
SubruleMethodOpts,
|
||||
TokenType
|
||||
} from "../../../../api"
|
||||
import {
|
||||
forEach,
|
||||
has,
|
||||
isArray,
|
||||
isFunction,
|
||||
peek,
|
||||
some
|
||||
} from "../../../utils/utils"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
import {
|
||||
Alternation,
|
||||
Alternative,
|
||||
NonTerminal,
|
||||
Option,
|
||||
Repetition,
|
||||
RepetitionMandatory,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
RepetitionWithSeparator,
|
||||
Rule,
|
||||
Terminal
|
||||
} from "../../grammar/gast/gast_public"
|
||||
import { Lexer } from "../../../scan/lexer_public"
|
||||
import { augmentTokenTypes, hasShortKeyProperty } from "../../../scan/tokens"
|
||||
import { createToken, createTokenInstance } from "../../../scan/tokens_public"
|
||||
import { END_OF_FILE } from "../parser"
|
||||
import { BITS_FOR_OCCURRENCE_IDX } from "../../grammar/keys"
|
||||
|
||||
type ProdWithDef = IProduction & { definition?: IProduction[] }
|
||||
const RECORDING_NULL_OBJECT = {
|
||||
description: "This Object indicates the Parser is during Recording Phase"
|
||||
}
|
||||
Object.freeze(RECORDING_NULL_OBJECT)
|
||||
|
||||
const HANDLE_SEPARATOR = true
|
||||
const MAX_METHOD_IDX = Math.pow(2, BITS_FOR_OCCURRENCE_IDX) - 1
|
||||
|
||||
const RFT = createToken({ name: "RECORDING_PHASE_TOKEN", pattern: Lexer.NA })
|
||||
augmentTokenTypes([RFT])
|
||||
const RECORDING_PHASE_TOKEN = createTokenInstance(
|
||||
RFT,
|
||||
"This IToken indicates the Parser is in Recording Phase\n\t" +
|
||||
"" +
|
||||
"See: https://sap.github.io/chevrotain/docs/guide/internals.html#grammar-recording for details",
|
||||
// Using "-1" instead of NaN (as in EOF) because an actual number is less likely to
|
||||
// cause errors if the output of LA or CONSUME would be (incorrectly) used during the recording phase.
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
-1
|
||||
)
|
||||
Object.freeze(RECORDING_PHASE_TOKEN)
|
||||
|
||||
const RECORDING_PHASE_CSTNODE: CstNode = {
|
||||
name:
|
||||
"This CSTNode indicates the Parser is in Recording Phase\n\t" +
|
||||
"See: https://sap.github.io/chevrotain/docs/guide/internals.html#grammar-recording for details",
|
||||
children: {}
|
||||
}
|
||||
|
||||
/**
|
||||
* This trait handles the creation of the GAST structure for Chevrotain Grammars
|
||||
*/
|
||||
export class GastRecorder {
|
||||
recordingProdStack: ProdWithDef[]
|
||||
RECORDING_PHASE: boolean
|
||||
|
||||
initGastRecorder(this: MixedInParser, config: IParserConfig): void {
|
||||
this.recordingProdStack = []
|
||||
this.RECORDING_PHASE = false
|
||||
}
|
||||
|
||||
enableRecording(this: MixedInParser): void {
|
||||
this.RECORDING_PHASE = true
|
||||
|
||||
this.TRACE_INIT("Enable Recording", () => {
|
||||
/**
|
||||
* Warning Dark Voodoo Magic upcoming!
|
||||
* We are "replacing" the public parsing DSL methods API
|
||||
* With **new** alternative implementations on the Parser **instance**
|
||||
*
|
||||
* So far this is the only way I've found to avoid performance regressions during parsing time.
|
||||
* - Approx 30% performance regression was measured on Chrome 75 Canary when attempting to replace the "internal"
|
||||
* implementations directly instead.
|
||||
*/
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const idx = i > 0 ? i : ""
|
||||
this[`CONSUME${idx}`] = function (arg1, arg2) {
|
||||
return this.consumeInternalRecord(arg1, i, arg2)
|
||||
}
|
||||
this[`SUBRULE${idx}`] = function (arg1, arg2) {
|
||||
return this.subruleInternalRecord(arg1, i, arg2)
|
||||
}
|
||||
this[`OPTION${idx}`] = function (arg1) {
|
||||
return this.optionInternalRecord(arg1, i)
|
||||
}
|
||||
this[`OR${idx}`] = function (arg1) {
|
||||
return this.orInternalRecord(arg1, i)
|
||||
}
|
||||
this[`MANY${idx}`] = function (arg1) {
|
||||
this.manyInternalRecord(i, arg1)
|
||||
}
|
||||
this[`MANY_SEP${idx}`] = function (arg1) {
|
||||
this.manySepFirstInternalRecord(i, arg1)
|
||||
}
|
||||
this[`AT_LEAST_ONE${idx}`] = function (arg1) {
|
||||
this.atLeastOneInternalRecord(i, arg1)
|
||||
}
|
||||
this[`AT_LEAST_ONE_SEP${idx}`] = function (arg1) {
|
||||
this.atLeastOneSepFirstInternalRecord(i, arg1)
|
||||
}
|
||||
}
|
||||
|
||||
// DSL methods with the idx(suffix) as an argument
|
||||
this[`consume`] = function (idx, arg1, arg2) {
|
||||
return this.consumeInternalRecord(arg1, idx, arg2)
|
||||
}
|
||||
this[`subrule`] = <any>function (idx, arg1, arg2) {
|
||||
return this.subruleInternalRecord(arg1, idx, arg2)
|
||||
}
|
||||
this[`option`] = function (idx, arg1) {
|
||||
return this.optionInternalRecord(arg1, idx)
|
||||
}
|
||||
this[`or`] = function (idx, arg1) {
|
||||
return this.orInternalRecord(arg1, idx)
|
||||
}
|
||||
this[`many`] = function (idx, arg1) {
|
||||
this.manyInternalRecord(idx, arg1)
|
||||
}
|
||||
this[`atLeastOne`] = function (idx, arg1) {
|
||||
this.atLeastOneInternalRecord(idx, arg1)
|
||||
}
|
||||
|
||||
this.ACTION = this.ACTION_RECORD
|
||||
this.BACKTRACK = this.BACKTRACK_RECORD
|
||||
this.LA = this.LA_RECORD
|
||||
})
|
||||
}
|
||||
|
||||
disableRecording(this: MixedInParser) {
|
||||
this.RECORDING_PHASE = false
|
||||
// By deleting these **instance** properties, any future invocation
|
||||
// will be deferred to the original methods on the **prototype** object
|
||||
// This seems to get rid of any incorrect optimizations that V8 may
|
||||
// do during the recording phase.
|
||||
this.TRACE_INIT("Deleting Recording methods", () => {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const idx = i > 0 ? i : ""
|
||||
delete this[`CONSUME${idx}`]
|
||||
delete this[`SUBRULE${idx}`]
|
||||
delete this[`OPTION${idx}`]
|
||||
delete this[`OR${idx}`]
|
||||
delete this[`MANY${idx}`]
|
||||
delete this[`MANY_SEP${idx}`]
|
||||
delete this[`AT_LEAST_ONE${idx}`]
|
||||
delete this[`AT_LEAST_ONE_SEP${idx}`]
|
||||
}
|
||||
|
||||
delete this[`consume`]
|
||||
delete this[`subrule`]
|
||||
delete this[`option`]
|
||||
delete this[`or`]
|
||||
delete this[`many`]
|
||||
delete this[`atLeastOne`]
|
||||
|
||||
delete this.ACTION
|
||||
delete this.BACKTRACK
|
||||
delete this.LA
|
||||
})
|
||||
}
|
||||
|
||||
// TODO: is there any way to use this method to check no
|
||||
// Parser methods are called inside an ACTION?
|
||||
// Maybe try/catch/finally on ACTIONS while disabling the recorders state changes?
|
||||
ACTION_RECORD<T>(this: MixedInParser, impl: () => T): T {
|
||||
// NO-OP during recording
|
||||
return
|
||||
}
|
||||
|
||||
// Executing backtracking logic will break our recording logic assumptions
|
||||
BACKTRACK_RECORD<T>(
|
||||
grammarRule: (...args: any[]) => T,
|
||||
args?: any[]
|
||||
): () => boolean {
|
||||
return () => true
|
||||
}
|
||||
|
||||
// LA is part of the official API and may be used for custom lookahead logic
|
||||
// by end users who may forget to wrap it in ACTION or inside a GATE
|
||||
LA_RECORD(howMuch: number): IToken {
|
||||
// We cannot use the RECORD_PHASE_TOKEN here because someone may depend
|
||||
// On LA return EOF at the end of the input so an infinite loop may occur.
|
||||
return END_OF_FILE
|
||||
}
|
||||
|
||||
topLevelRuleRecord(name: string, def: Function): Rule {
|
||||
try {
|
||||
const newTopLevelRule = new Rule({ definition: [], name: name })
|
||||
newTopLevelRule.name = name
|
||||
this.recordingProdStack.push(newTopLevelRule)
|
||||
def.call(this)
|
||||
this.recordingProdStack.pop()
|
||||
return newTopLevelRule
|
||||
} catch (originalError) {
|
||||
if (originalError.KNOWN_RECORDER_ERROR !== true) {
|
||||
try {
|
||||
originalError.message =
|
||||
originalError.message +
|
||||
'\n\t This error was thrown during the "grammar recording phase" For more info see:\n\t' +
|
||||
"https://sap.github.io/chevrotain/docs/guide/internals.html#grammar-recording"
|
||||
} catch (mutabilityError) {
|
||||
// We may not be able to modify the original error object
|
||||
throw originalError
|
||||
}
|
||||
}
|
||||
throw originalError
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation of parsing DSL
|
||||
optionInternalRecord<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>,
|
||||
occurrence: number
|
||||
): OUT {
|
||||
return recordProd.call(this, Option, actionORMethodDef, occurrence)
|
||||
}
|
||||
|
||||
atLeastOneInternalRecord<OUT>(
|
||||
this: MixedInParser,
|
||||
occurrence: number,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
recordProd.call(this, RepetitionMandatory, actionORMethodDef, occurrence)
|
||||
}
|
||||
|
||||
atLeastOneSepFirstInternalRecord<OUT>(
|
||||
this: MixedInParser,
|
||||
occurrence: number,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
recordProd.call(
|
||||
this,
|
||||
RepetitionMandatoryWithSeparator,
|
||||
options,
|
||||
occurrence,
|
||||
HANDLE_SEPARATOR
|
||||
)
|
||||
}
|
||||
|
||||
manyInternalRecord<OUT>(
|
||||
this: MixedInParser,
|
||||
occurrence: number,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
recordProd.call(this, Repetition, actionORMethodDef, occurrence)
|
||||
}
|
||||
|
||||
manySepFirstInternalRecord<OUT>(
|
||||
this: MixedInParser,
|
||||
occurrence: number,
|
||||
options: ManySepMethodOpts<OUT>
|
||||
): void {
|
||||
recordProd.call(
|
||||
this,
|
||||
RepetitionWithSeparator,
|
||||
options,
|
||||
occurrence,
|
||||
HANDLE_SEPARATOR
|
||||
)
|
||||
}
|
||||
|
||||
orInternalRecord<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>,
|
||||
occurrence: number
|
||||
): T {
|
||||
return recordOrProd.call(this, altsOrOpts, occurrence)
|
||||
}
|
||||
|
||||
subruleInternalRecord<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
occurrence: number,
|
||||
options?: SubruleMethodOpts
|
||||
): T | CstNode {
|
||||
assertMethodIdxIsValid(occurrence)
|
||||
if (!ruleToCall || has(ruleToCall, "ruleName") === false) {
|
||||
const error: any = new Error(
|
||||
`<SUBRULE${getIdxSuffix(occurrence)}> argument is invalid` +
|
||||
` expecting a Parser method reference but got: <${JSON.stringify(
|
||||
ruleToCall
|
||||
)}>` +
|
||||
`\n inside top level rule: <${
|
||||
(<Rule>this.recordingProdStack[0]).name
|
||||
}>`
|
||||
)
|
||||
error.KNOWN_RECORDER_ERROR = true
|
||||
throw error
|
||||
}
|
||||
|
||||
const prevProd: any = peek(this.recordingProdStack)
|
||||
const ruleName = ruleToCall["ruleName"]
|
||||
const newNoneTerminal = new NonTerminal({
|
||||
idx: occurrence,
|
||||
nonTerminalName: ruleName,
|
||||
// The resolving of the `referencedRule` property will be done once all the Rule's GASTs have been created
|
||||
referencedRule: undefined
|
||||
})
|
||||
prevProd.definition.push(newNoneTerminal)
|
||||
|
||||
return this.outputCst ? RECORDING_PHASE_CSTNODE : <any>RECORDING_NULL_OBJECT
|
||||
}
|
||||
|
||||
consumeInternalRecord(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
occurrence: number,
|
||||
options: ConsumeMethodOpts
|
||||
): IToken {
|
||||
assertMethodIdxIsValid(occurrence)
|
||||
if (!hasShortKeyProperty(tokType)) {
|
||||
const error: any = new Error(
|
||||
`<CONSUME${getIdxSuffix(occurrence)}> argument is invalid` +
|
||||
` expecting a TokenType reference but got: <${JSON.stringify(
|
||||
tokType
|
||||
)}>` +
|
||||
`\n inside top level rule: <${
|
||||
(<Rule>this.recordingProdStack[0]).name
|
||||
}>`
|
||||
)
|
||||
error.KNOWN_RECORDER_ERROR = true
|
||||
throw error
|
||||
}
|
||||
const prevProd: any = peek(this.recordingProdStack)
|
||||
const newNoneTerminal = new Terminal({
|
||||
idx: occurrence,
|
||||
terminalType: tokType
|
||||
})
|
||||
prevProd.definition.push(newNoneTerminal)
|
||||
|
||||
return RECORDING_PHASE_TOKEN
|
||||
}
|
||||
}
|
||||
|
||||
function recordProd(
|
||||
prodConstructor: any,
|
||||
mainProdArg: any,
|
||||
occurrence: number,
|
||||
handleSep: boolean = false
|
||||
): any {
|
||||
assertMethodIdxIsValid(occurrence)
|
||||
const prevProd: any = peek(this.recordingProdStack)
|
||||
const grammarAction = isFunction(mainProdArg) ? mainProdArg : mainProdArg.DEF
|
||||
|
||||
const newProd = new prodConstructor({ definition: [], idx: occurrence })
|
||||
if (handleSep) {
|
||||
newProd.separator = mainProdArg.SEP
|
||||
}
|
||||
if (has(mainProdArg, "MAX_LOOKAHEAD")) {
|
||||
newProd.maxLookahead = mainProdArg.MAX_LOOKAHEAD
|
||||
}
|
||||
|
||||
this.recordingProdStack.push(newProd)
|
||||
grammarAction.call(this)
|
||||
prevProd.definition.push(newProd)
|
||||
this.recordingProdStack.pop()
|
||||
|
||||
return RECORDING_NULL_OBJECT
|
||||
}
|
||||
|
||||
function recordOrProd(mainProdArg: any, occurrence: number): any {
|
||||
assertMethodIdxIsValid(occurrence)
|
||||
const prevProd: any = peek(this.recordingProdStack)
|
||||
// Only an array of alternatives
|
||||
const hasOptions = isArray(mainProdArg) === false
|
||||
const alts = hasOptions === false ? mainProdArg : mainProdArg.DEF
|
||||
|
||||
const newOrProd = new Alternation({
|
||||
definition: [],
|
||||
idx: occurrence,
|
||||
ignoreAmbiguities: hasOptions && mainProdArg.IGNORE_AMBIGUITIES === true
|
||||
})
|
||||
if (has(mainProdArg, "MAX_LOOKAHEAD")) {
|
||||
newOrProd.maxLookahead = mainProdArg.MAX_LOOKAHEAD
|
||||
}
|
||||
|
||||
const hasPredicates = some(alts, (currAlt: any) => isFunction(currAlt.GATE))
|
||||
newOrProd.hasPredicates = hasPredicates
|
||||
|
||||
prevProd.definition.push(newOrProd)
|
||||
|
||||
forEach(alts, (currAlt) => {
|
||||
const currAltFlat = new Alternative({ definition: [] })
|
||||
newOrProd.definition.push(currAltFlat)
|
||||
if (has(currAlt, "IGNORE_AMBIGUITIES")) {
|
||||
currAltFlat.ignoreAmbiguities = currAlt.IGNORE_AMBIGUITIES
|
||||
}
|
||||
// **implicit** ignoreAmbiguities due to usage of gate
|
||||
else if (has(currAlt, "GATE")) {
|
||||
currAltFlat.ignoreAmbiguities = true
|
||||
}
|
||||
this.recordingProdStack.push(currAltFlat)
|
||||
currAlt.ALT.call(this)
|
||||
this.recordingProdStack.pop()
|
||||
})
|
||||
return RECORDING_NULL_OBJECT
|
||||
}
|
||||
|
||||
function getIdxSuffix(idx: number): string {
|
||||
return idx === 0 ? "" : `${idx}`
|
||||
}
|
||||
|
||||
function assertMethodIdxIsValid(idx): void {
|
||||
if (idx < 0 || idx > MAX_METHOD_IDX) {
|
||||
const error: any = new Error(
|
||||
// The stack trace will contain all the needed details
|
||||
`Invalid DSL Method idx value: <${idx}>\n\t` +
|
||||
`Idx value must be a none negative value smaller than ${
|
||||
MAX_METHOD_IDX + 1
|
||||
}`
|
||||
)
|
||||
error.KNOWN_RECORDER_ERROR = true
|
||||
throw error
|
||||
}
|
||||
}
|
||||
86
node_modules/chevrotain/src/parse/parser/traits/lexer_adapter.ts
generated
vendored
Normal file
86
node_modules/chevrotain/src/parse/parser/traits/lexer_adapter.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import { END_OF_FILE } from "../parser"
|
||||
import { IToken } from "../../../../api"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
|
||||
/**
|
||||
* Trait responsible abstracting over the interaction with Lexer output (Token vector).
|
||||
*
|
||||
* This could be generalized to support other kinds of lexers, e.g.
|
||||
* - Just in Time Lexing / Lexer-Less parsing.
|
||||
* - Streaming Lexer.
|
||||
*/
|
||||
export class LexerAdapter {
|
||||
tokVector: IToken[]
|
||||
tokVectorLength
|
||||
currIdx: number
|
||||
|
||||
initLexerAdapter() {
|
||||
this.tokVector = []
|
||||
this.tokVectorLength = 0
|
||||
this.currIdx = -1
|
||||
}
|
||||
|
||||
set input(newInput: IToken[]) {
|
||||
// @ts-ignore - `this parameter` not supported in setters/getters
|
||||
// - https://www.typescriptlang.org/docs/handbook/functions.html#this-parameters
|
||||
if (this.selfAnalysisDone !== true) {
|
||||
throw Error(
|
||||
`Missing <performSelfAnalysis> invocation at the end of the Parser's constructor.`
|
||||
)
|
||||
}
|
||||
// @ts-ignore - `this parameter` not supported in setters/getters
|
||||
// - https://www.typescriptlang.org/docs/handbook/functions.html#this-parameters
|
||||
this.reset()
|
||||
this.tokVector = newInput
|
||||
this.tokVectorLength = newInput.length
|
||||
}
|
||||
|
||||
get input(): IToken[] {
|
||||
return this.tokVector
|
||||
}
|
||||
|
||||
// skips a token and returns the next token
|
||||
SKIP_TOKEN(this: MixedInParser): IToken {
|
||||
if (this.currIdx <= this.tokVector.length - 2) {
|
||||
this.consumeToken()
|
||||
return this.LA(1)
|
||||
} else {
|
||||
return END_OF_FILE
|
||||
}
|
||||
}
|
||||
|
||||
// Lexer (accessing Token vector) related methods which can be overridden to implement lazy lexers
|
||||
// or lexers dependent on parser context.
|
||||
LA(this: MixedInParser, howMuch: number): IToken {
|
||||
const soughtIdx = this.currIdx + howMuch
|
||||
if (soughtIdx < 0 || this.tokVectorLength <= soughtIdx) {
|
||||
return END_OF_FILE
|
||||
} else {
|
||||
return this.tokVector[soughtIdx]
|
||||
}
|
||||
}
|
||||
|
||||
consumeToken(this: MixedInParser) {
|
||||
this.currIdx++
|
||||
}
|
||||
|
||||
exportLexerState(this: MixedInParser): number {
|
||||
return this.currIdx
|
||||
}
|
||||
|
||||
importLexerState(this: MixedInParser, newState: number) {
|
||||
this.currIdx = newState
|
||||
}
|
||||
|
||||
resetLexerState(this: MixedInParser): void {
|
||||
this.currIdx = -1
|
||||
}
|
||||
|
||||
moveToTerminatedState(this: MixedInParser): void {
|
||||
this.currIdx = this.tokVector.length - 1
|
||||
}
|
||||
|
||||
getLexerPosition(this: MixedInParser): number {
|
||||
return this.exportLexerState()
|
||||
}
|
||||
}
|
||||
252
node_modules/chevrotain/src/parse/parser/traits/looksahead.ts
generated
vendored
Normal file
252
node_modules/chevrotain/src/parse/parser/traits/looksahead.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
import {
|
||||
buildAlternativesLookAheadFunc,
|
||||
buildLookaheadFuncForOptionalProd,
|
||||
buildLookaheadFuncForOr,
|
||||
buildSingleAlternativeLookaheadFunction,
|
||||
PROD_TYPE
|
||||
} from "../../grammar/lookahead"
|
||||
import { forEach, has, isES2015MapSupported } from "../../../utils/utils"
|
||||
import {
|
||||
DEFAULT_PARSER_CONFIG,
|
||||
lookAheadSequence,
|
||||
TokenMatcher
|
||||
} from "../parser"
|
||||
import { IOrAlt, IParserConfig } from "../../../../api"
|
||||
import {
|
||||
AT_LEAST_ONE_IDX,
|
||||
AT_LEAST_ONE_SEP_IDX,
|
||||
getKeyForAutomaticLookahead,
|
||||
MANY_IDX,
|
||||
MANY_SEP_IDX,
|
||||
OPTION_IDX,
|
||||
OR_IDX
|
||||
} from "../../grammar/keys"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
import { Rule } from "../../grammar/gast/gast_public"
|
||||
import { collectMethods, getProductionDslName } from "../../grammar/gast/gast"
|
||||
|
||||
/**
|
||||
* Trait responsible for the lookahead related utilities and optimizations.
|
||||
*/
|
||||
export class LooksAhead {
|
||||
maxLookahead: number
|
||||
lookAheadFuncsCache: any
|
||||
dynamicTokensEnabled: boolean
|
||||
|
||||
initLooksAhead(config: IParserConfig) {
|
||||
this.dynamicTokensEnabled = has(config, "dynamicTokensEnabled")
|
||||
? config.dynamicTokensEnabled
|
||||
: DEFAULT_PARSER_CONFIG.dynamicTokensEnabled
|
||||
|
||||
this.maxLookahead = has(config, "maxLookahead")
|
||||
? config.maxLookahead
|
||||
: DEFAULT_PARSER_CONFIG.maxLookahead
|
||||
|
||||
/* istanbul ignore next - Using plain array as dictionary will be tested on older node.js versions and IE11 */
|
||||
this.lookAheadFuncsCache = isES2015MapSupported() ? new Map() : []
|
||||
|
||||
// Performance optimization on newer engines that support ES6 Map
|
||||
// For larger Maps this is slightly faster than using a plain object (array in our case).
|
||||
/* istanbul ignore else - The else branch will be tested on older node.js versions and IE11 */
|
||||
if (isES2015MapSupported()) {
|
||||
this.getLaFuncFromCache = this.getLaFuncFromMap
|
||||
this.setLaFuncCache = this.setLaFuncCacheUsingMap
|
||||
} else {
|
||||
this.getLaFuncFromCache = this.getLaFuncFromObj
|
||||
this.setLaFuncCache = this.setLaFuncUsingObj
|
||||
}
|
||||
}
|
||||
|
||||
preComputeLookaheadFunctions(this: MixedInParser, rules: Rule[]): void {
|
||||
forEach(rules, (currRule) => {
|
||||
this.TRACE_INIT(`${currRule.name} Rule Lookahead`, () => {
|
||||
const {
|
||||
alternation,
|
||||
repetition,
|
||||
option,
|
||||
repetitionMandatory,
|
||||
repetitionMandatoryWithSeparator,
|
||||
repetitionWithSeparator
|
||||
} = collectMethods(currRule)
|
||||
|
||||
forEach(alternation, (currProd) => {
|
||||
const prodIdx = currProd.idx === 0 ? "" : currProd.idx
|
||||
this.TRACE_INIT(`${getProductionDslName(currProd)}${prodIdx}`, () => {
|
||||
const laFunc = buildLookaheadFuncForOr(
|
||||
currProd.idx,
|
||||
currRule,
|
||||
currProd.maxLookahead || this.maxLookahead,
|
||||
currProd.hasPredicates,
|
||||
this.dynamicTokensEnabled,
|
||||
this.lookAheadBuilderForAlternatives
|
||||
)
|
||||
|
||||
const key = getKeyForAutomaticLookahead(
|
||||
this.fullRuleNameToShort[currRule.name],
|
||||
OR_IDX,
|
||||
currProd.idx
|
||||
)
|
||||
this.setLaFuncCache(key, laFunc)
|
||||
})
|
||||
})
|
||||
|
||||
forEach(repetition, (currProd) => {
|
||||
this.computeLookaheadFunc(
|
||||
currRule,
|
||||
currProd.idx,
|
||||
MANY_IDX,
|
||||
PROD_TYPE.REPETITION,
|
||||
currProd.maxLookahead,
|
||||
getProductionDslName(currProd)
|
||||
)
|
||||
})
|
||||
|
||||
forEach(option, (currProd) => {
|
||||
this.computeLookaheadFunc(
|
||||
currRule,
|
||||
currProd.idx,
|
||||
OPTION_IDX,
|
||||
PROD_TYPE.OPTION,
|
||||
currProd.maxLookahead,
|
||||
getProductionDslName(currProd)
|
||||
)
|
||||
})
|
||||
|
||||
forEach(repetitionMandatory, (currProd) => {
|
||||
this.computeLookaheadFunc(
|
||||
currRule,
|
||||
currProd.idx,
|
||||
AT_LEAST_ONE_IDX,
|
||||
PROD_TYPE.REPETITION_MANDATORY,
|
||||
currProd.maxLookahead,
|
||||
getProductionDslName(currProd)
|
||||
)
|
||||
})
|
||||
|
||||
forEach(repetitionMandatoryWithSeparator, (currProd) => {
|
||||
this.computeLookaheadFunc(
|
||||
currRule,
|
||||
currProd.idx,
|
||||
AT_LEAST_ONE_SEP_IDX,
|
||||
PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR,
|
||||
currProd.maxLookahead,
|
||||
getProductionDslName(currProd)
|
||||
)
|
||||
})
|
||||
|
||||
forEach(repetitionWithSeparator, (currProd) => {
|
||||
this.computeLookaheadFunc(
|
||||
currRule,
|
||||
currProd.idx,
|
||||
MANY_SEP_IDX,
|
||||
PROD_TYPE.REPETITION_WITH_SEPARATOR,
|
||||
currProd.maxLookahead,
|
||||
getProductionDslName(currProd)
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
computeLookaheadFunc(
|
||||
this: MixedInParser,
|
||||
rule: Rule,
|
||||
prodOccurrence: number,
|
||||
prodKey: number,
|
||||
prodType: PROD_TYPE,
|
||||
prodMaxLookahead: number,
|
||||
dslMethodName: string
|
||||
): void {
|
||||
this.TRACE_INIT(
|
||||
`${dslMethodName}${prodOccurrence === 0 ? "" : prodOccurrence}`,
|
||||
() => {
|
||||
const laFunc = buildLookaheadFuncForOptionalProd(
|
||||
prodOccurrence,
|
||||
rule,
|
||||
prodMaxLookahead || this.maxLookahead,
|
||||
this.dynamicTokensEnabled,
|
||||
prodType,
|
||||
this.lookAheadBuilderForOptional
|
||||
)
|
||||
const key = getKeyForAutomaticLookahead(
|
||||
this.fullRuleNameToShort[rule.name],
|
||||
prodKey,
|
||||
prodOccurrence
|
||||
)
|
||||
this.setLaFuncCache(key, laFunc)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
lookAheadBuilderForOptional(
|
||||
this: MixedInParser,
|
||||
alt: lookAheadSequence,
|
||||
tokenMatcher: TokenMatcher,
|
||||
dynamicTokensEnabled: boolean
|
||||
): () => boolean {
|
||||
return buildSingleAlternativeLookaheadFunction(
|
||||
alt,
|
||||
tokenMatcher,
|
||||
dynamicTokensEnabled
|
||||
)
|
||||
}
|
||||
|
||||
lookAheadBuilderForAlternatives(
|
||||
this: MixedInParser,
|
||||
alts: lookAheadSequence[],
|
||||
hasPredicates: boolean,
|
||||
tokenMatcher: TokenMatcher,
|
||||
dynamicTokensEnabled: boolean
|
||||
): (orAlts?: IOrAlt<any>[]) => number | undefined {
|
||||
return buildAlternativesLookAheadFunc(
|
||||
alts,
|
||||
hasPredicates,
|
||||
tokenMatcher,
|
||||
dynamicTokensEnabled
|
||||
)
|
||||
}
|
||||
|
||||
// this actually returns a number, but it is always used as a string (object prop key)
|
||||
getKeyForAutomaticLookahead(
|
||||
this: MixedInParser,
|
||||
dslMethodIdx: number,
|
||||
occurrence: number
|
||||
): number {
|
||||
let currRuleShortName: any = this.getLastExplicitRuleShortName()
|
||||
return getKeyForAutomaticLookahead(
|
||||
currRuleShortName,
|
||||
dslMethodIdx,
|
||||
occurrence
|
||||
)
|
||||
}
|
||||
|
||||
/* istanbul ignore next */
|
||||
getLaFuncFromCache(this: MixedInParser, key: number): Function {
|
||||
return undefined
|
||||
}
|
||||
|
||||
getLaFuncFromMap(this: MixedInParser, key: number): Function {
|
||||
return this.lookAheadFuncsCache.get(key)
|
||||
}
|
||||
|
||||
/* istanbul ignore next - Using plain array as dictionary will be tested on older node.js versions and IE11 */
|
||||
getLaFuncFromObj(this: MixedInParser, key: number): Function {
|
||||
return this.lookAheadFuncsCache[key]
|
||||
}
|
||||
|
||||
/* istanbul ignore next */
|
||||
setLaFuncCache(this: MixedInParser, key: number, value: Function): void {}
|
||||
|
||||
setLaFuncCacheUsingMap(
|
||||
this: MixedInParser,
|
||||
key: number,
|
||||
value: Function
|
||||
): void {
|
||||
this.lookAheadFuncsCache.set(key, value)
|
||||
}
|
||||
|
||||
/* istanbul ignore next - Using plain array as dictionary will be tested on older node.js versions and IE11 */
|
||||
setLaFuncUsingObj(this: MixedInParser, key: number, value: Function): void {
|
||||
this.lookAheadFuncsCache[key] = value
|
||||
}
|
||||
}
|
||||
58
node_modules/chevrotain/src/parse/parser/traits/parser_traits.ts
generated
vendored
Normal file
58
node_modules/chevrotain/src/parse/parser/traits/parser_traits.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import { ErrorHandler } from "./error_handler"
|
||||
import { LexerAdapter } from "./lexer_adapter"
|
||||
import { LooksAhead } from "./looksahead"
|
||||
import { RecognizerApi } from "./recognizer_api"
|
||||
import { RecognizerEngine } from "./recognizer_engine"
|
||||
import { Recoverable } from "./recoverable"
|
||||
import { TreeBuilder } from "./tree_builder"
|
||||
import {
|
||||
Parser as ParserConstructorImpel,
|
||||
CstParser as CstParserConstructorImpel,
|
||||
EmbeddedActionsParser as EmbeddedActionsParserConstructorImpel
|
||||
} from "../parser"
|
||||
import * as defs from "../../../../api"
|
||||
import { ContentAssist } from "./context_assist"
|
||||
import { GastRecorder } from "./gast_recorder"
|
||||
import { PerformanceTracer } from "./perf_tracer"
|
||||
|
||||
/**
|
||||
* This Type combines all the Parser traits.
|
||||
* It is used in all traits in the "this type assertion"
|
||||
* - https://github.com/Microsoft/TypeScript/wiki/What%27s-new-in-TypeScript#specifying-the-type-of-this-for-functions
|
||||
* This enables strong Type Checks inside trait methods that invoke methods from other traits.
|
||||
* This pattern is very similar to "self types" in Scala.
|
||||
* - https://docs.scala-lang.org/tour/self-types.html
|
||||
*/
|
||||
export type MixedInParser = ParserConstructorImpel &
|
||||
ErrorHandler &
|
||||
LexerAdapter &
|
||||
LooksAhead &
|
||||
RecognizerApi &
|
||||
RecognizerEngine &
|
||||
Recoverable &
|
||||
TreeBuilder &
|
||||
ContentAssist &
|
||||
GastRecorder &
|
||||
PerformanceTracer
|
||||
|
||||
interface MixedInCstParserConstructor {
|
||||
new (
|
||||
tokenVocabulary: defs.TokenVocabulary,
|
||||
config?: defs.IParserConfig
|
||||
): defs.CstParser
|
||||
}
|
||||
|
||||
export const CstParser: MixedInCstParserConstructor = <any>(
|
||||
CstParserConstructorImpel
|
||||
)
|
||||
|
||||
interface MixedInEmbeddedActionsParserConstructor {
|
||||
new (
|
||||
tokenVocabulary: defs.TokenVocabulary,
|
||||
config?: defs.IParserConfig
|
||||
): defs.EmbeddedActionsParser
|
||||
}
|
||||
|
||||
export const EmbeddedActionsParser: MixedInEmbeddedActionsParserConstructor = <
|
||||
any
|
||||
>EmbeddedActionsParserConstructorImpel
|
||||
53
node_modules/chevrotain/src/parse/parser/traits/perf_tracer.ts
generated
vendored
Normal file
53
node_modules/chevrotain/src/parse/parser/traits/perf_tracer.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { IParserConfig } from "../../../../api"
|
||||
import { has, timer } from "../../../utils/utils"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser"
|
||||
|
||||
/**
|
||||
* Trait responsible for runtime parsing errors.
|
||||
*/
|
||||
export class PerformanceTracer {
|
||||
traceInitPerf: boolean | number
|
||||
traceInitMaxIdent: number
|
||||
traceInitIndent: number
|
||||
|
||||
initPerformanceTracer(config: IParserConfig) {
|
||||
if (has(config, "traceInitPerf")) {
|
||||
const userTraceInitPerf = config.traceInitPerf
|
||||
const traceIsNumber = typeof userTraceInitPerf === "number"
|
||||
this.traceInitMaxIdent = traceIsNumber
|
||||
? <number>userTraceInitPerf
|
||||
: Infinity
|
||||
this.traceInitPerf = traceIsNumber
|
||||
? userTraceInitPerf > 0
|
||||
: userTraceInitPerf
|
||||
} else {
|
||||
this.traceInitMaxIdent = 0
|
||||
this.traceInitPerf = DEFAULT_PARSER_CONFIG.traceInitPerf
|
||||
}
|
||||
|
||||
this.traceInitIndent = -1
|
||||
}
|
||||
|
||||
TRACE_INIT<T>(this: MixedInParser, phaseDesc: string, phaseImpl: () => T): T {
|
||||
// No need to optimize this using NOOP pattern because
|
||||
// It is not called in a hot spot...
|
||||
if (this.traceInitPerf === true) {
|
||||
this.traceInitIndent++
|
||||
const indent = new Array(this.traceInitIndent + 1).join("\t")
|
||||
if (this.traceInitIndent < this.traceInitMaxIdent) {
|
||||
console.log(`${indent}--> <${phaseDesc}>`)
|
||||
}
|
||||
const { time, value } = timer(phaseImpl)
|
||||
/* istanbul ignore next - Difficult to reproduce specific performance behavior (>10ms) in tests */
|
||||
const traceMethod = time > 10 ? console.warn : console.log
|
||||
if (this.traceInitIndent < this.traceInitMaxIdent) {
|
||||
traceMethod(`${indent}<-- <${phaseDesc}> time: ${time}ms`)
|
||||
}
|
||||
this.traceInitIndent--
|
||||
return value
|
||||
} else {
|
||||
return phaseImpl()
|
||||
}
|
||||
}
|
||||
}
|
||||
717
node_modules/chevrotain/src/parse/parser/traits/recognizer_api.ts
generated
vendored
Normal file
717
node_modules/chevrotain/src/parse/parser/traits/recognizer_api.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,717 @@
|
|||
import {
|
||||
AtLeastOneSepMethodOpts,
|
||||
ConsumeMethodOpts,
|
||||
DSLMethodOpts,
|
||||
DSLMethodOptsWithErr,
|
||||
GrammarAction,
|
||||
IOrAlt,
|
||||
IRuleConfig,
|
||||
ISerializedGast,
|
||||
IToken,
|
||||
ManySepMethodOpts,
|
||||
OrMethodOpts,
|
||||
SubruleMethodOpts,
|
||||
TokenType
|
||||
} from "../../../../api"
|
||||
import { contains, values } from "../../../utils/utils"
|
||||
import { isRecognitionException } from "../../exceptions_public"
|
||||
import { DEFAULT_RULE_CONFIG, ParserDefinitionErrorType } from "../parser"
|
||||
import { defaultGrammarValidatorErrorProvider } from "../../errors_public"
|
||||
import { validateRuleIsOverridden } from "../../grammar/checks"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
import { Rule, serializeGrammar } from "../../grammar/gast/gast_public"
|
||||
|
||||
/**
|
||||
* This trait is responsible for implementing the public API
|
||||
* for defining Chevrotain parsers, i.e:
|
||||
* - CONSUME
|
||||
* - RULE
|
||||
* - OPTION
|
||||
* - ...
|
||||
*/
|
||||
export class RecognizerApi {
|
||||
ACTION<T>(this: MixedInParser, impl: () => T): T {
|
||||
return impl.call(this)
|
||||
}
|
||||
|
||||
consume(
|
||||
this: MixedInParser,
|
||||
idx: number,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, idx, options)
|
||||
}
|
||||
|
||||
subrule<T>(
|
||||
this: MixedInParser,
|
||||
idx: number,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, idx, options)
|
||||
}
|
||||
|
||||
option<OUT>(
|
||||
this: MixedInParser,
|
||||
idx: number,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, idx)
|
||||
}
|
||||
|
||||
or(
|
||||
this: MixedInParser,
|
||||
idx: number,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<any>
|
||||
): any {
|
||||
return this.orInternal(altsOrOpts, idx)
|
||||
}
|
||||
|
||||
many(
|
||||
this: MixedInParser,
|
||||
idx: number,
|
||||
actionORMethodDef: GrammarAction<any> | DSLMethodOpts<any>
|
||||
): void {
|
||||
return this.manyInternal(idx, actionORMethodDef)
|
||||
}
|
||||
|
||||
atLeastOne(
|
||||
this: MixedInParser,
|
||||
idx: number,
|
||||
actionORMethodDef: GrammarAction<any> | DSLMethodOptsWithErr<any>
|
||||
): void {
|
||||
return this.atLeastOneInternal(idx, actionORMethodDef)
|
||||
}
|
||||
|
||||
CONSUME(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 0, options)
|
||||
}
|
||||
|
||||
CONSUME1(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 1, options)
|
||||
}
|
||||
|
||||
CONSUME2(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 2, options)
|
||||
}
|
||||
|
||||
CONSUME3(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 3, options)
|
||||
}
|
||||
|
||||
CONSUME4(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 4, options)
|
||||
}
|
||||
|
||||
CONSUME5(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 5, options)
|
||||
}
|
||||
|
||||
CONSUME6(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 6, options)
|
||||
}
|
||||
|
||||
CONSUME7(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 7, options)
|
||||
}
|
||||
|
||||
CONSUME8(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 8, options)
|
||||
}
|
||||
|
||||
CONSUME9(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
options?: ConsumeMethodOpts
|
||||
): IToken {
|
||||
return this.consumeInternal(tokType, 9, options)
|
||||
}
|
||||
|
||||
SUBRULE<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 0, options)
|
||||
}
|
||||
|
||||
SUBRULE1<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 1, options)
|
||||
}
|
||||
|
||||
SUBRULE2<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 2, options)
|
||||
}
|
||||
|
||||
SUBRULE3<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 3, options)
|
||||
}
|
||||
|
||||
SUBRULE4<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 4, options)
|
||||
}
|
||||
|
||||
SUBRULE5<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 5, options)
|
||||
}
|
||||
|
||||
SUBRULE6<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 6, options)
|
||||
}
|
||||
|
||||
SUBRULE7<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 7, options)
|
||||
}
|
||||
|
||||
SUBRULE8<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 8, options)
|
||||
}
|
||||
|
||||
SUBRULE9<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
options?: SubruleMethodOpts
|
||||
): T {
|
||||
return this.subruleInternal(ruleToCall, 9, options)
|
||||
}
|
||||
|
||||
OPTION<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 0)
|
||||
}
|
||||
|
||||
OPTION1<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 1)
|
||||
}
|
||||
|
||||
OPTION2<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 2)
|
||||
}
|
||||
|
||||
OPTION3<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 3)
|
||||
}
|
||||
|
||||
OPTION4<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 4)
|
||||
}
|
||||
|
||||
OPTION5<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 5)
|
||||
}
|
||||
|
||||
OPTION6<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 6)
|
||||
}
|
||||
|
||||
OPTION7<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 7)
|
||||
}
|
||||
|
||||
OPTION8<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 8)
|
||||
}
|
||||
|
||||
OPTION9<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): OUT {
|
||||
return this.optionInternal(actionORMethodDef, 9)
|
||||
}
|
||||
|
||||
OR<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 0)
|
||||
}
|
||||
|
||||
OR1<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 1)
|
||||
}
|
||||
|
||||
OR2<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 2)
|
||||
}
|
||||
|
||||
OR3<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 3)
|
||||
}
|
||||
|
||||
OR4<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 4)
|
||||
}
|
||||
|
||||
OR5<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 5)
|
||||
}
|
||||
|
||||
OR6<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 6)
|
||||
}
|
||||
|
||||
OR7<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 7)
|
||||
}
|
||||
|
||||
OR8<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 8)
|
||||
}
|
||||
|
||||
OR9<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>
|
||||
): T {
|
||||
return this.orInternal(altsOrOpts, 9)
|
||||
}
|
||||
|
||||
MANY<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(0, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY1<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(1, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY2<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(2, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY3<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(3, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY4<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(4, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY5<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(5, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY6<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(6, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY7<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(7, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY8<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(8, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY9<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
this.manyInternal(9, actionORMethodDef)
|
||||
}
|
||||
|
||||
MANY_SEP<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(0, options)
|
||||
}
|
||||
|
||||
MANY_SEP1<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(1, options)
|
||||
}
|
||||
|
||||
MANY_SEP2<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(2, options)
|
||||
}
|
||||
|
||||
MANY_SEP3<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(3, options)
|
||||
}
|
||||
|
||||
MANY_SEP4<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(4, options)
|
||||
}
|
||||
|
||||
MANY_SEP5<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(5, options)
|
||||
}
|
||||
|
||||
MANY_SEP6<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(6, options)
|
||||
}
|
||||
|
||||
MANY_SEP7<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(7, options)
|
||||
}
|
||||
|
||||
MANY_SEP8<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(8, options)
|
||||
}
|
||||
|
||||
MANY_SEP9<OUT>(this: MixedInParser, options: ManySepMethodOpts<OUT>): void {
|
||||
this.manySepFirstInternal(9, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
this.atLeastOneInternal(0, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE1<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
return this.atLeastOneInternal(1, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE2<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
this.atLeastOneInternal(2, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE3<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
this.atLeastOneInternal(3, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE4<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
this.atLeastOneInternal(4, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE5<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
this.atLeastOneInternal(5, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE6<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
this.atLeastOneInternal(6, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE7<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
this.atLeastOneInternal(7, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE8<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
this.atLeastOneInternal(8, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE9<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
this.atLeastOneInternal(9, actionORMethodDef)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(0, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP1<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(1, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP2<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(2, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP3<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(3, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP4<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(4, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP5<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(5, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP6<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(6, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP7<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(7, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP8<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(8, options)
|
||||
}
|
||||
|
||||
AT_LEAST_ONE_SEP9<OUT>(
|
||||
this: MixedInParser,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
this.atLeastOneSepFirstInternal(9, options)
|
||||
}
|
||||
|
||||
RULE<T>(
|
||||
this: MixedInParser,
|
||||
name: string,
|
||||
implementation: (...implArgs: any[]) => T,
|
||||
config: IRuleConfig<T> = DEFAULT_RULE_CONFIG
|
||||
): (idxInCallingRule?: number, ...args: any[]) => T | any {
|
||||
if (contains(this.definedRulesNames, name)) {
|
||||
const errMsg = defaultGrammarValidatorErrorProvider.buildDuplicateRuleNameError(
|
||||
{
|
||||
topLevelRule: name,
|
||||
grammarName: this.className
|
||||
}
|
||||
)
|
||||
|
||||
const error = {
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.DUPLICATE_RULE_NAME,
|
||||
ruleName: name
|
||||
}
|
||||
this.definitionErrors.push(error)
|
||||
}
|
||||
|
||||
this.definedRulesNames.push(name)
|
||||
|
||||
let ruleImplementation = this.defineRule(name, implementation, config)
|
||||
this[name] = ruleImplementation
|
||||
return ruleImplementation
|
||||
}
|
||||
|
||||
OVERRIDE_RULE<T>(
|
||||
this: MixedInParser,
|
||||
name: string,
|
||||
impl: (...implArgs: any[]) => T,
|
||||
config: IRuleConfig<T> = DEFAULT_RULE_CONFIG
|
||||
): (idxInCallingRule?: number, ...args: any[]) => T {
|
||||
let ruleErrors = []
|
||||
ruleErrors = ruleErrors.concat(
|
||||
validateRuleIsOverridden(name, this.definedRulesNames, this.className)
|
||||
)
|
||||
this.definitionErrors.push.apply(this.definitionErrors, ruleErrors) // mutability for the win
|
||||
|
||||
let ruleImplementation = this.defineRule(name, impl, config)
|
||||
this[name] = ruleImplementation
|
||||
return ruleImplementation
|
||||
}
|
||||
|
||||
BACKTRACK<T>(
|
||||
this: MixedInParser,
|
||||
grammarRule: (...args: any[]) => T,
|
||||
args?: any[]
|
||||
): () => boolean {
|
||||
return function () {
|
||||
// save org state
|
||||
this.isBackTrackingStack.push(1)
|
||||
const orgState = this.saveRecogState()
|
||||
try {
|
||||
grammarRule.apply(this, args)
|
||||
// if no exception was thrown we have succeed parsing the rule.
|
||||
return true
|
||||
} catch (e) {
|
||||
if (isRecognitionException(e)) {
|
||||
return false
|
||||
} else {
|
||||
throw e
|
||||
}
|
||||
} finally {
|
||||
this.reloadRecogState(orgState)
|
||||
this.isBackTrackingStack.pop()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GAST export APIs
|
||||
public getGAstProductions(this: MixedInParser): Record<string, Rule> {
|
||||
return this.gastProductionsCache
|
||||
}
|
||||
|
||||
public getSerializedGastProductions(this: MixedInParser): ISerializedGast[] {
|
||||
return serializeGrammar(values(this.gastProductionsCache))
|
||||
}
|
||||
}
|
||||
854
node_modules/chevrotain/src/parse/parser/traits/recognizer_engine.ts
generated
vendored
Normal file
854
node_modules/chevrotain/src/parse/parser/traits/recognizer_engine.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,854 @@
|
|||
import {
|
||||
AtLeastOneSepMethodOpts,
|
||||
ConsumeMethodOpts,
|
||||
DSLMethodOpts,
|
||||
DSLMethodOptsWithErr,
|
||||
GrammarAction,
|
||||
IOrAlt,
|
||||
IParserConfig,
|
||||
IRuleConfig,
|
||||
IToken,
|
||||
ManySepMethodOpts,
|
||||
OrMethodOpts,
|
||||
SubruleMethodOpts,
|
||||
TokenType,
|
||||
TokenVocabulary
|
||||
} from "../../../../api"
|
||||
import {
|
||||
cloneArr,
|
||||
cloneObj,
|
||||
every,
|
||||
flatten,
|
||||
has,
|
||||
isArray,
|
||||
isEmpty,
|
||||
isObject,
|
||||
reduce,
|
||||
uniq,
|
||||
values
|
||||
} from "../../../utils/utils"
|
||||
import {
|
||||
AT_LEAST_ONE_IDX,
|
||||
AT_LEAST_ONE_SEP_IDX,
|
||||
BITS_FOR_METHOD_TYPE,
|
||||
BITS_FOR_OCCURRENCE_IDX,
|
||||
MANY_IDX,
|
||||
MANY_SEP_IDX,
|
||||
OPTION_IDX,
|
||||
OR_IDX
|
||||
} from "../../grammar/keys"
|
||||
import {
|
||||
isRecognitionException,
|
||||
MismatchedTokenException,
|
||||
NotAllInputParsedException
|
||||
} from "../../exceptions_public"
|
||||
import { PROD_TYPE } from "../../grammar/lookahead"
|
||||
import {
|
||||
AbstractNextTerminalAfterProductionWalker,
|
||||
NextTerminalAfterAtLeastOneSepWalker,
|
||||
NextTerminalAfterAtLeastOneWalker,
|
||||
NextTerminalAfterManySepWalker,
|
||||
NextTerminalAfterManyWalker
|
||||
} from "../../grammar/interpreter"
|
||||
import { DEFAULT_RULE_CONFIG, IParserState, TokenMatcher } from "../parser"
|
||||
import { IN_RULE_RECOVERY_EXCEPTION } from "./recoverable"
|
||||
import { EOF } from "../../../scan/tokens_public"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
import {
|
||||
augmentTokenTypes,
|
||||
isTokenType,
|
||||
tokenStructuredMatcher,
|
||||
tokenStructuredMatcherNoCategories
|
||||
} from "../../../scan/tokens"
|
||||
import { classNameFromInstance } from "../../../lang/lang_extensions"
|
||||
import { Rule } from "../../grammar/gast/gast_public"
|
||||
|
||||
/**
|
||||
* This trait is responsible for the runtime parsing engine
|
||||
* Used by the official API (recognizer_api.ts)
|
||||
*/
|
||||
export class RecognizerEngine {
|
||||
isBackTrackingStack
|
||||
className: string
|
||||
RULE_STACK: string[]
|
||||
RULE_OCCURRENCE_STACK: number[]
|
||||
definedRulesNames: string[]
|
||||
tokensMap: { [fqn: string]: TokenType }
|
||||
gastProductionsCache: Record<string, Rule>
|
||||
shortRuleNameToFull: Record<string, string>
|
||||
fullRuleNameToShort: Record<string, number>
|
||||
// The shortName Index must be coded "after" the first 8bits to enable building unique lookahead keys
|
||||
ruleShortNameIdx: number
|
||||
tokenMatcher: TokenMatcher
|
||||
|
||||
initRecognizerEngine(
|
||||
tokenVocabulary: TokenVocabulary,
|
||||
config: IParserConfig
|
||||
) {
|
||||
this.className = classNameFromInstance(this)
|
||||
// TODO: would using an ES6 Map or plain object be faster (CST building scenario)
|
||||
this.shortRuleNameToFull = {}
|
||||
this.fullRuleNameToShort = {}
|
||||
this.ruleShortNameIdx = 256
|
||||
this.tokenMatcher = tokenStructuredMatcherNoCategories
|
||||
|
||||
this.definedRulesNames = []
|
||||
this.tokensMap = {}
|
||||
this.isBackTrackingStack = []
|
||||
this.RULE_STACK = []
|
||||
this.RULE_OCCURRENCE_STACK = []
|
||||
this.gastProductionsCache = {}
|
||||
|
||||
if (has(config, "serializedGrammar")) {
|
||||
throw Error(
|
||||
"The Parser's configuration can no longer contain a <serializedGrammar> property.\n" +
|
||||
"\tSee: https://sap.github.io/chevrotain/docs/changes/BREAKING_CHANGES.html#_6-0-0\n" +
|
||||
"\tFor Further details."
|
||||
)
|
||||
}
|
||||
|
||||
if (isArray(tokenVocabulary)) {
|
||||
// This only checks for Token vocabularies provided as arrays.
|
||||
// That is good enough because the main objective is to detect users of pre-V4.0 APIs
|
||||
// rather than all edge cases of empty Token vocabularies.
|
||||
if (isEmpty(tokenVocabulary as any[])) {
|
||||
throw Error(
|
||||
"A Token Vocabulary cannot be empty.\n" +
|
||||
"\tNote that the first argument for the parser constructor\n" +
|
||||
"\tis no longer a Token vector (since v4.0)."
|
||||
)
|
||||
}
|
||||
|
||||
if (typeof (tokenVocabulary as any[])[0].startOffset === "number") {
|
||||
throw Error(
|
||||
"The Parser constructor no longer accepts a token vector as the first argument.\n" +
|
||||
"\tSee: https://sap.github.io/chevrotain/docs/changes/BREAKING_CHANGES.html#_4-0-0\n" +
|
||||
"\tFor Further details."
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (isArray(tokenVocabulary)) {
|
||||
this.tokensMap = <any>reduce(
|
||||
<any>tokenVocabulary,
|
||||
(acc, tokType: TokenType) => {
|
||||
acc[tokType.name] = tokType
|
||||
return acc
|
||||
},
|
||||
{}
|
||||
)
|
||||
} else if (
|
||||
has(tokenVocabulary, "modes") &&
|
||||
every(flatten(values((<any>tokenVocabulary).modes)), isTokenType)
|
||||
) {
|
||||
let allTokenTypes = flatten(values((<any>tokenVocabulary).modes))
|
||||
let uniqueTokens = uniq(allTokenTypes)
|
||||
this.tokensMap = <any>reduce(
|
||||
uniqueTokens,
|
||||
(acc, tokType: TokenType) => {
|
||||
acc[tokType.name] = tokType
|
||||
return acc
|
||||
},
|
||||
{}
|
||||
)
|
||||
} else if (isObject(tokenVocabulary)) {
|
||||
this.tokensMap = cloneObj(tokenVocabulary)
|
||||
} else {
|
||||
throw new Error(
|
||||
"<tokensDictionary> argument must be An Array of Token constructors," +
|
||||
" A dictionary of Token constructors or an IMultiModeLexerDefinition"
|
||||
)
|
||||
}
|
||||
|
||||
// always add EOF to the tokenNames -> constructors map. it is useful to assure all the input has been
|
||||
// parsed with a clear error message ("expecting EOF but found ...")
|
||||
/* tslint:disable */
|
||||
this.tokensMap["EOF"] = EOF
|
||||
|
||||
// TODO: This check may not be accurate for multi mode lexers
|
||||
const noTokenCategoriesUsed = every(
|
||||
values(tokenVocabulary),
|
||||
(tokenConstructor) => isEmpty(tokenConstructor.categoryMatches)
|
||||
)
|
||||
|
||||
this.tokenMatcher = noTokenCategoriesUsed
|
||||
? tokenStructuredMatcherNoCategories
|
||||
: tokenStructuredMatcher
|
||||
|
||||
// Because ES2015+ syntax should be supported for creating Token classes
|
||||
// We cannot assume that the Token classes were created using the "extendToken" utilities
|
||||
// Therefore we must augment the Token classes both on Lexer initialization and on Parser initialization
|
||||
augmentTokenTypes(values(this.tokensMap))
|
||||
}
|
||||
|
||||
defineRule<T>(
|
||||
this: MixedInParser,
|
||||
ruleName: string,
|
||||
impl: (...implArgs: any[]) => T,
|
||||
config: IRuleConfig<T>
|
||||
): (idxInCallingRule?: number, ...args: any[]) => T {
|
||||
if (this.selfAnalysisDone) {
|
||||
throw Error(
|
||||
`Grammar rule <${ruleName}> may not be defined after the 'performSelfAnalysis' method has been called'\n` +
|
||||
`Make sure that all grammar rule definitions are done before 'performSelfAnalysis' is called.`
|
||||
)
|
||||
}
|
||||
let resyncEnabled = has(config, "resyncEnabled")
|
||||
? config.resyncEnabled
|
||||
: DEFAULT_RULE_CONFIG.resyncEnabled
|
||||
let recoveryValueFunc = has(config, "recoveryValueFunc")
|
||||
? config.recoveryValueFunc
|
||||
: DEFAULT_RULE_CONFIG.recoveryValueFunc
|
||||
|
||||
// performance optimization: Use small integers as keys for the longer human readable "full" rule names.
|
||||
// this greatly improves Map access time (as much as 8% for some performance benchmarks).
|
||||
/* tslint:disable */
|
||||
let shortName =
|
||||
this.ruleShortNameIdx << (BITS_FOR_METHOD_TYPE + BITS_FOR_OCCURRENCE_IDX)
|
||||
/* tslint:enable */
|
||||
|
||||
this.ruleShortNameIdx++
|
||||
this.shortRuleNameToFull[shortName] = ruleName
|
||||
this.fullRuleNameToShort[ruleName] = shortName
|
||||
|
||||
function invokeRuleWithTry(args: any[]) {
|
||||
try {
|
||||
if (this.outputCst === true) {
|
||||
impl.apply(this, args)
|
||||
const cst = this.CST_STACK[this.CST_STACK.length - 1]
|
||||
this.cstPostRule(cst)
|
||||
return cst
|
||||
} else {
|
||||
return impl.apply(this, args)
|
||||
}
|
||||
} catch (e) {
|
||||
return this.invokeRuleCatch(e, resyncEnabled, recoveryValueFunc)
|
||||
} finally {
|
||||
this.ruleFinallyStateUpdate()
|
||||
}
|
||||
}
|
||||
|
||||
let wrappedGrammarRule
|
||||
|
||||
wrappedGrammarRule = function (idxInCallingRule: number = 0, args: any[]) {
|
||||
this.ruleInvocationStateUpdate(shortName, ruleName, idxInCallingRule)
|
||||
return invokeRuleWithTry.call(this, args)
|
||||
}
|
||||
|
||||
let ruleNamePropName = "ruleName"
|
||||
wrappedGrammarRule[ruleNamePropName] = ruleName
|
||||
wrappedGrammarRule["originalGrammarAction"] = impl
|
||||
return wrappedGrammarRule
|
||||
}
|
||||
|
||||
invokeRuleCatch(
|
||||
this: MixedInParser,
|
||||
e: Error,
|
||||
resyncEnabledConfig: boolean,
|
||||
recoveryValueFunc: Function
|
||||
): void {
|
||||
let isFirstInvokedRule = this.RULE_STACK.length === 1
|
||||
// note the reSync is always enabled for the first rule invocation, because we must always be able to
|
||||
// reSync with EOF and just output some INVALID ParseTree
|
||||
// during backtracking reSync recovery is disabled, otherwise we can't be certain the backtracking
|
||||
// path is really the most valid one
|
||||
let reSyncEnabled =
|
||||
resyncEnabledConfig && !this.isBackTracking() && this.recoveryEnabled
|
||||
|
||||
if (isRecognitionException(e)) {
|
||||
const recogError: any = e
|
||||
if (reSyncEnabled) {
|
||||
let reSyncTokType = this.findReSyncTokenType()
|
||||
if (this.isInCurrentRuleReSyncSet(reSyncTokType)) {
|
||||
recogError.resyncedTokens = this.reSyncTo(reSyncTokType)
|
||||
if (this.outputCst) {
|
||||
let partialCstResult: any = this.CST_STACK[
|
||||
this.CST_STACK.length - 1
|
||||
]
|
||||
partialCstResult.recoveredNode = true
|
||||
return partialCstResult
|
||||
} else {
|
||||
return recoveryValueFunc()
|
||||
}
|
||||
} else {
|
||||
if (this.outputCst) {
|
||||
const partialCstResult: any = this.CST_STACK[
|
||||
this.CST_STACK.length - 1
|
||||
]
|
||||
partialCstResult.recoveredNode = true
|
||||
recogError.partialCstResult = partialCstResult
|
||||
}
|
||||
// to be handled Further up the call stack
|
||||
throw recogError
|
||||
}
|
||||
} else if (isFirstInvokedRule) {
|
||||
// otherwise a Redundant input error will be created as well and we cannot guarantee that this is indeed the case
|
||||
this.moveToTerminatedState()
|
||||
// the parser should never throw one of its own errors outside its flow.
|
||||
// even if error recovery is disabled
|
||||
return recoveryValueFunc()
|
||||
} else {
|
||||
// to be recovered Further up the call stack
|
||||
throw recogError
|
||||
}
|
||||
} else {
|
||||
// some other Error type which we don't know how to handle (for example a built in JavaScript Error)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// Implementation of parsing DSL
|
||||
optionInternal<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>,
|
||||
occurrence: number
|
||||
): OUT {
|
||||
let key = this.getKeyForAutomaticLookahead(OPTION_IDX, occurrence)
|
||||
return this.optionInternalLogic(actionORMethodDef, occurrence, key)
|
||||
}
|
||||
|
||||
optionInternalLogic<OUT>(
|
||||
this: MixedInParser,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>,
|
||||
occurrence: number,
|
||||
key: number
|
||||
): OUT {
|
||||
let lookAheadFunc = this.getLaFuncFromCache(key)
|
||||
let action
|
||||
let predicate
|
||||
if ((<DSLMethodOpts<OUT>>actionORMethodDef).DEF !== undefined) {
|
||||
action = (<DSLMethodOpts<OUT>>actionORMethodDef).DEF
|
||||
predicate = (<DSLMethodOpts<OUT>>actionORMethodDef).GATE
|
||||
// predicate present
|
||||
if (predicate !== undefined) {
|
||||
let orgLookaheadFunction = lookAheadFunc
|
||||
lookAheadFunc = () => {
|
||||
return predicate.call(this) && orgLookaheadFunction.call(this)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
action = actionORMethodDef
|
||||
}
|
||||
|
||||
if (lookAheadFunc.call(this) === true) {
|
||||
return action.call(this)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
atLeastOneInternal<OUT>(
|
||||
this: MixedInParser,
|
||||
prodOccurrence: number,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>
|
||||
): void {
|
||||
let laKey = this.getKeyForAutomaticLookahead(
|
||||
AT_LEAST_ONE_IDX,
|
||||
prodOccurrence
|
||||
)
|
||||
return this.atLeastOneInternalLogic(
|
||||
prodOccurrence,
|
||||
actionORMethodDef,
|
||||
laKey
|
||||
)
|
||||
}
|
||||
|
||||
atLeastOneInternalLogic<OUT>(
|
||||
this: MixedInParser,
|
||||
prodOccurrence: number,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOptsWithErr<OUT>,
|
||||
key: number
|
||||
): void {
|
||||
let lookAheadFunc = this.getLaFuncFromCache(key)
|
||||
|
||||
let action
|
||||
let predicate
|
||||
if ((<DSLMethodOptsWithErr<OUT>>actionORMethodDef).DEF !== undefined) {
|
||||
action = (<DSLMethodOptsWithErr<OUT>>actionORMethodDef).DEF
|
||||
predicate = (<DSLMethodOptsWithErr<OUT>>actionORMethodDef).GATE
|
||||
// predicate present
|
||||
if (predicate !== undefined) {
|
||||
let orgLookaheadFunction = lookAheadFunc
|
||||
lookAheadFunc = () => {
|
||||
return predicate.call(this) && orgLookaheadFunction.call(this)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
action = actionORMethodDef
|
||||
}
|
||||
|
||||
if ((<Function>lookAheadFunc).call(this) === true) {
|
||||
let notStuck = this.doSingleRepetition(action)
|
||||
while (
|
||||
(<Function>lookAheadFunc).call(this) === true &&
|
||||
notStuck === true
|
||||
) {
|
||||
notStuck = this.doSingleRepetition(action)
|
||||
}
|
||||
} else {
|
||||
throw this.raiseEarlyExitException(
|
||||
prodOccurrence,
|
||||
PROD_TYPE.REPETITION_MANDATORY,
|
||||
(<DSLMethodOptsWithErr<OUT>>actionORMethodDef).ERR_MSG
|
||||
)
|
||||
}
|
||||
|
||||
// note that while it may seem that this can cause an error because by using a recursive call to
|
||||
// AT_LEAST_ONE we change the grammar to AT_LEAST_TWO, AT_LEAST_THREE ... , the possible recursive call
|
||||
// from the tryInRepetitionRecovery(...) will only happen IFF there really are TWO/THREE/.... items.
|
||||
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
this.attemptInRepetitionRecovery(
|
||||
this.atLeastOneInternal,
|
||||
[prodOccurrence, actionORMethodDef],
|
||||
<any>lookAheadFunc,
|
||||
AT_LEAST_ONE_IDX,
|
||||
prodOccurrence,
|
||||
NextTerminalAfterAtLeastOneWalker
|
||||
)
|
||||
}
|
||||
|
||||
atLeastOneSepFirstInternal<OUT>(
|
||||
this: MixedInParser,
|
||||
prodOccurrence: number,
|
||||
options: AtLeastOneSepMethodOpts<OUT>
|
||||
): void {
|
||||
let laKey = this.getKeyForAutomaticLookahead(
|
||||
AT_LEAST_ONE_SEP_IDX,
|
||||
prodOccurrence
|
||||
)
|
||||
this.atLeastOneSepFirstInternalLogic(prodOccurrence, options, laKey)
|
||||
}
|
||||
|
||||
atLeastOneSepFirstInternalLogic<OUT>(
|
||||
this: MixedInParser,
|
||||
prodOccurrence: number,
|
||||
options: AtLeastOneSepMethodOpts<OUT>,
|
||||
key: number
|
||||
): void {
|
||||
let action = options.DEF
|
||||
let separator = options.SEP
|
||||
|
||||
let firstIterationLookaheadFunc = this.getLaFuncFromCache(key)
|
||||
|
||||
// 1st iteration
|
||||
if (firstIterationLookaheadFunc.call(this) === true) {
|
||||
;(<GrammarAction<OUT>>action).call(this)
|
||||
|
||||
// TODO: Optimization can move this function construction into "attemptInRepetitionRecovery"
|
||||
// because it is only needed in error recovery scenarios.
|
||||
let separatorLookAheadFunc = () => {
|
||||
return this.tokenMatcher(this.LA(1), separator)
|
||||
}
|
||||
|
||||
// 2nd..nth iterations
|
||||
while (this.tokenMatcher(this.LA(1), separator) === true) {
|
||||
// note that this CONSUME will never enter recovery because
|
||||
// the separatorLookAheadFunc checks that the separator really does exist.
|
||||
this.CONSUME(separator)
|
||||
// No need for checking infinite loop here due to consuming the separator.
|
||||
;(<GrammarAction<OUT>>action).call(this)
|
||||
}
|
||||
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
this.attemptInRepetitionRecovery(
|
||||
this.repetitionSepSecondInternal,
|
||||
[
|
||||
prodOccurrence,
|
||||
separator,
|
||||
separatorLookAheadFunc,
|
||||
action,
|
||||
NextTerminalAfterAtLeastOneSepWalker
|
||||
],
|
||||
separatorLookAheadFunc,
|
||||
AT_LEAST_ONE_SEP_IDX,
|
||||
prodOccurrence,
|
||||
NextTerminalAfterAtLeastOneSepWalker
|
||||
)
|
||||
} else {
|
||||
throw this.raiseEarlyExitException(
|
||||
prodOccurrence,
|
||||
PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR,
|
||||
options.ERR_MSG
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
manyInternal<OUT>(
|
||||
this: MixedInParser,
|
||||
prodOccurrence: number,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>
|
||||
): void {
|
||||
let laKey = this.getKeyForAutomaticLookahead(MANY_IDX, prodOccurrence)
|
||||
return this.manyInternalLogic(prodOccurrence, actionORMethodDef, laKey)
|
||||
}
|
||||
|
||||
manyInternalLogic<OUT>(
|
||||
this: MixedInParser,
|
||||
prodOccurrence: number,
|
||||
actionORMethodDef: GrammarAction<OUT> | DSLMethodOpts<OUT>,
|
||||
key: number
|
||||
) {
|
||||
let lookaheadFunction = this.getLaFuncFromCache(key)
|
||||
|
||||
let action
|
||||
let predicate
|
||||
if ((<DSLMethodOpts<OUT>>actionORMethodDef).DEF !== undefined) {
|
||||
action = (<DSLMethodOpts<OUT>>actionORMethodDef).DEF
|
||||
predicate = (<DSLMethodOpts<OUT>>actionORMethodDef).GATE
|
||||
// predicate present
|
||||
if (predicate !== undefined) {
|
||||
let orgLookaheadFunction = lookaheadFunction
|
||||
lookaheadFunction = () => {
|
||||
return predicate.call(this) && orgLookaheadFunction.call(this)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
action = actionORMethodDef
|
||||
}
|
||||
|
||||
let notStuck = true
|
||||
while (lookaheadFunction.call(this) === true && notStuck === true) {
|
||||
notStuck = this.doSingleRepetition(action)
|
||||
}
|
||||
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
this.attemptInRepetitionRecovery(
|
||||
this.manyInternal,
|
||||
[prodOccurrence, actionORMethodDef],
|
||||
<any>lookaheadFunction,
|
||||
MANY_IDX,
|
||||
prodOccurrence,
|
||||
NextTerminalAfterManyWalker,
|
||||
// The notStuck parameter is only relevant when "attemptInRepetitionRecovery"
|
||||
// is invoked from manyInternal, in the MANY_SEP case and AT_LEAST_ONE[_SEP]
|
||||
// An infinite loop cannot occur as:
|
||||
// - Either the lookahead is guaranteed to consume something (Single Token Separator)
|
||||
// - AT_LEAST_ONE by definition is guaranteed to consume something (or error out).
|
||||
notStuck
|
||||
)
|
||||
}
|
||||
|
||||
manySepFirstInternal<OUT>(
|
||||
this: MixedInParser,
|
||||
prodOccurrence: number,
|
||||
options: ManySepMethodOpts<OUT>
|
||||
): void {
|
||||
let laKey = this.getKeyForAutomaticLookahead(MANY_SEP_IDX, prodOccurrence)
|
||||
this.manySepFirstInternalLogic(prodOccurrence, options, laKey)
|
||||
}
|
||||
|
||||
manySepFirstInternalLogic<OUT>(
|
||||
this: MixedInParser,
|
||||
prodOccurrence: number,
|
||||
options: ManySepMethodOpts<OUT>,
|
||||
key: number
|
||||
): void {
|
||||
let action = options.DEF
|
||||
let separator = options.SEP
|
||||
let firstIterationLaFunc = this.getLaFuncFromCache(key)
|
||||
|
||||
// 1st iteration
|
||||
if (firstIterationLaFunc.call(this) === true) {
|
||||
action.call(this)
|
||||
|
||||
let separatorLookAheadFunc = () => {
|
||||
return this.tokenMatcher(this.LA(1), separator)
|
||||
}
|
||||
// 2nd..nth iterations
|
||||
while (this.tokenMatcher(this.LA(1), separator) === true) {
|
||||
// note that this CONSUME will never enter recovery because
|
||||
// the separatorLookAheadFunc checks that the separator really does exist.
|
||||
this.CONSUME(separator)
|
||||
// No need for checking infinite loop here due to consuming the separator.
|
||||
action.call(this)
|
||||
}
|
||||
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
this.attemptInRepetitionRecovery(
|
||||
this.repetitionSepSecondInternal,
|
||||
[
|
||||
prodOccurrence,
|
||||
separator,
|
||||
separatorLookAheadFunc,
|
||||
action,
|
||||
NextTerminalAfterManySepWalker
|
||||
],
|
||||
separatorLookAheadFunc,
|
||||
MANY_SEP_IDX,
|
||||
prodOccurrence,
|
||||
NextTerminalAfterManySepWalker
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
repetitionSepSecondInternal<OUT>(
|
||||
this: MixedInParser,
|
||||
prodOccurrence: number,
|
||||
separator: TokenType,
|
||||
separatorLookAheadFunc: () => boolean,
|
||||
action: GrammarAction<OUT>,
|
||||
nextTerminalAfterWalker: typeof AbstractNextTerminalAfterProductionWalker
|
||||
): void {
|
||||
while (separatorLookAheadFunc()) {
|
||||
// note that this CONSUME will never enter recovery because
|
||||
// the separatorLookAheadFunc checks that the separator really does exist.
|
||||
this.CONSUME(separator)
|
||||
action.call(this)
|
||||
}
|
||||
|
||||
// we can only arrive to this function after an error
|
||||
// has occurred (hence the name 'second') so the following
|
||||
// IF will always be entered, its possible to remove it...
|
||||
// however it is kept to avoid confusion and be consistent.
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
/* istanbul ignore else */
|
||||
this.attemptInRepetitionRecovery(
|
||||
this.repetitionSepSecondInternal,
|
||||
[
|
||||
prodOccurrence,
|
||||
separator,
|
||||
separatorLookAheadFunc,
|
||||
action,
|
||||
nextTerminalAfterWalker
|
||||
],
|
||||
separatorLookAheadFunc,
|
||||
AT_LEAST_ONE_SEP_IDX,
|
||||
prodOccurrence,
|
||||
nextTerminalAfterWalker
|
||||
)
|
||||
}
|
||||
|
||||
doSingleRepetition(this: MixedInParser, action: Function): any {
|
||||
const beforeIteration = this.getLexerPosition()
|
||||
action.call(this)
|
||||
const afterIteration = this.getLexerPosition()
|
||||
|
||||
// This boolean will indicate if this repetition progressed
|
||||
// or if we are "stuck" (potential infinite loop in the repetition).
|
||||
return afterIteration > beforeIteration
|
||||
}
|
||||
|
||||
orInternal<T>(
|
||||
this: MixedInParser,
|
||||
altsOrOpts: IOrAlt<any>[] | OrMethodOpts<unknown>,
|
||||
occurrence: number
|
||||
): T {
|
||||
let laKey = this.getKeyForAutomaticLookahead(OR_IDX, occurrence)
|
||||
let alts = isArray(altsOrOpts)
|
||||
? (altsOrOpts as IOrAlt<any>[])
|
||||
: (altsOrOpts as OrMethodOpts<unknown>).DEF
|
||||
|
||||
const laFunc = this.getLaFuncFromCache(laKey)
|
||||
let altIdxToTake = laFunc.call(this, alts)
|
||||
if (altIdxToTake !== undefined) {
|
||||
let chosenAlternative: any = alts[altIdxToTake]
|
||||
return chosenAlternative.ALT.call(this)
|
||||
}
|
||||
this.raiseNoAltException(
|
||||
occurrence,
|
||||
(altsOrOpts as OrMethodOpts<unknown>).ERR_MSG
|
||||
)
|
||||
}
|
||||
|
||||
ruleFinallyStateUpdate(this: MixedInParser): void {
|
||||
this.RULE_STACK.pop()
|
||||
this.RULE_OCCURRENCE_STACK.pop()
|
||||
|
||||
// NOOP when cst is disabled
|
||||
this.cstFinallyStateUpdate()
|
||||
|
||||
if (this.RULE_STACK.length === 0 && this.isAtEndOfInput() === false) {
|
||||
let firstRedundantTok = this.LA(1)
|
||||
let errMsg = this.errorMessageProvider.buildNotAllInputParsedMessage({
|
||||
firstRedundant: firstRedundantTok,
|
||||
ruleName: this.getCurrRuleFullName()
|
||||
})
|
||||
this.SAVE_ERROR(new NotAllInputParsedException(errMsg, firstRedundantTok))
|
||||
}
|
||||
}
|
||||
|
||||
subruleInternal<T>(
|
||||
this: MixedInParser,
|
||||
ruleToCall: (idx: number) => T,
|
||||
idx: number,
|
||||
options?: SubruleMethodOpts
|
||||
) {
|
||||
let ruleResult
|
||||
try {
|
||||
const args = options !== undefined ? options.ARGS : undefined
|
||||
ruleResult = ruleToCall.call(this, idx, args)
|
||||
this.cstPostNonTerminal(
|
||||
ruleResult,
|
||||
options !== undefined && options.LABEL !== undefined
|
||||
? options.LABEL
|
||||
: (<any>ruleToCall).ruleName
|
||||
)
|
||||
return ruleResult
|
||||
} catch (e) {
|
||||
this.subruleInternalError(e, options, (<any>ruleToCall).ruleName)
|
||||
}
|
||||
}
|
||||
|
||||
subruleInternalError(
|
||||
this: MixedInParser,
|
||||
e: any,
|
||||
options: SubruleMethodOpts,
|
||||
ruleName: string
|
||||
): void {
|
||||
if (isRecognitionException(e) && e.partialCstResult !== undefined) {
|
||||
this.cstPostNonTerminal(
|
||||
e.partialCstResult,
|
||||
options !== undefined && options.LABEL !== undefined
|
||||
? options.LABEL
|
||||
: ruleName
|
||||
)
|
||||
|
||||
delete e.partialCstResult
|
||||
}
|
||||
throw e
|
||||
}
|
||||
|
||||
consumeInternal(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
idx: number,
|
||||
options: ConsumeMethodOpts
|
||||
): IToken {
|
||||
let consumedToken
|
||||
try {
|
||||
let nextToken = this.LA(1)
|
||||
if (this.tokenMatcher(nextToken, tokType) === true) {
|
||||
this.consumeToken()
|
||||
consumedToken = nextToken
|
||||
} else {
|
||||
this.consumeInternalError(tokType, nextToken, options)
|
||||
}
|
||||
} catch (eFromConsumption) {
|
||||
consumedToken = this.consumeInternalRecovery(
|
||||
tokType,
|
||||
idx,
|
||||
eFromConsumption
|
||||
)
|
||||
}
|
||||
|
||||
this.cstPostTerminal(
|
||||
options !== undefined && options.LABEL !== undefined
|
||||
? options.LABEL
|
||||
: tokType.name,
|
||||
consumedToken
|
||||
)
|
||||
return consumedToken
|
||||
}
|
||||
|
||||
consumeInternalError(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
nextToken: IToken,
|
||||
options: ConsumeMethodOpts
|
||||
): void {
|
||||
let msg
|
||||
let previousToken = this.LA(0)
|
||||
if (options !== undefined && options.ERR_MSG) {
|
||||
msg = options.ERR_MSG
|
||||
} else {
|
||||
msg = this.errorMessageProvider.buildMismatchTokenMessage({
|
||||
expected: tokType,
|
||||
actual: nextToken,
|
||||
previous: previousToken,
|
||||
ruleName: this.getCurrRuleFullName()
|
||||
})
|
||||
}
|
||||
throw this.SAVE_ERROR(
|
||||
new MismatchedTokenException(msg, nextToken, previousToken)
|
||||
)
|
||||
}
|
||||
|
||||
consumeInternalRecovery(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
idx: number,
|
||||
eFromConsumption: Error
|
||||
): IToken {
|
||||
// no recovery allowed during backtracking, otherwise backtracking may recover invalid syntax and accept it
|
||||
// but the original syntax could have been parsed successfully without any backtracking + recovery
|
||||
if (
|
||||
this.recoveryEnabled &&
|
||||
// TODO: more robust checking of the exception type. Perhaps Typescript extending expressions?
|
||||
eFromConsumption.name === "MismatchedTokenException" &&
|
||||
!this.isBackTracking()
|
||||
) {
|
||||
let follows = this.getFollowsForInRuleRecovery(<any>tokType, idx)
|
||||
try {
|
||||
return this.tryInRuleRecovery(<any>tokType, follows)
|
||||
} catch (eFromInRuleRecovery) {
|
||||
if (eFromInRuleRecovery.name === IN_RULE_RECOVERY_EXCEPTION) {
|
||||
// failed in RuleRecovery.
|
||||
// throw the original error in order to trigger reSync error recovery
|
||||
throw eFromConsumption
|
||||
} else {
|
||||
throw eFromInRuleRecovery
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw eFromConsumption
|
||||
}
|
||||
}
|
||||
|
||||
saveRecogState(this: MixedInParser): IParserState {
|
||||
// errors is a getter which will clone the errors array
|
||||
let savedErrors = this.errors
|
||||
let savedRuleStack = cloneArr(this.RULE_STACK)
|
||||
return {
|
||||
errors: savedErrors,
|
||||
lexerState: this.exportLexerState(),
|
||||
RULE_STACK: savedRuleStack,
|
||||
CST_STACK: this.CST_STACK
|
||||
}
|
||||
}
|
||||
|
||||
reloadRecogState(this: MixedInParser, newState: IParserState) {
|
||||
this.errors = newState.errors
|
||||
this.importLexerState(newState.lexerState)
|
||||
this.RULE_STACK = newState.RULE_STACK
|
||||
}
|
||||
|
||||
ruleInvocationStateUpdate(
|
||||
this: MixedInParser,
|
||||
shortName: string,
|
||||
fullName: string,
|
||||
idxInCallingRule: number
|
||||
): void {
|
||||
this.RULE_OCCURRENCE_STACK.push(idxInCallingRule)
|
||||
this.RULE_STACK.push(shortName)
|
||||
// NOOP when cst is disabled
|
||||
this.cstInvocationStateUpdate(fullName, shortName)
|
||||
}
|
||||
|
||||
isBackTracking(this: MixedInParser): boolean {
|
||||
return this.isBackTrackingStack.length !== 0
|
||||
}
|
||||
|
||||
getCurrRuleFullName(this: MixedInParser): string {
|
||||
let shortName = this.getLastExplicitRuleShortName()
|
||||
return this.shortRuleNameToFull[shortName]
|
||||
}
|
||||
|
||||
shortRuleNameToFullName(this: MixedInParser, shortName: string) {
|
||||
return this.shortRuleNameToFull[shortName]
|
||||
}
|
||||
|
||||
public isAtEndOfInput(this: MixedInParser): boolean {
|
||||
return this.tokenMatcher(this.LA(1), EOF)
|
||||
}
|
||||
|
||||
public reset(this: MixedInParser): void {
|
||||
this.resetLexerState()
|
||||
|
||||
this.isBackTrackingStack = []
|
||||
this.errors = []
|
||||
this.RULE_STACK = []
|
||||
// TODO: extract a specific reset for TreeBuilder trait
|
||||
this.CST_STACK = []
|
||||
this.RULE_OCCURRENCE_STACK = []
|
||||
}
|
||||
}
|
||||
459
node_modules/chevrotain/src/parse/parser/traits/recoverable.ts
generated
vendored
Normal file
459
node_modules/chevrotain/src/parse/parser/traits/recoverable.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,459 @@
|
|||
import { createTokenInstance, EOF } from "../../../scan/tokens_public"
|
||||
import {
|
||||
AbstractNextTerminalAfterProductionWalker,
|
||||
IFirstAfterRepetition
|
||||
} from "../../grammar/interpreter"
|
||||
import {
|
||||
cloneArr,
|
||||
contains,
|
||||
dropRight,
|
||||
find,
|
||||
flatten,
|
||||
has,
|
||||
isEmpty,
|
||||
map
|
||||
} from "../../../utils/utils"
|
||||
import {
|
||||
IParserConfig,
|
||||
IToken,
|
||||
ITokenGrammarPath,
|
||||
TokenType
|
||||
} from "../../../../api"
|
||||
import { MismatchedTokenException } from "../../exceptions_public"
|
||||
import { IN } from "../../constants"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser"
|
||||
|
||||
export const EOF_FOLLOW_KEY: any = {}
|
||||
|
||||
export interface IFollowKey {
|
||||
ruleName: string
|
||||
idxInCallingRule: number
|
||||
inRule: string
|
||||
}
|
||||
|
||||
export const IN_RULE_RECOVERY_EXCEPTION = "InRuleRecoveryException"
|
||||
|
||||
export function InRuleRecoveryException(message: string) {
|
||||
this.name = IN_RULE_RECOVERY_EXCEPTION
|
||||
this.message = message
|
||||
}
|
||||
|
||||
InRuleRecoveryException.prototype = Error.prototype
|
||||
|
||||
/**
|
||||
* This trait is responsible for the error recovery and fault tolerant logic
|
||||
*/
|
||||
export class Recoverable {
|
||||
recoveryEnabled: boolean
|
||||
firstAfterRepMap: Record<string, IFirstAfterRepetition>
|
||||
resyncFollows: Record<string, TokenType[]>
|
||||
|
||||
initRecoverable(config: IParserConfig) {
|
||||
this.firstAfterRepMap = {}
|
||||
this.resyncFollows = {}
|
||||
|
||||
this.recoveryEnabled = has(config, "recoveryEnabled")
|
||||
? config.recoveryEnabled
|
||||
: DEFAULT_PARSER_CONFIG.recoveryEnabled
|
||||
|
||||
// performance optimization, NOOP will be inlined which
|
||||
// effectively means that this optional feature does not exist
|
||||
// when not used.
|
||||
if (this.recoveryEnabled) {
|
||||
this.attemptInRepetitionRecovery = attemptInRepetitionRecovery
|
||||
}
|
||||
}
|
||||
|
||||
public getTokenToInsert(tokType: TokenType): IToken {
|
||||
let tokToInsert = createTokenInstance(
|
||||
tokType,
|
||||
"",
|
||||
NaN,
|
||||
NaN,
|
||||
NaN,
|
||||
NaN,
|
||||
NaN,
|
||||
NaN
|
||||
)
|
||||
tokToInsert.isInsertedInRecovery = true
|
||||
return tokToInsert
|
||||
}
|
||||
|
||||
public canTokenTypeBeInsertedInRecovery(tokType: TokenType) {
|
||||
return true
|
||||
}
|
||||
|
||||
tryInRepetitionRecovery(
|
||||
this: MixedInParser,
|
||||
grammarRule: Function,
|
||||
grammarRuleArgs: any[],
|
||||
lookAheadFunc: () => boolean,
|
||||
expectedTokType: TokenType
|
||||
): void {
|
||||
// TODO: can the resyncTokenType be cached?
|
||||
let reSyncTokType = this.findReSyncTokenType()
|
||||
let savedLexerState = this.exportLexerState()
|
||||
let resyncedTokens = []
|
||||
let passedResyncPoint = false
|
||||
|
||||
let nextTokenWithoutResync = this.LA(1)
|
||||
let currToken = this.LA(1)
|
||||
|
||||
let generateErrorMessage = () => {
|
||||
let previousToken = this.LA(0)
|
||||
// we are preemptively re-syncing before an error has been detected, therefor we must reproduce
|
||||
// the error that would have been thrown
|
||||
let msg = this.errorMessageProvider.buildMismatchTokenMessage({
|
||||
expected: expectedTokType,
|
||||
actual: nextTokenWithoutResync,
|
||||
previous: previousToken,
|
||||
ruleName: this.getCurrRuleFullName()
|
||||
})
|
||||
let error = new MismatchedTokenException(
|
||||
msg,
|
||||
nextTokenWithoutResync,
|
||||
this.LA(0)
|
||||
)
|
||||
// the first token here will be the original cause of the error, this is not part of the resyncedTokens property.
|
||||
error.resyncedTokens = dropRight(resyncedTokens)
|
||||
this.SAVE_ERROR(error)
|
||||
}
|
||||
|
||||
while (!passedResyncPoint) {
|
||||
// re-synced to a point where we can safely exit the repetition/
|
||||
if (this.tokenMatcher(currToken, expectedTokType)) {
|
||||
generateErrorMessage()
|
||||
return // must return here to avoid reverting the inputIdx
|
||||
} else if (lookAheadFunc.call(this)) {
|
||||
// we skipped enough tokens so we can resync right back into another iteration of the repetition grammar rule
|
||||
generateErrorMessage()
|
||||
// recursive invocation in other to support multiple re-syncs in the same top level repetition grammar rule
|
||||
grammarRule.apply(this, grammarRuleArgs)
|
||||
return // must return here to avoid reverting the inputIdx
|
||||
} else if (this.tokenMatcher(currToken, reSyncTokType)) {
|
||||
passedResyncPoint = true
|
||||
} else {
|
||||
currToken = this.SKIP_TOKEN()
|
||||
this.addToResyncTokens(currToken, resyncedTokens)
|
||||
}
|
||||
}
|
||||
|
||||
// we were unable to find a CLOSER point to resync inside the Repetition, reset the state.
|
||||
// The parsing exception we were trying to prevent will happen in the NEXT parsing step. it may be handled by
|
||||
// "between rules" resync recovery later in the flow.
|
||||
this.importLexerState(savedLexerState)
|
||||
}
|
||||
|
||||
shouldInRepetitionRecoveryBeTried(
|
||||
this: MixedInParser,
|
||||
expectTokAfterLastMatch: TokenType,
|
||||
nextTokIdx: number,
|
||||
notStuck: boolean | undefined
|
||||
): boolean {
|
||||
// Edge case of arriving from a MANY repetition which is stuck
|
||||
// Attempting recovery in this case could cause an infinite loop
|
||||
if (notStuck === false) {
|
||||
return false
|
||||
}
|
||||
|
||||
// arguments to try and perform resync into the next iteration of the many are missing
|
||||
if (expectTokAfterLastMatch === undefined || nextTokIdx === undefined) {
|
||||
return false
|
||||
}
|
||||
|
||||
// no need to recover, next token is what we expect...
|
||||
if (this.tokenMatcher(this.LA(1), expectTokAfterLastMatch)) {
|
||||
return false
|
||||
}
|
||||
|
||||
// error recovery is disabled during backtracking as it can make the parser ignore a valid grammar path
|
||||
// and prefer some backtracking path that includes recovered errors.
|
||||
if (this.isBackTracking()) {
|
||||
return false
|
||||
}
|
||||
|
||||
// if we can perform inRule recovery (single token insertion or deletion) we always prefer that recovery algorithm
|
||||
// because if it works, it makes the least amount of changes to the input stream (greedy algorithm)
|
||||
//noinspection RedundantIfStatementJS
|
||||
if (
|
||||
this.canPerformInRuleRecovery(
|
||||
expectTokAfterLastMatch,
|
||||
this.getFollowsForInRuleRecovery(expectTokAfterLastMatch, nextTokIdx)
|
||||
)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Error Recovery functionality
|
||||
getFollowsForInRuleRecovery(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
tokIdxInRule: number
|
||||
): TokenType[] {
|
||||
let grammarPath = this.getCurrentGrammarPath(tokType, tokIdxInRule)
|
||||
let follows = this.getNextPossibleTokenTypes(grammarPath)
|
||||
return follows
|
||||
}
|
||||
|
||||
tryInRuleRecovery(
|
||||
this: MixedInParser,
|
||||
expectedTokType: TokenType,
|
||||
follows: TokenType[]
|
||||
): IToken {
|
||||
if (this.canRecoverWithSingleTokenInsertion(expectedTokType, follows)) {
|
||||
let tokToInsert = this.getTokenToInsert(expectedTokType)
|
||||
return tokToInsert
|
||||
}
|
||||
|
||||
if (this.canRecoverWithSingleTokenDeletion(expectedTokType)) {
|
||||
let nextTok = this.SKIP_TOKEN()
|
||||
this.consumeToken()
|
||||
return nextTok
|
||||
}
|
||||
|
||||
throw new InRuleRecoveryException("sad sad panda")
|
||||
}
|
||||
|
||||
canPerformInRuleRecovery(
|
||||
this: MixedInParser,
|
||||
expectedToken: TokenType,
|
||||
follows: TokenType[]
|
||||
): boolean {
|
||||
return (
|
||||
this.canRecoverWithSingleTokenInsertion(expectedToken, follows) ||
|
||||
this.canRecoverWithSingleTokenDeletion(expectedToken)
|
||||
)
|
||||
}
|
||||
|
||||
canRecoverWithSingleTokenInsertion(
|
||||
this: MixedInParser,
|
||||
expectedTokType: TokenType,
|
||||
follows: TokenType[]
|
||||
): boolean {
|
||||
if (!this.canTokenTypeBeInsertedInRecovery(expectedTokType)) {
|
||||
return false
|
||||
}
|
||||
|
||||
// must know the possible following tokens to perform single token insertion
|
||||
if (isEmpty(follows)) {
|
||||
return false
|
||||
}
|
||||
|
||||
let mismatchedTok = this.LA(1)
|
||||
let isMisMatchedTokInFollows =
|
||||
find(follows, (possibleFollowsTokType: TokenType) => {
|
||||
return this.tokenMatcher(mismatchedTok, possibleFollowsTokType)
|
||||
}) !== undefined
|
||||
|
||||
return isMisMatchedTokInFollows
|
||||
}
|
||||
|
||||
canRecoverWithSingleTokenDeletion(
|
||||
this: MixedInParser,
|
||||
expectedTokType: TokenType
|
||||
): boolean {
|
||||
let isNextTokenWhatIsExpected = this.tokenMatcher(
|
||||
this.LA(2),
|
||||
expectedTokType
|
||||
)
|
||||
return isNextTokenWhatIsExpected
|
||||
}
|
||||
|
||||
isInCurrentRuleReSyncSet(
|
||||
this: MixedInParser,
|
||||
tokenTypeIdx: TokenType
|
||||
): boolean {
|
||||
let followKey = this.getCurrFollowKey()
|
||||
let currentRuleReSyncSet = this.getFollowSetFromFollowKey(followKey)
|
||||
return contains(currentRuleReSyncSet, tokenTypeIdx)
|
||||
}
|
||||
|
||||
findReSyncTokenType(this: MixedInParser): TokenType {
|
||||
let allPossibleReSyncTokTypes = this.flattenFollowSet()
|
||||
// this loop will always terminate as EOF is always in the follow stack and also always (virtually) in the input
|
||||
let nextToken = this.LA(1)
|
||||
let k = 2
|
||||
while (true) {
|
||||
let nextTokenType: any = nextToken.tokenType
|
||||
if (contains(allPossibleReSyncTokTypes, nextTokenType)) {
|
||||
return nextTokenType
|
||||
}
|
||||
nextToken = this.LA(k)
|
||||
k++
|
||||
}
|
||||
}
|
||||
|
||||
getCurrFollowKey(this: MixedInParser): IFollowKey {
|
||||
// the length is at least one as we always add the ruleName to the stack before invoking the rule.
|
||||
if (this.RULE_STACK.length === 1) {
|
||||
return EOF_FOLLOW_KEY
|
||||
}
|
||||
let currRuleShortName = this.getLastExplicitRuleShortName()
|
||||
let currRuleIdx = this.getLastExplicitRuleOccurrenceIndex()
|
||||
let prevRuleShortName = this.getPreviousExplicitRuleShortName()
|
||||
|
||||
return {
|
||||
ruleName: this.shortRuleNameToFullName(currRuleShortName),
|
||||
idxInCallingRule: currRuleIdx,
|
||||
inRule: this.shortRuleNameToFullName(prevRuleShortName)
|
||||
}
|
||||
}
|
||||
|
||||
buildFullFollowKeyStack(this: MixedInParser): IFollowKey[] {
|
||||
let explicitRuleStack = this.RULE_STACK
|
||||
let explicitOccurrenceStack = this.RULE_OCCURRENCE_STACK
|
||||
|
||||
return map(explicitRuleStack, (ruleName, idx) => {
|
||||
if (idx === 0) {
|
||||
return EOF_FOLLOW_KEY
|
||||
}
|
||||
return {
|
||||
ruleName: this.shortRuleNameToFullName(ruleName),
|
||||
idxInCallingRule: explicitOccurrenceStack[idx],
|
||||
inRule: this.shortRuleNameToFullName(explicitRuleStack[idx - 1])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
flattenFollowSet(this: MixedInParser): TokenType[] {
|
||||
let followStack = map(this.buildFullFollowKeyStack(), (currKey) => {
|
||||
return this.getFollowSetFromFollowKey(currKey)
|
||||
})
|
||||
return <any>flatten(followStack)
|
||||
}
|
||||
|
||||
getFollowSetFromFollowKey(
|
||||
this: MixedInParser,
|
||||
followKey: IFollowKey
|
||||
): TokenType[] {
|
||||
if (followKey === EOF_FOLLOW_KEY) {
|
||||
return [EOF]
|
||||
}
|
||||
|
||||
let followName =
|
||||
followKey.ruleName + followKey.idxInCallingRule + IN + followKey.inRule
|
||||
|
||||
return this.resyncFollows[followName]
|
||||
}
|
||||
|
||||
// It does not make any sense to include a virtual EOF token in the list of resynced tokens
|
||||
// as EOF does not really exist and thus does not contain any useful information (line/column numbers)
|
||||
addToResyncTokens(
|
||||
this: MixedInParser,
|
||||
token: IToken,
|
||||
resyncTokens: IToken[]
|
||||
): IToken[] {
|
||||
if (!this.tokenMatcher(token, EOF)) {
|
||||
resyncTokens.push(token)
|
||||
}
|
||||
return resyncTokens
|
||||
}
|
||||
|
||||
reSyncTo(this: MixedInParser, tokType: TokenType): IToken[] {
|
||||
let resyncedTokens = []
|
||||
let nextTok = this.LA(1)
|
||||
while (this.tokenMatcher(nextTok, tokType) === false) {
|
||||
nextTok = this.SKIP_TOKEN()
|
||||
this.addToResyncTokens(nextTok, resyncedTokens)
|
||||
}
|
||||
// the last token is not part of the error.
|
||||
return dropRight(resyncedTokens)
|
||||
}
|
||||
|
||||
attemptInRepetitionRecovery(
|
||||
this: MixedInParser,
|
||||
prodFunc: Function,
|
||||
args: any[],
|
||||
lookaheadFunc: () => boolean,
|
||||
dslMethodIdx: number,
|
||||
prodOccurrence: number,
|
||||
nextToksWalker: typeof AbstractNextTerminalAfterProductionWalker,
|
||||
notStuck?: boolean
|
||||
): void {
|
||||
// by default this is a NO-OP
|
||||
// The actual implementation is with the function(not method) below
|
||||
}
|
||||
|
||||
getCurrentGrammarPath(
|
||||
this: MixedInParser,
|
||||
tokType: TokenType,
|
||||
tokIdxInRule: number
|
||||
): ITokenGrammarPath {
|
||||
let pathRuleStack: string[] = this.getHumanReadableRuleStack()
|
||||
let pathOccurrenceStack: number[] = cloneArr(this.RULE_OCCURRENCE_STACK)
|
||||
let grammarPath: any = {
|
||||
ruleStack: pathRuleStack,
|
||||
occurrenceStack: pathOccurrenceStack,
|
||||
lastTok: tokType,
|
||||
lastTokOccurrence: tokIdxInRule
|
||||
}
|
||||
|
||||
return grammarPath
|
||||
}
|
||||
getHumanReadableRuleStack(this: MixedInParser): string[] {
|
||||
return map(this.RULE_STACK, (currShortName) =>
|
||||
this.shortRuleNameToFullName(currShortName)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function attemptInRepetitionRecovery(
|
||||
this: MixedInParser,
|
||||
prodFunc: Function,
|
||||
args: any[],
|
||||
lookaheadFunc: () => boolean,
|
||||
dslMethodIdx: number,
|
||||
prodOccurrence: number,
|
||||
nextToksWalker: typeof AbstractNextTerminalAfterProductionWalker,
|
||||
notStuck?: boolean
|
||||
) {
|
||||
let key = this.getKeyForAutomaticLookahead(dslMethodIdx, prodOccurrence)
|
||||
let firstAfterRepInfo = this.firstAfterRepMap[key]
|
||||
if (firstAfterRepInfo === undefined) {
|
||||
let currRuleName = this.getCurrRuleFullName()
|
||||
let ruleGrammar = this.getGAstProductions()[currRuleName]
|
||||
let walker: AbstractNextTerminalAfterProductionWalker = new nextToksWalker(
|
||||
ruleGrammar,
|
||||
prodOccurrence
|
||||
)
|
||||
firstAfterRepInfo = walker.startWalking()
|
||||
this.firstAfterRepMap[key] = firstAfterRepInfo
|
||||
}
|
||||
|
||||
let expectTokAfterLastMatch = firstAfterRepInfo.token
|
||||
let nextTokIdx = firstAfterRepInfo.occurrence
|
||||
let isEndOfRule = firstAfterRepInfo.isEndOfRule
|
||||
|
||||
// special edge case of a TOP most repetition after which the input should END.
|
||||
// this will force an attempt for inRule recovery in that scenario.
|
||||
if (
|
||||
this.RULE_STACK.length === 1 &&
|
||||
isEndOfRule &&
|
||||
expectTokAfterLastMatch === undefined
|
||||
) {
|
||||
expectTokAfterLastMatch = EOF
|
||||
nextTokIdx = 1
|
||||
}
|
||||
|
||||
if (
|
||||
this.shouldInRepetitionRecoveryBeTried(
|
||||
expectTokAfterLastMatch,
|
||||
nextTokIdx,
|
||||
notStuck
|
||||
)
|
||||
) {
|
||||
// TODO: performance optimization: instead of passing the original args here, we modify
|
||||
// the args param (or create a new one) and make sure the lookahead func is explicitly provided
|
||||
// to avoid searching the cache for it once more.
|
||||
this.tryInRepetitionRecovery(
|
||||
prodFunc,
|
||||
args,
|
||||
lookaheadFunc,
|
||||
expectTokAfterLastMatch
|
||||
)
|
||||
}
|
||||
}
|
||||
275
node_modules/chevrotain/src/parse/parser/traits/tree_builder.ts
generated
vendored
Normal file
275
node_modules/chevrotain/src/parse/parser/traits/tree_builder.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
import {
|
||||
addNoneTerminalToCst,
|
||||
addTerminalToCst,
|
||||
setNodeLocationFull,
|
||||
setNodeLocationOnlyOffset
|
||||
} from "../../cst/cst"
|
||||
import { has, isUndefined, keys, NOOP } from "../../../utils/utils"
|
||||
import {
|
||||
createBaseSemanticVisitorConstructor,
|
||||
createBaseVisitorConstructorWithDefaults
|
||||
} from "../../cst/cst_visitor"
|
||||
import {
|
||||
CstNode,
|
||||
CstNodeLocation,
|
||||
ICstVisitor,
|
||||
IParserConfig,
|
||||
IToken,
|
||||
nodeLocationTrackingOptions
|
||||
} from "../../../../api"
|
||||
import { MixedInParser } from "./parser_traits"
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser"
|
||||
|
||||
/**
|
||||
* This trait is responsible for the CST building logic.
|
||||
*/
|
||||
export class TreeBuilder {
|
||||
outputCst: boolean
|
||||
CST_STACK: CstNode[]
|
||||
baseCstVisitorConstructor: Function
|
||||
baseCstVisitorWithDefaultsConstructor: Function
|
||||
|
||||
// dynamically assigned Methods
|
||||
setNodeLocationFromNode: (
|
||||
nodeLocation: CstNodeLocation,
|
||||
locationInformation: CstNodeLocation
|
||||
) => void
|
||||
setNodeLocationFromToken: (
|
||||
nodeLocation: CstNodeLocation,
|
||||
locationInformation: CstNodeLocation
|
||||
) => void
|
||||
cstPostRule: (this: MixedInParser, ruleCstNode: CstNode) => void
|
||||
|
||||
setInitialNodeLocation: (cstNode: CstNode) => void
|
||||
nodeLocationTracking: nodeLocationTrackingOptions
|
||||
|
||||
initTreeBuilder(this: MixedInParser, config: IParserConfig) {
|
||||
this.CST_STACK = []
|
||||
|
||||
// outputCst is no longer exposed/defined in the pubic API
|
||||
this.outputCst = (config as any).outputCst
|
||||
|
||||
this.nodeLocationTracking = has(config, "nodeLocationTracking")
|
||||
? config.nodeLocationTracking
|
||||
: DEFAULT_PARSER_CONFIG.nodeLocationTracking
|
||||
|
||||
if (!this.outputCst) {
|
||||
this.cstInvocationStateUpdate = NOOP
|
||||
this.cstFinallyStateUpdate = NOOP
|
||||
this.cstPostTerminal = NOOP
|
||||
this.cstPostNonTerminal = NOOP
|
||||
this.cstPostRule = NOOP
|
||||
} else {
|
||||
if (/full/i.test(this.nodeLocationTracking)) {
|
||||
if (this.recoveryEnabled) {
|
||||
this.setNodeLocationFromToken = setNodeLocationFull
|
||||
this.setNodeLocationFromNode = setNodeLocationFull
|
||||
this.cstPostRule = NOOP
|
||||
this.setInitialNodeLocation = this.setInitialNodeLocationFullRecovery
|
||||
} else {
|
||||
this.setNodeLocationFromToken = NOOP
|
||||
this.setNodeLocationFromNode = NOOP
|
||||
this.cstPostRule = this.cstPostRuleFull
|
||||
this.setInitialNodeLocation = this.setInitialNodeLocationFullRegular
|
||||
}
|
||||
} else if (/onlyOffset/i.test(this.nodeLocationTracking)) {
|
||||
if (this.recoveryEnabled) {
|
||||
this.setNodeLocationFromToken = <any>setNodeLocationOnlyOffset
|
||||
this.setNodeLocationFromNode = <any>setNodeLocationOnlyOffset
|
||||
this.cstPostRule = NOOP
|
||||
this.setInitialNodeLocation = this.setInitialNodeLocationOnlyOffsetRecovery
|
||||
} else {
|
||||
this.setNodeLocationFromToken = NOOP
|
||||
this.setNodeLocationFromNode = NOOP
|
||||
this.cstPostRule = this.cstPostRuleOnlyOffset
|
||||
this.setInitialNodeLocation = this.setInitialNodeLocationOnlyOffsetRegular
|
||||
}
|
||||
} else if (/none/i.test(this.nodeLocationTracking)) {
|
||||
this.setNodeLocationFromToken = NOOP
|
||||
this.setNodeLocationFromNode = NOOP
|
||||
this.cstPostRule = NOOP
|
||||
this.setInitialNodeLocation = NOOP
|
||||
} else {
|
||||
throw Error(
|
||||
`Invalid <nodeLocationTracking> config option: "${config.nodeLocationTracking}"`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setInitialNodeLocationOnlyOffsetRecovery(
|
||||
this: MixedInParser,
|
||||
cstNode: any
|
||||
): void {
|
||||
cstNode.location = {
|
||||
startOffset: NaN,
|
||||
endOffset: NaN
|
||||
}
|
||||
}
|
||||
|
||||
setInitialNodeLocationOnlyOffsetRegular(
|
||||
this: MixedInParser,
|
||||
cstNode: any
|
||||
): void {
|
||||
cstNode.location = {
|
||||
// without error recovery the starting Location of a new CstNode is guaranteed
|
||||
// To be the next Token's startOffset (for valid inputs).
|
||||
// For invalid inputs there won't be any CSTOutput so this potential
|
||||
// inaccuracy does not matter
|
||||
startOffset: this.LA(1).startOffset,
|
||||
endOffset: NaN
|
||||
}
|
||||
}
|
||||
|
||||
setInitialNodeLocationFullRecovery(this: MixedInParser, cstNode: any): void {
|
||||
cstNode.location = {
|
||||
startOffset: NaN,
|
||||
startLine: NaN,
|
||||
startColumn: NaN,
|
||||
endOffset: NaN,
|
||||
endLine: NaN,
|
||||
endColumn: NaN
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @see setInitialNodeLocationOnlyOffsetRegular for explanation why this work
|
||||
|
||||
* @param cstNode
|
||||
*/
|
||||
setInitialNodeLocationFullRegular(this: MixedInParser, cstNode: any): void {
|
||||
const nextToken = this.LA(1)
|
||||
cstNode.location = {
|
||||
startOffset: nextToken.startOffset,
|
||||
startLine: nextToken.startLine,
|
||||
startColumn: nextToken.startColumn,
|
||||
endOffset: NaN,
|
||||
endLine: NaN,
|
||||
endColumn: NaN
|
||||
}
|
||||
}
|
||||
|
||||
cstInvocationStateUpdate(
|
||||
this: MixedInParser,
|
||||
fullRuleName: string,
|
||||
shortName: string | number
|
||||
): void {
|
||||
const cstNode: CstNode = {
|
||||
name: fullRuleName,
|
||||
children: {}
|
||||
}
|
||||
|
||||
this.setInitialNodeLocation(cstNode)
|
||||
this.CST_STACK.push(cstNode)
|
||||
}
|
||||
|
||||
cstFinallyStateUpdate(this: MixedInParser): void {
|
||||
this.CST_STACK.pop()
|
||||
}
|
||||
|
||||
cstPostRuleFull(this: MixedInParser, ruleCstNode: CstNode): void {
|
||||
const prevToken = this.LA(0)
|
||||
const loc = ruleCstNode.location
|
||||
|
||||
// If this condition is true it means we consumed at least one Token
|
||||
// In this CstNode.
|
||||
if (loc.startOffset <= prevToken.startOffset === true) {
|
||||
loc.endOffset = prevToken.endOffset
|
||||
loc.endLine = prevToken.endLine
|
||||
loc.endColumn = prevToken.endColumn
|
||||
}
|
||||
// "empty" CstNode edge case
|
||||
else {
|
||||
loc.startOffset = NaN
|
||||
loc.startLine = NaN
|
||||
loc.startColumn = NaN
|
||||
}
|
||||
}
|
||||
|
||||
cstPostRuleOnlyOffset(this: MixedInParser, ruleCstNode: CstNode): void {
|
||||
const prevToken = this.LA(0)
|
||||
const loc = ruleCstNode.location
|
||||
|
||||
// If this condition is true it means we consumed at least one Token
|
||||
// In this CstNode.
|
||||
if (loc.startOffset <= prevToken.startOffset === true) {
|
||||
loc.endOffset = prevToken.endOffset
|
||||
}
|
||||
// "empty" CstNode edge case
|
||||
else {
|
||||
loc.startOffset = NaN
|
||||
}
|
||||
}
|
||||
|
||||
cstPostTerminal(
|
||||
this: MixedInParser,
|
||||
key: string,
|
||||
consumedToken: IToken
|
||||
): void {
|
||||
const rootCst = this.CST_STACK[this.CST_STACK.length - 1]
|
||||
addTerminalToCst(rootCst, consumedToken, key)
|
||||
// This is only used when **both** error recovery and CST Output are enabled.
|
||||
this.setNodeLocationFromToken(rootCst.location, <any>consumedToken)
|
||||
}
|
||||
|
||||
cstPostNonTerminal(
|
||||
this: MixedInParser,
|
||||
ruleCstResult: CstNode,
|
||||
ruleName: string
|
||||
): void {
|
||||
const preCstNode = this.CST_STACK[this.CST_STACK.length - 1]
|
||||
addNoneTerminalToCst(preCstNode, ruleName, ruleCstResult)
|
||||
// This is only used when **both** error recovery and CST Output are enabled.
|
||||
this.setNodeLocationFromNode(preCstNode.location, ruleCstResult.location)
|
||||
}
|
||||
|
||||
getBaseCstVisitorConstructor<IN = any, OUT = any>(
|
||||
this: MixedInParser
|
||||
): {
|
||||
new (...args: any[]): ICstVisitor<IN, OUT>
|
||||
} {
|
||||
if (isUndefined(this.baseCstVisitorConstructor)) {
|
||||
const newBaseCstVisitorConstructor = createBaseSemanticVisitorConstructor(
|
||||
this.className,
|
||||
keys(this.gastProductionsCache)
|
||||
)
|
||||
this.baseCstVisitorConstructor = newBaseCstVisitorConstructor
|
||||
return newBaseCstVisitorConstructor
|
||||
}
|
||||
|
||||
return <any>this.baseCstVisitorConstructor
|
||||
}
|
||||
|
||||
getBaseCstVisitorConstructorWithDefaults<IN = any, OUT = any>(
|
||||
this: MixedInParser
|
||||
): {
|
||||
new (...args: any[]): ICstVisitor<IN, OUT>
|
||||
} {
|
||||
if (isUndefined(this.baseCstVisitorWithDefaultsConstructor)) {
|
||||
const newConstructor = createBaseVisitorConstructorWithDefaults(
|
||||
this.className,
|
||||
keys(this.gastProductionsCache),
|
||||
this.getBaseCstVisitorConstructor()
|
||||
)
|
||||
this.baseCstVisitorWithDefaultsConstructor = newConstructor
|
||||
return newConstructor
|
||||
}
|
||||
|
||||
return <any>this.baseCstVisitorWithDefaultsConstructor
|
||||
}
|
||||
|
||||
getLastExplicitRuleShortName(this: MixedInParser): string {
|
||||
let ruleStack = this.RULE_STACK
|
||||
return ruleStack[ruleStack.length - 1]
|
||||
}
|
||||
|
||||
getPreviousExplicitRuleShortName(this: MixedInParser): string {
|
||||
let ruleStack = this.RULE_STACK
|
||||
return ruleStack[ruleStack.length - 2]
|
||||
}
|
||||
|
||||
getLastExplicitRuleOccurrenceIndex(this: MixedInParser): number {
|
||||
let occurrenceStack = this.RULE_OCCURRENCE_STACK
|
||||
return occurrenceStack[occurrenceStack.length - 1]
|
||||
}
|
||||
}
|
||||
1147
node_modules/chevrotain/src/scan/lexer.ts
generated
vendored
Normal file
1147
node_modules/chevrotain/src/scan/lexer.ts
generated
vendored
Normal file
File diff suppressed because it is too large
Load diff
21
node_modules/chevrotain/src/scan/lexer_errors_public.ts
generated
vendored
Normal file
21
node_modules/chevrotain/src/scan/lexer_errors_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { ILexerErrorMessageProvider, IToken } from "../../api"
|
||||
|
||||
export const defaultLexerErrorProvider: ILexerErrorMessageProvider = {
|
||||
buildUnableToPopLexerModeMessage(token: IToken): string {
|
||||
return `Unable to pop Lexer Mode after encountering Token ->${token.image}<- The Mode Stack is empty`
|
||||
},
|
||||
|
||||
buildUnexpectedCharactersMessage(
|
||||
fullText: string,
|
||||
startOffset: number,
|
||||
length: number,
|
||||
line?: number,
|
||||
column?: number
|
||||
): string {
|
||||
return (
|
||||
`unexpected character: ->${fullText.charAt(
|
||||
startOffset
|
||||
)}<- at offset: ${startOffset},` + ` skipped ${length} characters.`
|
||||
)
|
||||
}
|
||||
}
|
||||
922
node_modules/chevrotain/src/scan/lexer_public.ts
generated
vendored
Normal file
922
node_modules/chevrotain/src/scan/lexer_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,922 @@
|
|||
import {
|
||||
analyzeTokenTypes,
|
||||
charCodeToOptimizedIndex,
|
||||
cloneEmptyGroups,
|
||||
DEFAULT_MODE,
|
||||
LineTerminatorOptimizedTester,
|
||||
performRuntimeChecks,
|
||||
performWarningRuntimeChecks,
|
||||
SUPPORT_STICKY,
|
||||
validatePatterns
|
||||
} from "./lexer"
|
||||
import {
|
||||
cloneArr,
|
||||
cloneObj,
|
||||
forEach,
|
||||
IDENTITY,
|
||||
isArray,
|
||||
isEmpty,
|
||||
isUndefined,
|
||||
keys,
|
||||
last,
|
||||
map,
|
||||
merge,
|
||||
NOOP,
|
||||
PRINT_WARNING,
|
||||
reduce,
|
||||
reject,
|
||||
timer,
|
||||
toFastProperties
|
||||
} from "../utils/utils"
|
||||
import { augmentTokenTypes } from "./tokens"
|
||||
import {
|
||||
CustomPatternMatcherFunc,
|
||||
ILexerConfig,
|
||||
ILexerDefinitionError,
|
||||
ILexingError,
|
||||
IMultiModeLexerDefinition,
|
||||
IToken,
|
||||
TokenType
|
||||
} from "../../api"
|
||||
import { defaultLexerErrorProvider } from "../scan/lexer_errors_public"
|
||||
import { clearRegExpParserCache } from "./reg_exp_parser"
|
||||
|
||||
export interface ILexingResult {
|
||||
tokens: IToken[]
|
||||
groups: { [groupName: string]: IToken[] }
|
||||
errors: ILexingError[]
|
||||
}
|
||||
|
||||
export enum LexerDefinitionErrorType {
|
||||
MISSING_PATTERN,
|
||||
INVALID_PATTERN,
|
||||
EOI_ANCHOR_FOUND,
|
||||
UNSUPPORTED_FLAGS_FOUND,
|
||||
DUPLICATE_PATTERNS_FOUND,
|
||||
INVALID_GROUP_TYPE_FOUND,
|
||||
PUSH_MODE_DOES_NOT_EXIST,
|
||||
MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE,
|
||||
MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY,
|
||||
MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST,
|
||||
LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED,
|
||||
SOI_ANCHOR_FOUND,
|
||||
EMPTY_MATCH_PATTERN,
|
||||
NO_LINE_BREAKS_FLAGS,
|
||||
UNREACHABLE_PATTERN,
|
||||
IDENTIFY_TERMINATOR,
|
||||
CUSTOM_LINE_BREAK
|
||||
}
|
||||
|
||||
export interface IRegExpExec {
|
||||
exec: CustomPatternMatcherFunc
|
||||
}
|
||||
|
||||
const DEFAULT_LEXER_CONFIG: ILexerConfig = {
|
||||
deferDefinitionErrorsHandling: false,
|
||||
positionTracking: "full",
|
||||
lineTerminatorsPattern: /\n|\r\n?/g,
|
||||
lineTerminatorCharacters: ["\n", "\r"],
|
||||
ensureOptimizations: false,
|
||||
safeMode: false,
|
||||
errorMessageProvider: defaultLexerErrorProvider,
|
||||
traceInitPerf: false,
|
||||
skipValidations: false
|
||||
}
|
||||
|
||||
Object.freeze(DEFAULT_LEXER_CONFIG)
|
||||
|
||||
export class Lexer {
|
||||
public static SKIPPED =
|
||||
"This marks a skipped Token pattern, this means each token identified by it will" +
|
||||
"be consumed and then thrown into oblivion, this can be used to for example to completely ignore whitespace."
|
||||
|
||||
public static NA = /NOT_APPLICABLE/
|
||||
public lexerDefinitionErrors: ILexerDefinitionError[] = []
|
||||
public lexerDefinitionWarning: ILexerDefinitionError[] = []
|
||||
|
||||
protected patternIdxToConfig: any = {}
|
||||
protected charCodeToPatternIdxToConfig: any = {}
|
||||
|
||||
protected modes: string[] = []
|
||||
protected defaultMode: string
|
||||
protected emptyGroups: { [groupName: string]: IToken } = {}
|
||||
|
||||
private config: ILexerConfig = undefined
|
||||
private trackStartLines: boolean = true
|
||||
private trackEndLines: boolean = true
|
||||
private hasCustom: boolean = false
|
||||
private canModeBeOptimized: any = {}
|
||||
|
||||
private traceInitPerf: boolean | number
|
||||
private traceInitMaxIdent: number
|
||||
private traceInitIndent: number
|
||||
|
||||
constructor(
|
||||
protected lexerDefinition: TokenType[] | IMultiModeLexerDefinition,
|
||||
config: ILexerConfig = DEFAULT_LEXER_CONFIG
|
||||
) {
|
||||
if (typeof config === "boolean") {
|
||||
throw Error(
|
||||
"The second argument to the Lexer constructor is now an ILexerConfig Object.\n" +
|
||||
"a boolean 2nd argument is no longer supported"
|
||||
)
|
||||
}
|
||||
|
||||
// todo: defaults func?
|
||||
this.config = merge(DEFAULT_LEXER_CONFIG, config)
|
||||
|
||||
const traceInitVal = this.config.traceInitPerf
|
||||
if (traceInitVal === true) {
|
||||
this.traceInitMaxIdent = Infinity
|
||||
this.traceInitPerf = true
|
||||
} else if (typeof traceInitVal === "number") {
|
||||
this.traceInitMaxIdent = traceInitVal
|
||||
this.traceInitPerf = true
|
||||
}
|
||||
this.traceInitIndent = -1
|
||||
|
||||
this.TRACE_INIT("Lexer Constructor", () => {
|
||||
let actualDefinition: IMultiModeLexerDefinition
|
||||
let hasOnlySingleMode = true
|
||||
this.TRACE_INIT("Lexer Config handling", () => {
|
||||
if (
|
||||
this.config.lineTerminatorsPattern ===
|
||||
DEFAULT_LEXER_CONFIG.lineTerminatorsPattern
|
||||
) {
|
||||
// optimized built-in implementation for the defaults definition of lineTerminators
|
||||
this.config.lineTerminatorsPattern = LineTerminatorOptimizedTester
|
||||
} else {
|
||||
if (
|
||||
this.config.lineTerminatorCharacters ===
|
||||
DEFAULT_LEXER_CONFIG.lineTerminatorCharacters
|
||||
) {
|
||||
throw Error(
|
||||
"Error: Missing <lineTerminatorCharacters> property on the Lexer config.\n" +
|
||||
"\tFor details See: https://sap.github.io/chevrotain/docs/guide/resolving_lexer_errors.html#MISSING_LINE_TERM_CHARS"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (config.safeMode && config.ensureOptimizations) {
|
||||
throw Error(
|
||||
'"safeMode" and "ensureOptimizations" flags are mutually exclusive.'
|
||||
)
|
||||
}
|
||||
|
||||
this.trackStartLines = /full|onlyStart/i.test(
|
||||
this.config.positionTracking
|
||||
)
|
||||
this.trackEndLines = /full/i.test(this.config.positionTracking)
|
||||
|
||||
// Convert SingleModeLexerDefinition into a IMultiModeLexerDefinition.
|
||||
if (isArray(lexerDefinition)) {
|
||||
actualDefinition = <any>{ modes: {} }
|
||||
actualDefinition.modes[DEFAULT_MODE] = cloneArr(
|
||||
<TokenType[]>lexerDefinition
|
||||
)
|
||||
actualDefinition[DEFAULT_MODE] = DEFAULT_MODE
|
||||
} else {
|
||||
// no conversion needed, input should already be a IMultiModeLexerDefinition
|
||||
hasOnlySingleMode = false
|
||||
actualDefinition = cloneObj(
|
||||
<IMultiModeLexerDefinition>lexerDefinition
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
if (this.config.skipValidations === false) {
|
||||
this.TRACE_INIT("performRuntimeChecks", () => {
|
||||
this.lexerDefinitionErrors = this.lexerDefinitionErrors.concat(
|
||||
performRuntimeChecks(
|
||||
actualDefinition,
|
||||
this.trackStartLines,
|
||||
this.config.lineTerminatorCharacters
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
this.TRACE_INIT("performWarningRuntimeChecks", () => {
|
||||
this.lexerDefinitionWarning = this.lexerDefinitionWarning.concat(
|
||||
performWarningRuntimeChecks(
|
||||
actualDefinition,
|
||||
this.trackStartLines,
|
||||
this.config.lineTerminatorCharacters
|
||||
)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
// for extra robustness to avoid throwing an none informative error message
|
||||
actualDefinition.modes = actualDefinition.modes
|
||||
? actualDefinition.modes
|
||||
: {}
|
||||
|
||||
// an error of undefined TokenTypes will be detected in "performRuntimeChecks" above.
|
||||
// this transformation is to increase robustness in the case of partially invalid lexer definition.
|
||||
forEach(actualDefinition.modes, (currModeValue, currModeName) => {
|
||||
actualDefinition.modes[currModeName] = reject<TokenType>(
|
||||
currModeValue,
|
||||
(currTokType) => isUndefined(currTokType)
|
||||
)
|
||||
})
|
||||
|
||||
let allModeNames = keys(actualDefinition.modes)
|
||||
|
||||
forEach(
|
||||
actualDefinition.modes,
|
||||
(currModDef: TokenType[], currModName) => {
|
||||
this.TRACE_INIT(`Mode: <${currModName}> processing`, () => {
|
||||
this.modes.push(currModName)
|
||||
|
||||
if (this.config.skipValidations === false) {
|
||||
this.TRACE_INIT(`validatePatterns`, () => {
|
||||
this.lexerDefinitionErrors = this.lexerDefinitionErrors.concat(
|
||||
validatePatterns(<TokenType[]>currModDef, allModeNames)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
// If definition errors were encountered, the analysis phase may fail unexpectedly/
|
||||
// Considering a lexer with definition errors may never be used, there is no point
|
||||
// to performing the analysis anyhow...
|
||||
if (isEmpty(this.lexerDefinitionErrors)) {
|
||||
augmentTokenTypes(currModDef)
|
||||
|
||||
let currAnalyzeResult
|
||||
this.TRACE_INIT(`analyzeTokenTypes`, () => {
|
||||
currAnalyzeResult = analyzeTokenTypes(currModDef, {
|
||||
lineTerminatorCharacters: this.config
|
||||
.lineTerminatorCharacters,
|
||||
positionTracking: config.positionTracking,
|
||||
ensureOptimizations: config.ensureOptimizations,
|
||||
safeMode: config.safeMode,
|
||||
tracer: this.TRACE_INIT.bind(this)
|
||||
})
|
||||
})
|
||||
|
||||
this.patternIdxToConfig[currModName] =
|
||||
currAnalyzeResult.patternIdxToConfig
|
||||
|
||||
this.charCodeToPatternIdxToConfig[currModName] =
|
||||
currAnalyzeResult.charCodeToPatternIdxToConfig
|
||||
|
||||
this.emptyGroups = merge(
|
||||
this.emptyGroups,
|
||||
currAnalyzeResult.emptyGroups
|
||||
)
|
||||
|
||||
this.hasCustom = currAnalyzeResult.hasCustom || this.hasCustom
|
||||
|
||||
this.canModeBeOptimized[currModName] =
|
||||
currAnalyzeResult.canBeOptimized
|
||||
}
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
this.defaultMode = actualDefinition.defaultMode
|
||||
|
||||
if (
|
||||
!isEmpty(this.lexerDefinitionErrors) &&
|
||||
!this.config.deferDefinitionErrorsHandling
|
||||
) {
|
||||
let allErrMessages = map(this.lexerDefinitionErrors, (error) => {
|
||||
return error.message
|
||||
})
|
||||
let allErrMessagesString = allErrMessages.join(
|
||||
"-----------------------\n"
|
||||
)
|
||||
throw new Error(
|
||||
"Errors detected in definition of Lexer:\n" + allErrMessagesString
|
||||
)
|
||||
}
|
||||
|
||||
// Only print warning if there are no errors, This will avoid pl
|
||||
forEach(this.lexerDefinitionWarning, (warningDescriptor) => {
|
||||
PRINT_WARNING(warningDescriptor.message)
|
||||
})
|
||||
|
||||
this.TRACE_INIT("Choosing sub-methods implementations", () => {
|
||||
// Choose the relevant internal implementations for this specific parser.
|
||||
// These implementations should be in-lined by the JavaScript engine
|
||||
// to provide optimal performance in each scenario.
|
||||
if (SUPPORT_STICKY) {
|
||||
this.chopInput = <any>IDENTITY
|
||||
this.match = this.matchWithTest
|
||||
} else {
|
||||
this.updateLastIndex = NOOP
|
||||
this.match = this.matchWithExec
|
||||
}
|
||||
|
||||
if (hasOnlySingleMode) {
|
||||
this.handleModes = NOOP
|
||||
}
|
||||
|
||||
if (this.trackStartLines === false) {
|
||||
this.computeNewColumn = IDENTITY
|
||||
}
|
||||
|
||||
if (this.trackEndLines === false) {
|
||||
this.updateTokenEndLineColumnLocation = NOOP
|
||||
}
|
||||
|
||||
if (/full/i.test(this.config.positionTracking)) {
|
||||
this.createTokenInstance = this.createFullToken
|
||||
} else if (/onlyStart/i.test(this.config.positionTracking)) {
|
||||
this.createTokenInstance = this.createStartOnlyToken
|
||||
} else if (/onlyOffset/i.test(this.config.positionTracking)) {
|
||||
this.createTokenInstance = this.createOffsetOnlyToken
|
||||
} else {
|
||||
throw Error(
|
||||
`Invalid <positionTracking> config option: "${this.config.positionTracking}"`
|
||||
)
|
||||
}
|
||||
|
||||
if (this.hasCustom) {
|
||||
this.addToken = this.addTokenUsingPush
|
||||
this.handlePayload = this.handlePayloadWithCustom
|
||||
} else {
|
||||
this.addToken = this.addTokenUsingMemberAccess
|
||||
this.handlePayload = this.handlePayloadNoCustom
|
||||
}
|
||||
})
|
||||
|
||||
this.TRACE_INIT("Failed Optimization Warnings", () => {
|
||||
const unOptimizedModes = reduce(
|
||||
this.canModeBeOptimized,
|
||||
(cannotBeOptimized, canBeOptimized, modeName) => {
|
||||
if (canBeOptimized === false) {
|
||||
cannotBeOptimized.push(modeName)
|
||||
}
|
||||
return cannotBeOptimized
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
||||
if (config.ensureOptimizations && !isEmpty(unOptimizedModes)) {
|
||||
throw Error(
|
||||
`Lexer Modes: < ${unOptimizedModes.join(
|
||||
", "
|
||||
)} > cannot be optimized.\n` +
|
||||
'\t Disable the "ensureOptimizations" lexer config flag to silently ignore this and run the lexer in an un-optimized mode.\n' +
|
||||
"\t Or inspect the console log for details on how to resolve these issues."
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
this.TRACE_INIT("clearRegExpParserCache", () => {
|
||||
clearRegExpParserCache()
|
||||
})
|
||||
|
||||
this.TRACE_INIT("toFastProperties", () => {
|
||||
toFastProperties(this)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
public tokenize(
|
||||
text: string,
|
||||
initialMode: string = this.defaultMode
|
||||
): ILexingResult {
|
||||
if (!isEmpty(this.lexerDefinitionErrors)) {
|
||||
let allErrMessages = map(this.lexerDefinitionErrors, (error) => {
|
||||
return error.message
|
||||
})
|
||||
let allErrMessagesString = allErrMessages.join(
|
||||
"-----------------------\n"
|
||||
)
|
||||
throw new Error(
|
||||
"Unable to Tokenize because Errors detected in definition of Lexer:\n" +
|
||||
allErrMessagesString
|
||||
)
|
||||
}
|
||||
|
||||
let lexResult = this.tokenizeInternal(text, initialMode)
|
||||
|
||||
return lexResult
|
||||
}
|
||||
|
||||
// There is quite a bit of duplication between this and "tokenizeInternalLazy"
|
||||
// This is intentional due to performance considerations.
|
||||
private tokenizeInternal(text: string, initialMode: string): ILexingResult {
|
||||
let i,
|
||||
j,
|
||||
matchAltImage,
|
||||
longerAltIdx,
|
||||
matchedImage,
|
||||
payload,
|
||||
altPayload,
|
||||
imageLength,
|
||||
group,
|
||||
tokType,
|
||||
newToken,
|
||||
errLength,
|
||||
droppedChar,
|
||||
msg,
|
||||
match
|
||||
let orgText = text
|
||||
let orgLength = orgText.length
|
||||
let offset = 0
|
||||
let matchedTokensIndex = 0
|
||||
// initializing the tokensArray to the "guessed" size.
|
||||
// guessing too little will still reduce the number of array re-sizes on pushes.
|
||||
// guessing too large (Tested by guessing x4 too large) may cost a bit more of memory
|
||||
// but would still have a faster runtime by avoiding (All but one) array resizing.
|
||||
let guessedNumberOfTokens = this.hasCustom
|
||||
? 0 // will break custom token pattern APIs the matchedTokens array will contain undefined elements.
|
||||
: Math.floor(text.length / 10)
|
||||
let matchedTokens = new Array(guessedNumberOfTokens)
|
||||
let errors: ILexingError[] = []
|
||||
let line = this.trackStartLines ? 1 : undefined
|
||||
let column = this.trackStartLines ? 1 : undefined
|
||||
let groups: any = cloneEmptyGroups(this.emptyGroups)
|
||||
let trackLines = this.trackStartLines
|
||||
const lineTerminatorPattern = this.config.lineTerminatorsPattern
|
||||
|
||||
let currModePatternsLength = 0
|
||||
let patternIdxToConfig = []
|
||||
let currCharCodeToPatternIdxToConfig = []
|
||||
|
||||
let modeStack = []
|
||||
|
||||
const emptyArray = []
|
||||
Object.freeze(emptyArray)
|
||||
let getPossiblePatterns = undefined
|
||||
|
||||
function getPossiblePatternsSlow() {
|
||||
return patternIdxToConfig
|
||||
}
|
||||
|
||||
function getPossiblePatternsOptimized(charCode) {
|
||||
const optimizedCharIdx = charCodeToOptimizedIndex(charCode)
|
||||
const possiblePatterns =
|
||||
currCharCodeToPatternIdxToConfig[optimizedCharIdx]
|
||||
if (possiblePatterns === undefined) {
|
||||
return emptyArray
|
||||
} else {
|
||||
return possiblePatterns
|
||||
}
|
||||
}
|
||||
|
||||
let pop_mode = (popToken) => {
|
||||
// TODO: perhaps avoid this error in the edge case there is no more input?
|
||||
if (
|
||||
modeStack.length === 1 &&
|
||||
// if we have both a POP_MODE and a PUSH_MODE this is in-fact a "transition"
|
||||
// So no error should occur.
|
||||
popToken.tokenType.PUSH_MODE === undefined
|
||||
) {
|
||||
// if we try to pop the last mode there lexer will no longer have ANY mode.
|
||||
// thus the pop is ignored, an error will be created and the lexer will continue parsing in the previous mode.
|
||||
let msg = this.config.errorMessageProvider.buildUnableToPopLexerModeMessage(
|
||||
popToken
|
||||
)
|
||||
|
||||
errors.push({
|
||||
offset: popToken.startOffset,
|
||||
line:
|
||||
popToken.startLine !== undefined ? popToken.startLine : undefined,
|
||||
column:
|
||||
popToken.startColumn !== undefined
|
||||
? popToken.startColumn
|
||||
: undefined,
|
||||
length: popToken.image.length,
|
||||
message: msg
|
||||
})
|
||||
} else {
|
||||
modeStack.pop()
|
||||
let newMode = last(modeStack)
|
||||
patternIdxToConfig = this.patternIdxToConfig[newMode]
|
||||
currCharCodeToPatternIdxToConfig = this.charCodeToPatternIdxToConfig[
|
||||
newMode
|
||||
]
|
||||
currModePatternsLength = patternIdxToConfig.length
|
||||
const modeCanBeOptimized =
|
||||
this.canModeBeOptimized[newMode] && this.config.safeMode === false
|
||||
|
||||
if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
|
||||
getPossiblePatterns = getPossiblePatternsOptimized
|
||||
} else {
|
||||
getPossiblePatterns = getPossiblePatternsSlow
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function push_mode(newMode) {
|
||||
modeStack.push(newMode)
|
||||
currCharCodeToPatternIdxToConfig = this.charCodeToPatternIdxToConfig[
|
||||
newMode
|
||||
]
|
||||
|
||||
patternIdxToConfig = this.patternIdxToConfig[newMode]
|
||||
currModePatternsLength = patternIdxToConfig.length
|
||||
|
||||
currModePatternsLength = patternIdxToConfig.length
|
||||
const modeCanBeOptimized =
|
||||
this.canModeBeOptimized[newMode] && this.config.safeMode === false
|
||||
|
||||
if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
|
||||
getPossiblePatterns = getPossiblePatternsOptimized
|
||||
} else {
|
||||
getPossiblePatterns = getPossiblePatternsSlow
|
||||
}
|
||||
}
|
||||
|
||||
// this pattern seems to avoid a V8 de-optimization, although that de-optimization does not
|
||||
// seem to matter performance wise.
|
||||
push_mode.call(this, initialMode)
|
||||
|
||||
let currConfig
|
||||
|
||||
while (offset < orgLength) {
|
||||
matchedImage = null
|
||||
|
||||
let nextCharCode = orgText.charCodeAt(offset)
|
||||
const chosenPatternIdxToConfig = getPossiblePatterns(nextCharCode)
|
||||
let chosenPatternsLength = chosenPatternIdxToConfig.length
|
||||
|
||||
for (i = 0; i < chosenPatternsLength; i++) {
|
||||
currConfig = chosenPatternIdxToConfig[i]
|
||||
let currPattern = currConfig.pattern
|
||||
payload = null
|
||||
|
||||
// manually in-lined because > 600 chars won't be in-lined in V8
|
||||
let singleCharCode = currConfig.short
|
||||
if (singleCharCode !== false) {
|
||||
if (nextCharCode === singleCharCode) {
|
||||
// single character string
|
||||
matchedImage = currPattern
|
||||
}
|
||||
} else if (currConfig.isCustom === true) {
|
||||
match = currPattern.exec(orgText, offset, matchedTokens, groups)
|
||||
if (match !== null) {
|
||||
matchedImage = match[0]
|
||||
if (match.payload !== undefined) {
|
||||
payload = match.payload
|
||||
}
|
||||
} else {
|
||||
matchedImage = null
|
||||
}
|
||||
} else {
|
||||
this.updateLastIndex(currPattern, offset)
|
||||
matchedImage = this.match(currPattern, text, offset)
|
||||
}
|
||||
|
||||
if (matchedImage !== null) {
|
||||
// even though this pattern matched we must try a another longer alternative.
|
||||
// this can be used to prioritize keywords over identifiers
|
||||
longerAltIdx = currConfig.longerAlt
|
||||
if (longerAltIdx !== undefined) {
|
||||
// TODO: micro optimize, avoid extra prop access
|
||||
// by saving/linking longerAlt on the original config?
|
||||
let longerAltConfig = patternIdxToConfig[longerAltIdx]
|
||||
let longerAltPattern = longerAltConfig.pattern
|
||||
altPayload = null
|
||||
|
||||
// single Char can never be a longer alt so no need to test it.
|
||||
// manually in-lined because > 600 chars won't be in-lined in V8
|
||||
if (longerAltConfig.isCustom === true) {
|
||||
match = longerAltPattern.exec(
|
||||
orgText,
|
||||
offset,
|
||||
matchedTokens,
|
||||
groups
|
||||
)
|
||||
if (match !== null) {
|
||||
matchAltImage = match[0]
|
||||
if (match.payload !== undefined) {
|
||||
altPayload = match.payload
|
||||
}
|
||||
} else {
|
||||
matchAltImage = null
|
||||
}
|
||||
} else {
|
||||
this.updateLastIndex(longerAltPattern, offset)
|
||||
matchAltImage = this.match(longerAltPattern, text, offset)
|
||||
}
|
||||
|
||||
if (matchAltImage && matchAltImage.length > matchedImage.length) {
|
||||
matchedImage = matchAltImage
|
||||
payload = altPayload
|
||||
currConfig = longerAltConfig
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// successful match
|
||||
if (matchedImage !== null) {
|
||||
imageLength = matchedImage.length
|
||||
group = currConfig.group
|
||||
if (group !== undefined) {
|
||||
tokType = currConfig.tokenTypeIdx
|
||||
// TODO: "offset + imageLength" and the new column may be computed twice in case of "full" location information inside
|
||||
// createFullToken method
|
||||
newToken = this.createTokenInstance(
|
||||
matchedImage,
|
||||
offset,
|
||||
tokType,
|
||||
currConfig.tokenType,
|
||||
line,
|
||||
column,
|
||||
imageLength
|
||||
)
|
||||
|
||||
this.handlePayload(newToken, payload)
|
||||
|
||||
// TODO: optimize NOOP in case there are no special groups?
|
||||
if (group === false) {
|
||||
matchedTokensIndex = this.addToken(
|
||||
matchedTokens,
|
||||
matchedTokensIndex,
|
||||
newToken
|
||||
)
|
||||
} else {
|
||||
groups[group].push(newToken)
|
||||
}
|
||||
}
|
||||
text = this.chopInput(text, imageLength)
|
||||
offset = offset + imageLength
|
||||
|
||||
// TODO: with newlines the column may be assigned twice
|
||||
column = this.computeNewColumn(column, imageLength)
|
||||
|
||||
if (trackLines === true && currConfig.canLineTerminator === true) {
|
||||
let numOfLTsInMatch = 0
|
||||
let foundTerminator
|
||||
let lastLTEndOffset
|
||||
lineTerminatorPattern.lastIndex = 0
|
||||
do {
|
||||
foundTerminator = lineTerminatorPattern.test(matchedImage)
|
||||
if (foundTerminator === true) {
|
||||
lastLTEndOffset = lineTerminatorPattern.lastIndex - 1
|
||||
numOfLTsInMatch++
|
||||
}
|
||||
} while (foundTerminator === true)
|
||||
|
||||
if (numOfLTsInMatch !== 0) {
|
||||
line = line + numOfLTsInMatch
|
||||
column = imageLength - lastLTEndOffset
|
||||
this.updateTokenEndLineColumnLocation(
|
||||
newToken,
|
||||
group,
|
||||
lastLTEndOffset,
|
||||
numOfLTsInMatch,
|
||||
line,
|
||||
column,
|
||||
imageLength
|
||||
)
|
||||
}
|
||||
}
|
||||
// will be NOOP if no modes present
|
||||
this.handleModes(currConfig, pop_mode, push_mode, newToken)
|
||||
} else {
|
||||
// error recovery, drop characters until we identify a valid token's start point
|
||||
let errorStartOffset = offset
|
||||
let errorLine = line
|
||||
let errorColumn = column
|
||||
let foundResyncPoint = false
|
||||
while (!foundResyncPoint && offset < orgLength) {
|
||||
// drop chars until we succeed in matching something
|
||||
droppedChar = orgText.charCodeAt(offset)
|
||||
// Identity Func (when sticky flag is enabled)
|
||||
text = this.chopInput(text, 1)
|
||||
offset++
|
||||
for (j = 0; j < currModePatternsLength; j++) {
|
||||
let currConfig = patternIdxToConfig[j]
|
||||
let currPattern = currConfig.pattern
|
||||
|
||||
// manually in-lined because > 600 chars won't be in-lined in V8
|
||||
let singleCharCode = currConfig.short
|
||||
if (singleCharCode !== false) {
|
||||
if (orgText.charCodeAt(offset) === singleCharCode) {
|
||||
// single character string
|
||||
foundResyncPoint = true
|
||||
}
|
||||
} else if (currConfig.isCustom === true) {
|
||||
foundResyncPoint =
|
||||
currPattern.exec(orgText, offset, matchedTokens, groups) !==
|
||||
null
|
||||
} else {
|
||||
this.updateLastIndex(currPattern, offset)
|
||||
foundResyncPoint = currPattern.exec(text) !== null
|
||||
}
|
||||
|
||||
if (foundResyncPoint === true) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errLength = offset - errorStartOffset
|
||||
// at this point we either re-synced or reached the end of the input text
|
||||
msg = this.config.errorMessageProvider.buildUnexpectedCharactersMessage(
|
||||
orgText,
|
||||
errorStartOffset,
|
||||
errLength,
|
||||
errorLine,
|
||||
errorColumn
|
||||
)
|
||||
errors.push({
|
||||
offset: errorStartOffset,
|
||||
line: errorLine,
|
||||
column: errorColumn,
|
||||
length: errLength,
|
||||
message: msg
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// if we do have custom patterns which push directly into the
|
||||
// TODO: custom tokens should not push directly??
|
||||
if (!this.hasCustom) {
|
||||
// if we guessed a too large size for the tokens array this will shrink it to the right size.
|
||||
matchedTokens.length = matchedTokensIndex
|
||||
}
|
||||
|
||||
return {
|
||||
tokens: matchedTokens,
|
||||
groups: groups,
|
||||
errors: errors
|
||||
}
|
||||
}
|
||||
|
||||
private handleModes(config, pop_mode, push_mode, newToken) {
|
||||
if (config.pop === true) {
|
||||
// need to save the PUSH_MODE property as if the mode is popped
|
||||
// patternIdxToPopMode is updated to reflect the new mode after popping the stack
|
||||
let pushMode = config.push
|
||||
pop_mode(newToken)
|
||||
if (pushMode !== undefined) {
|
||||
push_mode.call(this, pushMode)
|
||||
}
|
||||
} else if (config.push !== undefined) {
|
||||
push_mode.call(this, config.push)
|
||||
}
|
||||
}
|
||||
|
||||
private chopInput(text, length): string {
|
||||
return text.substring(length)
|
||||
}
|
||||
|
||||
private updateLastIndex(regExp, newLastIndex): void {
|
||||
regExp.lastIndex = newLastIndex
|
||||
}
|
||||
|
||||
// TODO: decrease this under 600 characters? inspect stripping comments option in TSC compiler
|
||||
private updateTokenEndLineColumnLocation(
|
||||
newToken,
|
||||
group,
|
||||
lastLTIdx,
|
||||
numOfLTsInMatch,
|
||||
line,
|
||||
column,
|
||||
imageLength
|
||||
): void {
|
||||
let lastCharIsLT, fixForEndingInLT
|
||||
if (group !== undefined) {
|
||||
// a none skipped multi line Token, need to update endLine/endColumn
|
||||
lastCharIsLT = lastLTIdx === imageLength - 1
|
||||
fixForEndingInLT = lastCharIsLT ? -1 : 0
|
||||
if (!(numOfLTsInMatch === 1 && lastCharIsLT === true)) {
|
||||
// if a token ends in a LT that last LT only affects the line numbering of following Tokens
|
||||
newToken.endLine = line + fixForEndingInLT
|
||||
// the last LT in a token does not affect the endColumn either as the [columnStart ... columnEnd)
|
||||
// inclusive to exclusive range.
|
||||
newToken.endColumn = column - 1 + -fixForEndingInLT
|
||||
}
|
||||
// else single LT in the last character of a token, no need to modify the endLine/EndColumn
|
||||
}
|
||||
}
|
||||
|
||||
private computeNewColumn(oldColumn, imageLength) {
|
||||
return oldColumn + imageLength
|
||||
}
|
||||
|
||||
// Place holder, will be replaced by the correct variant according to the locationTracking option at runtime.
|
||||
/* istanbul ignore next - place holder */
|
||||
private createTokenInstance(...args: any[]): IToken {
|
||||
return null
|
||||
}
|
||||
|
||||
private createOffsetOnlyToken(image, startOffset, tokenTypeIdx, tokenType) {
|
||||
return {
|
||||
image,
|
||||
startOffset,
|
||||
tokenTypeIdx,
|
||||
tokenType
|
||||
}
|
||||
}
|
||||
|
||||
private createStartOnlyToken(
|
||||
image,
|
||||
startOffset,
|
||||
tokenTypeIdx,
|
||||
tokenType,
|
||||
startLine,
|
||||
startColumn
|
||||
) {
|
||||
return {
|
||||
image,
|
||||
startOffset,
|
||||
startLine,
|
||||
startColumn,
|
||||
tokenTypeIdx,
|
||||
tokenType
|
||||
}
|
||||
}
|
||||
|
||||
private createFullToken(
|
||||
image,
|
||||
startOffset,
|
||||
tokenTypeIdx,
|
||||
tokenType,
|
||||
startLine,
|
||||
startColumn,
|
||||
imageLength
|
||||
) {
|
||||
return {
|
||||
image,
|
||||
startOffset,
|
||||
endOffset: startOffset + imageLength - 1,
|
||||
startLine,
|
||||
endLine: startLine,
|
||||
startColumn,
|
||||
endColumn: startColumn + imageLength - 1,
|
||||
tokenTypeIdx,
|
||||
tokenType
|
||||
}
|
||||
}
|
||||
|
||||
// Place holder, will be replaced by the correct variant according to the locationTracking option at runtime.
|
||||
/* istanbul ignore next - place holder */
|
||||
private addToken(tokenVector, index, tokenToAdd): number {
|
||||
return 666
|
||||
}
|
||||
|
||||
private addTokenUsingPush(tokenVector, index, tokenToAdd): number {
|
||||
tokenVector.push(tokenToAdd)
|
||||
return index
|
||||
}
|
||||
|
||||
private addTokenUsingMemberAccess(tokenVector, index, tokenToAdd): number {
|
||||
tokenVector[index] = tokenToAdd
|
||||
index++
|
||||
return index
|
||||
}
|
||||
|
||||
// Place holder, will be replaced by the correct variant according to the hasCustom flag option at runtime.
|
||||
/* istanbul ignore next - place holder */
|
||||
private handlePayload(token: IToken, payload: any): void {}
|
||||
|
||||
private handlePayloadNoCustom(token: IToken, payload: any): void {}
|
||||
|
||||
private handlePayloadWithCustom(token: IToken, payload: any): void {
|
||||
if (payload !== null) {
|
||||
token.payload = payload
|
||||
}
|
||||
}
|
||||
|
||||
/* istanbul ignore next - place holder to be replaced with chosen alternative at runtime */
|
||||
private match(pattern: RegExp, text: string, offset?: number): string {
|
||||
return null
|
||||
}
|
||||
|
||||
private matchWithTest(pattern: RegExp, text: string, offset: number): string {
|
||||
let found = pattern.test(text)
|
||||
if (found === true) {
|
||||
return text.substring(offset, pattern.lastIndex)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private matchWithExec(pattern, text): string {
|
||||
let regExpArray = pattern.exec(text)
|
||||
return regExpArray !== null ? regExpArray[0] : regExpArray
|
||||
}
|
||||
|
||||
// Duplicated from the parser's perf trace trait to allow future extraction
|
||||
// of the lexer to a separate package.
|
||||
TRACE_INIT<T>(phaseDesc: string, phaseImpl: () => T): T {
|
||||
// No need to optimize this using NOOP pattern because
|
||||
// It is not called in a hot spot...
|
||||
if (this.traceInitPerf === true) {
|
||||
this.traceInitIndent++
|
||||
const indent = new Array(this.traceInitIndent + 1).join("\t")
|
||||
if (this.traceInitIndent < this.traceInitMaxIdent) {
|
||||
console.log(`${indent}--> <${phaseDesc}>`)
|
||||
}
|
||||
const { time, value } = timer(phaseImpl)
|
||||
/* istanbul ignore next - Difficult to reproduce specific performance behavior (>10ms) in tests */
|
||||
const traceMethod = time > 10 ? console.warn : console.log
|
||||
if (this.traceInitIndent < this.traceInitMaxIdent) {
|
||||
traceMethod(`${indent}<-- <${phaseDesc}> time: ${time}ms`)
|
||||
}
|
||||
this.traceInitIndent--
|
||||
return value
|
||||
} else {
|
||||
return phaseImpl()
|
||||
}
|
||||
}
|
||||
}
|
||||
307
node_modules/chevrotain/src/scan/reg_exp.ts
generated
vendored
Normal file
307
node_modules/chevrotain/src/scan/reg_exp.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,307 @@
|
|||
import { VERSION, BaseRegExpVisitor } from "regexp-to-ast"
|
||||
import {
|
||||
flatten,
|
||||
map,
|
||||
forEach,
|
||||
contains,
|
||||
PRINT_ERROR,
|
||||
PRINT_WARNING,
|
||||
find,
|
||||
isArray,
|
||||
every,
|
||||
values
|
||||
} from "../utils/utils"
|
||||
import { getRegExpAst } from "./reg_exp_parser"
|
||||
import { charCodeToOptimizedIndex, minOptimizationVal } from "./lexer"
|
||||
|
||||
const complementErrorMessage =
|
||||
"Complement Sets are not supported for first char optimization"
|
||||
export const failedOptimizationPrefixMsg =
|
||||
'Unable to use "first char" lexer optimizations:\n'
|
||||
|
||||
export function getOptimizedStartCodesIndices(
|
||||
regExp: RegExp,
|
||||
ensureOptimizations = false
|
||||
): number[] {
|
||||
try {
|
||||
const ast = getRegExpAst(regExp)
|
||||
const firstChars = firstCharOptimizedIndices(
|
||||
ast.value,
|
||||
{},
|
||||
ast.flags.ignoreCase
|
||||
)
|
||||
return firstChars
|
||||
} catch (e) {
|
||||
/* istanbul ignore next */
|
||||
// Testing this relies on the regexp-to-ast library having a bug... */
|
||||
// TODO: only the else branch needs to be ignored, try to fix with newer prettier / tsc
|
||||
if (e.message === complementErrorMessage) {
|
||||
if (ensureOptimizations) {
|
||||
PRINT_WARNING(
|
||||
`${failedOptimizationPrefixMsg}` +
|
||||
`\tUnable to optimize: < ${regExp.toString()} >\n` +
|
||||
"\tComplement Sets cannot be automatically optimized.\n" +
|
||||
"\tThis will disable the lexer's first char optimizations.\n" +
|
||||
"\tSee: https://sap.github.io/chevrotain/docs/guide/resolving_lexer_errors.html#COMPLEMENT for details."
|
||||
)
|
||||
}
|
||||
} else {
|
||||
let msgSuffix = ""
|
||||
if (ensureOptimizations) {
|
||||
msgSuffix =
|
||||
"\n\tThis will disable the lexer's first char optimizations.\n" +
|
||||
"\tSee: https://sap.github.io/chevrotain/docs/guide/resolving_lexer_errors.html#REGEXP_PARSING for details."
|
||||
}
|
||||
PRINT_ERROR(
|
||||
`${failedOptimizationPrefixMsg}\n` +
|
||||
`\tFailed parsing: < ${regExp.toString()} >\n` +
|
||||
`\tUsing the regexp-to-ast library version: ${VERSION}\n` +
|
||||
"\tPlease open an issue at: https://github.com/bd82/regexp-to-ast/issues" +
|
||||
msgSuffix
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return []
|
||||
}
|
||||
|
||||
export function firstCharOptimizedIndices(ast, result, ignoreCase): number[] {
|
||||
switch (ast.type) {
|
||||
case "Disjunction":
|
||||
for (let i = 0; i < ast.value.length; i++) {
|
||||
firstCharOptimizedIndices(ast.value[i], result, ignoreCase)
|
||||
}
|
||||
break
|
||||
case "Alternative":
|
||||
const terms = ast.value
|
||||
for (let i = 0; i < terms.length; i++) {
|
||||
const term = terms[i]
|
||||
|
||||
// skip terms that cannot effect the first char results
|
||||
switch (term.type) {
|
||||
case "EndAnchor":
|
||||
// A group back reference cannot affect potential starting char.
|
||||
// because if a back reference is the first production than automatically
|
||||
// the group being referenced has had to come BEFORE so its codes have already been added
|
||||
case "GroupBackReference":
|
||||
// assertions do not affect potential starting codes
|
||||
case "Lookahead":
|
||||
case "NegativeLookahead":
|
||||
case "StartAnchor":
|
||||
case "WordBoundary":
|
||||
case "NonWordBoundary":
|
||||
continue
|
||||
}
|
||||
|
||||
const atom = term
|
||||
switch (atom.type) {
|
||||
case "Character":
|
||||
addOptimizedIdxToResult(atom.value, result, ignoreCase)
|
||||
break
|
||||
case "Set":
|
||||
if (atom.complement === true) {
|
||||
throw Error(complementErrorMessage)
|
||||
}
|
||||
forEach(atom.value, (code) => {
|
||||
if (typeof code === "number") {
|
||||
addOptimizedIdxToResult(code, result, ignoreCase)
|
||||
} else {
|
||||
// range
|
||||
const range = code
|
||||
// cannot optimize when ignoreCase is
|
||||
if (ignoreCase === true) {
|
||||
for (
|
||||
let rangeCode = range.from;
|
||||
rangeCode <= range.to;
|
||||
rangeCode++
|
||||
) {
|
||||
addOptimizedIdxToResult(rangeCode, result, ignoreCase)
|
||||
}
|
||||
}
|
||||
// Optimization (2 orders of magnitude less work for very large ranges)
|
||||
else {
|
||||
// handle unoptimized values
|
||||
for (
|
||||
let rangeCode = range.from;
|
||||
rangeCode <= range.to && rangeCode < minOptimizationVal;
|
||||
rangeCode++
|
||||
) {
|
||||
addOptimizedIdxToResult(rangeCode, result, ignoreCase)
|
||||
}
|
||||
|
||||
// Less common charCode where we optimize for faster init time, by using larger "buckets"
|
||||
if (range.to >= minOptimizationVal) {
|
||||
const minUnOptVal =
|
||||
range.from >= minOptimizationVal
|
||||
? range.from
|
||||
: minOptimizationVal
|
||||
const maxUnOptVal = range.to
|
||||
const minOptIdx = charCodeToOptimizedIndex(minUnOptVal)
|
||||
const maxOptIdx = charCodeToOptimizedIndex(maxUnOptVal)
|
||||
|
||||
for (
|
||||
let currOptIdx = minOptIdx;
|
||||
currOptIdx <= maxOptIdx;
|
||||
currOptIdx++
|
||||
) {
|
||||
result[currOptIdx] = currOptIdx
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
break
|
||||
case "Group":
|
||||
firstCharOptimizedIndices(atom.value, result, ignoreCase)
|
||||
break
|
||||
/* istanbul ignore next */
|
||||
default:
|
||||
throw Error("Non Exhaustive Match")
|
||||
}
|
||||
|
||||
// reached a mandatory production, no more **start** codes can be found on this alternative
|
||||
const isOptionalQuantifier =
|
||||
atom.quantifier !== undefined && atom.quantifier.atLeast === 0
|
||||
if (
|
||||
// A group may be optional due to empty contents /(?:)/
|
||||
// or if everything inside it is optional /((a)?)/
|
||||
(atom.type === "Group" && isWholeOptional(atom) === false) ||
|
||||
// If this term is not a group it may only be optional if it has an optional quantifier
|
||||
(atom.type !== "Group" && isOptionalQuantifier === false)
|
||||
) {
|
||||
break
|
||||
}
|
||||
}
|
||||
break
|
||||
/* istanbul ignore next */
|
||||
default:
|
||||
throw Error("non exhaustive match!")
|
||||
}
|
||||
|
||||
// console.log(Object.keys(result).length)
|
||||
return values(result)
|
||||
}
|
||||
|
||||
function addOptimizedIdxToResult(
|
||||
code: number,
|
||||
result: number[],
|
||||
ignoreCase: boolean
|
||||
) {
|
||||
const optimizedCharIdx = charCodeToOptimizedIndex(code)
|
||||
result[optimizedCharIdx] = optimizedCharIdx
|
||||
|
||||
if (ignoreCase === true) {
|
||||
handleIgnoreCase(code, result)
|
||||
}
|
||||
}
|
||||
|
||||
function handleIgnoreCase(code: number, result: number[]) {
|
||||
const char = String.fromCharCode(code)
|
||||
const upperChar = char.toUpperCase()
|
||||
/* istanbul ignore else */
|
||||
if (upperChar !== char) {
|
||||
const optimizedCharIdx = charCodeToOptimizedIndex(upperChar.charCodeAt(0))
|
||||
result[optimizedCharIdx] = optimizedCharIdx
|
||||
} else {
|
||||
const lowerChar = char.toLowerCase()
|
||||
if (lowerChar !== char) {
|
||||
const optimizedCharIdx = charCodeToOptimizedIndex(lowerChar.charCodeAt(0))
|
||||
result[optimizedCharIdx] = optimizedCharIdx
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function findCode(setNode, targetCharCodes) {
|
||||
return find(setNode.value, (codeOrRange) => {
|
||||
if (typeof codeOrRange === "number") {
|
||||
return contains(targetCharCodes, codeOrRange)
|
||||
} else {
|
||||
// range
|
||||
const range = <any>codeOrRange
|
||||
return (
|
||||
find(
|
||||
targetCharCodes,
|
||||
(targetCode) => range.from <= targetCode && targetCode <= range.to
|
||||
) !== undefined
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function isWholeOptional(ast) {
|
||||
if (ast.quantifier && ast.quantifier.atLeast === 0) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (!ast.value) {
|
||||
return false
|
||||
}
|
||||
|
||||
return isArray(ast.value)
|
||||
? every(ast.value, isWholeOptional)
|
||||
: isWholeOptional(ast.value)
|
||||
}
|
||||
|
||||
class CharCodeFinder extends BaseRegExpVisitor {
|
||||
found: boolean = false
|
||||
|
||||
constructor(private targetCharCodes: number[]) {
|
||||
super()
|
||||
}
|
||||
|
||||
visitChildren(node) {
|
||||
// No need to keep looking...
|
||||
if (this.found === true) {
|
||||
return
|
||||
}
|
||||
|
||||
// switch lookaheads as they do not actually consume any characters thus
|
||||
// finding a charCode at lookahead context does not mean that regexp can actually contain it in a match.
|
||||
switch (node.type) {
|
||||
case "Lookahead":
|
||||
this.visitLookahead(node)
|
||||
return
|
||||
case "NegativeLookahead":
|
||||
this.visitNegativeLookahead(node)
|
||||
return
|
||||
}
|
||||
|
||||
super.visitChildren(node)
|
||||
}
|
||||
|
||||
visitCharacter(node) {
|
||||
if (contains(this.targetCharCodes, node.value)) {
|
||||
this.found = true
|
||||
}
|
||||
}
|
||||
|
||||
visitSet(node) {
|
||||
if (node.complement) {
|
||||
if (findCode(node, this.targetCharCodes) === undefined) {
|
||||
this.found = true
|
||||
}
|
||||
} else {
|
||||
if (findCode(node, this.targetCharCodes) !== undefined) {
|
||||
this.found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function canMatchCharCode(
|
||||
charCodes: number[],
|
||||
pattern: RegExp | string
|
||||
) {
|
||||
if (pattern instanceof RegExp) {
|
||||
const ast = getRegExpAst(pattern)
|
||||
const charCodeFinder = new CharCodeFinder(charCodes)
|
||||
charCodeFinder.visit(ast)
|
||||
return charCodeFinder.found
|
||||
} else {
|
||||
return (
|
||||
find(<any>pattern, (char) => {
|
||||
return contains(charCodes, (<string>char).charCodeAt(0))
|
||||
}) !== undefined
|
||||
)
|
||||
}
|
||||
}
|
||||
19
node_modules/chevrotain/src/scan/reg_exp_parser.ts
generated
vendored
Normal file
19
node_modules/chevrotain/src/scan/reg_exp_parser.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { RegExpParser, RegExpPattern } from "regexp-to-ast"
|
||||
|
||||
let regExpAstCache = {}
|
||||
const regExpParser = new RegExpParser()
|
||||
|
||||
export function getRegExpAst(regExp: RegExp): RegExpPattern {
|
||||
const regExpStr = regExp.toString()
|
||||
if (regExpAstCache.hasOwnProperty(regExpStr)) {
|
||||
return regExpAstCache[regExpStr]
|
||||
} else {
|
||||
const regExpAst = regExpParser.pattern(regExpStr)
|
||||
regExpAstCache[regExpStr] = regExpAst
|
||||
return regExpAst
|
||||
}
|
||||
}
|
||||
|
||||
export function clearRegExpParserCache() {
|
||||
regExpAstCache = {}
|
||||
}
|
||||
159
node_modules/chevrotain/src/scan/tokens.ts
generated
vendored
Normal file
159
node_modules/chevrotain/src/scan/tokens.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
import {
|
||||
cloneArr,
|
||||
compact,
|
||||
contains,
|
||||
difference,
|
||||
flatten,
|
||||
forEach,
|
||||
has,
|
||||
isArray,
|
||||
isEmpty,
|
||||
map
|
||||
} from "../utils/utils"
|
||||
import { TokenType } from "../../api"
|
||||
|
||||
export function tokenStructuredMatcher(tokInstance, tokConstructor) {
|
||||
const instanceType = tokInstance.tokenTypeIdx
|
||||
if (instanceType === tokConstructor.tokenTypeIdx) {
|
||||
return true
|
||||
} else {
|
||||
return (
|
||||
tokConstructor.isParent === true &&
|
||||
tokConstructor.categoryMatchesMap[instanceType] === true
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Optimized tokenMatcher in case our grammar does not use token categories
|
||||
// Being so tiny it is much more likely to be in-lined and this avoid the function call overhead
|
||||
export function tokenStructuredMatcherNoCategories(token, tokType) {
|
||||
return token.tokenTypeIdx === tokType.tokenTypeIdx
|
||||
}
|
||||
|
||||
export let tokenShortNameIdx = 1
|
||||
export const tokenIdxToClass = {}
|
||||
|
||||
export function augmentTokenTypes(tokenTypes: TokenType[]): void {
|
||||
// collect the parent Token Types as well.
|
||||
let tokenTypesAndParents = expandCategories(tokenTypes)
|
||||
|
||||
// add required tokenType and categoryMatches properties
|
||||
assignTokenDefaultProps(tokenTypesAndParents)
|
||||
|
||||
// fill up the categoryMatches
|
||||
assignCategoriesMapProp(tokenTypesAndParents)
|
||||
assignCategoriesTokensProp(tokenTypesAndParents)
|
||||
|
||||
forEach(tokenTypesAndParents, (tokType) => {
|
||||
tokType.isParent = tokType.categoryMatches.length > 0
|
||||
})
|
||||
}
|
||||
|
||||
export function expandCategories(tokenTypes: TokenType[]): TokenType[] {
|
||||
let result = cloneArr(tokenTypes)
|
||||
|
||||
let categories = tokenTypes
|
||||
let searching = true
|
||||
while (searching) {
|
||||
categories = compact(
|
||||
flatten(map(categories, (currTokType) => currTokType.CATEGORIES))
|
||||
)
|
||||
|
||||
let newCategories = difference(categories, result)
|
||||
|
||||
result = result.concat(newCategories)
|
||||
|
||||
if (isEmpty(newCategories)) {
|
||||
searching = false
|
||||
} else {
|
||||
categories = newCategories
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function assignTokenDefaultProps(tokenTypes: TokenType[]): void {
|
||||
forEach(tokenTypes, (currTokType) => {
|
||||
if (!hasShortKeyProperty(currTokType)) {
|
||||
tokenIdxToClass[tokenShortNameIdx] = currTokType
|
||||
;(<any>currTokType).tokenTypeIdx = tokenShortNameIdx++
|
||||
}
|
||||
|
||||
// CATEGORIES? : TokenType | TokenType[]
|
||||
if (
|
||||
hasCategoriesProperty(currTokType) &&
|
||||
!isArray(currTokType.CATEGORIES)
|
||||
// &&
|
||||
// !isUndefined(currTokType.CATEGORIES.PATTERN)
|
||||
) {
|
||||
currTokType.CATEGORIES = [currTokType.CATEGORIES]
|
||||
}
|
||||
|
||||
if (!hasCategoriesProperty(currTokType)) {
|
||||
currTokType.CATEGORIES = []
|
||||
}
|
||||
|
||||
if (!hasExtendingTokensTypesProperty(currTokType)) {
|
||||
currTokType.categoryMatches = []
|
||||
}
|
||||
|
||||
if (!hasExtendingTokensTypesMapProperty(currTokType)) {
|
||||
currTokType.categoryMatchesMap = {}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function assignCategoriesTokensProp(tokenTypes: TokenType[]): void {
|
||||
forEach(tokenTypes, (currTokType) => {
|
||||
// avoid duplications
|
||||
currTokType.categoryMatches = []
|
||||
forEach(currTokType.categoryMatchesMap, (val, key) => {
|
||||
currTokType.categoryMatches.push(tokenIdxToClass[key].tokenTypeIdx)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export function assignCategoriesMapProp(tokenTypes: TokenType[]): void {
|
||||
forEach(tokenTypes, (currTokType) => {
|
||||
singleAssignCategoriesToksMap([], currTokType)
|
||||
})
|
||||
}
|
||||
|
||||
export function singleAssignCategoriesToksMap(
|
||||
path: TokenType[],
|
||||
nextNode: TokenType
|
||||
): void {
|
||||
forEach(path, (pathNode) => {
|
||||
nextNode.categoryMatchesMap[pathNode.tokenTypeIdx] = true
|
||||
})
|
||||
|
||||
forEach(nextNode.CATEGORIES, (nextCategory) => {
|
||||
const newPath = path.concat(nextNode)
|
||||
// avoids infinite loops due to cyclic categories.
|
||||
if (!contains(newPath, nextCategory)) {
|
||||
singleAssignCategoriesToksMap(newPath, nextCategory)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function hasShortKeyProperty(tokType: TokenType): boolean {
|
||||
return has(tokType, "tokenTypeIdx")
|
||||
}
|
||||
|
||||
export function hasCategoriesProperty(tokType: TokenType): boolean {
|
||||
return has(tokType, "CATEGORIES")
|
||||
}
|
||||
|
||||
export function hasExtendingTokensTypesProperty(tokType: TokenType): boolean {
|
||||
return has(tokType, "categoryMatches")
|
||||
}
|
||||
|
||||
export function hasExtendingTokensTypesMapProperty(
|
||||
tokType: TokenType
|
||||
): boolean {
|
||||
return has(tokType, "categoryMatchesMap")
|
||||
}
|
||||
|
||||
export function isTokenType(tokType: TokenType): boolean {
|
||||
return has(tokType, "tokenTypeIdx")
|
||||
}
|
||||
1
node_modules/chevrotain/src/scan/tokens_constants.ts
generated
vendored
Normal file
1
node_modules/chevrotain/src/scan/tokens_constants.ts
generated
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
export const EOF_TOKEN_TYPE = 1
|
||||
119
node_modules/chevrotain/src/scan/tokens_public.ts
generated
vendored
Normal file
119
node_modules/chevrotain/src/scan/tokens_public.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import { has, isString, isUndefined } from "../utils/utils"
|
||||
import { Lexer } from "./lexer_public"
|
||||
import { augmentTokenTypes, tokenStructuredMatcher } from "./tokens"
|
||||
import { IToken, ITokenConfig, TokenType } from "../../api"
|
||||
|
||||
export function tokenLabel(tokType: TokenType): string {
|
||||
if (hasTokenLabel(tokType)) {
|
||||
return tokType.LABEL
|
||||
} else {
|
||||
return tokType.name
|
||||
}
|
||||
}
|
||||
|
||||
export function tokenName(tokType: TokenType): string {
|
||||
return tokType.name
|
||||
}
|
||||
|
||||
export function hasTokenLabel(obj: TokenType): boolean {
|
||||
return isString((<any>obj).LABEL) && (<any>obj).LABEL !== ""
|
||||
}
|
||||
|
||||
const PARENT = "parent"
|
||||
const CATEGORIES = "categories"
|
||||
const LABEL = "label"
|
||||
const GROUP = "group"
|
||||
const PUSH_MODE = "push_mode"
|
||||
const POP_MODE = "pop_mode"
|
||||
const LONGER_ALT = "longer_alt"
|
||||
const LINE_BREAKS = "line_breaks"
|
||||
const START_CHARS_HINT = "start_chars_hint"
|
||||
|
||||
export function createToken(config: ITokenConfig): TokenType {
|
||||
return createTokenInternal(config)
|
||||
}
|
||||
|
||||
function createTokenInternal(config: ITokenConfig): TokenType {
|
||||
let pattern = config.pattern
|
||||
|
||||
let tokenType: TokenType = <any>{}
|
||||
tokenType.name = config.name
|
||||
|
||||
if (!isUndefined(pattern)) {
|
||||
tokenType.PATTERN = pattern
|
||||
}
|
||||
|
||||
if (has(config, PARENT)) {
|
||||
throw (
|
||||
"The parent property is no longer supported.\n" +
|
||||
"See: https://github.com/SAP/chevrotain/issues/564#issuecomment-349062346 for details."
|
||||
)
|
||||
}
|
||||
|
||||
if (has(config, CATEGORIES)) {
|
||||
// casting to ANY as this will be fixed inside `augmentTokenTypes``
|
||||
tokenType.CATEGORIES = <any>config[CATEGORIES]
|
||||
}
|
||||
|
||||
augmentTokenTypes([tokenType])
|
||||
|
||||
if (has(config, LABEL)) {
|
||||
tokenType.LABEL = config[LABEL]
|
||||
}
|
||||
|
||||
if (has(config, GROUP)) {
|
||||
tokenType.GROUP = config[GROUP]
|
||||
}
|
||||
|
||||
if (has(config, POP_MODE)) {
|
||||
tokenType.POP_MODE = config[POP_MODE]
|
||||
}
|
||||
|
||||
if (has(config, PUSH_MODE)) {
|
||||
tokenType.PUSH_MODE = config[PUSH_MODE]
|
||||
}
|
||||
|
||||
if (has(config, LONGER_ALT)) {
|
||||
tokenType.LONGER_ALT = config[LONGER_ALT]
|
||||
}
|
||||
|
||||
if (has(config, LINE_BREAKS)) {
|
||||
tokenType.LINE_BREAKS = config[LINE_BREAKS]
|
||||
}
|
||||
|
||||
if (has(config, START_CHARS_HINT)) {
|
||||
tokenType.START_CHARS_HINT = config[START_CHARS_HINT]
|
||||
}
|
||||
|
||||
return tokenType
|
||||
}
|
||||
|
||||
export const EOF = createToken({ name: "EOF", pattern: Lexer.NA })
|
||||
augmentTokenTypes([EOF])
|
||||
|
||||
export function createTokenInstance(
|
||||
tokType: TokenType,
|
||||
image: string,
|
||||
startOffset: number,
|
||||
endOffset: number,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
startColumn: number,
|
||||
endColumn: number
|
||||
): IToken {
|
||||
return {
|
||||
image,
|
||||
startOffset,
|
||||
endOffset,
|
||||
startLine,
|
||||
endLine,
|
||||
startColumn,
|
||||
endColumn,
|
||||
tokenTypeIdx: (<any>tokType).tokenTypeIdx,
|
||||
tokenType: tokType
|
||||
}
|
||||
}
|
||||
|
||||
export function tokenMatcher(token: IToken, tokType: TokenType): boolean {
|
||||
return tokenStructuredMatcher(token, tokType)
|
||||
}
|
||||
46
node_modules/chevrotain/src/text/range.ts
generated
vendored
Normal file
46
node_modules/chevrotain/src/text/range.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
export interface IRange {
|
||||
start: number
|
||||
end: number
|
||||
|
||||
contains(num: number): boolean
|
||||
|
||||
containsRange(other: IRange): boolean
|
||||
|
||||
isContainedInRange(other: IRange): boolean
|
||||
|
||||
strictlyContainsRange(other: IRange): boolean
|
||||
|
||||
isStrictlyContainedInRange(other: IRange): boolean
|
||||
}
|
||||
|
||||
export class Range implements IRange {
|
||||
constructor(public start: number, public end: number) {
|
||||
if (!isValidRange(start, end)) {
|
||||
throw new Error("INVALID RANGE")
|
||||
}
|
||||
}
|
||||
|
||||
contains(num: number): boolean {
|
||||
return this.start <= num && this.end >= num
|
||||
}
|
||||
|
||||
containsRange(other: IRange): boolean {
|
||||
return this.start <= other.start && this.end >= other.end
|
||||
}
|
||||
|
||||
isContainedInRange(other: IRange): boolean {
|
||||
return other.containsRange(this)
|
||||
}
|
||||
|
||||
strictlyContainsRange(other: IRange): boolean {
|
||||
return this.start < other.start && this.end > other.end
|
||||
}
|
||||
|
||||
isStrictlyContainedInRange(other: IRange): boolean {
|
||||
return other.strictlyContainsRange(this)
|
||||
}
|
||||
}
|
||||
|
||||
export function isValidRange(start: number, end: number): boolean {
|
||||
return !(start < 0 || end < start)
|
||||
}
|
||||
491
node_modules/chevrotain/src/utils/utils.ts
generated
vendored
Normal file
491
node_modules/chevrotain/src/utils/utils.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,491 @@
|
|||
/*
|
||||
Utils using lodash style API. (not necessarily 100% compliant) for functional and other utils.
|
||||
These utils should replace usage of lodash in the production code base. not because they are any better...
|
||||
but for the purpose of being a dependency free library.
|
||||
|
||||
The hotspots in the code are already written in imperative style for performance reasons.
|
||||
so writing several dozen utils which may be slower than the original lodash, does not matter as much
|
||||
considering they will not be invoked in hotspots...
|
||||
*/
|
||||
|
||||
export function isEmpty(arr: any[]): boolean {
|
||||
return arr && arr.length === 0
|
||||
}
|
||||
|
||||
export function keys(obj: any): string[] {
|
||||
if (obj === undefined || obj === null) {
|
||||
return []
|
||||
}
|
||||
return Object.keys(obj)
|
||||
}
|
||||
|
||||
export function values(obj: any): any[] {
|
||||
let vals = []
|
||||
let keys = Object.keys(obj)
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
vals.push(obj[keys[i]])
|
||||
}
|
||||
return vals
|
||||
}
|
||||
|
||||
export function mapValues<I, O>(
|
||||
obj: Object,
|
||||
callback: (value: I, key?: string) => O
|
||||
): O[] {
|
||||
let result: O[] = []
|
||||
let objKeys = keys(obj)
|
||||
for (let idx = 0; idx < objKeys.length; idx++) {
|
||||
let currKey = objKeys[idx]
|
||||
result.push(callback.call(null, obj[currKey], currKey))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function map<I, O>(arr: I[], callback: (I, idx?: number) => O): O[] {
|
||||
let result: O[] = []
|
||||
for (let idx = 0; idx < arr.length; idx++) {
|
||||
result.push(callback.call(null, arr[idx], idx))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function flatten<T>(arr: any[]): T[] {
|
||||
let result = []
|
||||
|
||||
for (let idx = 0; idx < arr.length; idx++) {
|
||||
let currItem = arr[idx]
|
||||
if (Array.isArray(currItem)) {
|
||||
result = result.concat(flatten(currItem))
|
||||
} else {
|
||||
result.push(currItem)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function first<T>(arr: T[]): T {
|
||||
return isEmpty(arr) ? undefined : arr[0]
|
||||
}
|
||||
|
||||
export function last<T>(arr: T[]): T {
|
||||
let len = arr && arr.length
|
||||
return len ? arr[len - 1] : undefined
|
||||
}
|
||||
|
||||
export function forEach(collection: any, iteratorCallback: Function): void {
|
||||
/* istanbul ignore else */
|
||||
if (Array.isArray(collection)) {
|
||||
for (let i = 0; i < collection.length; i++) {
|
||||
iteratorCallback.call(null, collection[i], i)
|
||||
}
|
||||
} else if (isObject(collection)) {
|
||||
let colKeys = keys(collection)
|
||||
for (let i = 0; i < colKeys.length; i++) {
|
||||
let key = colKeys[i]
|
||||
let value = collection[key]
|
||||
iteratorCallback.call(null, value, key)
|
||||
}
|
||||
} else {
|
||||
throw Error("non exhaustive match")
|
||||
}
|
||||
}
|
||||
|
||||
export function isString(item: any): boolean {
|
||||
return typeof item === "string"
|
||||
}
|
||||
|
||||
export function isUndefined(item: any): boolean {
|
||||
return item === undefined
|
||||
}
|
||||
|
||||
export function isFunction(item: any): boolean {
|
||||
return item instanceof Function
|
||||
}
|
||||
|
||||
export function drop<T>(arr: T[], howMuch: number = 1): T[] {
|
||||
return arr.slice(howMuch, arr.length)
|
||||
}
|
||||
|
||||
export function dropRight<T>(arr: T[], howMuch: number = 1): T[] {
|
||||
return arr.slice(0, arr.length - howMuch)
|
||||
}
|
||||
|
||||
export function filter<T>(arr: T[], predicate: (T) => boolean): T[] {
|
||||
let result = []
|
||||
if (Array.isArray(arr)) {
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
let item = arr[i]
|
||||
if (predicate.call(null, item)) {
|
||||
result.push(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function reject<T>(arr: T[], predicate: (T) => boolean): T[] {
|
||||
return filter(arr, (item) => !predicate(item))
|
||||
}
|
||||
|
||||
export function pick(obj: Object, predicate: (item) => boolean) {
|
||||
let keys = Object.keys(obj)
|
||||
let result = {}
|
||||
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
let currKey = keys[i]
|
||||
let currItem = obj[currKey]
|
||||
if (predicate(currItem)) {
|
||||
result[currKey] = currItem
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export function has(obj: any, prop: string): boolean {
|
||||
if (isObject(obj)) {
|
||||
return obj.hasOwnProperty(prop)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
export function contains<T>(arr: T[], item): boolean {
|
||||
return find(arr, (currItem) => currItem === item) !== undefined ? true : false
|
||||
}
|
||||
|
||||
/**
|
||||
* shallow clone
|
||||
*/
|
||||
export function cloneArr<T>(arr: T[]): T[] {
|
||||
let newArr = []
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
newArr.push(arr[i])
|
||||
}
|
||||
return newArr
|
||||
}
|
||||
|
||||
/**
|
||||
* shallow clone
|
||||
*/
|
||||
export function cloneObj(obj: Object): any {
|
||||
let clonedObj = {}
|
||||
for (let key in obj) {
|
||||
/* istanbul ignore else */
|
||||
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
||||
clonedObj[key] = obj[key]
|
||||
}
|
||||
}
|
||||
return clonedObj
|
||||
}
|
||||
|
||||
export function find<T>(arr: T[], predicate: (item: T) => boolean): T {
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
let item = arr[i]
|
||||
if (predicate.call(null, item)) {
|
||||
return item
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function findAll<T>(arr: T[], predicate: (item: T) => boolean): T[] {
|
||||
let found = []
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
let item = arr[i]
|
||||
if (predicate.call(null, item)) {
|
||||
found.push(item)
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
export function reduce<T, A>(
|
||||
arrOrObj: Array<T> | Object,
|
||||
iterator: (result: A, item, idx?) => A,
|
||||
initial: A
|
||||
): A {
|
||||
const isArr = Array.isArray(arrOrObj)
|
||||
|
||||
let vals: T[] = isArr ? <Array<T>>arrOrObj : values(arrOrObj)
|
||||
let objKeys = isArr ? [] : keys(arrOrObj)
|
||||
|
||||
let accumulator = initial
|
||||
for (let i = 0; i < vals.length; i++) {
|
||||
accumulator = iterator.call(
|
||||
null,
|
||||
accumulator,
|
||||
vals[i],
|
||||
isArr ? i : objKeys[i]
|
||||
)
|
||||
}
|
||||
return accumulator
|
||||
}
|
||||
|
||||
export function compact<T>(arr: T[]): T[] {
|
||||
return reject(arr, (item) => item === null || item === undefined)
|
||||
}
|
||||
|
||||
export function uniq<T>(
|
||||
arr: T[],
|
||||
identity: (item: T) => any = (item) => item
|
||||
): T[] {
|
||||
let identities = []
|
||||
return reduce(
|
||||
arr,
|
||||
(result, currItem) => {
|
||||
let currIdentity = identity(currItem)
|
||||
if (contains(identities, currIdentity)) {
|
||||
return result
|
||||
} else {
|
||||
identities.push(currIdentity)
|
||||
return result.concat(currItem)
|
||||
}
|
||||
},
|
||||
[]
|
||||
)
|
||||
}
|
||||
|
||||
export function partial(func: Function, ...restArgs: any[]): Function {
|
||||
let firstArg = [null]
|
||||
let allArgs = firstArg.concat(restArgs)
|
||||
return Function.bind.apply(func, allArgs)
|
||||
}
|
||||
|
||||
export function isArray(obj: any): obj is any[] {
|
||||
return Array.isArray(obj)
|
||||
}
|
||||
|
||||
export function isRegExp(obj: any): obj is RegExp {
|
||||
return obj instanceof RegExp
|
||||
}
|
||||
|
||||
export function isObject(obj: any): obj is Object {
|
||||
return obj instanceof Object
|
||||
}
|
||||
|
||||
export function every<T>(
|
||||
arr: T[],
|
||||
predicate: (item: T, idx?) => boolean
|
||||
): boolean {
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
if (!predicate(arr[i], i)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
export function difference<T>(arr: T[], values: T[]): T[] {
|
||||
return reject(arr, (item) => contains(values, item))
|
||||
}
|
||||
|
||||
export function some<T>(arr: T[], predicate: (item: T) => boolean): boolean {
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
if (predicate(arr[i])) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
export function indexOf<T>(arr: T[], value: T): number {
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
if (arr[i] === value) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
export function sortBy<T>(arr: T[], orderFunc: (item: T) => number): T[] {
|
||||
let result = cloneArr(arr)
|
||||
result.sort((a, b) => orderFunc(a) - orderFunc(b))
|
||||
return result
|
||||
}
|
||||
|
||||
export function zipObject(keys: any[], values: any[]): Object {
|
||||
if (keys.length !== values.length) {
|
||||
throw Error("can't zipObject with different number of keys and values!")
|
||||
}
|
||||
|
||||
let result = {}
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
result[keys[i]] = values[i]
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* mutates! (and returns) target
|
||||
*/
|
||||
export function assign(target: Object, ...sources: Object[]): Object {
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
let curSource = sources[i]
|
||||
let currSourceKeys = keys(curSource)
|
||||
for (let j = 0; j < currSourceKeys.length; j++) {
|
||||
let currKey = currSourceKeys[j]
|
||||
target[currKey] = curSource[currKey]
|
||||
}
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
/**
|
||||
* mutates! (and returns) target
|
||||
*/
|
||||
export function assignNoOverwrite(
|
||||
target: Object,
|
||||
...sources: Object[]
|
||||
): Object {
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
let curSource = sources[i]
|
||||
let currSourceKeys = keys(curSource)
|
||||
for (let j = 0; j < currSourceKeys.length; j++) {
|
||||
let currKey = currSourceKeys[j]
|
||||
if (!has(target, currKey)) {
|
||||
target[currKey] = curSource[currKey]
|
||||
}
|
||||
}
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
export function defaults(...sources: any[]): any {
|
||||
return assignNoOverwrite.apply(null, [{}].concat(sources))
|
||||
}
|
||||
|
||||
export function groupBy<T>(
|
||||
arr: T[],
|
||||
groupKeyFunc: (item: T) => string
|
||||
): { [groupKey: string]: T[] } {
|
||||
let result: { [groupKey: string]: T[] } = {}
|
||||
|
||||
forEach(arr, (item) => {
|
||||
let currGroupKey = groupKeyFunc(item)
|
||||
let currGroupArr = result[currGroupKey]
|
||||
|
||||
if (currGroupArr) {
|
||||
currGroupArr.push(item)
|
||||
} else {
|
||||
result[currGroupKey] = [item]
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge obj2 into obj1.
|
||||
* Will overwrite existing properties with the same name
|
||||
*/
|
||||
export function merge(obj1: Object, obj2: Object): any {
|
||||
let result = cloneObj(obj1)
|
||||
let keys2 = keys(obj2)
|
||||
for (let i = 0; i < keys2.length; i++) {
|
||||
let key = keys2[i]
|
||||
let value = obj2[key]
|
||||
result[key] = value
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export function NOOP() {}
|
||||
|
||||
export function IDENTITY(item) {
|
||||
return item
|
||||
}
|
||||
|
||||
/**
|
||||
* Will return a new packed array with same values.
|
||||
*/
|
||||
export function packArray<T>(holeyArr: T[]): T[] {
|
||||
const result = []
|
||||
for (let i = 0; i < holeyArr.length; i++) {
|
||||
const orgValue = holeyArr[i]
|
||||
result.push(orgValue !== undefined ? orgValue : undefined)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function PRINT_ERROR(msg) {
|
||||
/* istanbul ignore else - can't override global.console in node.js */
|
||||
if (console && console.error) {
|
||||
console.error(`Error: ${msg}`)
|
||||
}
|
||||
}
|
||||
|
||||
export function PRINT_WARNING(msg) {
|
||||
/* istanbul ignore else - can't override global.console in node.js*/
|
||||
if (console && console.warn) {
|
||||
// TODO: modify docs accordingly
|
||||
console.warn(`Warning: ${msg}`)
|
||||
}
|
||||
}
|
||||
|
||||
export function isES2015MapSupported(): boolean {
|
||||
return typeof Map === "function"
|
||||
}
|
||||
|
||||
export function applyMixins(derivedCtor: any, baseCtors: any[]) {
|
||||
baseCtors.forEach((baseCtor) => {
|
||||
const baseProto = baseCtor.prototype
|
||||
Object.getOwnPropertyNames(baseProto).forEach((propName) => {
|
||||
if (propName === "constructor") {
|
||||
return
|
||||
}
|
||||
|
||||
const basePropDescriptor = Object.getOwnPropertyDescriptor(
|
||||
baseProto,
|
||||
propName
|
||||
)
|
||||
// Handle Accessors
|
||||
if (
|
||||
basePropDescriptor &&
|
||||
(basePropDescriptor.get || basePropDescriptor.set)
|
||||
) {
|
||||
Object.defineProperty(
|
||||
derivedCtor.prototype,
|
||||
propName,
|
||||
basePropDescriptor
|
||||
)
|
||||
} else {
|
||||
derivedCtor.prototype[propName] = baseCtor.prototype[propName]
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// base on: https://github.com/petkaantonov/bluebird/blob/b97c0d2d487e8c5076e8bd897e0dcd4622d31846/src/util.js#L201-L216
|
||||
export function toFastProperties(toBecomeFast) {
|
||||
function FakeConstructor() {}
|
||||
// If our object is used as a constructor it would receive
|
||||
FakeConstructor.prototype = toBecomeFast
|
||||
const fakeInstance = new FakeConstructor()
|
||||
function fakeAccess() {
|
||||
return typeof fakeInstance.bar
|
||||
}
|
||||
// help V8 understand this is a "real" prototype by actually using
|
||||
// the fake instance.
|
||||
fakeAccess()
|
||||
fakeAccess()
|
||||
|
||||
return toBecomeFast
|
||||
// Eval prevents optimization of this method (even though this is dead code)
|
||||
/* istanbul ignore next */
|
||||
// tslint:disable-next-line
|
||||
eval(toBecomeFast)
|
||||
}
|
||||
|
||||
export function peek<T>(arr: T[]): T {
|
||||
return arr[arr.length - 1]
|
||||
}
|
||||
|
||||
/* istanbul ignore next - for performance tracing*/
|
||||
export function timer<T>(func: () => T): { time: number; value: T } {
|
||||
const start = new Date().getTime()
|
||||
const val = func()
|
||||
const end = new Date().getTime()
|
||||
const total = end - start
|
||||
return { time: total, value: val }
|
||||
}
|
||||
4
node_modules/chevrotain/src/version.ts
generated
vendored
Normal file
4
node_modules/chevrotain/src/version.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
// needs a separate module as this is required inside chevrotain productive code
|
||||
// and also in the entry point for webpack(api.ts).
|
||||
// A separate file avoids cyclic dependencies and webpack errors.
|
||||
export const VERSION = "7.1.1"
|
||||
Loading…
Add table
Add a link
Reference in a new issue