From f69808d8782f71b0e0396d03e44b35b92d37abbe Mon Sep 17 00:00:00 2001 From: Sergey Solovyev Date: Fri, 17 Oct 2025 15:43:35 +0200 Subject: [PATCH 1/3] Add information about node's original location in the parsed string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior to this commit it was not possible to trace back the parsed node location in the original string. This information could be used to highlight the syntax of a math expression displayed in a rich editor (e.g. a spreadsheet or a calculator). This commit is based on the PR #2796 that has been abandoned for 2 years. I rebased the task branch and fixed the issues reported in the original PR. In the nutshell, each parsed node stores an array of sources (`SourceMapping[]`) that are set during parsing (see `tokenSource` and its usages for details). The node's constructor and clone method are adjusted to take an optional `MetaOptions` object containing the source mappings. In the future `MetaOptions` could be extended to store more information. Benchmarks showed no change in `evaluate`. `parse` became slower, from 3.68µs to 5.17µs with the changes from this commit. Closes #2795 --- docs/expressions/expression_trees.md | 83 ++++-- src/expression/node/AccessorNode.js | 11 +- src/expression/node/ArrayNode.js | 11 +- src/expression/node/AssignmentNode.js | 11 +- src/expression/node/BlockNode.js | 11 +- src/expression/node/ConditionalNode.js | 11 +- src/expression/node/ConstantNode.js | 11 +- src/expression/node/FunctionAssignmentNode.js | 12 +- src/expression/node/FunctionNode.js | 11 +- src/expression/node/IndexNode.js | 11 +- src/expression/node/Node.js | 16 ++ src/expression/node/ObjectNode.js | 11 +- src/expression/node/OperatorNode.js | 12 +- src/expression/node/ParenthesisNode.js | 11 +- src/expression/node/RangeNode.js | 13 +- src/expression/node/RelationalNode.js | 11 +- src/expression/node/SymbolNode.js | 11 +- src/expression/parse.js | 196 ++++++++++---- src/function/algebra/resolve.js | 15 +- test/node-tests/doc.test.js | 6 + .../expression/node/RangeNode.test.js | 2 +- test/unit-tests/expression/parse.test.js | 246 +++++++++++++++++- .../function/algebra/resolve.test.js | 60 ++++- types/index.d.ts | 11 + 24 files changed, 647 insertions(+), 157 deletions(-) diff --git a/docs/expressions/expression_trees.md b/docs/expressions/expression_trees.md index 945f4fb9af..ce80c2f695 100644 --- a/docs/expressions/expression_trees.md +++ b/docs/expressions/expression_trees.md @@ -39,10 +39,11 @@ tree generated by `math.parse('sqrt(2 + x)')`. All nodes have the following methods: -- `clone() : Node` +- `clone(options: MetaOptions) : Node` Create a shallow clone of the node. - The node itself is cloned, its childs are not cloned. + The node itself is cloned, its childs are not cloned. + Information on available options can be found at [Type Definitions](expression_trees.md#type-definitions) - `cloneDeep() : Node` @@ -263,7 +264,10 @@ Each `Node` has the following properties: - `type: string` The type of the node, for example `'SymbolNode'` in case of a `SymbolNode`. - + +- `sources: SourceMapping[]` + An array of sources mapping this node back to its tokens in the parsed string. + The exact mapping will depend on the type of node and is listed in more detail for each node below. ## Nodes @@ -276,7 +280,7 @@ namespace `math`. Construction: ``` -new AccessorNode(object: Node, index: IndexNode) +new AccessorNode(object: Node, index: IndexNode, meta: MetaOptions) ``` Properties: @@ -284,6 +288,7 @@ Properties: - `object: Node` - `index: IndexNode` - `name: string` (read-only) The function or method name. Returns an empty string when undefined. +- `sources: SourceMapping[]` mappings to tokens defining this accessor. This will be `[` and `]` for array accessors, and `.` for dot notation accessors. Examples: @@ -302,12 +307,13 @@ const node2 = new math.AccessorNode(object, index) Construction: ``` -new ArrayNode(items: Node[]) +new ArrayNode(items: Node[], meta: MetaOptions) ``` Properties: - `items: Node[]` +- `sources: SourceMapping[]` mappings to the `[`, `]`, `,`, and `;` used to define this array in the parsed string Examples: @@ -326,8 +332,8 @@ const node2 = new math.ArrayNode([one, two, three]) Construction: ``` -new AssignmentNode(object: SymbolNode, value: Node) -new AssignmentNode(object: SymbolNode | AccessorNode, index: IndexNode, value: Node) +new AssignmentNode(object: SymbolNode, value: Node, meta: MetaOptions) +new AssignmentNode(object: SymbolNode | AccessorNode, index: IndexNode, value: Node, meta: MetaOptions) ``` Properties: @@ -336,6 +342,7 @@ Properties: - `index: IndexNode | null` - `value: Node` - `name: string` (read-only) The function or method name. Returns an empty string when undefined. +- `sources: SourceMapping[]` mapping to the `=` defining this assignment node in the parsed string Examples: @@ -359,12 +366,13 @@ a semicolon). Construction: ``` -block = new BlockNode(Array.<{node: Node} | {node: Node, visible: boolean}>) +block = new BlockNode(Array.<{node: Node} | {node: Node, visible: boolean}>, meta: MetaOptions) ``` Properties: - `blocks: Array.<{node: Node, visible: boolean}>` +- `sources: SourceMapping[]` mappings to each `;` token delimiting blocks in the parsed string Examples: @@ -396,7 +404,7 @@ const block2 = new BlockNode([ Construction: ``` -new ConditionalNode(condition: Node, trueExpr: Node, falseExpr: Node) +new ConditionalNode(condition: Node, trueExpr: Node, falseExpr: Node, meta: MetaOptions) ``` Properties: @@ -404,6 +412,7 @@ Properties: - `condition: Node` - `trueExpr: Node` - `falseExpr: Node` +- `sources: SourceMapping[]` mappings to the `?`, and `:` tokens defining this conditional in the parsed string Examples: @@ -423,12 +432,13 @@ const node2 = new math.ConditionalNode(condition, trueExpr, falseExpr) Construction: ``` -new ConstantNode(value: *) +new ConstantNode(value: *, meta: MetaOptions) ``` Properties: - `value: *` +- `sources: SourceMapping[]` mapping to the token representing the constant in the parsed string. Examples: @@ -445,7 +455,7 @@ const node3 = new math.ConstantNode('foo') Construction: ``` -new FunctionAssignmentNode(name: string, params: string[], expr: Node) +new FunctionAssignmentNode(name: string, params: string[], expr: Node, meta: MetaOptions) ``` Properties: @@ -453,6 +463,7 @@ Properties: - `name: string` - `params: string[]` - `expr: Node` +- `sources: SourceMapping[]` mapping to the `=` for this assignment in the parsed string Examples: @@ -471,13 +482,14 @@ const node2 = new math.FunctionAssignmentNode('f', ['x'], expr) Construction: ``` -new FunctionNode(fn: Node | string, args: Node[]) +new FunctionNode(fn: Node | string, args: Node[], meta: MetaOptions) ``` Properties: - `fn: Node | string` (read-only) The object or function name which to invoke. - `args: Node[]` +- `sources: SourceMapping[]` mappings to the `(` and `)` defining this function, as well as any `,` delimiting its parameters. Static functions: @@ -499,8 +511,8 @@ const node3 = new math.FunctionNode(new SymbolNode('sqrt'), [four]) Construction: ``` -new IndexNode(dimensions: Node[]) -new IndexNode(dimensions: Node[], dotNotation: boolean) +new IndexNode(dimensions: Node[], meta: MetaOptions) +new IndexNode(dimensions: Node[], dotNotation: boolean, meta: MetaOptions) ``` Each dimension can be a single value, a range, or a property. The values of @@ -515,6 +527,7 @@ Properties: - `dimensions: Node[]` - `dotNotation: boolean` +- `sources: SourceMapping[]` mappings to `,` delimiting items in an array index. If `dotNotation = true`, this will map to the constant following the `.` instead. Examples: @@ -536,12 +549,13 @@ const node2 = new math.AccessorNode(A, index) Construction: ``` -new ObjectNode(properties: Object.) +new ObjectNode(properties: Object., meta: MetaOptions) ``` Properties: - `properties: Object.` +- `sources: SourceMapping[]` mappings to the `{`, `}`, `:`, and `,` tokens defining this object in the parsed string Examples: @@ -560,7 +574,7 @@ const node2 = new math.ObjectNode({a: a, b: b, c: c}) Construction: ``` -new OperatorNode(op: string, fn: string, args: Node[], implicit: boolean = false) +new OperatorNode(op: string, fn: string, args: Node[], implicit: boolean = false, meta: MetaOptions) ``` Additional methods: @@ -594,6 +608,7 @@ Properties: - `fn: string` - `args: Node[]` - `implicit: boolean` True in case of an implicit multiplication, false otherwise +- `sources: SourceMapping[]` mapping to the `+` or `-` defining this unary operator in the parsed string Examples: @@ -610,12 +625,13 @@ const node2 = new math.OperatorNode('+', 'add', [a, b]) Construction: ``` -new ParenthesisNode(content: Node) +new ParenthesisNode(content: Node, meta: MetaOptions) ``` Properties: - `content: Node` +- `sources: SourceMapping[]` mappings to the `(` and `)` for this node in the parsed string Examples: @@ -631,7 +647,7 @@ const node2 = new math.ParenthesisNode(a) Construction: ``` -new RangeNode(start: Node, end: Node [, step: Node]) +new RangeNode(start: Node, end: Node [, step: Node], meta: MetaOptions) ``` Properties: @@ -639,7 +655,8 @@ Properties: - `start: Node` - `end: Node` - `step: Node | null` - +- `sources: SourceMapping[]` mappings to the `:` defining this range node in the parsed string. There will be 1 or 2 mappings, depending on whether step size was defined for the range + Examples: ```js @@ -660,7 +677,7 @@ const node4 = new math.RangeNode(zero, ten, two) Construction: ``` -new RelationalNode(conditionals: string[], params: Node[]) +new RelationalNode(conditionals: string[], params: Node[], meta: MetaOptions) ``` `conditionals` is an array of strings, each of which may be 'smaller', 'larger', 'smallerEq', 'largerEq', 'equal', or 'unequal'. The `conditionals` array must contain exactly one fewer item than `params`. @@ -669,6 +686,7 @@ Properties: - `conditionals: string[]` - `params: Node[]` +- `sources: SourceMapping[]` mappings to the relational symbol `<`, `>`, `==`, `>=`, or `<=` defining this node in the parsed string. This may include multiple mappings if multiple relationals are chained: `10 < x < 20` A `RelationalNode` efficiently represents a chained conditional expression with two or more comparison operators, such as `10 < x <= 50`. The expression is equivalent to `10 < x and x <= 50`, except that `x` is evaluated only once, and evaluation stops (is "short-circuited") once any condition tests false. Operators that are subject to chaining are `<`, `>`, `<=`, `>=`, `==`, and `!=`. For backward compatibility, `math.parse` will return an `OperatorNode` if only a single conditional is present (such as `x > 2`). @@ -690,12 +708,13 @@ const node2 = math.parse('10 < x <= 50') Construction: ``` -new SymbolNode(name: string) +new SymbolNode(name: string, meta: MetaOptions) ``` Properties: - `name: string` +- `sources: SourceMapping[]` a single mapping to the symbol defining this node in the parsed string. The text will match whatever symbol is defined. Static functions: @@ -709,3 +728,25 @@ const node = math.parse('x') const x = new math.SymbolNode('x') ``` + +## Type Definitions + +A few node methods and properties have complex object structures as their parameters or return types. They are: + +### MetaOptions + +This object is passed as a final parameter in the constructor of any node, or as the parameter when calling `clone()`. + +Properties: + +- `sources: SourceMapping` sets the sources for the newly created or cloned node + +### SourceMapping + +Each node has an array of `SourceMapping` objects which map back to the node's corresponding tokens in the original source string + +Properties: + +- `text: string` the token representing this node in the parsed string +- `index; number` the index of the token in the parsed string + diff --git a/src/expression/node/AccessorNode.js b/src/expression/node/AccessorNode.js index 8806699197..6c59645454 100644 --- a/src/expression/node/AccessorNode.js +++ b/src/expression/node/AccessorNode.js @@ -12,6 +12,7 @@ import { import { getSafeProperty } from '../../utils/customs.js' import { factory } from '../../utils/factory.js' import { accessFactory } from './utils/access.js' +import { defaultMetaOptions } from './Node.js' const name = 'AccessorNode' const dependencies = [ @@ -47,9 +48,10 @@ export const createAccessorNode = /* #__PURE__ */ factory(name, dependencies, ({ * @param {Node} object The object from which to retrieve * a property or subset. * @param {IndexNode} index IndexNode containing ranges + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (object, index) { - super() + constructor (object, index, meta = defaultMetaOptions) { + super(meta) if (!isNode(object)) { throw new TypeError('Node expected for parameter "object"') } @@ -133,10 +135,11 @@ export const createAccessorNode = /* #__PURE__ */ factory(name, dependencies, ({ /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {AccessorNode} */ - clone () { - return new AccessorNode(this.object, this.index) + clone (meta) { + return new AccessorNode(this.object, this.index, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/ArrayNode.js b/src/expression/node/ArrayNode.js index 145e96152f..3e39499992 100644 --- a/src/expression/node/ArrayNode.js +++ b/src/expression/node/ArrayNode.js @@ -1,6 +1,7 @@ import { isArrayNode, isNode } from '../../utils/is.js' import { map } from '../../utils/array.js' import { factory } from '../../utils/factory.js' +import { defaultMetaOptions } from './Node.js' const name = 'ArrayNode' const dependencies = [ @@ -14,9 +15,10 @@ export const createArrayNode = /* #__PURE__ */ factory(name, dependencies, ({ No * @extends {Node} * Holds an 1-dimensional array with items * @param {Node[]} [items] 1 dimensional array with items + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (items) { - super() + constructor (items, meta = defaultMetaOptions) { + super(meta) this.items = items || [] // validate input @@ -91,10 +93,11 @@ export const createArrayNode = /* #__PURE__ */ factory(name, dependencies, ({ No /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {ArrayNode} */ - clone () { - return new ArrayNode(this.items.slice(0)) + clone (meta) { + return new ArrayNode(this.items.slice(0), meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/AssignmentNode.js b/src/expression/node/AssignmentNode.js index eca997f2c4..212cee0920 100644 --- a/src/expression/node/AssignmentNode.js +++ b/src/expression/node/AssignmentNode.js @@ -4,6 +4,7 @@ import { factory } from '../../utils/factory.js' import { accessFactory } from './utils/access.js' import { assignFactory } from './utils/assign.js' import { getPrecedence } from '../operators.js' +import { defaultMetaOptions } from './Node.js' const name = 'AssignmentNode' const dependencies = [ @@ -65,9 +66,10 @@ export const createAssignmentNode = /* #__PURE__ */ factory(name, dependencies, * global scope. * @param {Node} value * The value to be assigned + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (object, index, value) { - super() + constructor (object, index, value, meta = defaultMetaOptions) { + super(meta) this.object = object this.index = value ? index : null this.value = value || index @@ -228,10 +230,11 @@ export const createAssignmentNode = /* #__PURE__ */ factory(name, dependencies, /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {AssignmentNode} */ - clone () { - return new AssignmentNode(this.object, this.index, this.value) + clone (meta) { + return new AssignmentNode(this.object, this.index, this.value, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/BlockNode.js b/src/expression/node/BlockNode.js index 3fc540c79c..335fbb1215 100644 --- a/src/expression/node/BlockNode.js +++ b/src/expression/node/BlockNode.js @@ -1,6 +1,7 @@ import { isNode } from '../../utils/is.js' import { forEach, map } from '../../utils/array.js' import { factory } from '../../utils/factory.js' +import { defaultMetaOptions } from './Node.js' const name = 'BlockNode' const dependencies = [ @@ -19,9 +20,10 @@ export const createBlockNode = /* #__PURE__ */ factory(name, dependencies, ({ Re * Object with properties block, which is a Node, and visible, * which is a boolean. The property visible is optional and * is true by default + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (blocks) { - super() + constructor (blocks, meta = defaultMetaOptions) { + super(meta) // validate input, copy blocks if (!Array.isArray(blocks)) throw new Error('Array expected') this.blocks = blocks.map(function (block) { @@ -109,9 +111,10 @@ export const createBlockNode = /* #__PURE__ */ factory(name, dependencies, ({ Re /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {BlockNode} */ - clone () { + clone (meta) { const blocks = this.blocks.map(function (block) { return { node: block.node, @@ -119,7 +122,7 @@ export const createBlockNode = /* #__PURE__ */ factory(name, dependencies, ({ Re } }) - return new BlockNode(blocks) + return new BlockNode(blocks, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/ConditionalNode.js b/src/expression/node/ConditionalNode.js index 279094381d..d13d361875 100644 --- a/src/expression/node/ConditionalNode.js +++ b/src/expression/node/ConditionalNode.js @@ -1,6 +1,7 @@ import { isBigNumber, isComplex, isNode, isUnit, typeOf } from '../../utils/is.js' import { factory } from '../../utils/factory.js' import { getPrecedence } from '../operators.js' +import { defaultMetaOptions } from './Node.js' const name = 'ConditionalNode' const dependencies = [ @@ -48,12 +49,13 @@ export const createConditionalNode = /* #__PURE__ */ factory(name, dependencies, * @param {Node} condition Condition, must result in a boolean * @param {Node} trueExpr Expression evaluated when condition is true * @param {Node} falseExpr Expression evaluated when condition is true + * @param {MetaOptions} [meta] The object with additional options for building this node. * * @constructor ConditionalNode * @extends {Node} */ - constructor (condition, trueExpr, falseExpr) { - super() + constructor (condition, trueExpr, falseExpr, meta = defaultMetaOptions) { + super(meta) if (!isNode(condition)) { throw new TypeError('Parameter condition must be a Node') } if (!isNode(trueExpr)) { throw new TypeError('Parameter trueExpr must be a Node') } if (!isNode(falseExpr)) { throw new TypeError('Parameter falseExpr must be a Node') } @@ -118,10 +120,11 @@ export const createConditionalNode = /* #__PURE__ */ factory(name, dependencies, /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {ConditionalNode} */ - clone () { - return new ConditionalNode(this.condition, this.trueExpr, this.falseExpr) + clone (meta) { + return new ConditionalNode(this.condition, this.trueExpr, this.falseExpr, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/ConstantNode.js b/src/expression/node/ConstantNode.js index 7ab4443a22..4b9735287d 100644 --- a/src/expression/node/ConstantNode.js +++ b/src/expression/node/ConstantNode.js @@ -2,6 +2,7 @@ import { format } from '../../utils/string.js' import { typeOf } from '../../utils/is.js' import { escapeLatex } from '../../utils/latex.js' import { factory } from '../../utils/factory.js' +import { defaultMetaOptions } from './Node.js' const name = 'ConstantNode' const dependencies = [ @@ -19,11 +20,12 @@ export const createConstantNode = /* #__PURE__ */ factory(name, dependencies, ({ * new ConstantNode('hello') * * @param {*} value Value can be any type (number, BigNumber, bigint, string, ...) + * @param {MetaOptions} meta Optional object with additional options for building this node * @constructor ConstantNode * @extends {Node} */ - constructor (value) { - super() + constructor (value, meta = defaultMetaOptions) { + super(meta) this.value = value } @@ -72,10 +74,11 @@ export const createConstantNode = /* #__PURE__ */ factory(name, dependencies, ({ /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {ConstantNode} */ - clone () { - return new ConstantNode(this.value) + clone (meta) { + return new ConstantNode(this.value, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/FunctionAssignmentNode.js b/src/expression/node/FunctionAssignmentNode.js index d6406afecb..466e785a61 100644 --- a/src/expression/node/FunctionAssignmentNode.js +++ b/src/expression/node/FunctionAssignmentNode.js @@ -6,6 +6,7 @@ import { forEach, join } from '../../utils/array.js' import { toSymbol } from '../../utils/latex.js' import { getPrecedence } from '../operators.js' import { factory } from '../../utils/factory.js' +import { defaultMetaOptions } from './Node.js' const name = 'FunctionAssignmentNode' const dependencies = [ @@ -41,9 +42,10 @@ export const createFunctionAssignmentNode = /* #__PURE__ */ factory(name, depend * array with objects containing the name * and type of the parameter * @param {Node} expr The function expression + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (name, params, expr) { - super() + constructor (name, params, expr, meta = defaultMetaOptions) { + super(meta) // validate input if (typeof name !== 'string') { throw new TypeError('String expected for parameter "name"') } if (!Array.isArray(params)) { @@ -148,11 +150,11 @@ export const createFunctionAssignmentNode = /* #__PURE__ */ factory(name, depend /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {FunctionAssignmentNode} */ - clone () { - return new FunctionAssignmentNode( - this.name, this.params.slice(0), this.expr) + clone (meta) { + return new FunctionAssignmentNode(this.name, this.params.slice(0), this.expr, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/FunctionNode.js b/src/expression/node/FunctionNode.js index 7de973c39d..4376feb81b 100644 --- a/src/expression/node/FunctionNode.js +++ b/src/expression/node/FunctionNode.js @@ -5,6 +5,7 @@ import { getSafeProperty, getSafeMethod } from '../../utils/customs.js' import { createSubScope } from '../../utils/scope.js' import { factory } from '../../utils/factory.js' import { defaultTemplate, latexFunctions } from '../../utils/latex.js' +import { defaultMetaOptions } from './Node.js' const name = 'FunctionNode' const dependencies = [ @@ -94,9 +95,10 @@ export const createFunctionNode = /* #__PURE__ */ factory(name, dependencies, ({ * Item resolving to a function on which to invoke * the arguments, typically a SymbolNode or AccessorNode * @param {./Node[]} args + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (fn, args) { - super() + constructor (fn, args = [], meta = defaultMetaOptions) { + super(meta) if (typeof fn === 'string') { fn = new SymbolNode(fn) } @@ -311,10 +313,11 @@ export const createFunctionNode = /* #__PURE__ */ factory(name, dependencies, ({ /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {FunctionNode} */ - clone () { - return new FunctionNode(this.fn, this.args.slice(0)) + clone (meta) { + return new FunctionNode(this.fn, this.args.slice(0), meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/IndexNode.js b/src/expression/node/IndexNode.js index 8dc207179d..8aca0d2b1d 100644 --- a/src/expression/node/IndexNode.js +++ b/src/expression/node/IndexNode.js @@ -3,6 +3,7 @@ import { getSafeProperty } from '../../utils/customs.js' import { factory } from '../../utils/factory.js' import { isArray, isConstantNode, isMatrix, isNode, isString, typeOf } from '../../utils/is.js' import { escape } from '../../utils/string.js' +import { defaultMetaOptions } from './Node.js' const name = 'IndexNode' const dependencies = [ @@ -25,9 +26,10 @@ export const createIndexNode = /* #__PURE__ */ factory(name, dependencies, ({ No * Optional property describing whether this index was written using dot * notation like `a.b`, or using bracket notation like `a["b"]` * (which is the default). This property is used for string conversion. + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (dimensions, dotNotation) { - super() + constructor (dimensions, dotNotation = false, meta = defaultMetaOptions) { + super(meta) this.dimensions = dimensions this.dotNotation = dotNotation || false @@ -138,10 +140,11 @@ export const createIndexNode = /* #__PURE__ */ factory(name, dependencies, ({ No /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {IndexNode} */ - clone () { - return new IndexNode(this.dimensions.slice(0), this.dotNotation) + clone (meta) { + return new IndexNode(this.dimensions.slice(0), this.dotNotation, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/Node.js b/src/expression/node/Node.js index 06c67abc3b..662e173034 100644 --- a/src/expression/node/Node.js +++ b/src/expression/node/Node.js @@ -7,6 +7,9 @@ import { createMap } from '../../utils/map.js' const name = 'Node' const dependencies = ['mathWithTransform'] +export const defaultMetaOptions = { + sources: [] +} export const createNode = /* #__PURE__ */ factory(name, dependencies, ({ mathWithTransform }) => { /** @@ -23,9 +26,22 @@ export const createNode = /* #__PURE__ */ factory(name, dependencies, ({ mathWit } class Node { + sources = [] + get type () { return 'Node' } get isNode () { return true } + /** + * @constructor Node + * A generic node, the parent of other AST nodes + * @param {MetaOptions} [meta] The object with additional options for building this node. + */ + constructor (meta = defaultMetaOptions) { + if (meta.sources) { + this.sources = meta.sources + } + } + /** * Evaluate the node * @param {Object} [scope] Scope to read/write variables diff --git a/src/expression/node/ObjectNode.js b/src/expression/node/ObjectNode.js index f4d3fd8ec6..5acbdd40f5 100644 --- a/src/expression/node/ObjectNode.js +++ b/src/expression/node/ObjectNode.js @@ -3,6 +3,7 @@ import { factory } from '../../utils/factory.js' import { isNode } from '../../utils/is.js' import { hasOwnProperty } from '../../utils/object.js' import { escape, stringify } from '../../utils/string.js' +import { defaultMetaOptions } from './Node.js' const name = 'ObjectNode' const dependencies = [ @@ -16,9 +17,10 @@ export const createObjectNode = /* #__PURE__ */ factory(name, dependencies, ({ N * @extends {Node} * Holds an object with keys/values * @param {Object.} [properties] object with key/value pairs + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (properties) { - super() + constructor (properties, meta = defaultMetaOptions) { + super(meta) this.properties = properties || {} // validate input @@ -110,16 +112,17 @@ export const createObjectNode = /* #__PURE__ */ factory(name, dependencies, ({ N /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {ObjectNode} */ - clone () { + clone (meta) { const properties = {} for (const key in this.properties) { if (hasOwnProperty(this.properties, key)) { properties[key] = this.properties[key] } } - return new ObjectNode(properties) + return new ObjectNode(properties, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/OperatorNode.js b/src/expression/node/OperatorNode.js index 6890db6ece..8e67e9ede1 100644 --- a/src/expression/node/OperatorNode.js +++ b/src/expression/node/OperatorNode.js @@ -6,6 +6,7 @@ import { getSafeProperty, isSafeMethod } from '../../utils/customs.js' import { getAssociativity, getPrecedence, isAssociativeWith, properties } from '../operators.js' import { latexOperators } from '../../utils/latex.js' import { factory } from '../../utils/factory.js' +import { defaultMetaOptions } from './Node.js' const name = 'OperatorNode' const dependencies = [ @@ -250,9 +251,10 @@ export const createOperatorNode = /* #__PURE__ */ factory(name, dependencies, ({ * @param {Node[]} args Operator arguments * @param {boolean} [implicit] Is this an implicit multiplication? * @param {boolean} [isPercentage] Is this an percentage Operation? + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (op, fn, args, implicit, isPercentage) { - super() + constructor (op, fn, args, implicit = false, isPercentage = false, meta = defaultMetaOptions) { + super(meta) // validate input if (typeof op !== 'string') { throw new TypeError('string expected for parameter "op"') @@ -361,11 +363,11 @@ export const createOperatorNode = /* #__PURE__ */ factory(name, dependencies, ({ /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {OperatorNode} */ - clone () { - return new OperatorNode( - this.op, this.fn, this.args.slice(0), this.implicit, this.isPercentage) + clone (meta) { + return new OperatorNode(this.op, this.fn, this.args.slice(0), this.implicit, this.isPercentage, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/ParenthesisNode.js b/src/expression/node/ParenthesisNode.js index d530ba8215..ad471fbf34 100644 --- a/src/expression/node/ParenthesisNode.js +++ b/src/expression/node/ParenthesisNode.js @@ -1,5 +1,6 @@ import { isNode } from '../../utils/is.js' import { factory } from '../../utils/factory.js' +import { defaultMetaOptions } from './Node.js' const name = 'ParenthesisNode' const dependencies = [ @@ -13,10 +14,11 @@ export const createParenthesisNode = /* #__PURE__ */ factory(name, dependencies, * @extends {Node} * A parenthesis node describes manual parenthesis from the user input * @param {Node} content + * @param {MetaOptions} [meta] The object with additional options for building this node. * @extends {Node} */ - constructor (content) { - super() + constructor (content, meta = defaultMetaOptions) { + super(meta) // validate input if (!isNode(content)) { throw new TypeError('Node expected for parameter "content"') @@ -76,10 +78,11 @@ export const createParenthesisNode = /* #__PURE__ */ factory(name, dependencies, /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {ParenthesisNode} */ - clone () { - return new ParenthesisNode(this.content) + clone (meta) { + return new ParenthesisNode(this.content, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/RangeNode.js b/src/expression/node/RangeNode.js index f1997d0c1d..d9759d5c3f 100644 --- a/src/expression/node/RangeNode.js +++ b/src/expression/node/RangeNode.js @@ -1,6 +1,7 @@ import { isNode, isSymbolNode } from '../../utils/is.js' import { factory } from '../../utils/factory.js' import { getPrecedence } from '../operators.js' +import { defaultMetaOptions } from './Node.js' const name = 'RangeNode' const dependencies = [ @@ -45,14 +46,15 @@ export const createRangeNode = /* #__PURE__ */ factory(name, dependencies, ({ No * @param {Node} start included lower-bound * @param {Node} end included upper-bound * @param {Node} [step] optional step + * @param {MetaOptions} [meta] The object with additional options for building this node. */ - constructor (start, end, step) { - super() + constructor (start, end, step = null, meta = defaultMetaOptions) { + super(meta) // validate inputs if (!isNode(start)) throw new TypeError('Node expected') if (!isNode(end)) throw new TypeError('Node expected') if (step && !isNode(step)) throw new TypeError('Node expected') - if (arguments.length > 3) throw new Error('Too many arguments') + if (arguments.length > 4) throw new Error('Too many arguments') this.start = start // included lower-bound this.end = end // included upper-bound @@ -143,10 +145,11 @@ export const createRangeNode = /* #__PURE__ */ factory(name, dependencies, ({ No /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {RangeNode} */ - clone () { - return new RangeNode(this.start, this.end, this.step && this.step) + clone (meta) { + return new RangeNode(this.start, this.end, this.step && this.step, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/RelationalNode.js b/src/expression/node/RelationalNode.js index 5663ad71f6..e90f46d7fe 100644 --- a/src/expression/node/RelationalNode.js +++ b/src/expression/node/RelationalNode.js @@ -3,6 +3,7 @@ import { escape } from '../../utils/string.js' import { getSafeProperty } from '../../utils/customs.js' import { latexOperators } from '../../utils/latex.js' import { factory } from '../../utils/factory.js' +import { defaultMetaOptions } from './Node.js' const name = 'RelationalNode' const dependencies = [ @@ -27,12 +28,13 @@ export const createRelationalNode = /* #__PURE__ */ factory(name, dependencies, * An array of conditional operators used to compare the parameters * @param {Node[]} params * The parameters that will be compared + * @param {MetaOptions} [meta] The object with additional options for building this node. * * @constructor RelationalNode * @extends {Node} */ - constructor (conditionals, params) { - super() + constructor (conditionals, params, meta = defaultMetaOptions) { + super(meta) if (!Array.isArray(conditionals)) { throw new TypeError('Parameter conditionals must be an array') } if (!Array.isArray(params)) { throw new TypeError('Parameter params must be an array') } if (conditionals.length !== params.length - 1) { @@ -106,10 +108,11 @@ export const createRelationalNode = /* #__PURE__ */ factory(name, dependencies, /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {RelationalNode} */ - clone () { - return new RelationalNode(this.conditionals, this.params) + clone (meta = {}) { + return new RelationalNode(this.conditionals, this.params, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/node/SymbolNode.js b/src/expression/node/SymbolNode.js index 36391c751e..e0786d8841 100644 --- a/src/expression/node/SymbolNode.js +++ b/src/expression/node/SymbolNode.js @@ -2,6 +2,7 @@ import { escape } from '../../utils/string.js' import { getSafeProperty } from '../../utils/customs.js' import { factory } from '../../utils/factory.js' import { toSymbol } from '../../utils/latex.js' +import { defaultMetaOptions } from './Node.js' const name = 'SymbolNode' const dependencies = [ @@ -26,10 +27,11 @@ export const createSymbolNode = /* #__PURE__ */ factory(name, dependencies, ({ m * @extends {Node} * A symbol node can hold and resolve a symbol * @param {string} name + * @param {MetaOptions} [meta] The object with additional options for building this node. * @extends {Node} */ - constructor (name) { - super() + constructor (name, meta = defaultMetaOptions) { + super(meta) // validate input if (typeof name !== 'string') { throw new TypeError('String expected for parameter "name"') @@ -111,10 +113,11 @@ export const createSymbolNode = /* #__PURE__ */ factory(name, dependencies, ({ m /** * Create a clone of this node, a shallow copy + * @param {MetaOptions} [meta] An object with additional options for cloning this node * @return {SymbolNode} */ - clone () { - return new SymbolNode(this.name) + clone (meta) { + return new SymbolNode(this.name, meta ?? { sources: this.sources }) } /** diff --git a/src/expression/parse.js b/src/expression/parse.js index d3564cc76e..a948edd58a 100644 --- a/src/expression/parse.js +++ b/src/expression/parse.js @@ -214,6 +214,16 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } } + /** + * Returns a mapping of the current token in state to its place in the source expression + * @param {Object} state + * @return {SourceMapping} the source mapping + * @private + */ + function tokenSource (state) { + return { index: state.index - state.token.length, text: state.token } + } + /** * View upto `length` characters of the expression starting at the current character. * @@ -620,6 +630,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ function parseBlock (state) { let node const blocks = [] + const sources = [] let visible if (state.token !== '' && state.token !== '\n' && state.token !== ';') { @@ -631,6 +642,8 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // TODO: simplify this loop while (state.token === '\n' || state.token === ';') { // eslint-disable-line no-unmodified-loop-condition + sources.push(tokenSource(state)) + if (blocks.length === 0 && node) { visible = (state.token !== ';') blocks.push({ node, visible }) @@ -649,10 +662,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } if (blocks.length > 0) { - return new BlockNode(blocks) + return new BlockNode(blocks, { sources }) } else { if (!node) { - node = new ConstantNode(undefined) + node = new ConstantNode(undefined, { sources: [{ index: 0, text: '' }] }) if (state.comment) { node.comment = state.comment } @@ -675,18 +688,21 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ const node = parseConditional(state) + const source = tokenSource(state) + if (state.token === '=') { if (isSymbolNode(node)) { // parse a variable assignment like 'a = 2/3' name = node.name + const symbolNode = new SymbolNode(name, { sources: node.sources }) getTokenSkipNewline(state) value = parseAssignment(state) - return new AssignmentNode(new SymbolNode(name), value) + return new AssignmentNode(symbolNode, value, null, { sources: [source] }) } else if (isAccessorNode(node)) { // parse a matrix subset assignment like 'A[1,2] = 4' getTokenSkipNewline(state) value = parseAssignment(state) - return new AssignmentNode(node.object, node.index, value) + return new AssignmentNode(node.object, node.index, value, { sources: [source] }) } else if (isFunctionNode(node) && isSymbolNode(node.fn)) { // parse function assignment like 'f(x) = x^2' valid = true @@ -704,7 +720,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (valid) { getTokenSkipNewline(state) value = parseAssignment(state) - return new FunctionAssignmentNode(name, args, value) + return new FunctionAssignmentNode(name, args, value, { sources: [source] }) } } @@ -727,7 +743,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ function parseConditional (state) { let node = parseLogicalOr(state) + const condSources = [] while (state.token === '?') { // eslint-disable-line no-unmodified-loop-condition + condSources.push(tokenSource(state)) // set a conditional level, the range operator will be ignored as long // as conditionalLevel === state.nestingLevel. const prev = state.conditionalLevel @@ -739,12 +757,15 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ':') throw createSyntaxError(state, 'False part of conditional expression expected') + const condSource = condSources.pop() + const colonSource = tokenSource(state) + state.conditionalLevel = null getTokenSkipNewline(state) const falseExpr = parseAssignment(state) // Note: check for conditional operator again, right associativity - node = new ConditionalNode(condition, trueExpr, falseExpr) + node = new ConditionalNode(condition, trueExpr, falseExpr, { sources: [condSource, colonSource] }) // restore the previous conditional level state.conditionalLevel = prev @@ -762,8 +783,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseLogicalXor(state) while (state.token === 'or') { // eslint-disable-line no-unmodified-loop-condition + const source = tokenSource(state) getTokenSkipNewline(state) - node = new OperatorNode('or', 'or', [node, parseLogicalXor(state)]) + node = new OperatorNode('or', 'or', [node, parseLogicalXor(state)], false, false, { sources: [source] }) } return node @@ -778,8 +800,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseLogicalAnd(state) while (state.token === 'xor') { // eslint-disable-line no-unmodified-loop-condition + const source = tokenSource(state) getTokenSkipNewline(state) - node = new OperatorNode('xor', 'xor', [node, parseLogicalAnd(state)]) + node = new OperatorNode('xor', 'xor', [node, parseLogicalAnd(state)], false, false, { sources: [source] }) } return node @@ -794,8 +817,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseBitwiseOr(state) while (state.token === 'and') { // eslint-disable-line no-unmodified-loop-condition + const source = tokenSource(state) getTokenSkipNewline(state) - node = new OperatorNode('and', 'and', [node, parseBitwiseOr(state)]) + node = new OperatorNode('and', 'and', [node, parseBitwiseOr(state)], false, false, { sources: [source] }) } return node @@ -810,8 +834,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseBitwiseXor(state) while (state.token === '|') { // eslint-disable-line no-unmodified-loop-condition + const source = tokenSource(state) getTokenSkipNewline(state) - node = new OperatorNode('|', 'bitOr', [node, parseBitwiseXor(state)]) + node = new OperatorNode('|', 'bitOr', [node, parseBitwiseXor(state)], false, false, { sources: [source] }) } return node @@ -826,8 +851,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseBitwiseAnd(state) while (state.token === '^|') { // eslint-disable-line no-unmodified-loop-condition + const source = tokenSource(state) getTokenSkipNewline(state) - node = new OperatorNode('^|', 'bitXor', [node, parseBitwiseAnd(state)]) + node = new OperatorNode('^|', 'bitXor', [node, parseBitwiseAnd(state)], false, false, { sources: [source] }) } return node @@ -842,8 +868,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseRelational(state) while (state.token === '&') { // eslint-disable-line no-unmodified-loop-condition + const source = tokenSource(state) getTokenSkipNewline(state) - node = new OperatorNode('&', 'bitAnd', [node, parseRelational(state)]) + node = new OperatorNode('&', 'bitAnd', [node, parseRelational(state)], false, false, { sources: [source] }) } return node @@ -866,7 +893,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ '>=': 'largerEq' } + const sources = [] while (hasOwnProperty(operators, state.token)) { // eslint-disable-line no-unmodified-loop-condition + sources.push(tokenSource(state)) const cond = { name: state.token, fn: operators[state.token] } conditionals.push(cond) getTokenSkipNewline(state) @@ -876,9 +905,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (params.length === 1) { return params[0] } else if (params.length === 2) { - return new OperatorNode(conditionals[0].name, conditionals[0].fn, params) + return new OperatorNode(conditionals[0].name, conditionals[0].fn, params, false, false, { sources }) } else { - return new RelationalNode(conditionals.map(c => c.fn), params) + return new RelationalNode(conditionals.map(c => c.fn), params, { sources }) } } @@ -902,9 +931,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ name = state.token fn = operators[name] + const source = tokenSource(state) + getTokenSkipNewline(state) params = [node, parseConversion(state)] - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) } return node @@ -927,17 +958,19 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (hasOwnProperty(operators, state.token)) { name = state.token + const source = tokenSource(state) fn = operators[name] getTokenSkipNewline(state) if (name === 'in' && '])},;'.includes(state.token)) { // end of expression -> this is the unit 'in' ('inch') - node = new OperatorNode('*', 'multiply', [node, new SymbolNode('in')], true) + // no source mapping because this * operator is not explicitly in the source expression + node = new OperatorNode('*', 'multiply', [node, new SymbolNode('in', { sources: [source] })], true) } else { // operator 'a to b' or 'a in b' params = [node, parseRange(state)] - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) } } @@ -955,7 +988,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token === ':') { // implicit start=1 (one-based) - node = new ConstantNode(1) + const implicitSource = tokenSource(state) + implicitSource.text = '' + node = new ConstantNode(1, { sources: [implicitSource] }) } else { // explicit start node = parseAddSubtract(state) @@ -965,13 +1000,17 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // we ignore the range operator when a conditional operator is being processed on the same level params.push(node) + const sources = [] // parse step and end while (state.token === ':' && params.length < 3) { // eslint-disable-line no-unmodified-loop-condition + sources.push(tokenSource(state)) getTokenSkipNewline(state) if (state.token === ')' || state.token === ']' || state.token === ',' || state.token === '') { // implicit end - params.push(new SymbolNode('end')) + const implicitSource = tokenSource(state) + implicitSource.text = '' + params.push(new SymbolNode('end', { sources: [implicitSource] })) } else { // explicit end params.push(parseAddSubtract(state)) @@ -980,10 +1019,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (params.length === 3) { // params = [start, step, end] - node = new RangeNode(params[0], params[2], params[1]) // start, end, step + node = new RangeNode(params[0], params[2], params[1], { sources }) // start, end, step } else { // length === 2 // params = [start, end] - node = new RangeNode(params[0], params[1]) // start, end + node = new RangeNode(params[0], params[1], null, { sources }) // start, end } } @@ -1007,15 +1046,17 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (hasOwnProperty(operators, state.token)) { name = state.token fn = operators[name] + const source = tokenSource(state) getTokenSkipNewline(state) const rightNode = parseMultiplyDivideModulus(state) if (rightNode.isPercentage) { + // no mapping as this * operator is not in source expression params = [node, new OperatorNode('*', 'multiply', [node, rightNode])] } else { params = [node, rightNode] } - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) } return node @@ -1046,9 +1087,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // explicit operators name = state.token fn = operators[name] + const source = tokenSource(state) getTokenSkipNewline(state) last = parseImplicitMultiplication(state) - node = new OperatorNode(name, fn, [node, last]) + node = new OperatorNode(name, fn, [node, last], false, false, { sources: [source] }) } else { break } @@ -1081,8 +1123,15 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // symbol: implicit multiplication like '2a', '(2+3)a', 'a b' // number: implicit multiplication like '(2+3)2' // parenthesis: implicit multiplication like '2(3+4)', '(3+4)(1+2)' + const source = tokenSource(state) + + // mapping is an empty string at the index where * would be + // in the case of "in", the word itself represents the * + if (source.text !== 'in') { + source.text = '' + } last = parseRule2(state) - node = new OperatorNode('*', 'multiply', [node, last], true /* implicit */) + node = new OperatorNode('*', 'multiply', [node, last], true /* implicit */, false, { sources: [source] }) } else { break } @@ -1108,6 +1157,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (true) { // Match the "number /" part of the pattern "number / number symbol" if (state.token === '/' && rule2Node(last)) { + const source = tokenSource(state) // Look ahead to see if the next token is a number tokenStates.push(Object.assign({}, state)) getTokenSkipNewline(state) @@ -1125,7 +1175,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ Object.assign(state, tokenStates.pop()) tokenStates.pop() last = parseUnaryPercentage(state) - node = new OperatorNode('/', 'divide', [node, last]) + node = new OperatorNode('/', 'divide', [node, last], false, false, { sources: [source] }) } else { // Not a match, so rewind tokenStates.pop() @@ -1155,6 +1205,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token === '%') { const previousState = Object.assign({}, state) + const source = tokenSource(state) getTokenSkipNewline(state) // We need to decide if this is a unary percentage % or binary modulo % // So we attempt to parse a unary expression at this point. @@ -1168,7 +1219,8 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ Object.assign(state, previousState) } catch { // Not seeing a term at this point, so was a unary % - node = new OperatorNode('/', 'divide', [node, new ConstantNode(100)], false, true) + const constant = new ConstantNode(100, { sources: [source] }) + node = new OperatorNode('/', 'divide', [node, constant], false, true, { sources: [{ ...source }] }) } } @@ -1192,11 +1244,12 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (hasOwnProperty(operators, state.token)) { fn = operators[state.token] name = state.token + const source = tokenSource(state) getTokenSkipNewline(state) params = [parseUnary(state)] - return new OperatorNode(name, fn, params) + return new OperatorNode(name, fn, params, false, false, { sources: [source] }) } return parsePow(state) @@ -1216,10 +1269,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token === '^' || state.token === '.^') { name = state.token fn = (name === '^') ? 'pow' : 'dotPow' + const source = tokenSource(state) getTokenSkipNewline(state) params = [node, parseUnary(state)] // Go back to unary, we can have '2^-3' - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) } return node @@ -1259,11 +1313,12 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (hasOwnProperty(operators, state.token)) { name = state.token fn = operators[name] + const source = tokenSource(state) getToken(state) params = [node] - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) node = parseAccessors(state, node) } @@ -1304,11 +1359,14 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.tokenType === TOKENTYPE.SYMBOL && hasOwnProperty(state.extraNodes, state.token)) { const CustomNode = state.extraNodes[state.token] + const sources = [tokenSource(state)] + getToken(state) // parse parameters if (state.token === '(') { params = [] + sources.push(tokenSource(state)) openParams(state) getToken(state) @@ -1318,6 +1376,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // parse a list with parameters while (state.token === ',') { // eslint-disable-line no-unmodified-loop-condition + sources.push(tokenSource(state)) getToken(state) params.push(parseAssignment(state)) } @@ -1326,13 +1385,14 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ')') { throw createSyntaxError(state, 'Parenthesis ) expected') } + sources.push(tokenSource(state)) closeParams(state) getToken(state) } // create a new custom node // noinspection JSValidateTypes - return new CustomNode(params) + return new CustomNode(params, { sources }) } return parseSymbol(state) @@ -1350,14 +1410,16 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ (state.tokenType === TOKENTYPE.DELIMITER && state.token in NAMED_DELIMITERS)) { name = state.token + const source = tokenSource(state) + getToken(state) if (hasOwnProperty(CONSTANTS, name)) { // true, false, null, ... - node = new ConstantNode(CONSTANTS[name]) + node = new ConstantNode(CONSTANTS[name], { sources: [source] }) } else if (NUMERIC_CONSTANTS.includes(name)) { // NaN, Infinity - node = new ConstantNode(numeric(name, 'number')) + node = new ConstantNode(numeric(name, 'number'), { sources: [source] }) } else { - node = new SymbolNode(name) + node = new SymbolNode(name, { sources: [source] }) } // parse function parameters and matrix index @@ -1385,6 +1447,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ function parseAccessors (state, node, types) { let params + const sources = [tokenSource(state)] while ((state.token === '(' || state.token === '[' || state.token === '.') && (!types || types.includes(state.token))) { // eslint-disable-line no-unmodified-loop-condition params = [] @@ -1400,6 +1463,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // parse a list with parameters while (state.token === ',') { // eslint-disable-line no-unmodified-loop-condition + sources.push(tokenSource(state)) getToken(state) params.push(parseAssignment(state)) } @@ -1408,10 +1472,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ')') { throw createSyntaxError(state, 'Parenthesis ) expected') } + sources.push(tokenSource(state)) closeParams(state) getToken(state) - node = new FunctionNode(node, params) + node = new FunctionNode(node, params, { sources }) } else { // implicit multiplication like (2+3)(4+5) or sqrt(2)(1+2) // don't parse it here but let it be handled by parseImplicitMultiplication @@ -1422,12 +1487,14 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // index notation like variable[2, 3] openParams(state) getToken(state) + const indexSources = [] if (state.token !== ']') { params.push(parseAssignment(state)) // parse a list with parameters while (state.token === ',') { // eslint-disable-line no-unmodified-loop-condition + indexSources.push(tokenSource(state)) getToken(state) params.push(parseAssignment(state)) } @@ -1436,10 +1503,12 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ']') { throw createSyntaxError(state, 'Parenthesis ] expected') } + sources.push(tokenSource(state)) closeParams(state) getToken(state) - node = new AccessorNode(node, new IndexNode(params)) + const indexNode = new IndexNode(params, false, { sources: indexSources }) + node = new AccessorNode(node, indexNode, { sources }) } else { // dot notation like variable.prop getToken(state) @@ -1449,12 +1518,13 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (!isPropertyName) { throw createSyntaxError(state, 'Property name expected after dot') } - - params.push(new ConstantNode(state.token)) + const constantSource = tokenSource(state) + params.push(new ConstantNode(state.token, { sources: [constantSource] })) getToken(state) const dotNotation = true - node = new AccessorNode(node, new IndexNode(params, dotNotation)) + const indexNode = new IndexNode(params, dotNotation, { sources: [{ ...constantSource }] }) + node = new AccessorNode(node, indexNode, { sources }) } } @@ -1467,13 +1537,13 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ * @private */ function parseString (state) { - let node, str + let node if (state.token === '"' || state.token === "'") { - str = parseStringToken(state, state.token) + const parsedStringToken = parseStringToken(state, state.token) // create constant - node = new ConstantNode(str) + node = new ConstantNode(parsedStringToken.token, { sources: parsedStringToken.sources }) // parse index parameters node = parseAccessors(state, node) @@ -1488,9 +1558,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ * Parse a string surrounded by single or double quotes * @param {Object} state * @param {"'" | "\""} quote - * @return {string} + * @return {{token: string, sources: SourceMapping[]}} */ function parseStringToken (state, quote) { + const sources = [tokenSource(state)] let str = '' while (currentCharacter(state) !== '' && currentCharacter(state) !== quote) { @@ -1526,9 +1597,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== quote) { throw createSyntaxError(state, `End of string ${quote} expected`) } + sources.push(tokenSource(state)) getToken(state) - return str + return { token: str, sources } } /** @@ -1540,6 +1612,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let array, params, rows, cols if (state.token === '[') { + const sources = [tokenSource(state)] // matrix [...] openParams(state) getToken(state) @@ -1555,6 +1628,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // the rows of the matrix are separated by dot-comma's while (state.token === ';') { // eslint-disable-line no-unmodified-loop-condition + sources.push(tokenSource(state)) getToken(state) if (state.token !== ']') { @@ -1566,6 +1640,8 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ']') { throw createSyntaxError(state, 'End of matrix ] expected') } + + sources.push(tokenSource(state)) closeParams(state) getToken(state) @@ -1578,12 +1654,15 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } } - array = new ArrayNode(params) + array = new ArrayNode(params, { sources }) } else { // 1 dimensional vector if (state.token !== ']') { throw createSyntaxError(state, 'End of matrix ] expected') } + + // merge the [] sources with the ,,, sources from parseRow + row.sources = [...sources, ...row.sources, tokenSource(state)] closeParams(state) getToken(state) @@ -1591,9 +1670,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } } else { // this is an empty matrix "[ ]" + sources.push(tokenSource(state)) closeParams(state) getToken(state) - array = new ArrayNode([]) + array = new ArrayNode([], { sources }) } return parseAccessors(state, array) @@ -1610,7 +1690,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ const params = [parseAssignment(state)] let len = 1 + const sources = [] while (state.token === ',') { // eslint-disable-line no-unmodified-loop-condition + sources.push(tokenSource(state)) getToken(state) // parse expression @@ -1620,7 +1702,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } } - return new ArrayNode(params) + return new ArrayNode(params, { sources }) } /** @@ -1630,17 +1712,22 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ */ function parseObject (state) { if (state.token === '{') { + const sources = [tokenSource(state)] openParams(state) let key const properties = {} do { + if (state.token === ',') { + sources.push(tokenSource(state)) + } + getToken(state) if (state.token !== '}') { // parse key if (state.token === '"' || state.token === "'") { - key = parseStringToken(state, state.token) + key = parseStringToken(state, state.token).token } else if (state.tokenType === TOKENTYPE.SYMBOL || (state.tokenType === TOKENTYPE.DELIMITER && state.token in NAMED_DELIMITERS)) { key = state.token getToken(state) @@ -1652,6 +1739,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ':') { throw createSyntaxError(state, 'Colon : expected after object key') } + sources.push(tokenSource(state)) getToken(state) // parse key @@ -1663,10 +1751,13 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== '}') { throw createSyntaxError(state, 'Comma , or bracket } expected after object value') } + + sources.push(tokenSource(state)) + closeParams(state) getToken(state) - let node = new ObjectNode(properties) + let node = new ObjectNode(properties, { sources }) // parse index parameters node = parseAccessors(state, node) @@ -1688,12 +1779,13 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.tokenType === TOKENTYPE.NUMBER) { // this is a number numberStr = state.token + const source = tokenSource(state) getToken(state) const numericType = safeNumberType(numberStr, config) const value = numeric(numberStr, numericType) - return new ConstantNode(value) + return new ConstantNode(value, { sources: [source] }) } return parseParentheses(state) @@ -1709,6 +1801,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // check if it is a parenthesized expression if (state.token === '(') { + const sources = [tokenSource(state)] // parentheses (...) openParams(state) getToken(state) @@ -1718,10 +1811,13 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ')') { throw createSyntaxError(state, 'Parenthesis ) expected') } + + sources.push(tokenSource(state)) + closeParams(state) getToken(state) - node = new ParenthesisNode(node) + node = new ParenthesisNode(node, { sources }) node = parseAccessors(state, node) return node } diff --git a/src/function/algebra/resolve.js b/src/function/algebra/resolve.js index 6a1249ed93..b4875b6aac 100644 --- a/src/function/algebra/resolve.js +++ b/src/function/algebra/resolve.js @@ -65,9 +65,11 @@ export const createResolve = /* #__PURE__ */ factory(name, dependencies, ({ nextWithin.add(node.name) return _resolve(value, scope, nextWithin) } else if (typeof value === 'number') { - return parse(String(value)) + const parsed = parse(String(value)).clone({ sources: [] }) + return parsed } else if (value !== undefined) { - return new ConstantNode(value) + const parsed = new ConstantNode(value).clone({ sources: [] }) + return parsed } else { return node } @@ -75,14 +77,17 @@ export const createResolve = /* #__PURE__ */ factory(name, dependencies, ({ const args = node.args.map(function (arg) { return _resolve(arg, scope, within) }) - return new OperatorNode(node.op, node.fn, args, node.implicit) + const newNode = new OperatorNode(node.op, node.fn, args, node.implicit).clone({ sources: node.sources }) + return newNode } else if (isParenthesisNode(node)) { - return new ParenthesisNode(_resolve(node.content, scope, within)) + const parenNode = new ParenthesisNode(_resolve(node.content, scope, within)).clone({ sources: [] }) + return parenNode } else if (isFunctionNode(node)) { const args = node.args.map(function (arg) { return _resolve(arg, scope, within) }) - return new FunctionNode(node.name, args) + const fnNode = new FunctionNode(node.name, args).clone({ sources: [] }) + return fnNode } // Otherwise just recursively resolve any children (might also work diff --git a/test/node-tests/doc.test.js b/test/node-tests/doc.test.js index fdf8f51122..d89c442229 100644 --- a/test/node-tests/doc.test.js +++ b/test/node-tests/doc.test.js @@ -153,6 +153,12 @@ function checkExpectation (want, got) { } return approxEqual(got, want, 1e-9) } + if (want instanceof math.Node && got instanceof math.Node) { + got.clone({ sources: [] }) + want.clone({ sources: [] }) + + return assert.deepEqual(got, want) + } if ( typeof want === 'string' && typeof got === 'string' && diff --git a/test/unit-tests/expression/node/RangeNode.test.js b/test/unit-tests/expression/node/RangeNode.test.js index 97e7b2056d..e3c97c3d6a 100644 --- a/test/unit-tests/expression/node/RangeNode.test.js +++ b/test/unit-tests/expression/node/RangeNode.test.js @@ -39,7 +39,7 @@ describe('RangeNode', function () { assert.throws(function () { console.log(new RangeNode()) }, TypeError) assert.throws(function () { console.log(new RangeNode(start)) }, TypeError) assert.throws(function () { console.log(new RangeNode([])) }, TypeError) - assert.throws(function () { console.log(new RangeNode(start, end, start, end)) }, Error) + assert.throws(function () { console.log(new RangeNode(start, end, start, end, end)) }, Error) assert.throws(function () { console.log(new RangeNode(0, 10)) }, TypeError) }) diff --git a/test/unit-tests/expression/parse.test.js b/test/unit-tests/expression/parse.test.js index 309ad3512b..36fddf98d1 100644 --- a/test/unit-tests/expression/parse.test.js +++ b/test/unit-tests/expression/parse.test.js @@ -31,6 +31,16 @@ function parseAndStringifyWithParens (expr) { return parse(expr).toString({ parenthesis: 'all' }) } +/** + * Helper to delete sources from a node and return it. We use this because + * "identical" nodes will not be equal if parsed from different strings, + * which breaks deepStrictEqual assertions + */ +function emptySources (node) { + node.sources = [] + return node +} + describe('parse', function () { it('should parse a single expression', function () { approxEqual(parse('2 + 6 / 3').compile().evaluate(), 4) @@ -1103,18 +1113,18 @@ describe('parse', function () { }) it('should parse constants', function () { - assert.strictEqual(parse('true').type, 'ConstantNode') - assert.deepStrictEqual(parse('true'), new ConstantNode(true)) - assert.deepStrictEqual(parse('false'), new ConstantNode(false)) - assert.deepStrictEqual(parse('null'), new ConstantNode(null)) - assert.deepStrictEqual(parse('undefined'), new ConstantNode(undefined)) + assert.strictEqual(emptySources(parse('true')).type, 'ConstantNode') + assert.deepStrictEqual(emptySources(parse('true')), new ConstantNode(true)) + assert.deepStrictEqual(emptySources(parse('false')), new ConstantNode(false)) + assert.deepStrictEqual(emptySources(parse('null')), new ConstantNode(null)) + assert.deepStrictEqual(emptySources(parse('undefined')), new ConstantNode(undefined)) }) it('should parse numeric constants', function () { const nanConstantNode = parse('NaN') assert.deepStrictEqual(nanConstantNode.type, 'ConstantNode') assert.ok(isNaN(nanConstantNode.value)) - assert.deepStrictEqual(parse('Infinity'), new ConstantNode(Infinity)) + assert.deepStrictEqual(emptySources(parse('Infinity')), new ConstantNode(Infinity)) }) it('should evaluate constants', function () { @@ -2605,4 +2615,228 @@ describe('parse', function () { assert.strictEqual(mathClone.evaluate('2'), 2) }) + + describe('sources', function () { + it('adds sources for constants', function () { + const parsed = math.parse('4') + assert.deepStrictEqual(parsed.sources, [{ index: 0, text: '4' }]) + }) + + it('adds sources for symbols', function () { + const parsed = math.parse('foo') + assert.deepStrictEqual(parsed.sources, [{ index: 0, text: 'foo' }]) + }) + + it('adds sources for operators', function () { + const parsed = math.parse('1 + 2') + assert.deepStrictEqual(parsed.sources, [{ index: 2, text: '+' }]) + assert.deepStrictEqual(parsed.args[0].sources, [{ index: 0, text: '1' }]) + assert.deepStrictEqual(parsed.args[1].sources, [{ index: 4, text: '2' }]) + }) + + it('adds sources for blocks', function () { + const parsed = math.parse('1 + 1; 2 + 2\n3 + 3') + + // should have a source for each block delimiter + const expected = [ + { index: 5, text: ';' }, + { index: 12, text: '\n' } + ] + + assert.deepStrictEqual(parsed.sources, expected) + }) + + it('adds sources for 1D matrices', function () { + const parsed = math.parse('[1, 2, 3]') + + // should have a source for brackets and item delimiters + const expected = [ + { index: 0, text: '[' }, + { index: 2, text: ',' }, + { index: 5, text: ',' }, + { index: 8, text: ']' } + ] + + assert.deepStrictEqual(parsed.sources, expected) + }) + + it('adds sources for 2D matrices', function () { + const parsed = math.parse('[1, 2; 3, 4]') + + // outer matrix has sources for brackets and row delimeters + const expected = [ + { index: 0, text: '[' }, + { index: 5, text: ';' }, + { index: 11, text: ']' } + ] + + assert.deepStrictEqual(parsed.sources, expected) + + // inner matrices only have sources for item delimeters + assert.deepStrictEqual(parsed.items[0].sources, [{ index: 2, text: ',' }]) + assert.deepStrictEqual(parsed.items[1].sources, [{ index: 8, text: ',' }]) + }) + + it('adds sources for empty matrices', function () { + const parsed = math.parse('[]') + + // should have a source for brackets and item delimiters + const expected = [ + { index: 0, text: '[' }, + { index: 1, text: ']' } + ] + + assert.deepStrictEqual(parsed.sources, expected) + }) + + it('adds sources for ranges', function () { + const parsed = math.parse('1:2:3') + + assert.deepStrictEqual(parsed.start.sources, [{ index: 0, text: '1' }]) + assert.deepStrictEqual(parsed.step.sources, [{ index: 2, text: '2' }]) + assert.deepStrictEqual(parsed.end.sources, [{ index: 4, text: '3' }]) + + const delimiters = [ + { index: 1, text: ':' }, + { index: 3, text: ':' } + ] + + assert.deepStrictEqual(parsed.sources, delimiters) + + // implicit start and end sources point to where the value would be + + const implicitStart = math.parse(':1') + assert.deepStrictEqual(implicitStart.start.sources, [{ index: 0, text: '' }]) + + const implicitEnd = math.parse('1:') + assert.deepStrictEqual(implicitEnd.end.sources, [{ index: 2, text: '' }]) + }) + + it('adds sources for parentheses', function () { + // should properly match outer and inner parentheses + const outerParen = math.parse('( 1 + (2 + 3))') + const outerSources = [ + { index: 0, text: '(' }, + { index: 13, text: ')' } + ] + assert.deepStrictEqual(outerParen.sources, outerSources) + + const innerParen = outerParen.content.args[1] + const innerSources = [ + { index: 6, text: '(' }, + { index: 12, text: ')' } + ] + assert.deepStrictEqual(innerParen.sources, innerSources) + }) + + it('adds sources for the conditional operator', function () { + // should properly match outer and inner conditional delimeters + const outerCond = math.parse('true ? (false ? 1 : 2) : 3') + const outerSources = [ + { index: 5, text: '?' }, + { index: 23, text: ':' } + ] + assert.deepStrictEqual(outerCond.sources, outerSources) + + const innerCond = outerCond.trueExpr.content + const innerSources = [ + { index: 14, text: '?' }, + { index: 18, text: ':' } + ] + assert.deepStrictEqual(innerCond.sources, innerSources) + }) + + it('adds sources for assignments', function () { + const parsed = math.parse('val = 42') + assert.deepStrictEqual(parsed.sources, [{ index: 4, text: '=' }]) + }) + + it('adds sources for percents', function () { + const parsed = math.parse('13%') + assert.deepStrictEqual(parsed.sources, [{ index: 2, text: '%' }]) + }) + + it('adds sources for implicit multiplication', function () { + const parsed = math.parse('2a') + + // index is where the multiplication symbol would be + assert.deepStrictEqual(parsed.sources, [{ index: 1, text: '' }]) + }) + + it('adds sources for conversions', function () { + const parsedTo = math.parse('1 foot to in') + assert.deepStrictEqual(parsedTo.sources, [{ index: 7, text: 'to' }]) + + const parsedIn = math.parse('in in 1 foot') + assert.deepStrictEqual(parsedIn.sources, [{ index: 3, text: 'in' }]) + }) + + it('adds sources for unary operators', function () { + const unaryPlus = math.parse('+1') + const unaryMinus = math.parse('-1') + assert.deepStrictEqual(unaryPlus.sources, [{ index: 0, text: '+' }]) + assert.deepStrictEqual(unaryMinus.sources, [{ index: 0, text: '-' }]) + }) + + it('adds sources for power operators', function () { + const parsed = math.parse('2^4') + assert.deepStrictEqual(parsed.sources, [{ index: 1, text: '^' }]) + }) + + it('adds sources for constants', function () { + const parsedTrue = math.parse('true') + assert.deepStrictEqual(parsedTrue.sources, [{ index: 0, text: 'true' }]) + + const parsedNull = math.parse('null') + assert.deepStrictEqual(parsedNull.sources, [{ index: 0, text: 'null' }]) + + const parsedInfinity = math.parse('Infinity') + assert.deepStrictEqual(parsedInfinity.sources, [{ index: 0, text: 'Infinity' }]) + + const parsedNaN = math.parse('NaN') + assert.deepStrictEqual(parsedNaN.sources, [{ index: 0, text: 'NaN' }]) + }) + + it('adds sources for function calls', function () { + const parsed = math.parse('foo(1, 2)') + + // should have sources for parens and each param delimeter + const sources = [ + { index: 3, text: '(' }, + { index: 5, text: ',' }, + { index: 8, text: ')' } + ] + assert.deepStrictEqual(parsed.sources, sources) + }) + + it('adds sources for string literals', function () { + const singleQuote = math.parse("'hello'") + const singleSources = [ + { index: 0, text: "'" }, + { index: 6, text: "'" } + ] + assert.deepStrictEqual(singleQuote.sources, singleSources) + + const doubleQuote = math.parse('"hello"') + const doubleSources = [ + { index: 0, text: '"' }, + { index: 6, text: '"' } + ] + assert.deepStrictEqual(doubleQuote.sources, doubleSources) + }) + + it('adds sources for objects', function () { + const parsed = math.parse('{ foo: 13, bar: 25 }') + + // sources include brackets, key-value delimiters, and entry delimeters + const expected = [ + { index: 0, text: '{' }, + { index: 5, text: ':' }, + { index: 9, text: ',' }, + { index: 14, text: ':' }, + { index: 19, text: '}' } + ] + assert.deepStrictEqual(parsed.sources, expected) + }) + }) }) diff --git a/test/unit-tests/function/algebra/resolve.test.js b/test/unit-tests/function/algebra/resolve.test.js index c0e73b7504..981679fe9c 100644 --- a/test/unit-tests/function/algebra/resolve.test.js +++ b/test/unit-tests/function/algebra/resolve.test.js @@ -4,6 +4,30 @@ import assert from 'assert' import math from '../../../../src/defaultInstance.js' import { simplifyAndCompare } from './simplify.test.js' +import { defaultMetaOptions } from '../../../../src/expression/node/Node.js' + +function emptySources (...args) { + return args.map((item) => { + if (item.traverse != null) { + return emptySourcesFromTree(item) + } else if (item.forEach != null) { + return emptySourcesFromArray(item) + } else { + return item.clone(defaultMetaOptions) + } + }) +} + +function emptySourcesFromArray (array) { + return array.map((item) => emptySources(item)) +} + +function emptySourcesFromTree (tree) { + return tree.transform(function (node) { + if (node === tree) return node + return node.clone(defaultMetaOptions) + }).clone(defaultMetaOptions) +} describe('resolve', function () { it('should substitute scoped constants', function () { @@ -45,21 +69,24 @@ describe('resolve', function () { it('should operate directly on strings', function () { const collapsingScope = { x: math.parse('y'), y: math.parse('z') } - assert.deepStrictEqual(math.resolve('x+y', { x: 1 }), math.parse('1 + y')) - assert.deepStrictEqual( + assert.deepStrictEqual(...emptySources(math.resolve('x+y', { x: 1 }), math.parse('1 + y'))) + assert.deepStrictEqual(...emptySources( math.resolve('x + y', collapsingScope), - math.parse('z + z')) - assert.deepStrictEqual( + math.parse('z + z') + )) + assert.deepStrictEqual(...emptySources( math.resolve('[x, y, 1, w]', collapsingScope), - math.parse('[z, z, 1, w]')) + math.parse('[z, z, 1, w]') + )) }) it('should substitute scoped constants from Map like scopes', function () { assert.strictEqual( math.resolve(math.parse('x+y'), new Map([['x', 1]])).toString(), '1 + y' ) // direct - assert.deepStrictEqual( + assert.deepStrictEqual(...emptySources( math.resolve('x+y', new Map([['x', 1]])), math.parse('1 + y')) + ) simplifyAndCompare('x+y', 'x+y', new Map()) // operator simplifyAndCompare('x+y', 'y+1', new Map([['x', 1]])) simplifyAndCompare('x+y', 'y+1', new Map([['x', math.parse('1')]])) @@ -70,16 +97,16 @@ describe('resolve', function () { const scope = { x: 1, y: 2 } const expressions = [parse('x+z'), 'y+z', 'y-x'] let results = [parse('x+z'), parse('y+z'), parse('y-x')] - assert.deepStrictEqual(math.resolve(expressions), results) + assert.deepStrictEqual(...emptySources(math.resolve(expressions), results)) results = [parse('1+z'), parse('2+z'), parse('2-1')] - assert.deepStrictEqual(math.resolve(expressions, scope), results) - assert.deepStrictEqual( + assert.deepStrictEqual(...emptySources(math.resolve(expressions, scope), results)) + assert.deepStrictEqual(...emptySources( math.resolve(math.matrix(expressions), scope), math.matrix(results) - ) + )) const nested = ['z/y', ['x+x', 'gcd(x,y)'], '3+x'] results = [parse('z/2'), [parse('1+1'), parse('gcd(1,2)')], parse('3+1')] - assert.deepStrictEqual(math.resolve(nested, scope), results) + assert.deepStrictEqual(...emptySources(math.resolve(nested, scope), results)) }) it('should throw a readable error if one item is wrong type', function () { @@ -102,4 +129,15 @@ describe('resolve', function () { }), /ReferenceError.*\{x, y, z\}/) }) + + it('should set blank sources for resolved values', function () { + const resolved = math.resolve('1 + x', { x: 5 }) + + // standard nodes should still have sources + assert.deepStrictEqual(resolved.sources, [{ index: 2, text: '+' }]) + assert.deepStrictEqual(resolved.args[0].sources, [{ index: 0, text: '1' }]) + + // resolved variable should have no sources + assert.deepStrictEqual(resolved.args[1].sources, []) + }) }) diff --git a/types/index.d.ts b/types/index.d.ts index 2049487bae..9d12913869 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -41,6 +41,17 @@ export interface FactoryFunctionMap { [key: string]: FactoryFunction | FactoryFunctionMap } +// Maps a parsed node back to its place in the original source string +export interface SourceMapping { + index: number + text: string +} + +// Additional options when building or cloning a node +export interface MetaOptions { + sources: SourceMapping[] +} + /** Available options for parse */ export interface ParseOptions { /** a set of custom nodes */ From 1efd653d8a1dd2922ef118c17e0e696387c98bf1 Mon Sep 17 00:00:00 2001 From: Sergey Solovyev Date: Mon, 20 Oct 2025 12:05:33 +0200 Subject: [PATCH 2/3] Add a config option to disable source tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In some cases it's important to parse data as quickly as possible. As source tracing impacts the `parse` performance I've decided to add a new config option that allows disabling the feature. I set `traceSources` to `false` and ran `expression_parser.js` benchmark and got the following result: ``` (plain js) evaluate 0.04 µs ±0.01% (mathjs) evaluate 0.25 µs ±0.30% (mathjs) parse, compile, evaluate 7.22 µs ±0.79% (mathjs) parse, compile 7.19 µs ±5.51% (mathjs) parse 6.05 µs ±12.34% ``` If `traceSources` is set to `true` I'm getting these times: ``` (plain js) evaluate 0.04 µs ±0.00% (mathjs) evaluate 0.25 µs ±0.19% (mathjs) parse, compile, evaluate 7.25 µs ±0.59% (mathjs) parse, compile 7.59 µs ±10.22% (mathjs) parse 6.42 µs ±18.91% ``` The control times (prior to the changes in this PR): ``` (plain js) evaluate 0.04 µs ±0.00% (mathjs) evaluate 0.25 µs ±0.27% (mathjs) parse, compile, evaluate 6.48 µs ±15.36% (mathjs) parse, compile 6.42 µs ±14.74% (mathjs) parse 4.94 µs ±13.23% ``` --- src/core/config.js | 9 ++++- src/core/create.js | 2 + src/core/function/config.js | 7 ++++ src/expression/parse.js | 61 +++++++++++++++-------------- test/benchmark/expression_parser.js | 2 +- test/benchmark/load.js | 2 +- test/benchmark/matrix_operations.js | 2 +- 7 files changed, 51 insertions(+), 34 deletions(-) diff --git a/src/core/config.js b/src/core/config.js index f777ae6181..48fc14eea7 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -33,5 +33,12 @@ export const DEFAULT_CONFIG = { // legacy behavior for matrix subset. When true, the subset function // returns a matrix or array with the same size as the index (except for scalars). // When false, it returns a matrix or array with a size depending on the type of index. - legacySubset: false + legacySubset: false, + + // If set to `true` (the default value), `parse` records information about + // the original source location of each `Node` in the parsed string. See `SourceMapping` and `Node#sources`. + // If set to `false`, `Node#sources` is always be empty. + // The only time you want to set this to `false` is when you want to speed up parsing of + // a large amount of text. + traceSources: true } diff --git a/src/core/create.js b/src/core/create.js index bb3a7bbe04..e8931ed9cd 100644 --- a/src/core/create.js +++ b/src/core/create.js @@ -90,6 +90,8 @@ import { importFactory } from './function/import.js' * {string} randomSeed * Random seed for seeded pseudo random number generator. * Set to null to randomly seed. + * {boolean} traceSources + * Enables node's source tracing in the parsed string. Slows down parsing a bit. * @returns {Object} Returns a bare-bone math.js instance containing * functions: * - `import` to add new functions diff --git a/src/core/function/config.js b/src/core/function/config.js index 5be65558f6..11ab2b7659 100644 --- a/src/core/function/config.js +++ b/src/core/function/config.js @@ -47,6 +47,8 @@ export function configFactory (config, emit) { * {string} randomSeed * Random seed for seeded pseudo random number generator. * Set to null to randomly seed. + * {boolean} traceSources + * Enables node's source tracing in the parsed string. Slows down parsing a bit. * @return {Object} Returns the current configuration */ function _config (options) { @@ -71,6 +73,11 @@ export function configFactory (config, emit) { validateOption(options, 'matrix', MATRIX_OPTIONS) validateOption(options, 'number', NUMBER_OPTIONS) + if (options.traceSources !== undefined) { + if (typeof options.traceSources !== 'boolean') { + console.warn('Warning: The configuration option "traceSources" must be a boolean.') + } + } // merge options deepExtend(config, options) diff --git a/src/expression/parse.js b/src/expression/parse.js index a948edd58a..6985915891 100644 --- a/src/expression/parse.js +++ b/src/expression/parse.js @@ -221,7 +221,8 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ * @private */ function tokenSource (state) { - return { index: state.index - state.token.length, text: state.token } + if (config.traceSources) return { index: state.index - state.token.length, text: state.token } + else return {} } /** @@ -630,7 +631,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ function parseBlock (state) { let node const blocks = [] - const sources = [] + const sources = config.traceSources ? [] : null let visible if (state.token !== '' && state.token !== '\n' && state.token !== ';') { @@ -642,7 +643,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // TODO: simplify this loop while (state.token === '\n' || state.token === ';') { // eslint-disable-line no-unmodified-loop-condition - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) if (blocks.length === 0 && node) { visible = (state.token !== ';') @@ -665,7 +666,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ return new BlockNode(blocks, { sources }) } else { if (!node) { - node = new ConstantNode(undefined, { sources: [{ index: 0, text: '' }] }) + node = new ConstantNode(undefined, { sources: config.traceSources ? [{ index: 0, text: '' }] : null }) if (state.comment) { node.comment = state.comment } @@ -893,9 +894,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ '>=': 'largerEq' } - const sources = [] + const sources = config.traceSources ? [] : null while (hasOwnProperty(operators, state.token)) { // eslint-disable-line no-unmodified-loop-condition - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) const cond = { name: state.token, fn: operators[state.token] } conditionals.push(cond) getTokenSkipNewline(state) @@ -1000,10 +1001,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // we ignore the range operator when a conditional operator is being processed on the same level params.push(node) - const sources = [] + const sources = config.traceSources ? [] : null // parse step and end while (state.token === ':' && params.length < 3) { // eslint-disable-line no-unmodified-loop-condition - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) getTokenSkipNewline(state) if (state.token === ')' || state.token === ']' || state.token === ',' || state.token === '') { @@ -1359,7 +1360,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.tokenType === TOKENTYPE.SYMBOL && hasOwnProperty(state.extraNodes, state.token)) { const CustomNode = state.extraNodes[state.token] - const sources = [tokenSource(state)] + const sources = config.traceSources ? [tokenSource(state)] : null getToken(state) @@ -1376,7 +1377,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // parse a list with parameters while (state.token === ',') { // eslint-disable-line no-unmodified-loop-condition - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) getToken(state) params.push(parseAssignment(state)) } @@ -1385,7 +1386,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ')') { throw createSyntaxError(state, 'Parenthesis ) expected') } - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) closeParams(state) getToken(state) } @@ -1447,7 +1448,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ function parseAccessors (state, node, types) { let params - const sources = [tokenSource(state)] + const sources = config.traceSources ? [tokenSource(state)] : null while ((state.token === '(' || state.token === '[' || state.token === '.') && (!types || types.includes(state.token))) { // eslint-disable-line no-unmodified-loop-condition params = [] @@ -1463,7 +1464,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // parse a list with parameters while (state.token === ',') { // eslint-disable-line no-unmodified-loop-condition - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) getToken(state) params.push(parseAssignment(state)) } @@ -1472,7 +1473,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ')') { throw createSyntaxError(state, 'Parenthesis ) expected') } - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) closeParams(state) getToken(state) @@ -1503,7 +1504,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ']') { throw createSyntaxError(state, 'Parenthesis ] expected') } - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) closeParams(state) getToken(state) @@ -1561,7 +1562,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ * @return {{token: string, sources: SourceMapping[]}} */ function parseStringToken (state, quote) { - const sources = [tokenSource(state)] + const sources = config.traceSources ? [tokenSource(state)] : null let str = '' while (currentCharacter(state) !== '' && currentCharacter(state) !== quote) { @@ -1597,7 +1598,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== quote) { throw createSyntaxError(state, `End of string ${quote} expected`) } - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) getToken(state) return { token: str, sources } @@ -1612,7 +1613,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let array, params, rows, cols if (state.token === '[') { - const sources = [tokenSource(state)] + const sources = config.traceSources ? [tokenSource(state)] : null // matrix [...] openParams(state) getToken(state) @@ -1628,7 +1629,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // the rows of the matrix are separated by dot-comma's while (state.token === ';') { // eslint-disable-line no-unmodified-loop-condition - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) getToken(state) if (state.token !== ']') { @@ -1641,7 +1642,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ throw createSyntaxError(state, 'End of matrix ] expected') } - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) closeParams(state) getToken(state) @@ -1662,7 +1663,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } // merge the [] sources with the ,,, sources from parseRow - row.sources = [...sources, ...row.sources, tokenSource(state)] + row.sources = [...(sources ?? []), ...row.sources, tokenSource(state)] closeParams(state) getToken(state) @@ -1670,7 +1671,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } } else { // this is an empty matrix "[ ]" - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) closeParams(state) getToken(state) array = new ArrayNode([], { sources }) @@ -1690,9 +1691,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ const params = [parseAssignment(state)] let len = 1 - const sources = [] + const sources = config.traceSources ? [] : null while (state.token === ',') { // eslint-disable-line no-unmodified-loop-condition - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) getToken(state) // parse expression @@ -1712,14 +1713,14 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ */ function parseObject (state) { if (state.token === '{') { - const sources = [tokenSource(state)] + const sources = config.traceSources ? [tokenSource(state)] : null openParams(state) let key const properties = {} do { if (state.token === ',') { - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) } getToken(state) @@ -1739,7 +1740,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token !== ':') { throw createSyntaxError(state, 'Colon : expected after object key') } - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) getToken(state) // parse key @@ -1752,7 +1753,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ throw createSyntaxError(state, 'Comma , or bracket } expected after object value') } - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) closeParams(state) getToken(state) @@ -1801,7 +1802,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // check if it is a parenthesized expression if (state.token === '(') { - const sources = [tokenSource(state)] + const sources = config.traceSources ? [tokenSource(state)] : null // parentheses (...) openParams(state) getToken(state) @@ -1812,7 +1813,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ throw createSyntaxError(state, 'Parenthesis ) expected') } - sources.push(tokenSource(state)) + sources?.push(tokenSource(state)) closeParams(state) getToken(state) diff --git a/test/benchmark/expression_parser.js b/test/benchmark/expression_parser.js index 222341f98a..94faa8b42e 100644 --- a/test/benchmark/expression_parser.js +++ b/test/benchmark/expression_parser.js @@ -8,7 +8,7 @@ import { all, create } from '../../lib/esm/index.js' import { getSafeProperty } from '../../lib/esm/utils/customs.js' import { formatTaskResult } from './utils/formatTaskResult.js' -const math = create(all) +const math = create(all, { traceSources: false }) const expr = '2 + 3 * sin(pi / 4) - 4x' const scope = new Map([ diff --git a/test/benchmark/load.js b/test/benchmark/load.js index 8d4e6811a4..08a3d2652b 100644 --- a/test/benchmark/load.js +++ b/test/benchmark/load.js @@ -6,7 +6,7 @@ import { formatTaskResult } from './utils/formatTaskResult.js' const timeLabel = 'import, parse, and load time' console.time(timeLabel) -const math = create(all) +const math = create(all, { traceSources: false }) console.timeEnd(timeLabel) let calls diff --git a/test/benchmark/matrix_operations.js b/test/benchmark/matrix_operations.js index e0c189ab46..28a18d64df 100644 --- a/test/benchmark/matrix_operations.js +++ b/test/benchmark/matrix_operations.js @@ -22,7 +22,7 @@ import { all, create } from '../../lib/esm/index.js' import { formatTaskResult } from './utils/formatTaskResult.js' const bench = new Bench({ time: 10, iterations: 100 }) -const math = create(all) +const math = create(all, { traceSources: false }) // fiedler matrix 25 x 25 const fiedler = [ From d3427ea448a04d36514eec84d75b3786c753e12c Mon Sep 17 00:00:00 2001 From: Sergey Solovyev Date: Wed, 22 Oct 2025 11:11:25 +0200 Subject: [PATCH 3/3] Improve performance of parse.js when source tracing is disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It looks like the hotspot was `tokenSource` method and the object allocation followed by the method call: `{ sources: [tokenSource(state)] }`. Instead of using `tokenSource` directly let's add a new method `tokenSourceMetaOptions` and use it. This method returns a preallocated `defaultMetaOptions` if source tracing is disabled. `expression_parser.js` benchmark BEFORE this change: ``` (plain js) evaluate 0.04 µs ±0.00% (mathjs) evaluate 0.26 µs ±31.65% (mathjs) parse, compile, evaluate 6.23 µs ±13.74% (mathjs) parse, compile 6.01 µs ±10.45% (mathjs) parse 5.31 µs ±27.41% ``` AFTER this change: ``` (plain js) evaluate 0.04 µs ±0.00% (mathjs) evaluate 0.25 µs ±15.79% (mathjs) parse, compile, evaluate 6.00 µs ±0.41% (mathjs) parse, compile 5.89 µs ±6.29% (mathjs) parse 4.66 µs ±5.90% ``` BEFORE all the changes in the PR (control group): ``` (plain js) evaluate 0.04 µs ±0.00% (mathjs) evaluate 0.24 µs ±0.09% (mathjs) parse, compile, evaluate 4.89 µs ±0.28% (mathjs) parse, compile 4.97 µs ±10.20% (mathjs) parse 3.61 µs ±6.10% ``` --- src/expression/parse.js | 91 ++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 37 deletions(-) diff --git a/src/expression/parse.js b/src/expression/parse.js index 6985915891..e20aa96a98 100644 --- a/src/expression/parse.js +++ b/src/expression/parse.js @@ -3,6 +3,7 @@ import { isAccessorNode, isConstantNode, isFunctionNode, isOperatorNode, isSymbo import { deepMap } from '../utils/collection.js' import { safeNumberType } from '../utils/number.js' import { hasOwnProperty } from '../utils/object.js' +import { defaultMetaOptions } from './node/Node.js' const name = 'parse' const dependencies = [ @@ -225,6 +226,22 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ else return {} } + /** + * Returns a MetaOptions object containing source mapping for the current token. + * + * This is a convenience function that wraps tokenSource() in the MetaOptions format + * required by node constructors. It respects the config.traceSources flag to enable + * or disable source tracing at runtime. + * + * When source tracing is enabled, this function creates a MetaOptions object containing + * a single SourceMapping for the current token. When disabled, it returns the default + * empty MetaOptions to avoid the performance overhead of creating source mappings. + */ + function tokenSourceMetaOptions (state) { + if (config.traceSources) return { sources: [tokenSource(state)] } + else return defaultMetaOptions + } + /** * View upto `length` characters of the expression starting at the current character. * @@ -689,7 +706,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ const node = parseConditional(state) - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) if (state.token === '=') { if (isSymbolNode(node)) { @@ -698,12 +715,12 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ const symbolNode = new SymbolNode(name, { sources: node.sources }) getTokenSkipNewline(state) value = parseAssignment(state) - return new AssignmentNode(symbolNode, value, null, { sources: [source] }) + return new AssignmentNode(symbolNode, value, null, meta) } else if (isAccessorNode(node)) { // parse a matrix subset assignment like 'A[1,2] = 4' getTokenSkipNewline(state) value = parseAssignment(state) - return new AssignmentNode(node.object, node.index, value, { sources: [source] }) + return new AssignmentNode(node.object, node.index, value, meta) } else if (isFunctionNode(node) && isSymbolNode(node.fn)) { // parse function assignment like 'f(x) = x^2' valid = true @@ -721,7 +738,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (valid) { getTokenSkipNewline(state) value = parseAssignment(state) - return new FunctionAssignmentNode(name, args, value, { sources: [source] }) + return new FunctionAssignmentNode(name, args, value, meta) } } @@ -784,9 +801,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseLogicalXor(state) while (state.token === 'or') { // eslint-disable-line no-unmodified-loop-condition - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) - node = new OperatorNode('or', 'or', [node, parseLogicalXor(state)], false, false, { sources: [source] }) + node = new OperatorNode('or', 'or', [node, parseLogicalXor(state)], false, false, meta) } return node @@ -801,9 +818,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseLogicalAnd(state) while (state.token === 'xor') { // eslint-disable-line no-unmodified-loop-condition - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) - node = new OperatorNode('xor', 'xor', [node, parseLogicalAnd(state)], false, false, { sources: [source] }) + node = new OperatorNode('xor', 'xor', [node, parseLogicalAnd(state)], false, false, meta) } return node @@ -818,9 +835,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseBitwiseOr(state) while (state.token === 'and') { // eslint-disable-line no-unmodified-loop-condition - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) - node = new OperatorNode('and', 'and', [node, parseBitwiseOr(state)], false, false, { sources: [source] }) + node = new OperatorNode('and', 'and', [node, parseBitwiseOr(state)], false, false, meta) } return node @@ -835,9 +852,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseBitwiseXor(state) while (state.token === '|') { // eslint-disable-line no-unmodified-loop-condition - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) - node = new OperatorNode('|', 'bitOr', [node, parseBitwiseXor(state)], false, false, { sources: [source] }) + node = new OperatorNode('|', 'bitOr', [node, parseBitwiseXor(state)], false, false, meta) } return node @@ -852,9 +869,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseBitwiseAnd(state) while (state.token === '^|') { // eslint-disable-line no-unmodified-loop-condition - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) - node = new OperatorNode('^|', 'bitXor', [node, parseBitwiseAnd(state)], false, false, { sources: [source] }) + node = new OperatorNode('^|', 'bitXor', [node, parseBitwiseAnd(state)], false, false, meta) } return node @@ -869,9 +886,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node = parseRelational(state) while (state.token === '&') { // eslint-disable-line no-unmodified-loop-condition - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) - node = new OperatorNode('&', 'bitAnd', [node, parseRelational(state)], false, false, { sources: [source] }) + node = new OperatorNode('&', 'bitAnd', [node, parseRelational(state)], false, false, meta) } return node @@ -932,11 +949,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ name = state.token fn = operators[name] - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) params = [node, parseConversion(state)] - node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) + node = new OperatorNode(name, fn, params, false, false, meta) } return node @@ -959,7 +976,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (hasOwnProperty(operators, state.token)) { name = state.token - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) fn = operators[name] getTokenSkipNewline(state) @@ -967,11 +984,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (name === 'in' && '])},;'.includes(state.token)) { // end of expression -> this is the unit 'in' ('inch') // no source mapping because this * operator is not explicitly in the source expression - node = new OperatorNode('*', 'multiply', [node, new SymbolNode('in', { sources: [source] })], true) + node = new OperatorNode('*', 'multiply', [node, new SymbolNode('in', meta)], true) } else { // operator 'a to b' or 'a in b' params = [node, parseRange(state)] - node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) + node = new OperatorNode(name, fn, params, false, false, meta) } } @@ -1047,7 +1064,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (hasOwnProperty(operators, state.token)) { name = state.token fn = operators[name] - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) const rightNode = parseMultiplyDivideModulus(state) @@ -1057,7 +1074,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } else { params = [node, rightNode] } - node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) + node = new OperatorNode(name, fn, params, false, false, meta) } return node @@ -1088,10 +1105,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // explicit operators name = state.token fn = operators[name] - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) last = parseImplicitMultiplication(state) - node = new OperatorNode(name, fn, [node, last], false, false, { sources: [source] }) + node = new OperatorNode(name, fn, [node, last], false, false, meta) } else { break } @@ -1245,12 +1262,12 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (hasOwnProperty(operators, state.token)) { fn = operators[state.token] name = state.token - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) params = [parseUnary(state)] - return new OperatorNode(name, fn, params, false, false, { sources: [source] }) + return new OperatorNode(name, fn, params, false, false, meta) } return parsePow(state) @@ -1270,11 +1287,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token === '^' || state.token === '.^') { name = state.token fn = (name === '^') ? 'pow' : 'dotPow' - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getTokenSkipNewline(state) params = [node, parseUnary(state)] // Go back to unary, we can have '2^-3' - node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) + node = new OperatorNode(name, fn, params, false, false, meta) } return node @@ -1314,12 +1331,12 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (hasOwnProperty(operators, state.token)) { name = state.token fn = operators[name] - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getToken(state) params = [node] - node = new OperatorNode(name, fn, params, false, false, { sources: [source] }) + node = new OperatorNode(name, fn, params, false, false, meta) node = parseAccessors(state, node) } @@ -1411,16 +1428,16 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ (state.tokenType === TOKENTYPE.DELIMITER && state.token in NAMED_DELIMITERS)) { name = state.token - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getToken(state) if (hasOwnProperty(CONSTANTS, name)) { // true, false, null, ... - node = new ConstantNode(CONSTANTS[name], { sources: [source] }) + node = new ConstantNode(CONSTANTS[name], meta) } else if (NUMERIC_CONSTANTS.includes(name)) { // NaN, Infinity - node = new ConstantNode(numeric(name, 'number'), { sources: [source] }) + node = new ConstantNode(numeric(name, 'number'), meta) } else { - node = new SymbolNode(name, { sources: [source] }) + node = new SymbolNode(name, meta) } // parse function parameters and matrix index @@ -1780,13 +1797,13 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.tokenType === TOKENTYPE.NUMBER) { // this is a number numberStr = state.token - const source = tokenSource(state) + const meta = tokenSourceMetaOptions(state) getToken(state) const numericType = safeNumberType(numberStr, config) const value = numeric(numberStr, numericType) - return new ConstantNode(value, { sources: [source] }) + return new ConstantNode(value, meta) } return parseParentheses(state)