diff --git a/app/gui/e2e/project-view/tableVisualisation.spec.ts b/app/gui/e2e/project-view/tableVisualisation.spec.ts index 73fd06f10f..e0a38df726 100644 --- a/app/gui/e2e/project-view/tableVisualisation.spec.ts +++ b/app/gui/e2e/project-view/tableVisualisation.spec.ts @@ -2,6 +2,7 @@ import { test, type Page } from '@playwright/test' import * as actions from './actions' import { expect } from './customExpect' import { mockExpressionUpdate } from './expressionUpdates' +import { CONTROL_KEY } from './keyboard' import * as locate from './locate' import { graphNodeByBinding } from './locate' @@ -36,19 +37,19 @@ test('Copy from Table Visualization', async ({ page, context }) => { await context.grantPermissions(['clipboard-read', 'clipboard-write']) await actions.goToGraph(page) - actions.openVisualization(page, 'Table') + await actions.openVisualization(page, 'Table') const tableVisualization = locate.tableVisualization(page) await expect(tableVisualization).toExist() await tableVisualization.getByText('0,0').hover() await page.mouse.down() await tableVisualization.getByText('2,1').hover() await page.mouse.up() - await page.keyboard.press('Control+C') + await page.keyboard.press(`${CONTROL_KEY}+C`) // Paste to Node. await actions.clickAtBackground(page) const nodesCount = await locate.graphNode(page).count() - await page.keyboard.press('Control+V') + await page.keyboard.press(`${CONTROL_KEY}+V`) await expect(locate.graphNode(page)).toHaveCount(nodesCount + 1) await expect(locate.graphNode(page).last().locator('input')).toHaveValue( '0,0\t0,11,0\t1,12,0\t2,1', @@ -60,7 +61,7 @@ test('Copy from Table Visualization', async ({ page, context }) => { await expect(widget).toBeVisible() await widget.getByRole('button', { name: 'Add new column' }).click() await widget.locator('.ag-cell', { hasNotText: /0/ }).first().click() - await page.keyboard.press('Control+V') + await page.keyboard.press(`${CONTROL_KEY}+V`) await expect(widget.locator('.ag-cell')).toHaveText([ '0', '0,0', diff --git a/app/gui/src/dashboard/hooks/refreshHooks.ts b/app/gui/src/dashboard/hooks/refreshHooks.ts index 8c4328b344..138ab0f4c0 100644 --- a/app/gui/src/dashboard/hooks/refreshHooks.ts +++ b/app/gui/src/dashboard/hooks/refreshHooks.ts @@ -7,7 +7,6 @@ import * as React from 'react' // This must not be a `symbol` as it cannot be sent to Playright. /** The type of the state returned by {@link useRefresh}. */ -// eslint-disable-next-line @typescript-eslint/no-empty-object-type export interface RefreshState {} /** A hook that contains no state. It is used to trigger React re-renders. */ diff --git a/app/gui/src/dashboard/utilities/LocalStorage.ts b/app/gui/src/dashboard/utilities/LocalStorage.ts index 020bf1b646..b659d529d5 100644 --- a/app/gui/src/dashboard/utilities/LocalStorage.ts +++ b/app/gui/src/dashboard/utilities/LocalStorage.ts @@ -43,7 +43,6 @@ export interface LocalStorageKeyMetadata { * The data that can be stored in a {@link LocalStorage}. * Declaration merge into this interface to add a new key. */ -// eslint-disable-next-line @typescript-eslint/no-empty-object-type export interface LocalStorageData {} // ======================= diff --git a/app/gui/src/project-view/components/ComponentBrowser/ai.ts b/app/gui/src/project-view/components/ComponentBrowser/ai.ts index e94b501f9f..7f72c47a21 100644 --- a/app/gui/src/project-view/components/ComponentBrowser/ai.ts +++ b/app/gui/src/project-view/components/ComponentBrowser/ai.ts @@ -26,7 +26,7 @@ export function useAI( const lsRpc = project.lsRpcConnection const sourceNodeId = graphDb.getIdentDefiningNode(sourceIdentifier) const contextId = - sourceNodeId && graphDb.nodeIdToNode.get(sourceNodeId)?.outerExpr.externalId + sourceNodeId && graphDb.nodeIdToNode.get(sourceNodeId)?.outerAst.externalId if (!contextId) return Err(`Cannot find node with name ${sourceIdentifier}`) const prompt = await withContext( diff --git a/app/gui/src/project-view/components/GraphEditor.vue b/app/gui/src/project-view/components/GraphEditor.vue index d6c3305229..714a27467b 100644 --- a/app/gui/src/project-view/components/GraphEditor.vue +++ b/app/gui/src/project-view/components/GraphEditor.vue @@ -52,6 +52,7 @@ import type { SuggestionId } from '@/stores/suggestionDatabase/entry' import { suggestionDocumentationUrl, type Typename } from '@/stores/suggestionDatabase/entry' import { provideVisualizationStore } from '@/stores/visualization' import { bail } from '@/util/assert' +import { Ast } from '@/util/ast' import type { AstId } from '@/util/ast/abstract' import { colorFromString } from '@/util/colors' import { partition } from '@/util/data/array' @@ -579,7 +580,7 @@ function clearFocus() { function createNodesFromSource(sourceNode: NodeId, options: NodeCreationOptions[]) { const sourcePort = graphStore.db.getNodeFirstOutputPort(sourceNode) if (sourcePort == null) return - const sourcePortAst = graphStore.viewModule.get(sourcePort) + const sourcePortAst = graphStore.viewModule.get(sourcePort) as Ast.Expression const [toCommit, toEdit] = partition(options, (opts) => opts.commit) createNodes( toCommit.map((options: NodeCreationOptions) => ({ @@ -631,14 +632,14 @@ function collapseNodes() { } const selectedNodeRects = filterDefined(Array.from(selected, graphStore.visibleArea)) graphStore.edit((edit) => { - const { refactoredExpressionAstId, collapsedNodeIds, outputAstId } = performCollapse( + const { collapsedCallRoot, collapsedNodeIds, outputAstId } = performCollapse( info.value, edit.getVersion(topLevel), graphStore.db, currentMethodName, ) const position = collapsedNodePlacement(selectedNodeRects) - edit.get(refactoredExpressionAstId).mutableNodeMetadata().set('position', position.xy()) + edit.get(collapsedCallRoot).mutableNodeMetadata().set('position', position.xy()) if (outputAstId != null) { const collapsedNodeRects = filterDefined( Array.from(collapsedNodeIds, graphStore.visibleArea), diff --git a/app/gui/src/project-view/components/GraphEditor/GraphEdges.vue b/app/gui/src/project-view/components/GraphEditor/GraphEdges.vue index 73d217e790..389a66bee9 100644 --- a/app/gui/src/project-view/components/GraphEditor/GraphEdges.vue +++ b/app/gui/src/project-view/components/GraphEditor/GraphEdges.vue @@ -101,7 +101,7 @@ function createEdge(source: AstId, target: PortId) { // Creating this edge would create a circular dependency. Prevent that and display error. toast.error('Could not connect due to circular dependency.') } else { - const identAst = Ast.parse(ident, edit) + const identAst = Ast.parseExpression(ident, edit)! if (!graph.updatePortValue(edit, target, identAst)) { if (isAstId(target)) { console.warn(`Failed to connect edge to port ${target}, falling back to direct edit.`) diff --git a/app/gui/src/project-view/components/GraphEditor/GraphNodeComment.vue b/app/gui/src/project-view/components/GraphEditor/GraphNodeComment.vue index 911f0148df..2c3351ba67 100644 --- a/app/gui/src/project-view/components/GraphEditor/GraphNodeComment.vue +++ b/app/gui/src/project-view/components/GraphEditor/GraphNodeComment.vue @@ -13,10 +13,7 @@ const textEditor = ref>() const textEditorContent = computed(() => textEditor.value?.contentElement) const graphStore = useGraphStore() -const { documentation } = useAstDocumentation( - graphStore, - () => props.node.docs ?? props.node.outerExpr, -) +const { documentation } = useAstDocumentation(graphStore, () => props.node.outerAst) syncRef(editing, useFocusDelayed(textEditorContent).focused) diff --git a/app/gui/src/project-view/components/GraphEditor/GraphVisualization/visualizationData.ts b/app/gui/src/project-view/components/GraphEditor/GraphVisualization/visualizationData.ts index 8ba77ba4eb..6a1e1e7349 100644 --- a/app/gui/src/project-view/components/GraphEditor/GraphVisualization/visualizationData.ts +++ b/app/gui/src/project-view/components/GraphEditor/GraphVisualization/visualizationData.ts @@ -126,7 +126,10 @@ export function useVisualizationData({ const preprocessor = visPreprocessor.value const args = preprocessor.positionalArgumentsExpressions const tempModule = Ast.MutableModule.Transient() - const preprocessorModule = Ast.parse(preprocessor.visualizationModule, tempModule) + const preprocessorModule = Ast.parseExpression( + preprocessor.visualizationModule, + tempModule, + )! // TODO[ao]: it work with builtin visualization, but does not work in general case. // Tracked in https://github.com/orgs/enso-org/discussions/6832#discussioncomment-7754474. if (!isIdentifier(preprocessor.expression)) { @@ -140,9 +143,9 @@ export function useVisualizationData({ ) const preprocessorInvocation = Ast.App.PositionalSequence(preprocessorQn, [ Ast.Wildcard.new(tempModule), - ...args.map((arg) => Ast.Group.new(tempModule, Ast.parse(arg, tempModule))), + ...args.map((arg) => Ast.Group.new(tempModule, Ast.parseExpression(arg, tempModule)!)), ]) - const rhs = Ast.parse(dataSourceValue.expression, tempModule) + const rhs = Ast.parseExpression(dataSourceValue.expression, tempModule)! const expression = Ast.OprApp.new(tempModule, preprocessorInvocation, '<|', rhs) return projectStore.executeExpression(dataSourceValue.contextId, expression.code()) } catch (e) { diff --git a/app/gui/src/project-view/components/GraphEditor/NodeWidgetTree.vue b/app/gui/src/project-view/components/GraphEditor/NodeWidgetTree.vue index ea545d34b7..f18d72cf34 100644 --- a/app/gui/src/project-view/components/GraphEditor/NodeWidgetTree.vue +++ b/app/gui/src/project-view/components/GraphEditor/NodeWidgetTree.vue @@ -13,7 +13,7 @@ import { computed, toRef, watch } from 'vue' import { DisplayIcon } from './widgets/WidgetIcon.vue' const props = defineProps<{ - ast: Ast.Ast + ast: Ast.Expression nodeId: NodeId nodeElement: HTMLElement | undefined nodeType: NodeType diff --git a/app/gui/src/project-view/components/GraphEditor/__tests__/clipboard.test.ts b/app/gui/src/project-view/components/GraphEditor/__tests__/clipboard.test.ts index 9680e814ca..d50615c0be 100644 --- a/app/gui/src/project-view/components/GraphEditor/__tests__/clipboard.test.ts +++ b/app/gui/src/project-view/components/GraphEditor/__tests__/clipboard.test.ts @@ -68,7 +68,7 @@ const testNodeInputs: { { code: '## Documentation\nfoo = 2 + 2' }, ] const testNodes = testNodeInputs.map(({ code, visualization, colorOverride }) => { - const root = Ast.Ast.parse(code) + const root = [...Ast.parseBlock(code).statements()][0]! root.setNodeMetadata({ visualization, colorOverride }) const node = nodeFromAst(root, false) assertDefined(node) @@ -82,7 +82,9 @@ test.each([...testNodes.map((node) => [node]), testNodes])( const clipboardItem = clipboardItemFromTypes(nodesToClipboardData(sourceNodes)) const pastedNodes = await nodesFromClipboardContent([clipboardItem]) sourceNodes.forEach((sourceNode, i) => { - expect(pastedNodes[i]?.documentation).toBe(sourceNode.docs?.documentation()) + const documentation = + sourceNode.outerAst.isStatement() ? sourceNode.outerAst.documentationText() : undefined + expect(pastedNodes[i]?.documentation).toBe(documentation) expect(pastedNodes[i]?.expression).toBe(sourceNode.innerExpr.code()) expect(pastedNodes[i]?.metadata?.colorOverride).toBe(sourceNode.colorOverride) expect(pastedNodes[i]?.metadata?.visualization).toBe(sourceNode.vis) diff --git a/app/gui/src/project-view/components/GraphEditor/__tests__/collapsing.test.ts b/app/gui/src/project-view/components/GraphEditor/__tests__/collapsing.test.ts index 2bf20ad5c9..b8bdf41706 100644 --- a/app/gui/src/project-view/components/GraphEditor/__tests__/collapsing.test.ts +++ b/app/gui/src/project-view/components/GraphEditor/__tests__/collapsing.test.ts @@ -1,11 +1,18 @@ -import { prepareCollapsedInfo } from '@/components/GraphEditor/collapsing' +import { performCollapseImpl, prepareCollapsedInfo } from '@/components/GraphEditor/collapsing' import { GraphDb, type NodeId } from '@/stores/graph/graphDatabase' import { assert } from '@/util/assert' import { Ast, RawAst } from '@/util/ast' +import { findExpressions } from '@/util/ast/__tests__/testCase' import { unwrap } from '@/util/data/result' import { tryIdentifier } from '@/util/qualifiedName' import { expect, test } from 'vitest' import { watchEffect } from 'vue' +import { Identifier } from 'ydoc-shared/ast' +import { nodeIdFromOuterAst } from '../../../stores/graph/graphDatabase' + +// =============================== +// === Collapse Analysis Tests === +// =============================== function setupGraphDb(code: string, graphDb: GraphDb) { const { root, toRaw, getSpan } = Ast.parseExtended(code) @@ -211,3 +218,73 @@ main = expect(refactored.pattern).toEqual('sum') expect(refactored.arguments).toEqual(['input', 'four']) }) + +// ================================ +// === Collapse Execution Tests === +// ================================ + +test('Perform collapse', () => { + const root = Ast.parseModule( + [ + 'main =', + ' keep1 = 1', + ' extract1 = keep1', + ' keep2 = 2', + ' extract2 = extract1 + 1', + ' target = extract2', + ].join('\n'), + ) + root.module.setRoot(root) + const before = findExpressions(root, { + 'keep1 = 1': Ast.Assignment, + 'extract1 = keep1': Ast.Assignment, + 'keep2 = 2': Ast.Assignment, + 'extract2 = extract1 + 1': Ast.Assignment, + 'target = extract2': Ast.Assignment, + }) + const statementsToExtract = new Set() + const statementToReplace = before['target = extract2'].id + statementsToExtract.add(before['extract1 = keep1'].id) + statementsToExtract.add(before['extract2 = extract1 + 1'].id) + statementsToExtract.add(statementToReplace) + const { collapsedCallRoot, outputAstId, collapsedNodeIds } = performCollapseImpl( + root, + { + args: ['keep1' as Identifier], + statementsToExtract, + statementToReplace: before['target = extract2'].id, + }, + 'main', + ) + expect(root.code()).toBe( + [ + '## ICON group', + 'collapsed keep1 =', + ' extract1 = keep1', + ' extract2 = extract1 + 1', + ' target = extract2', + ' target', + '', + 'main =', + ' keep1 = 1', + ' keep2 = 2', + ' target = Main.collapsed keep1', + ].join('\n'), + ) + const after = findExpressions(root, { + 'extract1 = keep1': Ast.Assignment, + 'extract2 = extract1 + 1': Ast.Assignment, + 'target = extract2': Ast.Assignment, + target: Ast.ExpressionStatement, + 'keep1 = 1': Ast.Assignment, + 'keep2 = 2': Ast.Assignment, + 'target = Main.collapsed keep1': Ast.Assignment, + }) + expect(collapsedNodeIds).toStrictEqual( + [after['target = extract2'], after['extract2 = extract1 + 1'], after['extract1 = keep1']].map( + nodeIdFromOuterAst, + ), + ) + expect(outputAstId).toBe(after['target'].expression.id) + expect(collapsedCallRoot).toBe(after['target = Main.collapsed keep1'].expression.id) +}) diff --git a/app/gui/src/project-view/components/GraphEditor/clipboard.ts b/app/gui/src/project-view/components/GraphEditor/clipboard.ts index 5685ce3b9e..c4673d8b9b 100644 --- a/app/gui/src/project-view/components/GraphEditor/clipboard.ts +++ b/app/gui/src/project-view/components/GraphEditor/clipboard.ts @@ -148,7 +148,7 @@ const spreadsheetDecoder: ClipboardDecoder = { }, } -const toTable = computed(() => Pattern.parse('__.to Table')) +const toTable = computed(() => Pattern.parseExpression('__.to Table')) /** Create Enso Expression generating table from this tsvData. */ export function tsvTableToEnsoExpression(tsvData: string) { @@ -186,9 +186,10 @@ export function writeClipboard(data: MimeData) { // === Serializing nodes === function nodeStructuredData(node: Node): CopiedNode { + const documentation = node.outerAst.isStatement() ? node.outerAst.documentationText() : undefined return { expression: node.innerExpr.code(), - documentation: node.docs?.documentation(), + documentation, metadata: node.rootExpr.serializeMetadata(), ...(node.pattern ? { binding: node.pattern.code() } : {}), } @@ -204,6 +205,6 @@ export function clipboardNodeData(nodes: CopiedNode[]): MimeData { export function nodesToClipboardData(nodes: Node[]): MimeData { return { ...clipboardNodeData(nodes.map(nodeStructuredData)), - 'text/plain': nodes.map((node) => node.outerExpr.code()).join('\n'), + 'text/plain': nodes.map((node) => node.outerAst.code()).join('\n'), } } diff --git a/app/gui/src/project-view/components/GraphEditor/collapsing.ts b/app/gui/src/project-view/components/GraphEditor/collapsing.ts index 84de430131..689bc491e3 100644 --- a/app/gui/src/project-view/components/GraphEditor/collapsing.ts +++ b/app/gui/src/project-view/components/GraphEditor/collapsing.ts @@ -1,14 +1,9 @@ -import { asNodeId, GraphDb, nodeIdFromOuterExpr, type NodeId } from '@/stores/graph/graphDatabase' -import { assert, assertDefined } from '@/util/assert' +import { GraphDb, NodeId, nodeIdFromOuterAst } from '@/stores/graph/graphDatabase' +import { assert } from '@/util/assert' import { Ast } from '@/util/ast' -import { autospaced, isIdentifier, moduleMethodNames, type Identifier } from '@/util/ast/abstract' -import { filterDefined } from '@/util/data/iterable' -import { Err, Ok, unwrap, type Result } from '@/util/data/result' -import { - isIdentifierOrOperatorIdentifier, - tryIdentifier, - type IdentifierOrOperatorIdentifier, -} from '@/util/qualifiedName' +import { Identifier, isIdentifier, moduleMethodNames } from '@/util/ast/abstract' +import { Err, Ok, Result, unwrap } from '@/util/data/result' +import { tryIdentifier } from '@/util/qualifiedName' import * as set from 'lib0/set' // === Types === @@ -24,7 +19,7 @@ interface ExtractedInfo { /** Nodes with these ids should be moved to the function body, in their original order. */ ids: Set /** The output information of the function. */ - output: Output | null + output: Output /** The list of extracted function’s argument names. */ inputs: Identifier[] } @@ -110,9 +105,11 @@ export function prepareCollapsedInfo( output = { node: arbitraryLeaf, identifier } } - const pattern = graphDb.nodeIdToNode.get(output.node)?.pattern?.code() ?? '' - assert(isIdentifier(pattern)) + const pattern = graphDb.nodeIdToNode.get(output.node)?.pattern?.code() + assert(pattern != null && isIdentifier(pattern)) const inputs = Array.from(inputSet) + + assert(selected.has(output.node)) return Ok({ extracted: { ids: selected, @@ -128,10 +125,7 @@ export function prepareCollapsedInfo( } /** Generate a safe method name for a collapsed function using `baseName` as a prefix. */ -function findSafeMethodName( - topLevel: Ast.BodyBlock, - baseName: IdentifierOrOperatorIdentifier, -): IdentifierOrOperatorIdentifier { +function findSafeMethodName(topLevel: Ast.BodyBlock, baseName: Identifier): Identifier { const allIdentifiers = moduleMethodNames(topLevel) if (!allIdentifiers.has(baseName)) { return baseName @@ -141,107 +135,98 @@ function findSafeMethodName( index++ } const name = `${baseName}${index}` - assert(isIdentifierOrOperatorIdentifier(name)) + assert(isIdentifier(name)) return name } // === performCollapse === // We support working inside `Main` module of the project at the moment. -const MODULE_NAME = 'Main' as IdentifierOrOperatorIdentifier -const COLLAPSED_FUNCTION_NAME = 'collapsed' as IdentifierOrOperatorIdentifier +const MODULE_NAME = 'Main' as Identifier +const COLLAPSED_FUNCTION_NAME = 'collapsed' as Identifier interface CollapsingResult { /** The ID of the node refactored to the collapsed function call. */ - refactoredNodeId: NodeId - refactoredExpressionAstId: Ast.AstId + collapsedCallRoot: Ast.AstId /** * IDs of nodes inside the collapsed function, except the output node. * The order of these IDs is reversed comparing to the order of nodes in the source code. */ collapsedNodeIds: NodeId[] /** ID of the output AST node inside the collapsed function. */ - outputAstId?: Ast.AstId | undefined + outputAstId: Ast.AstId +} + +interface PreparedCollapseInfo { + args: Identifier[] + statementsToExtract: Set + statementToReplace: Ast.AstId } /** Perform the actual AST refactoring for collapsing nodes. */ export function performCollapse( info: CollapsedInfo, topLevel: Ast.MutableBodyBlock, - db: GraphDb, + graphDb: GraphDb, currentMethodName: string, ): CollapsingResult { + const nodeIdToStatementId = (nodeId: NodeId) => graphDb.nodeIdToNode.get(nodeId)!.outerAst.id + const preparedInfo = { + args: info.extracted.inputs, + statementsToExtract: new Set([...info.extracted.ids].map(nodeIdToStatementId)), + statementToReplace: nodeIdToStatementId(info.refactored.id), + outputIdentifier: info.extracted.output.identifier, + } + return performCollapseImpl(topLevel, preparedInfo, currentMethodName) +} + +/** @internal */ +export function performCollapseImpl( + topLevel: Ast.MutableBodyBlock, + info: PreparedCollapseInfo, + currentMethodName: string, +) { const edit = topLevel.module - const functionAst = Ast.findModuleMethod(topLevel, currentMethodName) - assertDefined(functionAst) - const functionBlock = edit.getVersion(functionAst).bodyAsBlock() - const posToInsert = findInsertionPos(topLevel, currentMethodName) const collapsedName = findSafeMethodName(topLevel, COLLAPSED_FUNCTION_NAME) - const astIdsToExtract = new Set( - [...info.extracted.ids].map((nodeId) => db.nodeIdToNode.get(nodeId)?.outerExpr.id), - ) - const astIdToReplace = db.nodeIdToNode.get(info.refactored.id)?.outerExpr.id - const { - ast: refactoredAst, - nodeId: refactoredNodeId, - expressionAstId: refactoredExpressionAstId, - } = collapsedCallAst(info, collapsedName, edit) - const collapsed: Ast.Owned[] = [] + const { statement: currentMethod, index: currentMethodLine } = Ast.findModuleMethod( + topLevel, + currentMethodName, + )! + // Update the definition of the refactored function. - functionBlock.updateLines((lines) => { - const refactored: Ast.OwnedBlockLine[] = [] - for (const line of lines) { - const ast = line.expression?.node - if (!ast) continue - if (astIdsToExtract.has(ast.id)) { - collapsed.push(ast) - if (ast.id === astIdToReplace) { - refactored.push({ expression: autospaced(refactoredAst) }) - } - } else { - refactored.push(line) - } - } - return refactored + const extractedLines = currentMethod + .bodyAsBlock() + .extractIf(({ id }) => info.statementsToExtract.has(id) && id !== info.statementToReplace) + const collapsedCall = Ast.App.PositionalSequence( + Ast.PropertyAccess.new(edit, Ast.Ident.new(edit, MODULE_NAME), collapsedName), + info.args.map((arg) => Ast.Ident.new(edit, arg)), + ) + const statementToReplace = edit.get(info.statementToReplace) + assert(statementToReplace instanceof Ast.MutableAssignment) + const outputIdentifier = statementToReplace.pattern.code() as Identifier + extractedLines.push({ + statement: { + whitespace: undefined, + node: statementToReplace.replace( + Ast.Assignment.new(outputIdentifier, collapsedCall, { edit }), + ), + }, }) + const collapsedNodeIds = extractedLines + .map(({ statement }) => statement && nodeIdFromOuterAst(statement.node)) + .filter((id) => id != null) + .reverse() + // Insert a new function. - const collapsedNodeIds = [...filterDefined(collapsed.map(nodeIdFromOuterExpr))].reverse() - let outputAstId: Ast.AstId | undefined - const outputIdentifier = info.extracted.output?.identifier - if (outputIdentifier != null) { - const ident = Ast.Ident.new(edit, outputIdentifier) - collapsed.push(ident) - outputAstId = ident.id - } - const argNames = info.extracted.inputs - const collapsedFunction = Ast.Function.fromStatements(edit, collapsedName, argNames, collapsed) - const collapsedFunctionWithIcon = Ast.Documented.new('ICON group', collapsedFunction) - topLevel.insert(posToInsert, collapsedFunctionWithIcon, undefined) - return { refactoredNodeId, refactoredExpressionAstId, collapsedNodeIds, outputAstId } -} - -/** Prepare a method call expression for collapsed method. */ -function collapsedCallAst( - info: CollapsedInfo, - collapsedName: IdentifierOrOperatorIdentifier, - edit: Ast.MutableModule, -): { ast: Ast.Owned; expressionAstId: Ast.AstId; nodeId: NodeId } { - const pattern = info.refactored.pattern - const args = info.refactored.arguments - const functionName = `${MODULE_NAME}.${collapsedName}` - const expression = functionName + (args.length > 0 ? ' ' : '') + args.join(' ') - const expressionAst = Ast.parse(expression, edit) - const ast = Ast.Assignment.new(edit, pattern, expressionAst) - return { ast, expressionAstId: expressionAst.id, nodeId: asNodeId(expressionAst.externalId) } -} - -/** Find the position before the current method to insert a collapsed one. */ -function findInsertionPos(topLevel: Ast.BodyBlock, currentMethodName: string): number { - const currentFuncPosition = topLevel.lines.findIndex((line) => { - const expr = line.expression?.node?.innerExpression() - return expr instanceof Ast.Function && expr.name?.code() === currentMethodName + const collapsedBody = Ast.BodyBlock.new(extractedLines, edit) + const outputAst = Ast.Ident.new(edit, outputIdentifier) + collapsedBody.push(outputAst) + const collapsedFunction = Ast.Function.new(collapsedName, info.args, collapsedBody, { + edit, + documentation: 'ICON group', }) + topLevel.insert(currentMethodLine, collapsedFunction, undefined) - return currentFuncPosition === -1 ? 0 : currentFuncPosition + return { collapsedCallRoot: collapsedCall.id, outputAstId: outputAst.id, collapsedNodeIds } } diff --git a/app/gui/src/project-view/components/GraphEditor/widgets/WidgetApplication.vue b/app/gui/src/project-view/components/GraphEditor/widgets/WidgetApplication.vue index a54f1525c0..3da81bfecc 100644 --- a/app/gui/src/project-view/components/GraphEditor/widgets/WidgetApplication.vue +++ b/app/gui/src/project-view/components/GraphEditor/widgets/WidgetApplication.vue @@ -45,7 +45,10 @@ const operatorStyle = computed(() => { application.value.appTree instanceof Ast.OprApp || application.value.appTree instanceof Ast.PropertyAccess ) { - const [_lhs, opr, rhs] = application.value.appTree.concreteChildren() + const [_lhs, opr, rhs] = application.value.appTree.concreteChildren({ + verbatim: true, + indent: '', + }) return { '--whitespace-pre': `${JSON.stringify(opr?.whitespace ?? '')}`, '--whitespace-post': `${JSON.stringify(rhs?.whitespace ?? '')}`, diff --git a/app/gui/src/project-view/components/GraphEditor/widgets/WidgetCheckbox.vue b/app/gui/src/project-view/components/GraphEditor/widgets/WidgetCheckbox.vue index fcc8cba1eb..b44fb026a1 100644 --- a/app/gui/src/project-view/components/GraphEditor/widgets/WidgetCheckbox.vue +++ b/app/gui/src/project-view/components/GraphEditor/widgets/WidgetCheckbox.vue @@ -64,14 +64,14 @@ const argumentName = computed(() => { diff --git a/app/gui/src/project-view/components/visualizations/tableVizToolbar.ts b/app/gui/src/project-view/components/visualizations/tableVizToolbar.ts index ea6f46c2ad..e0dd51298b 100644 --- a/app/gui/src/project-view/components/visualizations/tableVizToolbar.ts +++ b/app/gui/src/project-view/components/visualizations/tableVizToolbar.ts @@ -39,7 +39,7 @@ function useSortFilterNodesButton({ isFilterSortNodeEnabled, createNodes, }: SortFilterNodesButtonOptions): ComputedRef { - const sortPatternPattern = computed(() => Pattern.parse('(..Name __ __ )')) + const sortPatternPattern = computed(() => Pattern.parseExpression('(..Name __ __ )')!) const sortDirection = computed(() => ({ asc: '..Ascending', @@ -53,36 +53,36 @@ function useSortFilterNodesButton({ .map((sort) => sortPatternPattern.value.instantiateCopied([ Ast.TextLiteral.new(sort.columnName), - Ast.parse(sortDirection.value[sort.sortDirection as SortDirection]), + Ast.parseExpression(sortDirection.value[sort.sortDirection as SortDirection])!, ]), ) return Ast.Vector.new(module, columnSortExpressions) } - const filterPattern = computed(() => Pattern.parse('__ (__ __)')) + const filterPattern = computed(() => Pattern.parseExpression('__ (__ __)')!) function makeFilterPattern(module: Ast.MutableModule, columnName: string, items: string[]) { if ( (items?.length === 1 && items.indexOf('true') != -1) || (items?.length === 1 && items.indexOf('false') != -1) ) { - const boolToInclude = items.indexOf('false') != -1 ? Ast.parse('False') : Ast.parse('True') + const boolToInclude = Ast.Ident.tryParse(items.indexOf('false') != -1 ? 'False' : 'True')! return filterPattern.value.instantiateCopied([ Ast.TextLiteral.new(columnName), - Ast.parse('..Equal'), + Ast.parseExpression('..Equal')!, boolToInclude, ]) } const itemList = items.map((i) => Ast.TextLiteral.new(i)) return filterPattern.value.instantiateCopied([ Ast.TextLiteral.new(columnName), - Ast.parse('..Is_In'), + Ast.parseExpression('..Is_In')!, Ast.Vector.new(module, itemList), ]) } function getAstPatternSort() { - return Pattern.new((ast) => + return Pattern.new((ast) => Ast.App.positional( Ast.PropertyAccess.new(ast.module, ast, Ast.identifier('sort')!), makeSortPattern(ast.module), @@ -91,7 +91,7 @@ function useSortFilterNodesButton({ } function getAstPatternFilter(columnName: string, items: string[]) { - return Pattern.new((ast) => + return Pattern.new((ast) => Ast.App.positional( Ast.PropertyAccess.new(ast.module, ast, Ast.identifier('filter')!), makeFilterPattern(ast.module, columnName, items), @@ -100,7 +100,7 @@ function useSortFilterNodesButton({ } function getAstPatternFilterAndSort(columnName: string, items: string[]) { - return Pattern.new((ast) => + return Pattern.new((ast) => Ast.OprApp.new( ast.module, Ast.App.positional( diff --git a/app/gui/src/project-view/composables/__tests__/nodeCreation.test.ts b/app/gui/src/project-view/composables/__tests__/nodeCreation.test.ts index 4cb6de4281..a0de64dd11 100644 --- a/app/gui/src/project-view/composables/__tests__/nodeCreation.test.ts +++ b/app/gui/src/project-view/composables/__tests__/nodeCreation.test.ts @@ -15,7 +15,7 @@ test.each([ ])('New node location in block', (...linesWithInsertionPoint: string[]) => { const inputLines = linesWithInsertionPoint.filter((line) => line !== '*') const bodyBlock = Ast.parseBlock(inputLines.join('\n')) - insertNodeStatements(bodyBlock, [Ast.parse('newNodePositionMarker')]) + insertNodeStatements(bodyBlock, [Ast.parseStatement('newNodePositionMarker')!]) const lines = bodyBlock .code() .split('\n') @@ -26,11 +26,13 @@ test.each([ // This is a special case because when a block is empty, adding a line requires adding *two* linebreaks. test('Adding node to empty block', () => { const module = Ast.MutableModule.Transient() - const func = Ast.Function.fromStatements(module, identifier('f')!, [], []) + const func = Ast.Function.new(identifier('f')!, [], Ast.BodyBlock.new([], module), { + edit: module, + }) const rootBlock = Ast.BodyBlock.new([], module) rootBlock.push(func) expect(rootBlock.code().trimEnd()).toBe('f =') - insertNodeStatements(func.bodyAsBlock(), [Ast.parse('newNode')]) + insertNodeStatements(func.bodyAsBlock(), [Ast.parseStatement('newNode')!]) expect( rootBlock .code() diff --git a/app/gui/src/project-view/composables/astDocumentation.ts b/app/gui/src/project-view/composables/astDocumentation.ts index 81cf012b3b..dca29731df 100644 --- a/app/gui/src/project-view/composables/astDocumentation.ts +++ b/app/gui/src/project-view/composables/astDocumentation.ts @@ -1,26 +1,31 @@ import { type GraphStore } from '@/stores/graph' +import { Ast } from '@/util/ast' import { type ToValue } from '@/util/reactivity' import { computed, toValue } from 'vue' -import type { Ast } from 'ydoc-shared/ast' /** A composable for reactively retrieving and setting documentation from given Ast node. */ -export function useAstDocumentation(graphStore: GraphStore, ast: ToValue) { +export function useAstDocumentation(graphStore: GraphStore, ast: ToValue) { return { documentation: { - state: computed(() => toValue(ast)?.documentingAncestor()?.documentation() ?? ''), - set: (value: string) => { + state: computed(() => { const astValue = toValue(ast) - if (!astValue) return - if (value.trimStart() !== '') { - graphStore.getMutable(astValue).getOrInitDocumentation().setDocumentationText(value) - } else { - // Remove the documentation node. - const documented = astValue.documentingAncestor() - if (documented && documented.expression) - graphStore.edit((edit) => - edit.getVersion(documented).update((documented) => documented.expression!.take()), - ) - } + return (astValue?.isStatement() ? astValue.documentationText() : undefined) ?? '' + }), + set: (text: string | undefined) => { + const astValue = toValue(ast) + graphStore.edit((edit) => { + if (astValue?.isStatement()) { + const editAst = edit.getVersion(astValue) + // If the statement can have documentation attached (for example, it is a `Function`, `Assignment`, or + // `ExpressionStatement`), do so. If in cannot (for example, it is an `import` declaration), an error will + // be reported below. + if ('setDocumentationText' in editAst) { + editAst.setDocumentationText(text) + return + } + } + console.error('Unable to set documentation', astValue?.id) + }) }, }, } diff --git a/app/gui/src/project-view/composables/nodeCreation.ts b/app/gui/src/project-view/composables/nodeCreation.ts index 46dc3fcca5..915e534b61 100644 --- a/app/gui/src/project-view/composables/nodeCreation.ts +++ b/app/gui/src/project-view/composables/nodeCreation.ts @@ -126,9 +126,13 @@ export function useNodeCreation( const createdIdentifiers = new Set() const identifiersRenameMap = new Map() graphStore.edit((edit) => { - const statements = new Array() + const statements = new Array>() for (const options of placedNodes) { - const rhs = Ast.parse(options.expression, edit) + const rhs = Ast.parseExpression(options.expression, edit) + if (!rhs) { + console.error('Cannot create node: invalid expression', options.expression) + continue + } const ident = getIdentifier(rhs, options, createdIdentifiers) createdIdentifiers.add(ident) const { id, rootExpression } = newAssignmentNode( @@ -192,19 +196,16 @@ export function useNodeCreation( function newAssignmentNode( edit: Ast.MutableModule, ident: Ast.Identifier, - rhs: Ast.Owned, + rhs: Ast.Owned, options: NodeCreationOptions, identifiersRenameMap: Map, ) { rhs.setNodeMetadata(options.metadata ?? {}) - const assignment = Ast.Assignment.new(edit, ident, rhs) + const { documentation } = options + const assignment = Ast.Assignment.new(ident, rhs, { edit, documentation }) afterCreation(edit, assignment, ident, options, identifiersRenameMap) const id = asNodeId(rhs.externalId) - const rootExpression = - options.documentation != null ? - Ast.Documented.new(options.documentation, assignment) - : assignment - return { rootExpression, id } + return { rootExpression: assignment, id } } function getIdentifier( @@ -270,10 +271,14 @@ function existingNameToPrefix(name: string): string { * The location will be after any statements in the block that bind identifiers; if the block ends in an expression * statement, the location will be before it so that the value of the block will not be affected. */ -export function insertNodeStatements(bodyBlock: Ast.MutableBodyBlock, statements: Ast.Owned[]) { +export function insertNodeStatements( + bodyBlock: Ast.MutableBodyBlock, + statements: Ast.Owned[], +) { const lines = bodyBlock.lines + const lastStatement = lines[lines.length - 1]?.statement?.node const index = - lines[lines.length - 1]?.expression?.node.isBindingStatement !== false ? + lastStatement instanceof Ast.MutableAssignment || lastStatement instanceof Ast.MutableFunction ? lines.length : lines.length - 1 bodyBlock.insert(index, ...statements) diff --git a/app/gui/src/project-view/mock/engine.ts b/app/gui/src/project-view/mock/engine.ts index 5120490187..3b89a3696c 100644 --- a/app/gui/src/project-view/mock/engine.ts +++ b/app/gui/src/project-view/mock/engine.ts @@ -459,8 +459,10 @@ export const mockLSHandler: MockTransportData = async (method, data, transport) expressionId: ExpressionId expression: string } - const aiPromptPat = Pattern.parse('Standard.Visualization.AI.build_ai_prompt __ . to_json') - const exprAst = Ast.parse(data_.expression) + const aiPromptPat = Pattern.parseExpression( + 'Standard.Visualization.AI.build_ai_prompt __ . to_json', + ) + const exprAst = Ast.parseExpression(data_.expression)! if (aiPromptPat.test(exprAst)) { sendVizUpdate( data_.visualizationId, diff --git a/app/gui/src/project-view/providers/__tests__/widgetRegistry.test.ts b/app/gui/src/project-view/providers/__tests__/widgetRegistry.test.ts index 48d923dc86..3447f4a872 100644 --- a/app/gui/src/project-view/providers/__tests__/widgetRegistry.test.ts +++ b/app/gui/src/project-view/providers/__tests__/widgetRegistry.test.ts @@ -56,8 +56,8 @@ describe('WidgetRegistry', () => { }), ) - const someAst = WidgetInput.FromAst(Ast.parse('foo')) - const blankAst = WidgetInput.FromAst(Ast.parse('_')) + const someAst = WidgetInput.FromAst(Ast.parseExpression('foo')) + const blankAst = WidgetInput.FromAst(Ast.parseExpression('_')) const someArgPlaceholder: WidgetInput = { portId: '57d429dc-df85-49f8-b150-567c7d1fb502' as PortId, value: 'bar', diff --git a/app/gui/src/project-view/providers/widgetRegistry.ts b/app/gui/src/project-view/providers/widgetRegistry.ts index 3f4afbd3a5..e651a0d592 100644 --- a/app/gui/src/project-view/providers/widgetRegistry.ts +++ b/app/gui/src/project-view/providers/widgetRegistry.ts @@ -12,21 +12,20 @@ import type { WidgetEditHandlerParent } from './widgetRegistry/editHandler' export type WidgetComponent = Component> export namespace WidgetInput { - /** Create a basic {@link WidgetInput } from AST node. */ - export function FromAst(ast: A): WidgetInput & { value: A } { + /** Returns widget-input data for the given AST expression or token. */ + export function FromAst( + ast: A, + ): WidgetInput & { value: A } { return { portId: ast.id, value: ast, } } - /** Create a basic {@link WidgetInput } from AST node with enforced port. */ - export function FromAstWithPort( - ast: A, - ): WidgetInput & { value: A } { + /** Returns the input marked to be a port. */ + export function WithPort(input: T): T { return { - portId: ast.id, - value: ast, + ...input, forcePort: true, } } @@ -57,14 +56,14 @@ export namespace WidgetInput { } /** Check if input's value is existing AST node (not placeholder or token). */ - export function isAst(input: WidgetInput): input is WidgetInput & { value: Ast.Ast } { - return input.value instanceof Ast.Ast + export function isAst(input: WidgetInput): input is WidgetInput & { value: Ast.Expression } { + return input.value instanceof Ast.Ast && input.value.isExpression() } /** Check if input's value is existing AST node or placeholder. Rule out token inputs. */ export function isAstOrPlaceholder( input: WidgetInput, - ): input is WidgetInput & { value: Ast.Ast | string | undefined } { + ): input is WidgetInput & { value: Ast.Expression | string | undefined } { return isPlaceholder(input) || isAst(input) } @@ -74,9 +73,9 @@ export namespace WidgetInput { } /** Check if input's value is an AST which potentially may be a function call. */ - export function isFunctionCall( - input: WidgetInput, - ): input is WidgetInput & { value: Ast.App | Ast.Ident | Ast.PropertyAccess | Ast.OprApp } { + export function isFunctionCall(input: WidgetInput): input is WidgetInput & { + value: Ast.App | Ast.Ident | Ast.PropertyAccess | Ast.OprApp | Ast.AutoscopedIdentifier + } { return ( input.value instanceof Ast.App || input.value instanceof Ast.Ident || @@ -119,10 +118,10 @@ export interface WidgetInput { */ portId: PortId /** - * An expected widget value. If Ast.Ast or Ast.Token, the widget represents an existing part of + * An expected widget value. If Ast.Expression or Ast.Token, the widget represents an existing part of * code. If string, it may be e.g. a default value of an argument. */ - value: Ast.Ast | Ast.Token | string | undefined + value: Ast.Expression | Ast.Token | string | undefined /** An expected type which widget should set. */ expectedType?: Typename | undefined /** Configuration provided by engine. */ @@ -172,7 +171,7 @@ export interface WidgetProps { export interface WidgetUpdate { edit?: MutableModule | undefined portUpdate?: { origin: PortId } & ( - | { value: Ast.Owned | string | undefined } + | { value: Ast.Owned | string | undefined } | { metadataKey: string; metadata: unknown } ) } diff --git a/app/gui/src/project-view/providers/widgetRegistry/editHandler.ts b/app/gui/src/project-view/providers/widgetRegistry/editHandler.ts index f8a57ed9fc..fea1a7efa7 100644 --- a/app/gui/src/project-view/providers/widgetRegistry/editHandler.ts +++ b/app/gui/src/project-view/providers/widgetRegistry/editHandler.ts @@ -60,7 +60,7 @@ export abstract class WidgetEditHandlerParent { this.parent?.unsetActiveChild(this) } - protected onEdit(origin: PortId, value: Ast.Owned | string): void { + protected onEdit(origin: PortId, value: Ast.Owned | string): void { this.hooks.edit?.(origin, value) this.parent?.onEdit(origin, value) } @@ -265,8 +265,8 @@ export class WidgetEditHandler extends WidgetEditHandlerParent { this.onStart(this.portId) } - /** TODO: Add docs */ - edit(value: Ast.Owned | string) { + /** Emit an event updating the widget's value. */ + edit(value: Ast.Owned | string) { this.onEdit(this.portId, value) } } @@ -281,7 +281,7 @@ export interface WidgetEditHooks extends Interaction { end?(origin?: PortId | undefined): void childEnded?(origin?: PortId | undefined): void /** Hook called when a child widget provides an updated value. */ - edit?(origin: PortId, value: Ast.Owned | string): void + edit?(origin: PortId, value: Ast.Owned | string): void /** * Hook enabling a widget to provide a handler for the add-item intent of a child widget. The parent can return true * to indicate that creating the new item has been handled and the child should not perform its action in this case. diff --git a/app/gui/src/project-view/providers/widgetTree.ts b/app/gui/src/project-view/providers/widgetTree.ts index 3116ac695f..c659ec6199 100644 --- a/app/gui/src/project-view/providers/widgetTree.ts +++ b/app/gui/src/project-view/providers/widgetTree.ts @@ -10,7 +10,7 @@ export { injectFn as injectWidgetTree, provideFn as provideWidgetTree } const { provideFn, injectFn } = createContextStore( 'Widget tree', ( - astRoot: Ref, + astRoot: Ref, nodeId: Ref, nodeElement: Ref, nodeSize: Ref, diff --git a/app/gui/src/project-view/stores/graph/__tests__/graphDatabase.test.ts b/app/gui/src/project-view/stores/graph/__tests__/graphDatabase.test.ts index c006bff9bd..97063b271a 100644 --- a/app/gui/src/project-view/stores/graph/__tests__/graphDatabase.test.ts +++ b/app/gui/src/project-view/stores/graph/__tests__/graphDatabase.test.ts @@ -25,7 +25,7 @@ export function parseWithSpans>(code: stri const { root: ast, toRaw, getSpan } = Ast.parseExtended(code, idMap) const idFromExternal = new Map() - ast.visitRecursiveAst((ast) => { + ast.visitRecursive((ast) => { idFromExternal.set(ast.externalId, ast.id) }) const id = (name: keyof T) => idFromExternal.get(eid(name))! diff --git a/app/gui/src/project-view/stores/graph/graphDatabase.ts b/app/gui/src/project-view/stores/graph/graphDatabase.ts index 2c2174a677..5bcd1c9df5 100644 --- a/app/gui/src/project-view/stores/graph/graphDatabase.ts +++ b/app/gui/src/project-view/stores/graph/graphDatabase.ts @@ -5,14 +5,16 @@ import type { SuggestionEntry } from '@/stores/suggestionDatabase/entry' import { assert } from '@/util/assert' import { Ast, RawAst } from '@/util/ast' import type { AstId, NodeMetadata } from '@/util/ast/abstract' -import { autospaced, MutableModule } from '@/util/ast/abstract' +import { MutableModule } from '@/util/ast/abstract' import { AliasAnalyzer } from '@/util/ast/aliasAnalysis' import { inputNodeFromAst, nodeFromAst, nodeRootExpr } from '@/util/ast/node' import { MappedKeyMap, MappedSet } from '@/util/containers' import { tryGetIndex } from '@/util/data/array' import { recordEqual } from '@/util/data/object' +import { unwrap } from '@/util/data/result' import { Vec2 } from '@/util/data/vec2' import { ReactiveDb, ReactiveIndex, ReactiveMapping } from '@/util/database/reactiveDb' +import { tryIdentifier } from '@/util/qualifiedName' import { nonReactiveView, resumeReactivity, @@ -67,8 +69,10 @@ export class BindingsDb { // Add or update bindings. for (const [bindingRange, usagesRanges] of analyzer.aliases) { const aliasAst = bindingRangeToTree.get(bindingRange) - assert(aliasAst != null) - if (aliasAst == null) continue + if (aliasAst == null) { + console.warn(`Binding not found`, bindingRange) + continue + } const aliasAstId = aliasAst.id const info = this.bindings.get(aliasAstId) if (info == null) { @@ -121,7 +125,7 @@ export class BindingsDb { bindingRanges.add(binding) for (const usage of usages) bindingRanges.add(usage) } - ast.visitRecursiveAst((ast) => { + ast.visitRecursive((ast) => { const span = getSpan(ast.id) assert(span != null) if (bindingRanges.has(span)) { @@ -153,13 +157,13 @@ export class GraphDb { private nodeIdToPatternExprIds = new ReactiveIndex(this.nodeIdToNode, (id, entry) => { const exprs: AstId[] = [] - if (entry.pattern) entry.pattern.visitRecursiveAst((ast) => void exprs.push(ast.id)) + if (entry.pattern) entry.pattern.visitRecursive((ast) => void exprs.push(ast.id)) return Array.from(exprs, (expr) => [id, expr]) }) private nodeIdToExprIds = new ReactiveIndex(this.nodeIdToNode, (id, entry) => { const exprs: AstId[] = [] - entry.innerExpr.visitRecursiveAst((ast) => void exprs.push(ast.id)) + entry.innerExpr.visitRecursive((ast) => void exprs.push(ast.id)) return Array.from(exprs, (expr) => [id, expr]) }) @@ -195,7 +199,7 @@ export class GraphDb { nodeOutputPorts = new ReactiveIndex(this.nodeIdToNode, (id, entry) => { if (entry.pattern == null) return [] const ports = new Set() - entry.pattern.visitRecursiveAst((ast) => { + entry.pattern.visitRecursive((ast) => { if (this.bindings.bindings.has(ast.id)) { ports.add(ast.id) return false @@ -350,7 +354,7 @@ export class GraphDb { const args = functionAst_.argumentDefinitions const update = ( nodeId: NodeId, - ast: Ast.Ast, + ast: Ast.Expression | Ast.Statement, isInput: boolean, isOutput: boolean, argIndex: number | undefined, @@ -383,7 +387,7 @@ export class GraphDb { update(nodeId, argPattern, true, false, index) }) body.forEach((outerAst, index) => { - const nodeId = nodeIdFromOuterExpr(outerAst) + const nodeId = nodeIdFromOuterAst(outerAst) if (!nodeId) return const isLastInBlock = index === body.length - 1 update(nodeId, outerAst, false, isLastInBlock, undefined) @@ -400,12 +404,15 @@ export class GraphDb { /** Scan a node's content from its outer expression down to, but not including, its inner expression. */ private updateNodeStructure( nodeId: NodeId, - ast: Ast.Ast, + ast: Ast.Statement | Ast.Expression, isOutput: boolean, isInput: boolean, argIndex?: number, ) { - const newNode = isInput ? inputNodeFromAst(ast, argIndex ?? 0) : nodeFromAst(ast, isOutput) + const newNode = + isInput ? + inputNodeFromAst(ast as Ast.Expression, argIndex ?? 0) + : nodeFromAst(ast as Ast.Statement, isOutput) if (!newNode) return const oldNode = this.nodeIdToNode.getUntracked(nodeId) if (oldNode == null) { @@ -424,14 +431,13 @@ export class GraphDb { } else { const { type, - outerExpr, + outerAst, pattern, rootExpr, innerExpr, primarySubject, prefixes, conditionalPorts, - docs, argIndex, } = newNode const node = resumeReactivity(oldNode) @@ -440,7 +446,7 @@ export class GraphDb { const updateAst = (field: NodeAstField) => { if (oldNode[field]?.id !== newNode[field]?.id) node[field] = newNode[field] as any } - const astFields: NodeAstField[] = ['outerExpr', 'pattern', 'rootExpr', 'innerExpr', 'docs'] + const astFields: NodeAstField[] = ['outerAst', 'pattern', 'rootExpr', 'innerExpr'] astFields.forEach(updateAst) if (oldNode.primarySubject !== primarySubject) node.primarySubject = primarySubject if (!recordEqual(oldNode.prefixes, prefixes)) node.prefixes = prefixes @@ -448,14 +454,13 @@ export class GraphDb { // Ensure new fields can't be added to `NodeAstData` without this code being updated. const _allFieldsHandled = { type, - outerExpr, + outerAst, pattern, rootExpr, innerExpr, primarySubject, prefixes, conditionalPorts, - docs, argIndex, } satisfies NodeDataFromAst } @@ -475,7 +480,7 @@ export class GraphDb { updateExternalIds(topLevel: Ast.Ast) { const idToExternalNew = new Map() const idFromExternalNew = new Map() - topLevel.visitRecursiveAst((ast) => { + topLevel.visitRecursive((ast) => { idToExternalNew.set(ast.id, ast.externalId) idFromExternalNew.set(ast.externalId, ast.id) }) @@ -540,14 +545,10 @@ export class GraphDb { /** TODO: Add docs */ mockNode(binding: string, id: NodeId, code?: string): Node { const edit = MutableModule.Transient() - const pattern = Ast.parse(binding, edit) - const expression = Ast.parse(code ?? '0', edit) - const outerExpr = Ast.Assignment.concrete( - edit, - autospaced(pattern), - { node: Ast.Token.new('='), whitespace: ' ' }, - { node: expression, whitespace: ' ' }, - ) + const ident = unwrap(tryIdentifier(binding)) + const expression = Ast.parseExpression(code ?? '0', edit)! + const outerAst = Ast.Assignment.new(ident, expression, { edit }) + const pattern = outerAst.pattern const node: Node = { type: 'component', @@ -557,11 +558,10 @@ export class GraphDb { primarySubject: undefined, colorOverride: undefined, conditionalPorts: new Set(), - docs: undefined, - outerExpr, + outerAst, pattern, - rootExpr: Ast.parse(code ?? '0'), - innerExpr: Ast.parse(code ?? '0'), + rootExpr: expression, + innerExpr: expression, zIndex: this.highestZIndex, argIndex: undefined, } @@ -574,7 +574,7 @@ export class GraphDb { /** Source code data of the specific node. */ interface NodeSource { - /** The outer AST of the node (see {@link NodeDataFromAst.outerExpr}). */ + /** The outer AST of the node (see {@link NodeDataFromAst.outerAst}). */ outerAst: Ast.Ast /** * Whether the node is `output` of the function or not. Mutually exclusive with `isInput`. @@ -602,28 +602,37 @@ export function asNodeId(id: ExternalId | undefined): NodeId | undefined { return id != null ? (id as NodeId) : undefined } -/** Given an expression at the top level of a block, return the `NodeId` for the expression. */ -export function nodeIdFromOuterExpr(outerExpr: Ast.Ast) { - const { root } = nodeRootExpr(outerExpr) +/** Given the outermost AST for a node, returns its {@link NodeId}. */ +export function nodeIdFromOuterAst(outerAst: Ast.Statement | Ast.Expression) { + const { root } = nodeRootExpr(outerAst) return root && asNodeId(root.externalId) } export interface NodeDataFromAst { type: NodeType - /** The outer expression, usually an assignment expression (`a = b`). */ - outerExpr: Ast.Ast - /** The left side of the assignment expression, if `outerExpr` is an assignment expression. */ - pattern: Ast.Ast | undefined /** - * The value of the node. The right side of the assignment, if `outerExpr` is an assignment - * expression, else the entire `outerExpr`. + * The statement or top-level expression. + * + * If the function has a body block, the nodes derived from the block are statements: + * - Assignment expressions (`a = b`) + * - Expression-statements (unnamed nodes and output nodes) + * If the function has a single-line body, the corresponding node will be an expression. + * + * Nodes for the function's inputs have (pattern) expressions as their outer ASTs. */ - rootExpr: Ast.Ast + outerAst: Ast.Statement | Ast.Expression + /** The left side of the assignment expression, if `outerAst` is an assignment expression. */ + pattern: Ast.Expression | undefined + /** + * The value of the node. The right side of the assignment, if `outerAst` is an assignment + * expression, else the entire `outerAst`. + */ + rootExpr: Ast.Expression /** * The expression displayed by the node. This is `rootExpr`, minus the prefixes, which are in * `prefixes`. */ - innerExpr: Ast.Ast + innerExpr: Ast.Expression /** Prefixes that are present in `rootExpr` but omitted in `innerExpr` to ensure a clean output. */ @@ -632,8 +641,6 @@ export interface NodeDataFromAst { primarySubject: Ast.AstId | undefined /** Ports that are not targetable by default; they can be targeted while holding the modifier key. */ conditionalPorts: Set - /** An AST node containing the node's documentation comment. */ - docs: Ast.Documented | undefined /** The index of the argument in the function's argument list, if the node is an input node. */ argIndex: number | undefined } diff --git a/app/gui/src/project-view/stores/graph/imports.ts b/app/gui/src/project-view/stores/graph/imports.ts index 21e4f005e2..d2118ad3eb 100644 --- a/app/gui/src/project-view/stores/graph/imports.ts +++ b/app/gui/src/project-view/stores/graph/imports.ts @@ -99,9 +99,9 @@ export interface UnqualifiedImport { } /** Read imports from given module block */ -export function readImports(ast: Ast.Ast): Import[] { +export function readImports(ast: Ast.BodyBlock): Import[] { const imports: Import[] = [] - ast.visitRecursiveAst((node) => { + ast.visitRecursive((node) => { if (node instanceof Ast.Import) { const recognized = recognizeImport(node) if (recognized) { @@ -132,8 +132,8 @@ function newImportsLocation(scope: Ast.BodyBlock): number { const lines = scope.lines for (let i = 0; i < lines.length; i++) { const line = lines[i]! - if (line.expression) { - if (line.expression.node?.innerExpression() instanceof Ast.Import) { + if (line.statement) { + if (line.statement.node instanceof Ast.Import) { lastImport = i } else { break diff --git a/app/gui/src/project-view/stores/graph/index.ts b/app/gui/src/project-view/stores/graph/index.ts index f0756ff21f..5543c6d97e 100644 --- a/app/gui/src/project-view/stores/graph/index.ts +++ b/app/gui/src/project-view/stores/graph/index.ts @@ -2,7 +2,7 @@ import { usePlacement } from '@/components/ComponentBrowser/placement' import { createContextStore } from '@/providers' import type { PortId } from '@/providers/portInfo' import type { WidgetUpdate } from '@/providers/widgetRegistry' -import { GraphDb, nodeIdFromOuterExpr, type NodeId } from '@/stores/graph/graphDatabase' +import { GraphDb, nodeIdFromOuterAst, type NodeId } from '@/stores/graph/graphDatabase' import { addImports, detectImportConflicts, @@ -223,7 +223,7 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC return Err('Method pointer is not a module method') const method = Ast.findModuleMethod(topLevel, ptr.name) if (!method) return Err(`No method with name ${ptr.name} in ${modulePath.value}`) - return Ok(method) + return Ok(method.statement) } /** @@ -330,8 +330,8 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC updatePortValue(edit, usage, undefined) } - const outerExpr = edit.getVersion(node.outerExpr) - if (outerExpr) Ast.deleteFromParentBlock(outerExpr) + const outerAst = edit.getVersion(node.outerAst) + if (outerAst.isStatement()) Ast.deleteFromParentBlock(outerAst) nodeRects.delete(id) nodeHoverAnimations.delete(id) deletedNodes.add(id) @@ -576,7 +576,7 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC function updatePortValue( edit: MutableModule, id: PortId, - value: Ast.Owned | undefined, + value: Ast.Owned | undefined, ): boolean { const update = getPortPrimaryInstance(id)?.onUpdate if (!update) return false @@ -692,7 +692,7 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC const body = func.bodyExpressions() const result: NodeId[] = [] for (const expr of body) { - const nodeId = nodeIdFromOuterExpr(expr) + const nodeId = nodeIdFromOuterAst(expr) if (nodeId && ids.has(nodeId)) result.push(nodeId) } return result @@ -710,14 +710,14 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC sourceNodeId: NodeId, targetNodeId: NodeId, ) { - const sourceExpr = db.nodeIdToNode.get(sourceNodeId)?.outerExpr.id - const targetExpr = db.nodeIdToNode.get(targetNodeId)?.outerExpr.id + const sourceExpr = db.nodeIdToNode.get(sourceNodeId)?.outerAst.id + const targetExpr = db.nodeIdToNode.get(targetNodeId)?.outerAst.id const body = edit.getVersion(unwrap(getExecutedMethodAst(edit))).bodyAsBlock() assert(sourceExpr != null) assert(targetExpr != null) const lines = body.lines - const sourceIdx = lines.findIndex((line) => line.expression?.node.id === sourceExpr) - const targetIdx = lines.findIndex((line) => line.expression?.node.id === targetExpr) + const sourceIdx = lines.findIndex((line) => line.statement?.node.id === sourceExpr) + const targetIdx = lines.findIndex((line) => line.statement?.node.id === targetExpr) assert(sourceIdx != null) assert(targetIdx != null) @@ -727,7 +727,7 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC const deps = reachable([targetNodeId], (node) => db.nodeDependents.lookup(node)) const dependantLines = new Set( - Array.from(deps, (id) => db.nodeIdToNode.get(id)?.outerExpr.id), + Array.from(deps, (id) => db.nodeIdToNode.get(id)?.outerAst.id), ) // Include the new target itself in the set of lines that must be placed after source node. dependantLines.add(targetExpr) @@ -744,7 +744,7 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC // Split those lines into two buckets, whether or not they depend on the target. const [linesAfter, linesBefore] = partition(linesToSort, (line) => - dependantLines.has(line.expression?.node.id), + dependantLines.has(line.statement?.node.id), ) // Recombine all lines after splitting, keeping existing dependants below the target. diff --git a/app/gui/src/project-view/util/__tests__/callTree.test.ts b/app/gui/src/project-view/util/__tests__/callTree.test.ts index 5bff218583..73131960ca 100644 --- a/app/gui/src/project-view/util/__tests__/callTree.test.ts +++ b/app/gui/src/project-view/util/__tests__/callTree.test.ts @@ -101,7 +101,7 @@ test.each` expectedPattern, fixture: { allowInfix, mockSuggestion, argsParameters }, }: TestData) => { - const ast = Ast.parse(expression.trim()) + const ast = Ast.parseExpression(expression.trim()) const configuration: widgetCfg.FunctionCall = { kind: 'FunctionCall', @@ -207,7 +207,7 @@ test.each([ ({ code, subapplicationIndex, notAppliedArguments, expectedNotAppliedArguments }: TestCase) => { const { db, expectedMethodCall, expectedSuggestion, setExpressionInfo } = prepareMocksForGetMethodCallTest() - const ast = Ast.parse(code) + const ast = Ast.parseExpression(code) db.updateExternalIds(ast) const subApplication = nthSubapplication(ast, subapplicationIndex) assert(subApplication) @@ -345,7 +345,7 @@ test.each([ 'Computing IDs of arguments: $description', ({ code, subapplicationIndex, notAppliedArguments, expectedSameIds }: ArgsTestCase) => { const { db, expectedMethodCall, setExpressionInfo } = prepareMocksForGetMethodCallTest() - const ast = Ast.parse(code) + const ast = Ast.parseExpression(code) const subApplication = nthSubapplication(ast, subapplicationIndex) assert(subApplication) db.updateExternalIds(ast) diff --git a/app/gui/src/project-view/util/ast/__tests__/__snapshots__/raw.test.ts.snap b/app/gui/src/project-view/util/ast/__tests__/__snapshots__/raw.test.ts.snap deleted file mode 100644 index c99f75b007..0000000000 --- a/app/gui/src/project-view/util/ast/__tests__/__snapshots__/raw.test.ts.snap +++ /dev/null @@ -1,1473 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`Parsing '2 - + 3 - + 4' 1`] = ` -{ - "childrenLengthInCodeParsed": 11, - "spanCodeLengthLineChars16": 4, - "spanCodeLengthNewlines": 2, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "statements": [ - { - "expression": { - "childrenLengthInCodeParsed": 11, - "excess": [], - "expressions": [ - { - "expression": { - "expression": { - "base": undefined, - "childrenLengthInCodeParsed": 1, - "fractionalDigits": undefined, - "integer": { - "base": undefined, - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 3, - "codeStartLine": 1, - "codeStartUtf8": 5, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 3, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 5, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 5, - "type": "Digits", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 5, - }, - "spanCodeLengthLineChars16": 1, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 1, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 2, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 4, - "type": "Number", - "warnings": [], - "whitespaceLengthInCodeParsed": 1, - "whitespaceStartInCodeParsed": 4, - }, - "operator": { - "ok": true, - "value": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 1, - "codeStartLine": 1, - "codeStartUtf8": 3, - "leftOffsetCodeLenLineChars16": 1, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 2, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 3, - "type": "Operator", - "whitespaceLengthInCodeBuffer": 1, - "whitespaceStartInCodeBuffer": 2, - }, - }, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 1, - "codeStartCol16": 1, - "codeStartLine": 0, - "codeStartUtf8": 1, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 1, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 1, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 1, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 1, - }, - }, - { - "expression": { - "expression": { - "base": undefined, - "childrenLengthInCodeParsed": 1, - "fractionalDigits": undefined, - "integer": { - "base": undefined, - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 3, - "codeStartLine": 2, - "codeStartUtf8": 10, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 3, - "leftOffsetCodeStartLine": 2, - "leftOffsetCodeStartUtf8": 10, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 10, - "type": "Digits", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 10, - }, - "spanCodeLengthLineChars16": 1, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 1, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 2, - "spanLeftOffsetCodeStartLine": 2, - "spanLeftOffsetCodeStartUtf8": 9, - "type": "Number", - "warnings": [], - "whitespaceLengthInCodeParsed": 1, - "whitespaceStartInCodeParsed": 9, - }, - "operator": { - "ok": true, - "value": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 1, - "codeStartLine": 2, - "codeStartUtf8": 8, - "leftOffsetCodeLenLineChars16": 1, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 2, - "leftOffsetCodeStartUtf8": 7, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 8, - "type": "Operator", - "whitespaceLengthInCodeBuffer": 1, - "whitespaceStartInCodeBuffer": 7, - }, - }, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 1, - "codeStartCol16": 4, - "codeStartLine": 1, - "codeStartUtf8": 6, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 4, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 6, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 6, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 6, - }, - }, - ], - "lhs": { - "base": undefined, - "childrenLengthInCodeParsed": 1, - "fractionalDigits": undefined, - "integer": { - "base": undefined, - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 0, - "type": "Digits", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - "spanCodeLengthLineChars16": 1, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "type": "Number", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "spanCodeLengthLineChars16": 4, - "spanCodeLengthNewlines": 2, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "type": "OperatorBlockApplication", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 0, - "startInCodeBuffer": 0, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - }, - ], - "type": "BodyBlock", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, -} -`; - -exports[`Parsing 'Data.read -2 + 2' 1`] = ` -{ - "childrenLengthInCodeParsed": 15, - "spanCodeLengthLineChars16": 5, - "spanCodeLengthNewlines": 1, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "statements": [ - { - "expression": { - "childrenLengthInCodeParsed": 9, - "lhs": { - "childrenLengthInCodeParsed": 4, - "spanCodeLengthLineChars16": 4, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "token": { - "codeLenLineChars16": 4, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": true, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 4, - "liftLevel": 0, - "startInCodeBuffer": 0, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "opr": { - "ok": true, - "value": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 4, - "codeStartLine": 0, - "codeStartUtf8": 4, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 4, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 4, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 4, - "type": "Operator", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 4, - }, - }, - "rhs": { - "childrenLengthInCodeParsed": 4, - "spanCodeLengthLineChars16": 4, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 5, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 5, - "token": { - "codeLenLineChars16": 4, - "codeLenNewlines": 0, - "codeStartCol16": 5, - "codeStartLine": 0, - "codeStartUtf8": 5, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": false, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 5, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 5, - "lengthInCodeBuffer": 4, - "liftLevel": 0, - "startInCodeBuffer": 5, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 5, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 5, - }, - "spanCodeLengthLineChars16": 9, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "type": "OprApp", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 0, - "startInCodeBuffer": 0, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - }, - { - "expression": { - "childrenLengthInCodeParsed": 5, - "lhs": { - "base": undefined, - "childrenLengthInCodeParsed": 1, - "fractionalDigits": undefined, - "integer": { - "base": undefined, - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 1, - "codeStartUtf8": 10, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 10, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 10, - "type": "Digits", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 10, - }, - "spanCodeLengthLineChars16": 1, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 10, - "type": "Number", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 10, - }, - "opr": { - "ok": true, - "value": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 2, - "codeStartLine": 1, - "codeStartUtf8": 12, - "leftOffsetCodeLenLineChars16": 1, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 1, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 11, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 12, - "type": "Operator", - "whitespaceLengthInCodeBuffer": 1, - "whitespaceStartInCodeBuffer": 11, - }, - }, - "rhs": { - "base": undefined, - "childrenLengthInCodeParsed": 1, - "fractionalDigits": undefined, - "integer": { - "base": undefined, - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 4, - "codeStartLine": 1, - "codeStartUtf8": 14, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 4, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 14, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 14, - "type": "Digits", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 14, - }, - "spanCodeLengthLineChars16": 1, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 1, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 3, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 13, - "type": "Number", - "warnings": [], - "whitespaceLengthInCodeParsed": 1, - "whitespaceStartInCodeParsed": 13, - }, - "spanCodeLengthLineChars16": 5, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 10, - "type": "OprApp", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 10, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 1, - "codeStartCol16": 9, - "codeStartLine": 0, - "codeStartUtf8": 9, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 9, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 9, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 9, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 9, - }, - }, - ], - "type": "BodyBlock", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, -} -`; - -exports[`Parsing 'Data.read "File" -2 + 3' 1`] = ` -{ - "childrenLengthInCodeParsed": 22, - "spanCodeLengthLineChars16": 5, - "spanCodeLengthNewlines": 1, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "statements": [ - { - "expression": { - "arg": { - "childrenLengthInCodeParsed": 6, - "close": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 15, - "codeStartLine": 0, - "codeStartUtf8": 15, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 15, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 15, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 15, - "type": "TextEnd", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 15, - }, - "elements": [ - { - "text": { - "codeLenLineChars16": 4, - "codeLenNewlines": 0, - "codeStartCol16": 11, - "codeStartLine": 0, - "codeStartUtf8": 11, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 11, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 11, - "lengthInCodeBuffer": 4, - "startInCodeBuffer": 11, - "type": "TextSection", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 11, - }, - "type": "Section", - }, - ], - "newline": undefined, - "open": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 10, - "codeStartLine": 0, - "codeStartUtf8": 10, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 10, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 10, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 10, - "type": "TextStart", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 10, - }, - "spanCodeLengthLineChars16": 6, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 1, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 9, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 9, - "type": "TextLiteral", - "warnings": [], - "whitespaceLengthInCodeParsed": 1, - "whitespaceStartInCodeParsed": 9, - }, - "childrenLengthInCodeParsed": 16, - "func": { - "childrenLengthInCodeParsed": 9, - "lhs": { - "childrenLengthInCodeParsed": 4, - "spanCodeLengthLineChars16": 4, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "token": { - "codeLenLineChars16": 4, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": true, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 4, - "liftLevel": 0, - "startInCodeBuffer": 0, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "opr": { - "ok": true, - "value": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 4, - "codeStartLine": 0, - "codeStartUtf8": 4, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 4, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 4, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 4, - "type": "Operator", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 4, - }, - }, - "rhs": { - "childrenLengthInCodeParsed": 4, - "spanCodeLengthLineChars16": 4, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 5, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 5, - "token": { - "codeLenLineChars16": 4, - "codeLenNewlines": 0, - "codeStartCol16": 5, - "codeStartLine": 0, - "codeStartUtf8": 5, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": false, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 5, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 5, - "lengthInCodeBuffer": 4, - "liftLevel": 0, - "startInCodeBuffer": 5, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 5, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 5, - }, - "spanCodeLengthLineChars16": 9, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "type": "OprApp", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "spanCodeLengthLineChars16": 16, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "type": "App", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 0, - "startInCodeBuffer": 0, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - }, - { - "expression": { - "childrenLengthInCodeParsed": 5, - "lhs": { - "base": undefined, - "childrenLengthInCodeParsed": 1, - "fractionalDigits": undefined, - "integer": { - "base": undefined, - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 1, - "codeStartUtf8": 17, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 17, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 17, - "type": "Digits", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 17, - }, - "spanCodeLengthLineChars16": 1, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 17, - "type": "Number", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 17, - }, - "opr": { - "ok": true, - "value": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 2, - "codeStartLine": 1, - "codeStartUtf8": 19, - "leftOffsetCodeLenLineChars16": 1, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 1, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 18, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 19, - "type": "Operator", - "whitespaceLengthInCodeBuffer": 1, - "whitespaceStartInCodeBuffer": 18, - }, - }, - "rhs": { - "base": undefined, - "childrenLengthInCodeParsed": 1, - "fractionalDigits": undefined, - "integer": { - "base": undefined, - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 4, - "codeStartLine": 1, - "codeStartUtf8": 21, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 4, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 21, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 21, - "type": "Digits", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 21, - }, - "spanCodeLengthLineChars16": 1, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 1, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 3, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 20, - "type": "Number", - "warnings": [], - "whitespaceLengthInCodeParsed": 1, - "whitespaceStartInCodeParsed": 20, - }, - "spanCodeLengthLineChars16": 5, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 17, - "type": "OprApp", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 17, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 1, - "codeStartCol16": 16, - "codeStartLine": 0, - "codeStartUtf8": 16, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 16, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 16, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 16, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 16, - }, - }, - ], - "type": "BodyBlock", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, -} -`; - -exports[`Parsing 'Data.read File -2 + 3' 1`] = ` -{ - "childrenLengthInCodeParsed": 20, - "spanCodeLengthLineChars16": 5, - "spanCodeLengthNewlines": 1, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "statements": [ - { - "expression": { - "arg": { - "childrenLengthInCodeParsed": 4, - "spanCodeLengthLineChars16": 4, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 1, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 9, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 9, - "token": { - "codeLenLineChars16": 4, - "codeLenNewlines": 0, - "codeStartCol16": 10, - "codeStartLine": 0, - "codeStartUtf8": 10, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": true, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 10, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 10, - "lengthInCodeBuffer": 4, - "liftLevel": 0, - "startInCodeBuffer": 10, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 10, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 1, - "whitespaceStartInCodeParsed": 9, - }, - "childrenLengthInCodeParsed": 14, - "func": { - "childrenLengthInCodeParsed": 9, - "lhs": { - "childrenLengthInCodeParsed": 4, - "spanCodeLengthLineChars16": 4, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "token": { - "codeLenLineChars16": 4, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": true, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 4, - "liftLevel": 0, - "startInCodeBuffer": 0, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "opr": { - "ok": true, - "value": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 4, - "codeStartLine": 0, - "codeStartUtf8": 4, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 4, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 4, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 4, - "type": "Operator", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 4, - }, - }, - "rhs": { - "childrenLengthInCodeParsed": 4, - "spanCodeLengthLineChars16": 4, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 5, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 5, - "token": { - "codeLenLineChars16": 4, - "codeLenNewlines": 0, - "codeStartCol16": 5, - "codeStartLine": 0, - "codeStartUtf8": 5, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": false, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 5, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 5, - "lengthInCodeBuffer": 4, - "liftLevel": 0, - "startInCodeBuffer": 5, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 5, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 5, - }, - "spanCodeLengthLineChars16": 9, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "type": "OprApp", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "spanCodeLengthLineChars16": 14, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "type": "App", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 0, - "startInCodeBuffer": 0, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - }, - { - "expression": { - "childrenLengthInCodeParsed": 5, - "lhs": { - "base": undefined, - "childrenLengthInCodeParsed": 1, - "fractionalDigits": undefined, - "integer": { - "base": undefined, - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 1, - "codeStartUtf8": 15, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 15, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 15, - "type": "Digits", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 15, - }, - "spanCodeLengthLineChars16": 1, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 15, - "type": "Number", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 15, - }, - "opr": { - "ok": true, - "value": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 2, - "codeStartLine": 1, - "codeStartUtf8": 17, - "leftOffsetCodeLenLineChars16": 1, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 1, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 16, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 17, - "type": "Operator", - "whitespaceLengthInCodeBuffer": 1, - "whitespaceStartInCodeBuffer": 16, - }, - }, - "rhs": { - "base": undefined, - "childrenLengthInCodeParsed": 1, - "fractionalDigits": undefined, - "integer": { - "base": undefined, - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 4, - "codeStartLine": 1, - "codeStartUtf8": 19, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 4, - "leftOffsetCodeStartLine": 1, - "leftOffsetCodeStartUtf8": 19, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 19, - "type": "Digits", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 19, - }, - "spanCodeLengthLineChars16": 1, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 1, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 3, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 18, - "type": "Number", - "warnings": [], - "whitespaceLengthInCodeParsed": 1, - "whitespaceStartInCodeParsed": 18, - }, - "spanCodeLengthLineChars16": 5, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 1, - "spanLeftOffsetCodeStartUtf8": 15, - "type": "OprApp", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 15, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 1, - "codeStartCol16": 14, - "codeStartLine": 0, - "codeStartUtf8": 14, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 14, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 14, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 14, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 14, - }, - }, - ], - "type": "BodyBlock", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, -} -`; - -exports[`Parsing 'foo bar -' 1`] = ` -{ - "childrenLengthInCodeParsed": 8, - "spanCodeLengthLineChars16": 0, - "spanCodeLengthNewlines": 1, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "statements": [ - { - "expression": { - "arg": { - "childrenLengthInCodeParsed": 3, - "spanCodeLengthLineChars16": 3, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 1, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 3, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 3, - "token": { - "codeLenLineChars16": 3, - "codeLenNewlines": 0, - "codeStartCol16": 4, - "codeStartLine": 0, - "codeStartUtf8": 4, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": false, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 4, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 4, - "lengthInCodeBuffer": 3, - "liftLevel": 0, - "startInCodeBuffer": 4, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 4, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 1, - "whitespaceStartInCodeParsed": 3, - }, - "childrenLengthInCodeParsed": 7, - "func": { - "childrenLengthInCodeParsed": 3, - "spanCodeLengthLineChars16": 3, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "token": { - "codeLenLineChars16": 3, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": false, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 3, - "liftLevel": 0, - "startInCodeBuffer": 0, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "spanCodeLengthLineChars16": 7, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "type": "App", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 0, - "startInCodeBuffer": 0, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - }, - { - "expression": undefined, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 1, - "codeStartCol16": 7, - "codeStartLine": 0, - "codeStartUtf8": 7, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 7, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 7, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 7, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 7, - }, - }, - ], - "type": "BodyBlock", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, -} -`; - -exports[`Parsing 'foo bar=baz' 1`] = ` -{ - "childrenLengthInCodeParsed": 11, - "spanCodeLengthLineChars16": 11, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "statements": [ - { - "expression": { - "arg": { - "childrenLengthInCodeParsed": 3, - "spanCodeLengthLineChars16": 3, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 8, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 8, - "token": { - "codeLenLineChars16": 3, - "codeLenNewlines": 0, - "codeStartCol16": 8, - "codeStartLine": 0, - "codeStartUtf8": 8, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": false, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 8, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 8, - "lengthInCodeBuffer": 3, - "liftLevel": 0, - "startInCodeBuffer": 8, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 8, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 8, - }, - "childrenLengthInCodeParsed": 11, - "close": undefined, - "equals": { - "codeLenLineChars16": 1, - "codeLenNewlines": 0, - "codeStartCol16": 7, - "codeStartLine": 0, - "codeStartUtf8": 7, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 7, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 7, - "lengthInCodeBuffer": 1, - "startInCodeBuffer": 7, - "type": "AssignmentOperator", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 7, - }, - "func": { - "childrenLengthInCodeParsed": 3, - "spanCodeLengthLineChars16": 3, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "token": { - "codeLenLineChars16": 3, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": false, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 3, - "liftLevel": 0, - "startInCodeBuffer": 0, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - "type": "Ident", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "name": { - "codeLenLineChars16": 3, - "codeLenNewlines": 0, - "codeStartCol16": 4, - "codeStartLine": 0, - "codeStartUtf8": 4, - "isFree": false, - "isOperatorLexically": false, - "isTypeOrConstructor": false, - "leftOffsetCodeLenLineChars16": 1, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 3, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 3, - "lengthInCodeBuffer": 3, - "liftLevel": 0, - "startInCodeBuffer": 4, - "type": "Ident", - "whitespaceLengthInCodeBuffer": 1, - "whitespaceStartInCodeBuffer": 3, - }, - "open": undefined, - "spanCodeLengthLineChars16": 11, - "spanCodeLengthNewlines": 0, - "spanLeftOffsetCodeLenLineChars16": 0, - "spanLeftOffsetCodeLenNewlines": 0, - "spanLeftOffsetCodeStartCol16": 0, - "spanLeftOffsetCodeStartLine": 0, - "spanLeftOffsetCodeStartUtf8": 0, - "type": "NamedApp", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, - }, - "newline": { - "codeLenLineChars16": 0, - "codeLenNewlines": 0, - "codeStartCol16": 0, - "codeStartLine": 0, - "codeStartUtf8": 0, - "leftOffsetCodeLenLineChars16": 0, - "leftOffsetCodeLenNewlines": 0, - "leftOffsetCodeStartCol16": 0, - "leftOffsetCodeStartLine": 0, - "leftOffsetCodeStartUtf8": 0, - "lengthInCodeBuffer": 0, - "startInCodeBuffer": 0, - "type": "Newline", - "whitespaceLengthInCodeBuffer": 0, - "whitespaceStartInCodeBuffer": 0, - }, - }, - ], - "type": "BodyBlock", - "warnings": [], - "whitespaceLengthInCodeParsed": 0, - "whitespaceStartInCodeParsed": 0, -} -`; diff --git a/app/gui/src/project-view/util/ast/__tests__/abstract.test.ts b/app/gui/src/project-view/util/ast/__tests__/abstract.test.ts index c343fe7bc9..2f9848ed24 100644 --- a/app/gui/src/project-view/util/ast/__tests__/abstract.test.ts +++ b/app/gui/src/project-view/util/ast/__tests__/abstract.test.ts @@ -15,6 +15,7 @@ import { } from '@/util/ast/abstract' import { fc, test } from '@fast-check/vitest' import { describe, expect } from 'vitest' +import { BodyBlock } from 'ydoc-shared/ast' import { findExpressions, testCase, tryFindExpressions } from './testCase' test('Raw block abstracts to Ast.BodyBlock', () => { @@ -25,10 +26,21 @@ test('Raw block abstracts to Ast.BodyBlock', () => { expect(abstracted.root).toBeInstanceOf(Ast.BodyBlock) }) -//const disabledCases = [ -// ' a', -// 'a ', -//] +// FIXME: Parsing source code and reprinting it should produce exactly the same output as input. The following cases are +// known to be incorrectly handled. For each such case the test checks the result of parsing and reprinting to ensure +// it is at least a reasonable normalization of the input. +const normalizingCases = [ + { input: ' a', normalized: ' a' }, + { input: 'a ', normalized: 'a \n' }, + { + input: ['main =', ' foo', ' bar', ' baz'].join('\n'), + normalized: ['main =', ' foo', ' bar', ' baz'].join('\n'), + }, + { + input: ['main =', ' foo', ' bar', 'baz'].join('\n'), + normalized: ['main =', ' foo', ' bar', 'baz'].join('\n'), + }, +] const cases = [ 'Console.', '(', @@ -309,8 +321,6 @@ const cases = [ ['foo', ' + bar +'].join('\n'), ['foo', ' + bar', ' - baz'].join('\n'), ['main =', ' foo', 'bar'].join('\n'), - ['main =', ' foo', ' bar', ' baz'].join('\n'), - ['main =', ' foo', ' bar', 'baz'].join('\n'), ['main ~foo = x'].join('\n'), ['main =', ' ', ' x'].join('\n'), ['main =', ' ', ' x'].join('\n'), @@ -375,13 +385,18 @@ const cases = [ '\n\n', '\na', '\n\na', + ...normalizingCases, ] -test.each(cases)('parse/print round trip: %s', (code) => { +test.each(cases)('parse/print round-trip: %s', (testCase) => { + const code = typeof testCase === 'object' ? testCase.input : testCase + const expectedCode = typeof testCase === 'object' ? testCase.normalized : testCase + // Get an AST. - const { root } = Ast.parseModuleWithSpans(code) + const root = Ast.parseModule(code) + root.module.setRoot(root) // Print AST back to source. const printed = Ast.print(root) - expect(printed.code).toEqual(code) + expect(printed.code).toEqual(expectedCode) // Loading token IDs from IdMaps is not implemented yet, fix during sync. printed.info.tokens.clear() const idMap = Ast.spanMapToIdMap(printed.info) @@ -403,22 +418,29 @@ test.each(cases)('parse/print round trip: %s', (code) => { }) const parseCases = [ - { code: 'foo bar+baz', tree: ['', [['foo'], [['bar'], '+', ['baz']]]] }, - { code: '(foo)', tree: ['', ['(', ['foo'], ')']] }, + { code: 'foo bar+baz', tree: [['foo'], [['bar'], '+', ['baz']]] }, + { code: '(foo)', tree: ['(', ['foo'], ')'] }, ] test.each(parseCases)('parse: %s', (testCase) => { - const root = Ast.parseBlock(testCase.code) + const root = Ast.parseExpression(testCase.code) + assertDefined(root) expect(Ast.tokenTree(root)).toEqual(testCase.tree) }) +function functionBlock(topLevel: BodyBlock, name: string) { + const func = findModuleMethod(topLevel, name) + if (!(func?.statement.body instanceof BodyBlock)) return undefined + return func.statement.body +} + test('Insert new expression', () => { const code = 'main =\n text1 = "foo"\n' const root = Ast.parseBlock(code) - const main = Ast.functionBlock(root, 'main')! + const main = functionBlock(root, 'main')! expect(main).toBeDefined() const edit = root.module.edit() - const rhs = Ast.parse('42', edit) - const assignment = Ast.Assignment.new(edit, 'baz' as Identifier, rhs) + const rhs = Ast.parseExpression('42', edit)! + const assignment = Ast.Assignment.new('baz' as Identifier, rhs, { edit }) edit.getVersion(main).push(assignment) const printed = edit.getVersion(root).code() expect(printed).toEqual('main =\n text1 = "foo"\n baz = 42\n') @@ -433,7 +455,7 @@ type SimpleModule = { function simpleModule(): SimpleModule { const code = 'main =\n text1 = "foo"\n' const root = Ast.parseBlock(code) - const main = findModuleMethod(root, 'main')! + const main = findModuleMethod(root, 'main')!.statement const mainBlock = main.body instanceof Ast.BodyBlock ? main.body : null assert(mainBlock != null) expect(mainBlock).toBeInstanceOf(Ast.BodyBlock) @@ -475,8 +497,8 @@ test('Replace subexpression', () => { const newValue = Ast.TextLiteral.new('bar', edit) expect(newValue.code()).toBe("'bar'") edit.replace(assignment.expression!.id, newValue) - const assignment_ = edit.tryGet(assignment.id)! - assert(assignment_ instanceof Ast.Assignment) + const assignment_ = edit.tryGet(assignment.id) + assert(assignment_ instanceof Ast.MutableAssignment) expect(assignment_.expression!.id).toBe(newValue.id) expect(edit.tryGet(assignment_.expression!.id)?.code()).toBe("'bar'") const printed = edit.getVersion(root).code() @@ -487,14 +509,16 @@ test('Modify subexpression - setting a vector', () => { // A case where the #9357 bug was visible. const code = 'main =\n text1 = foo\n' const root = Ast.parseBlock(code) - const main = Ast.functionBlock(root, 'main')! + const main = functionBlock(root, 'main')! expect(main).not.toBeNull() const assignment: Ast.Assignment = main.statements().next().value expect(assignment).toBeInstanceOf(Ast.Assignment) const edit = root.module.edit() const transientModule = MutableModule.Transient() - const newValue = Ast.Vector.new(transientModule, [Ast.parse('bar')]) + const barExpression = Ast.parseExpression('bar') + assertDefined(barExpression) + const newValue = Ast.Vector.new(transientModule, [barExpression]) expect(newValue.code()).toBe('[bar]') edit.replaceValue(assignment.expression.id, newValue) const printed = edit.getVersion(root).code() @@ -520,10 +544,10 @@ test('Block lines interface', () => { const block = Ast.parseBlock('VLE \nSISI\nGNIK \n') // Sort alphabetically, but keep the blank line at the end. const reordered = block.takeLines().sort((a, b) => { - if (a.expression?.node.code() === b.expression?.node.code()) return 0 - if (!a.expression) return 1 - if (!b.expression) return -1 - return a.expression.node.code() < b.expression.node.code() ? -1 : 1 + if (a.statement?.node.code() === b.statement?.node.code()) return 0 + if (!a.statement) return 1 + if (!b.statement) return -1 + return a.statement.node.code() < b.statement.node.code() ? -1 : 1 }) const edit = block.module.edit() const newBlock = Ast.BodyBlock.new(reordered, edit) @@ -560,16 +584,19 @@ test('Construct app', () => { }) test('Automatic parenthesis', () => { - const block = Ast.parseBlock('main = func arg1 arg2') + const block = Ast.parseModule('main = func arg1 arg2') + block.module.setRoot(block) let arg1: Ast.MutableAst | undefined - block.visitRecursiveAst((ast) => { + block.visitRecursive((ast) => { if (ast instanceof Ast.MutableIdent && ast.code() === 'arg1') { assert(!arg1) arg1 = ast } }) assert(arg1 != null) - arg1.replace(Ast.parse('innerfunc innerarg', block.module)) + const replacementExpr = Ast.parseExpression('innerfunc innerarg', block.module) + assertDefined(replacementExpr) + arg1.replace(replacementExpr) const correctCode = 'main = func (innerfunc innerarg) arg2' // This assertion will fail when smart printing handles this case. // At that point we should test tree repair separately. @@ -583,7 +610,7 @@ test('Tree repair: Non-canonical block line attribution', () => { 'func a b =': Ast.Function, ' c = a + b': Ast.Assignment, 'main =': Ast.Function, - ' func arg1 arg2': Ast.App, + ' func arg1 arg2': Ast.ExpressionStatement, }) const before = beforeCase.statements @@ -601,7 +628,7 @@ test('Tree repair: Non-canonical block line attribution', () => { 'func a b =': Ast.Function, 'c = a + b': Ast.Assignment, 'main =': Ast.Function, - 'func arg1 arg2': Ast.App, + 'func arg1 arg2': Ast.ExpressionStatement, }) const repairedFunc = afterRepair['func a b ='] assert(repairedFunc.body instanceof Ast.BodyBlock) @@ -617,8 +644,9 @@ test('Tree repair: Non-canonical block line attribution', () => { describe('Code edit', () => { test('Change argument type', () => { - const beforeRoot = Ast.parse('func arg1 arg2') - beforeRoot.module.replaceRoot(beforeRoot) + const beforeRoot = Ast.parseExpression('func arg1 arg2') + assertDefined(beforeRoot) + beforeRoot.module.setRoot(beforeRoot) const before = findExpressions(beforeRoot, { func: Ast.Ident, arg1: Ast.Ident, @@ -646,8 +674,9 @@ describe('Code edit', () => { }) test('Insert argument names', () => { - const beforeRoot = Ast.parse('func arg1 arg2') - beforeRoot.module.replaceRoot(beforeRoot) + const beforeRoot = Ast.parseExpression('func arg1 arg2') + assertDefined(beforeRoot) + beforeRoot.module.setRoot(beforeRoot) const before = findExpressions(beforeRoot, { func: Ast.Ident, arg1: Ast.Ident, @@ -676,8 +705,9 @@ describe('Code edit', () => { }) test('Remove argument names', () => { - const beforeRoot = Ast.parse('func name1=arg1 name2=arg2') - beforeRoot.module.replaceRoot(beforeRoot) + const beforeRoot = Ast.parseExpression('func name1=arg1 name2=arg2') + assertDefined(beforeRoot) + beforeRoot.module.setRoot(beforeRoot) const before = findExpressions(beforeRoot, { func: Ast.Ident, arg1: Ast.Ident, @@ -768,8 +798,9 @@ describe('Code edit', () => { }) test('Inline expression change', () => { - const beforeRoot = Ast.parse('func name1=arg1 name2=arg2') - beforeRoot.module.replaceRoot(beforeRoot) + const beforeRoot = Ast.parseExpression('func name1=arg1 name2=arg2') + assertDefined(beforeRoot) + beforeRoot.module.setRoot(beforeRoot) const before = findExpressions(beforeRoot, { func: Ast.Ident, arg1: Ast.Ident, @@ -800,9 +831,10 @@ describe('Code edit', () => { test('No-op inline expression change', () => { const code = 'a = 1' - const expression = Ast.parse(code) + const expression = Ast.parseStatement(code) + assertDefined(expression) const module = expression.module - module.replaceRoot(expression) + module.setRoot(expression) expression.syncToCode(code) expect(module.root()?.code()).toBe(code) }) @@ -811,14 +843,14 @@ describe('Code edit', () => { const code = 'a = 1\nb = 2\n' const block = Ast.parseBlock(code) const module = block.module - module.replaceRoot(block) + module.setRoot(block) block.syncToCode(code) expect(module.root()?.code()).toBe(code) }) test('Shifting whitespace ownership', () => { - const beforeRoot = Ast.parseModuleWithSpans('value = 1 +\n').root - beforeRoot.module.replaceRoot(beforeRoot) + const beforeRoot = Ast.parseModule('value = 1 +\n') + beforeRoot.module.setRoot(beforeRoot) const before = findExpressions(beforeRoot, { value: Ast.Ident, '1': Ast.NumericLiteral, @@ -841,9 +873,9 @@ describe('Code edit', () => { }) test('merging', () => { - const block = Ast.parseModuleWithSpans('a = 1\nb = 2').root + const block = Ast.parseModule('a = 1\nb = 2') const module = block.module - module.replaceRoot(block) + module.setRoot(block) const editA = module.edit() editA.getVersion(block).syncToCode('a = 10\nb = 2') @@ -858,7 +890,8 @@ describe('Code edit', () => { }) test('Analyze app-like', () => { - const appLike = Ast.parse('(Preprocessor.default_preprocessor 3 _ 5 _ <| 4) <| 6') + const appLike = Ast.parseExpression('(Preprocessor.default_preprocessor 3 _ 5 _ <| 4) <| 6') + assertDefined(appLike) const { func, args } = Ast.analyzeAppLike(appLike) expect(func.code()).toBe('Preprocessor.default_preprocessor') expect(args.map((ast) => ast.code())).toEqual(['3', '4', '5', '6']) @@ -904,9 +937,9 @@ test.each([ ])( 'Substitute qualified name $pattern inside $original', ({ original, pattern, substitution, expected }) => { - const expression = Ast.parse(original) + const expression = Ast.parseExpression(original) ?? Ast.parseStatement(original) const module = expression.module - module.replaceRoot(expression) + module.setRoot(expression) const edit = expression.module.edit() substituteQualifiedName(expression, pattern as Ast.Identifier, substitution as Ast.Identifier) module.applyEdit(edit) @@ -960,9 +993,9 @@ test.each([ ])( 'Substitute identifier $pattern inside $original', ({ original, pattern, substitution, expected }) => { - const expression = Ast.parse(original) + const expression = Ast.parseExpression(original) ?? Ast.parseStatement(original) const module = expression.module - module.replaceRoot(expression) + module.setRoot(expression) const edit = expression.module.edit() substituteIdentifier(expression, pattern as Ast.Identifier, substitution as Ast.Identifier) module.applyEdit(edit) @@ -1037,79 +1070,6 @@ test('setRawTextContent promotes single-line uninterpolated text to interpolated expect(literal.code()).toBe(`'${escapeTextLiteral(rawText)}'`) }) -const docEditCases = [ - { code: '## Simple\nnode', documentation: 'Simple' }, - { - code: '## Preferred indent\n 2nd line\n 3rd line\nnode', - documentation: 'Preferred indent\n2nd line\n3rd line', - }, - { - code: '## Extra-indented child\n 2nd line\n 3rd line\nnode', - documentation: 'Extra-indented child\n2nd line\n3rd line', - normalized: '## Extra-indented child\n 2nd line\n 3rd line\nnode', - }, - { - code: '## Extra-indented child, beyond 4th column\n 2nd line\n 3rd line\nnode', - documentation: 'Extra-indented child, beyond 4th column\n2nd line\n 3rd line', - normalized: '## Extra-indented child, beyond 4th column\n 2nd line\n 3rd line\nnode', - }, - { - code: '##Preferred indent, no initial space\n 2nd line\n 3rd line\nnode', - documentation: 'Preferred indent, no initial space\n2nd line\n3rd line', - normalized: '## Preferred indent, no initial space\n 2nd line\n 3rd line\nnode', - }, - { - code: '## Minimum indent\n 2nd line\n 3rd line\nnode', - documentation: 'Minimum indent\n2nd line\n3rd line', - normalized: '## Minimum indent\n 2nd line\n 3rd line\nnode', - }, -] -test.each(docEditCases)('Documentation edit round trip: $code', (docCase) => { - const { code, documentation } = docCase - const parsed = Ast.Documented.tryParse(code) - assert(parsed != null) - const parsedDocumentation = parsed.documentation() - expect(parsedDocumentation).toBe(documentation) - const edited = MutableModule.Transient().copy(parsed) - edited.setDocumentationText(parsedDocumentation) - expect(edited.code()).toBe(docCase.normalized ?? code) -}) - -test.each([ - '## Some documentation\nf x = 123', - '## Some documentation\n and a second line\nf x = 123', - '## Some documentation## Another documentation??\nf x = 123', -])('Finding documentation: $code', (code) => { - const block = Ast.parseBlock(code) - const method = Ast.findModuleMethod(block, 'f') - assertDefined(method) - expect(method.documentingAncestor()).toBeDefined() -}) - -test.each([ - { - code: '## Already documented\nf x = 123', - expected: '## Already documented\nf x = 123', - }, - { - code: 'f x = 123', - expected: '## \nf x = 123', - }, -])('Adding documentation: $code', ({ code, expected }) => { - const block = Ast.parseBlock(code) - const module = block.module - const method = module.getVersion(Ast.findModuleMethod(block, 'f')!) - method.getOrInitDocumentation() - expect(block.code()).toBe(expected) -}) - -test('Creating comments', () => { - const expr = Ast.parse('2 + 2') - expr.module.replaceRoot(expr) - expr.update((expr) => Ast.Documented.new('Calculate five', expr)) - expect(expr.module.root()?.code()).toBe('## Calculate five\n2 + 2') -}) - test.each([ { code: 'operator1', expected: { subject: 'operator1', accesses: [] } }, { code: 'operator1 foo bar', expected: { subject: 'operator1 foo bar', accesses: [] } }, @@ -1132,7 +1092,7 @@ test.each([ }, { code: 'operator1 + operator2', expected: { subject: 'operator1 + operator2', accesses: [] } }, ])('Access chain in $code', ({ code, expected }) => { - const ast = Ast.parse(code) + const ast = Ast.parseExpression(code) const { subject, accessChain } = Ast.accessChain(ast) expect({ subject: subject.code(), @@ -1148,7 +1108,7 @@ test.each` `('Pushing $pushed to vector $initial', ({ initial, pushed, expected }) => { const vector = Ast.Vector.tryParse(initial) assertDefined(vector) - vector.push(Ast.parse(pushed, vector.module)) + vector.push(Ast.parseExpression(pushed, vector.module)) expect(vector.code()).toBe(expected) }) @@ -1228,7 +1188,7 @@ test.each` ({ initial, index, value, expected }) => { const vector = Ast.Vector.tryParse(initial) assertDefined(vector) - vector.set(index, Ast.parse(value, vector.module)) + vector.set(index, Ast.parseExpression(value, vector.module)) expect(vector.code()).toBe(expected) }, ) @@ -1250,7 +1210,7 @@ test.each` 'Conversions between enso literals and js numbers: $ensoNumber', ({ ensoNumber, jsNumber, expectedEnsoNumber }) => { if (ensoNumber != null) { - const literal = Ast.parse(ensoNumber) + const literal = Ast.parseExpression(ensoNumber) expect(tryEnsoToNumber(literal)).toBe(jsNumber) } if (jsNumber != null) { diff --git a/app/gui/src/project-view/util/ast/__tests__/documentation.test.ts b/app/gui/src/project-view/util/ast/__tests__/documentation.test.ts new file mode 100644 index 0000000000..df613ce05d --- /dev/null +++ b/app/gui/src/project-view/util/ast/__tests__/documentation.test.ts @@ -0,0 +1,90 @@ +import { assert } from '@/util/assert' +import { Ast } from '@/util/ast' +import { test } from '@fast-check/vitest' +import { expect } from 'vitest' + +test.each([ + { code: '## Simple\nnode', documentation: 'Simple' }, + { + code: '## Preferred indent\n 2nd line\n 3rd line\nnode', + documentation: 'Preferred indent\n2nd line\n3rd line', + }, + { + code: '## Extra-indented child\n 2nd line\n 3rd line\nnode', + documentation: 'Extra-indented child\n2nd line\n3rd line', + normalized: '## Extra-indented child\n 2nd line\n 3rd line\nnode', + }, + { + code: '## Extra-indented child, beyond 4th column\n 2nd line\n 3rd line\nnode', + documentation: 'Extra-indented child, beyond 4th column\n2nd line\n 3rd line', + normalized: '## Extra-indented child, beyond 4th column\n 2nd line\n 3rd line\nnode', + }, + { + code: '##Preferred indent, no initial space\n 2nd line\n 3rd line\nnode', + documentation: 'Preferred indent, no initial space\n2nd line\n3rd line', + normalized: '## Preferred indent, no initial space\n 2nd line\n 3rd line\nnode', + }, + { + code: '## Minimum indent\n 2nd line\n 3rd line\nnode', + documentation: 'Minimum indent\n2nd line\n3rd line', + normalized: '## Minimum indent\n 2nd line\n 3rd line\nnode', + }, +])('Documentation edit round-trip: $code', (docCase) => { + const { code, documentation } = docCase + const parsed = Ast.parseStatement(code)! + const parsedDocumentation = parsed.documentationText() + expect(parsedDocumentation).toBe(documentation) + const edited = Ast.MutableModule.Transient().copy(parsed) + assert('setDocumentationText' in edited) + edited.setDocumentationText(parsedDocumentation) + expect(edited.code()).toBe(docCase.normalized ?? code) +}) + +test.each([ + '## Some documentation\nf x = 123', + '## Some documentation\n and a second line\nf x = 123', + '## Some documentation## Another documentation??\nf x = 123', +])('Finding documentation: $code', (code) => { + const block = Ast.parseBlock(code) + const method = Ast.findModuleMethod(block, 'f')!.statement + expect(method.documentationText()).toBeTruthy() +}) + +test.each([ + { + code: '## Already documented\nf x = 123', + expected: '## Already documented\nf x = 123', + }, + { + code: 'f x = 123', + expected: '##\nf x = 123', + }, +])('Adding documentation: $code', ({ code, expected }) => { + const block = Ast.parseBlock(code) + const module = block.module + const method = module.getVersion(Ast.findModuleMethod(block, 'f')!.statement) + if (method.documentationText() === undefined) { + method.setDocumentationText('') + } + expect(block.code()).toBe(expected) +}) + +test('Creating comments', () => { + const block = Ast.parseBlock('2 + 2') + block.module.setRoot(block) + const statement = [...block.statements()][0]! as Ast.MutableExpressionStatement + const docText = 'Calculate five' + statement.setDocumentationText(docText) + expect(statement.module.root()?.code()).toBe(`## ${docText}\n2 + 2`) +}) + +test('Creating comments: indented', () => { + const block = Ast.parseBlock('main =\n x = 1') + const module = block.module + module.setRoot(block) + const main = module.getVersion(Ast.findModuleMethod(block, 'main')!.statement) + const statement = [...main.bodyAsBlock().statements()][0]! as Ast.MutableAssignment + const docText = 'The smallest natural number' + statement.setDocumentationText(docText) + expect(statement.module.root()?.code()).toBe(`main =\n ## ${docText}\n x = 1`) +}) diff --git a/app/gui/src/project-view/util/ast/__tests__/match.test.ts b/app/gui/src/project-view/util/ast/__tests__/match.test.ts index dad897734c..be7bd5445e 100644 --- a/app/gui/src/project-view/util/ast/__tests__/match.test.ts +++ b/app/gui/src/project-view/util/ast/__tests__/match.test.ts @@ -80,9 +80,9 @@ test.each([ extracted: ['with_enabled_context', "'current_context_name'", 'a + b'], }, ])('`isMatch` and `extractMatches`', ({ target, pattern, extracted }) => { - const targetAst = Ast.parse(target) + const targetAst = Ast.parseExpression(target) const module = targetAst.module - const patternAst = Pattern.parse(pattern) + const patternAst = Pattern.parseExpression(pattern) expect( patternAst.match(targetAst) !== undefined, `'${target}' has CST ${extracted != null ? '' : 'not '}matching '${pattern}'`, @@ -101,9 +101,9 @@ test.each([ { template: 'a __ c', source: 'b', result: 'a b c' }, { template: 'a . __ . c', source: 'b', result: 'a . b . c' }, ])('instantiate', ({ template, source, result }) => { - const pattern = Pattern.parse(template) + const pattern = Pattern.parseExpression(template) const edit = MutableModule.Transient() - const intron = Ast.parse(source, edit) + const intron = Ast.parseExpression(source, edit) const instantiated = pattern.instantiate(edit, [intron]) expect(instantiated.code()).toBe(result) }) diff --git a/app/gui/src/project-view/util/ast/__tests__/node.test.ts b/app/gui/src/project-view/util/ast/__tests__/node.test.ts index cc9dcccd08..a6a7151265 100644 --- a/app/gui/src/project-view/util/ast/__tests__/node.test.ts +++ b/app/gui/src/project-view/util/ast/__tests__/node.test.ts @@ -10,17 +10,17 @@ test.each` ${'## Documentation\n2 + 2'} | ${undefined} | ${'2 + 2'} | ${'Documentation'} ${'## Documentation\nfoo = 2 + 2'} | ${'foo'} | ${'2 + 2'} | ${'Documentation'} `('Node information from AST $line line', ({ line, pattern, rootExpr, documentation }) => { - const ast = Ast.Ast.parse(line) + const ast = [...Ast.parseBlock(line).statements()][0]! const node = nodeFromAst(ast, false) - expect(node?.outerExpr).toBe(ast) + expect(node?.outerAst).toBe(ast) expect(node?.pattern?.code()).toBe(pattern) expect(node?.rootExpr.code()).toBe(rootExpr) expect(node?.innerExpr.code()).toBe(rootExpr) - expect(node?.docs?.documentation()).toBe(documentation) + expect(node?.outerAst.isStatement() && node.outerAst.documentationText()).toBe(documentation) }) test.each(['## Documentation only'])("'%s' should not be a node", (line) => { - const ast = Ast.Ast.parse(line) + const ast = Ast.parseStatement(line) const node = nodeFromAst(ast, false) expect(node).toBeUndefined() }) @@ -47,7 +47,7 @@ test.each([ }, { code: 'operator1 + operator2', expected: undefined }, ])('Primary application subject of $code', ({ code, expected }) => { - const ast = Ast.Ast.parse(code) + const ast = Ast.parseExpression(code) const module = ast.module const primaryApplication = primaryApplicationSubject(ast) const analyzed = primaryApplication && { diff --git a/app/gui/src/project-view/util/ast/__tests__/opr.test.ts b/app/gui/src/project-view/util/ast/__tests__/opr.test.ts deleted file mode 100644 index 3dbd981976..0000000000 --- a/app/gui/src/project-view/util/ast/__tests__/opr.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { assert } from '@/util/assert' -import { RawAstExtended } from '@/util/ast/extended' -import { GeneralOprApp, operandsOfLeftAssocOprChain, type OperatorChain } from '@/util/ast/opr' -import { RawAst } from '@/util/ast/raw' -import { expect, test } from 'vitest' - -test.each([ - { code: '2 + 3', result: ['2', '+', '3'] }, - { code: '2 + 4 + 5', result: ['2 + 4', '+', '5'] }, - { code: '2\n + 3\n + 4', result: ['2', '+', '3', '+', '4'] }, - { code: '2\n - 4\n * 5', result: ['2', '-', '4', '*', '5'] }, - { code: 'foo . bar\n . baz', result: ['foo . bar', '.', 'baz'] }, - // See https://github.com/orgs/enso-org/discussions/8021 - // { code: '2 + 3\n + 4', result: ['2 + 3', '+', '4'] }, - { code: '+ 2', result: [null, '+', '2'] }, - { code: '2 +', result: ['2', '+', null] }, - { code: '.foo', result: [null, '.', 'foo'] }, - { code: 'foo.', result: ['foo', '.', null] }, -])('Generalized infix from $code', ({ code, result }) => { - let ast = RawAstExtended.parseLine(code) - if (ast.isTree(RawAst.Tree.Type.OprSectionBoundary)) { - ast = ast.map((boundary) => boundary.ast) - } - assert( - ast.isTree(RawAst.Tree.Type.OprApp) || ast.isTree(RawAst.Tree.Type.OperatorBlockApplication), - ) - const opr = new GeneralOprApp(ast as OperatorChain) - expect(Array.from(opr.componentsReprs())).toStrictEqual(result) -}) - -test.each([ - { - code: '2 + 3', - result: [ - { type: 'ast', repr: '2' }, - { type: 'ast', repr: '3' }, - ], - }, - { - code: '2 + 3 + 4', - result: [ - { type: 'ast', repr: '2' }, - { type: 'ast', repr: '3' }, - { type: 'ast', repr: '4' }, - ], - }, - { - code: '2 * 3 + 4', - result: [ - { type: 'ast', repr: '2 * 3' }, - { type: 'ast', repr: '4' }, - ], - }, - { - code: '2\n + 3\n + 4', - result: [ - { type: 'ast', repr: '2' }, - { type: 'ast', repr: '3' }, - { type: 'ast', repr: '4' }, - ], - }, - // See https://github.com/orgs/enso-org/discussions/8021 - // { - // code: '2 + 3\n + 4', - // result: [ - // { type: 'ast', repr: '2' }, - // { type: 'ast', repr: '3' }, - // { type: 'ast', repr: '4' }, - // ], - // }, - // There is a bug in AST spans in some OperatorBlockApplications. Fix this test once fixed - { - code: '2\n * 3\n + 44', - result: [ - { type: 'partOfOprBlockApp', repr: '2\n * 3\n + 44', statements: 1 }, - { type: 'ast', repr: '44' }, - ], - }, - { - code: '2\n + 3\n * 4\n + 55', - result: [ - { type: 'partOfOprBlockApp', repr: '2\n + 3\n * 4\n + 55', statements: 2 }, - { type: 'ast', repr: '55' }, - ], - }, - // https://github.com/orgs/enso-org/discussions/8021 - // { - // code: '2 * 3\n + 4', - // result: [ - // { type: 'ast', repr: '2 * 3' }, - // { type: 'ast', repr: '4' }, - // ], - // }, - { - code: 'foo bar', - result: [{ type: 'ast', repr: 'foo bar' }], - }, - { - code: '2 * 3', - opr: '+', - result: [{ type: 'ast', repr: '2 * 3' }], - }, -])( - 'Getting left-associative operator operands in $code', - ({ - code, - opr, - result, - }: { - code: string - opr?: string - result: { type: string; repr: string; statements?: number }[] - }) => { - const ast = RawAstExtended.parseLine(code) - const actual = operandsOfLeftAssocOprChain(ast, opr) - const actualWithExpected = Array.from(actual, (operand, i) => { - return { actual: operand, expected: result[i] } - }) - for (const { actual, expected } of actualWithExpected) { - if (expected === null) { - expect(actual).toBeNull() - } else { - expect(actual?.type).toStrictEqual(expected?.type) - if (actual?.type === 'ast') { - expect(actual.ast.repr()).toStrictEqual(expected?.repr) - } else { - assert(actual?.type == 'partOfOprBlockApp') - expect(actual.ast.repr()).toStrictEqual(expected?.repr) - expect(actual.statements).toStrictEqual(expected?.statements) - } - } - } - }, -) diff --git a/app/gui/src/project-view/util/ast/__tests__/prefixes.test.ts b/app/gui/src/project-view/util/ast/__tests__/prefixes.test.ts index c191b95647..fee5228fa7 100644 --- a/app/gui/src/project-view/util/ast/__tests__/prefixes.test.ts +++ b/app/gui/src/project-view/util/ast/__tests__/prefixes.test.ts @@ -1,4 +1,4 @@ -import { Ast } from '@/util/ast/abstract' +import { Ast } from '@/util/ast' import { Prefixes } from '@/util/ast/prefixes' import { expect, test } from 'vitest' @@ -67,13 +67,13 @@ test.each([ }, ])('modify', ({ prefixes: lines, modifications, source, target }) => { const prefixes = Prefixes.FromLines(lines as any) - const sourceAst = Ast.parse(source) - sourceAst.module.replaceRoot(sourceAst) + const sourceAst = Ast.parseExpression(source) + sourceAst.module.setRoot(sourceAst) const edit = sourceAst.module.edit() const modificationAsts = Object.fromEntries( Object.entries(modifications).map(([k, v]) => [ k, - v ? Array.from(v, (mod) => Ast.parse(mod, edit)) : undefined, + v ? Array.from(v, (mod) => Ast.parseExpression(mod, edit)) : undefined, ]), ) prefixes.modify(edit.getVersion(sourceAst), modificationAsts) diff --git a/app/gui/src/project-view/util/ast/__tests__/raw.test.ts b/app/gui/src/project-view/util/ast/__tests__/raw.test.ts index 6217d4608c..efacea35a3 100644 --- a/app/gui/src/project-view/util/ast/__tests__/raw.test.ts +++ b/app/gui/src/project-view/util/ast/__tests__/raw.test.ts @@ -1,17 +1,21 @@ -import { - astContainingChar, - childrenAstNodes, - debugAst, - rawParseLine, - rawParseModule, - readAstOrTokenSpan, - readAstSpan, - readTokenSpan, - walkRecursive, -} from '@/util/ast/raw' +import { RawAst, rawParseModule, readAstOrTokenSpan, walkRecursive } from '@/util/ast/raw' import { assert, expect, test } from 'vitest' import { Token, Tree } from 'ydoc-shared/ast/generated/ast' import type { LazyObject } from 'ydoc-shared/ast/parserSupport' +import { assertDefined } from 'ydoc-shared/util/assert' +import { tryGetSoleValue } from 'ydoc-shared/util/data/iterable' + +/** + * Read a single line of code + * + * Helper for tests. If the code is multiline, an exception is raised. + */ +function rawParseLine(code: string): RawAst.Tree { + const block = rawParseModule(code) + const soleExpression = tryGetSoleValue(block.statements)?.expression + assertDefined(soleExpression) + return soleExpression +} function validateSpans(obj: LazyObject, initialPos?: number): number { const state = { pos: initialPos ?? 0 } @@ -47,97 +51,17 @@ const parseCases = [ '2\n + 3\n + 4', ] -test.each(parseCases)("Parsing '%s'", (code) => { - expect(debugAst(rawParseModule(code))).toMatchSnapshot() -}) - test.each(parseCases)("AST spans of '%s' are valid", (input) => { const tree = rawParseModule(input) const endPos = validateSpans(tree) expect(endPos).toStrictEqual(input.length) }) -test("Reading AST node's code", () => { - const code = 'Data.read File\n2 + 3' - const ast = rawParseModule(code) - expect(readAstSpan(ast, code)).toStrictEqual(code) - assert(ast.type === Tree.Type.BodyBlock) - const statements = Array.from(ast.statements) - - assert(statements[0]?.expression != null) - expect(readAstSpan(statements[0].expression, code)).toStrictEqual('Data.read File') - assert(statements[0].expression.type === Tree.Type.App) - expect(readAstSpan(statements[0].expression.func, code)).toStrictEqual('Data.read') - expect(readAstSpan(statements[0].expression.arg, code)).toStrictEqual('File') - - assert(statements[1]?.expression != null) - expect(readAstSpan(statements[1].expression, code)).toStrictEqual('2 + 3') - assert(statements[1].expression.type === Tree.Type.OprApp) - assert(statements[1].expression.lhs != null) - assert(statements[1].expression.rhs != null) - assert(statements[1].expression.opr.ok) - expect(readAstSpan(statements[1].expression.lhs, code)).toStrictEqual('2') - expect(readTokenSpan(statements[1].expression.opr.value, code)).toStrictEqual('+') - expect(readAstSpan(statements[1].expression.rhs, code)).toStrictEqual('3') -}) - -test.each([ - [ - '2 + a', - [ - { type: Tree.Type.Number, repr: '2' }, - { type: Tree.Type.Ident, repr: 'a' }, - ], - ], - [ - 'a.b', - [ - { type: Tree.Type.Ident, repr: 'a' }, - { type: Tree.Type.Ident, repr: 'b' }, - ], - ], - [ - 'Data.read foo', - [ - { type: Tree.Type.OprApp, repr: 'Data.read' }, - { type: Tree.Type.Ident, repr: 'foo' }, - ], - ], - ['(2 + a)', [{ type: Tree.Type.OprApp, repr: '2 + a' }]], - [ - 'Data.read\n foo\n bar', - [ - { type: Tree.Type.OprApp, repr: 'Data.read' }, - { type: Tree.Type.Ident, repr: 'foo' }, - { type: Tree.Type.Ident, repr: 'bar' }, - ], - ], - [ - 'Data.read file=foo', - [ - { type: Tree.Type.OprApp, repr: 'Data.read' }, - { type: Tree.Type.Ident, repr: 'foo' }, - ], - ], - // These are Invalid nodes, so the child is a subtree containing the whole expression. - ['(', [{ type: Tree.Type.Group, repr: '(' }]], - ['(foo', [{ type: Tree.Type.Group, repr: '(foo' }]], -])("Reading children of '%s'", (code, expected) => { - const ast = rawParseLine(code) - const children = Array.from(childrenAstNodes(ast)) - const childrenWithExpected = children.map((child, i) => { - return { child, expected: expected[i] } - }) - for (const { child, expected } of childrenWithExpected) { - expect(child.type).toBe(expected?.type) - expect(readAstSpan(child, code)).toBe(expected?.repr) - } -}) - test.each([ [ '2 + a', [ + { tree: Tree.Type.ExpressionStatement, repr: '2 + a' }, { tree: Tree.Type.OprApp, repr: '2 + a' }, { tree: Tree.Type.Number, repr: '2' }, { token: Token.Type.Digits, repr: '2' }, @@ -158,61 +82,3 @@ test.each([ expect(visitedRepr).toStrictEqual(expected) }) - -test.each([ - [ - '2 + a', - 0, - [ - { type: Tree.Type.Number, repr: '2' }, - { type: Tree.Type.OprApp, repr: '2 + a' }, - { type: Tree.Type.BodyBlock, repr: '2 + a' }, - ], - ], - [ - 'Data.read foo', - 5, - [ - { type: Tree.Type.Ident, repr: 'read' }, - { type: Tree.Type.OprApp, repr: 'Data.read' }, - { type: Tree.Type.App, repr: 'Data.read foo' }, - { type: Tree.Type.BodyBlock, repr: 'Data.read foo' }, - ], - ], - [ - 'Data.read foo', - 4, - [ - { type: Tree.Type.OprApp, repr: 'Data.read' }, - { type: Tree.Type.App, repr: 'Data.read foo' }, - { type: Tree.Type.BodyBlock, repr: 'Data.read foo' }, - ], - ], - [ - 'Data.read foo', - 9, - [ - { type: Tree.Type.App, repr: 'Data.read foo' }, - { type: Tree.Type.BodyBlock, repr: 'Data.read foo' }, - ], - ], - [ - 'Data.', - 4, - [ - { type: Tree.Type.OprApp, repr: 'Data.' }, - { type: Tree.Type.OprSectionBoundary, repr: 'Data.' }, - { type: Tree.Type.BodyBlock, repr: 'Data.' }, - ], - ], -])("Reading AST from code '%s' and position %i", (code, position, expected) => { - const ast = rawParseModule(code) - const astAtPosition = astContainingChar(position, ast) - const resultWithExpected = astAtPosition.map((ast, i) => { - return { ast, expected: expected[i] } - }) - for (const { ast, expected } of resultWithExpected) { - expect(ast.type).toBe(expected?.type) - expect(readAstSpan(ast, code)).toBe(expected?.repr) - } -}) diff --git a/app/gui/src/project-view/util/ast/__tests__/reactive.test.ts b/app/gui/src/project-view/util/ast/__tests__/reactive.test.ts index 17dde6fb82..c4c6ec6680 100644 --- a/app/gui/src/project-view/util/ast/__tests__/reactive.test.ts +++ b/app/gui/src/project-view/util/ast/__tests__/reactive.test.ts @@ -5,21 +5,28 @@ import { nextTick, watchEffect } from 'vue' import * as Y from 'yjs' test('Module reactivity: applyEdit', async () => { - const beforeEdit = Ast.parse('func arg1 arg2') - beforeEdit.module.replaceRoot(beforeEdit) + const beforeEdit = Ast.parseBlock('func arg1 arg2') + beforeEdit.module.setRoot(beforeEdit) const module = reactiveModule(new Y.Doc(), () => {}) module.applyEdit(beforeEdit.module) expect(module.root()!.code()).toBe(beforeEdit.code()) - const app2 = module.root() as unknown as Ast.App + const app2 = ( + (module.root() as Ast.MutableBodyBlock).lines[0]!.statement! + .node as Ast.MutableExpressionStatement + ).expression as unknown as Ast.App let app2Code: string | undefined = undefined watchEffect(() => (app2Code = app2.argument.code())) expect(app2Code).toBe('arg2') const edit = beforeEdit.module.edit() - const editApp2 = edit.getVersion(beforeEdit) as any as Ast.MutableApp - editApp2.setArgument(Ast.Ident.tryParse('newArg', edit)!) + const editApp2 = ( + edit.getVersion(beforeEdit).lines[0]!.statement!.node as Ast.MutableExpressionStatement + ).expression as Ast.MutableApp + const newArg = Ast.Ident.tryParse('newArg', edit) + expect(newArg).toBeDefined() + editApp2.setArgument(newArg!) const codeAfterEdit = 'func arg1 newArg' expect(edit.root()!.code()).toBe(codeAfterEdit) @@ -30,8 +37,8 @@ test('Module reactivity: applyEdit', async () => { }) test('Module reactivity: Direct Edit', async () => { - const beforeEdit = Ast.parse('func arg1 arg2') - beforeEdit.module.replaceRoot(beforeEdit) + const beforeEdit = Ast.parseExpression('func arg1 arg2') + beforeEdit.module.setRoot(beforeEdit) const module = reactiveModule(new Y.Doc(), () => {}) module.applyEdit(beforeEdit.module) @@ -52,29 +59,37 @@ test('Module reactivity: Direct Edit', async () => { }) test('Module reactivity: Tracking access to ancestors', async () => { - const docsBeforeEdit = 'The main method' - const beforeEdit = Ast.parseBlock(`## ${docsBeforeEdit}\nmain =\n 23`) - beforeEdit.module.replaceRoot(beforeEdit) + const beforeEdit = Ast.parseModule('main = 23\nother = f') + beforeEdit.module.setRoot(beforeEdit) const module = reactiveModule(new Y.Doc(), () => {}) module.applyEdit(beforeEdit.module) expect(module.root()!.code()).toBe(beforeEdit.code()) const block = module.root() as any as Ast.BodyBlock - const expression = ([...block.statements()][0] as Ast.Documented).expression as Ast.Function - expect(expression.name.code()).toBe('main') - let mainDocs: string | undefined = undefined - watchEffect(() => (mainDocs = expression.documentingAncestor()?.documentation())) - expect(mainDocs).toBe(docsBeforeEdit) + + const [func, otherFunc] = block.statements() as [Ast.Function, Ast.Function] + expect(func.name.code()).toBe('main') + expect(otherFunc.name.code()).toBe('other') + const expression = Array.from(func.bodyExpressions())[0]! + expect(expression.code()).toBe('23') + const otherExpression = Array.from(otherFunc.bodyExpressions())[0]! + expect(otherExpression.code()).toBe('f') + + let parentAccesses = 0 + watchEffect(() => { + expect(expression.parent()).toBeDefined() + parentAccesses += 1 + }) + expect(parentAccesses).toBe(1) const edit = beforeEdit.module.edit() - const editBlock = edit.getVersion(beforeEdit) as any as Ast.MutableBodyBlock - const editDoc = [...editBlock.statements()][0] as Ast.MutableDocumented - const docsAfterEdit = 'The main method, now with more documentation' - editDoc.setDocumentationText(docsAfterEdit) - + const taken = edit.getVersion(expression).replaceValue(Ast.parseExpression('replacement', edit)) + edit.getVersion(otherExpression).updateValue((oe) => Ast.App.positional(oe, taken, edit)) module.applyEdit(edit) - expect(mainDocs).toBe(docsBeforeEdit) + + expect(module.root()?.code()).toBe('main = replacement\nother = f 23') + expect(parentAccesses).toBe(1) await nextTick() - expect(mainDocs).toBe(docsAfterEdit) + expect(parentAccesses).toBe(2) }) diff --git a/app/gui/src/project-view/util/ast/__tests__/sourceDocument.test.ts b/app/gui/src/project-view/util/ast/__tests__/sourceDocument.test.ts index e8130ab95d..570cffd43a 100644 --- a/app/gui/src/project-view/util/ast/__tests__/sourceDocument.test.ts +++ b/app/gui/src/project-view/util/ast/__tests__/sourceDocument.test.ts @@ -12,7 +12,7 @@ test('Test SourceDocument', () => { const code = '1 + 1' const edit1 = syncModule.edit() const root = Ast.parseBlock(code, edit1) - edit1.replaceRoot(root) + edit1.setRoot(root) syncModule.applyEdit(edit1) expect(sourceDoc.text).toBe(code) diff --git a/app/gui/src/project-view/util/ast/__tests__/testCase.ts b/app/gui/src/project-view/util/ast/__tests__/testCase.ts index be5687f2c3..0670abef74 100644 --- a/app/gui/src/project-view/util/ast/__tests__/testCase.ts +++ b/app/gui/src/project-view/util/ast/__tests__/testCase.ts @@ -20,9 +20,9 @@ export function testCase(spec: T): TestCase const statementIndex = new Map() const parsed = Ast.parseBlock(code) - parsed.module.replaceRoot(parsed) + parsed.module.setRoot(parsed) const statements = new Array() - parsed.visitRecursiveAst((ast) => { + parsed.visitRecursive((ast) => { if (ast instanceof Ast.BodyBlock) statements.push(...ast.statements()) }) for (const statement of statements) { @@ -54,7 +54,7 @@ export function tryFindExpressions( ): Partial> { const result: Partial> = {} const expressionsSought = new Set(Object.keys(expressions)) - root.visitRecursiveAst((ast) => { + root.visitRecursive((ast) => { const code = ast.code() const trimmedFirstLine = code.split('\n', 1)[0]!.trim() if (!expressionsSought.has(trimmedFirstLine)) return diff --git a/app/gui/src/project-view/util/ast/abstract.ts b/app/gui/src/project-view/util/ast/abstract.ts index c9c0659a4e..a17d065810 100644 --- a/app/gui/src/project-view/util/ast/abstract.ts +++ b/app/gui/src/project-view/util/ast/abstract.ts @@ -1,62 +1,52 @@ import { normalizeQualifiedName, qnFromSegments } from '@/util/qualifiedName' -import type { - AstId, - IdentifierOrOperatorIdentifier, - Mutable, - MutableAst, - NodeKey, - Owned, - QualifiedName, - TokenId, - TokenKey, -} from 'ydoc-shared/ast' import { Ast, BodyBlock, + Expression, Function, Ident, + IdentifierOrOperatorIdentifier, + Mutable, + MutableAst, MutableBodyBlock, + MutableExpression, + MutableFunction, MutableIdent, MutableModule, MutablePropertyAccess, + MutableStatement, NegationApp, NumericLiteral, OprApp, + Owned, PropertyAccess, + QualifiedName, + Statement, Token, isTokenId, + parseExpression, print, } from 'ydoc-shared/ast' + export * from 'ydoc-shared/ast' -/** TODO: Add docs */ -export function deserialize(serialized: string): Owned { - const parsed: SerializedPrintedSource = JSON.parse(serialized) +/** Given an output of {@link serializeExpression}, returns a deserialized expression. */ +export function deserializeExpression(serialized: string): Owned { // Not implemented: restoring serialized external IDs. This is not the best approach anyway; // Y.Js can't merge edits to objects when they're being serialized and deserialized. - return Ast.parse(parsed.code) + return parseExpression(serialized)! } -interface SerializedInfoMap { - nodes: Record - tokens: Record -} - -interface SerializedPrintedSource { - info: SerializedInfoMap - code: string -} - -/** TODO: Add docs */ -export function serialize(ast: Ast): string { - return JSON.stringify(print(ast)) +/** Returns a serialized representation of the expression. */ +export function serializeExpression(ast: Expression): string { + return print(ast).code } export type TokenTree = (TokenTree | string)[] -/** TODO: Add docs */ +/** Returns a debug representation. */ export function tokenTree(root: Ast): TokenTree { const module = root.module - return Array.from(root.concreteChildren(), (child) => { + return Array.from(root.concreteChildren({ verbatim: false, indent: '' }), (child) => { if (isTokenId(child.node)) { return module.getToken(child.node).code() } else { @@ -71,7 +61,7 @@ export function tokenTreeWithIds(root: Ast): TokenTree { const module = root.module return [ root.externalId, - ...Array.from(root.concreteChildren(), (child) => { + ...Array.from(root.concreteChildren({ verbatim: false, indent: '' }), (child) => { if (isTokenId(child.node)) { return module.getToken(child.node).code() } else { @@ -86,38 +76,45 @@ export function tokenTreeWithIds(root: Ast): TokenTree { export function moduleMethodNames(topLevel: BodyBlock): Set { const result = new Set() for (const statement of topLevel.statements()) { - const inner = statement.innerExpression() - if (inner instanceof Function) { - result.add(inner.name.code()) - } + if (statement instanceof Function) result.add(statement.name.code()) } return result } -// FIXME: We should use alias analysis to handle ambiguous names correctly. -/** TODO: Add docs */ -export function findModuleMethod(topLevel: BodyBlock, name: string): Function | undefined { - for (const statement of topLevel.statements()) { - const inner = statement.innerExpression() - if (inner instanceof Function && inner.name.code() === name) { - return inner - } +export function findModuleMethod( + topLevel: MutableBodyBlock, + name: string, +): { statement: MutableFunction; index: number } | undefined +export function findModuleMethod( + topLevel: BodyBlock, + name: string, +): { statement: Function; index: number } | undefined +/** Find the definition of the function with the specified name in the given block. */ +export function findModuleMethod( + topLevel: BodyBlock, + name: string, +): { statement: Function; index: number } | undefined { + // FIXME: We should use alias analysis to handle shadowing correctly. + const isFunctionWithName = (statement: Statement, name: string) => + statement instanceof Function && statement.name.code() === name + const index = topLevel.lines.findIndex( + (line) => line.statement && isFunctionWithName(line.statement.node, name), + ) + if (index === -1) return undefined + const statement = topLevel.lines[index]!.statement!.node as Function + return { + /** The `Function` definition. */ + statement, + /** The index into the block's `lines` where the definition was found. */ + index, } - return undefined } -/** TODO: Add docs */ -export function functionBlock(topLevel: BodyBlock, name: string) { - const func = findModuleMethod(topLevel, name) - if (!(func?.body instanceof BodyBlock)) return undefined - return func.body -} - -/** TODO: Add docs */ -export function deleteFromParentBlock(ast: MutableAst) { +/** Delete the specified statement from its containing block. */ +export function deleteFromParentBlock(ast: MutableStatement) { const parent = ast.mutableParent() if (parent instanceof MutableBodyBlock) - parent.updateLines((lines) => lines.filter((line) => line.expression?.node.id !== ast.id)) + parent.updateLines((lines) => lines.filter((line) => line.statement?.node.id !== ast.id)) } /** @@ -219,10 +216,10 @@ export function substituteQualifiedName( if (expr instanceof MutablePropertyAccess || expr instanceof MutableIdent) { const qn = parseQualifiedName(expr) if (qn === pattern) { - expr.updateValue(() => Ast.parse(to, expr.module)) + expr.updateValue(() => parseExpression(to, expr.module)!) } else if (qn && qn.startsWith(pattern)) { const withoutPattern = qn.replace(pattern, '') - expr.updateValue(() => Ast.parse(to + withoutPattern, expr.module)) + expr.updateValue(() => parseExpression(to + withoutPattern, expr.module)!) } } else { for (const child of expr.children()) { @@ -269,6 +266,11 @@ export function copyIntoNewModule(ast: T): Owned> { return module.getVersion(ast) as Owned> } +/** Safely cast a mutable or owned value to its base type. */ +export function dropMutability(value: Owned>): T { + return value as unknown as T +} + declare const tokenKey: unique symbol declare module '@/providers/widgetRegistry' { export interface WidgetInputTypes { diff --git a/app/gui/src/project-view/util/ast/aliasAnalysis.ts b/app/gui/src/project-view/util/ast/aliasAnalysis.ts index 222a46c7bb..0ab8e0bc8b 100644 --- a/app/gui/src/project-view/util/ast/aliasAnalysis.ts +++ b/app/gui/src/project-view/util/ast/aliasAnalysis.ts @@ -1,14 +1,13 @@ import { assert } from '@/util/assert' import { RawAst, - astPrettyPrintType, parsedTreeOrTokenRange, rawParseModule, readAstOrTokenSpan, readTokenSpan, } from '@/util/ast/raw' import { MappedKeyMap, MappedSet, NonEmptyStack } from '@/util/containers' -import type { LazyObject } from 'ydoc-shared/ast/parserSupport' +import { LazyObject } from 'ydoc-shared/ast/parserSupport' import { rangeIsBefore, sourceRangeKey, type SourceRange } from 'ydoc-shared/yjsModel' const ACCESSOR_OPERATOR = '.' @@ -323,7 +322,7 @@ export class AliasAnalyzer { } } break - case RawAst.Tree.Type.Documented: + case RawAst.Tree.Type.ExpressionStatement: // Intentionally omit documentation, as it is not a "real" code. this.processTree(node.expression) break @@ -348,3 +347,10 @@ function log(...messages: Array<() => any>) { console.log(...messages.map((message) => message())) } } + +function astPrettyPrintType(obj: unknown): string | undefined { + if (obj instanceof LazyObject && Object.hasOwnProperty.call(obj, 'type')) { + const proto = Object.getPrototypeOf(obj) + return proto?.constructor?.name + } +} diff --git a/app/gui/src/project-view/util/ast/match.ts b/app/gui/src/project-view/util/ast/match.ts index 09693007ae..0f4b9222ec 100644 --- a/app/gui/src/project-view/util/ast/match.ts +++ b/app/gui/src/project-view/util/ast/match.ts @@ -2,14 +2,21 @@ import { assert, assertDefined } from '@/util/assert' import { Ast } from '@/util/ast' import { zipLongest } from '@/util/data/iterable' -/** TODO: Add docs */ -export class Pattern { - private readonly template: Ast.Ast +/** + * A pattern is an AST object with "placeholder" expressions. + * + * It can be used in two ways: + * - It can be matched against an AST node, in which case each placeholder will match any expression, and the matches + * will be returned. + * - It can be instantiated, by providing an expression to be substituted for each placeholder. + */ +export class Pattern { + private readonly template: T private readonly placeholders: Ast.AstId[] private readonly placeholder: string - private constructor(template: Ast.Owned, placeholder: string) { - this.template = template + private constructor(template: Ast.Owned>, placeholder: string) { + this.template = Ast.dropMutability(template) this.placeholders = findPlaceholders(template, placeholder) this.placeholder = placeholder } @@ -18,13 +25,20 @@ export class Pattern { * Parse an expression template in which a specified identifier (by default `__`) * may match any arbitrary subtree. */ - static parse(template: string, placeholder: string = '__'): Pattern { - const ast = Ast.parse(template) + static parseExpression(template: string, placeholder: string = '__'): Pattern { + const ast = Ast.parseExpression(template) + assertDefined(ast) return new Pattern(ast, placeholder) } - /** TODO: Add docs */ - static new(f: (placeholder: Ast.Owned) => Ast.Owned, placeholder: string = '__'): Pattern { + /** + * Given a function that constructs an AST value when provided an expression, creates a `Pattern` that constructs an + * equivalent AST value when instantiated with an expression. + */ + static new( + f: (placeholder: Ast.Owned) => Ast.Owned>, + placeholder: string = '__', + ): Pattern { assert(Ast.isIdentifier(placeholder)) const module = Ast.MutableModule.Transient() return new Pattern(f(Ast.Ident.new(module, placeholder)), placeholder) @@ -48,7 +62,10 @@ export class Pattern { } /** Create a new concrete example of the pattern, with the placeholders replaced with the given subtrees. */ - instantiate(edit: Ast.MutableModule, subtrees: Ast.Owned[]): Ast.Owned { + instantiate( + edit: Ast.MutableModule, + subtrees: Ast.Owned[], + ): Ast.Owned> { const template = edit.copy(this.template) const placeholders = findPlaceholders(template, this.placeholder).map((ast) => edit.tryGet(ast)) for (const [placeholder, replacement] of zipLongest(placeholders, subtrees)) { @@ -59,20 +76,20 @@ export class Pattern { return template } - /** TODO: Add docs */ - instantiateCopied(subtrees: Ast.Ast[], edit?: Ast.MutableModule): Ast.Owned { + /** + * Helper that creates the AST described by the pattern, as {@link instantiate}, after first copying each of the + * referenced subtrees into a different module. + */ + instantiateCopied( + subtrees: (Ast.Expression | Ast.MutableExpression)[], + edit?: Ast.MutableModule, + ): Ast.Owned> { const module = edit ?? Ast.MutableModule.Transient() return this.instantiate( module, subtrees.map((ast) => module.copy(ast)), ) } - - /** TODO: Add docs */ - compose(f: (pattern: Ast.Owned) => Ast.Owned): Pattern { - const module = Ast.MutableModule.Transient() - return new Pattern(f(module.copy(this.template)), this.placeholder) - } } function findPlaceholders(ast: Ast.Ast, placeholder: string): Ast.AstId[] { diff --git a/app/gui/src/project-view/util/ast/node.ts b/app/gui/src/project-view/util/ast/node.ts index 0de13d9854..a436542196 100644 --- a/app/gui/src/project-view/util/ast/node.ts +++ b/app/gui/src/project-view/util/ast/node.ts @@ -8,57 +8,52 @@ export const prefixes = Prefixes.FromLines({ }) /** Given a node's outer expression, find the root expression and any statements wrapping it. */ -export function nodeRootExpr(ast: Ast.Ast): { - root: Ast.Ast | undefined - docs: Ast.Documented | undefined +export function nodeRootExpr(ast: Ast.Statement | Ast.Expression): { + root: Ast.Expression | undefined assignment: Ast.Assignment | undefined } { - const [withinDocs, docs] = - ast instanceof Ast.Documented ? [ast.expression, ast] : [ast, undefined] - const [withinAssignment, assignment] = - withinDocs instanceof Ast.Assignment ? - [withinDocs.expression, withinDocs] - : [withinDocs, undefined] + const assignment = ast instanceof Ast.Assignment ? ast : undefined + const root = + assignment ? assignment.expression + : ast instanceof Ast.ExpressionStatement ? ast.expression + : undefined return { - root: withinAssignment, - docs, + root, assignment, } } -/** TODO: Add docs */ -export function inputNodeFromAst(ast: Ast.Ast, argIndex: number): NodeDataFromAst { +/** Create a Node from the pattern of a function argument. */ +export function inputNodeFromAst(ast: Ast.Expression, argIndex: number): NodeDataFromAst { return { type: 'input', - outerExpr: ast, + outerAst: ast, pattern: undefined, rootExpr: ast, innerExpr: ast, prefixes: { enableRecording: undefined }, primarySubject: undefined, conditionalPorts: new Set(), - docs: undefined, argIndex, } } /** Given a node's outer expression, return all the `Node` fields that depend on its AST structure. */ -export function nodeFromAst(ast: Ast.Ast, isOutput: boolean): NodeDataFromAst | undefined { - const { root, docs, assignment } = nodeRootExpr(ast) +export function nodeFromAst(ast: Ast.Statement, isOutput: boolean): NodeDataFromAst | undefined { + const { root, assignment } = nodeRootExpr(ast) if (!root) return const { innerExpr, matches } = prefixes.extractMatches(root) const type = assignment == null && isOutput ? 'output' : 'component' const primaryApplication = primaryApplicationSubject(innerExpr) return { type, - outerExpr: ast, + outerAst: ast, pattern: assignment?.pattern, rootExpr: root, innerExpr, prefixes: matches, primarySubject: primaryApplication?.subject, conditionalPorts: new Set(primaryApplication?.accessChain ?? []), - docs, argIndex: undefined, } } @@ -68,7 +63,7 @@ export function nodeFromAst(ast: Ast.Ast, isOutput: boolean): NodeDataFromAst | * application. */ export function primaryApplicationSubject( - ast: Ast.Ast, + ast: Ast.Expression, ): { subject: Ast.AstId; accessChain: Ast.AstId[] } | undefined { // Descend into LHS of any sequence of applications. while (ast instanceof Ast.App) ast = ast.function diff --git a/app/gui/src/project-view/util/ast/opr.ts b/app/gui/src/project-view/util/ast/opr.ts deleted file mode 100644 index ee30573e71..0000000000 --- a/app/gui/src/project-view/util/ast/opr.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { assert } from '@/util/assert' -import { RawAstExtended } from '@/util/ast/extended' -import { RawAst } from '@/util/ast/raw' -import { zip } from '@/util/data/iterable' -import { mapIterator } from 'lib0/iterator' - -/** An operand of one of the applications inside `GeneralOprApp` */ -export type GeneralOperand = - | Operand - // A part of `GeneralOprApp`, consisting of lhs and first `statements` of applications. - | { type: 'partOfGeneralOprApp'; oprApp: GeneralOprApp; statements: number } - -export type OperatorChain = RawAstExtended< - RawAst.Tree.OprApp | RawAst.Tree.OperatorBlockApplication, - HasIdMap -> - -/** A structure unifying API of OprApp and OperatorBlockApplication */ -export class GeneralOprApp { - lhs: RawAstExtended | null - apps: { - opr: RawAstExtended | null - expr: RawAstExtended | null - }[] - - /** TODO: Add docs */ - constructor(ast: OperatorChain) { - this.lhs = ast.tryMap((t) => t.lhs) ?? null - if (ast.isTree(RawAst.Tree.Type.OprApp)) { - const rhs = ast.tryMap((t) => t.rhs) ?? null - const opr = ast.tryMap((t) => (t.opr.ok ? t.opr.value : undefined)) ?? null - this.apps = [{ opr, expr: rhs }] - } else { - const blockApplication = ast as RawAstExtended - const expressions = (line: RawAst.OperatorLine): RawAst.OperatorBlockExpression[] => - line.expression ? [line.expression] : [] - const operators = blockApplication.tryMapIter((ast) => - [...ast.expressions] - .flatMap(expressions) - .map((expr) => (expr.operator.ok ? expr.operator.value : null)) - .values(), - ) - const exprs = blockApplication.mapIter((ast) => - [...ast.expressions] - .flatMap(expressions) - .map((expr) => expr.expression) - .values(), - ) - this.apps = Array.from( - mapIterator(zip(operators, exprs), ([opr, expr]) => ({ - opr: opr ? opr : null, - expr: expr ? expr : null, - })), - ) - } - } - - /** Last operator */ - lastOpr(): RawAstExtended | null { - return this.apps[this.apps.length - 1]?.opr ?? null - } - - /** Returns representation of all operands interleaved with appropriate operators */ - *componentsReprs(): Generator { - yield this.lhs != null ? this.lhs.repr() : null - for (const app of this.apps) { - yield app.opr != null ? app.opr.repr() : null - yield app.expr != null ? app.expr.repr() : null - } - } - - /** - * Read operands of an operator chain. Operator is assumed to be left-associative. - * - * Works like `operandsOfLeftAssocOprChain` defined in this module, see its docs for details. - */ - *operandsOfLeftAssocOprChain(expectedOpr?: string): Generator | null> { - // If this represents an OperatorBlockApplication, there may be many different operators. Our chain - // ends with the first not matching starting from the end. - let matchingOprs - for (matchingOprs = 0; matchingOprs < this.apps.length; matchingOprs++) { - const app = this.apps[this.apps.length - matchingOprs - 1]! - if (!app.opr) break - const oprCode = app.opr.repr() - if (expectedOpr != null && oprCode != expectedOpr) break - expectedOpr = oprCode - } - if (matchingOprs === this.apps.length) { - // If all operators matched, the lhs may be a continuation of this chain. - if (this.lhs != null) yield* operandsOfLeftAssocOprChain(this.lhs, expectedOpr) - else yield null - } else { - // Not all operators matched; the first operand will be a part of this GeneralOprApp. - yield { - type: 'partOfGeneralOprApp', - oprApp: this, - statements: this.apps.length - matchingOprs, - } - } - for (let i = this.apps.length - matchingOprs; i < this.apps.length; ++i) { - const app = this.apps[i] - if (app?.expr != null) yield { type: 'ast', ast: app.expr } - else yield null - } - } -} - -/** - * An operand of some operator application chain. - * - * There is a special case, where operand is a part of OperatorBlockApplication which is not - * representable by any AST structure. - */ -export type Operand = - | { type: 'ast'; ast: RawAstExtended } - | { - type: 'partOfOprBlockApp' - ast: RawAstExtended - statements: number - } - -/** - * Read operands of an operator chain. Operator is assumed to be left-associative. - * - * It flattens applications of same operator, e.g. for `2 + 3 + 4` will return `2`, `3`, and `4`, - * but `2 - 3 + 4` will return `2 - 3` as first operand, and then `4`. If the ast is not - * an operator application (of this specific operator if provided), `this` will be returned as - * a single operand. - * @param ast the subtree which we assume is an operator application chain. - * @param expectedOpr if specified, the chain will be of specific operator. - */ -export function* operandsOfLeftAssocOprChain( - ast: RawAstExtended, - expectedOpr?: string, -): Generator | null> { - switch (ast.inner.type) { - case RawAst.Tree.Type.OprApp: - case RawAst.Tree.Type.OperatorBlockApplication: { - const oprApp = new GeneralOprApp(ast as OperatorChain) - for (const operand of oprApp.operandsOfLeftAssocOprChain(expectedOpr)) { - if (operand == null || operand.type !== 'partOfGeneralOprApp') yield operand - else { - const isEntireOprApp = operand.statements === oprApp.apps.length - if (isEntireOprApp) { - yield { type: 'ast', ast } - } else { - assert(ast.isTree(RawAst.Tree.Type.OperatorBlockApplication)) - yield { type: 'partOfOprBlockApp', ast, statements: operand.statements } - } - } - } - break - } - default: - yield { type: 'ast', ast } - } -} diff --git a/app/gui/src/project-view/util/ast/prefixes.ts b/app/gui/src/project-view/util/ast/prefixes.ts index 08a9cd1530..3314c93c82 100644 --- a/app/gui/src/project-view/util/ast/prefixes.ts +++ b/app/gui/src/project-view/util/ast/prefixes.ts @@ -5,7 +5,7 @@ import { unsafeKeys } from '@/util/record' type Matches = Record interface MatchResult { - innerExpr: Ast.Ast + innerExpr: Ast.Expression matches: Record } @@ -21,18 +21,18 @@ export class Prefixes> { static FromLines(lines: Record) { return new Prefixes( Object.fromEntries( - Object.entries(lines).map(([name, line]) => [name, Pattern.parse(line)]), + Object.entries(lines).map(([name, line]) => [name, Pattern.parseExpression(line)]), ) as Record, ) } /** TODO: Add docs */ - extractMatches(expression: Ast.Ast): MatchResult { + extractMatches(expression: Ast.Expression): MatchResult { const matches = Object.fromEntries( Object.entries(this.prefixes).map(([name, pattern]) => { const matches = pattern.match(expression) const lastMatch = matches != null ? matches[matches.length - 1] : undefined - if (lastMatch) expression = expression.module.get(lastMatch) + if (lastMatch) expression = expression.module.get(lastMatch) as Ast.Expression return [name, matches] }), ) as Matches @@ -40,15 +40,19 @@ export class Prefixes> { } /** TODO: Add docs */ - modify(expression: Ast.Mutable, replacements: Partial>) { + modify( + expression: Ast.MutableExpression, + replacements: Partial[] | undefined>>, + ) { expression.updateValue((expression) => { - const matches = this.extractMatches(expression) + const matches = this.extractMatches(expression as Ast.Owned) const edit = expression.module - let result = edit.take(matches.innerExpr.id) + let result = edit.getVersion(matches.innerExpr).take() for (const key of unsafeKeys(this.prefixes).reverse()) { if (key in replacements && !replacements[key]) continue - const replacement: Ast.Owned[] | undefined = - replacements[key] ?? matches.matches[key]?.map((match) => edit.take(match)!) + const replacement: Ast.Owned[] | undefined = + replacements[key] ?? + matches.matches[key]?.map((match) => edit.take(match) as Ast.Owned) if (!replacement) continue const pattern = this.prefixes[key] const parts = [...replacement, result] diff --git a/app/gui/src/project-view/util/ast/raw.ts b/app/gui/src/project-view/util/ast/raw.ts index 313640dd61..f6197bbbde 100644 --- a/app/gui/src/project-view/util/ast/raw.ts +++ b/app/gui/src/project-view/util/ast/raw.ts @@ -1,28 +1,12 @@ -import { assertDefined } from '@/util/assert' -import * as map from 'lib0/map' import * as RawAst from 'ydoc-shared/ast/generated/ast' import { rawParseModule } from 'ydoc-shared/ast/parse' -import { LazyObject, LazySequence } from 'ydoc-shared/ast/parserSupport' -import { tryGetSoleValue } from 'ydoc-shared/util/data/iterable' -import { isResult, mapOk } from 'ydoc-shared/util/data/result' +import { LazyObject } from 'ydoc-shared/ast/parserSupport' import type { SourceRange } from 'ydoc-shared/yjsModel' export { RawAst, rawParseModule } export type HasAstRange = SourceRange | RawAst.Tree | RawAst.Token -/** - * Read a single line of code - * - * Is meant to be a helper for tests. If the code is multiline, an exception is raised. - */ -export function rawParseLine(code: string): RawAst.Tree { - const block = rawParseModule(code) - const soleExpression = tryGetSoleValue(block.statements)?.expression - assertDefined(soleExpression) - return soleExpression -} - /** * Read span of code represented by given AST node, not including left whitespace offset. * @@ -34,16 +18,6 @@ export function readAstOrTokenSpan(node: RawAst.Tree | RawAst.Token, code: strin return code.substring(range[0], range[1]) } -/** - * Read span of code represented by given RawAst.Tree. - * - * The Tree is assumed to be a part of AST generated from `code`. - */ -export function readAstSpan(node: RawAst.Tree, code: string): string { - const range = parsedTreeRange(node) - return code.substring(range[0], range[1]) -} - /** * Read span of code represented by given RawAst.Token. * @@ -64,6 +38,7 @@ export function childrenAstNodes(obj: LazyObject): RawAst.Tree[] { obj.visitChildren(visitor) return children } + /** TODO: Add docs */ export function childrenAstNodesOrTokens(obj: LazyObject): (RawAst.Tree | RawAst.Token)[] { const children: (RawAst.Tree | RawAst.Token)[] = [] @@ -78,46 +53,6 @@ export function childrenAstNodesOrTokens(obj: LazyObject): (RawAst.Tree | RawAst return children } -/** Returns all AST nodes from `root` tree containing given char, starting from the most nested. */ -export function astContainingChar(charIndex: number, root: RawAst.Tree): RawAst.Tree[] { - return treePath(root, (node) => { - const begin = node.whitespaceStartInCodeParsed + node.whitespaceLengthInCodeParsed - const end = begin + node.childrenLengthInCodeParsed - return charIndex >= begin && charIndex < end - }).reverse() -} - -/** - * Given a predicate, return a path from the root down the tree containing the - * first node at each level found to satisfy the predicate. - */ -function treePath(obj: LazyObject, pred: (node: RawAst.Tree) => boolean): RawAst.Tree[] { - const path: RawAst.Tree[] = [] - const visitor = (obj: LazyObject) => { - if (RawAst.Tree.isInstance(obj)) { - const isMatch = pred(obj) - if (isMatch) path.push(obj) - return obj.visitChildren(visitor) || isMatch - } else { - return obj.visitChildren(visitor) - } - } - obj.visitChildren(visitor) - return path -} - -/** TODO: Add docs */ -export function findAstWithRange( - root: RawAst.Tree | RawAst.Token, - range: SourceRange, -): RawAst.Tree | RawAst.Token | undefined { - for (const child of childrenAstNodes(root)) { - const [begin, end] = parsedTreeOrTokenRange(child) - if (begin === range[0] && end === range[1]) return child - if (begin <= range[0] && end >= range[1]) return findAstWithRange(child, range) - } -} - /** TODO: Add docs */ export function* walkRecursive( node: RawAst.Tree | RawAst.Token, @@ -158,14 +93,14 @@ export function visitRecursive( * @returns Object with `start` and `end` properties; index of first character in the `node` * and first character _not_ being in the `node`. */ -export function parsedTreeRange(tree: RawAst.Tree): SourceRange { +function parsedTreeRange(tree: RawAst.Tree): SourceRange { const start = tree.whitespaceStartInCodeParsed + tree.whitespaceLengthInCodeParsed const end = start + tree.childrenLengthInCodeParsed return [start, end] } /** TODO: Add docs */ -export function parsedTokenRange(token: RawAst.Token): SourceRange { +function parsedTokenRange(token: RawAst.Token): SourceRange { const start = token.startInCodeBuffer const end = start + token.lengthInCodeBuffer return [start, end] @@ -177,65 +112,3 @@ export function parsedTreeOrTokenRange(node: HasAstRange): SourceRange { else if (RawAst.Token.isInstance(node)) return parsedTokenRange(node) else return node } - -/** TODO: Add docs */ -export function astPrettyPrintType(obj: unknown): string | undefined { - if (obj instanceof LazyObject && Object.hasOwnProperty.call(obj, 'type')) { - const proto = Object.getPrototypeOf(obj) - return proto?.constructor?.name - } -} - -/** TODO: Add docs */ -export function debugAst(obj: unknown): unknown { - if (obj instanceof LazyObject) { - const fields = Object.fromEntries( - allGetterNames(obj).map((k) => [k, debugAst((obj as any)[k])]), - ) - if (Object.hasOwnProperty.call(obj, 'type')) { - const className = astPrettyPrintType(obj) - return { type: className, ...fields } - } else { - return fields - } - } else if (obj instanceof LazySequence) { - return Array.from(obj, debugAst) - } else if (isResult(obj)) { - return mapOk(obj, debugAst) - } else { - return obj - } -} - -const protoGetters = new Map() -function allGetterNames(obj: object): string[] { - let proto = Object.getPrototypeOf(obj) - return map.setIfUndefined(protoGetters, proto, () => { - const props = new Map() - do { - for (const [name, prop] of Object.entries(Object.getOwnPropertyDescriptors(proto))) { - if (!props.has(name)) props.set(name, prop) - } - } while ((proto = Object.getPrototypeOf(proto))) - const getters = new Set() - for (const [name, prop] of props.entries()) { - if (prop.get != null && prop.configurable && !debugHideFields.includes(name)) { - getters.add(name) - } - } - return [...getters] - }) -} - -const debugHideFields = [ - '_v', - '__proto__', - 'codeReprBegin', - 'codeReprLen', - 'leftOffsetCodeReprBegin', - 'leftOffsetCodeReprLen', - 'leftOffsetVisible', - 'spanLeftOffsetCodeReprBegin', - 'spanLeftOffsetCodeReprLen', - 'spanLeftOffsetVisible', -] diff --git a/app/gui/src/project-view/util/callTree.ts b/app/gui/src/project-view/util/callTree.ts index aabe01d85d..5c34e4cf58 100644 --- a/app/gui/src/project-view/util/callTree.ts +++ b/app/gui/src/project-view/util/callTree.ts @@ -34,7 +34,11 @@ class ArgumentFactory { ) } - argument(ast: Ast.Ast, index: number | undefined, info: SuggestionEntryArgument | undefined) { + argument( + ast: Ast.Expression, + index: number | undefined, + info: SuggestionEntryArgument | undefined, + ) { return new ArgumentAst( this.callId, this.kind, @@ -51,7 +55,7 @@ class ArgumentFactory { } type ArgWidgetConfiguration = WidgetConfiguration & { display?: DisplayMode } -type WidgetInputValue = Ast.Ast | Ast.Token | string | undefined +type WidgetInputValue = Ast.Expression | Ast.Token | string | undefined abstract class Argument { protected constructor( public callId: string, @@ -127,7 +131,7 @@ export class ArgumentAst extends Argument { dynamicConfig: ArgWidgetConfiguration | undefined, index: number | undefined, argInfo: SuggestionEntryArgument | undefined, - public ast: Ast.Ast, + public ast: Ast.Expression, ) { super(callId, kind, dynamicConfig, index, argInfo) } @@ -149,24 +153,24 @@ interface InterpretedInfix { kind: 'infix' appTree: Ast.OprApp operator: Ast.Token | undefined - lhs: Ast.Ast | undefined - rhs: Ast.Ast | undefined + lhs: Ast.Expression | undefined + rhs: Ast.Expression | undefined } interface InterpretedPrefix { kind: 'prefix' - func: Ast.Ast + func: Ast.Expression args: FoundApplication[] } interface FoundApplication { appTree: Ast.App - argument: Ast.Ast + argument: Ast.Expression argName: string | undefined } /** TODO: Add docs */ -export function interpretCall(callRoot: Ast.Ast): InterpretedCall { +export function interpretCall(callRoot: Ast.Expression): InterpretedCall { if (callRoot instanceof Ast.OprApp) { // Infix chains are handled one level at a time. Each application may have at most 2 arguments. return { @@ -209,8 +213,8 @@ interface CallInfo { /** TODO: Add docs */ export class ArgumentApplication { private constructor( - public appTree: Ast.Ast, - public target: ArgumentApplication | Ast.Ast | ArgumentPlaceholder | ArgumentAst, + public appTree: Ast.Expression, + public target: ArgumentApplication | Ast.Expression | ArgumentPlaceholder | ArgumentAst, public infixOperator: Ast.Token | undefined, public argument: ArgumentAst | ArgumentPlaceholder, public calledFunction: SuggestionEntry | undefined, @@ -260,7 +264,7 @@ export class ArgumentApplication { ) const resolvedArgs: Array<{ - appTree: Ast.Ast + appTree: Ast.Expression argument: ArgumentAst | ArgumentPlaceholder }> = [] @@ -378,7 +382,7 @@ export class ArgumentApplication { } return resolvedArgs.reduce( - (target: ArgumentApplication | Ast.Ast, toDisplay) => + (target: ArgumentApplication | Ast.Expression, toDisplay) => new ArgumentApplication( toDisplay.appTree, target, @@ -395,7 +399,7 @@ export class ArgumentApplication { static FromInterpretedWithInfo( interpreted: InterpretedCall, callInfo: CallInfo = {}, - ): ArgumentApplication | Ast.Ast { + ): ArgumentApplication | Ast.Expression { if (interpreted.kind === 'infix') { return ArgumentApplication.FromInterpretedInfix(interpreted, callInfo) } else { @@ -487,7 +491,7 @@ const unknownArgInfoNamed = (name: string) => ({ }) /** TODO: Add docs */ -export function getAccessOprSubject(app: Ast.Ast): Ast.Ast | undefined { +export function getAccessOprSubject(app: Ast.Expression): Ast.Expression | undefined { if (app instanceof Ast.PropertyAccess) return app.lhs } @@ -500,7 +504,7 @@ export function getAccessOprSubject(app: Ast.Ast): Ast.Ast | undefined { * We also don’t consider infix applications here, as using them inside a prefix chain would require additional syntax (like parenthesis). */ export function getMethodCallInfoRecursively( - ast: Ast.Ast, + ast: Ast.Expression, graphDb: { getMethodCallInfo(id: AstId): MethodCallInfo | undefined }, ): MethodCallInfo | undefined { let appliedArgs = 0 diff --git a/app/ydoc-server/src/edits.ts b/app/ydoc-server/src/edits.ts index 2e72a1b79b..a557a7552e 100644 --- a/app/ydoc-server/src/edits.ts +++ b/app/ydoc-server/src/edits.ts @@ -77,7 +77,7 @@ export function applyDocumentUpdates( // Update the metadata object. // Depth-first key order keeps diffs small. newMetadata = { node: {}, widget: {} } - root.visitRecursiveAst(ast => { + root.visitRecursive(ast => { let pos = ast.nodeMetadata.get('position') const vis = ast.nodeMetadata.get('visualization') const colorOverride = ast.nodeMetadata.get('colorOverride') diff --git a/app/ydoc-server/src/languageServerSession.ts b/app/ydoc-server/src/languageServerSession.ts index 98d7923d3b..82b28373a5 100644 --- a/app/ydoc-server/src/languageServerSession.ts +++ b/app/ydoc-server/src/languageServerSession.ts @@ -606,7 +606,7 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> { // the code was externally edited. In this case we try to fix the spans by running // the `syncToCode` on the saved code snapshot. const { root, spans } = Ast.parseModuleWithSpans(snapshotCode, syncModule) - syncModule.syncRoot(root) + syncModule.setRoot(root) parsedIdMap = deserializeIdMap(idMapJson) const edit = syncModule.edit() @@ -615,7 +615,7 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> { syncModule.applyEdit(edit) } else { const { root, spans } = Ast.parseModuleWithSpans(code, syncModule) - syncModule.syncRoot(root) + syncModule.setRoot(root) parsedSpans = spans } } @@ -651,7 +651,7 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> { (nodeMeta.length !== 0 || widgetMeta.length !== 0) ) { const externalIdToAst = new Map() - astRoot.visitRecursiveAst(ast => { + astRoot.visitRecursive(ast => { if (!externalIdToAst.has(ast.externalId)) externalIdToAst.set(ast.externalId, ast) }) const missing = new Set() diff --git a/app/ydoc-shared/src/ast/debug.ts b/app/ydoc-shared/src/ast/debug.ts index 0dfae575f6..1daa78f348 100644 --- a/app/ydoc-shared/src/ast/debug.ts +++ b/app/ydoc-shared/src/ast/debug.ts @@ -1,11 +1,11 @@ import { Ast } from './tree' /** Returns a GraphViz graph illustrating parent/child relationships in the given subtree. */ -export function graphParentPointers(ast: Ast) { +export function graphParentPointers(ast: Ast, bidirectional?: true): string { const sanitize = (id: string) => id.replace('ast:', '').replace(/[^A-Za-z0-9]/g, '') const parentToChild = new Array<{ parent: string; child: string }>() const childToParent = new Array<{ child: string; parent: string }>() - ast.visitRecursiveAst(ast => { + ast.visitRecursive(ast => { for (const child of ast.children()) { if (child instanceof Ast) parentToChild.push({ child: sanitize(child.id), parent: sanitize(ast.id) }) @@ -15,8 +15,10 @@ export function graphParentPointers(ast: Ast) { }) let result = 'digraph parentPointers {\n' for (const { parent, child } of parentToChild) result += `${parent} -> ${child};\n` - for (const { child, parent } of childToParent) - result += `${child} -> ${parent} [weight=0; color=red; style=dotted];\n` + if (bidirectional) { + for (const { child, parent } of childToParent) + result += `${child} -> ${parent} [weight=0; color=red; style=dotted];\n` + } result += '}\n' return result } diff --git a/app/ydoc-shared/src/ast/index.ts b/app/ydoc-shared/src/ast/index.ts index dbd9e65372..c6410b40c7 100644 --- a/app/ydoc-shared/src/ast/index.ts +++ b/app/ydoc-shared/src/ast/index.ts @@ -3,8 +3,8 @@ import { reachable } from '../util/data/graph' import type { ExternalId } from '../yjsModel' import type { Module } from './mutableModule' import type { SyncTokenId } from './token' -import type { AstId } from './tree' -import { App, Ast, Group, MutableAst, OprApp, Wildcard } from './tree' +import type { AstId, MutableAst } from './tree' +import { App, Ast, Group, OprApp, Wildcard } from './tree' export * from './mutableModule' export * from './parse' diff --git a/app/ydoc-shared/src/ast/mutableModule.ts b/app/ydoc-shared/src/ast/mutableModule.ts index 6271b05453..9dd23e09a2 100644 --- a/app/ydoc-shared/src/ast/mutableModule.ts +++ b/app/ydoc-shared/src/ast/mutableModule.ts @@ -1,7 +1,19 @@ import * as random from 'lib0/random' import * as Y from 'yjs' -import type { AstId, NodeChild, Owned, RawNodeChild, SyncTokenId } from '.' -import { Token, asOwned, isTokenId, newExternalId, subtreeRoots } from '.' +import { + AstId, + MutableBodyBlock, + NodeChild, + Owned, + RawNodeChild, + SyncTokenId, + Token, + asOwned, + isTokenId, + newExternalId, + parseModule, + subtreeRoots, +} from '.' import { assert, assertDefined } from '../util/assert' import type { SourceRangeEdit } from '../util/data/text' import { defaultLocalOrigin, tryAsOrigin, type ExternalId, type Origin } from '../yjsModel' @@ -85,46 +97,36 @@ export class MutableModule implements Module { return this.ydoc.transact(f, origin) } - /** TODO: Add docs */ + /** Return the top-level block of the module. */ root(): MutableAst | undefined { return this.rootPointer()?.expression } - /** TODO: Add docs */ - replaceRoot(newRoot: Owned | undefined): Owned | undefined { + /** Set the given block to be the top-level block of the module. */ + setRoot(newRoot: Owned | undefined) { if (newRoot) { const rootPointer = this.rootPointer() if (rootPointer) { - return rootPointer.expression.replace(newRoot) + rootPointer.expression.replace(newRoot) } else { invalidFields(this, this.baseObject('Invalid', undefined, ROOT_ID), { whitespace: '', node: newRoot, }) - return undefined } } else { const oldRoot = this.root() - if (!oldRoot) return - this.nodes.delete(ROOT_ID) - oldRoot.fields.set('parent', undefined) - return asOwned(oldRoot) + if (oldRoot) oldRoot.fields.set('parent', undefined) } } - /** TODO: Add docs */ - syncRoot(root: Owned) { - this.replaceRoot(root) - this.gc() - } - /** TODO: Add docs */ syncToCode(code: string) { const root = this.root() if (root) { root.syncToCode(code) } else { - this.replaceRoot(Ast.parse(code, this)) + this.setRoot(parseModule(code, this)) } } @@ -169,7 +171,7 @@ export class MutableModule implements Module { /** @internal */ importCopy(ast: T): Owned> { assert(ast.module !== this) - ast.visitRecursiveAst(ast => this.nodes.set(ast.id, ast.fields.clone() as any)) + ast.visitRecursive(ast => this.nodes.set(ast.id, ast.fields.clone() as any)) const fields = this.nodes.get(ast.id) assertDefined(fields) fields.set('parent', undefined) @@ -333,11 +335,6 @@ export class MutableModule implements Module { return this.replace(id, Wildcard.new(this)) || asOwned(this.get(id)) } - /** TODO: Add docs */ - updateValue(id: AstId, f: (x: Owned) => Owned): T | undefined { - return this.tryGet(id)?.updateValue(f) - } - ///////////////////////////////////////////// /** TODO: Add docs */ diff --git a/app/ydoc-shared/src/ast/parse.ts b/app/ydoc-shared/src/ast/parse.ts index 94901d28bf..ac7c135e62 100644 --- a/app/ydoc-shared/src/ast/parse.ts +++ b/app/ydoc-shared/src/ast/parse.ts @@ -1,15 +1,14 @@ import * as map from 'lib0/map' -import type { +import { AstId, FunctionFields, Module, + MutableInvalid, NodeChild, Owned, OwnedRefs, TextElement, TextToken, -} from '.' -import { Token, asOwned, isTokenId, @@ -39,7 +38,6 @@ import { type SourceRange, type SourceRangeKey, } from '../yjsModel' -import { graphParentPointers } from './debug' import { parse_block, parse_module, xxHash128 } from './ffi' import * as RawAst from './generated/ast' import { MutableModule } from './mutableModule' @@ -50,7 +48,7 @@ import { Ast, AutoscopedIdentifier, BodyBlock, - Documented, + ExpressionStatement, Function, Generic, Group, @@ -60,7 +58,10 @@ import { MutableAssignment, MutableAst, MutableBodyBlock, + MutableExpression, + MutableExpressionStatement, MutableIdent, + MutableStatement, NegationApp, NumericLiteral, OprApp, @@ -95,7 +96,7 @@ export function normalize(rootIn: Ast): Ast { const module = MutableModule.Transient() const tree = rawParseModule(printed.code) const { root: parsed, spans } = abstract(module, tree, printed.code) - module.replaceRoot(parsed) + module.setRoot(parsed) setExternalIds(module, spans, idMap) return parsed } @@ -154,6 +155,20 @@ class Abstractor { this.toRaw = new Map() } + abstractStatement(tree: RawAst.Tree): { + whitespace: string | undefined + node: Owned + } { + return this.abstractTree(tree) as any + } + + abstractExpression(tree: RawAst.Tree): { + whitespace: string | undefined + node: Owned + } { + return this.abstractTree(tree) as any + } + abstractTree(tree: RawAst.Tree): { whitespace: string | undefined; node: Owned } { const whitespaceStart = tree.whitespaceStartInCodeParsed const whitespaceEnd = whitespaceStart + tree.whitespaceLengthInCodeParsed @@ -168,54 +183,14 @@ class Abstractor { case RawAst.Tree.Type.BodyBlock: { const lines = Array.from(tree.statements, line => { const newline = this.abstractToken(line.newline) - const expression = line.expression ? this.abstractTree(line.expression) : undefined - return { newline, expression } + const statement = line.expression ? this.abstractStatement(line.expression) : undefined + return { newline, statement } }) node = BodyBlock.concrete(this.module, lines) break } case RawAst.Tree.Type.Function: { - const annotationLines = Array.from(tree.annotationLines, anno => ({ - annotation: { - operator: this.abstractToken(anno.annotation.operator), - annotation: this.abstractToken(anno.annotation.annotation), - argument: anno.annotation.argument && this.abstractTree(anno.annotation.argument), - }, - newlines: Array.from(anno.newlines, this.abstractToken.bind(this)), - })) - const signatureLine = tree.signatureLine && { - signature: this.abstractTypeSignature(tree.signatureLine.signature), - newlines: Array.from(tree.signatureLine.newlines, this.abstractToken.bind(this)), - } - const private_ = tree.private && this.abstractToken(tree.private) - const name = this.abstractTree(tree.name) - const argumentDefinitions = Array.from(tree.args, arg => ({ - open: arg.open && this.abstractToken(arg.open), - open2: arg.open2 && this.abstractToken(arg.open2), - suspension: arg.suspension && this.abstractToken(arg.suspension), - pattern: this.abstractTree(arg.pattern), - type: arg.typeNode && { - operator: this.abstractToken(arg.typeNode.operator), - type: this.abstractTree(arg.typeNode.typeNode), - }, - close2: arg.close2 && this.abstractToken(arg.close2), - defaultValue: arg.default && { - equals: this.abstractToken(arg.default.equals), - expression: this.abstractTree(arg.default.expression), - }, - close: arg.close && this.abstractToken(arg.close), - })) - const equals = this.abstractToken(tree.equals) - const body = tree.body !== undefined ? this.abstractTree(tree.body) : undefined - node = Function.concrete(this.module, { - annotationLines, - signatureLine, - private_, - name, - argumentDefinitions, - equals, - body, - } satisfies FunctionFields) + node = this.abstractFunction(tree) break } case RawAst.Tree.Type.Ident: { @@ -224,24 +199,25 @@ class Abstractor { break } case RawAst.Tree.Type.Assignment: { - const pattern = this.abstractTree(tree.pattern) + const docLine = tree.docLine && this.abstractDocLine(tree.docLine) + const pattern = this.abstractExpression(tree.pattern) const equals = this.abstractToken(tree.equals) - const value = this.abstractTree(tree.expr) - node = Assignment.concrete(this.module, pattern, equals, value) + const value = this.abstractExpression(tree.expr) + node = Assignment.concrete(this.module, docLine, pattern, equals, value) break } case RawAst.Tree.Type.App: { - const func = this.abstractTree(tree.func) - const arg = this.abstractTree(tree.arg) + const func = this.abstractExpression(tree.func) + const arg = this.abstractExpression(tree.arg) node = App.concrete(this.module, func, undefined, undefined, arg) break } case RawAst.Tree.Type.NamedApp: { - const func = this.abstractTree(tree.func) + const func = this.abstractExpression(tree.func) const open = tree.open ? this.abstractToken(tree.open) : undefined const name = this.abstractToken(tree.name) const equals = this.abstractToken(tree.equals) - const arg = this.abstractTree(tree.arg) + const arg = this.abstractExpression(tree.arg) const close = tree.close ? this.abstractToken(tree.close) : undefined const parens = open && close ? { open, close } : undefined const nameSpecification = { name, equals } @@ -250,7 +226,7 @@ class Abstractor { } case RawAst.Tree.Type.UnaryOprApp: { const opr = this.abstractToken(tree.opr) - const arg = tree.rhs ? this.abstractTree(tree.rhs) : undefined + const arg = tree.rhs ? this.abstractExpression(tree.rhs) : undefined if (arg && opr.node.code() === '-') { node = NegationApp.concrete(this.module, opr, arg) } else { @@ -265,12 +241,12 @@ class Abstractor { break } case RawAst.Tree.Type.OprApp: { - const lhs = tree.lhs ? this.abstractTree(tree.lhs) : undefined + const lhs = tree.lhs ? this.abstractExpression(tree.lhs) : undefined const opr = tree.opr.ok ? [this.abstractToken(tree.opr.value)] : Array.from(tree.opr.error.payload.operators, this.abstractToken.bind(this)) - const rhs = tree.rhs ? this.abstractTree(tree.rhs) : undefined + const rhs = tree.rhs ? this.abstractExpression(tree.rhs) : undefined const soleOpr = tryGetSoleValue(opr) if (soleOpr?.node.code() === '.' && rhs?.node instanceof MutableIdent) { // Propagate type. @@ -302,7 +278,7 @@ class Abstractor { // (which makes it impossible to give them unique IDs in the current IdMap format). case RawAst.Tree.Type.OprSectionBoundary: case RawAst.Tree.Type.TemplateFunction: - return { whitespace, node: this.abstractTree(tree.ast).node } + return { whitespace, node: this.abstractExpression(tree.ast).node } case RawAst.Tree.Type.Invalid: { const expression = this.abstractTree(tree.ast) node = Invalid.concrete(this.module, expression) @@ -310,7 +286,7 @@ class Abstractor { } case RawAst.Tree.Type.Group: { const open = tree.open ? this.abstractToken(tree.open) : undefined - const expression = tree.body ? this.abstractTree(tree.body) : undefined + const expression = tree.body ? this.abstractExpression(tree.body) : undefined const close = tree.close ? this.abstractToken(tree.close) : undefined node = Group.concrete(this.module, open, expression, close) break @@ -323,18 +299,16 @@ class Abstractor { node = TextLiteral.concrete(this.module, open, newline, elements, close) break } - case RawAst.Tree.Type.Documented: { - const open = this.abstractToken(tree.documentation.open) - const elements = Array.from(tree.documentation.elements, this.abstractTextToken.bind(this)) - const newlines = Array.from(tree.documentation.newlines, this.abstractToken.bind(this)) - const expression = tree.expression ? this.abstractTree(tree.expression) : undefined - node = Documented.concrete(this.module, open, elements, newlines, expression) + case RawAst.Tree.Type.ExpressionStatement: { + const docLine = tree.docLine && this.abstractDocLine(tree.docLine) + const expression = this.abstractExpression(tree.expression) + node = ExpressionStatement.concrete(this.module, docLine, expression) break } case RawAst.Tree.Type.Import: { const recurseBody = (tree: RawAst.Tree) => { - const body = this.abstractTree(tree) - if (body.node instanceof Invalid && body.node.code() === '') return undefined + const body = this.abstractExpression(tree) + if (body.node instanceof MutableInvalid && body.node.code() === '') return undefined return body } const recurseSegment = (segment: RawAst.MultiSegmentAppSegment) => ({ @@ -353,12 +327,12 @@ class Abstractor { case RawAst.Tree.Type.Array: { const left = this.abstractToken(tree.left) const elements = [] - if (tree.first) elements.push({ value: this.abstractTree(tree.first) }) + if (tree.first) elements.push({ value: this.abstractExpression(tree.first) }) else if (!tree.rest.next().done) elements.push({ value: undefined }) for (const rawElement of tree.rest) { elements.push({ delimiter: this.abstractToken(rawElement.operator), - value: rawElement.body && this.abstractTree(rawElement.body), + value: rawElement.body && this.abstractExpression(rawElement.body), }) } const right = this.abstractToken(tree.right) @@ -374,6 +348,52 @@ class Abstractor { return { node, whitespace } } + private abstractFunction(tree: RawAst.Tree.Function) { + const docLine = tree.docLine && this.abstractDocLine(tree.docLine) + const annotationLines = Array.from(tree.annotationLines, anno => ({ + annotation: { + operator: this.abstractToken(anno.annotation.operator), + annotation: this.abstractToken(anno.annotation.annotation), + argument: anno.annotation.argument && this.abstractExpression(anno.annotation.argument), + }, + newlines: Array.from(anno.newlines, this.abstractToken.bind(this)), + })) + const signatureLine = tree.signatureLine && { + signature: this.abstractTypeSignature(tree.signatureLine.signature), + newlines: Array.from(tree.signatureLine.newlines, this.abstractToken.bind(this)), + } + const private_ = tree.private && this.abstractToken(tree.private) + const name = this.abstractExpression(tree.name) + const argumentDefinitions = Array.from(tree.args, arg => ({ + open: arg.open && this.abstractToken(arg.open), + open2: arg.open2 && this.abstractToken(arg.open2), + suspension: arg.suspension && this.abstractToken(arg.suspension), + pattern: this.abstractExpression(arg.pattern), + type: arg.typeNode && { + operator: this.abstractToken(arg.typeNode.operator), + type: this.abstractExpression(arg.typeNode.typeNode), + }, + close2: arg.close2 && this.abstractToken(arg.close2), + defaultValue: arg.default && { + equals: this.abstractToken(arg.default.equals), + expression: this.abstractExpression(arg.default.expression), + }, + close: arg.close && this.abstractToken(arg.close), + })) + const equals = this.abstractToken(tree.equals) + const body = tree.body !== undefined ? this.abstractExpression(tree.body) : undefined + return Function.concrete(this.module, { + docLine, + annotationLines, + signatureLine, + private_, + name, + argumentDefinitions, + equals, + body, + } satisfies FunctionFields) + } + private abstractToken(token: RawAst.Token): { whitespace: string; node: Token } { const whitespaceStart = token.whitespaceStartInCodeBuffer const whitespaceEnd = whitespaceStart + token.whitespaceLengthInCodeBuffer @@ -412,7 +432,7 @@ class Abstractor { return { type: 'splice', open: this.abstractToken(raw.open), - expression: raw.expression && this.abstractTree(raw.expression), + expression: raw.expression && this.abstractExpression(raw.expression), close: this.abstractToken(raw.close), } } @@ -440,9 +460,19 @@ class Abstractor { private abstractTypeSignature(signature: RawAst.TypeSignature) { return { - name: this.abstractTree(signature.name), + name: this.abstractExpression(signature.name), operator: this.abstractToken(signature.operator), - type: this.abstractTree(signature.typeNode), + type: this.abstractExpression(signature.typeNode), + } + } + + private abstractDocLine(docLine: RawAst.DocLine) { + return { + docs: { + open: this.abstractToken(docLine.docs.open), + elements: Array.from(docLine.docs.elements, this.abstractTextToken.bind(this)), + }, + newlines: Array.from(docLine.newlines, this.abstractToken.bind(this)), } } } @@ -512,144 +542,81 @@ export function print(ast: Ast): PrintedSource { nodes: new Map(), tokens: new Map(), } - const code = ast.printSubtree(info, 0, undefined) + const code = ast.printSubtree(info, 0, null) return { info, code } } /** - * Used by `Ast.printSubtree`. Note that some AST types have overrides. + * Used by `Ast.printSubtree`. * @internal */ export function printAst( ast: Ast, info: SpanMap, offset: number, - parentIndent: string | undefined, - verbatim?: boolean, + parentIndent: string | null, + verbatim: boolean = false, ): string { let code = '' - for (const child of ast.concreteChildren(verbatim)) { + let currentLineIndent = parentIndent + let prevIsNewline = false + let isFirstToken = offset === 0 + for (const child of ast.concreteChildren({ verbatim, indent: parentIndent })) { if (!isTokenId(child.node) && ast.module.get(child.node) === undefined) continue - if (child.whitespace != null) { - code += child.whitespace - } else if (code.length != 0) { - code += ' ' + if (prevIsNewline) currentLineIndent = child.whitespace + const token = isTokenId(child.node) ? ast.module.getToken(child.node) : undefined + // Every line in a block starts with a newline token. In an AST produced by the parser, the newline token at the + // first line of a module is zero-length. In order to handle whitespace correctly if the lines of a module are + // rearranged, if a zero-length newline is encountered within a block, it is printed as an ordinary newline + // character, and if an ordinary newline is found at the beginning of the output, it is not printed; however if the + // output begins with a newline including a (plain) comment, we print the line as we would in any other block. + if ( + token?.tokenType_ == RawAst.Token.Type.Newline && + isFirstToken && + (!token.code_ || token.code_ === '\n') + ) { + prevIsNewline = true + isFirstToken = false + continue } - if (isTokenId(child.node)) { + code += child.whitespace + if (token) { const tokenStart = offset + code.length - const token = ast.module.getToken(child.node) - const span = tokenKey(tokenStart, token.code().length) + prevIsNewline = token.tokenType_ == RawAst.Token.Type.Newline + let tokenCode = token.code_ + if (token.tokenType_ == RawAst.Token.Type.Newline) { + tokenCode = tokenCode || '\n' + } + const span = tokenKey(tokenStart, tokenCode.length) info.tokens.set(span, token) - code += token.code() + code += tokenCode } else { + assert(!isTokenId(child.node)) + prevIsNewline = false const childNode = ast.module.get(child.node) - code += childNode.printSubtree(info, offset + code.length, parentIndent, verbatim) + code += childNode.printSubtree(info, offset + code.length, currentLineIndent, verbatim) // Extra structural validation. assertEqual(childNode.id, child.node) if (parentId(childNode) !== ast.id) { - console.error( - `Inconsistent parent pointer (expected ${ast.id})`, - childNode, - graphParentPointers(ast.module.root()!), - ) + console.error(`Inconsistent parent pointer (expected ${ast.id})`, childNode) } assertEqual(parentId(childNode), ast.id) } + isFirstToken = false } - const span = nodeKey(offset, code.length) + // Adjustment to handle an edge case: A module starts with a zero-length newline token. If its first line is indented, + // the initial whitespace belongs to the first line because it isn't hoisted past the (zero-length) newline to be the + // leading whitespace for the block. In that case, our representation of the block contains leading whitespace at the + // beginning, which must be excluded when calculating spans. + const leadingWhitespace = code.match(/ */)?.[0].length ?? 0 + const span = nodeKey(offset + leadingWhitespace, code.length - leadingWhitespace) map.setIfUndefined(info.nodes, span, (): Ast[] => []).unshift(ast) return code } -/** - * Use `Ast.code()' to stringify. - * @internal - */ -export function printBlock( - block: BodyBlock, - info: SpanMap, - offset: number, - parentIndent: string | undefined, - verbatim?: boolean, -): string { - let blockIndent: string | undefined - let code = '' - block.fields.get('lines').forEach((line, index) => { - code += line.newline.whitespace ?? '' - const newlineCode = block.module.getToken(line.newline.node).code() - // Only print a newline if this isn't the first line in the output, or it's a comment. - if (offset || index || newlineCode.startsWith('#')) { - // If this isn't the first line in the output, but there is a concrete newline token: - // if it's a zero-length newline, ignore it and print a normal newline. - code += newlineCode || '\n' - } - if (line.expression) { - if (blockIndent === undefined) { - if ((line.expression.whitespace?.length ?? 0) > (parentIndent?.length ?? 0)) { - blockIndent = line.expression.whitespace! - } else if (parentIndent !== undefined) { - blockIndent = parentIndent + ' ' - } else { - blockIndent = '' - } - } - const validIndent = (line.expression.whitespace?.length ?? 0) > (parentIndent?.length ?? 0) - code += validIndent ? line.expression.whitespace : blockIndent - const lineNode = block.module.get(line.expression.node) - assertEqual(lineNode.id, line.expression.node) - assertEqual(parentId(lineNode), block.id) - code += lineNode.printSubtree(info, offset + code.length, blockIndent, verbatim) - } - }) - const span = nodeKey(offset, code.length) - map.setIfUndefined(info.nodes, span, (): Ast[] => []).unshift(block) - return code -} - -/** - * Use `Ast.code()' to stringify. - * @internal - */ -export function printDocumented( - documented: Documented, - info: SpanMap, - offset: number, - parentIndent: string | undefined, - verbatim?: boolean, -): string { - const open = documented.fields.get('open') - const topIndent = parentIndent ?? open.whitespace ?? '' - let code = '' - code += open.node.code_ - const minWhitespaceLength = topIndent.length + 1 - let preferredWhitespace = topIndent + ' ' - documented.fields.get('elements').forEach(({ token }, i) => { - if (i === 0) { - const whitespace = token.whitespace ?? ' ' - code += whitespace - code += token.node.code_ - preferredWhitespace += whitespace - } else if (token.node.tokenType_ === RawAst.Token.Type.TextSection) { - if (token.whitespace && (verbatim || token.whitespace.length >= minWhitespaceLength)) - code += token.whitespace - else code += preferredWhitespace - code += token.node.code_ - } else { - code += token.whitespace ?? '' - code += token.node.code_ - } - }) - code += documented.fields - .get('newlines') - .map(({ whitespace, node }) => (whitespace ?? '') + node.code_) - .join('') - if (documented.expression) { - code += documented.fields.get('expression')?.whitespace ?? topIndent - code += documented.expression.printSubtree(info, offset + code.length, topIndent, verbatim) - } - const span = nodeKey(offset, code.length) - map.setIfUndefined(info.nodes, span, (): Ast[] => []).unshift(documented) - return code +/** Parse the input as a complete module. */ +export function parseModule(code: string, module?: MutableModule): Owned { + return parseModuleWithSpans(code, module).root } /** Parse the input as a body block, not the top level of a module. */ @@ -659,19 +626,43 @@ export function parseBlock(code: string, module?: MutableModule): Owned | undefined { const module_ = module ?? MutableModule.Transient() const ast = parseBlock(code, module) const soleStatement = tryGetSoleValue(ast.statements()) - if (!soleStatement) return ast + if (!soleStatement) return const parent = parentId(soleStatement) if (parent) module_.delete(parent) soleStatement.fields.set('parent', undefined) return asOwned(soleStatement) } +/** + * Parse the input as an expression. If it cannot be parsed as an expression (e.g. it is a statement or block), returns + * `undefined`. + */ +export function parseExpression( + code: string, + module?: MutableModule, +): Owned | undefined { + const module_ = module ?? MutableModule.Transient() + const ast = parseBlock(code, module) + const soleStatement = tryGetSoleValue(ast.statements()) + if (!(soleStatement instanceof MutableExpressionStatement)) return undefined + const expression = soleStatement.expression + module_.delete(soleStatement.id) + const parent = parentId(expression) + if (parent) module_.delete(parent) + expression.fields.set('parent', undefined) + return asOwned(expression) +} + /** Parse a module, and return it along with a mapping from source locations to parsed objects. */ export function parseModuleWithSpans( code: string, @@ -690,7 +681,7 @@ export function parseExtended(code: string, idMap?: IdMap | undefined, inModule? const module = inModule ?? MutableModule.Transient() const { root, spans, toRaw } = module.transact(() => { const { root, spans, toRaw } = abstract(module, rawRoot, code) - root.module.replaceRoot(root) + root.module.setRoot(root) if (idMap) setExternalIds(root.module, spans, idMap) return { root, spans, toRaw } }) @@ -702,7 +693,7 @@ export function parseExtended(code: string, idMap?: IdMap | undefined, inModule? /** Return the number of `Ast`s in the tree, including the provided root. */ export function astCount(ast: Ast): number { let count = 0 - ast.visitRecursiveAst(_subtree => { + ast.visitRecursive(_subtree => { count += 1 }) return count @@ -784,11 +775,11 @@ export function repair( const fixes = module ?? root.module.edit() for (const ast of lostInline) { if (ast instanceof Group) continue - fixes.getVersion(ast).update(ast => Group.new(fixes, ast)) + fixes.getVersion(ast).update(ast => Group.new(fixes, ast as any)) } // Verify that it's fixed. - const printed2 = print(fixes.getVersion(root)) + const printed2 = print(fixes.root()!) const reparsed2 = parseModuleWithSpans(printed2.code) const { lostInline: lostInline2, lostBlock: lostBlock2 } = checkSpans( printed2.info.nodes, @@ -851,7 +842,7 @@ function resync( function hashSubtreeSyntax(ast: Ast, hashesOut: Map): SyntaxHash { let content = '' content += ast.typeName + ':' - for (const child of ast.concreteChildren()) { + for (const child of ast.concreteChildren({ verbatim: false, indent: '' })) { content += child.whitespace ?? '?' if (isTokenId(child.node)) { content += 'Token:' + hashString(ast.module.getToken(child.node).code()) @@ -885,12 +876,6 @@ function syntaxHash(root: Ast) { return { root: rootHash, hashes } } -/** If the input is a block containing a single expression, return the expression; otherwise return the input. */ -function rawBlockToInline(tree: RawAst.Tree.Tree) { - if (tree.type !== RawAst.Tree.Type.BodyBlock) return tree - return tryGetSoleValue(tree.statements)?.expression ?? tree -} - /** Update `ast` to match the given source code, while modifying it as little as possible. */ export function syncToCode(ast: MutableAst, code: string, metadataSource?: Module) { const codeBefore = ast.code() @@ -984,9 +969,22 @@ export function applyTextEditsToAst( ) { const printed = print(ast) const code = applyTextEdits(printed.code, textEdits) - const rawParsedBlock = rawParseModule(code) - const rawParsed = - ast instanceof MutableBodyBlock ? rawParsedBlock : rawBlockToInline(rawParsedBlock) + const astModuleRoot = ast.module.root() + const rawParsedBlock = + ast instanceof MutableBodyBlock && astModuleRoot && ast.is(astModuleRoot) ? + rawParseModule(code) + : rawParseBlock(code) + const rawParsedStatement = + ast instanceof MutableBodyBlock ? undefined : ( + tryGetSoleValue(rawParsedBlock.statements)?.expression + ) + const rawParsedExpression = + ast.isExpression() ? + rawParsedStatement?.type === RawAst.Tree.Type.ExpressionStatement ? + rawParsedStatement.expression + : undefined + : undefined + const rawParsed = rawParsedExpression ?? rawParsedStatement ?? rawParsedBlock const parsed = abstract(ast.module, rawParsed, code) const toSync = calculateCorrespondence( ast, @@ -1031,7 +1029,7 @@ function syncTree( target.fields.get('metadata').set('externalId', newExternalId()) } const newRoot = syncRoot ? target : newContent - newRoot.visitRecursiveAst(ast => { + newRoot.visitRecursive(ast => { const syncFieldsFrom = toSync.get(ast.id) const editAst = edit.getVersion(ast) if (syncFieldsFrom) { diff --git a/app/ydoc-shared/src/ast/token.ts b/app/ydoc-shared/src/ast/token.ts index e5aa86097e..70006062bc 100644 --- a/app/ydoc-shared/src/ast/token.ts +++ b/app/ydoc-shared/src/ast/token.ts @@ -157,7 +157,7 @@ export function isOperator(code: string): code is Operator { /** @internal */ export function isTokenId( - t: DeepReadonly | Owned>, + t: DeepReadonly, ): t is DeepReadonly { return typeof t === 'object' && !(t instanceof Ast) } diff --git a/app/ydoc-shared/src/ast/tree.ts b/app/ydoc-shared/src/ast/tree.ts index 267e8fd4b6..574f68d61f 100644 --- a/app/ydoc-shared/src/ast/tree.ts +++ b/app/ydoc-shared/src/ast/tree.ts @@ -1,22 +1,21 @@ +// Declaration-merging is used to implement mixin types in this file. /* eslint-disable @typescript-eslint/no-unsafe-declaration-merging */ import * as Y from 'yjs' -import type { +import { Identifier, IdentifierOrOperatorIdentifier, IdentifierOrOperatorIdentifierToken, IdentifierToken, Module, + MutableModule, NodeChild, Owned, + ROOT_ID, RawNodeChild, SpanMap, SyncTokenId, - TypeOrConstructorIdentifier, -} from '.' -import { - MutableModule, - ROOT_ID, Token, + TypeOrConstructorIdentifier, asOwned, escapeTextLiteral, isIdentifier, @@ -37,12 +36,10 @@ import { is_numeric_literal } from './ffi' import * as RawAst from './generated/ast' import { applyTextEditsToAst, - parse, - parseBlock, + parseExpression, + parseStatement, print, printAst, - printBlock, - printDocumented, syncToCode, } from './parse' @@ -51,6 +48,9 @@ export type DeepReadonly = T declare const brandAstId: unique symbol export type AstId = string & { [brandAstId]: never } +export type ConcreteChild = { whitespace: string; node: T } +export type RawConcreteChild = ConcreteChild | ConcreteChild + /** @internal */ export interface MetadataFields { externalId: ExternalId @@ -84,13 +84,24 @@ const astFieldKeys = allKeys({ metadata: null, }) -/** TODO: Add docs */ +/** + * Base class for "first class" AST types. The kinds of AST data include: + * - Subtypes of `Ast`, including {@link Expression} and {@link Statement}. These are high-level types representing + * meaningful substructures of the AST; they have unique IDs, can be mutated, and support many common operations. + * - {@link Token}s: All AST types are ultimately composed of tokens representing their constituent code. + * - Structured field data: Types organizing {@link Token}s into structures representing parts of {@link Ast}s. Some + * structured field data includes Y.Js types, such as the metadata maps; field data is otherwise immutable, and is + * get/set through accessors of the `Ast` types. + */ export abstract class Ast { readonly module: Module /** @internal */ readonly fields: FixedMapView - /** TODO: Add docs */ + /** + * Return a stable unique identifier for this AST object. This can be used to retrieve the object from the module, or + * from any module derived from this module (e.g. by calling {@link MutableModule.edit}). + */ get id(): AstId { return this.fields.get('id') } @@ -102,6 +113,26 @@ export abstract class Ast { return id } + /** Returns whether this type can be an expression. */ + isAllowedInExpressionContext(): boolean { + return false + } + + /** Returns whether this type can be a statement. */ + isAllowedInStatementContext(): boolean { + return false + } + + /** Type predicate for {@link Expression}. */ + isExpression(): this is Expression { + return this.isAllowedInExpressionContext() + } + + /** Type predicate for {@link Statement}. */ + isStatement(): this is Statement { + return this.isAllowedInStatementContext() + } + /** TODO: Add docs */ get nodeMetadata(): NodeMetadata { const metadata = this.fields.get('metadata') @@ -133,64 +164,16 @@ export abstract class Ast { return this.id === other.id } - /** TODO: Add docs */ - innerExpression(): Ast { - return this.wrappedExpression()?.innerExpression() ?? this - } - - /** TODO: Add docs */ - wrappedExpression(): Ast | undefined { - return undefined - } - - /** TODO: Add docs */ - wrappingExpression(): Ast | undefined { - const parent = this.parent() - return parent?.wrappedExpression()?.is(this) ? parent : undefined - } - - /** TODO: Add docs */ - wrappingExpressionRoot(): Ast { - return this.wrappingExpression()?.wrappingExpressionRoot() ?? this - } - - /** TODO: Add docs */ - documentingAncestor(): Documented | undefined { - return this.wrappingExpression()?.documentingAncestor() - } - - /** TODO: Add docs */ - get isBindingStatement(): boolean { - const inner = this.wrappedExpression() - if (inner) { - return inner.isBindingStatement - } else { - return false - } - } - - /** TODO: Add docs */ + /** Return source code representing this node. */ code(): string { return print(this).code } /** TODO: Add docs */ - visitRecursive(visit: (node: Ast | Token) => void): void { - visit(this) - for (const child of this.children()) { - if (isToken(child)) { - visit(child) - } else { - child.visitRecursive(visit) - } - } - } - - /** TODO: Add docs */ - visitRecursiveAst(visit: (ast: Ast) => void | boolean): void { + visitRecursive(visit: (ast: Ast) => void | boolean): void { if (visit(this) === false) return for (const child of this.children()) { - if (!isToken(child)) child.visitRecursiveAst(visit) + if (!isToken(child)) child.visitRecursive(visit) } } @@ -198,7 +181,7 @@ export abstract class Ast { printSubtree( info: SpanMap, offset: number, - parentIndent: string | undefined, + parentIndent: string | null, verbatim?: boolean, ): string { return printAst(this, info, offset, parentIndent, verbatim) @@ -206,7 +189,7 @@ export abstract class Ast { /** Returns child subtrees, without information about the whitespace between them. */ *children(): IterableIterator { - for (const child of this.concreteChildren()) { + for (const child of this.concreteChildren({ verbatim: false, indent: '' })) { if (isTokenId(child.node)) { yield this.module.getToken(child.node) } else { @@ -227,16 +210,6 @@ export abstract class Ast { return this.module.get(this.parentId) } - /** TODO: Add docs */ - static parseBlock(source: string, module?: MutableModule) { - return parseBlock(source, module) - } - - /** TODO: Add docs */ - static parse(source: string, module?: MutableModule) { - return parse(source, module) - } - //////////////////// protected constructor(module: Module, fields: FixedMapView) { @@ -248,9 +221,8 @@ export abstract class Ast { * Returns child subtrees, including information about the whitespace between them. * @internal */ - abstract concreteChildren(verbatim?: boolean): IterableIterator + abstract concreteChildren(printContext: PrintContext): IterableIterator } -// eslint-disable-next-line @typescript-eslint/no-empty-object-type export interface MutableAst {} /** TODO: Add docs */ export abstract class MutableAst extends Ast { @@ -411,21 +383,12 @@ export abstract class MutableAst extends Ast { applyTextEditsToAst(this, textEdits, metadataSource ?? this.module) } - /** TODO: Add docs */ - getOrInitDocumentation(): MutableDocumented { - const existing = this.documentingAncestor() - if (existing) return this.module.getVersion(existing) - return this.module - .getVersion(this.wrappingExpressionRoot()) - .updateValue(ast => Documented.new('', ast)) - } - /////////////////// /** @internal */ importReferences(module: Module) { if (module === this.module) return - for (const child of this.concreteChildren()) { + for (const child of this.concreteChildren({ verbatim: false, indent: '' })) { if (!isTokenId(child.node)) { const childInForeignModule = module.get(child.node) assert(childInForeignModule !== undefined) @@ -476,6 +439,8 @@ type StructuralField = | SignatureLine | FunctionAnnotation | AnnotationLine + | DocComment + | DocLine /** Type whose fields are all suitable for storage as `Ast` fields. */ interface FieldObject { @@ -593,6 +558,10 @@ function mapRefs( field: VectorElement, f: MapRef, ): VectorElement +function mapRefs( + field: DocLine, + f: MapRef, +): DocLine function mapRefs( field: AnnotationLine, f: MapRef, @@ -652,6 +621,74 @@ function applyMixins(derivedCtor: any, constructors: any[]) { }) } +abstract class BaseStatement extends Ast { + override isAllowedInStatementContext(): true { + return true + } + override isAllowedInExpressionContext() { + return false + } + documentationText(): string | undefined { + return + } +} +/** + * A statement, i.e. the contents of a line which is either at the top level of a module (a module declaration), or + * within a body block (a function body statement). + */ +export interface Statement extends BaseStatement { + /** If this statement type supports attached documentation, and documentation is present, parse and return it. */ + documentationText(): string | undefined +} +abstract class BaseMutableStatement extends MutableAst implements Statement { + override isAllowedInStatementContext(): true { + return true + } + override isAllowedInExpressionContext() { + return false + } + documentationText(): string | undefined { + return + } + setDocumentationText?: (text: string | undefined) => void +} +/** A mutable {@link Statement}. */ +export interface MutableStatement extends BaseMutableStatement { + /** + * Set (or clear) the documentation associated with this statement. This method is only present on statement types + * that support attaching documentation. + */ + setDocumentationText?: (text: string | undefined) => void +} + +abstract class BaseExpression extends Ast { + override isAllowedInStatementContext() { + return false + } + override isAllowedInExpressionContext(): true { + return true + } +} +/** An expression, pattern, or type-expression. */ +export interface Expression extends BaseExpression {} +abstract class BaseMutableExpression extends MutableAst implements Expression { + override isAllowedInStatementContext() { + return false + } + override isAllowedInExpressionContext(): true { + return true + } +} +/** A mutable {@link Expression}. */ +export interface MutableExpression extends BaseMutableExpression {} + +function toStatement( + ast: Owned | Owned, + edit?: MutableModule, +) { + return ast.isAllowedInStatementContext() ? ast : ExpressionStatement.new(ast as any, { edit }) +} + interface AppFields { function: NodeChild parens: OpenCloseTokens | undefined @@ -667,7 +704,7 @@ interface NameSpecification { equals: T['token'] } /** TODO: Add docs */ -export class App extends Ast { +export class App extends BaseExpression { declare fields: FixedMap /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -676,17 +713,17 @@ export class App extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableApp) return parsed } /** TODO: Add docs */ static concrete( module: MutableModule, - func: NodeChild, + func: NodeChild>, parens: OpenCloseTokens | undefined, nameSpecification: NameSpecification | undefined, - argument: NodeChild, + argument: NodeChild>, ) { const base = module.baseObject('App') const id_ = base.get('id') @@ -702,9 +739,9 @@ export class App extends Ast { /** TODO: Add docs */ static new( module: MutableModule, - func: Owned, + func: Owned, argumentName: StrictIdentLike | undefined, - argument: Owned, + argument: Owned, ) { return App.concrete( module, @@ -716,35 +753,42 @@ export class App extends Ast { } /** TODO: Add docs */ - static positional(func: Owned, argument: Owned, module?: MutableModule): Owned { + static positional( + func: Owned, + argument: Owned, + module?: MutableModule, + ): Owned { return App.new(module ?? MutableModule.Transient(), func, undefined, argument) } - /** TODO: Add docs */ - static PositionalSequence(func: Owned, args: Owned[]): Owned { - return args.reduce( + /** Build a tree of 0-or-more applications applying all given arguments to the given function. */ + static PositionalSequence( + func: Owned, + args: Owned[], + ): Owned { + return args.reduce>( (expression, argument) => App.new(func.module, expression, undefined, argument), func, ) } /** TODO: Add docs */ - get function(): Ast { - return this.module.get(this.fields.get('function').node) + get function(): Expression { + return this.module.get(this.fields.get('function').node) as Expression } /** TODO: Add docs */ get argumentName(): Token | undefined { return this.module.getToken(this.fields.get('nameSpecification')?.name.node) } /** TODO: Add docs */ - get argument(): Ast { - return this.module.get(this.fields.get('argument').node) + get argument(): Expression { + return this.module.get(this.fields.get('argument').node) as Expression } /** TODO: Add docs */ - *concreteChildren(verbatim?: boolean): IterableIterator { + *concreteChildren({ verbatim }: PrintContext): IterableIterator { const { function: function_, parens, nameSpecification, argument } = getAll(this.fields) - yield ensureUnspaced(function_, verbatim) + yield firstChild(function_) const useParens = !!(parens && (nameSpecification || verbatim)) const spacedEquals = useParens && !!nameSpecification?.equals.whitespace if (useParens) yield ensureSpaced(parens.open, verbatim) @@ -761,58 +805,79 @@ export class App extends Ast { if (useParens) yield preferUnspaced(parens.close) } } -function ensureSpacedOnlyIf( - child: NodeChild, +interface OptionalWhitespace { + whitespace?: string | undefined +} +interface WithWhitespace extends OptionalWhitespace { + whitespace: string +} +function withWhitespace(node: T, whitespace: string): T & WithWhitespace { + return { ...node, whitespace } +} +function ensureSpacedOnlyIf( + child: T, condition: boolean, verbatim: boolean | undefined, -): ConcreteChild { +): T & WithWhitespace { return condition ? ensureSpaced(child, verbatim) : ensureUnspaced(child, verbatim) } -type ConcreteChild = { whitespace: string; node: T } -function isConcrete(child: NodeChild): child is ConcreteChild { +function isConcrete(child: T): child is T & WithWhitespace { return child.whitespace !== undefined } -function tryAsConcrete(child: NodeChild): ConcreteChild | undefined { +function tryAsConcrete(child: T): (T & WithWhitespace) | undefined { return isConcrete(child) ? child : undefined } -function ensureSpaced(child: NodeChild, verbatim: boolean | undefined): ConcreteChild { +function ensureSpaced( + child: T, + verbatim: boolean | undefined, +): T & WithWhitespace { const concreteInput = tryAsConcrete(child) if (verbatim && concreteInput) return concreteInput return concreteInput?.whitespace ? concreteInput : { ...child, whitespace: ' ' } } -function ensureUnspaced(child: NodeChild, verbatim: boolean | undefined): ConcreteChild { +function ensureUnspaced( + child: T, + verbatim: boolean | undefined, +): T & WithWhitespace { const concreteInput = tryAsConcrete(child) if (verbatim && concreteInput) return concreteInput return concreteInput?.whitespace === '' ? concreteInput : { ...child, whitespace: '' } } -function preferSpacedIf(child: NodeChild, condition: boolean): ConcreteChild { +function firstChild(child: T): T & WithWhitespace { + const concreteInput = tryAsConcrete(child) + return concreteInput?.whitespace === '' ? concreteInput : { ...child, whitespace: '' } +} +function preferSpacedIf( + child: T, + condition: boolean, +): T & WithWhitespace { return condition ? preferSpaced(child) : preferUnspaced(child) } -function preferUnspaced(child: NodeChild): ConcreteChild { +function preferUnspaced(child: T): T & WithWhitespace { return tryAsConcrete(child) ?? { ...child, whitespace: '' } } -function preferSpaced(child: NodeChild): ConcreteChild { +function preferSpaced(child: T): T & WithWhitespace { return tryAsConcrete(child) ?? { ...child, whitespace: ' ' } } /** TODO: Add docs */ -export class MutableApp extends App implements MutableAst { +export class MutableApp extends App implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap - setFunction(value: Owned) { + setFunction(value: Owned) { setNode(this.fields, 'function', this.claimChild(value)) } setArgumentName(name: StrictIdentLike | undefined) { this.fields.set('nameSpecification', nameSpecification(name)) } - setArgument(value: Owned) { + setArgument(value: Owned) { setNode(this.fields, 'argument', this.claimChild(value)) } } -export interface MutableApp extends App, MutableAst { - get function(): MutableAst - get argument(): MutableAst +export interface MutableApp extends App, MutableExpression { + get function(): MutableExpression + get argument(): MutableExpression } applyMixins(MutableApp, [MutableAst]) @@ -821,7 +886,7 @@ interface UnaryOprAppFields { argument: NodeChild | undefined } /** TODO: Add docs */ -export class UnaryOprApp extends Ast { +export class UnaryOprApp extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -830,7 +895,7 @@ export class UnaryOprApp extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableUnaryOprApp) return parsed } @@ -838,7 +903,7 @@ export class UnaryOprApp extends Ast { static concrete( module: MutableModule, operator: NodeChild, - argument: NodeChild | undefined, + argument: NodeChild> | undefined, ) { const base = module.baseObject('UnaryOprApp') const id_ = base.get('id') @@ -850,7 +915,11 @@ export class UnaryOprApp extends Ast { } /** TODO: Add docs */ - static new(module: MutableModule, operator: Token, argument: Owned | undefined) { + static new( + module: MutableModule, + operator: Token, + argument: Owned | undefined, + ) { return this.concrete(module, unspaced(operator), argument ? autospaced(argument) : undefined) } @@ -859,31 +928,31 @@ export class UnaryOprApp extends Ast { return this.module.getToken(this.fields.get('operator').node) } /** TODO: Add docs */ - get argument(): Ast | undefined { - return this.module.get(this.fields.get('argument')?.node) + get argument(): Expression | undefined { + return this.module.get(this.fields.get('argument')?.node) as Expression | undefined } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { + *concreteChildren({ verbatim }: PrintContext): IterableIterator { const { operator, argument } = getAll(this.fields) - yield operator - if (argument) yield argument + yield firstChild(operator) + if (argument) yield ensureUnspaced(argument, verbatim) } } /** TODO: Add docs */ -export class MutableUnaryOprApp extends UnaryOprApp implements MutableAst { +export class MutableUnaryOprApp extends UnaryOprApp implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap setOperator(value: Token) { this.fields.set('operator', unspaced(value)) } - setArgument(argument: Owned | undefined) { + setArgument(argument: Owned | undefined) { setNode(this.fields, 'argument', this.claimChild(argument)) } } -export interface MutableUnaryOprApp extends UnaryOprApp, MutableAst { - get argument(): MutableAst | undefined +export interface MutableUnaryOprApp extends UnaryOprApp, MutableExpression { + get argument(): MutableExpression | undefined } applyMixins(MutableUnaryOprApp, [MutableAst]) @@ -892,7 +961,7 @@ interface AutoscopedIdentifierFields { identifier: NodeChild } /** TODO: Add docs */ -export class AutoscopedIdentifier extends Ast { +export class AutoscopedIdentifier extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -909,7 +978,7 @@ export class AutoscopedIdentifier extends Ast { source: string, module?: MutableModule, ): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableAutoscopedIdentifier) return parsed } @@ -935,14 +1004,14 @@ export class AutoscopedIdentifier extends Ast { } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { + *concreteChildren({ verbatim }: PrintContext): IterableIterator { const { operator, identifier } = getAll(this.fields) - yield operator - yield identifier + yield firstChild(operator) + yield ensureUnspaced(identifier, verbatim) } } /** TODO: Add docs */ -export class MutableAutoscopedIdentifier extends AutoscopedIdentifier implements MutableAst { +export class MutableAutoscopedIdentifier extends AutoscopedIdentifier implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap @@ -951,7 +1020,7 @@ export class MutableAutoscopedIdentifier extends AutoscopedIdentifier implements this.fields.set('identifier', unspaced(token)) } } -export interface MutableAutoscopedIdentifier extends AutoscopedIdentifier, MutableAst {} +export interface MutableAutoscopedIdentifier extends AutoscopedIdentifier, MutableExpression {} applyMixins(MutableAutoscopedIdentifier, [MutableAst]) interface NegationAppFields { @@ -959,7 +1028,7 @@ interface NegationAppFields { argument: NodeChild } /** TODO: Add docs */ -export class NegationApp extends Ast { +export class NegationApp extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -968,12 +1037,16 @@ export class NegationApp extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableNegationApp) return parsed } /** TODO: Add docs */ - static concrete(module: MutableModule, operator: NodeChild, argument: NodeChild) { + static concrete( + module: MutableModule, + operator: NodeChild, + argument: NodeChild>, + ) { const base = module.baseObject('NegationApp') const id_ = base.get('id') const fields = composeFieldData(base, { @@ -984,7 +1057,7 @@ export class NegationApp extends Ast { } /** TODO: Add docs */ - static new(module: MutableModule, argument: Owned) { + static new(module: MutableModule, argument: Owned) { const minus = Token.new('-', RawAst.Token.Type.Operator) return this.concrete(module, unspaced(minus), unspaced(argument)) } @@ -994,28 +1067,28 @@ export class NegationApp extends Ast { return this.module.getToken(this.fields.get('operator').node) } /** TODO: Add docs */ - get argument(): Ast { - return this.module.get(this.fields.get('argument').node) + get argument(): Expression { + return this.module.get(this.fields.get('argument').node) as Expression } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { + *concreteChildren({ verbatim }: PrintContext): IterableIterator { const { operator, argument } = getAll(this.fields) - yield operator - if (argument) yield argument + yield firstChild(operator) + if (argument) yield ensureUnspaced(argument, verbatim) } } /** TODO: Add docs */ -export class MutableNegationApp extends NegationApp implements MutableAst { +export class MutableNegationApp extends NegationApp implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap - setArgument(value: Owned) { + setArgument(value: Owned) { setNode(this.fields, 'argument', this.claimChild(value)) } } -export interface MutableNegationApp extends NegationApp, MutableAst { - get argument(): MutableAst +export interface MutableNegationApp extends NegationApp, MutableExpression { + get argument(): MutableExpression } applyMixins(MutableNegationApp, [MutableAst]) @@ -1025,7 +1098,7 @@ interface OprAppFields { rhs: NodeChild | undefined } /** TODO: Add docs */ -export class OprApp extends Ast { +export class OprApp extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -1034,16 +1107,16 @@ export class OprApp extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableOprApp) return parsed } /** TODO: Add docs */ static concrete( module: MutableModule, - lhs: NodeChild | undefined, + lhs: NodeChild> | undefined, operators: NodeChild[], - rhs: NodeChild | undefined, + rhs: NodeChild> | undefined, ) { const base = module.baseObject('OprApp') const id_ = base.get('id') @@ -1058,9 +1131,9 @@ export class OprApp extends Ast { /** TODO: Add docs */ static new( module: MutableModule, - lhs: Owned | undefined, + lhs: Owned | undefined, operator: Token | string, - rhs: Owned | undefined, + rhs: Owned | undefined, ) { const operatorToken = operator instanceof Token ? operator : Token.new(operator, RawAst.Token.Type.Operator) @@ -1068,8 +1141,8 @@ export class OprApp extends Ast { } /** TODO: Add docs */ - get lhs(): Ast | undefined { - return this.module.get(this.fields.get('lhs')?.node) + get lhs(): Expression | undefined { + return this.module.get(this.fields.get('lhs')?.node) as Expression | undefined } /** TODO: Add docs */ get operator(): Result[]> { @@ -1082,36 +1155,47 @@ export class OprApp extends Ast { return opr ? Ok(opr.node) : Err(operators_) } /** TODO: Add docs */ - get rhs(): Ast | undefined { - return this.module.get(this.fields.get('rhs')?.node) + get rhs(): Expression | undefined { + return this.module.get(this.fields.get('rhs')?.node) as Expression | undefined } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { + *concreteChildren({ verbatim }: PrintContext): IterableIterator { const { lhs, operators, rhs } = getAll(this.fields) - if (lhs) yield lhs - yield* operators - if (rhs) yield rhs + if (lhs) yield firstChild(lhs) + const [operator0, ...extraOperators] = operators + assertDefined(operator0) + const spaced = ((lhs && operator0.whitespace) ?? rhs?.whitespace ?? ' ') !== '' + const isAsymmetricallySpaceableOperator = + operator0.node.code_ === '->' || operator0.node.code_ === '=' || operator0.node.code_ === ',' + const applySpacing = + isAsymmetricallySpaceableOperator ? + (token: RawNodeChild) => preferSpacedIf(token, spaced) + : (token: RawNodeChild) => ensureSpacedOnlyIf(token, spaced, verbatim) + yield lhs ? applySpacing(operator0) : firstChild(operator0) + const extrasSpaced = (extraOperators[0]?.whitespace ?? ' ') !== '' + for (const op of extraOperators) yield ensureSpacedOnlyIf(op, extrasSpaced, verbatim) + if (rhs) yield applySpacing(rhs) } } /** TODO: Add docs */ -export class MutableOprApp extends OprApp implements MutableAst { +export class MutableOprApp extends OprApp implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap - setLhs(value: Owned) { + setLhs(value: Owned) { setNode(this.fields, 'lhs', this.claimChild(value)) } setOperator(value: Token) { this.fields.set('operators', [unspaced(value)]) } - setRhs(value: Owned) { + setRhs(value: Owned) { setNode(this.fields, 'rhs', this.claimChild(value)) } } -export interface MutableOprApp extends OprApp, MutableAst { - get lhs(): MutableAst | undefined - get rhs(): MutableAst | undefined +export interface MutableOprApp extends OprApp, MutableExpression { + get lhs(): MutableExpression | undefined + get rhs(): MutableExpression | undefined } applyMixins(MutableOprApp, [MutableAst]) @@ -1121,7 +1205,7 @@ interface PropertyAccessFields { rhs: NodeChild } /** TODO: Add docs */ -export class PropertyAccess extends Ast { +export class PropertyAccess extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -1133,12 +1217,17 @@ export class PropertyAccess extends Ast { source: string, module?: MutableModule, ): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutablePropertyAccess) return parsed } /** TODO: Add docs */ - static new(module: MutableModule, lhs: Owned, rhs: IdentLike, style?: { spaced?: boolean }) { + static new( + module: MutableModule, + lhs: Owned, + rhs: IdentLike, + style?: { spaced?: boolean }, + ) { const dot = Token.new('.', RawAst.Token.Type.Operator) const whitespace = style?.spaced ? ' ' : '' return this.concrete( @@ -1182,7 +1271,7 @@ export class PropertyAccess extends Ast { /** TODO: Add docs */ static concrete( module: MutableModule, - lhs: NodeChild | undefined, + lhs: NodeChild> | undefined, operator: NodeChild, rhs: NodeChild>, ) { @@ -1197,8 +1286,8 @@ export class PropertyAccess extends Ast { } /** TODO: Add docs */ - get lhs(): Ast | undefined { - return this.module.get(this.fields.get('lhs')?.node) + get lhs(): Expression | undefined { + return this.module.get(this.fields.get('lhs')?.node) as Expression | undefined } /** TODO: Add docs */ get operator(): Token { @@ -1212,19 +1301,20 @@ export class PropertyAccess extends Ast { } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { + *concreteChildren({ verbatim }: PrintContext): IterableIterator { const { lhs, operator, rhs } = getAll(this.fields) - if (lhs) yield lhs - yield operator - yield rhs + if (lhs) yield firstChild(lhs) + const spaced = ((lhs && operator.whitespace) ?? rhs.whitespace ?? '') !== '' + yield lhs ? ensureSpacedOnlyIf(operator, spaced, verbatim) : firstChild(operator) + yield ensureSpacedOnlyIf(rhs, spaced, verbatim) } } /** TODO: Add docs */ -export class MutablePropertyAccess extends PropertyAccess implements MutableAst { +export class MutablePropertyAccess extends PropertyAccess implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap - setLhs(value: Owned | undefined) { + setLhs(value: Owned | undefined) { setNode(this.fields, 'lhs', this.claimChild(value)) } setRhs(ident: IdentLike) { @@ -1233,8 +1323,8 @@ export class MutablePropertyAccess extends PropertyAccess implements MutableAst this.fields.set('rhs', old ? { ...old, node } : unspaced(node)) } } -export interface MutablePropertyAccess extends PropertyAccess, MutableAst { - get lhs(): MutableAst | undefined +export interface MutablePropertyAccess extends PropertyAccess, MutableExpression { + get lhs(): MutableExpression | undefined } applyMixins(MutablePropertyAccess, [MutableAst]) @@ -1242,7 +1332,10 @@ applyMixins(MutablePropertyAccess, [MutableAst]) * Unroll the provided chain of `PropertyAccess` nodes, returning the first non-access as `subject` and the accesses * from left-to-right. */ -export function accessChain(ast: Ast): { subject: Ast; accessChain: PropertyAccess[] } { +export function accessChain(ast: Expression): { + subject: Expression + accessChain: PropertyAccess[] +} { const accessChain = new Array() while (ast instanceof PropertyAccess && ast.lhs) { accessChain.push(ast) @@ -1256,7 +1349,19 @@ interface GenericFields { children: RawNodeChild[] } /** TODO: Add docs */ -export class Generic extends Ast { +export class Generic extends Ast implements Expression, Statement { + /** See {@link Ast.isAllowedInStatementContext}. */ + override isAllowedInStatementContext(): true { + return true + } + /** See {@link Ast.isAllowedInExpressionContext}. */ + override isAllowedInExpressionContext(): true { + return true + } + /** See {@link Statement['documentationText']}. */ + documentationText() { + return undefined + } declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -1274,8 +1379,9 @@ export class Generic extends Ast { } /** TODO: Add docs */ - concreteChildren(_verbatim?: boolean): IterableIterator { - return this.fields.get('children')[Symbol.iterator]() + concreteChildren(_printContext: PrintContext): IterableIterator { + // This type is only created by the parser, so its children are always concrete. + return this.fields.get('children')[Symbol.iterator]() as IterableIterator } } /** TODO: Add docs */ @@ -1283,22 +1389,25 @@ export class MutableGeneric extends Generic implements MutableAst { declare readonly module: MutableModule declare readonly fields: FixedMap } -export interface MutableGeneric extends Generic, MutableAst {} +export interface MutableGeneric extends Generic, MutableAst { + isAllowedInStatementContext(): true + isAllowedInExpressionContext(): true +} applyMixins(MutableGeneric, [MutableAst]) interface MultiSegmentAppSegment { header: T['token'] body: T['ast'] | undefined } -function multiSegmentAppSegment( +function multiSegmentAppSegment( header: string, body: Owned, ): MultiSegmentAppSegment -function multiSegmentAppSegment( +function multiSegmentAppSegment( header: string, body: Owned | undefined, ): MultiSegmentAppSegment | undefined -function multiSegmentAppSegment( +function multiSegmentAppSegment( header: string, body: Owned | undefined, ): MultiSegmentAppSegment | undefined { @@ -1329,7 +1438,7 @@ interface ImportFields extends FieldObject { } /** TODO: Add docs */ -export class Import extends Ast { +export class Import extends BaseStatement { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -1338,33 +1447,33 @@ export class Import extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseStatement(source, module) if (parsed instanceof MutableImport) return parsed } /** TODO: Add docs */ - get polyglot(): Ast | undefined { - return this.module.get(this.fields.get('polyglot')?.body?.node) + get polyglot(): Expression | undefined { + return this.module.get(this.fields.get('polyglot')?.body?.node) as Expression | undefined } /** TODO: Add docs */ - get from(): Ast | undefined { - return this.module.get(this.fields.get('from')?.body?.node) + get from(): Expression | undefined { + return this.module.get(this.fields.get('from')?.body?.node) as Expression | undefined } /** TODO: Add docs */ - get import_(): Ast | undefined { - return this.module.get(this.fields.get('import').body?.node) + get import_(): Expression | undefined { + return this.module.get(this.fields.get('import').body?.node) as Expression | undefined } /** TODO: Add docs */ get all(): Token | undefined { return this.module.getToken(this.fields.get('all')?.node) } /** TODO: Add docs */ - get as(): Ast | undefined { - return this.module.get(this.fields.get('as')?.body?.node) + get as(): Expression | undefined { + return this.module.get(this.fields.get('as')?.body?.node) as Expression | undefined } /** TODO: Add docs */ - get hiding(): Ast | undefined { - return this.module.get(this.fields.get('hiding')?.body?.node) + get hiding(): Expression | undefined { + return this.module.get(this.fields.get('hiding')?.body?.node) as Expression | undefined } /** TODO: Add docs */ @@ -1428,24 +1537,25 @@ export class Import extends Ast { } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { - const segment = (segment: MultiSegmentAppSegment | undefined) => { - const parts = [] - if (segment) parts.push(segment.header) - if (segment?.body) parts.push(segment.body) - return parts + *concreteChildren({ verbatim }: PrintContext): IterableIterator { + let isFirst = true + function* segment(segment: MultiSegmentAppSegment) { + yield isFirst ? firstChild(segment.header) : ensureSpaced(segment.header, verbatim) + if (segment.body) yield ensureSpaced(segment.body, verbatim) + isFirst = false } const { polyglot, from, import: import_, all, as, hiding } = getAll(this.fields) - yield* segment(polyglot) - yield* segment(from) - yield* segment(import_) - if (all) yield all - yield* segment(as) - yield* segment(hiding) + if (polyglot) yield* segment(polyglot) + if (from) yield* segment(from) + if (import_) yield* segment(import_) + assert(!isFirst) // There's always a `from` or `import` + if (all) yield ensureSpaced(all, verbatim) + if (as) yield* segment(as) + if (hiding) yield* segment(hiding) } } /** TODO: Add docs */ -export class MutableImport extends Import implements MutableAst { +export class MutableImport extends Import implements MutableStatement { declare readonly module: MutableModule declare readonly fields: FixedMap @@ -1459,40 +1569,42 @@ export class MutableImport extends Import implements MutableAst { return multiSegmentAppSegmentToRaw(this.module, msas, this.id) } - setPolyglot(value: Owned | undefined) { + setPolyglot(value: Owned | undefined) { this.fields.set( 'polyglot', value ? this.toRaw(multiSegmentAppSegment('polyglot', value)) : undefined, ) } - setFrom(value: Owned | undefined) { + setFrom(value: Owned | undefined) { this.fields.set('from', value ? this.toRaw(multiSegmentAppSegment('from', value)) : value) } - setImport(value: Owned) { + setImport(value: Owned) { this.fields.set('import', this.toRaw(multiSegmentAppSegment('import', value))) } setAll(value: Token | undefined) { this.fields.set('all', spaced(value)) } - setAs(value: Owned | undefined) { + setAs(value: Owned | undefined) { this.fields.set('as', this.toRaw(multiSegmentAppSegment('as', value))) } - setHiding(value: Owned | undefined) { + setHiding(value: Owned | undefined) { this.fields.set('hiding', this.toRaw(multiSegmentAppSegment('hiding', value))) } } -export interface MutableImport extends Import, MutableAst { - get polyglot(): MutableAst | undefined - get from(): MutableAst | undefined - get import_(): MutableAst | undefined - get as(): MutableAst | undefined - get hiding(): MutableAst | undefined +export interface MutableImport extends Import, MutableStatement { + get polyglot(): MutableExpression | undefined + get from(): MutableExpression | undefined + get import_(): MutableExpression | undefined + get as(): MutableExpression | undefined + get hiding(): MutableExpression | undefined } applyMixins(MutableImport, [MutableAst]) interface TreeRefs { token: any ast: any + expression: any + statement: any } type RefMap = ( field: FieldData, @@ -1500,14 +1612,26 @@ type RefMap = ( type RawRefs = { token: NodeChild ast: NodeChild + expression: NodeChild + statement: NodeChild } export type OwnedRefs = { token: NodeChild ast: NodeChild + expression: NodeChild> + statement: NodeChild> } type ConcreteRefs = { token: NodeChild ast: NodeChild + expression: NodeChild + statement: NodeChild +} +type MutableRefs = { + token: NodeChild + ast: NodeChild + expression: NodeChild + statement: NodeChild } function ownedToRaw(module: MutableModule, parentId: AstId): RefMap { return (child: FieldData) => { @@ -1554,7 +1678,7 @@ function textElementValue(element: TextElement): string { case 'token': { if (element.interpreted != null) return element.interpreted // The logical newline is not necessarily the same as the concrete token, e.g. the token could be a CRLF. - if (element.token.node.tokenType_ === RawAst.Token.Type.TextNewline) return '\n' + if (element.token.node.tokenType_ === RawAst.Token.Type.Newline) return '\n' // The token is an invalid escape-sequence or a text-section; return it verbatim. return element.token.node.code() } @@ -1580,7 +1704,7 @@ function uninterpolatedText(elements: DeepReadonly, module: Modul return elements.reduce((s, e) => s + rawTextElementValue(e, module), '') } -function fieldConcreteChildren(field: FieldData) { +function fieldRawChildren(field: FieldData) { const children = new Array() rewriteFieldRefs(field, (subfield: FieldData) => { if (typeof subfield === 'object' && 'node' in subfield) children.push(subfield) @@ -1595,7 +1719,7 @@ interface TextLiteralFields { close: NodeChild | undefined } /** TODO: Add docs */ -export class TextLiteral extends Ast { +export class TextLiteral extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -1604,7 +1728,7 @@ export class TextLiteral extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableTextLiteral) return parsed } @@ -1630,7 +1754,7 @@ export class TextLiteral extends Ast { /** TODO: Add docs */ static new(rawText: string, module?: MutableModule): Owned { const escaped = escapeTextLiteral(rawText) - const parsed = parse(`'${escaped}'`, module) + const parsed = parseExpression(`'${escaped}'`, module) if (!(parsed instanceof MutableTextLiteral)) { console.error(`Failed to escape string for interpolated text`, rawText, escaped, parsed) const safeText = rawText.replaceAll(/[^-+A-Za-z0-9_. ]/g, '') @@ -1648,12 +1772,45 @@ export class TextLiteral extends Ast { } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { + *concreteChildren({ verbatim, indent }: PrintContext): IterableIterator { const { open, newline, elements, close } = getAll(this.fields) - if (open) yield open - if (newline) yield newline - for (const e of elements) yield* fieldConcreteChildren(e) - if (close) yield close + if (open) yield firstChild(open) + const rawChildren = new Array() + if (newline) rawChildren.push(newline) + rawChildren.push(...fieldRawChildren(elements)) + let nextTokenSpacing: 'unspaced' | 'maybe-spaced' | 'indented' = 'unspaced' + let blockIndent: string | undefined = undefined + for (const raw of rawChildren) { + const tokenType = isToken(raw.node) ? raw.node.tokenType_ : null + const specialToken = + tokenType === RawAst.Token.Type.Newline ? 'newline' + : ( + tokenType === RawAst.Token.Type.OpenSymbol || tokenType === RawAst.Token.Type.CloseSymbol + ) ? + 'splice-delimiter' + : null + if (specialToken === 'newline') { + yield preferUnspaced(raw) + } else if (nextTokenSpacing === 'indented') { + const whitespace: string = + blockIndent ?? + (raw.whitespace && raw.whitespace.length > (indent || '').length ? + raw.whitespace + : undefined) ?? + (indent || '') + ' ' + blockIndent = whitespace + yield withWhitespace(raw, whitespace) + } else if (specialToken === 'splice-delimiter' || nextTokenSpacing === 'maybe-spaced') { + yield preferUnspaced(raw) + } else { + yield ensureUnspaced(raw, verbatim) + } + nextTokenSpacing = + specialToken === 'newline' ? 'indented' + : specialToken === 'splice-delimiter' ? 'maybe-spaced' + : 'unspaced' + } + if (close) yield ensureUnspaced(close, verbatim) } /** TODO: Add docs */ @@ -1683,7 +1840,7 @@ export class TextLiteral extends Ast { } } /** TODO: Add docs */ -export class MutableTextLiteral extends TextLiteral implements MutableAst { +export class MutableTextLiteral extends TextLiteral implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap @@ -1713,135 +1870,157 @@ export class MutableTextLiteral extends TextLiteral implements MutableAst { } const literalContents = isInterpolated || mustBecomeInterpolated ? escapeTextLiteral(rawText) : rawText - const parsed = parse(`${boundary}${literalContents}${boundary}`) + const parsed = parseExpression(`${boundary}${literalContents}${boundary}`) assert(parsed instanceof TextLiteral) const elements = parsed.elements.map(e => mapRefs(e, concreteToOwned(this.module))) this.setElements(elements) } } -export interface MutableTextLiteral extends TextLiteral, MutableAst {} +export interface MutableTextLiteral extends TextLiteral, MutableExpression {} applyMixins(MutableTextLiteral, [MutableAst]) -interface DocumentedFields { - open: NodeChild - elements: TextToken[] - newlines: NodeChild[] - expression: NodeChild | undefined +interface ExpressionStatementFields { + docLine: DocLine | undefined + expression: NodeChild } /** TODO: Add docs */ -export class Documented extends Ast { - declare fields: FixedMapView +export class ExpressionStatement extends BaseStatement { + declare fields: FixedMapView /** TODO: Add docs */ - constructor(module: Module, fields: FixedMapView) { + constructor(module: Module, fields: FixedMapView) { super(module, fields) } /** TODO: Add docs */ - static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) - if (parsed instanceof MutableDocumented) return parsed + static tryParse( + source: string, + module?: MutableModule, + ): Owned | undefined { + const parsed = parseStatement(source, module) + if (parsed instanceof MutableExpressionStatement) return parsed } /** TODO: Add docs */ - static new(text: string, expression: Owned) { + static new( + expression: Owned, + options: { + documentation?: string | undefined + edit?: MutableModule | undefined + } = {}, + ) { return this.concrete( - expression.module, - undefined, - textToUninterpolatedElements(text), - undefined, + options.edit ?? MutableModule.Transient(), + options.documentation != null ? + elementsToDocLine(textToUninterpolatedElements(options.documentation)) + : undefined, autospaced(expression), ) } /** TODO: Add docs */ - static concrete( - module: MutableModule, - open: NodeChild | undefined, - elements: TextToken[], - newlines: NodeChild[] | undefined, - expression: NodeChild | undefined, - ) { - const base = module.baseObject('Documented') - const id_ = base.get('id') - const fields = composeFieldData(base, { - open: open ?? unspaced(Token.new('##', RawAst.Token.Type.Operator)), - elements: elements.map(e => mapRefs(e, ownedToRaw(module, id_))), - newlines: newlines ?? [unspaced(Token.new('\n', RawAst.Token.Type.Newline))], - expression: concreteChild(module, expression, id_), - }) - return asOwned(new MutableDocumented(module, fields)) + static documented(documentation: string, expression: Owned) { + return this.new(expression, { documentation, edit: expression.module }) } /** TODO: Add docs */ - get expression(): Ast | undefined { - return this.module.get(this.fields.get('expression')?.node) + static concrete( + module: MutableModule, + docLine: DocLine | undefined, + expression: NodeChild>, + ) { + const base = module.baseObject('ExpressionStatement') + const id_ = base.get('id') + const fields = composeFieldData(base, { + docLine: docLine && mapRefs(docLine, ownedToRaw(module, id_)), + expression: concreteChild(module, expression, id_), + }) + return asOwned(new MutableExpressionStatement(module, fields)) + } + + /** TODO: Add docs */ + get expression(): Expression { + return this.module.get(this.fields.get('expression').node) as Expression } /** Return the string value of the documentation. */ - documentation(): string { - const raw = uninterpolatedText(this.fields.get('elements'), this.module) - return raw.startsWith(' ') ? raw.slice(1) : raw + override documentationText(): string | undefined { + return docLineToText(this.fields.get('docLine'), this.module) } /** TODO: Add docs */ - override wrappedExpression(): Ast | undefined { - return this.expression - } - - /** TODO: Add docs */ - override documentingAncestor(): Documented | undefined { - return this - } - - /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { - const { open, elements, newlines, expression } = getAll(this.fields) - yield open - for (const { token } of elements) yield token - yield* newlines - if (expression) yield expression - } - - /** TODO: Add docs */ - override printSubtree( - info: SpanMap, - offset: number, - parentIndent: string | undefined, - verbatim?: boolean, - ): string { - return printDocumented(this, info, offset, parentIndent, verbatim) + *concreteChildren({ indent, verbatim }: PrintContext): IterableIterator { + const { docLine, expression } = getAll(this.fields) + if (docLine) yield* docLineToConcrete(docLine, indent || '') + yield docLine ? + { whitespace: indent || '', node: expression.node } + : ensureUnspaced(expression, verbatim) } } +function* docLineToConcrete(docLine: DocLine, indent: string): IterableIterator { + yield firstChild(docLine.docs.open) + let prevType = undefined + let extraIndent = '' + for (const { token } of docLine.docs.elements) { + if (token.node.tokenType_ === RawAst.Token.Type.Newline) { + yield ensureUnspaced(token, false) + } else { + if (prevType === RawAst.Token.Type.Newline) { + yield { whitespace: indent + extraIndent, node: token.node } + } else { + if (prevType === undefined) { + const leadingSpace = token.node.code_.match(/ */) + extraIndent = ' ' + (leadingSpace ? leadingSpace[0] : '') + } + yield { whitespace: '', node: token.node } + } + } + prevType = token.node.tokenType_ + } + for (const newline of docLine.newlines) yield preferUnspaced(newline) +} +function docLineToText(docLine: DocLine | undefined, module: Module): string | undefined { + if (docLine == null) return + const raw = uninterpolatedText(docLine.docs.elements, module) + return raw.startsWith(' ') ? raw.slice(1) : raw +} +function docLineFromText( + text: string | undefined, + ast: { module: MutableModule; id: AstId }, +): DocLine | undefined { + if (text == null) return + return mapRefs( + elementsToDocLine(textToUninterpolatedElements(text)), + ownedToRaw(ast.module, ast.id), + ) +} /** TODO: Add docs */ -export class MutableDocumented extends Documented implements MutableAst { +export class MutableExpressionStatement extends ExpressionStatement implements MutableStatement { declare readonly module: MutableModule - declare readonly fields: FixedMap + declare readonly fields: FixedMap - setDocumentationText(text: string) { - this.fields.set( - 'elements', - textToUninterpolatedElements(text).map(owned => - mapRefs(owned, ownedToRaw(this.module, this.id)), - ), - ) + setDocumentationText(text: string | undefined) { + this.fields.set('docLine', docLineFromText(text, this)) } - setExpression(value: Owned | undefined) { + setExpression(value: Owned) { this.fields.set('expression', unspaced(this.claimChild(value))) } } -export interface MutableDocumented extends Documented, MutableAst { - get expression(): MutableAst | undefined +export interface MutableExpressionStatement extends ExpressionStatement, MutableAst { + isAllowedInStatementContext(): true + isAllowedInExpressionContext(): boolean + get expression(): MutableExpression } -applyMixins(MutableDocumented, [MutableAst]) +applyMixins(MutableExpressionStatement, [MutableAst]) function textToUninterpolatedElements(text: string): TextToken[] { + text = text && ' ' + text const elements = new Array>() text.split('\n').forEach((line, i) => { if (i) elements.push({ type: 'token', - token: unspaced(Token.new('\n', RawAst.Token.Type.TextNewline)), + token: unspaced(Token.new('\n', RawAst.Token.Type.Newline)), }) elements.push({ type: 'token', @@ -1851,11 +2030,33 @@ function textToUninterpolatedElements(text: string): TextToken[] { return elements } +function elementsToDocLine(elements: TextToken[]): DocLine { + return { + docs: { + open: unspaced(Token.new('##', RawAst.Token.Type.TextStart)), + elements, + }, + newlines: [unspaced(Token.new('\n', RawAst.Token.Type.Newline))], + } +} + interface InvalidFields { expression: NodeChild } /** TODO: Add docs */ -export class Invalid extends Ast { +export class Invalid extends Ast implements Statement, Expression { + /** See {@link Ast.isAllowedInStatementContext}. */ + override isAllowedInStatementContext(): true { + return true + } + /** See {@link Ast.isAllowedInExpressionContext}. */ + override isAllowedInExpressionContext(): true { + return true + } + /** See {@link Statement['documentationText']}. */ + documentationText() { + return undefined + } declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -1874,15 +2075,15 @@ export class Invalid extends Ast { } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { - yield this.fields.get('expression') + *concreteChildren(_printContext: PrintContext): IterableIterator { + yield firstChild(this.fields.get('expression')) } /** TODO: Add docs */ override printSubtree( info: SpanMap, offset: number, - parentIndent: string | undefined, + parentIndent: string | null, _verbatim?: boolean, ): string { return super.printSubtree(info, offset, parentIndent, true) @@ -1898,11 +2099,13 @@ export function invalidFields( return composeFieldData(base, { expression: concreteChild(module, expression, id_) }) } /** TODO: Add docs */ -export class MutableInvalid extends Invalid implements MutableAst { +export class MutableInvalid extends Invalid implements MutableStatement, MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap } export interface MutableInvalid extends Invalid, MutableAst { + isAllowedInStatementContext(): true + isAllowedInExpressionContext(): true /** * The `expression` getter is intentionally not narrowed to provide mutable access: * It makes more sense to `.replace` the `Invalid` node. @@ -1916,7 +2119,7 @@ interface GroupFields { close: NodeChild | undefined } /** TODO: Add docs */ -export class Group extends Ast { +export class Group extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -1925,7 +2128,7 @@ export class Group extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableGroup) return parsed } @@ -1933,7 +2136,7 @@ export class Group extends Ast { static concrete( module: MutableModule, open: NodeChild | undefined, - expression: NodeChild | undefined, + expression: NodeChild> | undefined, close: NodeChild | undefined, ) { const base = module.baseObject('Group') @@ -1947,41 +2150,37 @@ export class Group extends Ast { } /** TODO: Add docs */ - static new(module: MutableModule, expression: Owned) { + static new(module: MutableModule, expression: Owned) { const open = unspaced(Token.new('(', RawAst.Token.Type.OpenSymbol)) const close = unspaced(Token.new(')', RawAst.Token.Type.CloseSymbol)) return this.concrete(module, open, unspaced(expression), close) } /** TODO: Add docs */ - get expression(): Ast | undefined { - return this.module.get(this.fields.get('expression')?.node) + get expression(): Expression | undefined { + return this.module.get(this.fields.get('expression')?.node) as Expression | undefined } /** TODO: Add docs */ - override wrappedExpression(): Ast | undefined { - return this.expression - } - - /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { + *concreteChildren(_printContext: PrintContext): IterableIterator { const { open, expression, close } = getAll(this.fields) - if (open) yield open - if (expression) yield expression - if (close) yield close + if (open) yield firstChild(open) + const spaced = ((open && expression?.whitespace) ?? '') !== '' + if (expression) yield open ? preferSpacedIf(expression, spaced) : firstChild(expression) + if (close) yield open ?? expression ? preferSpacedIf(close, spaced) : firstChild(close) } } /** TODO: Add docs */ -export class MutableGroup extends Group implements MutableAst { +export class MutableGroup extends Group implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap - setExpression(value: Owned | undefined) { + setExpression(value: Owned | undefined) { this.fields.set('expression', unspaced(this.claimChild(value))) } } -export interface MutableGroup extends Group, MutableAst { - get expression(): MutableAst | undefined +export interface MutableGroup extends Group, MutableExpression { + get expression(): MutableExpression | undefined } applyMixins(MutableGroup, [MutableAst]) @@ -1989,7 +2188,7 @@ interface NumericLiteralFields { tokens: NodeChild[] } /** TODO: Add docs */ -export class NumericLiteral extends Ast { +export class NumericLiteral extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -2001,7 +2200,7 @@ export class NumericLiteral extends Ast { source: string, module?: MutableModule, ): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableNumericLiteral) return parsed } @@ -2010,7 +2209,7 @@ export class NumericLiteral extends Ast { source: string, module?: MutableModule, ): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if ( parsed instanceof MutableNumericLiteral || (parsed instanceof MutableNegationApp && parsed.argument instanceof MutableNumericLiteral) @@ -2026,16 +2225,19 @@ export class NumericLiteral extends Ast { } /** TODO: Add docs */ - concreteChildren(_verbatim?: boolean): IterableIterator { - return this.fields.get('tokens')[Symbol.iterator]() + *concreteChildren({ verbatim }: PrintContext): IterableIterator { + const [first, ...rest] = this.fields.get('tokens') + assertDefined(first) + yield firstChild(first) + for (const child of rest) yield ensureUnspaced(child, verbatim) } } /** TODO: Add docs */ -export class MutableNumericLiteral extends NumericLiteral implements MutableAst { +export class MutableNumericLiteral extends NumericLiteral implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap } -export interface MutableNumericLiteral extends NumericLiteral, MutableAst {} +export interface MutableNumericLiteral extends NumericLiteral, MutableExpression {} applyMixins(MutableNumericLiteral, [MutableAst]) /** TODO: Add docs */ @@ -2047,7 +2249,7 @@ export interface ArgumentDefinition { open?: T['token'] | undefined open2?: T['token'] | undefined suspension?: T['token'] | undefined - pattern: T['ast'] + pattern: T['expression'] type?: ArgumentType | undefined close2?: T['token'] | undefined defaultValue?: ArgumentDefault | undefined @@ -2064,6 +2266,16 @@ interface ArgumentType { type: T['ast'] } +interface DocComment { + open: T['token'] + elements: TextToken[] +} + +export interface DocLine { + docs: DocComment + newlines: T['token'][] +} + interface FunctionAnnotation { operator: T['token'] annotation: T['token'] @@ -2087,6 +2299,7 @@ interface SignatureLine { } export interface FunctionFields { + docLine: DocLine | undefined annotationLines: AnnotationLine[] signatureLine: SignatureLine | undefined private_: T['token'] | undefined @@ -2096,7 +2309,7 @@ export interface FunctionFields { body: T['ast'] | undefined } /** TODO: Add docs */ -export class Function extends Ast { +export class Function extends BaseStatement { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -2105,17 +2318,17 @@ export class Function extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseStatement(source, module) if (parsed instanceof MutableFunction) return parsed } /** TODO: Add docs */ - get name(): Ast { - return this.module.get(this.fields.get('name').node) + get name(): Expression { + return this.module.get(this.fields.get('name').node) as Expression } /** TODO: Add docs */ - get body(): Ast | undefined { - return this.module.get(this.fields.get('body')?.node) + get body(): BodyBlock | Expression | undefined { + return this.module.get(this.fields.get('body')?.node) as Expression | BodyBlock | undefined } /** TODO: Add docs */ get argumentDefinitions(): ArgumentDefinition[] { @@ -2132,6 +2345,7 @@ export class Function extends Ast { const base = module.baseObject('Function') const id_ = base.get('id') const rawFields = composeFieldData(base, { + docLine: fields.docLine && mapRefs(fields.docLine, ownedToRaw(module, id_)), annotationLines: (fields.annotationLines ?? []).map(anno => mapRefs(anno, ownedToRaw(module, id_)), ), @@ -2149,15 +2363,30 @@ export class Function extends Ast { /** TODO: Add docs */ static new( - module: MutableModule, name: IdentLike, - argumentDefinitions: ArgumentDefinition[], - body: Owned, + args: (ArgumentDefinition | StrictIdentLike)[], + body: Owned | Owned, + options: { + documentation?: string + edit?: MutableModule + } = {}, ): Owned { - // Note that a function name may not be an operator if the function is not in the body of a type definition, but we - // can't easily enforce that because we don't currently make a syntactic distinction between top-level functions and - // type methods. + const module = options.edit ?? MutableModule.Transient() + const argumentDefinitions = args.map(arg => + typeof arg === 'string' || isToken(arg) ? + { + pattern: spaced(Ident.new(module, arg)), + } + : arg, + ) return MutableFunction.concrete(module, { + docLine: + options.documentation != null ? + elementsToDocLine(textToUninterpolatedElements(options.documentation)) + : undefined, + // Note that a function name may not be an operator if the function is not in the body of a type definition, but + // we can't easily enforce that because we don't currently make a syntactic distinction between top-level + // functions and type methods. name: unspaced(Ident.newAllowingOperators(module, name)), argumentDefinitions, equals: spaced(makeEquals()), @@ -2165,25 +2394,8 @@ export class Function extends Ast { }) } - /** Construct a function with simple (name-only) arguments and a body block. */ - static fromStatements( - module: MutableModule, - name: IdentLike, - argumentNames: StrictIdentLike[], - statements: Owned[], - ): Owned { - const statements_: OwnedBlockLine[] = statements.map(statement => ({ - expression: unspaced(statement), - })) - const argumentDefinitions = argumentNames.map(name => ({ - pattern: spaced(Ident.new(module, name)), - })) - const body = BodyBlock.new(statements_, module) - return MutableFunction.new(module, name, argumentDefinitions, body) - } - /** TODO: Add docs */ - *bodyExpressions(): IterableIterator { + *bodyExpressions(): IterableIterator { const body = this.body if (body instanceof BodyBlock) { yield* body.statements() @@ -2193,61 +2405,112 @@ export class Function extends Ast { } /** TODO: Add docs */ - override get isBindingStatement(): boolean { - return true - } - - /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { - const { annotationLines, signatureLine, private_, name, argumentDefinitions, equals, body } = - getAll(this.fields) + *concreteChildren({ indent, verbatim }: PrintContext): IterableIterator { + const { + docLine, + annotationLines, + signatureLine, + private_, + name, + argumentDefinitions, + equals, + body, + } = getAll(this.fields) + // True/false: Previous is newline or non-newline; undefined: No prev yet. + let prevIsNewline: boolean | undefined = undefined + const maybeIndented = (nodeChild: NodeChild) => { + if (prevIsNewline) { + prevIsNewline = false + return { whitespace: indent || '', node: nodeChild.node } + } else if (prevIsNewline == null) { + prevIsNewline = false + return firstChild(nodeChild) + } else { + return preferUnspaced(nodeChild) + } + } + if (docLine) { + yield* docLineToConcrete(docLine, indent || '') + prevIsNewline = true + } for (const anno of annotationLines) { const { operator, annotation, argument } = anno.annotation - yield operator - yield annotation - if (argument) yield argument - yield* anno.newlines + yield maybeIndented(operator) + yield preferUnspaced(annotation) + if (argument) yield ensureSpaced(argument, verbatim) + for (const newline of anno.newlines) yield preferUnspaced(newline) + prevIsNewline = true } if (signatureLine) { - const { signature, newlines } = signatureLine - const { name, operator, type } = signature - yield name - yield operator - yield type - yield* newlines + const { name, operator, type } = signatureLine.signature + yield maybeIndented(name) + const spaced = (operator.whitespace ?? type.whitespace ?? ' ') !== '' + yield ensureSpacedOnlyIf(operator, spaced, verbatim) + yield ensureSpacedOnlyIf(type, spaced, verbatim) + for (const newline of signatureLine.newlines) yield preferUnspaced(newline) + prevIsNewline = true } - if (private_) yield private_ - yield name + if (private_) yield maybeIndented(private_) + yield maybeIndented(name) for (const def of argumentDefinitions) { const { open, open2, suspension, pattern, type, close2, defaultValue, close } = def - if (open) yield open - if (open2) yield open2 - if (suspension) yield suspension - yield pattern + if (open) yield ensureSpaced(open, verbatim) + const spacedInsideParen1 = open && ((open2 ?? suspension ?? pattern).whitespace ?? '') !== '' + if (open2) yield ensureSpacedOnlyIf(open2, spacedInsideParen1 ?? true, verbatim) + const spacedInsideParen2 = open2 && ((suspension ?? pattern).whitespace ?? '') !== '' + if (suspension) { + yield ensureSpacedOnlyIf( + suspension, + spacedInsideParen2 ?? spacedInsideParen1 ?? true, + verbatim, + ) + yield ensureUnspaced(pattern, verbatim) + } else { + yield ensureSpacedOnlyIf( + pattern, + spacedInsideParen2 ?? spacedInsideParen1 ?? true, + verbatim, + ) + } if (type) { - yield type.operator - yield type.type + const spaced = (type.operator.whitespace ?? type.type.whitespace ?? ' ') !== '' + yield ensureSpacedOnlyIf(type.operator, spaced, verbatim) + yield ensureSpacedOnlyIf(type.type, spaced, verbatim) } if (defaultValue) { - yield defaultValue.equals - yield defaultValue.expression + const spaced = + (defaultValue.equals.whitespace ?? defaultValue.expression.whitespace ?? ' ') !== '' + yield ensureSpacedOnlyIf(defaultValue.equals, spaced, verbatim) + yield ensureSpacedOnlyIf(defaultValue.expression, spaced, verbatim) } - if (close2) yield close2 - if (close) yield close + if (close2) yield ensureSpacedOnlyIf(close2, spacedInsideParen2 ?? false, verbatim) + if (close) yield ensureSpacedOnlyIf(close, spacedInsideParen1 ?? false, verbatim) } yield { whitespace: equals.whitespace ?? ' ', node: this.module.getToken(equals.node) } - if (body) yield preferSpacedIf(body, this.module.tryGet(body.node) instanceof BodyBlock) + if (body) + yield preferSpacedIf( + body, + !!equals.whitespace && !(this.module.tryGet(body.node) instanceof BodyBlock), + ) + } + + /** Return the string value of the documentation. */ + override documentationText(): string | undefined { + return docLineToText(this.fields.get('docLine'), this.module) } } /** TODO: Add docs */ -export class MutableFunction extends Function implements MutableAst { +export class MutableFunction extends Function implements MutableStatement { declare readonly module: MutableModule declare readonly fields: FixedMap - setName(value: Owned) { + setDocumentationText(text: string | undefined) { + this.fields.set('docLine', docLineFromText(text, this)) + } + setName(value: Owned) { this.fields.set('name', unspaced(this.claimChild(value))) } - setBody(value: Owned | undefined) { + setBody(value: Owned | undefined) { this.fields.set('body', unspaced(this.claimChild(value))) } setArgumentDefinitions(defs: ArgumentDefinition[]) { @@ -2267,19 +2530,25 @@ export class MutableFunction extends Function implements MutableAst { return newBody } } -export interface MutableFunction extends Function, MutableAst { - get name(): MutableAst - get body(): MutableAst | undefined +export interface MutableFunction extends Function, MutableStatement { + get name(): MutableExpression + get body(): MutableExpression | MutableBodyBlock | undefined } applyMixins(MutableFunction, [MutableAst]) +interface PrintContext { + verbatim: boolean + indent: string | null +} + interface AssignmentFields { + docLine: DocLine | undefined pattern: NodeChild equals: NodeChild expression: NodeChild } /** TODO: Add docs */ -export class Assignment extends Ast { +export class Assignment extends BaseStatement { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -2288,20 +2557,22 @@ export class Assignment extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseStatement(source, module) if (parsed instanceof MutableAssignment) return parsed } /** TODO: Add docs */ static concrete( module: MutableModule, - pattern: NodeChild, + docLine: DocLine | undefined, + pattern: NodeChild>, equals: NodeChild, - expression: NodeChild, + expression: NodeChild>, ) { const base = module.baseObject('Assignment') const id_ = base.get('id') const fields = composeFieldData(base, { + docLine: docLine && mapRefs(docLine, ownedToRaw(module, id_)), pattern: concreteChild(module, pattern, id_), equals, expression: concreteChild(module, expression, id_), @@ -2310,52 +2581,68 @@ export class Assignment extends Ast { } /** TODO: Add docs */ - static new(module: MutableModule, ident: StrictIdentLike, expression: Owned) { + static new( + ident: StrictIdentLike | Owned, + expression: Owned, + options: { edit?: MutableModule | undefined; documentation?: string | undefined }, + ) { + const module = options.edit ?? MutableModule.Transient() return Assignment.concrete( module, - unspaced(Ident.new(module, ident)), + options.documentation != null ? + elementsToDocLine(textToUninterpolatedElements(options.documentation)) + : undefined, + autospaced(ident instanceof MutableIdent ? ident : Ident.new(module, ident)), spaced(makeEquals()), spaced(expression), ) } /** TODO: Add docs */ - get pattern(): Ast { - return this.module.get(this.fields.get('pattern').node) + get pattern(): Expression { + return this.module.get(this.fields.get('pattern').node) as Expression } /** TODO: Add docs */ - get expression(): Ast { - return this.module.get(this.fields.get('expression').node) + get expression(): Expression { + return this.module.get(this.fields.get('expression').node) as Expression } /** TODO: Add docs */ - override get isBindingStatement(): boolean { - return true - } - - /** TODO: Add docs */ - *concreteChildren(verbatim?: boolean): IterableIterator { - const { pattern, equals, expression } = getAll(this.fields) - yield ensureUnspaced(pattern, verbatim) + *concreteChildren({ verbatim, indent }: PrintContext): IterableIterator { + const { docLine, pattern, equals, expression } = getAll(this.fields) + if (docLine) yield* docLineToConcrete(docLine, indent || '') + yield docLine ? + { whitespace: indent || '', node: pattern.node } + : ensureUnspaced(pattern, verbatim) yield ensureSpacedOnlyIf(equals, expression.whitespace !== '', verbatim) yield preferSpaced(expression) } + + /** Return the string value of the documentation. */ + override documentationText(): string | undefined { + return docLineToText(this.fields.get('docLine'), this.module) + } } /** TODO: Add docs */ -export class MutableAssignment extends Assignment implements MutableAst { +export class MutableAssignment extends Assignment implements MutableStatement { declare readonly module: MutableModule declare readonly fields: FixedMap - setPattern(value: Owned) { + setDocumentationText(text: string | undefined) { + this.fields.set('docLine', docLineFromText(text, this)) + } + setPattern(value: Owned) { this.fields.set('pattern', unspaced(this.claimChild(value))) } - setExpression(value: Owned) { + setExpression(value: Owned) { setNode(this.fields, 'expression', this.claimChild(value)) } } export interface MutableAssignment extends Assignment, MutableAst { - get pattern(): MutableAst - get expression(): MutableAst + isAllowedInStatementContext(): true + isAllowedInExpressionContext(): boolean + get pattern(): MutableExpression + get expression(): MutableExpression } applyMixins(MutableAssignment, [MutableAst]) @@ -2363,19 +2650,13 @@ interface BodyBlockFields { lines: RawBlockLine[] } /** TODO: Add docs */ -export class BodyBlock extends Ast { +export class BodyBlock extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { super(module, fields) } - /** TODO: Add docs */ - static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) - if (parsed instanceof MutableBodyBlock) return parsed - } - /** TODO: Add docs */ static concrete(module: MutableModule, lines: OwnedBlockLine[]) { const base = module.baseObject('BodyBlock') @@ -2397,17 +2678,27 @@ export class BodyBlock extends Ast { } /** TODO: Add docs */ - *statements(): IterableIterator { + *statements(): IterableIterator { for (const line of this.lines) { - if (line.expression) yield line.expression.node + if (line.statement) yield line.statement.node } } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { + *concreteChildren({ indent }: PrintContext): IterableIterator { + let linesIndent: string | undefined = undefined for (const line of this.fields.get('lines')) { yield preferUnspaced(line.newline) - if (line.expression) yield line.expression + if (line.statement) { + const whitespace: string = + linesIndent ?? + (line.statement.whitespace && line.statement.whitespace.length > (indent || '').length ? + line.statement.whitespace + : undefined) ?? + (indent != null ? indent + ' ' : '') + linesIndent = whitespace + yield { whitespace, node: line.statement.node } + } } } @@ -2415,14 +2706,14 @@ export class BodyBlock extends Ast { override printSubtree( info: SpanMap, offset: number, - parentIndent: string | undefined, + parentIndent: string | null, verbatim?: boolean, ): string { - return printBlock(this, info, offset, parentIndent, verbatim) + return super.printSubtree(info, offset, parentIndent, verbatim) } } /** TODO: Add docs */ -export class MutableBodyBlock extends BodyBlock implements MutableAst { +export class MutableBodyBlock extends BodyBlock implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap @@ -2439,62 +2730,79 @@ export class MutableBodyBlock extends BodyBlock implements MutableAst { ) } + /** Remove statements matching the given predicate from the block and return them. */ + extractIf(predicate: (statement: Statement) => boolean): OwnedBlockLine[] { + const extracted = new Array() + this.updateLines(lines => { + const remaining: OwnedBlockLine[] = [] + for (const line of lines) { + const ast = line.statement?.node + if (!ast) continue + ;(predicate(ast) ? extracted : remaining).push(line) + } + return remaining + }) + return extracted + } + /** Insert the given statement(s) starting at the specified line index. */ - insert(index: number, ...statements: (Owned | undefined)[]) { + insert(index: number, ...statements: (Owned | undefined)[]) { const before = this.fields.get('lines').slice(0, index) const insertions = statements.map(statement => ({ newline: unspaced(Token.new('\n', RawAst.Token.Type.Newline)), - expression: statement && unspaced(this.claimChild(statement)), + statement: statement && unspaced(this.claimChild(statement)), })) const after = this.fields.get('lines').slice(index) this.fields.set('lines', [...before, ...insertions, ...after]) } - push(statement: Owned) { + push(statementOrExpression: Owned | Owned) { const oldLines = this.fields.get('lines') + const statement = toStatement(statementOrExpression, this.module) const newLine = { newline: unspaced(Token.new('\n', RawAst.Token.Type.Newline)), - expression: unspaced(this.claimChild(statement)), + statement: unspaced(this.claimChild(statement)), } this.fields.set('lines', [...oldLines, newLine]) } - filter(keep: (ast: MutableAst) => boolean) { + filter(keep: (ast: MutableStatement) => boolean) { const oldLines = this.fields.get('lines') const filteredLines = oldLines.filter(line => { - if (!line.expression) return true - return keep(this.module.get(line.expression.node)) + if (!line.statement) return true + return keep(this.module.get(line.statement.node) as MutableStatement) }) this.fields.set('lines', filteredLines) } } -export interface MutableBodyBlock extends BodyBlock, MutableAst { - statements(): IterableIterator +export interface MutableBodyBlock extends BodyBlock, MutableExpression { + statements(): IterableIterator } applyMixins(MutableBodyBlock, [MutableAst]) interface RawLine { newline: T['token'] - expression: T['ast'] | undefined + statement: T['statement'] | undefined } interface Line { newline?: T['token'] | undefined - expression: T['ast'] | undefined + statement: T['statement'] | undefined } type RawBlockLine = RawLine export type BlockLine = Line export type OwnedBlockLine = Line +export type MutableBlockLine = Line function lineFromRaw(raw: RawBlockLine, module: Module): BlockLine { - const expression = raw.expression ? module.get(raw.expression.node) : undefined + const expression = raw.statement ? (module.get(raw.statement.node) as Statement) : undefined return { newline: { ...raw.newline, node: module.getToken(raw.newline.node) }, - expression: + statement: expression ? { - whitespace: raw.expression?.whitespace, + whitespace: raw.statement?.whitespace, node: expression, } : undefined, @@ -2502,13 +2810,16 @@ function lineFromRaw(raw: RawBlockLine, module: Module): BlockLine { } function ownedLineFromRaw(raw: RawBlockLine, module: MutableModule): OwnedBlockLine { - const expression = raw.expression ? module.get(raw.expression.node).takeIfParented() : undefined + const expression = + raw.statement ? + (module.get(raw.statement.node).takeIfParented() as Owned) + : undefined return { newline: { ...raw.newline, node: module.getToken(raw.newline.node) }, - expression: + statement: expression ? { - whitespace: raw.expression?.whitespace, + whitespace: raw.statement?.whitespace, node: expression, } : undefined, @@ -2518,11 +2829,11 @@ function ownedLineFromRaw(raw: RawBlockLine, module: MutableModule): OwnedBlockL function lineToRaw(line: OwnedBlockLine, module: MutableModule, block: AstId): RawBlockLine { return { newline: line.newline ?? unspaced(Token.new('\n', RawAst.Token.Type.Newline)), - expression: - line.expression ? + statement: + line.statement ? { - whitespace: line.expression?.whitespace, - node: claimChild(module, line.expression.node, block), + whitespace: line.statement?.whitespace, + node: claimChild(module, line.statement.node, block), } : undefined, } @@ -2532,7 +2843,7 @@ interface IdentFields { token: NodeChild } /** TODO: Add docs */ -export class Ident extends Ast { +export class Ident extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -2541,7 +2852,7 @@ export class Ident extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableIdent) return parsed } @@ -2573,8 +2884,8 @@ export class Ident extends Ast { } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { - yield this.fields.get('token') + *concreteChildren(_printContext: PrintContext): IterableIterator { + yield firstChild(this.fields.get('token')) } /** TODO: Add docs */ @@ -2583,7 +2894,7 @@ export class Ident extends Ast { } } /** TODO: Add docs */ -export class MutableIdent extends Ident implements MutableAst { +export class MutableIdent extends Ident implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap @@ -2595,14 +2906,14 @@ export class MutableIdent extends Ident implements MutableAst { return this.token.code() } } -export interface MutableIdent extends Ident, MutableAst {} +export interface MutableIdent extends Ident, MutableExpression {} applyMixins(MutableIdent, [MutableAst]) interface WildcardFields { token: NodeChild } /** TODO: Add docs */ -export class Wildcard extends Ast { +export class Wildcard extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -2611,7 +2922,7 @@ export class Wildcard extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableWildcard) return parsed } @@ -2634,17 +2945,17 @@ export class Wildcard extends Ast { } /** TODO: Add docs */ - *concreteChildren(_verbatim?: boolean): IterableIterator { - yield this.fields.get('token') + *concreteChildren(_printContext: PrintContext): IterableIterator { + yield firstChild(this.fields.get('token')) } } /** TODO: Add docs */ -export class MutableWildcard extends Wildcard implements MutableAst { +export class MutableWildcard extends Wildcard implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap } -export interface MutableWildcard extends Wildcard, MutableAst {} +export interface MutableWildcard extends Wildcard, MutableExpression {} applyMixins(MutableWildcard, [MutableAst]) type AbstractVectorElement = { @@ -2664,7 +2975,7 @@ interface VectorFields { close: NodeChild } /** TODO: Add docs */ -export class Vector extends Ast { +export class Vector extends BaseExpression { declare fields: FixedMapView /** TODO: Add docs */ constructor(module: Module, fields: FixedMapView) { @@ -2673,7 +2984,7 @@ export class Vector extends Ast { /** TODO: Add docs */ static tryParse(source: string, module?: MutableModule): Owned | undefined { - const parsed = parse(source, module) + const parsed = parseExpression(source, module) if (parsed instanceof MutableVector) return parsed } @@ -2742,7 +3053,7 @@ export class Vector extends Ast { } /** TODO: Add docs */ - *concreteChildren(verbatim?: boolean): IterableIterator { + *concreteChildren({ verbatim }: PrintContext): IterableIterator { const { open, elements, close } = getAll(this.fields) yield ensureUnspaced(open, verbatim) let isFirst = true @@ -2759,15 +3070,15 @@ export class Vector extends Ast { } /** TODO: Add docs */ - *values(): IterableIterator { + *values(): IterableIterator { for (const element of this.fields.get('elements')) - if (element.value) yield this.module.get(element.value.node) + if (element.value) yield this.module.get(element.value.node) as Expression } /** TODO: Add docs */ - *enumerate(): IterableIterator<[number, Ast | undefined]> { + *enumerate(): IterableIterator<[number, Expression | undefined]> { for (const [index, element] of this.fields.get('elements').entries()) { - yield [index, this.module.get(element.value?.node)] + yield [index, this.module.get(element.value?.node) as Expression] } } @@ -2777,7 +3088,7 @@ export class Vector extends Ast { } } /** TODO: Add docs */ -export class MutableVector extends Vector implements MutableAst { +export class MutableVector extends Vector implements MutableExpression { declare readonly module: MutableModule declare readonly fields: FixedMap @@ -2800,7 +3111,7 @@ export class MutableVector extends Vector implements MutableAst { } } - set(index: number, value: Owned) { + set(index: number, value: Owned) { const elements = [...this.fields.get('elements')] elements[index] = { delimiter: elements[index]!.delimiter, @@ -2836,12 +3147,13 @@ export class MutableVector extends Vector implements MutableAst { } } - keep(predicate: (ast: Ast) => boolean) { + keep(predicate: (ast: Expression) => boolean) { const elements = this.fields.get('elements') // Spacing around opening brackets should be preserved, as it's more natural (see tests). const firstSpacing = elements[0]?.value?.whitespace const filtered = elements.filter( - element => element.value && predicate(this.module.get(element.value.node)), + element => + element.value && predicate(this.module.get(element.value.node) as MutableExpression), ) if (firstSpacing != null && filtered[0]?.value != null) { filtered[0].value.whitespace = firstSpacing @@ -2863,8 +3175,8 @@ export class MutableVector extends Vector implements MutableAst { } } } -export interface MutableVector extends Vector, MutableAst { - values(): IterableIterator +export interface MutableVector extends Vector, MutableExpression { + values(): IterableIterator } applyMixins(MutableVector, [MutableAst]) @@ -2872,7 +3184,7 @@ export type Mutable = T extends App ? MutableApp : T extends Assignment ? MutableAssignment : T extends BodyBlock ? MutableBodyBlock - : T extends Documented ? MutableDocumented + : T extends ExpressionStatement ? MutableExpressionStatement : T extends Function ? MutableFunction : T extends Generic ? MutableGeneric : T extends Group ? MutableGroup @@ -2887,6 +3199,8 @@ export type Mutable = : T extends UnaryOprApp ? MutableUnaryOprApp : T extends Vector ? MutableVector : T extends Wildcard ? MutableWildcard + : T extends Statement ? MutableStatement + : T extends Expression ? MutableExpression : MutableAst /** TODO: Add docs */ @@ -2900,8 +3214,8 @@ export function materializeMutable(module: MutableModule, fields: FixedMap): As return new Assignment(module, fields_) case 'BodyBlock': return new BodyBlock(module, fields_) - case 'Documented': - return new Documented(module, fields_) + case 'ExpressionStatement': + return new ExpressionStatement(module, fields_) case 'Function': return new Function(module, fields_) case 'Generic': diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/ErrorCompilerTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/ErrorCompilerTest.java index f41d853e26..fa6cde35a9 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/ErrorCompilerTest.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/ErrorCompilerTest.java @@ -462,7 +462,7 @@ public class ErrorCompilerTest extends CompilerTests { public void illegalForeignBody4() throws Exception { var ir = parse("foreign js foo = 4"); assertSingleSyntaxError( - ir, Syntax.UnexpectedExpression$.MODULE$, "Unexpected expression", 0, 18); + ir, new Syntax.InvalidForeignDefinition("Expected text literal as body"), null, 0, 18); } @Test diff --git a/engine/runtime-integration-tests/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala b/engine/runtime-integration-tests/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala index 4375c599e3..663bd5492b 100644 --- a/engine/runtime-integration-tests/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala +++ b/engine/runtime-integration-tests/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala @@ -2275,7 +2275,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers { | |## My sweet type |type Mtp - | ## My sweet type + | ## My sweet value | MyType a b""".stripMargin val module = code.preprocessModule() @@ -2306,7 +2306,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers { .Argument("b", SuggestionBuilder.Any, false, false, None) ), returnType = "Test.Mtp", - documentation = Some(" My sweet type"), + documentation = Some(" My sweet value"), annotations = Seq() ), Vector() diff --git a/engine/runtime-parser/src/main/java/org/enso/compiler/core/TreeToIr.java b/engine/runtime-parser/src/main/java/org/enso/compiler/core/TreeToIr.java index 14c473067c..6e791b94e1 100644 --- a/engine/runtime-parser/src/main/java/org/enso/compiler/core/TreeToIr.java +++ b/engine/runtime-parser/src/main/java/org/enso/compiler/core/TreeToIr.java @@ -37,6 +37,7 @@ import org.enso.compiler.core.ir.module.scope.imports.Polyglot; import org.enso.syntax2.ArgumentDefinition; import org.enso.syntax2.Base; import org.enso.syntax2.DocComment; +import org.enso.syntax2.DocLine; import org.enso.syntax2.FunctionAnnotation; import org.enso.syntax2.Line; import org.enso.syntax2.Parser; @@ -170,21 +171,18 @@ final class TreeToIr { List diag = nil(); for (Line line : module.getStatements()) { var expr = line.getExpression(); - // Documentation found among imports/exports or at the top of the module (if it starts with imports) is - // placed in `bindings` because that's what the Scala parser used to do. - while (expr instanceof Tree.Documented doc) { - Definition c; - try { - c = translateComment(doc, doc.getDocumentation()); - } catch (SyntaxException ex) { - c = ex.toError(); - } - bindings = join(c, bindings); - expr = doc.getExpression(); - } switch (expr) { case Tree.Import imp -> imports = join(translateImport(imp), imports); case Tree.Export exp -> exports = join(translateExport(exp), exports); + case Tree.Documentation docs -> { + Definition docsIr; + try { + docsIr = translateComment(docs, docs.getDocs()); + } catch (SyntaxException ex) { + docsIr = ex.toError(); + } + bindings = join(docsIr, bindings); + } case Tree.Private priv -> { if (isPrivate) { var error = translateSyntaxError(priv, Syntax.UnexpectedExpression$.MODULE$); @@ -245,22 +243,12 @@ final class TreeToIr { case Tree.Function fn -> translateMethodBinding(fn, appendTo); - case Tree.ForeignFunction fn when fn.getBody() instanceof Tree.TextLiteral body -> { + case Tree.ForeignFunction fn -> { var name = fn.getName(); var nameLoc = getIdentifiedLocation(name); var methodRef = new Name.MethodReference(Option.empty(), buildName(name), nameLoc, meta()); var args = translateArgumentsDefinition(fn.getArgs()); - var languageName = fn.getLanguage().codeRepr(); - var language = languageName; - if (language == null) { - var message = "Language '" + languageName + "' is not a supported polyglot language."; - var error = translateSyntaxError(inputAst, new Syntax.InvalidForeignDefinition(message)); - yield join(error, appendTo); - } - var text = buildTextConstant(body, body.getElements()); - var def = - new Foreign.Definition(language, text, getIdentifiedLocation(fn.getBody()), meta()); - // Foreign functions are always considered private + var def = translateForeignFunction(fn); var binding = new Method.Binding( methodRef, args, true, def, getIdentifiedLocation(inputAst), meta()); yield join(binding, appendTo); @@ -272,11 +260,14 @@ final class TreeToIr { yield translateModuleSymbol(anno.getExpression(), join(annotation, appendTo)); } - case Tree.Documented doc -> { - var comment = translateComment(doc, doc.getDocumentation()); - yield translateModuleSymbol(doc.getExpression(), join(comment, appendTo)); + case Tree.ExpressionStatement statement -> { + if (statement.getDocLine() instanceof DocLine docLine) + appendTo = join(translateComment(statement, docLine.getDocs()), appendTo); + yield translateModuleSymbol(statement.getExpression(), appendTo); } + case Tree.Documentation doc -> join(translateComment(doc, doc.getDocs()), appendTo); + case Tree.TypeSignatureDeclaration sig -> { var ascription = translateMethodTypeSignature(sig.getSignature()); yield join(ascription, appendTo); @@ -289,6 +280,22 @@ final class TreeToIr { }; } + private Expression translateForeignFunction(Tree.ForeignFunction fn) throws SyntaxException { + var languageName = fn.getLanguage().codeRepr(); + var language = languageName; + if (language == null) { + var message = "Language '" + languageName + "' is not a supported polyglot language."; + return translateSyntaxError(fn, new Syntax.InvalidForeignDefinition(message)); + } + String text; + if (fn.getBody() instanceof Tree.TextLiteral body) { + text = buildTextConstant(body, body.getElements()); + } else { + return translateSyntaxError(fn, new Syntax.InvalidForeignDefinition("Expected text literal as body")); + } + return new Foreign.Definition(language, text, getIdentifiedLocation(fn.getBody()), meta()); + } + private List translateArgumentsDefinition( java.util.List args) throws SyntaxException { List res = nil(); @@ -300,6 +307,15 @@ final class TreeToIr { } List translateConstructorDefinition(Tree.ConstructorDefinition cons, List appendTo) { + if (cons.getDocLine() instanceof DocLine docLine) { + Definition docsIr; + try { + docsIr = translateComment(cons, docLine.getDocs()); + } catch (SyntaxException ex) { + docsIr = ex.toError(); + } + appendTo = join(docsIr, appendTo); + } for (var annoLine : cons.getAnnotationLines()) { appendTo = join(translateAnnotation(annoLine.getAnnotation()), appendTo); } @@ -360,26 +376,21 @@ final class TreeToIr { case Tree.Function fun -> translateTypeMethodBinding(fun, appendTo); - case Tree.ForeignFunction fn when fn.getBody() instanceof Tree.TextLiteral body -> { + case Tree.ForeignFunction fn -> { var name = buildName(fn.getName()); var args = translateArgumentsDefinition(fn.getArgs()); - var languageName = fn.getLanguage().codeRepr(); - var language = languageName; - if (language == null) { - var message = "Language '" + languageName + "' is not a supported polyglot language."; - var error = translateSyntaxError(inputAst, new Syntax.InvalidForeignDefinition(message)); - yield join(error, appendTo); - } - var text = buildTextConstant(body, body.getElements()); - var def = - new Foreign.Definition(language, text, getIdentifiedLocation(fn.getBody()), meta()); + var def = translateForeignFunction(fn); var binding = new Function.Binding(name, args, def, false, getIdentifiedLocation(fn), true, meta()); yield join(binding, appendTo); } - case Tree.Documented doc -> { - var irDoc = translateComment(doc, doc.getDocumentation()); - yield translateTypeBodyExpression(doc.getExpression(), join(irDoc, appendTo)); + + case Tree.Documentation docs -> join(translateComment(docs, docs.getDocs()), appendTo); + + case Tree.ExpressionStatement statement -> { + if (statement.getDocLine() instanceof DocLine docLine) + appendTo = join(translateComment(statement, docLine.getDocs()), appendTo); + yield translateTypeBodyExpression(statement.getExpression(), appendTo); } case Tree.AnnotatedBuiltin anno -> { @@ -459,6 +470,9 @@ final class TreeToIr { private List translateMethodBinding(Tree.Function fn, List appendTo) throws SyntaxException { + if (fn.getDocLine() instanceof DocLine docLine) { + appendTo = join(translateComment(fn, docLine.getDocs()), appendTo); + } for (var annoLine : fn.getAnnotationLines()) { appendTo = join(translateAnnotation(annoLine.getAnnotation()), appendTo); } @@ -494,6 +508,15 @@ final class TreeToIr { } private List translateTypeMethodBinding(Tree.Function fun, List appendTo) { + if (fun.getDocLine() instanceof DocLine docLine) { + IR comment; + try { + comment = translateComment(fun, docLine.getDocs()); + } catch (SyntaxException ex) { + comment = ex.toError(); + } + appendTo = join(comment, appendTo); + } for (var annoLine : fun.getAnnotationLines()) { appendTo = join(translateAnnotation(annoLine.getAnnotation()), appendTo); } @@ -972,8 +995,8 @@ final class TreeToIr { continue; } var branch = line.getCase(); - if (branch.getDocumentation() != null) { - var comment = translateComment(cas, branch.getDocumentation()); + if (branch.getDocLine() instanceof DocLine docLine) { + var comment = translateComment(cas, docLine.getDocs()); var loc = getIdentifiedLocation(cas); var doc = new Pattern.Documentation(comment.doc(), loc, meta()); var br = new Case.Branch( @@ -1023,9 +1046,7 @@ final class TreeToIr { getIdentifiedLocation(anno), meta()); yield translateBuiltinAnnotation(ir, anno.getExpression(), nil()); } - // Documentation can be attached to an expression in a few cases, like if someone documents a line of an - // `ArgumentBlockApplication`. The documentation is ignored. - case Tree.Documented docu -> translateExpression(docu.getExpression()); + case Tree.App app -> { var fn = translateExpression(app.getFunc(), isMethod); var loc = getIdentifiedLocation(app); @@ -1099,10 +1120,28 @@ final class TreeToIr { private void translateBlockStatement(Tree tree, Collection appendTo) { switch (tree) { case null -> {} - case Tree.Assignment assign -> { - appendTo.add(translateAssignment(assign)); + case Tree.Assignment assignment -> { + if (assignment.getDocLine() instanceof DocLine docLine) { + Expression docsIr; + try { + docsIr = translateComment(assignment, docLine.getDocs()); + } catch (SyntaxException ex) { + docsIr = ex.toError(); + } + appendTo.add(docsIr); + } + appendTo.add(translateAssignment(assignment)); } case Tree.Function fun -> { + if (fun.getDocLine() instanceof DocLine docLine) { + Expression comment; + try { + comment = translateComment(fun, docLine.getDocs()); + } catch (SyntaxException ex) { + comment = ex.toError(); + } + appendTo.add(comment); + } for (var annoLine : fun.getAnnotationLines()) { appendTo.add((Expression)translateAnnotation(annoLine.getAnnotation())); } @@ -1114,15 +1153,26 @@ final class TreeToIr { case Tree.TypeSignatureDeclaration sig -> { appendTo.add(translateTypeSignatureToOprApp(sig.getSignature())); } - case Tree.Documented doc -> { + case Tree.Documentation docs -> { Expression ir; try { - ir = translateComment(doc, doc.getDocumentation()); + ir = translateComment(docs, docs.getDocs()); } catch (SyntaxException ex) { ir = ex.toError(); } appendTo.add(ir); - translateBlockStatement(doc.getExpression(), appendTo); + } + case Tree.ExpressionStatement statement -> { + if (statement.getDocLine() instanceof DocLine docLine) { + Expression ir; + try { + ir = translateComment(statement, docLine.getDocs()); + } catch (SyntaxException ex) { + ir = ex.toError(); + } + appendTo.add(ir); + } + translateBlockStatement(statement.getExpression(), appendTo); } default -> { var expressionStatement = translateExpression(tree); @@ -1203,7 +1253,7 @@ final class TreeToIr { case Tree.OprSectionBoundary section -> section.getAst(); case Tree.TemplateFunction function -> function.getAst(); case Tree.AnnotatedBuiltin annotated -> annotated.getExpression(); - case Tree.Documented documented -> documented.getExpression(); + case Tree.ExpressionStatement statement -> statement.getExpression(); case Tree.Assignment assignment -> assignment.getExpr(); case Tree.TypeAnnotated annotated -> annotated.getExpression(); case Tree.App app when isApplication(app.getFunc()) -> app.getFunc(); @@ -1217,6 +1267,7 @@ final class TreeToIr { case Tree.CaseOf ignored -> null; case Tree.Array ignored -> null; case Tree.Tuple ignored -> null; + case Tree.Documentation ignored -> null; default -> null; }; } diff --git a/eslint.config.mjs b/eslint.config.mjs index b4c9f9feb6..8c2a4343fc 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -233,7 +233,9 @@ export default [ }, ], '@typescript-eslint/no-namespace': 'off', - '@typescript-eslint/no-empty-object-type': ['error'], + // Empty interfaces have valid uses; e.g. although an empty interface extending a class is semantically equivalent + // to a type alias, it is not resolved by IDEs to the base type (which may be internal). + '@typescript-eslint/no-empty-object-type': ['error', { allowInterfaces: 'always' }], 'no-unused-labels': 'off', // Taken care of by prettier 'vue/max-attributes-per-line': 'off', diff --git a/lib/rust/parser/debug/src/lib.rs b/lib/rust/parser/debug/src/lib.rs index a00d750b5a..3c55e29423 100644 --- a/lib/rust/parser/debug/src/lib.rs +++ b/lib/rust/parser/debug/src/lib.rs @@ -95,15 +95,29 @@ where T: serde::Serialize + Reflect { let (car, _) = cons.into_pair(); Value::cons(car, Value::Null) }; + let simplify_tree = |list: Value| { + let list = strip_hidden_fields(list); + let vec = list.to_vec().unwrap(); + if vec[0].as_symbol().unwrap() == "ExpressionStatement" { + match &vec[1..] { + [Value::Cons(doc_line), Value::Cons(expr)] if doc_line.cdr().is_null() => + return expr.cdr().to_owned(), + _ => {} + } + }; + list + }; let line = rust_to_meta[&tree::block::Line::reflect().id]; let operator_line = rust_to_meta[&tree::block::OperatorLine::reflect().id]; let type_signature_line = rust_to_meta[&tree::TypeSignatureLine::reflect().id]; let invalid = rust_to_meta[&tree::Invalid::reflect().id]; + let tree = rust_to_meta[&tree::Tree::reflect().id]; to_s_expr.mapper(line, into_car); to_s_expr.mapper(operator_line, into_car); to_s_expr.mapper(type_signature_line, into_car); to_s_expr.mapper(invalid, strip_invalid); to_s_expr.mapper(text_escape_token, simplify_escape); + to_s_expr.mapper(tree, simplify_tree); tuplify(to_s_expr.value(ast_ty, &value)) } diff --git a/lib/rust/parser/debug/tests/parse.rs b/lib/rust/parser/debug/tests/parse.rs index c470756576..2dcbd876d9 100644 --- a/lib/rust/parser/debug/tests/parse.rs +++ b/lib/rust/parser/debug/tests/parse.rs @@ -156,31 +156,44 @@ fn plain_comments() { } #[test] -fn doc_comments() { - let lines = [ - "## The Identity Function", - "", - " Arguments:", - " - x: value to do nothing to", - "id x = x", - ]; - test!(lines.join("\n"), - (Documented - (#((Section " The Identity Function") (Newline) - (Newline) - (Section "Arguments:") (Newline) - (Section "- x: value to do nothing to")) - #(())) - ,(Function::new("id", sexp![(Ident x)]).with_arg("x")))); - let lines = ["type Foo", " ## Test indent handling", " ", " foo bar = foo"]; - test!(&lines.join("\n"), +fn function_documentation() { + test!([ + "## The Identity Function", + "", + " Arguments:", + " - x: value to do nothing to", + "id x = x", + ].join("\n"), + ,(Function::new("id", sexp![(Ident x)]) + .with_docs(sexp![ + ((#((Section " The Identity Function") (Newline) (Newline) + (Section "Arguments:") (Newline) + (Section "- x: value to do nothing to"))) + #(()))]) + .with_arg("x"))); + test!(&["type Foo", " ## Test indent handling", " ", " foo bar = foo"].join("\n"), (TypeDef Foo #() #( - (Documented - (#((Section " Test indent handling")) #(() ())) - ,(Function::new("foo", sexp![(Ident foo)]).with_arg("bar")))))); + ,(Function::new("foo", sexp![(Ident foo)]) + .with_docs(sexp![((#((Section " Test indent handling"))) #(() ()))]) + .with_arg("bar"))))); expect_invalid_node("expression ## unexpected doc comment on same line"); } +#[test] +fn expression_documentation() { + test_block!("## The value of x\nx", + (ExpressionStatement ((#((Section " The value of x"))) #(())) (Ident x))); +} + +#[test] +fn unused_documentation() { + test!("## First docs\n## More docs\n\n## More docs after a gap", + (Documentation (#((Section " First docs")))) + (Documentation (#((Section " More docs")))) + () + (Documentation (#((Section " More docs after a gap"))))); +} + // === Type Definitions === @@ -215,14 +228,17 @@ fn type_constructors() { ,(Constructor::new("Bar") .with_arg(sexp![(() (Ident a) (":" (Ident B)) ((OprApp (Ident C) (Ok ".") (Ident D))))])) ))); - test!("type Foo\n ## Bar\n Baz", (TypeDef Foo #() #( - (Documented (#((Section " Bar")) #(())) - ,(Constructor::new("Baz")))))); test!(["type A", " Foo (a : Integer, b : Integer)"].join("\n"), (TypeDef A #() #(,(Constructor::new("Foo").with_arg(sexp![(() (Ident a) (":" (Invalid)) ())]))))); } +#[test] +fn type_constructor_documentation() { + test!("type Foo\n ## Bar\n Baz", (TypeDef Foo #() #( + ,(Constructor::new("Baz").with_docs(sexp![((#((Section " Bar"))) #(()))]))))); +} + #[test] fn type_constructor_private() { test!(["type Foo", " private Bar"].join("\n"), @@ -257,20 +273,18 @@ fn type_methods() { #(,(Function::new("number", block![(Ident x)])) ,(Function::new("area", sexp![(OprApp (Ident x) (Ok "+") (Ident x))]) .with_arg("self"))))); - let code = [ - "type Problem_Builder", - " ## Returns a vector containing all reported problems, aggregated.", - " build_problemset : Vector", - " build_problemset self =", - " self", - ]; - test!(code.join("\n"), + test!([ + "type Problem_Builder", + " ## Returns a vector containing all reported problems, aggregated.", + " build_problemset : Vector", + " build_problemset self =", + " self", + ].join("\n"), (TypeDef Problem_Builder #() #( - (Documented - (#((Section " Returns a vector containing all reported problems, aggregated.")) #(())) - ,(Function::new("build_problemset", block![(Ident self)]) - .with_sig(sexp![(Ident Vector)]) - .with_arg("self")))))); + ,(Function::new("build_problemset", block![(Ident self)]) + .with_docs(sexp![((#((Section " Returns a vector containing all reported problems, aggregated."))) #(()))]) + .with_sig(sexp![(Ident Vector)]) + .with_arg("self"))))); test!("[foo., bar.]", (Array (OprSectionBoundary 1 (OprApp (Ident foo) (Ok ".") ())) #(("," (OprSectionBoundary 1 (OprApp (Ident bar) (Ok ".") ())))))); @@ -358,12 +372,19 @@ fn assignment_simple() { test!("foo = x", ,(Function::new("foo", sexp![(Ident x)]))); // In a body block, this is a variable binding. test_block!("main =\n foo = x", - ,(Function::new("main", block![(Assignment (Ident foo) (Ident x))]))); - test_block!("foo=x", (Assignment (Ident foo) (Ident x))); - test_block!("foo= x", (Assignment (Ident foo) (Ident x))); + ,(Function::new("main", block![,(Assignment::new("foo", sexp![(Ident x)]))]))); + test_block!("foo=x", ,(Assignment::new("foo", sexp![(Ident x)]))); + test_block!("foo= x", ,(Assignment::new("foo", sexp![(Ident x)]))); expect_invalid_node("foo =x"); } +#[test] +fn assignment_documentation() { + test_block!("## The Foo\nfoo = x", + ,(Assignment::new("foo", sexp![(Ident x)]) + .with_docs(sexp![((#((Section " The Foo"))) #(()))]))); +} + // === Functions === @@ -563,13 +584,13 @@ fn code_block_body() { #[test] fn code_block_operator() { - let code = ["value = nums", " * each random", " + constant"]; - test_block!(code.join("\n"), - (Assignment (Ident value) - (OperatorBlockApplication (Ident nums) - #(((Ok "*") (App (Ident each) (Ident random))) - ((Ok "+") (Ident constant))) - #()))); + test_block!(["value = nums", " * each random", " + constant"].join("\n"), + ,(Assignment::new("value", sexp![ + (OperatorBlockApplication (Ident nums) + #(((Ok "*") (App (Ident each) (Ident random))) + ((Ok "+") (Ident constant))) + #()) + ]))); } #[test] @@ -588,14 +609,14 @@ fn code_block_argument_list() { test!("foo\n bar", (ArgumentBlockApplication (Ident foo) #((Ident bar)))); test_block!("value = foo\n bar", - (Assignment (Ident value) (ArgumentBlockApplication (Ident foo) #((Ident bar))))); + ,(Assignment::new("value", sexp![(ArgumentBlockApplication (Ident foo) #((Ident bar)))]))); let code = ["value = foo", " +x", " bar"]; test_block!(code.join("\n"), - (Assignment (Ident value) - (ArgumentBlockApplication (Ident foo) #( - (OprSectionBoundary 1 (OprApp () (Ok "+") (Ident x))) - (Ident bar))))); + ,(Assignment::new("value", sexp![ + (ArgumentBlockApplication (Ident foo) #( + (OprSectionBoundary 1 (OprApp () (Ok "+") (Ident x))) + (Ident bar)))]))); } #[test] @@ -733,8 +754,8 @@ fn operator_sections() { (OprSectionBoundary 1 (OprApp (OprApp () (Ok "+") (Number () "1" ())) (Ok "+") (Ident x)))); test_block!("increment = 1 +", - (Assignment (Ident increment) - (OprSectionBoundary 1 (OprApp (Number () "1" ()) (Ok "+") ())))); + ,(Assignment::new("increment", sexp![ + (OprSectionBoundary 1 (OprApp (Number () "1" ()) (Ok "+") ()))]))); test!("1+ << 2*", (OprSectionBoundary 1 (OprApp (OprApp (Number () "1" ()) (Ok "+") ()) @@ -805,17 +826,18 @@ fn unary_operator_at_end_of_expression() { fn unspaced_operator_sequence() { // Add a negated value. test_block!("x = y+-z", - (Assignment (Ident x) (OprApp (Ident y) (Ok "+") (UnaryOprApp "-" (Ident z))))); + ,(Assignment::new("x", sexp![(OprApp (Ident y) (Ok "+") (UnaryOprApp "-" (Ident z)))]))); // Create an operator section that adds a negated value to its input. test_block!("x = +-z", - (Assignment (Ident x) (OprSectionBoundary 1 - (OprApp () (Ok "+") (UnaryOprApp "-" (Ident z)))))); + ,(Assignment::new("x", sexp![ + (OprSectionBoundary 1 + (OprApp () (Ok "+") (UnaryOprApp "-" (Ident z))))]))); // The `-` can only be lexed as a unary operator, and unary operators cannot form sections. expect_invalid_node("main =\n x = y+-"); // Assign a negative number to x. - test_block!("x=-1", (Assignment (Ident x) (UnaryOprApp "-" (Number () "1" ())))); + test_block!("x=-1", ,(Assignment::new("x", sexp![(UnaryOprApp "-" (Number () "1" ()))]))); // Assign a negated value to x. - test_block!("x=-y", (Assignment (Ident x) (UnaryOprApp "-" (Ident y)))); + test_block!("x=-y", ,(Assignment::new("x", sexp![(UnaryOprApp "-" (Ident y))]))); } #[test] @@ -844,7 +866,7 @@ fn minus_unary() { test!("-x", (UnaryOprApp "-" (Ident x))); test!("(-x)", (Group (UnaryOprApp "-" (Ident x)))); test!("-(x * x)", (UnaryOprApp "-" (Group (OprApp (Ident x) (Ok "*") (Ident x))))); - test_block!("x=-x", (Assignment (Ident x) (UnaryOprApp "-" (Ident x)))); + test_block!("x=-x", ,(Assignment::new("x", sexp![(UnaryOprApp "-" (Ident x))]))); test!("-x+x", (OprApp (UnaryOprApp "-" (Ident x)) (Ok "+") (Ident x))); test!("-x*x", (OprApp (UnaryOprApp "-" (Ident x)) (Ok "*") (Ident x))); } @@ -871,9 +893,9 @@ fn method_app_in_minus_unary() { #[test] fn autoscope_operator() { test!("x : ..True", (TypeSignatureDeclaration ((Ident x) ":" (AutoscopedIdentifier ".." True)))); - test_block!("x = ..True", (Assignment (Ident x) (AutoscopedIdentifier ".." True))); + test_block!("x = ..True", ,(Assignment::new("x", sexp![(AutoscopedIdentifier ".." True)]))); test_block!("x = f ..True", - (Assignment (Ident x) (App (Ident f) (AutoscopedIdentifier ".." True)))); + ,(Assignment::new("x", sexp![(App (Ident f) (AutoscopedIdentifier ".." True))]))); expect_invalid_node("x = ..not_a_constructor"); expect_invalid_node("x = case a of ..True -> True"); expect_invalid_node("x = ..4"); @@ -1002,6 +1024,12 @@ fn type_signatures() { test!("val : Bool\nval = True", ,(Function::new("val", sexp![(Ident True)]) .with_sig(sexp![(Ident Bool)]))); + test!("val : Bool\n\nval = True", + ,(Function::new("val", sexp![(Ident True)]) + .with_sig(sexp![(Ident Bool)]))); + test!("val : Bool\n\n\nval = True", + ,(Function::new("val", sexp![(Ident True)]) + .with_sig(sexp![(Ident Bool)]))); test!("val : Bool\ndifferent_name = True", (TypeSignatureDeclaration ((Ident val) ":" (Ident Bool))) ,(Function::new("different_name", sexp![(Ident True)]))); @@ -1024,11 +1052,11 @@ fn type_signatures() { #[test] fn type_annotations() { test_block!("val = x : Int", - (Assignment (Ident val) (TypeAnnotated (Ident x) ":" (Ident Int)))); + ,(Assignment::new("val", sexp![(TypeAnnotated (Ident x) ":" (Ident Int))]))); test_block!("val = foo (x : Int)", - (Assignment (Ident val) - (App (Ident foo) - (Group (TypeAnnotated (Ident x) ":" (Ident Int)))))); + ,(Assignment::new("val", sexp![ + (App (Ident foo) + (Group (TypeAnnotated (Ident x) ":" (Ident Int))))]))); test!("(x : My_Type _)", (Group (TypeAnnotated (Ident x) @@ -1048,10 +1076,11 @@ fn type_annotations() { #[test] fn inline_text_literals() { test!(r#""I'm an inline raw text!""#, (TextLiteral #((Section "I'm an inline raw text!")))); - test_block!(r#"zero_length = """#, (Assignment (Ident zero_length) (TextLiteral #()))); + test_block!(r#"zero_length = """#, ,(Assignment::new("zero_length", sexp![(TextLiteral #())]))); test!(r#""type""#, (TextLiteral #((Section "type")))); - test_block!(r#"unclosed = ""#, (Assignment (Ident unclosed) (TextLiteral #()))); - test_block!(r#"unclosed = "a"#, (Assignment (Ident unclosed) (TextLiteral #((Section "a"))))); + test_block!(r#"unclosed = ""#, ,(Assignment::new("unclosed", sexp![(TextLiteral #())]))); + test_block!(r#"unclosed = "a"#, + ,(Assignment::new("unclosed", sexp![(TextLiteral #((Section "a")))]))); test!(r#"'Other quote type'"#, (TextLiteral #((Section "Other quote type")))); test!(r#""Non-escape: \n""#, (TextLiteral #((Section "Non-escape: \\n")))); test!(r#""Non-escape: \""#, (TextLiteral #((Section "Non-escape: \\")))); @@ -1093,15 +1122,16 @@ x"#, (TextLiteral #((Section "multiline string that doesn't end in a newline"))) (Ident x)); test_block!("x = \"\"\"\n Indented multiline\nx", - (Assignment (Ident x) (TextLiteral #((Section "Indented multiline")))) + ,(Assignment::new("x", sexp![(TextLiteral #((Section "Indented multiline")))])) (Ident x)); test!("'''\n \\nEscape at start\n", (TextLiteral #((Escape 0x0A) (Section "Escape at start"))) ()); test!("x =\n x = '''\n x\nx", - ,(Function::new("x", block![(Assignment (Ident x) (TextLiteral #((Section "x"))))])) + ,(Function::new("x", block![ + ,(Assignment::new("x", sexp![(TextLiteral #((Section "x")))]))])) (Ident x)); test_block!("foo = bar '''\n baz", - (Assignment (Ident foo) (App (Ident bar) (TextLiteral #((Section "baz")))))); + ,(Assignment::new("foo", sexp![(App (Ident bar) (TextLiteral #((Section "baz"))))]))); test!("'''\n \\t'", (TextLiteral #((Escape 0x09) (Section "'")))); test!("'''\n x\n \\t'", (TextLiteral #((Section "x") (Newline) (Escape 0x09) (Section "'")))); } @@ -1188,10 +1218,10 @@ fn old_lambdas() { (App (Ident f) (OprApp (Ident x) (Ok "->") (BodyBlock #((Ident y)))))); test!("x->y-> z", (OprApp (Ident x) (Ok "->") (OprApp (Ident y) (Ok "->") (Ident z)))); test_block!("foo = x -> (y = bar x) -> x + y", - (Assignment (Ident foo) - (OprApp (Ident x) (Ok "->") - (OprApp (Group (OprApp (Ident y) (Ok "=") (App (Ident bar) (Ident x)))) (Ok "->") - (OprApp (Ident x) (Ok "+") (Ident y)))))); + ,(Assignment::new("foo", sexp![ + (OprApp (Ident x) (Ok "->") + (OprApp (Group (OprApp (Ident y) (Ok "=") (App (Ident bar) (Ident x)))) (Ok "->") + (OprApp (Ident x) (Ok "+") (Ident y))))]))); } @@ -1200,8 +1230,10 @@ fn old_lambdas() { #[test] fn pattern_irrefutable() { test_block!("Point x_val = my_point", - (Assignment (App (Ident Point) (Ident x_val)) (Ident my_point))); - test_block!("Vector _ = x", (Assignment (App (Ident Vector) (Wildcard -1)) (Ident x))); + ,(Assignment::pattern(sexp![(App (Ident Point) (Ident x_val))], + sexp![(Ident my_point)]))); + test_block!("Vector _ = x", + ,(Assignment::pattern(sexp![(App (Ident Vector) (Wildcard -1))], sexp![(Ident x)]))); test_block!("X.y = z", ,(Function::named(sexp![(OprApp (Ident X) (Ok ".") (Ident y))], sexp![(Ident z)]))); } @@ -1261,23 +1293,18 @@ fn case_expression() { #[test] fn case_documentation() { - #[rustfmt::skip] - let code = [ - "case a of", - " ## The Some case", - " Some -> x", - " ## The Int case", - " Int -> x", - ]; - #[rustfmt::skip] - let expected = block![ + test!([ + "case a of", + " ## The Some case", + " Some -> x", + " ## The Int case", + " Int -> x", + ].join("\n"), (CaseOf (Ident a) #( - (((#((Section " The Some case")) #()) () () ())) + ((((#((Section " The Some case"))) #()) () () ())) ((() (Ident Some) "->" (Ident x))) - (((#((Section " The Int case")) #()) () () ())) - ((() (Ident Int) "->" (Ident x))))) - ]; - test(code.join("\n"), expected); + ((((#((Section " The Int case"))) #()) () () ())) + ((() (Ident Int) "->" (Ident x)))))); } #[test] @@ -1313,11 +1340,11 @@ fn suspended_default_arguments_in_pattern() { #[test] fn suspended_default_arguments_in_expression() { test_block!("c = self.value ...", - (Assignment (Ident c) - (App (OprApp (Ident self) (Ok ".") (Ident value)) (SuspendedDefaultArguments)))); + ,(Assignment::new("c", sexp![ + (App (OprApp (Ident self) (Ok ".") (Ident value)) (SuspendedDefaultArguments))]))); test_block!("c = self.value...", - (Assignment (Ident c) - (App (OprApp (Ident self) (Ok ".") (Ident value)) (SuspendedDefaultArguments)))); + ,(Assignment::new("c", sexp![ + (App (OprApp (Ident self) (Ok ".") (Ident value)) (SuspendedDefaultArguments))]))); } // === Private (project-private) keyword === @@ -1403,7 +1430,7 @@ mod numbers { #[test] fn with_decimal() { - test_block!("pi = 3.14", (Assignment (Ident pi) (Number () "3" ("." "14")))); + test_block!("pi = 3.14", ,(Assignment::new("pi", sexp![(Number () "3" ("." "14"))]))); } #[test] @@ -1557,11 +1584,11 @@ fn skip() { #[test] fn statement_in_expression_context() { - test_block!("x = y = z", (Assignment (Ident x) (Invalid))); + test_block!("x = y = z", ,(Assignment::new("x", sexp![(Invalid)]))); test!("(y = z)", (Group(Invalid))); test!("(y = z) x", (App (Group (Invalid)) (Ident x))); test_block!("(f x = x)", (Group(Invalid))); - test_block!("y = f x = x", (Assignment (Ident y) (Invalid))); + test_block!("y = f x = x", ,(Assignment::new("y", sexp![(Invalid)]))); } @@ -1857,6 +1884,7 @@ fn expect_valid(code: &str) { /// Builder for function definitions. struct Function { + docs: lexpr::Value, annotations: Vec, signature: lexpr::Value, private: lexpr::Value, @@ -1874,6 +1902,7 @@ impl Function { fn named(name: lexpr::Value, body: lexpr::Value) -> Self { Self { + docs: sexp![()], annotations: vec![], signature: sexp![()], private: sexp![()], @@ -1884,6 +1913,11 @@ impl Function { } } + #[rustfmt::skip] + fn with_docs(self, docs: lexpr::Value) -> Self { + Self { docs, ..self } + } + #[rustfmt::skip] fn with_annotation(mut self, annotation: &str, arg: lexpr::Value) -> Self { let annotation = lexpr::Value::symbol(annotation); @@ -1913,8 +1947,8 @@ impl Function { impl From for lexpr::Value { #[rustfmt::skip] - fn from(Function { annotations, signature, private, name, args, ret, body }: Function) -> Self { - sexp![(Function ,annotations ,signature ,private ,name ,args ,ret ,body)] + fn from(Function { docs, annotations, signature, private, name, args, ret, body }: Function) -> Self { + sexp![(Function ,docs ,annotations ,signature ,private ,name ,args ,ret ,body)] } } @@ -1943,6 +1977,7 @@ impl From for lexpr::Value { /// Builder for type constructor definitions. struct Constructor { + docs: lexpr::Value, annotations: Vec, private: lexpr::Value, name: lexpr::Value, @@ -1953,6 +1988,7 @@ struct Constructor { impl Constructor { fn new(name: &str) -> Self { Self { + docs: sexp![()], annotations: vec![], private: sexp![()], name: lexpr::Value::symbol(name), @@ -1961,6 +1997,11 @@ impl Constructor { } } + #[rustfmt::skip] + fn with_docs(self, docs: lexpr::Value) -> Self { + Self { docs, ..self } + } + #[rustfmt::skip] fn with_annotation(mut self, annotation: &str, arg: lexpr::Value) -> Self { let annotation = lexpr::Value::symbol(annotation); @@ -1986,7 +2027,39 @@ impl Constructor { impl From for lexpr::Value { #[rustfmt::skip] - fn from(Constructor { annotations, private, name, args, arg_lines }: Constructor) -> Self { - sexp![(ConstructorDefinition ,annotations ,private ,name ,args, arg_lines)] + fn from(Constructor { docs, annotations, private, name, args, arg_lines }: Constructor) -> Self { + sexp![(ConstructorDefinition ,docs ,annotations ,private ,name ,args, arg_lines)] + } +} + +// === Assignments === + +/// Builder for variable assignments. +struct Assignment { + docs: lexpr::Value, + pattern: lexpr::Value, + value: lexpr::Value, +} + +impl Assignment { + fn new(name: &str, body: lexpr::Value) -> Self { + let name = lexpr::Value::symbol(name); + Self::pattern(sexp![(Ident, name)], body) + } + + fn pattern(pattern: lexpr::Value, value: lexpr::Value) -> Self { + Self { docs: sexp![()], pattern, value } + } + + #[rustfmt::skip] + fn with_docs(self, docs: lexpr::Value) -> Self { + Self { docs, ..self } + } +} + +impl From for lexpr::Value { + #[rustfmt::skip] + fn from(Assignment { docs, pattern, value }: Assignment) -> Self { + sexp![(Assignment ,docs ,pattern ,value)] } } diff --git a/lib/rust/parser/doc-parser/src/main.rs b/lib/rust/parser/doc-parser/src/main.rs deleted file mode 100644 index 82f745d914..0000000000 --- a/lib/rust/parser/doc-parser/src/main.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! Prints a debug representation of Enso documentation found in the given Enso source file(s). - -// === Non-Standard Linter Configuration === -#![allow(clippy::option_map_unit_fn)] -#![allow(clippy::precedence)] -#![allow(dead_code)] -#![deny(unconditional_recursion)] -#![warn(missing_docs)] -#![warn(trivial_casts)] -#![warn(unused_qualifications)] - -use enso_doc_parser::*; -use enso_parser::prelude::*; - -use enso_parser::syntax::tree::DocComment; -use enso_parser::syntax::tree::TextElement; - - - -// ==================================== -// === Debug Representation Printer === -// ==================================== - -fn main() { - let args = std::env::args().skip(1); - if args.len() == 0 { - use std::io::Read; - let mut input = String::new(); - std::io::stdin().read_to_string(&mut input).unwrap(); - check_doc_parse("", input.as_str()); - } else { - args.for_each(|path| check_doc_parse(&path, &std::fs::read_to_string(&path).unwrap())); - } -} - -/// Print the token for the input file. -fn check_doc_parse(filename: &str, code: &str) { - println!("File: {filename}"); - let docs = extract_docs(filename, code); - for doc in &docs { - for token in parse(doc) { - println!("{token:?}"); - } - } -} - -/// Extract docs from the input file. -fn extract_docs(_filename: &str, mut code: &str) -> Vec { - if let Some((_meta, code_)) = enso_parser::metadata::parse(code) { - code = code_; - } - let ast = enso_parser::Parser::new().parse_module(code); - let docs = RefCell::new(vec![]); - ast.visit_trees(|tree| match &tree.variant { - enso_parser::syntax::tree::Variant::Documented(doc) => { - docs.borrow_mut().push(doc.documentation.clone()); - } - enso_parser::syntax::tree::Variant::CaseOf(case_of) => { - for case in case_of.cases.iter().filter_map(|c| c.case.as_ref()) { - docs.borrow_mut().extend(case.documentation.clone()); - } - } - _ => {} - }); - docs.take().iter().map(content).collect() -} - -/// Return the contents of the comment, with leading whitespace, the `##` token, and following -/// empty lines removed; newlines will be normalized. -pub fn content(node: &DocComment) -> String { - let mut buf = String::new(); - for element in &node.elements { - match element { - TextElement::Section { text } => buf.push_str(&text.code.repr), - TextElement::Newline { .. } => buf.push('\n'), - TextElement::Escape { - token: - token @ enso_parser::syntax::token::TextEscape { - variant: enso_parser::syntax::token::variant::TextEscape { value }, - .. - }, - } => { - if let Some(c) = value.to_char() { - buf.push(c); - } else { - // Invalid escape character, or unpaired surrogate that can't be represented in - // a Rust string. - buf.push_str(**token.code) - } - } - // Unreachable. - TextElement::Splice { .. } => continue, - } - } - buf -} - -/// Lex the given documentation, and return the sequence of tokens. -fn parse(input: &str) -> Vec { - let mut docs = TokenCollector::::default(); - let mut lexer = Lexer::default(); - for (line_number, line) in input.trim_start().lines().enumerate() { - let location = Location::start_of_line(line_number); - let line = Span { location, text: line }; - lexer.line::(line, &mut docs); - } - lexer.finish(&mut docs); - docs.tokens -} - - - -// ======================= -// === Token Collector === -// ======================= - -/// Token consumer that reifies the sequence of tokens for debugging and tests. -#[derive(Default, Debug)] -struct TokenCollector { - tokens: Vec, - location_type: ZST, -} - -#[derive(Debug)] -enum Token { - Tag { tag: Tag, description: String }, - EnterMarkedSection { mark: Mark, header: String }, - EnterKeyedSection { header: String }, - Start(ScopeType), - End(ScopeType), - StartQuote, - EndQuote, - Text(String), - RawLine(String), -} - -impl TokenConsumer for TokenCollector { - fn tag(&mut self, tag: Tag, description: Option>) { - self.tokens.push(Token::Tag { - tag, - description: description.map(String::from).unwrap_or_default(), - }) - } - - fn enter_marked_section(&mut self, mark: Mark, header: Option>) { - self.tokens.push(Token::EnterMarkedSection { - mark, - header: header.map(String::from).unwrap_or_default(), - }) - } - - fn enter_keyed_section(&mut self, header: Span) { - self.tokens.push(Token::EnterKeyedSection { header: header.into() }) - } - - fn text(&mut self, text: Span) { - match self.tokens.last_mut() { - Some(Token::Text(current)) => { - current.push(' '); - current.push_str(text.text.as_ref()) - } - _ => self.tokens.push(Token::Text(text.text.into())), - } - } - - fn start_list(&mut self) { - self.tokens.push(Token::Start(ScopeType::List)); - } - - fn start_list_item(&mut self) { - self.tokens.push(Token::Start(ScopeType::ListItem)); - } - - fn start_paragraph(&mut self) { - self.tokens.push(Token::Start(ScopeType::Paragraph)); - } - - fn start_raw(&mut self) { - self.tokens.push(Token::Start(ScopeType::Raw)); - } - - fn start_quote(&mut self) { - self.tokens.push(Token::StartQuote); - } - - fn end_quote(&mut self) { - self.tokens.push(Token::EndQuote); - } - - fn whitespace(&mut self) { - self.tokens.push(Token::Text(" ".to_owned())); - } - - fn raw_line(&mut self, text: Span) { - self.tokens.push(Token::RawLine(text.text.into())); - } - - fn end(&mut self, scope: ScopeType) { - self.tokens.push(Token::End(scope)); - } -} diff --git a/lib/rust/parser/src/macros/built_in.rs b/lib/rust/parser/src/macros/built_in.rs index a15a67ec8b..97af2301b1 100644 --- a/lib/rust/parser/src/macros/built_in.rs +++ b/lib/rust/parser/src/macros/built_in.rs @@ -9,6 +9,7 @@ use crate::expression_to_pattern; use crate::source::Code; use crate::syntax::maybe_with_error; use crate::syntax::operator; +use crate::syntax::statement::try_parse_doc_comment; use crate::syntax::token; use crate::syntax::tree::SyntaxError; use crate::syntax::Item; @@ -333,11 +334,12 @@ fn parse_case_line<'s>( precedence: &mut operator::Precedence<'s>, ) -> (syntax::tree::CaseLine<'s>, Option) { let syntax::item::Line { newline, mut items } = line; - if let documentation @ Some(_) = try_parse_doc_comment(&mut items, precedence) { + if let Some(docs) = try_parse_doc_comment(&mut items) { + let doc_line = Some(syntax::tree::DocLine { docs, newlines: vec![] }); return ( syntax::tree::CaseLine { newline: newline.into(), - case: Some(syntax::tree::Case { documentation, ..default() }), + case: Some(syntax::tree::Case { doc_line, ..default() }), }, default(), ); @@ -398,30 +400,6 @@ fn find_top_level_arrow(items: &[Item]) -> Option { } } -fn try_parse_doc_comment<'s>( - items: &mut Vec>, - precedence: &mut operator::Precedence<'s>, -) -> Option> { - if matches!( - items.first(), - Some(Item::Token(token @ Token { variant: token::Variant::TextStart(_), .. })) if token.code == "##" - ) { - let Some(syntax::Tree { - variant: syntax::tree::Variant::Documented(mut documented), - span, - .. - }) = precedence.resolve(items) - else { - unreachable!() - }; - debug_assert_eq!(documented.expression, None); - documented.documentation.open.left_offset += span.left_offset; - Some(documented.documentation) - } else { - None - } -} - /// Array literal. pub fn array<'s>() -> Definition<'s> { crate::macro_definition! {("[", everything(), "]", nothing()) array_body} diff --git a/lib/rust/parser/src/syntax/statement.rs b/lib/rust/parser/src/syntax/statement.rs index df72e826ef..b2977efcd1 100644 --- a/lib/rust/parser/src/syntax/statement.rs +++ b/lib/rust/parser/src/syntax/statement.rs @@ -18,10 +18,14 @@ use crate::syntax::statement::type_def::try_parse_type_def; use crate::syntax::token; use crate::syntax::tree; use crate::syntax::tree::block; +use crate::syntax::tree::AnnotationLine; use crate::syntax::tree::ArgumentDefinition; +use crate::syntax::tree::DocComment; +use crate::syntax::tree::DocLine; use crate::syntax::tree::FunctionAnnotation; use crate::syntax::tree::SyntaxError; use crate::syntax::tree::TypeSignature; +use crate::syntax::tree::TypeSignatureLine; use crate::syntax::treebuilding::Spacing; use crate::syntax::Item; use crate::syntax::Token; @@ -44,7 +48,7 @@ impl<'s> BodyBlockParser<'s> { ) -> Tree<'s> { let lines = compound_lines_with_tail_expression(lines, |prefixes, line, is_tail| { if is_tail { - self.statement_parser.parse_tail_expression(line, precedence) + self.statement_parser.parse_tail_expression(prefixes, line, precedence) } else { self.statement_parser.parse_statement(prefixes, line, precedence) } @@ -68,7 +72,7 @@ impl<'s> BodyBlockParser<'s> { fn compound_lines<'s>( lines: &mut Vec>, mut parse_line: impl FnMut( - &mut Vec>>, + &mut StatementPrefixes<'s>, item::Line<'s>, ) -> Line<'s, StatementOrPrefix<'s>>, ) -> Vec> { @@ -82,7 +86,7 @@ fn compound_lines<'s>( fn compound_lines_with_tail_expression<'s>( lines: &mut Vec>, parse_line: impl FnMut( - &mut Vec>>, + &mut StatementPrefixes<'s>, item::Line<'s>, bool, ) -> Line<'s, StatementOrPrefix<'s>>, @@ -97,36 +101,32 @@ fn compound_lines_with_tail_expression<'s>( fn compound_lines_maybe_with_tail_expression<'s>( lines: &mut Vec>, mut parse_line: impl FnMut( - &mut Vec>>, + &mut StatementPrefixes<'s>, item::Line<'s>, bool, ) -> Line<'s, StatementOrPrefix<'s>>, tail_index: Option, ) -> Vec> { let mut block_lines = Vec::new(); - let mut line_prefixes = Vec::new(); + let mut line_prefixes = StatementPrefixes::default(); for (i, line) in lines.drain(..).enumerate() { let is_tail = tail_index == Some(i); match parse_line(&mut line_prefixes, line, is_tail) { Line { newline, content: Some(StatementOrPrefix::Statement(statement)) } => { - for Line { newline, content } in line_prefixes.drain(..) { - block_lines.push(block::Line { newline, expression: content.map(Tree::from) }) - } + line_prefixes.drain_unused_into(&mut block_lines); block_lines.push(block::Line { newline, expression: Some(statement) }) } Line { newline, content: Some(StatementOrPrefix::Prefix(prefix)) } => - line_prefixes.push(Line { newline, content: Some(prefix) }), + line_prefixes.push(newline, prefix), Line { newline, content: None } => - if line_prefixes.is_empty() { + if line_prefixes.prefixes.is_empty() { block_lines.push(newline.into()); } else { - line_prefixes.push(newline.into()); + line_prefixes.push_newline(newline); }, } } - for Line { newline, content } in line_prefixes { - block_lines.push(block::Line { newline, expression: content.map(Tree::from) }) - } + line_prefixes.drain_unused_into(&mut block_lines); block_lines } @@ -157,7 +157,7 @@ struct StatementParser<'s> { impl<'s> StatementParser<'s> { fn parse_statement( &mut self, - prefixes: &mut Vec>>, + prefixes: &mut StatementPrefixes<'s>, line: item::Line<'s>, precedence: &mut Precedence<'s>, ) -> Line<'s, StatementOrPrefix<'s>> { @@ -170,10 +170,11 @@ impl<'s> StatementParser<'s> { fn parse_tail_expression( &mut self, + prefixes: &mut StatementPrefixes<'s>, line: item::Line<'s>, precedence: &mut Precedence<'s>, ) -> Line<'s, StatementOrPrefix<'s>> { - parse_statement(&mut vec![], line, precedence, &mut self.args_buffer, StatementContext { + parse_statement(prefixes, line, precedence, &mut self.args_buffer, StatementContext { evaluation_context: EvaluationContext::Eager, visibility_context: VisibilityContext::Private, tail_expression: true, @@ -182,7 +183,7 @@ impl<'s> StatementParser<'s> { fn parse_module_statement( &mut self, - prefixes: &mut Vec>>, + prefixes: &mut StatementPrefixes<'s>, line: item::Line<'s>, precedence: &mut Precedence<'s>, ) -> Line<'s, StatementOrPrefix<'s>> { @@ -219,6 +220,7 @@ fn scan_private_keywords<'s>(items: impl IntoIterator enum StatementPrefix<'s> { TypeSignature(TypeSignature<'s>), Annotation(FunctionAnnotation<'s>), + Documentation(DocComment<'s>), } impl<'s> From> for Tree<'s> { @@ -228,6 +230,7 @@ impl<'s> From> for Tree<'s> { Tree::type_signature_declaration(signature), StatementPrefix::Annotation(annotation) => Tree::annotation(annotation).with_error(SyntaxError::AnnotationExpectedDefinition), + StatementPrefix::Documentation(docs) => Tree::documentation(docs), } } } @@ -261,8 +264,27 @@ impl<'s> From> for StatementOrPrefix<'s> { } } +enum StatementPrefixLine<'s> { + TypeSignature(TypeSignatureLine<'s>), + Annotation(AnnotationLine<'s>), + Documentation(DocLine<'s>), +} + +impl<'s> StatementPrefixLine<'s> { + fn new(prefix: StatementPrefix<'s>, newlines: NonEmptyVec>) -> Self { + match prefix { + StatementPrefix::TypeSignature(signature) => + Self::TypeSignature(TypeSignatureLine { signature, newlines }), + StatementPrefix::Annotation(annotation) => + Self::Annotation(AnnotationLine { annotation, newlines }), + StatementPrefix::Documentation(docs) => + Self::Documentation(DocLine { docs, newlines: newlines.into() }), + } + } +} + fn parse_statement<'s>( - prefixes: &mut Vec>>, + prefixes: &mut StatementPrefixes<'s>, mut line: item::Line<'s>, precedence: &mut Precedence<'s>, args_buffer: &mut Vec>, @@ -273,29 +295,32 @@ fn parse_statement<'s>( let private_keywords = scan_private_keywords(&line.items); let start = private_keywords; let items = &mut line.items; - if let Some(annotation) = try_parse_annotation(items, start, precedence) { + let parsed = None + .or_else(|| { + try_parse_annotation(items, start, precedence) + .map(StatementPrefix::Annotation) + .map(StatementOrPrefix::Prefix) + }) + .or_else(|| { + try_parse_type_def(items, start, precedence, args_buffer) + .map(StatementOrPrefix::Statement) + }) + .or_else(|| { + try_parse_doc_comment(items) + .map(StatementPrefix::Documentation) + .map(StatementOrPrefix::Prefix) + }); + if let Some(parsed) = parsed { debug_assert_eq!(items.len(), start); return Line { newline, content: apply_private_keywords( - Some(StatementOrPrefix::Prefix(StatementPrefix::Annotation(annotation))), + Some(parsed), items.drain(..), statement_context.visibility_context, ), }; } - if let Some(type_def) = try_parse_type_def(items, start, precedence, args_buffer) { - debug_assert_eq!(items.len(), start); - return Line { - newline, - content: apply_private_keywords( - Some(type_def), - items.drain(..), - statement_context.visibility_context, - ) - .map(StatementOrPrefix::Statement), - }; - } let top_level_operator = match find_top_level_operator(&items[start..]) { Ok(top_level_operator) => top_level_operator.map(|(i, t)| (i + start, t)), Err(e) => @@ -334,19 +359,190 @@ fn parse_statement<'s>( } } Some(_) => unreachable!(), - None => { - let statement = precedence.resolve_offset(start, items); - debug_assert!(items.len() <= start); - Line { - newline, - content: apply_private_keywords( - statement, - items.drain(..), - statement_context.visibility_context, - ) - .map(StatementOrPrefix::Statement), - } + None => parse_expression_statement( + prefixes, + start, + item::Line { newline, items: mem::take(items) }, + precedence, + statement_context.visibility_context, + ) + .map_content(StatementOrPrefix::Statement), + } +} + +#[derive(Default)] +struct StatementPrefixes<'s> { + prefixes: Vec<(token::Newline<'s>, StatementPrefix<'s>, usize)>, + newlines: Vec>, +} + +impl<'s> StatementPrefixes<'s> { + fn push(&mut self, newline: token::Newline<'s>, prefix: StatementPrefix<'s>) { + let newlines_start = self.newlines.len(); + self.prefixes.push((newline, prefix, newlines_start)) + } + + fn push_newline(&mut self, newline: token::Newline<'s>) { + self.newlines.push(newline) + } + + fn last(&self) -> Option<&StatementPrefix<'s>> { + self.prefixes.last().map(|(_, prefix, _)| prefix) + } + + /// `first_newline`: + /// - Before the call, must contain the first newline after the prefix. + /// - Upon return, will contain the newline before the prefix. + fn pop(&mut self, first_newline: &mut token::Newline<'s>) -> StatementPrefixLine<'s> { + let (newline_before_prefix, prefix, trailing_newlines_start) = self.prefixes.pop().unwrap(); + let original_first_newline = mem::replace(first_newline, newline_before_prefix); + let trailing_newlines = self.newlines.drain(trailing_newlines_start..); + let mut newlines = Vec::with_capacity(trailing_newlines.len() + 1); + newlines.extend(trailing_newlines); + let newlines = NonEmptyVec::from_vec_and_last(newlines, original_first_newline); + StatementPrefixLine::new(prefix, newlines) + } + + fn drain_unused_into(&mut self, lines: &mut Vec>) { + lines.reserve(self.prefixes.len() + self.newlines.len()); + let mut empty_lines = self.newlines.drain(..).map(block::Line::from); + let mut prev_trailing_newlines_start = 0; + for (newline_before_prefix, prefix, trailing_newlines_start) in self.prefixes.drain(..) { + let trailing_newlines = trailing_newlines_start - prev_trailing_newlines_start; + prev_trailing_newlines_start = trailing_newlines_start; + lines.extend((&mut empty_lines).take(trailing_newlines)); + lines.push(block::Line { + newline: newline_before_prefix, + expression: Some(prefix.into()), + }); } + lines.extend(empty_lines); + } +} + +fn take_doc_line<'s>( + prefixes: &mut StatementPrefixes<'s>, + first_newline: &mut token::Newline<'s>, +) -> Option> { + if let Some(StatementPrefix::Documentation(_)) = prefixes.last() { + let StatementPrefixLine::Documentation(doc_line) = prefixes.pop(first_newline) else { + unreachable!() + }; + Some(doc_line) + } else { + None + } +} + +fn parse_expression_statement<'s>( + prefixes: &mut StatementPrefixes<'s>, + start: usize, + mut line: item::Line<'s>, + precedence: &mut Precedence<'s>, + visibility_context: VisibilityContext, +) -> Line<'s, Tree<'s>> { + let expression = precedence.resolve_offset(start, &mut line.items); + debug_assert!(line.items.len() <= start); + let expression = apply_private_keywords(expression, line.items.drain(..), visibility_context); + let mut first_newline = line.newline; + let expression = + expression.map(|expression| to_statement(prefixes, &mut first_newline, expression)); + Line { newline: first_newline, content: expression } +} + +/// `first_newline`: +/// - Before the call, must contain the first newline before `expression_or_statement`. +/// - Upon return, will contain the newline before the returned `Tree` (which will be different from +/// the passed value if any prefixes were consumed). +fn to_statement<'s>( + prefixes: &mut StatementPrefixes<'s>, + first_newline: &mut token::Newline<'s>, + expression_or_statement: Tree<'s>, +) -> Tree<'s> { + use tree::Variant::*; + let is_expression = match &expression_or_statement.variant { + // Currently could be expression or statement--treating as expression. + Invalid(_) => true, + // Currently could be expression or statement--treating as statement so prefix-line + // annotations don't affect how documentation is attached to a type. + AnnotatedBuiltin(_) => false, + // Expression + ArgumentBlockApplication(_) + | OperatorBlockApplication(_) + | Ident(_) + | Number(_) + | Wildcard(_) + | SuspendedDefaultArguments(_) + | TextLiteral(_) + | App(_) + | NamedApp(_) + | OprApp(_) + | UnaryOprApp(_) + | AutoscopedIdentifier(_) + | OprSectionBoundary(_) + | TemplateFunction(_) + | MultiSegmentApp(_) + | Group(_) + | TypeAnnotated(_) + | CaseOf(_) + | Lambda(_) + | Array(_) + | Tuple(_) => true, + // Statement + Private(_) + | TypeDef(_) + | Assignment(_) + | Function(_) + | ForeignFunction(_) + | Import(_) + | Export(_) + | TypeSignatureDeclaration(_) + | Annotation(_) + | Documentation(_) + | ConstructorDefinition(_) => false, + // Unexpected here + BodyBlock(_) | ExpressionStatement(_) => false, + }; + if is_expression { + let doc_line = take_doc_line(prefixes, first_newline); + Tree::expression_statement(doc_line, expression_or_statement) + } else { + expression_or_statement + } +} + +/// Parse the input as a documentation comment, if it matches the syntax. +pub fn try_parse_doc_comment<'s>(items: &mut Vec>) -> Option> { + match items.first() { + Some(Item::Token(token @ Token { variant: token::Variant::TextStart(_), .. })) + if token.code.repr.0 == "##" => + { + let mut items = items.drain(..); + let Some(Item::Token(open)) = items.next() else { unreachable!() }; + let elements = items + .filter_map(|item| { + let Item::Token(token) = item else { unreachable!() }; + match token.variant { + token::Variant::TextSection(variant) => { + let token = token.with_variant(variant); + Some(tree::TextElement::Section { text: token }) + } + token::Variant::TextEscape(variant) => { + let token = token.with_variant(variant); + Some(tree::TextElement::Escape { token }) + } + token::Variant::TextNewline(_) => { + let token = token::newline(token.left_offset, token.code); + Some(tree::TextElement::Newline { newline: token }) + } + token::Variant::TextEnd(_) => None, + _ => unreachable!(), + } + }) + .collect(); + Some(DocComment { open: open.with_variant(token::variant::TextStart()), elements }) + } + _ => None, } } @@ -473,7 +669,7 @@ enum VisibilityContext { } fn parse_assignment_like_statement<'s>( - prefixes: &mut Vec>>, + prefixes: &mut StatementPrefixes<'s>, mut line: item::Line<'s>, start: usize, operator: usize, @@ -537,14 +733,14 @@ fn parse_assignment_like_statement<'s>( (expression, Some(qn_len)) => Type::Function { expression, qn_len }, (None, None) => Type::InvalidNoExpressionNoQn, } { - Type::Assignment { expression } => Line { - newline, - content: apply_private_keywords( - Some(parse_assignment(start, items, operator, expression, precedence)), - items.drain(..), - visibility_context, - ), - }, + Type::Assignment { expression } => AssignmentBuilder::new( + start, + item::Line { newline, items: mem::take(items) }, + operator, + expression, + precedence, + ) + .build(prefixes, visibility_context), Type::Function { expression, qn_len } => FunctionBuilder::new( item::Line { newline, items: mem::take(items) }, start, @@ -567,16 +763,45 @@ fn parse_assignment_like_statement<'s>( } } -fn parse_assignment<'s>( - start: usize, - items: &mut Vec>, - operator: token::AssignmentOperator<'s>, - expression: Tree<'s>, - precedence: &mut Precedence<'s>, -) -> Tree<'s> { - let pattern = - expression_to_pattern(precedence.resolve_non_section_offset(start, items).unwrap()); - Tree::assignment(pattern, operator, expression) +struct AssignmentBuilder<'s> { + newline: token::Newline<'s>, + pattern: Tree<'s>, + operator: token::AssignmentOperator<'s>, + expression: Tree<'s>, + excess_items: Vec>, +} + +impl<'s> AssignmentBuilder<'s> { + fn new( + start: usize, + mut line: item::Line<'s>, + operator: token::AssignmentOperator<'s>, + expression: Tree<'s>, + precedence: &mut Precedence<'s>, + ) -> Self { + let pattern = expression_to_pattern( + precedence.resolve_non_section_offset(start, &mut line.items).unwrap(), + ); + Self { newline: line.newline, pattern, operator, expression, excess_items: line.items } + } + + fn build( + self, + prefixes: &mut StatementPrefixes<'s>, + visibility_context: VisibilityContext, + ) -> Line<'s, Tree<'s>> { + let Self { newline, pattern, operator, expression, excess_items } = self; + let mut first_newline = newline; + let doc_line = take_doc_line(prefixes, &mut first_newline); + Line { + newline: first_newline, + content: apply_private_keywords( + Some(Tree::assignment(doc_line, pattern, operator, expression)), + excess_items.into_iter(), + visibility_context, + ), + } + } } fn parse_pattern<'s>( diff --git a/lib/rust/parser/src/syntax/statement/function_def.rs b/lib/rust/parser/src/syntax/statement/function_def.rs index 9b94873399..41feead9c5 100644 --- a/lib/rust/parser/src/syntax/statement/function_def.rs +++ b/lib/rust/parser/src/syntax/statement/function_def.rs @@ -10,6 +10,8 @@ use crate::syntax::statement::find_top_level_operator; use crate::syntax::statement::parse_pattern; use crate::syntax::statement::Line; use crate::syntax::statement::StatementPrefix; +use crate::syntax::statement::StatementPrefixLine; +use crate::syntax::statement::StatementPrefixes; use crate::syntax::statement::VisibilityContext; use crate::syntax::token; use crate::syntax::tree; @@ -18,6 +20,7 @@ use crate::syntax::tree::ArgumentDefault; use crate::syntax::tree::ArgumentDefinition; use crate::syntax::tree::ArgumentDefinitionLine; use crate::syntax::tree::ArgumentType; +use crate::syntax::tree::DocLine; use crate::syntax::tree::ReturnSpecification; use crate::syntax::tree::SyntaxError; use crate::syntax::tree::TypeSignatureLine; @@ -70,7 +73,7 @@ impl<'s> FunctionBuilder<'s> { pub fn build( mut self, - prefixes: &mut Vec>>, + prefixes: &mut StatementPrefixes<'s>, operator: token::AssignmentOperator<'s>, expression: Option>, visibility_context: VisibilityContext, @@ -86,6 +89,7 @@ impl<'s> FunctionBuilder<'s> { #[derive(Default)] struct PrefixesAccumulator<'s> { + docs: Option>, annotations: Option>>, signature: Option>, } @@ -93,41 +97,36 @@ impl<'s> FunctionBuilder<'s> { let mut acc = PrefixesAccumulator::default(); while let Some(prefix) = prefixes.last() { - let Some(content) = prefix.content.as_ref() else { break }; - match (&acc, &content) { + match (&acc, &prefix) { ( - PrefixesAccumulator { annotations: None, signature: None }, + PrefixesAccumulator { docs: None, annotations: None, signature: None }, StatementPrefix::TypeSignature(signature), ) if qn_equivalent(&self.name, &signature.name) => { - let Some(Line { - newline: outer_newline, - content: Some(StatementPrefix::TypeSignature(signature)), - }) = prefixes.pop() + let StatementPrefixLine::TypeSignature(signature_line) = + prefixes.pop(&mut first_newline) else { unreachable!() }; - let newline = mem::replace(&mut first_newline, outer_newline); - acc.signature = Some(TypeSignatureLine { - signature, - newlines: NonEmptyVec::singleton(newline), - }); + acc.signature = Some(signature_line); } - (PrefixesAccumulator { .. }, StatementPrefix::Annotation(_)) => { - let Some(Line { - newline: outer_newline, - content: Some(StatementPrefix::Annotation(annotation)), - }) = prefixes.pop() + (PrefixesAccumulator { docs: None, .. }, StatementPrefix::Annotation(_)) => { + let StatementPrefixLine::Annotation(annotation_line) = + prefixes.pop(&mut first_newline) else { unreachable!() }; - let newline = mem::replace(&mut first_newline, outer_newline); let mut annotations = acc.annotations.take().unwrap_or_default(); - annotations.push(AnnotationLine { - annotation, - newlines: NonEmptyVec::singleton(newline), - }); + annotations.push(annotation_line); acc.annotations = Some(annotations); } + (PrefixesAccumulator { docs: None, .. }, StatementPrefix::Documentation(_)) => { + let StatementPrefixLine::Documentation(doc_line) = + prefixes.pop(&mut first_newline) + else { + unreachable!() + }; + acc.docs = Some(doc_line); + } _ => break, } } @@ -137,11 +136,13 @@ impl<'s> FunctionBuilder<'s> { annotations.reverse(); annotations }; + let docs = acc.docs; Line { newline: first_newline, content: apply_private_keywords( Some(Tree::function( + docs, annotations, signature, private, @@ -198,7 +199,7 @@ pub fn parse_args<'s>( } pub fn parse_constructor_definition<'s>( - prefixes: &mut Vec>>, + prefixes: &mut StatementPrefixes<'s>, mut line: item::Line<'s>, private_keywords_start: usize, start: usize, @@ -221,21 +222,28 @@ pub fn parse_constructor_definition<'s>( let mut first_newline = newline; let mut annotations_reversed = vec![]; + let mut doc_line = None; while let Some(prefix) = prefixes.last() { - let Some(content) = prefix.content.as_ref() else { break }; - if let StatementPrefix::Annotation(_) = &content { - let Some(Line { - newline: outer_newline, - content: Some(StatementPrefix::Annotation(annotation)), - }) = prefixes.pop() - else { - unreachable!() - }; - let newline = mem::replace(&mut first_newline, outer_newline); - annotations_reversed - .push(AnnotationLine { annotation, newlines: NonEmptyVec::singleton(newline) }); - } else { - break; + match &prefix { + StatementPrefix::Annotation(_) => { + let StatementPrefixLine::Annotation(annotation_line) = + prefixes.pop(&mut first_newline) + else { + unreachable!() + }; + annotations_reversed.push(annotation_line); + } + StatementPrefix::Documentation(_) => { + let StatementPrefixLine::Documentation(line) = prefixes.pop(&mut first_newline) + else { + unreachable!() + }; + doc_line = Some(line); + break; + } + _ => { + break; + } } } let annotations = { @@ -243,7 +251,8 @@ pub fn parse_constructor_definition<'s>( annotations_reversed }; - let def = Tree::constructor_definition(annotations, private, name, inline_args, block_args); + let def = + Tree::constructor_definition(doc_line, annotations, private, name, inline_args, block_args); Line { newline: first_newline, diff --git a/lib/rust/parser/src/syntax/statement/type_def.rs b/lib/rust/parser/src/syntax/statement/type_def.rs index 3518ed1e0d..828a22dfaf 100644 --- a/lib/rust/parser/src/syntax/statement/type_def.rs +++ b/lib/rust/parser/src/syntax/statement/type_def.rs @@ -13,7 +13,7 @@ use crate::syntax::statement::EvaluationContext; use crate::syntax::statement::Line; use crate::syntax::statement::StatementContext; use crate::syntax::statement::StatementOrPrefix; -use crate::syntax::statement::StatementPrefix; +use crate::syntax::statement::StatementPrefixes; use crate::syntax::statement::VisibilityContext; use crate::syntax::token; use crate::syntax::tree; @@ -79,7 +79,7 @@ pub fn try_parse_type_def<'s>( } fn parse_type_body_statement<'s>( - prefixes: &mut Vec>>, + prefixes: &mut StatementPrefixes<'s>, mut line: item::Line<'s>, precedence: &mut Precedence<'s>, args_buffer: &mut Vec>, @@ -121,7 +121,8 @@ fn parse_type_body_statement<'s>( tree::Variant::Function(_) | tree::Variant::ForeignFunction(_) | tree::Variant::Assignment(_) - | tree::Variant::Documented(_) + | tree::Variant::Documentation(_) + | tree::Variant::ExpressionStatement(_) | tree::Variant::Annotation(_) | tree::Variant::AnnotatedBuiltin(_) => None, tree::Variant::TypeSignatureDeclaration(_) => None, diff --git a/lib/rust/parser/src/syntax/tree.rs b/lib/rust/parser/src/syntax/tree.rs index 025841bce2..3c50f6146d 100644 --- a/lib/rust/parser/src/syntax/tree.rs +++ b/lib/rust/parser/src/syntax/tree.rs @@ -206,6 +206,8 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args) }, /// A variable assignment, like `foo = bar 23`. Assignment { + /// Documentation applied to the variable. + pub doc_line: Option>, /// The pattern which should be unified with the expression. pub pattern: Tree<'s>, /// The `=` token. @@ -215,6 +217,8 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args) }, /// A function definition, like `add x y = x + y`. Function { + /// Documentation applied to the function. + pub doc_line: Option>, /// Annotations applied to the function. pub annotation_lines: Vec>, /// A type signature for the function, on its own line. @@ -326,15 +330,21 @@ macro_rules! with_ast_definition { ($f:ident ($($args:tt)*)) => { $f! { $($args) pub newlines: Vec>, pub expression: Option>, }, - /// An expression preceded by a doc comment. - Documented { - /// The documentation. - pub documentation: DocComment<'s>, - /// The item being documented. - pub expression: Option>, + /// A documentation comment that wasn't attached to a following documentable item. + Documentation { + pub docs: DocComment<'s>, + }, + /// An expression at the top level of a block. + ExpressionStatement { + /// Documentation applied to the expression. + pub doc_line: Option>, + /// The expression. + pub expression: Tree<'s>, }, /// Defines a type constructor. ConstructorDefinition { + /// Documentation applied to the constructor. + pub doc_line: Option>, /// Annotations applied to the constructor. pub annotation_lines: Vec>, /// The `private` keyword, if present. @@ -486,7 +496,23 @@ impl<'s> span::Builder<'s> for TextElement<'s> { // === Documentation === -/// A documentation comment. +/// A documentation comment line. +#[cfg_attr(feature = "debug", derive(Visitor))] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Reflect, Deserialize)] +pub struct DocLine<'s> { + /// The documentation. + pub docs: DocComment<'s>, + /// Empty lines between the comment and the item. + pub newlines: Vec>, +} + +impl<'s> span::Builder<'s> for DocLine<'s> { + fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> { + span.add(&mut self.docs).add(&mut self.newlines) + } +} + +/// Contents of a documentation comment. #[cfg_attr(feature = "debug", derive(Visitor))] #[derive(Debug, Clone, Eq, PartialEq, Serialize, Reflect, Deserialize)] pub struct DocComment<'s> { @@ -494,13 +520,11 @@ pub struct DocComment<'s> { pub open: token::TextStart<'s>, /// The documentation text. pub elements: Vec>, - /// Empty lines between the comment and the item. - pub newlines: Vec>, } impl<'s> span::Builder<'s> for DocComment<'s> { fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> { - span.add(&mut self.open).add(&mut self.elements).add(&mut self.newlines) + span.add(&mut self.open).add(&mut self.elements) } } @@ -728,20 +752,20 @@ impl<'s> span::Builder<'s> for CaseLine<'s> { #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Reflect, Deserialize)] pub struct Case<'s> { /// Documentation, if present. - pub documentation: Option>, + pub doc_line: Option>, /// The pattern being matched. It is an error for this to be absent. - pub pattern: Option>, + pub pattern: Option>, /// Token. - pub arrow: Option>, + pub arrow: Option>, /// The expression associated with the pattern. It is an error for this to be empty. - pub expression: Option>, + pub expression: Option>, } impl<'s> Case<'s> { /// Return a mutable reference to the `left_offset` of this object (which will actually belong /// to one of the object's children, if it has any). pub fn left_offset_mut(&mut self) -> Option<&mut Offset<'s>> { - None.or_else(|| self.documentation.as_mut().map(|t| &mut t.open.left_offset)) + None.or_else(|| self.doc_line.as_mut().map(|t| &mut t.docs.open.left_offset)) .or_else(|| self.pattern.as_mut().map(|t| &mut t.span.left_offset)) .or_else(|| self.arrow.as_mut().map(|t| &mut t.left_offset)) .or_else(|| self.expression.as_mut().map(|e| &mut e.span.left_offset)) @@ -750,7 +774,7 @@ impl<'s> Case<'s> { impl<'s> span::Builder<'s> for Case<'s> { fn add_to_span(&mut self, span: Span<'s>) -> Span<'s> { - span.add(&mut self.documentation) + span.add(&mut self.doc_line) .add(&mut self.pattern) .add(&mut self.arrow) .add(&mut self.expression) diff --git a/lib/rust/parser/src/syntax/tree/block.rs b/lib/rust/parser/src/syntax/tree/block.rs index 904f8d9560..73ae885f44 100644 --- a/lib/rust/parser/src/syntax/tree/block.rs +++ b/lib/rust/parser/src/syntax/tree/block.rs @@ -141,7 +141,6 @@ where I: Iterator> #[derive(Debug)] enum Prefix<'s> { BuiltinAnnotation { node: Box>, span: Span<'s> }, - Documentation { node: Box>, span: Span<'s> }, } impl<'s> TryFrom> for Prefix<'s> { @@ -150,7 +149,6 @@ impl<'s> TryFrom> for Prefix<'s> { match tree.variant { Variant::AnnotatedBuiltin(node) if node.expression.is_none() => Ok(Prefix::BuiltinAnnotation { node, span: tree.span }), - Variant::Documented(node) => Ok(Prefix::Documentation { node, span: tree.span }), _ => Err(tree), } } @@ -160,7 +158,6 @@ impl<'s> Prefix<'s> { fn push_newline(&mut self, newline: token::Newline<'s>) { let (newlines, span) = match self { Prefix::BuiltinAnnotation { node, span } => (&mut node.newlines, span), - Prefix::Documentation { node, span } => (&mut node.documentation.newlines, span), }; span.code_length += newline.left_offset.code.length() + newline.code.length(); newlines.push(newline); @@ -169,7 +166,6 @@ impl<'s> Prefix<'s> { fn apply_to(mut self, expression: Tree<'s>) -> Tree<'s> { let (expr, span) = match &mut self { Prefix::BuiltinAnnotation { node, span } => (&mut node.expression, span), - Prefix::Documentation { node, span } => (&mut node.expression, span), }; span.code_length += expression.span.left_offset.code.length() + expression.span.code_length; *expr = Some(expression); @@ -182,8 +178,6 @@ impl<'s> From> for Tree<'s> { match prefix { Prefix::BuiltinAnnotation { node, span } => Tree { variant: Variant::AnnotatedBuiltin(node), span, warnings: default() }, - Prefix::Documentation { node, span } => - Tree { variant: Variant::Documented(node), span, warnings: default() }, } } } diff --git a/lib/rust/parser/src/syntax/treebuilding/compound_token.rs b/lib/rust/parser/src/syntax/treebuilding/compound_token.rs index 103de827b9..b67ca301e8 100644 --- a/lib/rust/parser/src/syntax/treebuilding/compound_token.rs +++ b/lib/rust/parser/src/syntax/treebuilding/compound_token.rs @@ -287,8 +287,7 @@ impl<'s> TextLiteralBuilder<'s> { let Self { open, newline, elements, has_preceding_item } = self; if open.code.starts_with('#') { assert_eq!(newline, None); - let doc = syntax::tree::DocComment { open, elements, newlines: default() }; - let tree = Tree::documented(doc, default()); + let tree = Tree::documentation(syntax::tree::DocComment { open, elements }); let error = has_preceding_item.then_some(SyntaxError::DocumentationUnexpectedNonInitial); maybe_with_error(tree, error) diff --git a/lib/rust/prelude/src/data/non_empty_vec.rs b/lib/rust/prelude/src/data/non_empty_vec.rs index d69e26eca7..29c9741a32 100644 --- a/lib/rust/prelude/src/data/non_empty_vec.rs +++ b/lib/rust/prelude/src/data/non_empty_vec.rs @@ -57,6 +57,12 @@ impl NonEmptyVec { Self { elems } } + /// Construct a new `NonEmptyVec` from a `Vec` and an element `last`. + pub fn from_vec_and_last(mut vec: Vec, last: T) -> NonEmptyVec { + vec.push(last); + Self { elems: vec } + } + /// Construct a new, `NonEmptyVec` containing the provided element and with the /// provided `capacity`. ///