enso/app/gui2/shared/ast/parse.ts

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

936 lines
35 KiB
TypeScript
Raw Normal View History

import * as map from 'lib0/map'
import type { AstId, Module, NodeChild, Owned, OwnedRefs, TextElement, TextToken } from '.'
import {
Token,
asOwned,
isTokenId,
newExternalId,
parentId,
rewriteRefs,
subtreeRoots,
syncFields,
syncNodeMetadata,
} from '.'
import { assert, assertDefined, assertEqual } from '../util/assert'
import { tryGetSoleValue, zip } from '../util/data/iterable'
import type { SourceRangeEdit, SpanTree } from '../util/data/text'
import {
applyTextEdits,
applyTextEditsToSpans,
enclosingSpans,
textChangeToEdits,
trimEnd,
} from '../util/data/text'
import {
IdMap,
isUuid,
rangeLength,
sourceRangeFromKey,
sourceRangeKey,
type SourceRange,
type SourceRangeKey,
} from '../yjsModel'
import { graphParentPointers } from './debug'
import { parse_tree, xxHash128 } from './ffi'
import * as RawAst from './generated/ast'
import { MutableModule } from './mutableModule'
import type { LazyObject } from './parserSupport'
import {
App,
Assignment,
Ast,
BodyBlock,
Documented,
Function,
Generic,
Group,
Ident,
Import,
Invalid,
MutableAssignment,
MutableAst,
MutableBodyBlock,
MutableIdent,
NegationApp,
NumericLiteral,
OprApp,
PropertyAccess,
TextLiteral,
UnaryOprApp,
Wildcard,
} from './tree'
/** Return the raw parser output for the given code. */
export function parseEnso(code: string): RawAst.Tree.BodyBlock {
const blob = parse_tree(code)
const tree = RawAst.Tree.read(new DataView(blob.buffer), blob.byteLength - 4)
// The root of the parser output is always a body block.
assert(tree.type === RawAst.Tree.Type.BodyBlock)
return tree
}
/** Print the AST and re-parse it, copying `externalId`s (but not other metadata) from the original. */
export function normalize(rootIn: Ast): Ast {
const printed = print(rootIn)
const idMap = spanMapToIdMap(printed.info)
const module = MutableModule.Transient()
const tree = parseEnso(printed.code)
const { root: parsed, spans } = abstract(module, tree, printed.code)
module.replaceRoot(parsed)
setExternalIds(module, spans, idMap)
return parsed
}
/** Produce `Ast` types from `RawAst` parser output. */
export function abstract(
module: MutableModule,
tree: RawAst.Tree.BodyBlock,
code: string,
substitutor?: (key: NodeKey) => Owned | undefined,
): { root: Owned<MutableBodyBlock>; spans: SpanMap; toRaw: Map<AstId, RawAst.Tree> }
export function abstract(
module: MutableModule,
tree: RawAst.Tree,
code: string,
substitutor?: (key: NodeKey) => Owned | undefined,
): { root: Owned; spans: SpanMap; toRaw: Map<AstId, RawAst.Tree> }
export function abstract(
module: MutableModule,
tree: RawAst.Tree,
code: string,
substitutor?: (key: NodeKey) => Owned | undefined,
): { root: Owned; spans: SpanMap; toRaw: Map<AstId, RawAst.Tree> } {
const abstractor = new Abstractor(module, code, substitutor)
const root = abstractor.abstractTree(tree).node
const spans = { tokens: abstractor.tokens, nodes: abstractor.nodes }
return { root: root as Owned<MutableBodyBlock>, spans, toRaw: abstractor.toRaw }
}
/** Produces `Ast` types from `RawAst` parser output. */
class Abstractor {
private readonly module: MutableModule
private readonly code: string
private readonly substitutor: ((key: NodeKey) => Owned | undefined) | undefined
readonly nodes: NodeSpanMap
readonly tokens: TokenSpanMap
readonly toRaw: Map<AstId, RawAst.Tree>
/**
* @param module - Where to allocate the new nodes.
* @param code - Source code that will be used to resolve references in any passed `RawAst` objects.
* @param substitutor - A function that can inject subtrees for some spans, instead of the abstractor producing them.
* This can be used for incremental abstraction.
*/
constructor(
module: MutableModule,
code: string,
substitutor?: (key: NodeKey) => Owned | undefined,
) {
this.module = module
this.code = code
this.substitutor = substitutor
this.nodes = new Map()
this.tokens = new Map()
this.toRaw = new Map()
}
abstractTree(tree: RawAst.Tree): { whitespace: string | undefined; node: Owned } {
const whitespaceStart = tree.whitespaceStartInCodeParsed
const whitespaceEnd = whitespaceStart + tree.whitespaceLengthInCodeParsed
const whitespace = this.code.substring(whitespaceStart, whitespaceEnd)
const codeStart = whitespaceEnd
const codeEnd = codeStart + tree.childrenLengthInCodeParsed
const spanKey = nodeKey(codeStart, codeEnd - codeStart)
const substitute = this.substitutor?.(spanKey)
if (substitute) return { node: substitute, whitespace }
let node: Owned
switch (tree.type) {
case RawAst.Tree.Type.BodyBlock: {
const lines = Array.from(tree.statements, (line) => {
const newline = this.abstractToken(line.newline)
const expression = line.expression ? this.abstractTree(line.expression) : undefined
return { newline, expression }
})
node = BodyBlock.concrete(this.module, lines)
break
}
case RawAst.Tree.Type.Function: {
const name = this.abstractTree(tree.name)
const argumentDefinitions = Array.from(tree.args, (arg) => this.abstractChildren(arg))
const equals = this.abstractToken(tree.equals)
const body = tree.body !== undefined ? this.abstractTree(tree.body) : undefined
node = Function.concrete(this.module, name, argumentDefinitions, equals, body)
break
}
case RawAst.Tree.Type.Ident: {
const token = this.abstractToken(tree.token)
node = Ident.concrete(this.module, token)
break
}
case RawAst.Tree.Type.Assignment: {
const pattern = this.abstractTree(tree.pattern)
const equals = this.abstractToken(tree.equals)
const value = this.abstractTree(tree.expr)
node = Assignment.concrete(this.module, pattern, equals, value)
break
}
case RawAst.Tree.Type.App: {
const func = this.abstractTree(tree.func)
const arg = this.abstractTree(tree.arg)
node = App.concrete(this.module, func, undefined, undefined, arg)
break
}
case RawAst.Tree.Type.NamedApp: {
const func = this.abstractTree(tree.func)
const open = tree.open ? this.abstractToken(tree.open) : undefined
const name = this.abstractToken(tree.name)
const equals = this.abstractToken(tree.equals)
const arg = this.abstractTree(tree.arg)
const close = tree.close ? this.abstractToken(tree.close) : undefined
const parens = open && close ? { open, close } : undefined
const nameSpecification = { name, equals }
node = App.concrete(this.module, func, parens, nameSpecification, arg)
break
}
case RawAst.Tree.Type.UnaryOprApp: {
const opr = this.abstractToken(tree.opr)
const arg = tree.rhs ? this.abstractTree(tree.rhs) : undefined
if (arg && opr.node.code() === '-') {
node = NegationApp.concrete(this.module, opr, arg)
} else {
node = UnaryOprApp.concrete(this.module, opr, arg)
}
break
}
case RawAst.Tree.Type.OprApp: {
const lhs = tree.lhs ? this.abstractTree(tree.lhs) : undefined
const opr = tree.opr.ok
? [this.abstractToken(tree.opr.value)]
: Array.from(tree.opr.error.payload.operators, this.abstractToken.bind(this))
const rhs = tree.rhs ? this.abstractTree(tree.rhs) : undefined
const soleOpr = tryGetSoleValue(opr)
if (soleOpr?.node.code() === '.' && rhs?.node instanceof MutableIdent) {
// Propagate type.
const rhs_ = { ...rhs, node: rhs.node }
node = PropertyAccess.concrete(this.module, lhs, soleOpr, rhs_)
} else {
node = OprApp.concrete(this.module, lhs, opr, rhs)
}
break
}
case RawAst.Tree.Type.Number: {
const tokens = []
if (tree.base) tokens.push(this.abstractToken(tree.base))
if (tree.integer) tokens.push(this.abstractToken(tree.integer))
if (tree.fractionalDigits) {
tokens.push(this.abstractToken(tree.fractionalDigits.dot))
tokens.push(this.abstractToken(tree.fractionalDigits.digits))
}
node = NumericLiteral.concrete(this.module, tokens)
break
}
case RawAst.Tree.Type.Wildcard: {
const token = this.abstractToken(tree.token)
node = Wildcard.concrete(this.module, token)
break
}
// These expression types are (or will be) used for backend analysis.
// The frontend can ignore them, avoiding some problems with expressions sharing spans
// (which makes it impossible to give them unique IDs in the current IdMap format).
case RawAst.Tree.Type.OprSectionBoundary:
case RawAst.Tree.Type.TemplateFunction:
return { whitespace, node: this.abstractTree(tree.ast).node }
case RawAst.Tree.Type.Invalid: {
const expression = this.abstractTree(tree.ast)
node = Invalid.concrete(this.module, expression)
break
}
case RawAst.Tree.Type.Group: {
const open = tree.open ? this.abstractToken(tree.open) : undefined
const expression = tree.body ? this.abstractTree(tree.body) : undefined
const close = tree.close ? this.abstractToken(tree.close) : undefined
node = Group.concrete(this.module, open, expression, close)
break
}
case RawAst.Tree.Type.TextLiteral: {
const open = tree.open ? this.abstractToken(tree.open) : undefined
const newline = tree.newline ? this.abstractToken(tree.newline) : undefined
const elements = Array.from(tree.elements, (raw) => this.abstractTextElement(raw))
const close = tree.close ? this.abstractToken(tree.close) : undefined
node = TextLiteral.concrete(this.module, open, newline, elements, close)
break
}
case RawAst.Tree.Type.Documented: {
const open = this.abstractToken(tree.documentation.open)
const elements = Array.from(tree.documentation.elements, (raw) =>
this.abstractTextToken(raw),
)
const newlines = Array.from(tree.documentation.newlines, this.abstractToken.bind(this))
const expression = tree.expression ? this.abstractTree(tree.expression) : undefined
node = Documented.concrete(this.module, open, elements, newlines, expression)
break
}
case RawAst.Tree.Type.Import: {
const recurseBody = (tree: RawAst.Tree) => {
const body = this.abstractTree(tree)
if (body.node instanceof Invalid && body.node.code() === '') return undefined
return body
}
const recurseSegment = (segment: RawAst.MultiSegmentAppSegment) => ({
header: this.abstractToken(segment.header),
body: segment.body ? recurseBody(segment.body) : undefined,
})
const polyglot = tree.polyglot ? recurseSegment(tree.polyglot) : undefined
const from = tree.from ? recurseSegment(tree.from) : undefined
const import_ = recurseSegment(tree.import)
const all = tree.all ? this.abstractToken(tree.all) : undefined
const as = tree.as ? recurseSegment(tree.as) : undefined
const hiding = tree.hiding ? recurseSegment(tree.hiding) : undefined
node = Import.concrete(this.module, polyglot, from, import_, all, as, hiding)
break
}
default: {
node = Generic.concrete(this.module, this.abstractChildren(tree))
}
}
this.toRaw.set(node.id, tree)
map.setIfUndefined(this.nodes, spanKey, (): Ast[] => []).unshift(node)
return { node, whitespace }
}
private abstractToken(token: RawAst.Token): { whitespace: string; node: Token } {
const whitespaceStart = token.whitespaceStartInCodeBuffer
const whitespaceEnd = whitespaceStart + token.whitespaceLengthInCodeBuffer
const whitespace = this.code.substring(whitespaceStart, whitespaceEnd)
const codeStart = token.startInCodeBuffer
const codeEnd = codeStart + token.lengthInCodeBuffer
const tokenCode = this.code.substring(codeStart, codeEnd)
const key = tokenKey(codeStart, codeEnd - codeStart)
const node = Token.new(tokenCode, token.type)
this.tokens.set(key, node)
return { whitespace, node }
}
private abstractChildren(tree: LazyObject): (NodeChild<Owned> | NodeChild<Token>)[] {
const children: (NodeChild<Owned> | NodeChild<Token>)[] = []
const visitor = (child: LazyObject) => {
if (RawAst.Tree.isInstance(child)) {
children.push(this.abstractTree(child))
} else if (RawAst.Token.isInstance(child)) {
children.push(this.abstractToken(child))
} else {
child.visitChildren(visitor)
}
}
tree.visitChildren(visitor)
return children
}
private abstractTextElement(raw: RawAst.TextElement): TextElement<OwnedRefs> {
switch (raw.type) {
case RawAst.TextElement.Type.Newline:
case RawAst.TextElement.Type.Escape:
case RawAst.TextElement.Type.Section:
return this.abstractTextToken(raw)
case RawAst.TextElement.Type.Splice:
return {
type: 'splice',
open: this.abstractToken(raw.open),
expression: raw.expression && this.abstractTree(raw.expression),
close: this.abstractToken(raw.close),
}
}
}
private abstractTextToken(raw: RawAst.TextElement): TextToken<OwnedRefs> {
switch (raw.type) {
case RawAst.TextElement.Type.Newline:
return { type: 'token', token: this.abstractToken(raw.newline) }
case RawAst.TextElement.Type.Escape: {
const negativeOneU32 = 4294967295
return {
type: 'token',
token: this.abstractToken(raw.token),
interpreted:
raw.token.value !== negativeOneU32 ? String.fromCodePoint(raw.token.value) : undefined,
}
}
case RawAst.TextElement.Type.Section:
return { type: 'token', token: this.abstractToken(raw.text) }
case RawAst.TextElement.Type.Splice:
throw new Error('Unreachable: Splice in non-interpolated text field')
}
}
}
declare const nodeKeyBrand: unique symbol
/** A source-range key for an `Ast`. */
export type NodeKey = SourceRangeKey & { [nodeKeyBrand]: never }
declare const tokenKeyBrand: unique symbol
/** A source-range key for a `Token`. */
export type TokenKey = SourceRangeKey & { [tokenKeyBrand]: never }
/** Create a source-range key for an `Ast`. */
export function nodeKey(start: number, length: number): NodeKey {
return sourceRangeKey([start, start + length]) as NodeKey
}
/** Create a source-range key for a `Token`. */
export function tokenKey(start: number, length: number): TokenKey {
return sourceRangeKey([start, start + length]) as TokenKey
}
/** Maps from source ranges to `Ast`s. */
export type NodeSpanMap = Map<NodeKey, Ast[]>
/** Maps from source ranges to `Token`s. */
export type TokenSpanMap = Map<TokenKey, Token>
/** Maps from source ranges to `Ast`s and `Token`s. */
export interface SpanMap {
nodes: NodeSpanMap
tokens: TokenSpanMap
}
/** Code with an associated mapping to `Ast` types. */
interface PrintedSource {
info: SpanMap
code: string
}
/** Generate an `IdMap` from a `SpanMap`. */
export function spanMapToIdMap(spans: SpanMap): IdMap {
const idMap = new IdMap()
for (const [key, token] of spans.tokens.entries()) {
assert(isUuid(token.id))
idMap.insertKnownId(sourceRangeFromKey(key), token.id)
}
for (const [key, asts] of spans.nodes.entries()) {
for (const ast of asts) {
assert(isUuid(ast.externalId))
idMap.insertKnownId(sourceRangeFromKey(key), ast.externalId)
}
}
return idMap
}
/** Given a `SpanMap`, return a function that can look up source ranges by AST ID. */
export function spanMapToSpanGetter(spans: SpanMap): (id: AstId) => SourceRange | undefined {
const reverseMap = new Map<AstId, SourceRange>()
for (const [key, asts] of spans.nodes) {
for (const ast of asts) {
reverseMap.set(ast.id, sourceRangeFromKey(key))
}
}
return (id) => reverseMap.get(id)
}
/** Return stringification with associated ID map. This is only exported for testing. */
export function print(ast: Ast): PrintedSource {
const info: SpanMap = {
nodes: new Map(),
tokens: new Map(),
}
const code = ast.printSubtree(info, 0, undefined)
return { info, code }
}
/** @internal Used by `Ast.printSubtree`. Note that some AST types have overrides. */
export function printAst(
ast: Ast,
info: SpanMap,
offset: number,
parentIndent: string | undefined,
verbatim?: boolean,
): string {
let code = ''
for (const child of ast.concreteChildren(verbatim)) {
if (!isTokenId(child.node) && ast.module.get(child.node) === undefined) continue
if (child.whitespace != null) {
code += child.whitespace
} else if (code.length != 0) {
code += ' '
}
if (isTokenId(child.node)) {
const tokenStart = offset + code.length
const token = ast.module.getToken(child.node)
const span = tokenKey(tokenStart, token.code().length)
info.tokens.set(span, token)
code += token.code()
} else {
const childNode = ast.module.get(child.node)
code += childNode.printSubtree(info, offset + code.length, parentIndent, verbatim)
// Extra structural validation.
assertEqual(childNode.id, child.node)
if (parentId(childNode) !== ast.id) {
console.error(
`Inconsistent parent pointer (expected ${ast.id})`,
childNode,
graphParentPointers(ast.module.root()!),
)
}
assertEqual(parentId(childNode), ast.id)
}
}
const span = nodeKey(offset, code.length)
map.setIfUndefined(info.nodes, span, (): Ast[] => []).unshift(ast)
return code
}
/** @internal Use `Ast.code()' to stringify. */
export function printBlock(
block: BodyBlock,
info: SpanMap,
offset: number,
parentIndent: string | undefined,
verbatim?: boolean,
): string {
let blockIndent: string | undefined
let code = ''
for (const line of block.fields.get('lines')) {
code += line.newline.whitespace ?? ''
const newlineCode = block.module.getToken(line.newline.node).code()
// Only print a newline if this isn't the first line in the output, or it's a comment.
if (offset || code || newlineCode.startsWith('#')) {
// If this isn't the first line in the output, but there is a concrete newline token:
// if it's a zero-length newline, ignore it and print a normal newline.
code += newlineCode || '\n'
}
if (line.expression) {
if (blockIndent === undefined) {
if ((line.expression.whitespace?.length ?? 0) > (parentIndent?.length ?? 0)) {
blockIndent = line.expression.whitespace!
} else if (parentIndent !== undefined) {
blockIndent = parentIndent + ' '
} else {
blockIndent = ''
}
}
const validIndent = (line.expression.whitespace?.length ?? 0) > (parentIndent?.length ?? 0)
code += validIndent ? line.expression.whitespace : blockIndent
const lineNode = block.module.get(line.expression.node)
assertEqual(lineNode.id, line.expression.node)
assertEqual(parentId(lineNode), block.id)
code += lineNode.printSubtree(info, offset + code.length, blockIndent, verbatim)
}
}
const span = nodeKey(offset, code.length)
map.setIfUndefined(info.nodes, span, (): Ast[] => []).unshift(block)
return code
}
/** Parse the input as a block. */
export function parseBlock(code: string, inModule?: MutableModule): Owned<MutableBodyBlock> {
return parseBlockWithSpans(code, inModule).root
}
/** Parse the input. If it contains a single expression at the top level, return it; otherwise, return a block. */
export function parse(code: string, module?: MutableModule): Owned {
const module_ = module ?? MutableModule.Transient()
const ast = parseBlock(code, module_)
const soleStatement = tryGetSoleValue(ast.statements())
if (!soleStatement) return ast
const parent = parentId(soleStatement)
if (parent) module_.delete(parent)
soleStatement.fields.set('parent', undefined)
return asOwned(soleStatement)
}
/** Parse a block, and return it along with a mapping from source locations to parsed objects. */
export function parseBlockWithSpans(
code: string,
inModule?: MutableModule,
): { root: Owned<MutableBodyBlock>; spans: SpanMap } {
const tree = parseEnso(code)
const module = inModule ?? MutableModule.Transient()
return abstract(module, tree, code)
}
/** Parse the input, and apply the given `IdMap`. Return the parsed tree, the updated `IdMap`, the span map, and a
* mapping to the `RawAst` representation.
*/
export function parseExtended(code: string, idMap?: IdMap | undefined, inModule?: MutableModule) {
const rawRoot = parseEnso(code)
const module = inModule ?? MutableModule.Transient()
const { root, spans, toRaw } = module.transact(() => {
const { root, spans, toRaw } = abstract(module, rawRoot, code)
root.module.replaceRoot(root)
if (idMap) setExternalIds(root.module, spans, idMap)
return { root, spans, toRaw }
})
const getSpan = spanMapToSpanGetter(spans)
const idMapOut = spanMapToIdMap(spans)
return { root, idMap: idMapOut, getSpan, toRaw }
}
/** Return the number of `Ast`s in the tree, including the provided root. */
export function astCount(ast: Ast): number {
let count = 0
ast.visitRecursiveAst((_subtree) => {
count += 1
})
return count
}
/** Apply an `IdMap` to a module, using the given `SpanMap`.
* @returns The number of IDs that were assigned from the map.
*/
export function setExternalIds(edit: MutableModule, spans: SpanMap, ids: IdMap): number {
let astsMatched = 0
for (const [key, externalId] of ids.entries()) {
const asts = spans.nodes.get(key as NodeKey)
if (asts) {
for (const ast of asts) {
astsMatched += 1
const editAst = edit.getVersion(ast)
if (editAst.externalId !== externalId) editAst.setExternalId(externalId)
}
}
}
return astsMatched
}
/** Try to find all the spans in `expected` in `encountered`. If any are missing, use the provided `code` to determine
* whether the lost spans are single-line or multi-line.
*/
function checkSpans(expected: NodeSpanMap, encountered: NodeSpanMap, code: string) {
const lost = new Array<readonly [NodeKey, Ast]>()
for (const [key, asts] of expected) {
const outermostPrinted = asts[0]
if (!outermostPrinted) continue
for (let i = 1; i < asts.length; ++i) assertEqual(asts[i]?.parentId, asts[i - 1]?.id)
const encounteredAsts = encountered.get(key)
if (encounteredAsts === undefined) lost.push([key, outermostPrinted])
}
const lostInline = new Array<Ast>()
const lostBlock = new Array<Ast>()
for (const [key, ast] of lost) {
const [start, end] = sourceRangeFromKey(key)
;(code.substring(start, end).match(/[\r\n]/) ? lostBlock : lostInline).push(ast)
}
return { lostInline, lostBlock }
}
/** If the input tree's concrete syntax has precedence errors (i.e. its expected code would not parse back to the same
* structure), try to fix it. If possible, it will be repaired by inserting parentheses; if that doesn't fix it, the
* affected subtree will be re-synced to faithfully represent the source code the incorrect tree prints to.
*/
export function repair(
root: BodyBlock,
module?: MutableModule,
): { code: string; fixes: MutableModule | undefined } {
// Print the input to see what spans its nodes expect to have in the output.
const printed = print(root)
// Parse the printed output to see what spans actually correspond to nodes in the printed code.
const reparsed = parseBlockWithSpans(printed.code)
// See if any span we expected to be a node isn't; if so, it likely merged with its parent due to wrong precedence.
const { lostInline, lostBlock } = checkSpans(
printed.info.nodes,
reparsed.spans.nodes,
printed.code,
)
if (lostInline.length === 0) {
if (lostBlock.length !== 0) {
console.warn(`repair: Bad block elements, but all inline elements OK?`)
const fixes = module ?? root.module.edit()
resync(lostBlock, printed.info.nodes, reparsed.spans.nodes, fixes)
return { code: printed.code, fixes }
}
return { code: printed.code, fixes: undefined }
}
// Wrap any "lost" nodes in parentheses.
const fixes = module ?? root.module.edit()
for (const ast of lostInline) {
if (ast instanceof Group) continue
fixes.getVersion(ast).update((ast) => Group.new(fixes, ast))
}
// Verify that it's fixed.
const printed2 = print(fixes.getVersion(root))
const reparsed2 = parseBlockWithSpans(printed2.code)
const { lostInline: lostInline2, lostBlock: lostBlock2 } = checkSpans(
printed2.info.nodes,
reparsed2.spans.nodes,
printed2.code,
)
if (lostInline2.length !== 0 || lostBlock2.length !== 0)
resync([...lostInline2, ...lostBlock2], printed2.info.nodes, reparsed2.spans.nodes, fixes)
return { code: printed2.code, fixes }
}
/**
* Replace subtrees in the module to ensure that the module contents are consistent with the module's code.
*
* @param badAsts - ASTs that, if printed, would not parse to exactly their current content.
* @param badSpans - Span map produced by printing the `badAsts` nodes and all their parents.
* @param goodSpans - Span map produced by parsing the code from the module of `badAsts`.
* @param edit - Module to apply the fixes to; must contain all ASTs in `badAsts`.
*/
function resync(
badAsts: Iterable<Ast>,
badSpans: NodeSpanMap,
goodSpans: NodeSpanMap,
edit: MutableModule,
) {
const parentsOfBadSubtrees = new Set<AstId>()
const badAstIds = new Set(Array.from(badAsts, (ast) => ast.id))
for (const id of subtreeRoots(edit, badAstIds)) {
const parent = edit.get(id)?.parentId
if (parent) parentsOfBadSubtrees.add(parent)
}
const spanOfBadParent = new Array<readonly [AstId, NodeKey]>()
for (const [span, asts] of badSpans) {
for (const ast of asts) {
if (parentsOfBadSubtrees.has(ast.id)) spanOfBadParent.push([ast.id, span])
}
}
// All ASTs in the module of badAsts should have entries in badSpans.
assertEqual(spanOfBadParent.length, parentsOfBadSubtrees.size)
for (const [id, span] of spanOfBadParent) {
const parent = edit.get(id)
const goodAst = goodSpans.get(span)?.[0]
// The parent of the root of a bad subtree must be a good AST.
assertDefined(goodAst)
parent.syncToCode(goodAst.code())
}
console.warn(
`repair: Replaced ${parentsOfBadSubtrees.size} subtrees with their reparsed equivalents.`,
parentsOfBadSubtrees,
)
}
/** @internal Recursion helper for {@link syntaxHash}. */
function hashSubtreeSyntax(ast: Ast, hashesOut: Map<SyntaxHash, Ast[]>): SyntaxHash {
let content = ''
content += ast.typeName + ':'
for (const child of ast.concreteChildren()) {
content += child.whitespace ?? '?'
if (isTokenId(child.node)) {
content += 'Token:' + hashString(ast.module.getToken(child.node).code())
} else {
content += hashSubtreeSyntax(ast.module.get(child.node), hashesOut)
}
}
const astHash = hashString(content)
map.setIfUndefined(hashesOut, astHash, (): Ast[] => []).unshift(ast)
return astHash
}
declare const brandHash: unique symbol
/** See {@link syntaxHash}. */
type SyntaxHash = string & { [brandHash]: never }
/** Applies the syntax-data hashing function to the input, and brands the result as a `SyntaxHash`. */
function hashString(input: string): SyntaxHash {
return xxHash128(input) as SyntaxHash
}
/** Calculates `SyntaxHash`es for the given node and all its children.
*
* Each `SyntaxHash` summarizes the syntactic content of an AST. If two ASTs have the same code and were parsed the
* same way (i.e. one was not parsed in a context that resulted in a different interpretation), they will have the same
* hash. Note that the hash is invariant to metadata, including `externalId` assignments.
*/
function syntaxHash(root: Ast) {
const hashes = new Map<SyntaxHash, Ast[]>()
const rootHash = hashSubtreeSyntax(root, hashes)
return { root: rootHash, hashes }
}
/** If the input is a block containing a single expression, return the expression; otherwise return the input. */
function rawBlockToInline(tree: RawAst.Tree.Tree) {
if (tree.type !== RawAst.Tree.Type.BodyBlock) return tree
return tryGetSoleValue(tree.statements)?.expression ?? tree
}
/** Update `ast` to match the given source code, while modifying it as little as possible. */
export function syncToCode(ast: MutableAst, code: string, metadataSource?: Module) {
const codeBefore = ast.code()
const textEdits = textChangeToEdits(codeBefore, code)
applyTextEditsToAst(ast, textEdits, metadataSource ?? ast.module)
}
/** Find nodes in the input `ast` that should be treated as equivalents of nodes in `parsedRoot`. */
function calculateCorrespondence(
ast: Ast,
astSpans: NodeSpanMap,
parsedRoot: Ast,
parsedSpans: NodeSpanMap,
textEdits: SourceRangeEdit[],
codeAfter: string,
): Map<AstId, Ast> {
const newSpans = new Map<AstId, SourceRange>()
for (const [key, asts] of parsedSpans) {
for (const ast of asts) newSpans.set(ast.id, sourceRangeFromKey(key))
}
// Retained-code matching: For each new tree, check for some old tree of the same type such that the new tree is the
// smallest node to contain all characters of the old tree's code that were not deleted in the edit.
//
// If the new node's span exactly matches the retained code, add the match to `toSync`. If the new node's span
// contains additional code, add the match to `candidates`.
const toSync = new Map<AstId, Ast>()
const candidates = new Map<AstId, Ast>()
const allSpansBefore = Array.from(astSpans.keys(), sourceRangeFromKey)
const spansBeforeAndAfter = applyTextEditsToSpans(textEdits, allSpansBefore).map(
([before, after]) => [before, trimEnd(after, codeAfter)] satisfies [any, any],
)
const partAfterToAstBefore = new Map<SourceRangeKey, Ast>()
for (const [spanBefore, partAfter] of spansBeforeAndAfter) {
const astBefore = astSpans.get(sourceRangeKey(spanBefore) as NodeKey)?.[0]!
partAfterToAstBefore.set(sourceRangeKey(partAfter), astBefore)
}
const matchingPartsAfter = spansBeforeAndAfter.map(([_before, after]) => after)
const parsedSpanTree = new AstWithSpans(parsedRoot, (id) => newSpans.get(id)!)
const astsMatchingPartsAfter = enclosingSpans(parsedSpanTree, matchingPartsAfter)
for (const [astAfter, partsAfter] of astsMatchingPartsAfter) {
for (const partAfter of partsAfter) {
const astBefore = partAfterToAstBefore.get(sourceRangeKey(partAfter))!
if (astBefore.typeName() === astAfter.typeName()) {
;(rangeLength(newSpans.get(astAfter.id)!) === rangeLength(partAfter)
? toSync
: candidates
).set(astBefore.id, astAfter)
break
}
}
}
// Index the matched nodes.
const oldIdsMatched = new Set<AstId>()
const newIdsMatched = new Set<AstId>()
for (const [oldId, newAst] of toSync) {
oldIdsMatched.add(oldId)
newIdsMatched.add(newAst.id)
}
// Movement matching: For each new tree that hasn't been matched, match it with any identical unmatched old tree.
const newHashes = syntaxHash(parsedRoot).hashes
const oldHashes = syntaxHash(ast).hashes
for (const [hash, newAsts] of newHashes) {
const unmatchedNewAsts = newAsts.filter((ast) => !newIdsMatched.has(ast.id))
const unmatchedOldAsts = oldHashes.get(hash)?.filter((ast) => !oldIdsMatched.has(ast.id)) ?? []
for (const [unmatchedNew, unmatchedOld] of zip(unmatchedNewAsts, unmatchedOldAsts)) {
toSync.set(unmatchedOld.id, unmatchedNew)
// Update the matched-IDs indices.
oldIdsMatched.add(unmatchedOld.id)
newIdsMatched.add(unmatchedNew.id)
}
}
// Apply any non-optimal span matches from `candidates`, if the nodes involved were not matched during
// movement-matching.
for (const [beforeId, after] of candidates) {
if (oldIdsMatched.has(beforeId) || newIdsMatched.has(after.id)) continue
toSync.set(beforeId, after)
}
return toSync
}
/** Update `ast` according to changes to its corresponding source code. */
export function applyTextEditsToAst(
ast: MutableAst,
textEdits: SourceRangeEdit[],
metadataSource: Module,
) {
const printed = print(ast)
const code = applyTextEdits(printed.code, textEdits)
const rawParsedBlock = parseEnso(code)
const rawParsed =
ast instanceof MutableBodyBlock ? rawParsedBlock : rawBlockToInline(rawParsedBlock)
const parsed = abstract(ast.module, rawParsed, code)
const toSync = calculateCorrespondence(
ast,
printed.info.nodes,
parsed.root,
parsed.spans.nodes,
textEdits,
code,
)
syncTree(ast, parsed.root, toSync, ast.module, metadataSource)
}
/** Replace `target` with `newContent`, reusing nodes according to the correspondence in `toSync`. */
function syncTree(
target: Ast,
newContent: Owned,
toSync: Map<AstId, Ast>,
edit: MutableModule,
metadataSource: Module,
) {
const newIdToEquivalent = new Map<AstId, AstId>()
for (const [beforeId, after] of toSync) newIdToEquivalent.set(after.id, beforeId)
const childReplacerFor = (parentId: AstId) => (id: AstId) => {
const original = newIdToEquivalent.get(id)
if (original) {
const replacement = edit.get(original)
if (replacement.parentId !== parentId) replacement.fields.set('parent', parentId)
return original
} else {
const child = edit.get(id)
if (child.parentId !== parentId) child.fields.set('parent', parentId)
}
}
const parentId = target.fields.get('parent')
assertDefined(parentId)
const parent = edit.get(parentId)
const targetSyncEquivalent = toSync.get(target.id)
const syncRoot = targetSyncEquivalent?.id === newContent.id ? targetSyncEquivalent : undefined
if (!syncRoot) {
parent.replaceChild(target.id, newContent)
newContent.fields.set('metadata', target.fields.get('metadata').clone())
target.fields.get('metadata').set('externalId', newExternalId())
}
const newRoot = syncRoot ? target : newContent
newRoot.visitRecursiveAst((ast) => {
const syncFieldsFrom = toSync.get(ast.id)
const editAst = edit.getVersion(ast)
if (syncFieldsFrom) {
const originalAssignmentExpression =
ast instanceof Assignment
? metadataSource.get(ast.fields.get('expression').node)
: undefined
syncFields(edit.getVersion(ast), syncFieldsFrom, childReplacerFor(ast.id))
if (editAst instanceof MutableAssignment && originalAssignmentExpression) {
if (editAst.expression.externalId !== originalAssignmentExpression.externalId)
editAst.expression.setExternalId(originalAssignmentExpression.externalId)
syncNodeMetadata(
editAst.expression.mutableNodeMetadata(),
originalAssignmentExpression.nodeMetadata,
)
}
} else {
rewriteRefs(editAst, childReplacerFor(ast.id))
}
return true
})
return newRoot
}
/** Provides a `SpanTree` view of an `Ast`, given span information. */
class AstWithSpans implements SpanTree<Ast> {
private readonly ast: Ast
private readonly getSpan: (astId: AstId) => SourceRange
constructor(ast: Ast, getSpan: (astId: AstId) => SourceRange) {
this.ast = ast
this.getSpan = getSpan
}
id(): Ast {
return this.ast
}
span(): SourceRange {
return this.getSpan(this.ast.id)
}
*children(): IterableIterator<SpanTree<Ast>> {
for (const child of this.ast.children()) {
if (child instanceof Ast) yield new AstWithSpans(child, this.getSpan)
}
}
}