enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-12-23 20:16:47 +03:00
Code Issues Projects Releases Wiki Activity

Enable the Code Editor, with new apply-text-edits algo. (#9055)

Browse Source 
- Fix the UI problems with our CodeMirror integration (Fixed view stability; Fixed a focus bug; Fixed errors caused by diagnostics range exceptions; Fixed linter invalidation--see https://discuss.codemirror.net/t/problem-trying-to-force-linting/5823; Implemented edit-coalescing for performance).
- Introduce an algorithm for applying text edits to an AST. Compared to the GUI1 approach, the new algorithm supports deeper identity-stability for expressions (which is important for subexpression metadata and Y.Js sync), as well as reordered-subtree identification.
- Enable the code editor.
This commit is contained in:
Kaz Wesley 2024-02-19 15:57:42 -08:00 committed by GitHub
parent d75523b46f
commit c811a5ae8b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
35 changed files with 1832 additions and 535 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/gui2/mock/index.ts
View File

@ -67,7 +67,7 @@ export function projectStore() {
const mod = projectStore.projectModel.createNewModule('Main.enso')
mod.doc.ydoc.emit('load', [])
const syncModule = new Ast.MutableModule(mod.doc.ydoc)
mod.transact(() => {
syncModule.transact(() => {
const root = Ast.parseBlock('main =\n', syncModule)
syncModule.replaceRoot(root)
})

22
app/gui2/shared/ast/debug.ts Normal file
View File

@ -0,0 +1,22 @@
import { Ast } from './tree'
/// Returns a GraphViz graph illustrating parent/child relationships in the given subtree.
export function graphParentPointers(ast: Ast) {
const sanitize = (id: string) => id.replace('ast:', '').replace(/[^A-Za-z0-9]/g, '')
const parentToChild = new Array<{ parent: string; child: string }>()
const childToParent = new Array<{ child: string; parent: string }>()
ast.visitRecursiveAst((ast) => {
for (const child of ast.children()) {
if (child instanceof Ast)
parentToChild.push({ child: sanitize(child.id), parent: sanitize(ast.id) })
}
const parent = ast.parentId
if (parent) childToParent.push({ child: sanitize(ast.id), parent: sanitize(parent) })
})
let result = 'digraph parentPointers {\n'
for (const { parent, child } of parentToChild) result += `${parent} -> ${child};\n`
for (const { child, parent } of childToParent)
result += `${child} -> ${parent} [weight=0; color=red; style=dotted];\n`
result += '}\n'
return result
}

11
app/gui2/shared/ast/ffi.ts
View File

@ -1,6 +1,16 @@
import { createXXHash128 } from 'hash-wasm'
import init, { is_ident_or_operator, parse, parse_doc_to_json } from '../../rust-ffi/pkg/rust_ffi'
import { assertDefined } from '../util/assert'
import { isNode } from '../util/detect'
let xxHasher128: Awaited<ReturnType<typeof createXXHash128>> | undefined
export function xxHash128(input: string) {
assertDefined(xxHasher128, 'Module should have been loaded with `initializeFFI`.')
xxHasher128.init()
xxHasher128.update(input)
return xxHasher128.digest()
}
export async function initializeFFI(path?: string | undefined) {
if (isNode) {
const fs = await import('node:fs/promises')
@ -9,6 +19,7 @@ export async function initializeFFI(path?: string | undefined) {
} else {
await init()
}
xxHasher128 = await createXXHash128()
}
// TODO[ao]: We cannot to that, because the ffi is used by cjs modules.

10
app/gui2/shared/ast/index.ts
View File

@ -40,7 +40,7 @@ export function parentId(ast: Ast): AstId | undefined {
export function subtrees(module: Module, ids: Iterable<AstId>) {
const subtrees = new Set<AstId>()
for (const id of ids) {
let ast = module.get(id)
let ast = module.tryGet(id)
while (ast != null && !subtrees.has(ast.id)) {
subtrees.add(ast.id)
ast = ast.parent()
@ -50,10 +50,10 @@ export function subtrees(module: Module, ids: Iterable<AstId>) {
}
/** Returns the IDs of the ASTs that are not descendants of any others in the given set. */
export function subtreeRoots(module: Module, ids: Set<AstId>) {
const roots = new Array<AstId>()
export function subtreeRoots(module: Module, ids: Set<AstId>): Set<AstId> {
const roots = new Set<AstId>()
for (const id of ids) {
const astInModule = module.get(id)
const astInModule = module.tryGet(id)
if (!astInModule) continue
let ast = astInModule.parent()
let hasParentInSet
@ -64,7 +64,7 @@ export function subtreeRoots(module: Module, ids: Set<AstId>) {
}
ast = ast.parent()
}
if (!hasParentInSet) roots.push(id)
if (!hasParentInSet) roots.add(id)
}
return roots
}

141
app/gui2/shared/ast/mutableModule.ts
View File

@ -1,16 +1,25 @@
import * as random from 'lib0/random'
import * as Y from 'yjs'
import type { AstId, Owned, SyncTokenId } from '.'
import { Token, asOwned, isTokenId, newExternalId } from '.'
import { assert } from '../util/assert'
import type { ExternalId } from '../yjsModel'
import {
Token,
asOwned,
isTokenId,
newExternalId,
subtreeRoots,
type AstId,
type Owned,
type SyncTokenId,
} from '.'
import { assert, assertDefined } from '../util/assert'
import type { SourceRangeEdit } from '../util/data/text'
import { defaultLocalOrigin, tryAsOrigin, type ExternalId, type Origin } from '../yjsModel'
import type { AstFields, FixedMap, Mutable } from './tree'
import {
Ast,
Invalid,
MutableAst,
MutableInvalid,
Wildcard,
composeFieldData,
invalidFields,
materializeMutable,
setAll,
@ -19,13 +28,13 @@ import {
export interface Module {
edit(): MutableModule
root(): Ast | undefined
get(id: AstId): Ast | undefined
get(id: AstId | undefined): Ast | undefined
tryGet(id: AstId | undefined): Ast | undefined
/////////////////////////////////
checkedGet(id: AstId): Ast
checkedGet(id: AstId | undefined): Ast | undefined
/** Return the specified AST. Throws an exception if no AST with the provided ID was found. */
get(id: AstId): Ast
get(id: AstId | undefined): Ast | undefined
getToken(token: SyncTokenId): Token
getToken(token: SyncTokenId | undefined): Token | undefined
getAny(node: AstId | SyncTokenId): Ast | Token
@ -33,10 +42,12 @@ export interface Module {
}
export interface ModuleUpdate {
nodesAdded: AstId[]
nodesDeleted: AstId[]
fieldsUpdated: { id: AstId; fields: (readonly [string, unknown])[] }[]
nodesAdded: Set<AstId>
nodesDeleted: Set<AstId>
nodesUpdated: Set<AstId>
updateRoots: Set<AstId>
metadataUpdated: { id: AstId; changes: Map<string, unknown> }[]
origin: Origin | undefined
}
type YNode = FixedMap<AstFields>
@ -45,7 +56,7 @@ type YNodes = Y.Map<YNode>
export class MutableModule implements Module {
private readonly nodes: YNodes
get ydoc() {
private get ydoc() {
const ydoc = this.nodes.doc
assert(ydoc != null)
return ydoc
@ -53,7 +64,7 @@ export class MutableModule implements Module {
/** Return this module's copy of `ast`, if this module was created by cloning `ast`'s module. */
getVersion<T extends Ast>(ast: T): Mutable<T> {
const instance = this.checkedGet(ast.id)
const instance = this.get(ast.id)
return instance as Mutable<T>
}
@ -63,6 +74,14 @@ export class MutableModule implements Module {
return new MutableModule(doc)
}
applyEdit(edit: MutableModule, origin: Origin = defaultLocalOrigin) {
Y.applyUpdateV2(this.ydoc, Y.encodeStateAsUpdateV2(edit.ydoc), origin)
}
transact<T>(f: () => T, origin: Origin = defaultLocalOrigin): T {
return this.ydoc.transact(f, origin)
}
root(): MutableAst | undefined {
return this.rootPointer()?.expression
}
@ -93,6 +112,22 @@ export class MutableModule implements Module {
this.gc()
}
syncToCode(code: string) {
const root = this.root()
if (root) {
root.syncToCode(code)
} else {
this.replaceRoot(Ast.parse(code, this))
}
}
/** Update the module according to changes to its corresponding source code. */
applyTextEdits(textEdits: SourceRangeEdit[], metadataSource?: Module) {
const root = this.root()
assertDefined(root)
root.applyTextEdits(textEdits, metadataSource)
}
private gc() {
const live = new Set<AstId>()
const active = new Array<Ast>()
@ -129,7 +164,9 @@ export class MutableModule implements Module {
}
observe(observer: (update: ModuleUpdate) => void) {
const handle = (events: Y.YEvent<any>[]) => observer(this.observeEvents(events))
const handle = (events: Y.YEvent<any>[], transaction: Y.Transaction) => {
observer(this.observeEvents(events, tryAsOrigin(transaction.origin)))
}
// Attach the observer first, so that if an update hook causes changes in reaction to the initial state update, we
// won't miss them.
this.nodes.observeDeep(handle)
@ -142,15 +179,15 @@ export class MutableModule implements Module {
}
getStateAsUpdate(): ModuleUpdate {
const updateBuilder = new UpdateBuilder(this.nodes)
const updateBuilder = new UpdateBuilder(this, this.nodes, undefined)
for (const id of this.nodes.keys()) updateBuilder.addNode(id as AstId)
return updateBuilder
return updateBuilder.finish()
}
applyUpdate(update: Uint8Array, origin?: string): ModuleUpdate | undefined {
applyUpdate(update: Uint8Array, origin: Origin): ModuleUpdate | undefined {
let summary: ModuleUpdate | undefined
const observer = (events: Y.YEvent<any>[]) => {
summary = this.observeEvents(events)
summary = this.observeEvents(events, origin)
}
this.nodes.observeDeep(observer)
Y.applyUpdate(this.ydoc, update, origin)
@ -158,8 +195,8 @@ export class MutableModule implements Module {
return summary
}
private observeEvents(events: Y.YEvent<any>[]): ModuleUpdate {
const updateBuilder = new UpdateBuilder(this.nodes)
private observeEvents(events: Y.YEvent<any>[], origin: Origin | undefined): ModuleUpdate {
const updateBuilder = new UpdateBuilder(this, this.nodes, origin)
for (const event of events) {
if (event.target === this.nodes) {
// Updates to the node map.
@ -201,25 +238,23 @@ export class MutableModule implements Module {
updateBuilder.updateMetadata(id, changes)
}
}
return updateBuilder
return updateBuilder.finish()
}
clear() {
this.nodes.clear()
}
checkedGet(id: AstId): Mutable
checkedGet(id: AstId | undefined): Mutable | undefined
checkedGet(id: AstId | undefined): Mutable | undefined {
get(id: AstId): Mutable
get(id: AstId | undefined): Mutable | undefined
get(id: AstId | undefined): Mutable | undefined {
if (!id) return undefined
const ast = this.get(id)
const ast = this.tryGet(id)
assert(ast !== undefined, 'id in module')
return ast
}
get(id: AstId): Mutable | undefined
get(id: AstId | undefined): Mutable | undefined
get(id: AstId | undefined): Mutable | undefined {
tryGet(id: AstId | undefined): Mutable | undefined {
if (!id) return undefined
const nodeData = this.nodes.get(id)
if (!nodeData) return undefined
@ -228,19 +263,19 @@ export class MutableModule implements Module {
}
replace(id: AstId, value: Owned): Owned | undefined {
return this.get(id)?.replace(value)
return this.tryGet(id)?.replace(value)
}
replaceValue(id: AstId, value: Owned): Owned | undefined {
return this.get(id)?.replaceValue(value)
return this.tryGet(id)?.replaceValue(value)
}
take(id: AstId): Owned {
return this.replace(id, Wildcard.new(this)) || asOwned(this.checkedGet(id))
return this.replace(id, Wildcard.new(this)) || asOwned(this.get(id))
}
updateValue<T extends MutableAst>(id: AstId, f: (x: Owned) => Owned<T>): T | undefined {
return this.get(id)?.updateValue(f)
return this.tryGet(id)?.updateValue(f)
}
/////////////////////////////////////////////
@ -250,7 +285,7 @@ export class MutableModule implements Module {
}
private rootPointer(): MutableRootPointer | undefined {
const rootPointer = this.get(ROOT_ID)
const rootPointer = this.tryGet(ROOT_ID)
if (rootPointer) return rootPointer as MutableRootPointer
}
@ -269,8 +304,9 @@ export class MutableModule implements Module {
parent: undefined,
metadata: metadataFields,
})
this.nodes.set(id, fields)
return fields
const fieldObject = composeFieldData(fields, {})
this.nodes.set(id, fieldObject)
return fieldObject
}
/** @internal */
@ -283,7 +319,7 @@ export class MutableModule implements Module {
}
getAny(node: AstId | SyncTokenId): MutableAst | Token {
return isTokenId(node) ? this.getToken(node) : this.checkedGet(node)
return isTokenId(node) ? this.getToken(node) : this.get(node)
}
/** @internal Copy a node into the module, if it is bound to a different module. */
@ -306,8 +342,6 @@ export class MutableModule implements Module {
}
type MutableRootPointer = MutableInvalid & { get expression(): MutableAst | undefined }
/** @internal */
export interface RootPointer extends Invalid {}
function newAstId(type: string): AstId {
return `ast:${type}#${random.uint53()}` as AstId
@ -318,20 +352,24 @@ export function isAstId(value: string): value is AstId {
}
export const ROOT_ID = `Root` as AstId
class UpdateBuilder implements ModuleUpdate {
readonly nodesAdded: AstId[] = []
readonly nodesDeleted: AstId[] = []
readonly fieldsUpdated: { id: AstId; fields: (readonly [string, unknown])[] }[] = []
class UpdateBuilder {
readonly nodesAdded = new Set<AstId>()
readonly nodesDeleted = new Set<AstId>()
readonly nodesUpdated = new Set<AstId>()
readonly metadataUpdated: { id: AstId; changes: Map<string, unknown> }[] = []
readonly origin: Origin | undefined
private readonly module: Module
private readonly nodes: YNodes
constructor(nodes: YNodes) {
constructor(module: Module, nodes: YNodes, origin: Origin | undefined) {
this.module = module
this.nodes = nodes
this.origin = origin
}
addNode(id: AstId) {
this.nodesAdded.push(id)
this.nodesAdded.add(id)
this.updateAllFields(id)
}
@ -340,7 +378,7 @@ class UpdateBuilder implements ModuleUpdate {
}
updateFields(id: AstId, changes: Iterable<readonly [string, unknown]>) {
const fields = new Array<readonly [string, unknown]>()
let fieldsChanged = false
let metadataChanges = undefined
for (const entry of changes) {
const [key, value] = entry
@ -349,10 +387,10 @@ class UpdateBuilder implements ModuleUpdate {
metadataChanges = new Map<string, unknown>(value.entries())
} else {
assert(!(value instanceof Y.AbstractType))
fields.push(entry)
fieldsChanged = true
}
}
if (fields.length !== 0) this.fieldsUpdated.push({ id, fields })
if (fieldsChanged) this.nodesUpdated.add(id)
if (metadataChanges) this.metadataUpdated.push({ id, changes: metadataChanges })
}
@ -363,6 +401,11 @@ class UpdateBuilder implements ModuleUpdate {
}
deleteNode(id: AstId) {
this.nodesDeleted.push(id)
this.nodesDeleted.add(id)
}
finish(): ModuleUpdate {
const updateRoots = subtreeRoots(this.module, new Set(this.nodesUpdated.keys()))
return { ...this, updateRoots }
}
}

412
app/gui2/shared/ast/parse.ts
View File

@ -1,18 +1,36 @@
import * as map from 'lib0/map'
import type { AstId, Module, NodeChild, Owned } from '.'
import {
Token,
asOwned,
isTokenId,
parentId,
rewriteRefs,
subtreeRoots,
type AstId,
type NodeChild,
type Owned,
syncFields,
syncNodeMetadata,
} from '.'
import { assert, assertDefined, assertEqual } from '../util/assert'
import type { SourceRange, SourceRangeKey } from '../yjsModel'
import { IdMap, isUuid, sourceRangeFromKey, sourceRangeKey } from '../yjsModel'
import { parse_tree } from './ffi'
import { tryGetSoleValue, zip } from '../util/data/iterable'
import type { SourceRangeEdit, SpanTree } from '../util/data/text'
import {
applyTextEdits,
applyTextEditsToSpans,
enclosingSpans,
textChangeToEdits,
trimEnd,
} from '../util/data/text'
import {
IdMap,
isUuid,
rangeLength,
sourceRangeFromKey,
sourceRangeKey,
type SourceRange,
type SourceRangeKey,
} from '../yjsModel'
import { graphParentPointers } from './debug'
import { parse_tree, xxHash128 } from './ffi'
import * as RawAst from './generated/ast'
import { MutableModule } from './mutableModule'
import type { LazyObject } from './parserSupport'
@ -28,6 +46,8 @@ import {
Ident,
Import,
Invalid,
MutableAssignment,
MutableAst,
MutableBodyBlock,
MutableIdent,
NegationApp,
@ -39,11 +59,16 @@ import {
Wildcard,
} from './tree'
export function parseEnso(code: string): RawAst.Tree {
/** Return the raw parser output for the given code. */
export function parseEnso(code: string): RawAst.Tree.BodyBlock {
const blob = parse_tree(code)
return RawAst.Tree.read(new DataView(blob.buffer), blob.byteLength - 4)
const tree = RawAst.Tree.read(new DataView(blob.buffer), blob.byteLength - 4)
// The root of the parser output is always a body block.
assert(tree.type === RawAst.Tree.Type.BodyBlock)
return tree
}
/** Print the AST and re-parse it, copying `externalId`s (but not other metadata) from the original. */
export function normalize(rootIn: Ast): Ast {
const printed = print(rootIn)
const idMap = spanMapToIdMap(printed.info)
@ -55,27 +80,54 @@ export function normalize(rootIn: Ast): Ast {
return parsed
}
/** Produce `Ast` types from `RawAst` parser output. */
export function abstract(
module: MutableModule,
tree: RawAst.Tree.BodyBlock,
code: string,
substitutor?: (key: NodeKey) => Owned | undefined,
): { root: Owned<MutableBodyBlock>; spans: SpanMap; toRaw: Map<AstId, RawAst.Tree> }
export function abstract(
module: MutableModule,
tree: RawAst.Tree,
code: string,
substitutor?: (key: NodeKey) => Owned | undefined,
): { root: Owned; spans: SpanMap; toRaw: Map<AstId, RawAst.Tree> }
export function abstract(
module: MutableModule,
tree: RawAst.Tree,
code: string,
substitutor?: (key: NodeKey) => Owned | undefined,
): { root: Owned; spans: SpanMap; toRaw: Map<AstId, RawAst.Tree> } {
const abstractor = new Abstractor(module, code)
const abstractor = new Abstractor(module, code, substitutor)
const root = abstractor.abstractTree(tree).node
const spans = { tokens: abstractor.tokens, nodes: abstractor.nodes }
return { root, spans, toRaw: abstractor.toRaw }
return { root: root as Owned<MutableBodyBlock>, spans, toRaw: abstractor.toRaw }
}
/** Produces `Ast` types from `RawAst` parser output. */
class Abstractor {
private readonly module: MutableModule
private readonly code: string
private readonly substitutor: ((key: NodeKey) => Owned | undefined) | undefined
readonly nodes: NodeSpanMap
readonly tokens: TokenSpanMap
readonly toRaw: Map<AstId, RawAst.Tree>
constructor(module: MutableModule, code: string) {
/**
* @param module - Where to allocate the new nodes.
* @param code - Source code that will be used to resolve references in any passed `RawAst` objects.
* @param substitutor - A function that can inject subtrees for some spans, instead of the abstractor producing them.
* This can be used for incremental abstraction.
*/
constructor(
module: MutableModule,
code: string,
substitutor?: (key: NodeKey) => Owned | undefined,
) {
this.module = module
this.code = code
this.substitutor = substitutor
this.nodes = new Map()
this.tokens = new Map()
this.toRaw = new Map()
@ -88,6 +140,8 @@ class Abstractor {
const codeStart = whitespaceEnd
const codeEnd = codeStart + tree.childrenLengthInCodeParsed
const spanKey = nodeKey(codeStart, codeEnd - codeStart)
const substitute = this.substitutor?.(spanKey)
if (substitute) return { node: substitute, whitespace }
let node: Owned
switch (tree.type) {
case RawAst.Tree.Type.BodyBlock: {
@ -153,10 +207,11 @@ class Abstractor {
? [this.abstractToken(tree.opr.value)]
: Array.from(tree.opr.error.payload.operators, this.abstractToken.bind(this))
const rhs = tree.rhs ? this.abstractTree(tree.rhs) : undefined
if (opr.length === 1 && opr[0]?.node.code() === '.' && rhs?.node instanceof MutableIdent) {
const soleOpr = tryGetSoleValue(opr)
if (soleOpr?.node.code() === '.' && rhs?.node instanceof MutableIdent) {
// Propagate type.
const rhs_ = { ...rhs, node: rhs.node }
node = PropertyAccess.concrete(this.module, lhs, opr[0], rhs_)
node = PropertyAccess.concrete(this.module, lhs, soleOpr, rhs_)
} else {
node = OprApp.concrete(this.module, lhs, opr, rhs)
}
@ -259,7 +314,7 @@ class Abstractor {
return { whitespace, node }
}
private abstractChildren(tree: LazyObject) {
private abstractChildren(tree: LazyObject): NodeChild<Owned | Token>[] {
const children: NodeChild<Owned | Token>[] = []
const visitor = (child: LazyObject) => {
if (RawAst.Tree.isInstance(child)) {
@ -276,29 +331,38 @@ class Abstractor {
}
declare const nodeKeyBrand: unique symbol
/** A source-range key for an `Ast`. */
export type NodeKey = SourceRangeKey & { [nodeKeyBrand]: never }
declare const tokenKeyBrand: unique symbol
/** A source-range key for a `Token`. */
export type TokenKey = SourceRangeKey & { [tokenKeyBrand]: never }
/** Create a source-range key for an `Ast`. */
export function nodeKey(start: number, length: number): NodeKey {
return sourceRangeKey([start, start + length]) as NodeKey
}
/** Create a source-range key for a `Token`. */
export function tokenKey(start: number, length: number): TokenKey {
return sourceRangeKey([start, start + length]) as TokenKey
}
/** Maps from source ranges to `Ast`s. */
export type NodeSpanMap = Map<NodeKey, Ast[]>
/** Maps from source ranges to `Token`s. */
export type TokenSpanMap = Map<TokenKey, Token>
/** Maps from source ranges to `Ast`s and `Token`s. */
export interface SpanMap {
nodes: NodeSpanMap
tokens: TokenSpanMap
}
/** Code with an associated mapping to `Ast` types. */
interface PrintedSource {
info: SpanMap
code: string
}
/** Generate an `IdMap` from a `SpanMap`. */
export function spanMapToIdMap(spans: SpanMap): IdMap {
const idMap = new IdMap()
for (const [key, token] of spans.tokens.entries()) {
@ -314,6 +378,7 @@ export function spanMapToIdMap(spans: SpanMap): IdMap {
return idMap
}
/** Given a `SpanMap`, return a function that can look up source ranges by AST ID. */
export function spanMapToSpanGetter(spans: SpanMap): (id: AstId) => SourceRange | undefined {
const reverseMap = new Map<AstId, SourceRange>()
for (const [key, asts] of spans.nodes) {
@ -344,7 +409,7 @@ export function printAst(
): string {
let code = ''
for (const child of ast.concreteChildren(verbatim)) {
if (!isTokenId(child.node) && ast.module.checkedGet(child.node) === undefined) continue
if (!isTokenId(child.node) && ast.module.get(child.node) === undefined) continue
if (child.whitespace != null) {
code += child.whitespace
} else if (code.length != 0) {
@ -357,13 +422,16 @@ export function printAst(
info.tokens.set(span, token)
code += token.code()
} else {
const childNode = ast.module.checkedGet(child.node)
assert(childNode != null)
const childNode = ast.module.get(child.node)
code += childNode.printSubtree(info, offset + code.length, parentIndent, verbatim)
// Extra structural validation.
assertEqual(childNode.id, child.node)
if (parentId(childNode) !== ast.id) {
console.error(`Inconsistent parent pointer (expected ${ast.id})`, childNode)
console.error(
`Inconsistent parent pointer (expected ${ast.id})`,
childNode,
graphParentPointers(ast.module.root()!),
)
}
assertEqual(parentId(childNode), ast.id)
}
@ -404,7 +472,7 @@ export function printBlock(
}
const validIndent = (line.expression.whitespace?.length ?? 0) > (parentIndent?.length ?? 0)
code += validIndent ? line.expression.whitespace : blockIndent
const lineNode = block.module.checkedGet(line.expression.node)
const lineNode = block.module.get(line.expression.node)
assertEqual(lineNode.id, line.expression.node)
assertEqual(parentId(lineNode), block.id)
code += lineNode.printSubtree(info, offset + code.length, blockIndent, verbatim)
@ -416,7 +484,7 @@ export function printBlock(
}
/** Parse the input as a block. */
export function parseBlock(code: string, inModule?: MutableModule) {
export function parseBlock(code: string, inModule?: MutableModule): Owned<MutableBodyBlock> {
return parseBlockWithSpans(code, inModule).root
}
@ -424,58 +492,55 @@ export function parseBlock(code: string, inModule?: MutableModule) {
export function parse(code: string, module?: MutableModule): Owned {
const module_ = module ?? MutableModule.Transient()
const ast = parseBlock(code, module_)
const [expr] = ast.statements()
if (!expr) return ast
const parent = parentId(expr)
const soleStatement = tryGetSoleValue(ast.statements())
if (!soleStatement) return ast
const parent = parentId(soleStatement)
if (parent) module_.delete(parent)
expr.fields.set('parent', undefined)
return asOwned(expr)
soleStatement.fields.set('parent', undefined)
return asOwned(soleStatement)
}
/** Parse a block, and return it along with a mapping from source locations to parsed objects. */
export function parseBlockWithSpans(
code: string,
inModule?: MutableModule,
): { root: Owned<MutableBodyBlock>; spans: SpanMap } {
const tree = parseEnso(code)
const module = inModule ?? MutableModule.Transient()
return fromRaw(tree, code, module)
}
function fromRaw(
tree: RawAst.Tree,
code: string,
inModule?: MutableModule,
): {
root: Owned<MutableBodyBlock>
spans: SpanMap
toRaw: Map<AstId, RawAst.Tree>
} {
const module = inModule ?? MutableModule.Transient()
const ast = abstract(module, tree, code)
const spans = ast.spans
// The root of the tree produced by the parser is always a `BodyBlock`.
const root = ast.root as Owned<MutableBodyBlock>
return { root, spans, toRaw: ast.toRaw }
return abstract(module, tree, code)
}
/** Parse the input, and apply the given `IdMap`. Return the parsed tree, the updated `IdMap`, the span map, and a
* mapping to the `RawAst` representation.
*/
export function parseExtended(code: string, idMap?: IdMap | undefined, inModule?: MutableModule) {
const rawRoot = parseEnso(code)
const module = inModule ?? MutableModule.Transient()
const { root, spans, toRaw, idMapUpdates } = module.ydoc.transact(() => {
const { root, spans, toRaw } = fromRaw(rawRoot, code, module)
const { root, spans, toRaw } = module.transact(() => {
const { root, spans, toRaw } = abstract(module, rawRoot, code)
root.module.replaceRoot(root)
const idMapUpdates = idMap ? setExternalIds(root.module, spans, idMap) : 0
return { root, spans, toRaw, idMapUpdates }
}, 'local')
if (idMap) setExternalIds(root.module, spans, idMap)
return { root, spans, toRaw }
})
const getSpan = spanMapToSpanGetter(spans)
const idMapOut = spanMapToIdMap(spans)
return { root, idMap: idMapOut, getSpan, toRaw, idMapUpdates }
return { root, idMap: idMapOut, getSpan, toRaw }
}
export function setExternalIds(edit: MutableModule, spans: SpanMap, ids: IdMap) {
/** Return the number of `Ast`s in the tree, including the provided root. */
export function astCount(ast: Ast): number {
let count = 0
ast.visitRecursiveAst((_subtree) => {
count += 1
})
return count
}
/** Apply an `IdMap` to a module, using the given `SpanMap`.
* @returns The number of IDs that were assigned from the map.
*/
export function setExternalIds(edit: MutableModule, spans: SpanMap, ids: IdMap): number {
let astsMatched = 0
let asts = 0
edit.root()?.visitRecursiveAst((_ast) => (asts += 1))
for (const [key, externalId] of ids.entries()) {
const asts = spans.nodes.get(key as NodeKey)
if (asts) {
@ -486,9 +551,12 @@ export function setExternalIds(edit: MutableModule, spans: SpanMap, ids: IdMap)
}
}
}
return edit.root() ? asts - astsMatched : 0
return astsMatched
}
/** Try to find all the spans in `expected` in `encountered`. If any are missing, use the provided `code` to determine
* whether the lost spans are single-line or multi-line.
*/
function checkSpans(expected: NodeSpanMap, encountered: NodeSpanMap, code: string) {
const lost = new Array<readonly [NodeKey, Ast]>()
for (const [key, asts] of expected) {
@ -573,7 +641,7 @@ function resync(
const parentsOfBadSubtrees = new Set<AstId>()
const badAstIds = new Set(Array.from(badAsts, (ast) => ast.id))
for (const id of subtreeRoots(edit, badAstIds)) {
const parent = edit.checkedGet(id)?.parentId
const parent = edit.get(id)?.parentId
if (parent) parentsOfBadSubtrees.add(parent)
}
@ -587,11 +655,11 @@ function resync(
assertEqual(spanOfBadParent.length, parentsOfBadSubtrees.size)
for (const [id, span] of spanOfBadParent) {
const parent = edit.checkedGet(id)
const parent = edit.get(id)
const goodAst = goodSpans.get(span)?.[0]
// The parent of the root of a bad subtree must be a good AST.
assertDefined(goodAst)
parent.replaceValue(edit.copy(goodAst))
parent.syncToCode(goodAst.code())
}
console.warn(
@ -599,3 +667,235 @@ function resync(
parentsOfBadSubtrees,
)
}
/** @internal Recursion helper for {@link syntaxHash}. */
function hashSubtreeSyntax(ast: Ast, hashesOut: Map<SyntaxHash, Ast[]>): SyntaxHash {
let content = ''
content += ast.typeName + ':'
for (const child of ast.concreteChildren()) {
content += child.whitespace ?? '?'
if (isTokenId(child.node)) {
content += 'Token:' + hashString(ast.module.getToken(child.node).code())
} else {
content += hashSubtreeSyntax(ast.module.get(child.node), hashesOut)
}
}
const astHash = hashString(content)
map.setIfUndefined(hashesOut, astHash, (): Ast[] => []).unshift(ast)
return astHash
}
declare const brandHash: unique symbol
/** See {@link syntaxHash}. */
type SyntaxHash = string & { [brandHash]: never }
/** Applies the syntax-data hashing function to the input, and brands the result as a `SyntaxHash`. */
function hashString(input: string): SyntaxHash {
return xxHash128(input) as SyntaxHash
}
/** Calculates `SyntaxHash`es for the given node and all its children.
*
* Each `SyntaxHash` summarizes the syntactic content of an AST. If two ASTs have the same code and were parsed the
* same way (i.e. one was not parsed in a context that resulted in a different interpretation), they will have the same
* hash. Note that the hash is invariant to metadata, including `externalId` assignments.
*/
function syntaxHash(root: Ast) {
const hashes = new Map<SyntaxHash, Ast[]>()
const rootHash = hashSubtreeSyntax(root, hashes)
return { root: rootHash, hashes }
}
/** If the input is a block containing a single expression, return the expression; otherwise return the input. */
function rawBlockToInline(tree: RawAst.Tree.Tree) {
if (tree.type !== RawAst.Tree.Type.BodyBlock) return tree
return tryGetSoleValue(tree.statements)?.expression ?? tree
}
/** Update `ast` to match the given source code, while modifying it as little as possible. */
export function syncToCode(ast: MutableAst, code: string, metadataSource?: Module) {
const codeBefore = ast.code()
const textEdits = textChangeToEdits(codeBefore, code)
applyTextEditsToAst(ast, textEdits, metadataSource ?? ast.module)
}
/** Find nodes in the input `ast` that should be treated as equivalents of nodes in `parsedRoot`. */
function calculateCorrespondence(
ast: Ast,
astSpans: NodeSpanMap,
parsedRoot: Ast,
parsedSpans: NodeSpanMap,
textEdits: SourceRangeEdit[],
codeAfter: string,
): Map<AstId, Ast> {
const newSpans = new Map<AstId, SourceRange>()
for (const [key, asts] of parsedSpans) {
for (const ast of asts) newSpans.set(ast.id, sourceRangeFromKey(key))
}
// Retained-code matching: For each new tree, check for some old tree of the same type such that the new tree is the
// smallest node to contain all characters of the old tree's code that were not deleted in the edit.
//
// If the new node's span exactly matches the retained code, add the match to `toSync`. If the new node's span
// contains additional code, add the match to `candidates`.
const toSync = new Map<AstId, Ast>()
const candidates = new Map<AstId, Ast>()
const allSpansBefore = Array.from(astSpans.keys(), sourceRangeFromKey)
const spansBeforeAndAfter = applyTextEditsToSpans(textEdits, allSpansBefore).map(
([before, after]) => [before, trimEnd(after, codeAfter)] satisfies [any, any],
)
const partAfterToAstBefore = new Map<SourceRangeKey, Ast>()
for (const [spanBefore, partAfter] of spansBeforeAndAfter) {
const astBefore = astSpans.get(sourceRangeKey(spanBefore) as NodeKey)?.[0]!
partAfterToAstBefore.set(sourceRangeKey(partAfter), astBefore)
}
const matchingPartsAfter = spansBeforeAndAfter.map(([_before, after]) => after)
const parsedSpanTree = new AstWithSpans(parsedRoot, (id) => newSpans.get(id)!)
const astsMatchingPartsAfter = enclosingSpans(parsedSpanTree, matchingPartsAfter)
for (const [astAfter, partsAfter] of astsMatchingPartsAfter) {
for (const partAfter of partsAfter) {
const astBefore = partAfterToAstBefore.get(sourceRangeKey(partAfter))!
if (astBefore.typeName() === astAfter.typeName()) {
;(rangeLength(newSpans.get(astAfter.id)!) === rangeLength(partAfter)
? toSync
: candidates
).set(astBefore.id, astAfter)
break
}
}
}
// Index the matched nodes.
const oldIdsMatched = new Set<AstId>()
const newIdsMatched = new Set<AstId>()
for (const [oldId, newAst] of toSync) {
oldIdsMatched.add(oldId)
newIdsMatched.add(newAst.id)
}
// Movement matching: For each new tree that hasn't been matched, match it with any identical unmatched old tree.
const newHashes = syntaxHash(parsedRoot).hashes
const oldHashes = syntaxHash(ast).hashes
for (const [hash, newAsts] of newHashes) {
const unmatchedNewAsts = newAsts.filter((ast) => !newIdsMatched.has(ast.id))
const unmatchedOldAsts = oldHashes.get(hash)?.filter((ast) => !oldIdsMatched.has(ast.id)) ?? []
for (const [unmatchedNew, unmatchedOld] of zip(unmatchedNewAsts, unmatchedOldAsts)) {
toSync.set(unmatchedOld.id, unmatchedNew)
// Update the matched-IDs indices.
oldIdsMatched.add(unmatchedOld.id)
newIdsMatched.add(unmatchedNew.id)
}
}
// Apply any non-optimal span matches from `candidates`, if the nodes involved were not matched during
// movement-matching.
for (const [beforeId, after] of candidates) {
if (oldIdsMatched.has(beforeId) || newIdsMatched.has(after.id)) continue
toSync.set(beforeId, after)
}
return toSync
}
/** Update `ast` according to changes to its corresponding source code. */
export function applyTextEditsToAst(
ast: MutableAst,
textEdits: SourceRangeEdit[],
metadataSource: Module,
) {
const printed = print(ast)
const code = applyTextEdits(printed.code, textEdits)
const rawParsedBlock = parseEnso(code)
const rawParsed =
ast instanceof MutableBodyBlock ? rawParsedBlock : rawBlockToInline(rawParsedBlock)
const parsed = abstract(ast.module, rawParsed, code)
const toSync = calculateCorrespondence(
ast,
printed.info.nodes,
parsed.root,
parsed.spans.nodes,
textEdits,
code,
)
syncTree(ast, parsed.root, toSync, ast.module, metadataSource)
}
/** Replace `target` with `newContent`, reusing nodes according to the correspondence in `toSync`. */
function syncTree(
target: Ast,
newContent: Owned,
toSync: Map<AstId, Ast>,
edit: MutableModule,
metadataSource: Module,
) {
const newIdToEquivalent = new Map<AstId, AstId>()
for (const [beforeId, after] of toSync) newIdToEquivalent.set(after.id, beforeId)
const childReplacerFor = (parentId: AstId) => (id: AstId) => {
const original = newIdToEquivalent.get(id)
if (original) {
const replacement = edit.get(original)
if (replacement.parentId !== parentId) replacement.fields.set('parent', parentId)
return original
} else {
const child = edit.get(id)
if (child.parentId !== parentId) child.fields.set('parent', parentId)
}
}
const parentId = target.fields.get('parent')
assertDefined(parentId)
const parent = edit.get(parentId)
const targetSyncEquivalent = toSync.get(target.id)
const syncRoot = targetSyncEquivalent?.id === newContent.id ? targetSyncEquivalent : undefined
if (!syncRoot) {
parent.replaceChild(target.id, newContent)
newContent.fields.set('metadata', target.fields.get('metadata').clone())
}
const newRoot = syncRoot ? target : newContent
newRoot.visitRecursiveAst((ast) => {
const syncFieldsFrom = toSync.get(ast.id)
const editAst = edit.getVersion(ast)
if (syncFieldsFrom) {
const originalAssignmentExpression =
ast instanceof Assignment
? metadataSource.get(ast.fields.get('expression').node)
: undefined
syncFields(edit.getVersion(ast), syncFieldsFrom, childReplacerFor(ast.id))
if (editAst instanceof MutableAssignment && originalAssignmentExpression) {
if (editAst.expression.externalId !== originalAssignmentExpression.externalId)
editAst.expression.setExternalId(originalAssignmentExpression.externalId)
syncNodeMetadata(
editAst.expression.mutableNodeMetadata(),
originalAssignmentExpression.nodeMetadata,
)
}
} else {
rewriteRefs(editAst, childReplacerFor(ast.id))
}
return true
})
return newRoot
}
/** Provides a `SpanTree` view of an `Ast`, given span information. */
class AstWithSpans implements SpanTree<Ast> {
private readonly ast: Ast
private readonly getSpan: (astId: AstId) => SourceRange
constructor(ast: Ast, getSpan: (astId: AstId) => SourceRange) {
this.ast = ast
this.getSpan = getSpan
}
id(): Ast {
return this.ast
}
span(): SourceRange {
return this.getSpan(this.ast.id)
}
*children(): IterableIterator<SpanTree<Ast>> {
for (const child of this.ast.children()) {
if (child instanceof Ast) yield new AstWithSpans(child, this.getSpan)
}
}
}

51
app/gui2/shared/ast/sourceDocument.ts
View File

@ -1,5 +1,9 @@
import { print, type AstId, type Module, type ModuleUpdate } from '.'
import { rangeEquals, sourceRangeFromKey, type SourceRange } from '../yjsModel'
import { assertDefined } from '../util/assert'
import type { SourceRangeEdit } from '../util/data/text'
import { offsetEdit, textChangeToEdits } from '../util/data/text'
import type { Origin, SourceRange } from '../yjsModel'
import { rangeEquals, sourceRangeFromKey } from '../yjsModel'
/** Provides a view of the text representation of a module,
* and information about the correspondence between the text and the ASTs,
@ -8,10 +12,12 @@ import { rangeEquals, sourceRangeFromKey, type SourceRange } from '../yjsModel'
export class SourceDocument {
private text_: string
private readonly spans: Map<AstId, SourceRange>
private readonly observers: SourceDocumentObserver[]
private constructor(text: string, spans: Map<AstId, SourceRange>) {
this.text_ = text
this.spans = spans
this.observers = []
}
static Empty() {
@ -19,23 +25,43 @@ export class SourceDocument {
}
clear() {
if (this.text_ !== '') this.text_ = ''
if (this.spans.size !== 0) this.spans.clear()
if (this.text_ !== '') {
const range: SourceRange = [0, this.text_.length]
this.text_ = ''
this.notifyObservers([{ range, insert: '' }], undefined)
}
}
applyUpdate(module: Module, update: ModuleUpdate) {
for (const id of update.nodesDeleted) this.spans.delete(id)
const root = module.root()
if (!root) return
const subtreeTextEdits = new Array<SourceRangeEdit>()
const printed = print(root)
for (const [key, nodes] of printed.info.nodes) {
const range = sourceRangeFromKey(key)
for (const node of nodes) {
const oldSpan = this.spans.get(node.id)
if (!oldSpan || !rangeEquals(range, oldSpan)) this.spans.set(node.id, range)
if (update.updateRoots.has(node.id) && node.id !== root.id) {
assertDefined(oldSpan)
const oldCode = this.text_.slice(oldSpan[0], oldSpan[1])
const newCode = printed.code.slice(range[0], range[1])
const subedits = textChangeToEdits(oldCode, newCode).map((textEdit) =>
offsetEdit(textEdit, oldSpan[0]),
)
subtreeTextEdits.push(...subedits)
}
}
}
if (printed.code !== this.text_) this.text_ = printed.code
if (printed.code !== this.text_) {
const textEdits = update.updateRoots.has(root.id)
? [{ range: [0, this.text_.length] satisfies SourceRange, insert: printed.code }]
: subtreeTextEdits
this.text_ = printed.code
this.notifyObservers(textEdits, update.origin)
}
}
get text(): string {
@ -45,4 +71,23 @@ export class SourceDocument {
getSpan(id: AstId): SourceRange | undefined {
return this.spans.get(id)
}
observe(observer: SourceDocumentObserver) {
this.observers.push(observer)
if (this.text_.length) observer([{ range: [0, 0], insert: this.text_ }], undefined)
}
unobserve(observer: SourceDocumentObserver) {
const index = this.observers.indexOf(observer)
if (index !== undefined) this.observers.splice(index, 1)
}
private notifyObservers(textEdits: SourceRangeEdit[], origin: Origin | undefined) {
for (const o of this.observers) o(textEdits, origin)
}
}
export type SourceDocumentObserver = (
textEdits: SourceRangeEdit[],
origin: Origin | undefined,
) => void

5
app/gui2/shared/ast/token.ts
View File

@ -23,6 +23,7 @@ export interface SyncTokenId {
code_: string
tokenType_: RawAst.Token.Type | undefined
}
export class Token implements SyncTokenId {
readonly id: TokenId
code_: string
@ -47,6 +48,10 @@ export class Token implements SyncTokenId {
return new this(code, type, id)
}
static equal(a: SyncTokenId, b: SyncTokenId): boolean {
return a.tokenType_ === b.tokenType_ && a.code_ === b.code_
}
code(): string {
return this.code_
}

463
app/gui2/shared/ast/tree.ts
View File

@ -11,6 +11,7 @@ import type {
} from '.'
import {
MutableModule,
ROOT_ID,
Token,
asOwned,
isIdentifier,
@ -18,17 +19,23 @@ import {
isTokenId,
newExternalId,
parentId,
} from '.'
import { assert, assertDefined, assertEqual, bail } from '../util/assert'
import type { Result } from '../util/data/result'
import { Err, Ok } from '../util/data/result'
import type { SourceRangeEdit } from '../util/data/text'
import type { ExternalId, VisualizationMetadata } from '../yjsModel'
import { visMetadataEquals } from '../yjsModel'
import * as RawAst from './generated/ast'
import {
applyTextEditsToAst,
parse,
parseBlock,
print,
printAst,
printBlock,
} from '.'
import { assert, assertDefined, assertEqual, bail } from '../util/assert'
import type { Result } from '../util/data/result'
import { Err, Ok } from '../util/data/result'
import type { ExternalId, VisualizationMetadata } from '../yjsModel'
import * as RawAst from './generated/ast'
syncToCode,
} from './parse'
declare const brandAstId: unique symbol
export type AstId = string & { [brandAstId]: never }
@ -47,12 +54,22 @@ export function asNodeMetadata(map: Map<string, unknown>): NodeMetadata {
return map as unknown as NodeMetadata
}
/** @internal */
export interface AstFields {
interface RawAstFields {
id: AstId
type: string
parent: AstId | undefined
metadata: FixedMap<MetadataFields>
}
export interface AstFields extends RawAstFields, LegalFieldContent {}
function allKeys<T>(keys: Record<keyof T, any>): (keyof T)[] {
return Object.keys(keys) as any
}
const astFieldKeys = allKeys<RawAstFields>({
id: null,
type: null,
parent: null,
metadata: null,
})
export abstract class Ast {
readonly module: Module
/** @internal */
@ -104,8 +121,8 @@ export abstract class Ast {
}
}
visitRecursiveAst(visit: (ast: Ast) => void): void {
visit(this)
visitRecursiveAst(visit: (ast: Ast) => void | boolean): void {
if (visit(this) === false) return
for (const child of this.children()) {
if (!isToken(child)) child.visitRecursiveAst(visit)
}
@ -126,7 +143,7 @@ export abstract class Ast {
if (isTokenId(child.node)) {
yield this.module.getToken(child.node)
} else {
const node = this.module.checkedGet(child.node)
const node = this.module.get(child.node)
if (node) yield node
}
}
@ -134,11 +151,11 @@ export abstract class Ast {
get parentId(): AstId | undefined {
const parentId = this.fields.get('parent')
if (parentId !== 'ROOT_ID') return parentId
if (parentId !== ROOT_ID) return parentId
}
parent(): Ast | undefined {
return this.module.checkedGet(this.parentId)
return this.module.get(this.parentId)
}
static parseBlock(source: string, inModule?: MutableModule) {
@ -185,7 +202,7 @@ export abstract class MutableAst extends Ast {
replace<T extends MutableAst>(replacement: Owned<T>): Owned<typeof this> {
const parentId = this.fields.get('parent')
if (parentId) {
const parent = this.module.checkedGet(parentId)
const parent = this.module.get(parentId)
parent.replaceChild(this.id, replacement)
this.fields.set('parent', undefined)
}
@ -230,7 +247,7 @@ export abstract class MutableAst extends Ast {
takeIfParented(): Owned<typeof this> {
const parent = parentId(this)
if (parent) {
const parentAst = this.module.checkedGet(parent)
const parentAst = this.module.get(parent)
const placeholder = Wildcard.new(this.module)
parentAst.replaceChild(this.id, placeholder)
this.fields.set('parent', undefined)
@ -277,7 +294,17 @@ export abstract class MutableAst extends Ast {
mutableParent(): MutableAst | undefined {
const parentId = this.fields.get('parent')
if (parentId === 'ROOT_ID') return
return this.module.checkedGet(parentId)
return this.module.get(parentId)
}
/** Modify this tree to represent the given code, while minimizing changes from the current set of `Ast`s. */
syncToCode(code: string, metadataSource?: Module) {
syncToCode(this, code, metadataSource)
}
/** Update the AST according to changes to its corresponding source code. */
applyTextEdits(textEdits: SourceRangeEdit[], metadataSource?: Module) {
applyTextEditsToAst(this, textEdits, metadataSource ?? this.module)
}
///////////////////
@ -287,7 +314,7 @@ export abstract class MutableAst extends Ast {
if (module === this.module) return
for (const child of this.concreteChildren()) {
if (!isTokenId(child.node)) {
const childInForeignModule = module.checkedGet(child.node)
const childInForeignModule = module.get(child.node)
assert(childInForeignModule !== undefined)
const importedChild = this.module.copy(childInForeignModule)
importedChild.fields.set('parent', undefined)
@ -297,7 +324,11 @@ export abstract class MutableAst extends Ast {
}
/** @internal */
abstract replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>): void
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
const replacementId = this.claimChild(replacement)
const changes = rewriteRefs(this, (id) => (id === target ? replacementId : undefined))
assertEqual(changes, 1)
}
protected claimChild<T extends MutableAst>(child: Owned<T>): AstId
protected claimChild<T extends MutableAst>(child: Owned<T> | undefined): AstId | undefined
@ -306,6 +337,130 @@ export abstract class MutableAst extends Ast {
}
}
/** Values that may be found in fields of `Ast` subtypes. */
type FieldData =
| NodeChild<AstId>
| NodeChild
| NodeChild<SyncTokenId>
| FieldData[]
| undefined
| StructuralField
/** Objects that do not directly contain `AstId`s or `SyncTokenId`s, but may have `NodeChild` fields. */
type StructuralField =
| RawMultiSegmentAppSegment
| RawBlockLine
| RawOpenCloseTokens
| RawNameSpecification
/** Type whose fields are all suitable for storage as `Ast` fields. */
interface FieldObject {
[field: string]: FieldData
}
/** Returns the fields of an `Ast` subtype that are not part of `AstFields`. */
function* fieldDataEntries<Fields>(map: FixedMapView<Fields>) {
for (const entry of map.entries()) {
// All fields that are not from `AstFields` are `FieldData`.
if (!astFieldKeys.includes(entry[0] as any)) yield entry as [string, FieldData]
}
}
/** Apply the given function to each `AstId` in the fields of `ast`. For each value that it returns an output, that
* output will be substituted for the input ID.
*/
export function rewriteRefs(ast: MutableAst, f: (id: AstId) => AstId | undefined) {
let fieldsChanged = 0
for (const [key, value] of fieldDataEntries(ast.fields)) {
const newValue = rewriteFieldRefs(value, f)
if (newValue !== undefined) {
ast.fields.set(key as any, newValue)
fieldsChanged += 1
}
}
return fieldsChanged
}
/** Copy all fields except the `Ast` base fields from `ast2` to `ast1`. A reference-rewriting function will be applied
* to `AstId`s in copied fields; see {@link rewriteRefs}.
*/
export function syncFields(ast1: MutableAst, ast2: Ast, f: (id: AstId) => AstId | undefined) {
for (const [key, value] of fieldDataEntries(ast2.fields)) {
const changedValue = rewriteFieldRefs(value, f)
const newValue = changedValue ?? value
if (!fieldEqual(ast1.fields.get(key as any), newValue)) ast1.fields.set(key as any, newValue)
}
}
export function syncNodeMetadata(target: MutableNodeMetadata, source: NodeMetadata) {
const oldPos = target.get('position')
const newPos = source.get('position')
if (oldPos?.x !== newPos?.x || oldPos?.y !== newPos?.y) target.set('position', newPos)
const newVis = source.get('visualization')
if (!visMetadataEquals(target.get('visualization'), newVis)) target.set('visualization', newVis)
}
function rewriteFieldRefs(field: FieldData, f: (id: AstId) => AstId | undefined): FieldData {
if (field === undefined) return field
if ('node' in field) {
const child = field.node
if (isTokenId(child)) return
const newValue = f(child)
if (newValue !== undefined) {
field.node = newValue
return field
}
} else if (Array.isArray(field)) {
let fieldChanged = false
field.forEach((subfield, i) => {
const newValue = rewriteFieldRefs(subfield, f)
if (newValue !== undefined) {
field[i] = newValue
fieldChanged = true
}
})
if (fieldChanged) return field
} else {
const fieldObject = field satisfies StructuralField
let fieldChanged = false
for (const [key, value] of Object.entries(fieldObject)) {
const newValue = rewriteFieldRefs(value, f)
if (newValue !== undefined) {
// This update is safe because `newValue` was obtained by reading `fieldObject[key]` and modifying it in a
// type-preserving way.
;(fieldObject as any)[key] = newValue
fieldChanged = true
}
}
if (fieldChanged) return fieldObject
}
}
function fieldEqual(field1: FieldData, field2: FieldData): boolean {
if (field1 === undefined) return field2 === undefined
if (field2 === undefined) return false
if ('node' in field1 && 'node' in field2) {
if (field1['whitespace'] !== field2['whitespace']) return false
if (isTokenId(field1.node) && isTokenId(field2.node))
return Token.equal(field1.node, field2.node)
else return field1.node === field2.node
} else if ('node' in field1 || 'node' in field2) {
return false
} else if (Array.isArray(field1) && Array.isArray(field2)) {
return (
field1.length === field2.length && field1.every((value1, i) => fieldEqual(value1, field2[i]))
)
} else if (Array.isArray(field1) || Array.isArray(field2)) {
return false
} else {
const fieldObject1 = field1 satisfies StructuralField
const fieldObject2 = field2 satisfies StructuralField
const keys = new Set<string>()
for (const key of Object.keys(fieldObject1)) keys.add(key)
for (const key of Object.keys(fieldObject2)) keys.add(key)
for (const key of keys)
if (!fieldEqual((fieldObject1 as any)[key], (fieldObject2 as any)[key])) return false
return true
}
}
function applyMixins(derivedCtor: any, constructors: any[]) {
constructors.forEach((baseCtor) => {
Object.getOwnPropertyNames(baseCtor.prototype).forEach((name) => {
@ -320,10 +475,18 @@ function applyMixins(derivedCtor: any, constructors: any[]) {
interface AppFields {
function: NodeChild<AstId>
parens: { open: NodeChild<SyncTokenId>; close: NodeChild<SyncTokenId> } | undefined
nameSpecification: { name: NodeChild<SyncTokenId>; equals: NodeChild<SyncTokenId> } | undefined
parens: RawOpenCloseTokens | undefined
nameSpecification: RawNameSpecification | undefined
argument: NodeChild<AstId>
}
interface RawOpenCloseTokens {
open: NodeChild<SyncTokenId>
close: NodeChild<SyncTokenId>
}
interface RawNameSpecification {
name: NodeChild<SyncTokenId>
equals: NodeChild<SyncTokenId>
}
export class App extends Ast {
declare fields: FixedMap<AstFields & AppFields>
constructor(module: Module, fields: FixedMapView<AstFields & AppFields>) {
@ -344,7 +507,7 @@ export class App extends Ast {
) {
const base = module.baseObject('App')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
function: concreteChild(module, func, id_),
parens,
nameSpecification,
@ -361,7 +524,7 @@ export class App extends Ast {
) {
return App.concrete(
module,
unspaced(func),
autospaced(func),
undefined,
nameSpecification(argumentName),
autospaced(argument),
@ -369,26 +532,27 @@ export class App extends Ast {
}
get function(): Ast {
return this.module.checkedGet(this.fields.get('function').node)
return this.module.get(this.fields.get('function').node)
}
get argumentName(): Token | undefined {
return this.module.getToken(this.fields.get('nameSpecification')?.name.node)
}
get argument(): Ast {
return this.module.checkedGet(this.fields.get('argument').node)
return this.module.get(this.fields.get('argument').node)
}
*concreteChildren(verbatim?: boolean): IterableIterator<NodeChild> {
const { function: function_, parens, nameSpecification, argument } = getAll(this.fields)
yield function_
if (parens) yield parens.open
const spacedEquals = !!parens && !!nameSpecification?.equals.whitespace
yield ensureUnspaced(function_, verbatim)
const useParens = !!(parens && (nameSpecification || verbatim))
const spacedEquals = useParens && !!nameSpecification?.equals.whitespace
if (useParens) yield ensureSpaced(parens.open, verbatim)
if (nameSpecification) {
yield ensureSpacedIf(nameSpecification.name, !parens, verbatim)
yield ensureSpacedIf(nameSpecification.name, !useParens, verbatim)
yield ensureSpacedOnlyIf(nameSpecification.equals, spacedEquals, verbatim)
}
yield ensureSpacedOnlyIf(argument, !nameSpecification || spacedEquals, verbatim)
if (parens) yield parens.close
if (useParens) yield preferUnspaced(parens.close)
}
printSubtree(
@ -424,6 +588,9 @@ function ensureUnspaced<T>(child: NodeChild<T>, verbatim: boolean | undefined):
if (verbatim && child.whitespace != null) return child
return child.whitespace === '' ? child : { whitespace: '', ...child }
}
function preferUnspaced<T>(child: NodeChild<T>): NodeChild<T> {
return child.whitespace === undefined ? { whitespace: '', ...child } : child
}
export class MutableApp extends App implements MutableAst {
declare readonly module: MutableModule
declare readonly fields: FixedMap<AstFields & AppFields>
@ -437,14 +604,6 @@ export class MutableApp extends App implements MutableAst {
setArgument<T extends MutableAst>(value: Owned<T>) {
setNode(this.fields, 'argument', this.claimChild(value))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
if (this.fields.get('function').node === target) {
this.setFunction(replacement)
} else if (this.fields.get('argument').node === target) {
this.setArgument(replacement)
}
}
}
export interface MutableApp extends App, MutableAst {
get function(): MutableAst
@ -474,7 +633,7 @@ export class UnaryOprApp extends Ast {
) {
const base = module.baseObject('UnaryOprApp')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
operator,
argument: concreteChild(module, argument, id_),
})
@ -489,7 +648,7 @@ export class UnaryOprApp extends Ast {
return this.module.getToken(this.fields.get('operator').node)
}
get argument(): Ast | undefined {
return this.module.checkedGet(this.fields.get('argument')?.node)
return this.module.get(this.fields.get('argument')?.node)
}
*concreteChildren(_verbatim?: boolean): IterableIterator<NodeChild> {
@ -508,12 +667,6 @@ export class MutableUnaryOprApp extends UnaryOprApp implements MutableAst {
setArgument<T extends MutableAst>(argument: Owned<T> | undefined) {
setNode(this.fields, 'argument', this.claimChild(argument))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
if (this.fields.get('argument')?.node === target) {
this.setArgument(replacement)
}
}
}
export interface MutableUnaryOprApp extends UnaryOprApp, MutableAst {
get argument(): MutableAst | undefined
@ -538,7 +691,7 @@ export class NegationApp extends Ast {
static concrete(module: MutableModule, operator: NodeChild<Token>, argument: NodeChild<Owned>) {
const base = module.baseObject('NegationApp')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
operator,
argument: concreteChild(module, argument, id_),
})
@ -553,7 +706,7 @@ export class NegationApp extends Ast {
return this.module.getToken(this.fields.get('operator').node)
}
get argument(): Ast {
return this.module.checkedGet(this.fields.get('argument').node)
return this.module.get(this.fields.get('argument').node)
}
*concreteChildren(_verbatim?: boolean): IterableIterator<NodeChild> {
@ -569,12 +722,6 @@ export class MutableNegationApp extends NegationApp implements MutableAst {
setArgument<T extends MutableAst>(value: Owned<T>) {
setNode(this.fields, 'argument', this.claimChild(value))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
if (this.fields.get('argument')?.node === target) {
this.setArgument(replacement)
}
}
}
export interface MutableNegationApp extends NegationApp, MutableAst {
get argument(): MutableAst
@ -605,7 +752,7 @@ export class OprApp extends Ast {
) {
const base = module.baseObject('OprApp')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
lhs: concreteChild(module, lhs, id_),
operators,
rhs: concreteChild(module, rhs, id_),
@ -623,7 +770,7 @@ export class OprApp extends Ast {
}
get lhs(): Ast | undefined {
return this.module.checkedGet(this.fields.get('lhs')?.node)
return this.module.get(this.fields.get('lhs')?.node)
}
get operator(): Result<Token, NodeChild<Token>[]> {
const operators = this.fields.get('operators')
@ -635,7 +782,7 @@ export class OprApp extends Ast {
return opr ? Ok(opr.node) : Err(operators_)
}
get rhs(): Ast | undefined {
return this.module.checkedGet(this.fields.get('rhs')?.node)
return this.module.get(this.fields.get('rhs')?.node)
}
*concreteChildren(_verbatim?: boolean): IterableIterator<NodeChild> {
@ -658,14 +805,6 @@ export class MutableOprApp extends OprApp implements MutableAst {
setRhs<T extends MutableAst>(value: Owned<T>) {
setNode(this.fields, 'rhs', this.claimChild(value))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
if (this.fields.get('lhs')?.node === target) {
this.setLhs(replacement)
} else if (this.fields.get('rhs')?.node === target) {
this.setRhs(replacement)
}
}
}
export interface MutableOprApp extends OprApp, MutableAst {
get lhs(): MutableAst | undefined
@ -736,7 +875,7 @@ export class PropertyAccess extends Ast {
) {
const base = module.baseObject('PropertyAccess')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
lhs: concreteChild(module, lhs, id_),
operator,
rhs: concreteChild(module, rhs, id_),
@ -745,13 +884,13 @@ export class PropertyAccess extends Ast {
}
get lhs(): Ast | undefined {
return this.module.checkedGet(this.fields.get('lhs')?.node)
return this.module.get(this.fields.get('lhs')?.node)
}
get operator(): Token {
return this.module.getToken(this.fields.get('operator').node)
}
get rhs(): IdentifierOrOperatorIdentifierToken {
const ast = this.module.checkedGet(this.fields.get('rhs').node)
const ast = this.module.get(this.fields.get('rhs').node)
assert(ast instanceof Ident)
return ast.token as IdentifierOrOperatorIdentifierToken
}
@ -775,15 +914,6 @@ export class MutablePropertyAccess extends PropertyAccess implements MutableAst
const old = this.fields.get('rhs')
this.fields.set('rhs', old ? { ...old, node } : unspaced(node))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
if (this.fields.get('lhs')?.node === target) {
this.setLhs(replacement)
} else if (this.fields.get('rhs')?.node === target) {
assert(replacement instanceof MutableIdent)
this.setRhs(replacement.token)
}
}
}
export interface MutablePropertyAccess extends PropertyAccess, MutableAst {
get lhs(): MutableAst | undefined
@ -802,7 +932,7 @@ export class Generic extends Ast {
static concrete(module: MutableModule, children: NodeChild<Owned | Token>[]) {
const base = module.baseObject('Generic')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
children: children.map((child) => concreteChild(module, child, id_)),
})
return asOwned(new MutableGeneric(module, fields))
@ -815,14 +945,6 @@ export class Generic extends Ast {
export class MutableGeneric extends Generic implements MutableAst {
declare readonly module: MutableModule
declare readonly fields: FixedMap<AstFields & GenericFields>
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
const replacement_ = autospaced(this.claimChild(replacement))
this.fields.set(
'children',
this.fields.get('children').map((child) => (child.node === target ? replacement_ : child)),
)
}
}
export interface MutableGeneric extends Generic, MutableAst {}
applyMixins(MutableGeneric, [MutableAst])
@ -899,22 +1021,22 @@ export class Import extends Ast {
}
get polyglot(): Ast | undefined {
return this.module.checkedGet(this.fields.get('polyglot')?.body?.node)
return this.module.get(this.fields.get('polyglot')?.body?.node)
}
get from(): Ast | undefined {
return this.module.checkedGet(this.fields.get('from')?.body?.node)
return this.module.get(this.fields.get('from')?.body?.node)
}
get import_(): Ast | undefined {
return this.module.checkedGet(this.fields.get('import').body?.node)
return this.module.get(this.fields.get('import').body?.node)
}
get all(): Token | undefined {
return this.module.getToken(this.fields.get('all')?.node)
}
get as(): Ast | undefined {
return this.module.checkedGet(this.fields.get('as')?.body?.node)
return this.module.get(this.fields.get('as')?.body?.node)
}
get hiding(): Ast | undefined {
return this.module.checkedGet(this.fields.get('hiding')?.body?.node)
return this.module.get(this.fields.get('hiding')?.body?.node)
}
static concrete(
@ -928,7 +1050,7 @@ export class Import extends Ast {
) {
const base = module.baseObject('Import')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
polyglot: multiSegmentAppSegmentToRaw(module, polyglot, id_),
from: multiSegmentAppSegmentToRaw(module, from, id_),
import: multiSegmentAppSegmentToRaw(module, import_, id_),
@ -1023,21 +1145,6 @@ export class MutableImport extends Import implements MutableAst {
setHiding<T extends MutableAst>(value: Owned<T> | undefined) {
this.fields.set('hiding', this.toRaw(multiSegmentAppSegment('hiding', value)))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
const { polyglot, from, import: import_, as, hiding } = getAll(this.fields)
polyglot?.body?.node === target
? this.setPolyglot(replacement)
: from?.body?.node === target
? this.setFrom(replacement)
: import_.body?.node === target
? this.setImport(replacement)
: as?.body?.node === target
? this.setAs(replacement)
: hiding?.body?.node === target
? this.setHiding(replacement)
: bail(`Failed to find child ${target} in node ${this.externalId}.`)
}
}
export interface MutableImport extends Import, MutableAst {
get polyglot(): MutableAst | undefined
@ -1092,7 +1199,7 @@ export class TextLiteral extends Ast {
) {
const base = module.baseObject('TextLiteral')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
open,
newline,
elements: elements.map((elem) => concreteChild(module, elem, id_)),
@ -1119,14 +1226,6 @@ export class TextLiteral extends Ast {
export class MutableTextLiteral extends TextLiteral implements MutableAst {
declare readonly module: MutableModule
declare readonly fields: FixedMap<AstFields & TextLiteralFields>
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
const replacement_ = autospaced(this.claimChild(replacement))
this.fields.set(
'elements',
this.fields.get('elements').map((child) => (child.node === target ? replacement_ : child)),
)
}
}
export interface MutableTextLiteral extends TextLiteral, MutableAst {}
applyMixins(MutableTextLiteral, [MutableAst])
@ -1157,7 +1256,7 @@ export class Documented extends Ast {
) {
const base = module.baseObject('Documented')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
open,
elements: elements.map((elem) => concreteChild(module, elem, id_)),
newlines,
@ -1167,7 +1266,7 @@ export class Documented extends Ast {
}
get expression(): Ast | undefined {
return this.module.checkedGet(this.fields.get('expression')?.node)
return this.module.get(this.fields.get('expression')?.node)
}
*concreteChildren(_verbatim?: boolean): IterableIterator<NodeChild> {
@ -1185,18 +1284,6 @@ export class MutableDocumented extends Documented implements MutableAst {
setExpression<T extends MutableAst>(value: Owned<T> | undefined) {
this.fields.set('expression', unspaced(this.claimChild(value)))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
if (this.fields.get('expression')?.node === target) {
this.setExpression(replacement)
} else {
const replacement_ = unspaced(this.claimChild(replacement))
this.fields.set(
'elements',
this.fields.get('elements').map((child) => (child.node === target ? replacement_ : child)),
)
}
}
}
export interface MutableDocumented extends Documented, MutableAst {
get expression(): MutableAst | undefined
@ -1218,7 +1305,7 @@ export class Invalid extends Ast {
}
get expression(): Ast {
return this.module.checkedGet(this.fields.get('expression').node)
return this.module.get(this.fields.get('expression').node)
}
*concreteChildren(_verbatim?: boolean): IterableIterator<NodeChild> {
@ -1240,7 +1327,7 @@ export function invalidFields(
expression: NodeChild<Owned>,
): FixedMap<AstFields & InvalidFields> {
const id_ = base.get('id')
return setAll(base, { expression: concreteChild(module, expression, id_) })
return composeFieldData(base, { expression: concreteChild(module, expression, id_) })
}
export class MutableInvalid extends Invalid implements MutableAst {
declare readonly module: MutableModule
@ -1250,11 +1337,6 @@ export class MutableInvalid extends Invalid implements MutableAst {
private setExpression<T extends MutableAst>(value: Owned<T>) {
this.fields.set('expression', unspaced(this.claimChild(value)))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
assertEqual(this.fields.get('expression').node, target)
this.setExpression(replacement)
}
}
export interface MutableInvalid extends Invalid, MutableAst {
/** The `expression` getter is intentionally not narrowed to provide mutable access:
@ -1286,7 +1368,11 @@ export class Group extends Ast {
) {
const base = module.baseObject('Group')
const id_ = base.get('id')
const fields = setAll(base, { open, expression: concreteChild(module, expression, id_), close })
const fields = composeFieldData(base, {
open,
expression: concreteChild(module, expression, id_),
close,
})
return asOwned(new MutableGroup(module, fields))
}
@ -1297,7 +1383,7 @@ export class Group extends Ast {
}
get expression(): Ast | undefined {
return this.module.checkedGet(this.fields.get('expression')?.node)
return this.module.get(this.fields.get('expression')?.node)
}
*concreteChildren(_verbatim?: boolean): IterableIterator<NodeChild> {
@ -1314,11 +1400,6 @@ export class MutableGroup extends Group implements MutableAst {
setExpression<T extends MutableAst>(value: Owned<T> | undefined) {
this.fields.set('expression', unspaced(this.claimChild(value)))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
assertEqual(this.fields.get('expression')?.node, target)
this.setExpression(replacement)
}
}
export interface MutableGroup extends Group, MutableAst {
get expression(): MutableAst | undefined
@ -1344,7 +1425,7 @@ export class NumericLiteral extends Ast {
static concrete(module: MutableModule, tokens: NodeChild<Token>[]) {
const base = module.baseObject('NumericLiteral')
const fields = setAll(base, { tokens })
const fields = composeFieldData(base, { tokens })
return asOwned(new MutableNumericLiteral(module, fields))
}
@ -1355,8 +1436,6 @@ export class NumericLiteral extends Ast {
export class MutableNumericLiteral extends NumericLiteral implements MutableAst {
declare readonly module: MutableModule
declare readonly fields: FixedMap<AstFields & NumericLiteralFields>
replaceChild<T extends MutableAst>(_target: AstId, _replacement: Owned<T>) {}
}
export interface MutableNumericLiteral extends NumericLiteral, MutableAst {}
applyMixins(MutableNumericLiteral, [MutableAst])
@ -1398,10 +1477,10 @@ export class Function extends Ast {
}
get name(): Ast {
return this.module.checkedGet(this.fields.get('name').node)
return this.module.get(this.fields.get('name').node)
}
get body(): Ast | undefined {
return this.module.checkedGet(this.fields.get('body')?.node)
return this.module.get(this.fields.get('body')?.node)
}
get argumentDefinitions(): ArgumentDefinition[] {
return this.fields.get('argumentDefinitions').map((raw) =>
@ -1421,7 +1500,7 @@ export class Function extends Ast {
) {
const base = module.baseObject('Function')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
name: concreteChild(module, name, id_),
argumentDefinitions: argumentDefinitionsToRaw(module, argumentDefinitions, id_),
equals,
@ -1478,7 +1557,11 @@ export class Function extends Ast {
for (const def of argumentDefinitions) yield* def
yield { whitespace: equals.whitespace ?? ' ', node: this.module.getToken(equals.node) }
if (body)
yield ensureSpacedOnlyIf(body, !(this.module.get(body.node) instanceof BodyBlock), verbatim)
yield ensureSpacedOnlyIf(
body,
!(this.module.tryGet(body.node) instanceof BodyBlock),
verbatim,
)
}
}
export class MutableFunction extends Function implements MutableAst {
@ -1503,23 +1586,6 @@ export class MutableFunction extends Function implements MutableAst {
if (oldBody) newBody.push(oldBody.take())
return newBody
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
const { name, argumentDefinitions, body } = getAll(this.fields)
if (name.node === target) {
this.setName(replacement)
} else if (body?.node === target) {
this.setBody(replacement)
} else {
const replacement_ = this.claimChild(replacement)
const replaceChild = (child: NodeChild) =>
child.node === target ? { ...child, node: replacement_ } : child
this.fields.set(
'argumentDefinitions',
argumentDefinitions.map((def) => def.map(replaceChild)),
)
}
}
}
export interface MutableFunction extends Function, MutableAst {
get name(): MutableAst
@ -1551,7 +1617,7 @@ export class Assignment extends Ast {
) {
const base = module.baseObject('Assignment')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
pattern: concreteChild(module, pattern, id_),
equals,
expression: concreteChild(module, expression, id_),
@ -1569,10 +1635,10 @@ export class Assignment extends Ast {
}
get pattern(): Ast {
return this.module.checkedGet(this.fields.get('pattern').node)
return this.module.get(this.fields.get('pattern').node)
}
get expression(): Ast {
return this.module.checkedGet(this.fields.get('expression').node)
return this.module.get(this.fields.get('expression').node)
}
*concreteChildren(verbatim?: boolean): IterableIterator<NodeChild> {
@ -1592,15 +1658,6 @@ export class MutableAssignment extends Assignment implements MutableAst {
setExpression<T extends MutableAst>(value: Owned<T>) {
setNode(this.fields, 'expression', this.claimChild(value))
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
const { pattern, expression } = getAll(this.fields)
if (pattern.node === target) {
this.setPattern(replacement)
} else if (expression.node === target) {
this.setExpression(replacement)
}
}
}
export interface MutableAssignment extends Assignment, MutableAst {
get pattern(): MutableAst
@ -1625,7 +1682,7 @@ export class BodyBlock extends Ast {
static concrete(module: MutableModule, lines: OwnedBlockLine[]) {
const base = module.baseObject('BodyBlock')
const id_ = base.get('id')
const fields = setAll(base, {
const fields = composeFieldData(base, {
lines: lines.map((line) => lineToRaw(line, module, id_)),
})
return asOwned(new MutableBodyBlock(module, fields))
@ -1702,19 +1759,10 @@ export class MutableBodyBlock extends BodyBlock implements MutableAst {
const oldLines = this.fields.get('lines')
const filteredLines = oldLines.filter((line) => {
if (!line.expression) return true
return keep(this.module.checkedGet(line.expression.node))
return keep(this.module.get(line.expression.node))
})
this.fields.set('lines', filteredLines)
}
replaceChild<T extends MutableAst>(target: AstId, replacement: Owned<T>) {
const replacement_ = this.claimChild(replacement)
const updateLine = (line: RawBlockLine) =>
line.expression?.node === target
? { ...line, expression: { ...line.expression, node: replacement_ } }
: line
this.fields.set('lines', this.fields.get('lines').map(updateLine))
}
}
export interface MutableBodyBlock extends BodyBlock, MutableAst {
statements(): IterableIterator<MutableAst>
@ -1730,12 +1778,12 @@ interface Line<T> {
expression: NodeChild<T> | undefined
}
type RawBlockLine = RawLine<AstId>
interface RawBlockLine extends RawLine<AstId> {}
export type BlockLine = Line<Ast>
export type OwnedBlockLine = Line<Owned>
function lineFromRaw(raw: RawBlockLine, module: Module): BlockLine {
const expression = raw.expression ? module.checkedGet(raw.expression.node) : undefined
const expression = raw.expression ? module.get(raw.expression.node) : undefined
return {
newline: { ...raw.newline, node: module.getToken(raw.newline.node) },
expression: expression
@ -1748,9 +1796,7 @@ function lineFromRaw(raw: RawBlockLine, module: Module): BlockLine {
}
function ownedLineFromRaw(raw: RawBlockLine, module: MutableModule): OwnedBlockLine {
const expression = raw.expression
? module.checkedGet(raw.expression.node).takeIfParented()
: undefined
const expression = raw.expression ? module.get(raw.expression.node).takeIfParented() : undefined
return {
newline: { ...raw.newline, node: module.getToken(raw.newline.node) },
expression: expression
@ -1794,7 +1840,7 @@ export class Ident extends Ast {
static concrete(module: MutableModule, token: NodeChild<Token>) {
const base = module.baseObject('Ident')
const fields = setAll(base, { token })
const fields = composeFieldData(base, { token })
return asOwned(new MutableIdent(module, fields))
}
@ -1823,8 +1869,6 @@ export class MutableIdent extends Ident implements MutableAst {
this.fields.set('token', unspaced(toIdent(ident)))
}
replaceChild<T extends MutableAst>(_target: AstId, _replacement: Owned<T>) {}
code(): Identifier {
return this.token.code()
}
@ -1852,7 +1896,7 @@ export class Wildcard extends Ast {
static concrete(module: MutableModule, token: NodeChild<Token>) {
const base = module.baseObject('Wildcard')
const fields = setAll(base, { token })
const fields = composeFieldData(base, { token })
return asOwned(new MutableWildcard(module, fields))
}
@ -1869,8 +1913,6 @@ export class Wildcard extends Ast {
export class MutableWildcard extends Wildcard implements MutableAst {
declare readonly module: MutableModule
declare readonly fields: FixedMap<AstFields & WildcardFields>
replaceChild<T extends MutableAst>(_target: AstId, _replacement: Owned<T>) {}
}
export interface MutableWildcard extends Wildcard, MutableAst {}
applyMixins(MutableWildcard, [MutableAst])
@ -2010,19 +2052,36 @@ function getAll<Fields extends object>(map: FixedMapView<Fields>): Fields {
return Object.fromEntries(map.entries()) as Fields
}
declare const brandLegalFieldContent: unique symbol
/** Used to add a constraint to all `AstFields`s subtypes ensuring that they were produced by `composeFieldData`, which
* enforces a requirement that the provided fields extend `FieldObject`.
*/
interface LegalFieldContent {
[brandLegalFieldContent]: never
}
/** Modifies the input `map`. Returns the same object with an extended type. */
export function setAll<Fields1, Fields2 extends object>(
export function setAll<Fields1, Fields2 extends Record<string, any>>(
map: FixedMap<Fields1>,
fields: Fields2,
): FixedMap<Fields1 & Fields2> {
const map_ = map as FixedMap<Fields1 & Fields2>
for (const [k, v] of Object.entries(fields)) {
const k_ = k as string & (keyof Fields1 | keyof Fields2)
map_.set(k_, v)
map_.set(k_, v as any)
}
return map_
}
/** Modifies the input `map`. Returns the same object with an extended type. The added fields are required to have only
* types extending `FieldData`; the returned object is branded as `LegalFieldContent`. */
export function composeFieldData<Fields1, Fields2 extends FieldObject>(
map: FixedMap<Fields1>,
fields: Fields2,
): FixedMap<Fields1 & Fields2 & LegalFieldContent> {
return setAll(map, fields) as FixedMap<Fields1 & Fields2 & LegalFieldContent>
}
function claimChild<T extends MutableAst>(
module: MutableModule,
child: Owned<T>,

8
app/gui2/shared/util/data/__tests__/iterable.test.ts Normal file
View File

@ -0,0 +1,8 @@
import { tryGetSoleValue } from 'shared/util/data/iterable'
import { expect, test } from 'vitest'
test('tryGetSoleValue', () => {
expect(tryGetSoleValue([])).toBeUndefined()
expect(tryGetSoleValue([1])).toEqual(1)
expect(tryGetSoleValue([1, 2])).toBeUndefined()
})

137
app/gui2/shared/util/data/__tests__/text.test.ts Normal file
View File

@ -0,0 +1,137 @@
import { expect, test } from 'vitest'
import { applyTextEditsToSpans, textChangeToEdits, trimEnd } from '../text'
/** Tests that:
* - When the code in `a[0]` is edited to become the code in `b[0]`,
* `applyTextEditsToSpans` followed by `trimEnd` transforms the spans in `a.slice(1)` into the spans in `b.slice(1)`.
* - The same holds when editing from `b` to `a`.
*/
function checkCorrespondence(a: string[], b: string[]) {
checkCorrespondenceForward(a, b)
checkCorrespondenceForward(b, a)
}
/** Performs the same check as {@link checkCorrespondence}, for correspondences that are not expected to be reversible.
*/
function checkCorrespondenceForward(before: string[], after: string[]) {
const leadingSpacesAndLength = (input: string): [number, number] => [
input.lastIndexOf(' ') + 1,
input.length,
]
const spacesAndHyphens = ([spaces, length]: readonly [number, number]) => {
return ' '.repeat(spaces) + '-'.repeat(length - spaces)
}
const edits = textChangeToEdits(before[0]!, after[0]!)
const spansAfter = applyTextEditsToSpans(edits, before.slice(1).map(leadingSpacesAndLength)).map(
([_spanBefore, spanAfter]) => trimEnd(spanAfter, after[0]!),
)
expect([after[0]!, ...spansAfter.map(spacesAndHyphens)]).toEqual(after)
}
test('applyTextEditsToSpans: Add and remove argument names.', () => {
checkCorrespondence(
[
'func arg1 arg2', // prettier-ignore
'----',
' ----',
'---------',
' ----',
'--------------',
],
[
'func name1=arg1 name2=arg2',
'----',
' ----',
'---------------',
' ----',
'--------------------------',
],
)
})
test('applyTextEditsToSpans: Lengthen and shorten argument names.', () => {
checkCorrespondence(
[
'func name1=arg1 name2=arg2',
'----',
' ----',
'---------------',
' ----',
'--------------------------',
],
[
'func longName1=arg1 longName2=arg2',
'----',
' ----',
'-------------------',
' ----',
'----------------------------------',
],
)
})
test('applyTextEditsToSpans: Add and remove inner application.', () => {
checkCorrespondence(
[
'func bbb2', // prettier-ignore
'----',
' ----',
'---------',
],
[
'func aaa1 bbb2', // prettier-ignore
'----',
' ----',
'--------------',
],
)
})
test('applyTextEditsToSpans: Add and remove outer application.', () => {
checkCorrespondence(
[
'func arg1', // prettier-ignore
'----',
' ----',
'---------',
],
[
'func arg1 arg2', // prettier-ignore
'----',
' ----',
'---------',
],
)
})
test('applyTextEditsToSpans: Distinguishing repeated subexpressions.', () => {
checkCorrespondence(
[
'foo (2 + 2) bar () (2 + 2)', // prettier-ignore
' -----',
' -------',
' -----',
' -------',
],
[
'foo (2 + 2) bar (2 + 2) (2 + 2)', // prettier-ignore
' -----',
' -------',
' -----',
' -------',
],
)
})
test('applyTextEditsToSpans: Space after line content.', () => {
checkCorrespondenceForward(
[
'value = 1 +', // prettier-ignore
'-----------',
],
[
'value = 1 ', // prettier-ignore
'---------',
],
)
})

98
app/gui2/shared/util/data/iterable.ts Normal file
View File

@ -0,0 +1,98 @@
/** @file Functions for manipulating {@link Iterable}s. */
export function* empty(): Generator<never> {}
export function* range(start: number, stop: number, step = start <= stop ? 1 : -1) {
if ((step > 0 && start > stop) || (step < 0 && start < stop)) {
throw new Error(
"The range's step is in the wrong direction - please use Infinity or -Infinity as the endpoint for an infinite range.",
)
}
if (start <= stop) {
while (start < stop) {
yield start
start += step
}
} else {
while (start > stop) {
yield start
start += step
}
}
}
export function* map<T, U>(iter: Iterable<T>, map: (value: T) => U) {
for (const value of iter) {
yield map(value)
}
}
export function* chain<T>(...iters: Iterable<T>[]) {
for (const iter of iters) {
yield* iter
}
}
export function* zip<T, U>(left: Iterable<T>, right: Iterable<U>): Generator<[T, U]> {
const leftIterator = left[Symbol.iterator]()
const rightIterator = right[Symbol.iterator]()
while (true) {
const leftResult = leftIterator.next()
const rightResult = rightIterator.next()
if (leftResult.done || rightResult.done) break
yield [leftResult.value, rightResult.value]
}
}
export function* zipLongest<T, U>(
left: Iterable<T>,
right: Iterable<U>,
): Generator<[T | undefined, U | undefined]> {
const leftIterator = left[Symbol.iterator]()
const rightIterator = right[Symbol.iterator]()
while (true) {
const leftResult = leftIterator.next()
const rightResult = rightIterator.next()
if (leftResult.done && rightResult.done) break
yield [
leftResult.done ? undefined : leftResult.value,
rightResult.done ? undefined : rightResult.value,
]
}
}
export function tryGetSoleValue<T>(iter: Iterable<T>): T | undefined {
const iterator = iter[Symbol.iterator]()
const result = iterator.next()
if (result.done) return
const excessResult = iterator.next()
if (!excessResult.done) return
return result.value
}
/** Utility to simplify consuming an iterator a part at a time. */
export class Resumable<T> {
private readonly iterator: Iterator<T>
private current: IteratorResult<T>
constructor(iterable: Iterable<T>) {
this.iterator = iterable[Symbol.iterator]()
this.current = this.iterator.next()
}
/** The given function peeks at the current value. If the function returns `true`, the current value will be advanced
* and the function called again; if it returns `false`, the peeked value remains current and `advanceWhile` returns.
*/
advanceWhile(f: (value: T) => boolean) {
while (!this.current.done && f(this.current.value)) {
this.current = this.iterator.next()
}
}
/** Apply the given function to all values remaining in the iterator. */
forEach(f: (value: T) => void) {
while (!this.current.done) {
f(this.current.value)
this.current = this.iterator.next()
}
}
}

149
app/gui2/shared/util/data/text.ts Normal file
View File

@ -0,0 +1,149 @@
import diff from 'fast-diff'
import { rangeEncloses, rangeLength, type SourceRange } from '../../yjsModel'
import { Resumable } from './iterable'
export type SourceRangeEdit = { range: SourceRange; insert: string }
/** Given text and a set of `TextEdit`s, return the result of applying the edits to the text. */
export function applyTextEdits(oldText: string, textEdits: SourceRangeEdit[]) {
textEdits.sort((a, b) => a.range[0] - b.range[0])
let start = 0
let newText = ''
for (const textEdit of textEdits) {
newText += oldText.slice(start, textEdit.range[0])
newText += textEdit.insert
start = textEdit.range[1]
}
newText += oldText.slice(start)
return newText
}
/** Given text before and after a change, return one possible set of {@link SourceRangeEdit}s describing the change. */
export function textChangeToEdits(before: string, after: string): SourceRangeEdit[] {
const textEdits: SourceRangeEdit[] = []
let nextEdit: undefined | SourceRangeEdit
let pos = 0
// Sequences fast-diff emits:
// EQUAL, INSERT
// EQUAL, DELETE
// DELETE, EQUAL
// DELETE, INSERT
// INSERT, EQUAL
for (const [op, text] of diff(before, after)) {
switch (op) {
case diff.INSERT:
if (!nextEdit) nextEdit = { range: [pos, pos], insert: '' }
nextEdit.insert = text
break
case diff.EQUAL:
if (nextEdit) {
textEdits.push(nextEdit)
nextEdit = undefined
}
pos += text.length
break
case diff.DELETE: {
if (nextEdit) textEdits.push(nextEdit)
const endPos = pos + text.length
nextEdit = { range: [pos, endPos], insert: '' }
pos = endPos
break
}
}
}
if (nextEdit) textEdits.push(nextEdit)
return textEdits
}
/** Translate a `TextEdit` by the specified offset. */
export function offsetEdit(textEdit: SourceRangeEdit, offset: number): SourceRangeEdit {
return { ...textEdit, range: [textEdit.range[0] + offset, textEdit.range[1] + offset] }
}
/** Given:
* @param textEdits - A change described by a set of text edits.
* @param spansBefore - A collection of spans in the text before the edit.
* @returns - A sequence of: Each span from `spansBefore` paired with the smallest span of the text after the edit that
* contains all text that was in the original span and has not been deleted. */
export function applyTextEditsToSpans(textEdits: SourceRangeEdit[], spansBefore: SourceRange[]) {
// Gather start and end points.
const numerically = (a: number, b: number) => a - b
const starts = new Resumable(spansBefore.map(([start, _end]) => start).sort(numerically))
const ends = new Resumable(spansBefore.map(([_start, end]) => end).sort(numerically))
// Construct translations from old locations to new locations for all start and end points.
const startMap = new Map<number, number>()
const endMap = new Map<number, number>()
let offset = 0
for (const { range, insert } of textEdits) {
starts.advanceWhile((start) => {
if (start < range[0]) {
startMap.set(start, start + offset)
return true
} else if (start <= range[1]) {
startMap.set(start, range[0] + offset + insert.length)
return true
}
return false
})
ends.advanceWhile((end) => {
if (end <= range[0]) {
endMap.set(end, end + offset)
return true
} else if (end <= range[1]) {
endMap.set(end, range[0] + offset)
return true
}
return false
})
offset += insert.length - rangeLength(range)
}
starts.forEach((start) => startMap.set(start, start + offset))
ends.forEach((end) => endMap.set(end, end + offset))
// Apply the translations to the map.
const spansBeforeAndAfter = new Array<readonly [SourceRange, SourceRange]>()
for (const spanBefore of spansBefore) {
const startAfter = startMap.get(spanBefore[0])!
const endAfter = endMap.get(spanBefore[1])!
if (endAfter > startAfter) spansBeforeAndAfter.push([spanBefore, [startAfter, endAfter]])
}
return spansBeforeAndAfter
}
export interface SpanTree<NodeId> {
id(): NodeId
span(): SourceRange
children(): IterableIterator<SpanTree<NodeId>>
}
/** Given a span tree and some ranges, for each range find the smallest node that fully encloses it.
* Return nodes paired with the ranges that are most closely enclosed by them.
*/
export function enclosingSpans<NodeId>(
tree: SpanTree<NodeId>,
ranges: SourceRange[],
resultsOut?: [NodeId, SourceRange[]][],
) {
const results = resultsOut ?? []
for (const child of tree.children()) {
const childSpan = child.span()
const childRanges: SourceRange[] = []
ranges = ranges.filter((range) => {
if (rangeEncloses(childSpan, range)) {
childRanges.push(range)
return false
}
return true
})
if (childRanges.length) enclosingSpans(child, childRanges, results)
}
if (ranges.length) results.push([tree.id(), ranges])
return results
}
/** Return the given range with any trailing spaces stripped. */
export function trimEnd(range: SourceRange, text: string): SourceRange {
const trimmedLength = text.slice(range[0], range[1]).search(/ +$/)
return trimmedLength === -1 ? range : [range[0], range[0] + trimmedLength]
}

26
app/gui2/shared/yjsModel.ts
View File

@ -120,15 +120,25 @@ export class DistributedModule {
this.undoManager = new Y.UndoManager([this.doc.nodes])
}
transact<T>(fn: () => T): T {
return this.doc.ydoc.transact(fn, 'local')
}
dispose(): void {
this.doc.ydoc.destroy()
}
}
export const localOrigins = ['local', 'local:CodeEditor'] as const
export type LocalOrigin = (typeof localOrigins)[number]
export type Origin = LocalOrigin | 'remote'
/** Locally-originated changes not otherwise specified. */
export const defaultLocalOrigin: LocalOrigin = 'local'
export function isLocalOrigin(origin: string): origin is LocalOrigin {
const localOriginNames: readonly string[] = localOrigins
return localOriginNames.includes(origin)
}
export function tryAsOrigin(origin: string): Origin | undefined {
if (isLocalOrigin(origin)) return origin
if (origin === 'remote') return origin
}
export type SourceRange = readonly [start: number, end: number]
declare const brandSourceRangeKey: unique symbol
export type SourceRangeKey = string & { [brandSourceRangeKey]: never }
@ -230,6 +240,14 @@ export function rangeEquals(a: SourceRange, b: SourceRange): boolean {
return a[0] == b[0] && a[1] == b[1]
}
export function rangeIncludes(a: SourceRange, b: number): boolean {
return a[0] <= b && a[1] >= b
}
export function rangeLength(a: SourceRange): number {
return a[1] - a[0]
}
export function rangeEncloses(a: SourceRange, b: SourceRange): boolean {
return a[0] <= b[0] && a[1] >= b[1]
}

187
app/gui2/src/components/CodeEditor.vue
View File

@ -1,8 +1,8 @@
<script setup lang="ts">
import type { Diagnostic, Highlighter } from '@/components/CodeEditor/codemirror'
import type { ChangeSet, Diagnostic, Highlighter } from '@/components/CodeEditor/codemirror'
import { Annotation, StateEffect, StateField } from '@/components/CodeEditor/codemirror'
import { usePointer } from '@/composables/events'
import { useGraphStore, type NodeId } from '@/stores/graph'
import { asNodeId } from '@/stores/graph/graphDatabase'
import { useProjectStore } from '@/stores/project'
import { useSuggestionDbStore } from '@/stores/suggestionDatabase'
import { useAutoBlur } from '@/util/autoBlur'
@ -10,8 +10,11 @@ import { chain } from '@/util/data/iterable'
import { unwrap } from '@/util/data/result'
import { qnJoin, tryQualifiedName } from '@/util/qualifiedName'
import { useLocalStorage } from '@vueuse/core'
import { rangeEncloses } from 'shared/yjsModel'
import { computed, onMounted, ref, watch, watchEffect } from 'vue'
import { createDebouncer } from 'lib0/eventloop'
import { MutableModule } from 'shared/ast'
import { textChangeToEdits, type SourceRangeEdit } from 'shared/util/data/text'
import { rangeEncloses, type Origin } from 'shared/yjsModel'
import { computed, onMounted, onUnmounted, ref, shallowRef, watch, watchEffect } from 'vue'
// Use dynamic imports to aid code splitting. The codemirror dependency is quite large.
const {
@ -38,35 +41,41 @@ const suggestionDbStore = useSuggestionDbStore()
const rootElement = ref<HTMLElement>()
useAutoBlur(rootElement)
const executionContextDiagnostics = computed(() =>
projectStore.module && graphStore.moduleSource.text
? lsDiagnosticsToCMDiagnostics(graphStore.moduleSource.text, projectStore.diagnostics)
: [],
)
const executionContextDiagnostics = shallowRef<Diagnostic[]>([])
// Effect that can be applied to the document to invalidate the linter state.
const diagnosticsUpdated = StateEffect.define()
// State value that is perturbed by any `diagnosticsUpdated` effect.
const diagnosticsVersion = StateField.define({
create: (_state) => 0,
update: (value, transaction) => {
for (const effect of transaction.effects) {
if (effect.is(diagnosticsUpdated)) value += 1
}
return value
},
})
const expressionUpdatesDiagnostics = computed(() => {
const nodeMap = graphStore.db.nodeIdToNode
const updates = projectStore.computedValueRegistry.db
const panics = updates.type.reverseLookup('Panic')
const errors = updates.type.reverseLookup('DataflowError')
const diagnostics: Diagnostic[] = []
for (const id of chain(panics, errors)) {
const update = updates.get(id)
for (const externalId of chain(panics, errors)) {
const update = updates.get(externalId)
if (!update) continue
const externalId = graphStore.db.idFromExternal(id)
if (!externalId) continue
const node = nodeMap.get(asNodeId(externalId))
if (!node) continue
const rootSpan = graphStore.moduleSource.getSpan(node.rootSpan.id)
if (!rootSpan) continue
const [from, to] = rootSpan
const astId = graphStore.db.idFromExternal(externalId)
if (!astId) continue
const span = graphStore.moduleSource.getSpan(astId)
if (!span) continue
const [from, to] = span
switch (update.payload.type) {
case 'Panic': {
diagnostics.push({ from, to, message: update.payload.message, severity: 'error' })
break
}
case 'DataflowError': {
const error = projectStore.dataflowErrors.lookup(id)
const error = projectStore.dataflowErrors.lookup(externalId)
if (error?.value?.message) {
diagnostics.push({ from, to, message: error.value.message, severity: 'error' })
}
@ -80,21 +89,16 @@ const expressionUpdatesDiagnostics = computed(() => {
// == CodeMirror editor setup ==
const editorView = new EditorView()
const viewInitialized = ref(false)
watchEffect(() => {
const module = projectStore.module
if (!module) return
/*
const yText = module.doc.contents
const undoManager = module.undoManager
const awareness = projectStore.awareness.internal
extensions: [yCollab(yText, awareness, { undoManager }), ...]
*/
if (!graphStore.moduleSource.text) return
editorView.setState(
EditorState.create({
doc: graphStore.moduleSource.text,
extensions: [
minimalSetup,
updateListener(),
diagnosticsVersion,
syntaxHighlighting(defaultHighlightStyle as Highlighter),
bracketMatching(),
foldGutter(),
@ -157,13 +161,136 @@ watchEffect(() => {
return { dom }
}),
enso(),
linter(() => [...executionContextDiagnostics.value, ...expressionUpdatesDiagnostics.value]),
linter(
() => [...executionContextDiagnostics.value, ...expressionUpdatesDiagnostics.value],
{
needsRefresh(update) {
return (
update.state.field(diagnosticsVersion) !==
update.startState.field(diagnosticsVersion)
)
},
},
),
],
}),
)
viewInitialized.value = true
})
watch([executionContextDiagnostics, expressionUpdatesDiagnostics], () => forceLinting(editorView))
function changeSetToTextEdits(changes: ChangeSet) {
const textEdits = new Array<SourceRangeEdit>()
changes.iterChanges((from, to, _fromB, _toB, insert) =>
textEdits.push({ range: [from, to], insert: insert.toString() }),
)
return textEdits
}
function textEditToChangeSpec({ range: [from, to], insert }: SourceRangeEdit) {
return { from, to, insert }
}
let pendingChanges: ChangeSet | undefined
let currentModule: MutableModule | undefined
/** Set the editor contents the current module state, discarding any pending editor-initiated changes. */
function resetView() {
console.info(`Resetting the editor to the module code.`)
pendingChanges = undefined
currentModule = undefined
const viewText = editorView.state.doc.toString()
const code = graphStore.moduleSource.text
editorView.dispatch({
changes: textChangeToEdits(viewText, code).map(textEditToChangeSpec),
annotations: synchronizedModule.of(graphStore.startEdit()),
})
}
/** Apply any pending changes to the currently-synchronized module, clearing the set of pending changes. */
function commitPendingChanges() {
if (!pendingChanges || !currentModule) return
try {
currentModule.applyTextEdits(changeSetToTextEdits(pendingChanges), graphStore.viewModule())
graphStore.commitEdit(currentModule, undefined, 'local:CodeEditor')
} catch (error) {
console.error(`Code Editor failed to modify module`, error)
resetView()
}
pendingChanges = undefined
}
function updateListener() {
const debouncer = createDebouncer(0)
return EditorView.updateListener.of((update) => {
for (const transaction of update.transactions) {
const newModule = transaction.annotation(synchronizedModule)
if (newModule) {
// Flush the pipeline of edits that were based on the old module.
commitPendingChanges()
currentModule = newModule
} else if (transaction.docChanged && currentModule) {
pendingChanges = pendingChanges
? pendingChanges.compose(transaction.changes)
: transaction.changes
// Defer the update until after pending events have been processed, so that if changes are arriving faster than
// we would be able to apply them individually we coalesce them to keep up.
debouncer(commitPendingChanges)
}
}
})
}
let needResync = false
// Indicates a change updating the text to correspond to the given module state.
const synchronizedModule = Annotation.define<MutableModule>()
watch(
viewInitialized,
(ready) => {
if (ready) graphStore.moduleSource.observe(observeSourceChange)
},
{ immediate: true },
)
onUnmounted(() => graphStore.moduleSource.unobserve(observeSourceChange))
function observeSourceChange(textEdits: SourceRangeEdit[], origin: Origin | undefined) {
// If we received an update from outside the Code Editor while the editor contained uncommitted changes, we cannot
// proceed incrementally; we wait for the changes to be merged as Y.Js AST updates, and then set the view to the
// resulting code.
if (needResync) {
if (!pendingChanges) {
resetView()
needResync = false
}
return
}
// When we aren't in the `needResync` state, we can ignore updates that originated in the Code Editor.
if (origin === 'local:CodeEditor') return
if (pendingChanges) {
console.info(`Deferring update (editor dirty).`)
needResync = true
return
}
// If none of the above exit-conditions were reached, the transaction is applicable to our current state.
editorView.dispatch({
changes: textEdits.map(textEditToChangeSpec),
annotations: synchronizedModule.of(graphStore.startEdit()),
})
}
// The LS protocol doesn't identify what version of the file updates are in reference to. When diagnostics are received
// from the LS, we map them to the text assuming that they are applicable to the current version of the module. This
// will be correct if there is no one else editing, and we aren't editing faster than the LS can send updates. Typing
// too quickly can result in incorrect ranges, but at idle it should correct itself when we receive new diagnostics.
watch([viewInitialized, () => projectStore.diagnostics], ([ready, diagnostics]) => {
if (!ready) return
executionContextDiagnostics.value = graphStore.moduleSource.text
? lsDiagnosticsToCMDiagnostics(graphStore.moduleSource.text, diagnostics)
: []
})
watch([executionContextDiagnostics, expressionUpdatesDiagnostics], () => {
editorView.dispatch({ effects: diagnosticsUpdated.of(null) })
forceLinting(editorView)
})
onMounted(() => {
editorView.focus()

26
app/gui2/src/components/CodeEditor/codemirror.ts
View File

@ -13,7 +13,7 @@ export {
} from '@codemirror/language'
export { forceLinting, lintGutter, linter, type Diagnostic } from '@codemirror/lint'
export { highlightSelectionMatches } from '@codemirror/search'
export { EditorState } from '@codemirror/state'
export { Annotation, EditorState, StateEffect, StateField, type ChangeSet } from '@codemirror/state'
export { EditorView, tooltips, type TooltipView } from '@codemirror/view'
export { type Highlighter } from '@lezer/highlight'
export { minimalSetup } from 'codemirror'
@ -42,6 +42,7 @@ import {
import { styleTags, tags } from '@lezer/highlight'
import { EditorView } from 'codemirror'
import type { Diagnostic as LSDiagnostic } from 'shared/languageServerTypes'
import { tryGetSoleValue } from 'shared/util/data/iterable'
export function lsDiagnosticsToCMDiagnostics(
source: string,
@ -57,15 +58,17 @@ export function lsDiagnosticsToCMDiagnostics(
}
for (const diagnostic of diagnostics) {
if (!diagnostic.location) continue
results.push({
from:
(lineStartIndices[diagnostic.location.start.line] ?? 0) +
diagnostic.location.start.character,
to: (lineStartIndices[diagnostic.location.end.line] ?? 0) + diagnostic.location.end.character,
message: diagnostic.message,
severity:
diagnostic.kind === 'Error' ? 'error' : diagnostic.kind === 'Warning' ? 'warning' : 'info',
})
const from =
(lineStartIndices[diagnostic.location.start.line] ?? 0) + diagnostic.location.start.character
const to =
(lineStartIndices[diagnostic.location.end.line] ?? 0) + diagnostic.location.end.character
if (to > source.length || from > source.length) {
// Suppress temporary errors if the source is not the version of the document the LS is reporting diagnostics for.
continue
}
const severity =
diagnostic.kind === 'Error' ? 'error' : diagnostic.kind === 'Warning' ? 'warning' : 'info'
results.push({ from, to, message: diagnostic.message, severity })
}
return results
}
@ -115,8 +118,7 @@ function astToCodeMirrorTree(
const [start, end] = ast.span()
const children = ast.children()
const hasSingleTokenChild = children.length === 1 && children[0]!.isToken()
const childrenToConvert = hasSingleTokenChild ? [] : children
const childrenToConvert = tryGetSoleValue(children)?.isToken() ? [] : children
const tree = new Tree(
nodeSet.types[ast.inner.type + (ast.isToken() ? RawAst.Tree.typeNames.length : 0)]!,

8
app/gui2/src/components/GraphEditor.vue
View File

@ -163,7 +163,8 @@ function sourcePortForSelection() {
}
useEvent(window, 'keydown', (event) => {
interactionBindingsHandler(event) || graphBindingsHandler(event) || codeEditorHandler(event)
;(!keyboardBusy() && (interactionBindingsHandler(event) || graphBindingsHandler(event))) ||
(!keyboardBusyExceptIn(codeEditorArea.value) && codeEditorHandler(event))
})
useEvent(window, 'pointerdown', interactionBindingsHandler, { capture: true })
@ -281,7 +282,7 @@ const graphBindingsHandler = graphBindings.handler({
// For collapsed function, only selected nodes would affect placement of the output node.
collapsedFunctionEnv.nodeRects = collapsedFunctionEnv.selectedNodeRects
edit
.checkedGet(refactoredNodeId)
.get(refactoredNodeId)
.mutableNodeMetadata()
.set('position', { x: position.x, y: position.y })
if (outputNodeId != null) {
@ -290,7 +291,7 @@ const graphBindingsHandler = graphBindings.handler({
collapsedFunctionEnv,
)
edit
.checkedGet(outputNodeId)
.get(outputNodeId)
.mutableNodeMetadata()
.set('position', { x: position.x, y: position.y })
}
@ -325,7 +326,6 @@ const codeEditorArea = ref<HTMLElement>()
const showCodeEditor = ref(false)
const codeEditorHandler = codeEditorBindings.handler({
toggle() {
if (keyboardBusyExceptIn(codeEditorArea.value)) return false
showCodeEditor.value = !showCodeEditor.value
},
})

4
app/gui2/src/components/GraphEditor/GraphNode.vue
View File

@ -170,9 +170,7 @@ const dragPointer = usePointer((pos, event, type) => {
})
const matches = computed(() => prefixes.extractMatches(props.node.rootSpan))
const displayedExpression = computed(() =>
props.node.rootSpan.module.checkedGet(matches.value.innerExpr),
)
const displayedExpression = computed(() => props.node.rootSpan.module.get(matches.value.innerExpr))
const isOutputContextOverridden = computed({
get() {

12
app/gui2/src/components/GraphEditor/NodeWidgetTree.vue
View File

@ -36,12 +36,12 @@ function handleWidgetUpdates(update: WidgetUpdate) {
const { value, origin } = update.portUpdate
if (Ast.isAstId(origin)) {
const ast =
value instanceof Ast.Ast
? value
: value == null
? Ast.Wildcard.new(edit)
: Ast.parse(value, edit)
edit.replaceValue(origin as Ast.AstId, ast)
value instanceof Ast.Ast ? value : value == null ? Ast.Wildcard.new(edit) : undefined
if (ast) {
edit.replaceValue(origin as Ast.AstId, ast)
} else if (typeof value === 'string') {
edit.tryGet(origin)?.syncToCode(value)
}
} else {
console.error(`[UPDATE ${origin}] Invalid top-level origin. Expected expression ID.`)
}

4
app/gui2/src/stores/graph/graphDatabase.ts
View File

@ -140,13 +140,13 @@ export class GraphDb {
private nodeIdToPatternExprIds = new ReactiveIndex(this.nodeIdToNode, (id, entry) => {
const exprs: AstId[] = []
if (entry.pattern) entry.pattern.visitRecursiveAst((ast) => exprs.push(ast.id))
if (entry.pattern) entry.pattern.visitRecursiveAst((ast) => void exprs.push(ast.id))
return Array.from(exprs, (expr) => [id, expr])
})
private nodeIdToExprIds = new ReactiveIndex(this.nodeIdToNode, (id, entry) => {
const exprs: AstId[] = []
entry.rootSpan.visitRecursiveAst((ast) => exprs.push(ast.id))
entry.rootSpan.visitRecursiveAst((ast) => void exprs.push(ast.id))
return Array.from(exprs, (expr) => [id, expr])
})

51
app/gui2/src/stores/graph/index.ts
View File

@ -29,15 +29,14 @@ import { iteratorFilter } from 'lib0/iterator'
import { defineStore } from 'pinia'
import { SourceDocument } from 'shared/ast/sourceDocument'
import type { ExpressionUpdate, StackItem } from 'shared/languageServerTypes'
import {
sourceRangeKey,
visMetadataEquals,
type SourceRangeKey,
type VisualizationIdentifier,
type VisualizationMetadata,
import type {
LocalOrigin,
SourceRangeKey,
VisualizationIdentifier,
VisualizationMetadata,
} from 'shared/yjsModel'
import { defaultLocalOrigin, sourceRangeKey, visMetadataEquals } from 'shared/yjsModel'
import { computed, markRaw, reactive, ref, toRef, watch, type ShallowRef } from 'vue'
import * as Y from 'yjs'
export { type Node, type NodeId } from '@/stores/graph/graphDatabase'
@ -112,7 +111,7 @@ export const useGraphStore = defineStore('graph', () => {
id: AstId
changes: NodeMetadata
}[]
const dirtyNodeSet = new Set(update.fieldsUpdated.map(({ id }) => id))
const dirtyNodeSet = new Set(update.nodesUpdated)
if (moduleChanged || dirtyNodeSet.size !== 0) {
db.updateExternalIds(root)
toRaw = new Map()
@ -258,7 +257,7 @@ export const useGraphStore = defineStore('graph', () => {
for (const id of ids) {
const node = db.nodeIdToNode.get(id)
if (!node) continue
const outerExpr = edit.get(node.outerExprId)
const outerExpr = edit.tryGet(node.outerExprId)
if (outerExpr) Ast.deleteFromParentBlock(outerExpr)
nodeRects.delete(id)
}
@ -272,12 +271,12 @@ export const useGraphStore = defineStore('graph', () => {
const node = db.nodeIdToNode.get(id)
if (!node) return
edit((edit) => {
edit.getVersion(node.rootSpan).replaceValue(Ast.parse(content, edit))
edit.getVersion(node.rootSpan).syncToCode(content)
})
}
function transact(fn: () => void) {
return proj.module?.transact(fn)
syncModule.value!.transact(fn)
}
function stopCapturingUndo() {
@ -285,7 +284,7 @@ export const useGraphStore = defineStore('graph', () => {
}
function setNodePosition(nodeId: NodeId, position: Vec2) {
const nodeAst = syncModule.value?.get(nodeId)
const nodeAst = syncModule.value?.tryGet(nodeId)
if (!nodeAst) return
const oldPos = nodeAst.nodeMetadata.get('position')
if (oldPos?.x !== position.x || oldPos?.y !== position.y) {
@ -305,7 +304,7 @@ export const useGraphStore = defineStore('graph', () => {
}
function setNodeVisualizationId(nodeId: NodeId, vis: Opt<VisualizationIdentifier>) {
const nodeAst = syncModule.value?.get(nodeId)
const nodeAst = syncModule.value?.tryGet(nodeId)
if (!nodeAst) return
editNodeMetadata(nodeAst, (metadata) =>
metadata.set(
@ -316,7 +315,7 @@ export const useGraphStore = defineStore('graph', () => {
}
function setNodeVisualizationVisible(nodeId: NodeId, visible: boolean) {
const nodeAst = syncModule.value?.get(nodeId)
const nodeAst = syncModule.value?.tryGet(nodeId)
if (!nodeAst) return
editNodeMetadata(nodeAst, (metadata) =>
metadata.set(
@ -327,7 +326,7 @@ export const useGraphStore = defineStore('graph', () => {
}
function updateNodeRect(nodeId: NodeId, rect: Rect) {
const nodeAst = syncModule.value?.get(nodeId)
const nodeAst = syncModule.value?.tryGet(nodeId)
if (!nodeAst) return
if (rect.pos.equals(Vec2.Zero) && !nodeAst.nodeMetadata.get('position')) {
const { position } = nonDictatedPlacement(rect.size, {
@ -419,16 +418,18 @@ export const useGraphStore = defineStore('graph', () => {
* @param skipTreeRepair - If the edit is known not to require any parenthesis insertion, this may be set to `true`
* for better performance.
*/
function commitEdit(edit: MutableModule, skipTreeRepair?: boolean) {
function commitEdit(
edit: MutableModule,
skipTreeRepair?: boolean,
origin: LocalOrigin = defaultLocalOrigin,
) {
const root = edit.root()
if (!(root instanceof Ast.BodyBlock)) {
console.error(`BUG: Cannot commit edit: No module root block.`)
return
}
const module_ = proj.module
if (!module_) return
if (!skipTreeRepair) Ast.repair(root, edit)
Y.applyUpdateV2(syncModule.value!.ydoc, Y.encodeStateAsUpdateV2(edit.ydoc), 'local')
syncModule.value!.applyEdit(edit, origin)
}
/** Edit the AST module.
@ -444,16 +445,15 @@ export const useGraphStore = defineStore('graph', () => {
const edit = direct ? syncModule.value : syncModule.value?.edit()
assert(edit != null)
let result
edit.ydoc.transact(() => {
edit.transact(() => {
result = f(edit)
if (!skipTreeRepair) {
const root = edit.root()
assert(root instanceof Ast.BodyBlock)
Ast.repair(root, edit)
}
}, 'local')
if (!direct)
Y.applyUpdateV2(syncModule.value!.ydoc, Y.encodeStateAsUpdateV2(edit.ydoc), 'local')
if (!direct) syncModule.value!.applyEdit(edit)
})
return result!
}
@ -461,6 +461,10 @@ export const useGraphStore = defineStore('graph', () => {
edit((edit) => f(edit.getVersion(ast).mutableNodeMetadata()), true, true)
}
function viewModule(): Module {
return syncModule.value!
}
function mockExpressionUpdate(
locator: string | { binding: string; expr: string },
update: Partial<ExpressionUpdate>,
@ -587,6 +591,7 @@ export const useGraphStore = defineStore('graph', () => {
startEdit,
commitEdit,
edit,
viewModule,
addMissingImports,
}
})

6
app/gui2/src/stores/project/index.ts
View File

@ -35,7 +35,7 @@ import type {
StackItem,
VisualizationConfiguration,
} from 'shared/languageServerTypes'
import { DistributedProject, type ExternalId, type Uuid } from 'shared/yjsModel'
import { DistributedProject, localOrigins, type ExternalId, type Uuid } from 'shared/yjsModel'
import {
computed,
markRaw,
@ -544,7 +544,7 @@ export const useProjectStore = defineStore('project', () => {
const moduleName = projectModel.findModuleByDocId(guid)
if (moduleName == null) return null
const mod = await projectModel.openModule(moduleName)
mod?.undoManager.addTrackedOrigin('local')
for (const origin of localOrigins) mod?.undoManager.addTrackedOrigin(origin)
return mod
})
@ -573,7 +573,7 @@ export const useProjectStore = defineStore('project', () => {
const visualizationDataRegistry = new VisualizationDataRegistry(executionContext, dataConnection)
const computedValueRegistry = ComputedValueRegistry.WithExecutionContext(executionContext)
const diagnostics = ref<Diagnostic[]>([])
const diagnostics = shallowRef<Diagnostic[]>([])
executionContext.on('executionStatus', (newDiagnostics) => {
diagnostics.value = newDiagnostics
})

283
app/gui2/src/util/ast/__tests__/abstract.test.ts
View File

@ -3,9 +3,18 @@ import { Ast } from '@/util/ast'
import { initializeFFI } from 'shared/ast/ffi'
import { expect, test } from 'vitest'
import { MutableModule, escape, unescape, type Identifier } from '../abstract'
import { findExpressions, testCase, tryFindExpressions } from './testCase'
await initializeFFI()
test('Raw block abstracts to Ast.BodyBlock', () => {
const code = 'value = 2 + 2'
const rawBlock = Ast.parseEnso(code)
const edit = MutableModule.Transient()
const abstracted = Ast.abstract(edit, rawBlock, code)
expect(abstracted.root).toBeInstanceOf(Ast.BodyBlock)
})
//const disabledCases = [
// ' a',
// 'a ',
@ -432,10 +441,10 @@ test('Replace subexpression', () => {
const newValue = Ast.TextLiteral.new('bar', edit)
expect(newValue.code()).toBe("'bar'")
edit.replace(assignment.expression!.id, newValue)
const assignment_ = edit.get(assignment.id)!
const assignment_ = edit.tryGet(assignment.id)!
assert(assignment_ instanceof Ast.Assignment)
expect(assignment_.expression!.id).toBe(newValue.id)
expect(edit.get(assignment_.expression!.id)?.code()).toBe("'bar'")
expect(edit.tryGet(assignment_.expression!.id)?.code()).toBe("'bar'")
const printed = edit.getVersion(root).code()
expect(printed).toEqual("main =\n text1 = 'bar'\n")
})
@ -492,10 +501,10 @@ test('Construct app', () => {
const namedApp = Ast.App.new(
edit,
Ast.Ident.new(edit, 'func' as Identifier),
'argName' as Identifier,
'name' as Identifier,
Ast.Ident.new(edit, 'arg' as Identifier),
)
expect(namedApp.code()).toBe('func argName=arg')
expect(namedApp.code()).toBe('func name=arg')
})
test.each([
@ -534,34 +543,248 @@ test('Automatic parenthesis', () => {
expect(block.code()).toBe(correctCode)
})
test('Resync', () => {
const root = Ast.parseBlock('main = func arg1 arg2')
const module = root.module
module.replaceRoot(root)
const arg1 = 'arg1' as Identifier
const func = Ast.Function.fromStatements(
module,
'func' as Identifier,
[arg1],
[Ast.Ident.new(module, arg1)],
)
test('Tree repair: Non-canonical block line attribution', () => {
const beforeCase = testCase({
'func a b =': Ast.Function,
' c = a + b': Ast.Assignment,
'main =': Ast.Function,
' func arg1 arg2': Ast.App,
})
const before = beforeCase.statements
const edit = beforeCase.module.edit()
// Add a trailing line to the function's block. This is syntactically non-canonical; it should belong to the parent.
func.bodyAsBlock().insert(1, undefined)
expect(func.bodyAsBlock().lines.length).toBe(2)
root.insert(0, func)
const codeBeforeRepair = root.code()
const rootExternalIdBeforeRepair = root.externalId
const funcExternalIdBeforeRepair = func.externalId
Ast.repair(root, module)
const repairedRoot = module.root()
assert(repairedRoot instanceof Ast.BodyBlock)
const repairedFunc = repairedRoot.statements().next().value
assert(repairedFunc instanceof Ast.Function)
edit.getVersion(before['func a b =']).bodyAsBlock().insert(1, undefined)
const editedRoot = edit.root()
assert(editedRoot instanceof Ast.BodyBlock)
const editedCode = editedRoot.code()
expect(editedCode).toContain('\n\n')
const repair = edit.edit()
Ast.repair(editedRoot, repair)
const afterRepair = findExpressions(repair.root()!, {
'func a b =': Ast.Function,
'c = a + b': Ast.Assignment,
'main =': Ast.Function,
'func arg1 arg2': Ast.App,
})
const repairedFunc = afterRepair['func a b =']
assert(repairedFunc.body instanceof Ast.BodyBlock)
// The function's body has been corrected.
expect(repairedFunc.body.lines.length).toBe(1)
expect(repairedRoot.code()).toBe(codeBeforeRepair)
expect(repairedRoot.externalId).toBe(rootExternalIdBeforeRepair)
// The resync operation loses metadata within the non-canonical subtree.
expect(repairedFunc.body?.externalId).not.toBe(funcExternalIdBeforeRepair)
expect(repair.root()?.code()).toBe(editedCode)
// The repair maintains identities in all nodes.
expect(afterRepair['c = a + b'].id).toBe(before[' c = a + b'].id)
expect(afterRepair['func arg1 arg2'].id).toBe(before[' func arg1 arg2'].id)
// The repair maintains identities of other functions.
expect(afterRepair['main ='].id).toBe(before['main ='].id)
})
test('Code edit: Change argument type', () => {
const beforeRoot = Ast.parse('func arg1 arg2')
beforeRoot.module.replaceRoot(beforeRoot)
const before = findExpressions(beforeRoot, {
func: Ast.Ident,
arg1: Ast.Ident,
arg2: Ast.Ident,
'func arg1': Ast.App,
'func arg1 arg2': Ast.App,
})
const edit = beforeRoot.module.edit()
const newCode = 'func 123 arg2'
edit.getVersion(beforeRoot).syncToCode(newCode)
// Ensure the change was made.
expect(edit.root()?.code()).toBe(newCode)
// Ensure the identities of all the original nodes were maintained.
const after = findExpressions(edit.root()!, {
func: Ast.Ident,
'123': Ast.NumericLiteral,
arg2: Ast.Ident,
'func 123': Ast.App,
'func 123 arg2': Ast.App,
})
expect(after.func.id).toBe(before.func.id)
expect(after.arg2.id).toBe(before.arg2.id)
expect(after['func 123'].id).toBe(before['func arg1'].id)
expect(after['func 123 arg2'].id).toBe(before['func arg1 arg2'].id)
})
test('Code edit: Insert argument names', () => {
const beforeRoot = Ast.parse('func arg1 arg2')
beforeRoot.module.replaceRoot(beforeRoot)
const before = findExpressions(beforeRoot, {
func: Ast.Ident,
arg1: Ast.Ident,
arg2: Ast.Ident,
'func arg1': Ast.App,
'func arg1 arg2': Ast.App,
})
const edit = beforeRoot.module.edit()
const newCode = 'func name1=arg1 name2=arg2'
edit.getVersion(beforeRoot).syncToCode(newCode)
// Ensure the change was made.
expect(edit.root()?.code()).toBe(newCode)
// Ensure the identities of all the original nodes were maintained.
const after = findExpressions(edit.root()!, {
func: Ast.Ident,
arg1: Ast.Ident,
arg2: Ast.Ident,
'func name1=arg1': Ast.App,
'func name1=arg1 name2=arg2': Ast.App,
})
expect(after.func.id).toBe(before.func.id)
expect(after.arg1.id).toBe(before.arg1.id)
expect(after.arg2.id).toBe(before.arg2.id)
expect(after['func name1=arg1'].id).toBe(before['func arg1'].id)
expect(after['func name1=arg1 name2=arg2'].id).toBe(before['func arg1 arg2'].id)
})
test('Code edit: Remove argument names', () => {
const beforeRoot = Ast.parse('func name1=arg1 name2=arg2')
beforeRoot.module.replaceRoot(beforeRoot)
const before = findExpressions(beforeRoot, {
func: Ast.Ident,
arg1: Ast.Ident,
arg2: Ast.Ident,
'func name1=arg1': Ast.App,
'func name1=arg1 name2=arg2': Ast.App,
})
const edit = beforeRoot.module.edit()
const newCode = 'func arg1 arg2'
edit.getVersion(beforeRoot).syncToCode(newCode)
// Ensure the change was made.
expect(edit.root()?.code()).toBe(newCode)
// Ensure the identities of all the original nodes were maintained.
const after = findExpressions(edit.root()!, {
func: Ast.Ident,
arg1: Ast.Ident,
arg2: Ast.Ident,
'func arg1': Ast.App,
'func arg1 arg2': Ast.App,
})
expect(after.func.id).toBe(before.func.id)
expect(after.arg1.id).toBe(before.arg1.id)
expect(after.arg2.id).toBe(before.arg2.id)
expect(after['func arg1'].id).toBe(before['func name1=arg1'].id)
expect(after['func arg1 arg2'].id).toBe(before['func name1=arg1 name2=arg2'].id)
})
test('Code edit: Rearrange block', () => {
const beforeCase = testCase({
'main =': Ast.Function,
' call_result = func sum 12': Ast.Assignment,
' sum = value + 23': Ast.Assignment,
' value = 42': Ast.Assignment,
})
const before = beforeCase.statements
const edit = beforeCase.module.edit()
const newCode = [
'main =',
'\n value = 42',
'\n sum = value + 23',
'\n call_result = func sum 12',
].join('')
edit.root()!.syncToCode(newCode)
// Ensure the change was made.
expect(edit.root()?.code()).toBe(newCode)
// Ensure the identities of all the original nodes were maintained.
const after = tryFindExpressions(edit.root()!, {
'main =': Ast.Function,
'call_result = func sum 12': Ast.Assignment,
'sum = value + 23': Ast.Assignment,
'value = 42': Ast.Assignment,
})
expect(after['call_result = func sum 12']?.id).toBe(before[' call_result = func sum 12'].id)
expect(after['sum = value + 23']?.id).toBe(before[' sum = value + 23'].id)
expect(after['value = 42']?.id).toBe(before[' value = 42'].id)
})
test('Code edit: Inline expression change', () => {
const beforeRoot = Ast.parse('func name1=arg1 name2=arg2')
beforeRoot.module.replaceRoot(beforeRoot)
const before = findExpressions(beforeRoot, {
func: Ast.Ident,
arg1: Ast.Ident,
arg2: Ast.Ident,
'func name1=arg1': Ast.App,
'func name1=arg1 name2=arg2': Ast.App,
})
const edit = beforeRoot.module.edit()
const newArg1Code = 'arg1+1'
edit.getVersion(before['arg1']).syncToCode(newArg1Code)
// Ensure the change was made.
expect(edit.root()?.code()).toBe('func name1=arg1+1 name2=arg2')
// Ensure the identities of all the original nodes were maintained.
const after = findExpressions(edit.root()!, {
func: Ast.Ident,
arg1: Ast.Ident,
arg2: Ast.Ident,
'arg1+1': Ast.OprApp,
'func name1=arg1+1': Ast.App,
'func name1=arg1+1 name2=arg2': Ast.App,
})
expect(after.func.id).toBe(before.func.id)
expect(after.arg1.id).toBe(before.arg1.id)
expect(after.arg2.id).toBe(before.arg2.id)
expect(after['func name1=arg1+1'].id).toBe(before['func name1=arg1'].id)
expect(after['func name1=arg1+1 name2=arg2'].id).toBe(before['func name1=arg1 name2=arg2'].id)
})
test('Code edit: No-op inline expression change', () => {
const code = 'a = 1'
const expression = Ast.parse(code)
const module = expression.module
module.replaceRoot(expression)
expression.syncToCode(code)
expect(module.root()?.code()).toBe(code)
})
test('Code edit: No-op block change', () => {
const code = 'a = 1\nb = 2\n'
const block = Ast.parseBlock(code)
const module = block.module
module.replaceRoot(block)
block.syncToCode(code)
expect(module.root()?.code()).toBe(code)
})
test('Code edit: Shifting whitespace ownership', () => {
const beforeRoot = Ast.parseBlock('value = 1 +\n')
beforeRoot.module.replaceRoot(beforeRoot)
const before = findExpressions(beforeRoot, {
value: Ast.Ident,
'1': Ast.NumericLiteral,
'value = 1 +': Ast.Assignment,
})
const edit = beforeRoot.module.edit()
const newCode = 'value = 1 \n'
edit.getVersion(beforeRoot).syncToCode(newCode)
// Ensure the change was made.
expect(edit.root()?.code()).toBe(newCode)
// Ensure the identities of all the original nodes were maintained.
const after = findExpressions(edit.root()!, {
value: Ast.Ident,
'1': Ast.NumericLiteral,
'value = 1': Ast.Assignment,
})
expect(after.value.id).toBe(before.value.id)
expect(after['1'].id).toBe(before['1'].id)
expect(after['value = 1'].id).toBe(before['value = 1 +'].id)
})
test('Code edit merging', () => {
const block = Ast.parseBlock('a = 1\nb = 2')
const module = block.module
module.replaceRoot(block)
const editA = module.edit()
editA.getVersion(block).syncToCode('a = 10\nb = 2')
const editB = module.edit()
editB.getVersion(block).syncToCode('a = 1\nb = 20')
module.applyEdit(editA)
module.applyEdit(editB)
expect(module.root()?.code()).toBe('a = 10\nb = 20')
})

2
app/gui2/src/util/ast/__tests__/match.test.ts
View File

@ -91,7 +91,7 @@ test.each([
`'${target}' has CST ${extracted != null ? '' : 'not '}matching '${pattern}'`,
).toBe(extracted != null)
expect(
patternAst.match(targetAst)?.map((match) => module.get(match)?.code()),
patternAst.match(targetAst)?.map((match) => module.tryGet(match)?.code()),
extracted != null
? `'${target}' matches '${pattern}' with '__'s corresponding to ${JSON.stringify(extracted)
.slice(1, -1)

34
app/gui2/src/util/ast/__tests__/sourceDocument.test.ts Normal file
View File

@ -0,0 +1,34 @@
import { Ast } from '@/util/ast'
import { MutableModule } from 'shared/ast'
import { initializeFFI } from 'shared/ast/ffi'
import { SourceDocument } from 'shared/ast/sourceDocument'
import { applyTextEdits } from 'shared/util/data/text'
import { expect, test } from 'vitest'
import * as Y from 'yjs'
await initializeFFI()
test('Test SourceDocument', () => {
const syncModule = new MutableModule(new Y.Doc())
const sourceDoc = SourceDocument.Empty()
syncModule.observe((update) => sourceDoc.applyUpdate(syncModule, update))
const code = '1 + 1'
const edit1 = syncModule.edit()
const root = Ast.parseBlock(code, edit1)
edit1.replaceRoot(root)
syncModule.applyEdit(edit1)
expect(sourceDoc.text).toBe(code)
let observedText = ''
sourceDoc.observe((textEdits) => {
observedText = applyTextEdits(observedText, textEdits)
})
expect(observedText).toBe(code)
const newCode = '2'
const edit2 = syncModule.edit()
edit2.getVersion(root).syncToCode(newCode)
syncModule.applyEdit(edit2)
expect(sourceDoc.text).toBe(newCode)
expect(observedText).toBe(newCode)
})

74
app/gui2/src/util/ast/__tests__/testCase.ts Normal file
View File

@ -0,0 +1,74 @@
import { assert, assertDefined } from 'shared/util/assert'
import { Ast } from '..'
export type StringsWithTypeValues = Record<string, any>
export type WithValuesInstantiated<Spec extends StringsWithTypeValues> = {
[Name in keyof Spec]: InstanceType<Spec[Name]>
}
export type TestCase<T extends StringsWithTypeValues> = {
statements: WithValuesInstantiated<T>
module: Ast.Module
}
export function testCase<T extends StringsWithTypeValues>(spec: T): TestCase<T> {
let code = ''
for (const lineCode of Object.keys(spec)) {
code += lineCode
code += '\n'
}
const statementIndex = new Map<string, Ast.Ast>()
const parsed = Ast.parseBlock(code)
parsed.module.replaceRoot(parsed)
const statements = new Array<Ast.Ast>()
parsed.visitRecursiveAst((ast) => {
if (ast instanceof Ast.BodyBlock) statements.push(...ast.statements())
})
for (const statement of statements) {
const code = statement.code()
const trimmedFirstLine = code.split('\n', 1)[0]!.trim()
assert(
!statementIndex.has(trimmedFirstLine),
'Not implemented: Disambiguating duplicate statements.',
)
statementIndex.set(trimmedFirstLine, statement)
}
const result: Partial<WithValuesInstantiated<T>> = {}
for (const [lineCode, lineType] of Object.entries(spec)) {
const trimmed = lineCode.trim()
const statement = statementIndex.get(trimmed)
assertDefined(statement)
assert(statement instanceof lineType)
const key: keyof WithValuesInstantiated<T> = lineCode
result[key] = statement as any
}
return { statements: result as any, module: parsed.module }
}
export function tryFindExpressions<T extends StringsWithTypeValues>(
root: Ast.Ast,
expressions: T,
): Partial<WithValuesInstantiated<T>> {
const result: Partial<WithValuesInstantiated<T>> = {}
const expressionsSought = new Set(Object.keys(expressions))
root.visitRecursiveAst((ast) => {
const code = ast.code()
const trimmedFirstLine = code.split('\n', 1)[0]!.trim()
if (!expressionsSought.has(trimmedFirstLine)) return
const key: keyof WithValuesInstantiated<T> = trimmedFirstLine
if (!(ast instanceof expressions[trimmedFirstLine])) return
assert(!(key in result))
result[key] = ast as any
})
return result
}
export function findExpressions<T extends StringsWithTypeValues>(
root: Ast.Ast,
expressions: T,
): WithValuesInstantiated<T> {
const result = tryFindExpressions(root, expressions)
for (const key of Object.keys(expressions)) assert(key in result)
return result as any
}

4
app/gui2/src/util/ast/abstract.ts
View File

@ -71,7 +71,7 @@ export function tokenTree(root: Ast): TokenTree {
if (isTokenId(child.node)) {
return module.getToken(child.node).code()
} else {
const node = module.get(child.node)
const node = module.tryGet(child.node)
return node ? tokenTree(node) : '<missing>'
}
})
@ -85,7 +85,7 @@ export function tokenTreeWithIds(root: Ast): TokenTree {
if (isTokenId(child.node)) {
return module.getToken(child.node).code()
} else {
const node = module.get(child.node)
const node = module.tryGet(child.node)
return node ? tokenTreeWithIds(node) : ['<missing>']
}
}),

12
app/gui2/src/util/ast/extended.ts
View File

@ -1,4 +1,4 @@
import { assert } from '@/util/assert'
import { assert, assertDefined } from '@/util/assert'
import {
childrenAstNodesOrTokens,
parseEnso,
@ -14,6 +14,7 @@ import * as sha256 from 'lib0/hash/sha256'
import * as map from 'lib0/map'
import * as Ast from 'shared/ast/generated/ast'
import { Token, Tree } from 'shared/ast/generated/ast'
import { tryGetSoleValue } from 'shared/util/data/iterable'
import type { ExternalId, IdMap, SourceRange } from 'shared/yjsModel'
import { markRaw } from 'vue'
@ -61,12 +62,9 @@ export class AstExtended<T extends Tree | Token = Tree | Token, HasIdMap extends
const block = AstExtended.parse(code)
assert(block.isTree(Tree.Type.BodyBlock))
return block.map((block) => {
const statements = block.statements[Symbol.iterator]()
const firstLine = statements.next()
assert(!firstLine.done)
assert(!!statements.next().done)
assert(firstLine.value.expression != null)
return firstLine.value.expression
const soleStatement = tryGetSoleValue(block.statements)
assertDefined(soleStatement?.expression)
return soleStatement.expression
})
}

12
app/gui2/src/util/ast/index.ts
View File

@ -1,4 +1,4 @@
import { assert } from '@/util/assert'
import { assert, assertDefined } from '@/util/assert'
import * as Ast from '@/util/ast/abstract'
import { parseEnso } from '@/util/ast/abstract'
import { AstExtended as RawAstExtended } from '@/util/ast/extended'
@ -6,6 +6,7 @@ import { isResult, mapOk } from '@/util/data/result'
import * as map from 'lib0/map'
import * as RawAst from 'shared/ast/generated/ast'
import { LazyObject, LazySequence } from 'shared/ast/parserSupport'
import { tryGetSoleValue } from 'shared/util/data/iterable'
import type { SourceRange } from 'shared/yjsModel'
export { Ast, RawAst, RawAstExtended, parseEnso }
@ -19,12 +20,9 @@ export type HasAstRange = SourceRange | RawAst.Tree | RawAst.Token
export function parseEnsoLine(code: string): RawAst.Tree {
const block = Ast.parseEnso(code)
assert(block.type === RawAst.Tree.Type.BodyBlock)
const statements = block.statements[Symbol.iterator]()
const firstLine = statements.next()
assert(!firstLine.done)
assert(!!statements.next().done)
assert(firstLine.value.expression != null)
return firstLine.value.expression
const soleExpression = tryGetSoleValue(block.statements)?.expression
assertDefined(soleExpression)
return soleExpression
}
/**

2
app/gui2/src/util/ast/match.ts
View File

@ -36,7 +36,7 @@ export class Pattern {
/** Create a new concrete example of the pattern, with the placeholders replaced with the given subtrees. */
instantiate(edit: MutableModule, subtrees: Ast.Owned[]): Ast.Owned {
const template = edit.copy(this.template)
const placeholders = findPlaceholders(template, this.placeholder).map((ast) => edit.get(ast))
const placeholders = findPlaceholders(template, this.placeholder).map((ast) => edit.tryGet(ast))
for (const [placeholder, replacement] of zipLongest(placeholders, subtrees)) {
assertDefined(placeholder)
assertDefined(replacement)

2
app/gui2/src/util/ast/prefixes.ts
View File

@ -29,7 +29,7 @@ export class Prefixes<T extends Record<keyof T, Pattern>> {
Object.entries<Pattern>(this.prefixes).map(([name, pattern]) => {
const matches = pattern.match(expression)
const lastMatch = matches != null ? matches[matches.length - 1] : undefined
if (lastMatch) expression = expression.module.checkedGet(lastMatch)
if (lastMatch) expression = expression.module.get(lastMatch)
return [name, matches]
}),
) as Matches<T>

63
app/gui2/src/util/data/iterable.ts
View File

@ -1,62 +1 @@
/** @file Functions for manipulating {@link Iterable}s. */
export function* empty(): Generator<never> {}
export function* range(start: number, stop: number, step = start <= stop ? 1 : -1) {
if ((step > 0 && start > stop) || (step < 0 && start < stop)) {
throw new Error(
"The range's step is in the wrong direction - please use Infinity or -Infinity as the endpoint for an infinite range.",
)
}
if (start <= stop) {
while (start < stop) {
yield start
start += step
}
} else {
while (start > stop) {
yield start
start += step
}
}
}
export function* map<T, U>(iter: Iterable<T>, map: (value: T) => U) {
for (const value of iter) {
yield map(value)
}
}
export function* chain<T>(...iters: Iterable<T>[]) {
for (const iter of iters) {
yield* iter
}
}
export function* zip<T, U>(left: Iterable<T>, right: Iterable<U>): Generator<[T, U]> {
const leftIterator = left[Symbol.iterator]()
const rightIterator = right[Symbol.iterator]()
while (true) {
const leftResult = leftIterator.next()
const rightResult = rightIterator.next()
if (leftResult.done || rightResult.done) break
yield [leftResult.value, rightResult.value]
}
}
export function* zipLongest<T, U>(
left: Iterable<T>,
right: Iterable<U>,
): Generator<[T | undefined, U | undefined]> {
const leftIterator = left[Symbol.iterator]()
const rightIterator = right[Symbol.iterator]()
while (true) {
const leftResult = leftIterator.next()
const rightResult = rightIterator.next()
if (leftResult.done && rightResult.done) break
yield [
leftResult.done ? undefined : leftResult.value,
rightResult.done ? undefined : rightResult.value,
]
}
}
export * from 'shared/util/data/iterable'

2
app/gui2/ydoc-server/edits.ts
View File

@ -23,7 +23,7 @@ export function applyDocumentUpdates(
synced: EnsoFileParts,
update: ModuleUpdate,
): AppliedUpdates {
const codeChanged = update.fieldsUpdated.length !== 0
const codeChanged = update.nodesUpdated.size !== 0
let idsChanged = false
let metadataChanged = false
for (const { changes } of update.metadataUpdated) {

43
app/gui2/ydoc-server/languageServerSession.ts
View File

@ -4,14 +4,8 @@ import * as map from 'lib0/map'
import { ObservableV2 } from 'lib0/observable'
import * as random from 'lib0/random'
import * as Y from 'yjs'
import {
Ast,
MutableModule,
parseBlockWithSpans,
setExternalIds,
spanMapToIdMap,
} from '../shared/ast'
import { print } from '../shared/ast/parse'
import * as Ast from '../shared/ast'
import { astCount } from '../shared/ast'
import { EnsoFileParts, combineFileParts, splitFileContents } from '../shared/ensoFile'
import { LanguageServer, computeTextChecksum } from '../shared/languageServer'
import { Checksum, FileEdit, Path, TextEdit, response } from '../shared/languageServerTypes'
@ -416,7 +410,7 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
const update = this.updateToApply
this.updateToApply = null
const syncModule = new MutableModule(this.doc.ydoc)
const syncModule = new Ast.MutableModule(this.doc.ydoc)
const moduleUpdate = syncModule.applyUpdate(update, 'remote')
if (moduleUpdate && this.syncedContent) {
const synced = splitFileContents(this.syncedContent)
@ -511,24 +505,35 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
const nodeMeta = Object.entries(metadata.ide.node)
let parsedSpans
const syncModule = new MutableModule(this.doc.ydoc)
const syncModule = new Ast.MutableModule(this.doc.ydoc)
if (code !== this.syncedCode) {
const { root, spans } = parseBlockWithSpans(code, syncModule)
syncModule.syncRoot(root)
parsedSpans = spans
const syncRoot = syncModule.root()
if (syncRoot) {
const edit = syncModule.edit()
edit.getVersion(syncRoot).syncToCode(code)
const editedRoot = edit.root()
if (editedRoot instanceof Ast.BodyBlock) Ast.repair(editedRoot, edit)
syncModule.applyEdit(edit)
} else {
const { root, spans } = Ast.parseBlockWithSpans(code, syncModule)
syncModule.syncRoot(root)
parsedSpans = spans
}
}
const astRoot = syncModule.root()
if (!astRoot) return
if ((code !== this.syncedCode || idMapJson !== this.syncedIdMap) && idMapJson) {
const idMap = deserializeIdMap(idMapJson)
const spans = parsedSpans ?? print(astRoot).info
const newExternalIds = setExternalIds(syncModule, spans, idMap)
if (newExternalIds !== 0) {
const spans = parsedSpans ?? Ast.print(astRoot).info
const idsAssigned = Ast.setExternalIds(syncModule, spans, idMap)
const numberOfAsts = astCount(astRoot)
const idsNotSetByMap = numberOfAsts - idsAssigned
if (idsNotSetByMap > 0) {
if (code !== this.syncedCode) {
unsyncedIdMap = spanMapToIdMap(spans)
unsyncedIdMap = Ast.spanMapToIdMap(spans)
} else {
console.warn(
`The LS sent an IdMap-only edit that is missing ${newExternalIds} of our expected ASTs.`,
`The LS sent an IdMap-only edit that is missing ${idsNotSetByMap} of our expected ASTs.`,
)
}
}
@ -539,7 +544,7 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
metadataJson !== this.syncedMetaJson) &&
nodeMeta.length !== 0
) {
const externalIdToAst = new Map<ExternalId, Ast>()
const externalIdToAst = new Map<ExternalId, Ast.Ast>()
astRoot.visitRecursiveAst((ast) => {
if (!externalIdToAst.has(ast.externalId)) externalIdToAst.set(ast.externalId, ast)
})

1
app/ide-desktop/lib/dashboard/src/layouts/dashboard/Chat.tsx
View File

@ -748,7 +748,6 @@ export default function Chat(props: ChatProps) {
<textarea
ref={messageInputRef}
rows={1}
autoFocus
required
placeholder="Type your message ..."
className="w-full rounded-lg bg-transparent resize-none p-1"